dead_end 1.1.6 → 2.0.1

data/lib/dead_end/code_frontier.rb

@@ -3,11 +3,19 @@
 module DeadEnd
   # The main function of the frontier is to hold the edges of our search and to
   # evaluate when we can stop searching.
+
+  # There are three main phases in the algorithm:
+  #
+  # 1. Sanitize/format input source
+  # 2. Search for invalid blocks
+  # 3. Format invalid blocks into something meaninful
+  #
+  # The Code frontier is a critical part of the second step
   #
   # ## Knowing where we've been
   #
-  # Once a code block is generated it is added onto the frontier where it will be
-  # sorted and then the frontier can be filtered. Large blocks that totally contain a
+  # Once a code block is generated it is added onto the frontier. Then it will be
+  # sorted by indentation and frontier can be filtered. Large blocks that fully enclose a
   # smaller block will cause the smaller block to be evicted.
   #
   #   CodeFrontier#<<(block) # Adds block to frontier
@@ -15,11 +23,11 @@ module DeadEnd
   #
   # ## Knowing where we can go
   #
-  # Internally it keeps track of "unvisited" lines which is exposed via `next_indent_line`
-  # when called this will return a line of code with the most indentation.
+  # Internally the frontier keeps track of "unvisited" lines which are exposed via `next_indent_line`
+  # when called, this method returns, a line of code with the highest indentation.
   #
-  # This line of code can be used to build a CodeBlock and then when that code block
-  # is added back to the frontier, then the lines are removed from the
+  # The returned line of code can be used to build a CodeBlock and then that code block
+  # is added back to the frontier. Then, the lines are removed from the
   # "unvisited" so we don't double-create the same block.
   #
   #   CodeFrontier#next_indent_line # Shows next line
@@ -27,19 +35,22 @@ module DeadEnd
   #
   # ## Knowing when to stop
   #
-  # The frontier holds the syntax error when removing all code blocks from the original
-  # source document allows it to be parsed as syntatically valid:
+  # The frontier knows how to check the entire document for a syntax error. When blocks
+  # are added onto the frontier, they're removed from the document. When all code containing
+  # syntax errors has been added to the frontier, the document will be parsable without a
+  # syntax error and the search can stop.
   #
-  #   CodeFrontier#holds_all_syntax_errors?
+  #   CodeFrontier#holds_all_syntax_errors? # Returns true when frontier holds all syntax errors
   #
   # ## Filtering false positives
   #
-  # Once the search is completed, the frontier will have many blocks that do not contain
-  # the syntax error. To filter to the smallest subset that does call:
+  # Once the search is completed, the frontier may have multiple blocks that do not contain
+  # the syntax error. To limit the result to the smallest subset of "invalid blocks" call:
   #
   #   CodeFrontier#detect_invalid_blocks
+  #
   class CodeFrontier
-    def initialize(code_lines: )
+    def initialize(code_lines:)
       @code_lines = code_lines
       @frontier = []
       @unvisited_lines = @code_lines.sort_by(&:indent_index)
@@ -66,7 +77,7 @@ module DeadEnd
 
     # Returns a code block with the largest indentation possible
     def pop
-      return @frontier.pop
+      @frontier.pop
     end
 
     def next_indent_line
@@ -78,14 +89,14 @@ module DeadEnd
       return true if @unvisited_lines.empty?
 
       frontier_indent = @frontier.last.current_indent
-      unvisited_indent= next_indent_line.indent
+      unvisited_indent = next_indent_line.indent
 
       if ENV["DEBUG"]
         puts "```"
         puts @frontier.last.to_s
         puts "```"
-        puts "  @frontier indent: #{frontier_indent}"
-        puts "  @unvisited indent:     #{unvisited_indent}"
+        puts "  @frontier indent:  #{frontier_indent}"
+        puts "  @unvisited indent: #{unvisited_indent}"
       end
 
       # Expand all blocks before moving to unvisited lines
@@ -106,7 +117,7 @@ module DeadEnd
       register_indent_block(block)
 
       # Make sure we don't double expand, if a code block fully engulfs another code block, keep the bigger one
-      @frontier.reject! {|b|
+      @frontier.reject! { |b|
         b.starts_at >= block.starts_at && b.ends_at <= block.ends_at
       }
       @frontier << block

data/lib/dead_end/code_line.rb

@@ -4,44 +4,47 @@ module DeadEnd
   # Represents a single line of code of a given source file
   #
   # This object contains metadata about the line such as
-  # amount of indentation. An if it is empty or not.
+  # amount of indentation, if it is empty or not, and
+  # lexical data, such as if it has an `end` or a keyword
+  # in it.
   #
-  # While a given search for syntax errors is being performed
-  # state about the search can be stored in individual lines such
-  # as :valid or :invalid.
-  #
-  # Visibility of lines can be toggled on and off.
+  # Visibility of lines can be toggled off. Marking a line as invisible
+  # indicates that it should not be used for syntax checks.
+  # It's functionally the same as commenting it out.
   #
   # Example:
   #
-  #   line = CodeLine.new(line: "def foo\n", index: 0)
-  #   line.line_number => 1
+  #   line = CodeLine.from_source("def foo\n").first
+  #   line.number => 1
   #   line.empty? # => false
   #   line.visible? # => true
   #   line.mark_invisible
   #   line.visible? # => false
   #
-  # A CodeBlock is made of multiple CodeLines
-  #
-  # Marking a line as invisible indicates that it should not be used
-  # for syntax checks. It's essentially the same as commenting it out
-  #
-  # Marking a line as invisible also lets the overall program know
-  # that it should not check that area for syntax errors.
   class CodeLine
     TRAILING_SLASH = ("\\" + $/).freeze
 
-    def self.parse(source)
+    # Returns an array of CodeLine objects
+    # from the source string
+    def self.from_source(source)
+      lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
       source.lines.map.with_index do |line, index|
-        CodeLine.new(line: line, index: index)
+        CodeLine.new(
+          line: line,
+          index: index,
+          lex: lex_array_for_line[index + 1]
+        )
       end
     end
 
-    attr_reader :line, :index, :indent, :original_line
+    attr_reader :line, :index, :lex, :line_number, :indent
+    def initialize(line:, index:, lex:)
+      @lex = lex
+      @line = line
+      @index = index
+      @original = line.freeze
+      @line_number = @index + 1
 
-    def initialize(line: , index:)
-      @original_line = line.freeze
-      @line = @original_line
       if line.strip.empty?
         @empty = true
         @indent = 0
@@ -49,102 +52,182 @@ module DeadEnd
         @empty = false
         @indent = SpaceCount.indent(line)
       end
-      @index = index
-      @status = nil # valid, invalid, unknown
-      @invalid = false
-
-      lex_detect!
-    end
 
-    private def lex_detect!
-      lex_array = LexAll.new(source: line)
       kw_count = 0
       end_count = 0
-      lex_array.each_with_index do |lex, index|
-        next unless lex.type == :on_kw
-
-        case lex.token
-        when 'if', 'unless', 'while', 'until'
-          # Only count if/unless when it's not a "trailing" if/unless
-          # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375
-          kw_count += 1 if !lex.expr_label?
-        when 'def', 'case', 'for', 'begin', 'class', 'module', 'do'
-          kw_count += 1
-        when 'end'
-          end_count += 1
-        end
+      @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
       end
 
-      @is_comment = lex_array.detect {|lex| lex.type != :on_sp}&.type == :on_comment
-      return if @is_comment
+      kw_count -= oneliner_method_count
+
       @is_kw = (kw_count - end_count) > 0
       @is_end = (end_count - kw_count) > 0
-      @is_trailing_slash = lex_array.last.token == TRAILING_SLASH
-    end
-
-    alias :original :original_line
-
-    def trailing_slash?
-      @is_trailing_slash
     end
 
+    # Used for stable sort via indentation level
+    #
+    # Ruby's sort is not "stable" meaning that when
+    # multiple elements have the same value, they are
+    # not guaranteed to return in the same order they
+    # were put in.
+    #
+    # So when multiple code lines have the same indentation
+    # level, they're sorted by their index value which is unique
+    # and consistent.
+    #
+    # This is mostly needed for consistency of the test suite
     def indent_index
       @indent_index ||= [indent, index]
     end
+    alias_method :number, :line_number
 
-    def <=>(b)
-      self.index <=> b.index
-    end
-
-    def is_comment?
-      @is_comment
-    end
-
-    def not_comment?
-      !is_comment?
-    end
-
+    # Returns true if the code line is determined
+    # to contain a keyword that matches with an `end`
+    #
+    # For example: `def`, `do`, `begin`, `ensure`, etc.
     def is_kw?
       @is_kw
     end
 
+    # Returns true if the code line is determined
+    # to contain an `end` keyword
     def is_end?
       @is_end
     end
 
+    # Used to hide lines
+    #
+    # The search alorithm will group lines into blocks
+    # then if those blocks are determined to represent
+    # valid code they will be hidden
     def mark_invisible
       @line = ""
-      self
-    end
-
-    def mark_visible
-      @line = @original_line
-      self
     end
 
+    # Means the line was marked as "invisible"
+    # Confusingly, "empty" lines are visible...they
+    # just don't contain any source code other than a newline ("\n").
     def visible?
       !line.empty?
     end
 
+    # Opposite or `visible?` (note: different than `empty?`)
     def hidden?
       !visible?
     end
 
-    def line_number
-      index + 1
+    # An `empty?` line is one that was originally left
+    # empty in the source code, while a "hidden" line
+    # is one that we've since marked as "invisible"
+    def empty?
+      @empty
     end
-    alias :number :line_number
 
+    # Opposite of `empty?` (note: different than `visible?`)
     def not_empty?
       !empty?
     end
 
-    def empty?
-      @empty
-    end
-
+    # Renders the given line
+    #
+    # Also allows us to represent source code as
+    # an array of code lines.
+    #
+    # When we have an array of code line elements
+    # calling `join` on the array will call `to_s`
+    # on each element, which essentially converts
+    # it back into it's original source string.
     def to_s
-      self.line
+      line
+    end
+
+    # When the code line is marked invisible
+    # we retain the original value of it's line
+    # this is useful for debugging and for
+    # showing extra context
+    #
+    # DisplayCodeWithLineNumbers will render
+    # all lines given to it, not just visible
+    # lines, it uses the original method to
+    # obtain them.
+    attr_reader :original
+
+    # Comparison operator, needed for equality
+    # and sorting
+    def <=>(other)
+      index <=> other.index
+    end
+
+    # [Not stable API]
+    #
+    # Lines that have a `on_ignored_nl` type token and NOT
+    # a `BEG` type seem to be a good proxy for the ability
+    # to join multiple lines into one.
+    #
+    # This predicate method is used to determine when those
+    # two criteria have been met.
+    #
+    # The one known case this doesn't handle is:
+    #
+    #     Ripper.lex <<~EOM
+    #       a &&
+    #        b ||
+    #        c
+    #     EOM
+    #
+    # For some reason this introduces `on_ignore_newline` but with BEG type
+    def ignore_newline_not_beg?
+      lex_value = lex.detect { |l| l.type == :on_ignored_nl }
+      !!(lex_value && !lex_value.expr_beg?)
+    end
+
+    # Determines if the given line has a trailing slash
+    #
+    #     lines = CodeLine.from_source(<<~EOM)
+    #       it "foo" \
+    #     EOM
+    #     expect(lines.first.trailing_slash?).to eq(true)
+    #
+    def trailing_slash?
+      last = @lex.last
+      return false unless last
+      return false unless last.type == :on_sp
+
+      last.token == TRAILING_SLASH
+    end
+
+    # Endless method detection
+    #
+    # From https://github.com/ruby/irb/commit/826ae909c9c93a2ddca6f9cfcd9c94dbf53d44ab
+    # Detecting a "oneliner" seems to need a state machine.
+    # This can be done by looking mostly at the "state" (last value):
+    #
+    #   ENDFN -> BEG (token = '=' ) -> END
+    #
+    private def oneliner_method_count
+      oneliner_count = 0
+      in_oneliner_def = nil
+
+      @lex.each do |lex|
+        if in_oneliner_def.nil?
+          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          # Continue
+        elsif lex.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if lex.token == "="
+        elsif lex.state.allbits?(Ripper::EXPR_END)
+          # We found an endless method, count it
+          oneliner_count += 1 if in_oneliner_def == :BODY
+
+          in_oneliner_def = nil
+        else
+          in_oneliner_def = nil
+        end
+      end
+
+      oneliner_count
     end
   end
 end

data/lib/dead_end/code_search.rb

@@ -3,11 +3,19 @@
 module DeadEnd
   # Searches code for a syntax error
   #
+  # There are three main phases in the algorithm:
+  #
+  # 1. Sanitize/format input source
+  # 2. Search for invalid blocks
+  # 3. Format invalid blocks into something meaninful
+  #
+  # This class handles the part.
+  #
   # The bulk of the heavy lifting is done in:
   #
   #  - CodeFrontier (Holds information for generating blocks and determining if we can stop searching)
   #  - ParseBlocksFromLine (Creates blocks into the frontier)
-  #  - BlockExpand (Expands existing blocks to search more code
+  #  - BlockExpand (Expands existing blocks to search more code)
   #
   # ## Syntax error detection
   #
@@ -25,65 +33,64 @@ module DeadEnd
   #   # => ["def lol\n"]
   #
   class CodeSearch
-    private; attr_reader :frontier; public
-    public; attr_reader :invalid_blocks, :record_dir, :code_lines
+    private
+
+    attr_reader :frontier
+
+    public
+
+    attr_reader :invalid_blocks, :record_dir, :code_lines
 
     def initialize(source, record_dir: ENV["DEAD_END_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
-      @source = source
       if record_dir
-        @time = Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')
-        @record_dir = Pathname(record_dir).join(@time).tap {|p| p.mkpath }
+        @time = Time.now.strftime("%Y-%m-%d-%H-%M-%s-%N")
+        @record_dir = Pathname(record_dir).join(@time).tap { |p| p.mkpath }
         @write_count = 0
       end
-      code_lines = source.lines.map.with_index do |line, i|
-        CodeLine.new(line: line, index: i)
-      end
 
-      @code_lines = TrailingSlashJoin.new(code_lines: code_lines).call
+      @tick = 0
+      @source = source
+      @name_tick = Hash.new { |hash, k| hash[k] = 0 }
+      @invalid_blocks = []
+
+      @code_lines = CleanDocument.new(source: source).call.lines
 
       @frontier = CodeFrontier.new(code_lines: @code_lines)
-      @invalid_blocks = []
-      @name_tick = Hash.new {|hash, k| hash[k] = 0 }
-      @tick = 0
-      @block_expand = BlockExpand.new(code_lines: code_lines)
+      @block_expand = BlockExpand.new(code_lines: @code_lines)
       @parse_blocks_from_indent_line = ParseBlocksFromIndentLine.new(code_lines: @code_lines)
     end
 
     # Used for debugging
     def record(block:, name: "record")
-      return if !@record_dir
+      return unless @record_dir
       @name_tick[name] += 1
       filename = "#{@write_count += 1}-#{name}-#{@name_tick[name]}.txt"
       if ENV["DEBUG"]
         puts "\n\n==== #{filename} ===="
-        puts "\n```#{block.starts_at}:#{block.ends_at}"
-        puts "#{block.to_s}"
+        puts "\n```#{block.starts_at}..#{block.ends_at}"
+        puts block.to_s
         puts "```"
-        puts "  block indent:     #{block.current_indent}"
+        puts "  block indent:      #{block.current_indent}"
       end
       @record_dir.join(filename).open(mode: "a") do |f|
         display = DisplayInvalidBlocks.new(
           blocks: block,
           terminal: false,
-          code_lines: @code_lines,
+          code_lines: @code_lines
         )
-        f.write(display.indent display.code_with_lines)
+        f.write(display.indent(display.code_with_lines))
       end
     end
 
-    def push(block, name: )
+    def push(block, name:)
       record(block: block, name: name)
 
-      if block.valid?
-        block.mark_invisible
-        frontier << block
-      else
-        frontier << block
-      end
+      block.mark_invisible if block.valid?
+      frontier << block
     end
 
     # Removes the block without putting it back in the frontier
-    def sweep(block:, name: )
+    def sweep(block:, name:)
       record(block: block, name: name)
 
       block.lines.each(&:mark_invisible)
@@ -119,26 +126,8 @@ module DeadEnd
       push(block, name: "expand")
     end
 
-    def sweep_heredocs
-      HeredocBlockParse.new(
-        source: @source,
-        code_lines: @code_lines
-      ).call.each do |block|
-        push(block, name: "heredoc")
-      end
-    end
-
-    def sweep_comments
-      lines = @code_lines.select(&:is_comment?)
-      return if lines.empty?
-      block = CodeBlock.new(lines: lines)
-      sweep(block: block, name: "comments")
-    end
-
     # Main search loop
     def call
-      sweep_heredocs
-      sweep_comments
       until frontier.holds_all_syntax_errors?
         @tick += 1
 
@@ -149,8 +138,8 @@ module DeadEnd
         end
       end
 
-      @invalid_blocks.concat(frontier.detect_invalid_blocks )
-      @invalid_blocks.sort_by! {|block| block.starts_at }
+      @invalid_blocks.concat(frontier.detect_invalid_blocks)
+      @invalid_blocks.sort_by! { |block| block.starts_at }
       self
     end
   end

data/lib/dead_end/display_code_with_line_numbers.rb

@@ -7,7 +7,6 @@ module DeadEnd
   # even if it is "marked invisible" any filtering of
   # output should be done before calling this class.
   #
-  #
   #   DisplayCodeWithLineNumbers.new(
   #     lines: lines,
   #     highlight_lines: [lines[2], lines[3]]
@@ -23,10 +22,10 @@ module DeadEnd
     TERMINAL_HIGHLIGHT = "\e[1;3m" # Bold, italics
     TERMINAL_END = "\e[0m"
 
-    def initialize(lines: , highlight_lines: [], terminal: false)
+    def initialize(lines:, highlight_lines: [], terminal: false)
       @lines = Array(lines).sort
       @terminal = terminal
-      @highlight_line_hash = Array(highlight_lines).each_with_object({}) {|line, h| h[line] = true  }
+      @highlight_line_hash = Array(highlight_lines).each_with_object({}) { |line, h| h[line] = true }
       @digit_count = @lines.last&.line_number.to_s.length
     end
 
@@ -48,12 +47,12 @@ module DeadEnd
       end.join
     end
 
-    private def format(contents: , number: , highlight: false, empty:)
-      string = String.new("")
-      if highlight
-        string << "❯ "
+    private def format(contents:, number:, empty:, highlight: false)
+      string = +""
+      string << if highlight
+        "❯ "
       else
-        string << "  "
+        "  "
       end
 
       string << number.rjust(@digit_count).to_s