dead_end 1.2.0 → 3.0.0

data/lib/dead_end/code_line.rb

@@ -4,44 +4,47 @@ module DeadEnd
   # Represents a single line of code of a given source file
   #
   # This object contains metadata about the line such as
-  # amount of indentation. An if it is empty or not.
+  # amount of indentation, if it is empty or not, and
+  # lexical data, such as if it has an `end` or a keyword
+  # in it.
   #
-  # While a given search for syntax errors is being performed
-  # state about the search can be stored in individual lines such
-  # as :valid or :invalid.
-  #
-  # Visibility of lines can be toggled on and off.
+  # Visibility of lines can be toggled off. Marking a line as invisible
+  # indicates that it should not be used for syntax checks.
+  # It's functionally the same as commenting it out.
   #
   # Example:
   #
-  #   line = CodeLine.new(line: "def foo\n", index: 0)
-  #   line.line_number => 1
+  #   line = CodeLine.from_source("def foo\n").first
+  #   line.number => 1
   #   line.empty? # => false
   #   line.visible? # => true
   #   line.mark_invisible
   #   line.visible? # => false
   #
-  # A CodeBlock is made of multiple CodeLines
-  #
-  # Marking a line as invisible indicates that it should not be used
-  # for syntax checks. It's essentially the same as commenting it out
-  #
-  # Marking a line as invisible also lets the overall program know
-  # that it should not check that area for syntax errors.
   class CodeLine
     TRAILING_SLASH = ("\\" + $/).freeze
 
-    def self.parse(source)
+    # Returns an array of CodeLine objects
+    # from the source string
+    def self.from_source(source)
+      lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
       source.lines.map.with_index do |line, index|
-        CodeLine.new(line: line, index: index)
+        CodeLine.new(
+          line: line,
+          index: index,
+          lex: lex_array_for_line[index + 1]
+        )
       end
     end
 
-    attr_reader :line, :index, :indent, :original_line
+    attr_reader :line, :index, :lex, :line_number, :indent
+    def initialize(line:, index:, lex:)
+      @lex = lex
+      @line = line
+      @index = index
+      @original = line.freeze
+      @line_number = @index + 1
 
-    def initialize(line:, index:)
-      @original_line = line.freeze
-      @line = @original_line
       if line.strip.empty?
         @empty = true
         @indent = 0
@@ -49,102 +52,182 @@ module DeadEnd
         @empty = false
         @indent = SpaceCount.indent(line)
       end
-      @index = index
-      @status = nil # valid, invalid, unknown
-      @invalid = false
 
-      lex_detect!
-    end
-
-    private def lex_detect!
-      lex_array = LexAll.new(source: line)
       kw_count = 0
       end_count = 0
-      lex_array.each_with_index do |lex, index|
-        next unless lex.type == :on_kw
-
-        case lex.token
-        when "if", "unless", "while", "until"
-          # Only count if/unless when it's not a "trailing" if/unless
-          # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375
-          kw_count += 1 unless lex.expr_label?
-        when "def", "case", "for", "begin", "class", "module", "do"
-          kw_count += 1
-        when "end"
-          end_count += 1
-        end
+      @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
       end
 
-      @is_comment = lex_array.detect { |lex| lex.type != :on_sp }&.type == :on_comment
-      return if @is_comment
+      kw_count -= oneliner_method_count
+
       @is_kw = (kw_count - end_count) > 0
       @is_end = (end_count - kw_count) > 0
-      @is_trailing_slash = lex_array.last.token == TRAILING_SLASH
-    end
-
-    alias_method :original, :original_line
-
-    def trailing_slash?
-      @is_trailing_slash
     end
 
+    # Used for stable sort via indentation level
+    #
+    # Ruby's sort is not "stable" meaning that when
+    # multiple elements have the same value, they are
+    # not guaranteed to return in the same order they
+    # were put in.
+    #
+    # So when multiple code lines have the same indentation
+    # level, they're sorted by their index value which is unique
+    # and consistent.
+    #
+    # This is mostly needed for consistency of the test suite
     def indent_index
       @indent_index ||= [indent, index]
     end
+    alias_method :number, :line_number
 
-    def <=>(other)
-      index <=> other.index
-    end
-
-    def is_comment?
-      @is_comment
-    end
-
-    def not_comment?
-      !is_comment?
-    end
-
+    # Returns true if the code line is determined
+    # to contain a keyword that matches with an `end`
+    #
+    # For example: `def`, `do`, `begin`, `ensure`, etc.
     def is_kw?
       @is_kw
     end
 
+    # Returns true if the code line is determined
+    # to contain an `end` keyword
     def is_end?
       @is_end
     end
 
+    # Used to hide lines
+    #
+    # The search alorithm will group lines into blocks
+    # then if those blocks are determined to represent
+    # valid code they will be hidden
     def mark_invisible
       @line = ""
-      self
-    end
-
-    def mark_visible
-      @line = @original_line
-      self
     end
 
+    # Means the line was marked as "invisible"
+    # Confusingly, "empty" lines are visible...they
+    # just don't contain any source code other than a newline ("\n").
     def visible?
       !line.empty?
     end
 
+    # Opposite or `visible?` (note: different than `empty?`)
     def hidden?
       !visible?
     end
 
-    def line_number
-      index + 1
+    # An `empty?` line is one that was originally left
+    # empty in the source code, while a "hidden" line
+    # is one that we've since marked as "invisible"
+    def empty?
+      @empty
     end
-    alias_method :number, :line_number
 
+    # Opposite of `empty?` (note: different than `visible?`)
     def not_empty?
       !empty?
     end
 
-    def empty?
-      @empty
-    end
-
+    # Renders the given line
+    #
+    # Also allows us to represent source code as
+    # an array of code lines.
+    #
+    # When we have an array of code line elements
+    # calling `join` on the array will call `to_s`
+    # on each element, which essentially converts
+    # it back into it's original source string.
     def to_s
       line
     end
+
+    # When the code line is marked invisible
+    # we retain the original value of it's line
+    # this is useful for debugging and for
+    # showing extra context
+    #
+    # DisplayCodeWithLineNumbers will render
+    # all lines given to it, not just visible
+    # lines, it uses the original method to
+    # obtain them.
+    attr_reader :original
+
+    # Comparison operator, needed for equality
+    # and sorting
+    def <=>(other)
+      index <=> other.index
+    end
+
+    # [Not stable API]
+    #
+    # Lines that have a `on_ignored_nl` type token and NOT
+    # a `BEG` type seem to be a good proxy for the ability
+    # to join multiple lines into one.
+    #
+    # This predicate method is used to determine when those
+    # two criteria have been met.
+    #
+    # The one known case this doesn't handle is:
+    #
+    #     Ripper.lex <<~EOM
+    #       a &&
+    #        b ||
+    #        c
+    #     EOM
+    #
+    # For some reason this introduces `on_ignore_newline` but with BEG type
+    def ignore_newline_not_beg?
+      lex_value = lex.detect { |l| l.type == :on_ignored_nl }
+      !!(lex_value && !lex_value.expr_beg?)
+    end
+
+    # Determines if the given line has a trailing slash
+    #
+    #     lines = CodeLine.from_source(<<~EOM)
+    #       it "foo" \
+    #     EOM
+    #     expect(lines.first.trailing_slash?).to eq(true)
+    #
+    def trailing_slash?
+      last = @lex.last
+      return false unless last
+      return false unless last.type == :on_sp
+
+      last.token == TRAILING_SLASH
+    end
+
+    # Endless method detection
+    #
+    # From https://github.com/ruby/irb/commit/826ae909c9c93a2ddca6f9cfcd9c94dbf53d44ab
+    # Detecting a "oneliner" seems to need a state machine.
+    # This can be done by looking mostly at the "state" (last value):
+    #
+    #   ENDFN -> BEG (token = '=' ) -> END
+    #
+    private def oneliner_method_count
+      oneliner_count = 0
+      in_oneliner_def = nil
+
+      @lex.each do |lex|
+        if in_oneliner_def.nil?
+          in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
+        elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
+          # Continue
+        elsif lex.state.allbits?(Ripper::EXPR_BEG)
+          in_oneliner_def = :BODY if lex.token == "="
+        elsif lex.state.allbits?(Ripper::EXPR_END)
+          # We found an endless method, count it
+          oneliner_count += 1 if in_oneliner_def == :BODY
+
+          in_oneliner_def = nil
+        else
+          in_oneliner_def = nil
+        end
+      end
+
+      oneliner_count
+    end
   end
 end

data/lib/dead_end/code_search.rb

@@ -3,11 +3,19 @@
 module DeadEnd
   # Searches code for a syntax error
   #
+  # There are three main phases in the algorithm:
+  #
+  # 1. Sanitize/format input source
+  # 2. Search for invalid blocks
+  # 3. Format invalid blocks into something meaninful
+  #
+  # This class handles the part.
+  #
   # The bulk of the heavy lifting is done in:
   #
   #  - CodeFrontier (Holds information for generating blocks and determining if we can stop searching)
   #  - ParseBlocksFromLine (Creates blocks into the frontier)
-  #  - BlockExpand (Expands existing blocks to search more code
+  #  - BlockExpand (Expands existing blocks to search more code)
   #
   # ## Syntax error detection
   #
@@ -31,28 +39,24 @@ module DeadEnd
 
     public
 
-    public
-
     attr_reader :invalid_blocks, :record_dir, :code_lines
 
     def initialize(source, record_dir: ENV["DEAD_END_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
-      @source = source
       if record_dir
         @time = Time.now.strftime("%Y-%m-%d-%H-%M-%s-%N")
         @record_dir = Pathname(record_dir).join(@time).tap { |p| p.mkpath }
         @write_count = 0
       end
-      code_lines = source.lines.map.with_index do |line, i|
-        CodeLine.new(line: line, index: i)
-      end
 
-      @code_lines = TrailingSlashJoin.new(code_lines: code_lines).call
+      @tick = 0
+      @source = source
+      @name_tick = Hash.new { |hash, k| hash[k] = 0 }
+      @invalid_blocks = []
+
+      @code_lines = CleanDocument.new(source: source).call.lines
 
       @frontier = CodeFrontier.new(code_lines: @code_lines)
-      @invalid_blocks = []
-      @name_tick = Hash.new { |hash, k| hash[k] = 0 }
-      @tick = 0
-      @block_expand = BlockExpand.new(code_lines: code_lines)
+      @block_expand = BlockExpand.new(code_lines: @code_lines)
       @parse_blocks_from_indent_line = ParseBlocksFromIndentLine.new(code_lines: @code_lines)
     end
 
@@ -63,18 +67,19 @@ module DeadEnd
       filename = "#{@write_count += 1}-#{name}-#{@name_tick[name]}.txt"
       if ENV["DEBUG"]
         puts "\n\n==== #{filename} ===="
-        puts "\n```#{block.starts_at}:#{block.ends_at}"
+        puts "\n```#{block.starts_at}..#{block.ends_at}"
         puts block.to_s
         puts "```"
-        puts "  block indent:     #{block.current_indent}"
+        puts "  block indent:      #{block.current_indent}"
       end
       @record_dir.join(filename).open(mode: "a") do |f|
-        display = DisplayInvalidBlocks.new(
-          blocks: block,
+        document = DisplayCodeWithLineNumbers.new(
+          lines: @code_lines.select(&:visible?),
           terminal: false,
-          code_lines: @code_lines
-        )
-        f.write(display.indent(display.code_with_lines))
+          highlight_lines: block.lines
+        ).call
+
+        f.write(document)
       end
     end
 
@@ -122,26 +127,8 @@ module DeadEnd
       push(block, name: "expand")
     end
 
-    def sweep_heredocs
-      HeredocBlockParse.new(
-        source: @source,
-        code_lines: @code_lines
-      ).call.each do |block|
-        push(block, name: "heredoc")
-      end
-    end
-
-    def sweep_comments
-      lines = @code_lines.select(&:is_comment?)
-      return if lines.empty?
-      block = CodeBlock.new(lines: lines)
-      sweep(block: block, name: "comments")
-    end
-
     # Main search loop
     def call
-      sweep_heredocs
-      sweep_comments
       until frontier.holds_all_syntax_errors?
         @tick += 1
 

data/lib/dead_end/display_code_with_line_numbers.rb

@@ -7,7 +7,6 @@ module DeadEnd
   # even if it is "marked invisible" any filtering of
   # output should be done before calling this class.
   #
-  #
   #   DisplayCodeWithLineNumbers.new(
   #     lines: lines,
   #     highlight_lines: [lines[2], lines[3]]

data/lib/dead_end/display_invalid_blocks.rb

@@ -8,96 +8,67 @@ module DeadEnd
   class DisplayInvalidBlocks
     attr_reader :filename
 
-    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: false, invalid_obj: WhoDisSyntaxError::Null.new)
-      @terminal = terminal
-      @filename = filename
+    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: DEFAULT_VALUE)
       @io = io
-
       @blocks = Array(blocks)
-
-      @invalid_lines = @blocks.map(&:lines).flatten
+      @filename = filename
       @code_lines = code_lines
 
-      @invalid_obj = invalid_obj
+      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
+    end
+
+    def document_ok?
+      @blocks.none? { |b| !b.hidden? }
     end
 
     def call
-      if @blocks.any? { |b| !b.hidden? }
-        found_invalid_blocks
-      else
+      if document_ok?
         @io.puts "Syntax OK"
+        return self
       end
-      self
-    end
 
-    private def no_invalid_blocks
-      @io.puts <<~EOM
-      EOM
-    end
-
-    private def found_invalid_blocks
-      @io.puts
-      @io.puts banner
-      @io.puts
-      @io.puts("file: #{filename}") if filename
-      @io.puts <<~EOM
-        simplified:
+      if filename
+        @io.puts("--> #{filename}")
+        @io.puts
+      end
+      @blocks.each do |block|
+        display_block(block)
+      end
 
-        #{indent(code_block)}
-      EOM
+      self
     end
 
-    def banner
-      case @invalid_obj.error_symbol
-      when :missing_end
-        <<~EOM
-          DeadEnd: Missing `end` detected
-
-          This code has a missing `end`. Ensure that all
-          syntax keywords (`def`, `do`, etc.) have a matching `end`.
-        EOM
-      when :unmatched_syntax
-        case @invalid_obj.unmatched_symbol
-        when :end
-          <<~EOM
-            DeadEnd: Unmatched `end` detected
-
-            This code has an unmatched `end`. Ensure that all `end` lines
-            in your code have a matching syntax keyword  (`def`,  `do`, etc.)
-            and that you don't have any extra `end` lines.
-          EOM
-        when :|
-          <<~EOM
-            DeadEnd: Unmatched `|` character detected
+    private def display_block(block)
+      # Build explanation
+      explain = ExplainSyntax.new(
+        code_lines: block.lines
+      ).call
 
-            Example:
+      # Enhance code output
+      # Also handles several ambiguious cases
+      lines = CaptureCodeContext.new(
+        blocks: block,
+        code_lines: @code_lines
+      ).call
 
-              `do |x` should be `do |x|`
-          EOM
-        when :"}"
-          <<~EOM
-            DeadEnd: Unmatched `}` character detected
+      # Build code output
+      document = DisplayCodeWithLineNumbers.new(
+        lines: lines,
+        terminal: @terminal,
+        highlight_lines: block.lines
+      ).call
 
-            This code has an unmatched `}`. Ensure that opening curly braces are
-            closed: `{ }`.
-          EOM
-        else
-          "DeadEnd: Unmatched `#{@invalid_obj.unmatched_symbol}` detected"
-        end
+      # Output syntax error explanation
+      explain.errors.each do |e|
+        @io.puts e
       end
-    end
-
-    def indent(string, with: "    ")
-      string.each_line.map { |l| with + l }.join
-    end
+      @io.puts
 
-    def code_block
-      string = +""
-      string << code_with_context
-      string
+      # Output code
+      @io.puts(document)
     end
 
-    def code_with_context
+    private def code_with_context
       lines = CaptureCodeContext.new(
         blocks: @blocks,
         code_lines: @code_lines
@@ -109,13 +80,5 @@ module DeadEnd
         highlight_lines: @invalid_lines
       ).call
     end
-
-    def code_with_lines
-      DisplayCodeWithLineNumbers.new(
-        lines: @code_lines.select(&:visible?),
-        terminal: @terminal,
-        highlight_lines: @invalid_lines
-      ).call
-    end
   end
 end

data/lib/dead_end/explain_syntax.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative "left_right_lex_count"
+
+module DeadEnd
+  # Explains syntax errors based on their source
+  #
+  # example:
+  #
+  #   source = "def foo; puts 'lol'" # Note missing end
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "Unmatched keyword, missing `end' ?"
+  #
+  # When the error cannot be determined by lexical counting
+  # then ripper is run against the input and the raw ripper
+  # errors returned.
+  #
+  # Example:
+  #
+  #   source = "1 * " # Note missing a second number
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "syntax error, unexpected end-of-input"
+  class ExplainSyntax
+    INVERSE = {
+      "{" => "}",
+      "}" => "{",
+      "[" => "]",
+      "]" => "[",
+      "(" => ")",
+      ")" => "(",
+      "|" => "|"
+    }.freeze
+
+    def initialize(code_lines:)
+      @code_lines = code_lines
+      @left_right = LeftRightLexCount.new
+      @missing = nil
+    end
+
+    def call
+      @code_lines.each do |line|
+        line.lex.each do |lex|
+          @left_right.count_lex(lex)
+        end
+      end
+
+      self
+    end
+
+    # Returns an array of missing elements
+    #
+    # For example this:
+    #
+    #   ExplainSyntax.new(code_lines: lines).missing
+    #   # => ["}"]
+    #
+    # Would indicate that the source is missing
+    # a `}` character in the source code
+    def missing
+      @missing ||= @left_right.missing
+    end
+
+    # Converts a missing string to
+    # an human understandable explanation.
+    #
+    # Example:
+    #
+    #   explain.why("}")
+    #   # => "Unmatched `{', missing `}' ?"
+    #
+    def why(miss)
+      case miss
+      when "keyword"
+        "Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ?"
+      when "end"
+        "Unmatched keyword, missing `end' ?"
+      else
+        inverse = INVERSE.fetch(miss) {
+          raise "Unknown explain syntax char or key: #{miss.inspect}"
+        }
+        "Unmatched `#{inverse}', missing `#{miss}' ?"
+      end
+    end
+
+    # Returns an array of syntax error messages
+    #
+    # If no missing pairs are found it falls back
+    # on the original ripper error messages
+    def errors
+      if missing.empty?
+        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+      end
+
+      missing.map { |miss| why(miss) }
+    end
+  end
+end