dead_end 2.0.2 → 3.0.3

data/lib/dead_end/code_frontier.rb

@@ -52,20 +52,46 @@ module DeadEnd
   class CodeFrontier
     def initialize(code_lines:)
       @code_lines = code_lines
-      @frontier = []
+      @frontier = InsertionSort.new
       @unvisited_lines = @code_lines.sort_by(&:indent_index)
+      @visited_lines = {}
+
+      @has_run = false
+      @check_next = true
     end
 
     def count
-      @frontier.count
+      @frontier.to_a.length
+    end
+
+    # Performance optimization
+    #
+    # Parsing with ripper is expensive
+    # If we know we don't have any blocks with invalid
+    # syntax, then we know we cannot have found
+    # the incorrect syntax yet.
+    #
+    # When an invalid block is added onto the frontier
+    # check document state
+    private def can_skip_check?
+      check_next = @check_next
+      @check_next = false
+
+      if check_next
+        false
+      else
+        true
+      end
     end
 
     # Returns true if the document is valid with all lines
     # removed. By default it checks all blocks in present in
     # the frontier array, but can be used for arbitrary arrays
     # of codeblocks as well
-    def holds_all_syntax_errors?(block_array = @frontier)
-      without_lines = block_array.map do |block|
+    def holds_all_syntax_errors?(block_array = @frontier, can_cache: true)
+      return false if can_cache && can_skip_check?
+
+      without_lines = block_array.to_a.flat_map do |block|
         block.lines
       end
 
@@ -77,7 +103,7 @@ module DeadEnd
 
     # Returns a code block with the largest indentation possible
     def pop
-      @frontier.pop
+      @frontier.to_a.pop
     end
 
     def next_indent_line
@@ -85,15 +111,15 @@ module DeadEnd
     end
 
     def expand?
-      return false if @frontier.empty?
-      return true if @unvisited_lines.empty?
+      return false if @frontier.to_a.empty?
+      return true if @unvisited_lines.to_a.empty?
 
-      frontier_indent = @frontier.last.current_indent
+      frontier_indent = @frontier.to_a.last.current_indent
       unvisited_indent = next_indent_line.indent
 
       if ENV["DEBUG"]
         puts "```"
-        puts @frontier.last.to_s
+        puts @frontier.to_a.last.to_s
         puts "```"
         puts "  @frontier indent:  #{frontier_indent}"
         puts "  @unvisited indent: #{unvisited_indent}"
@@ -104,7 +130,13 @@ module DeadEnd
     end
 
     def register_indent_block(block)
-      @unvisited_lines -= block.lines
+      block.lines.each do |line|
+        next if @visited_lines[line]
+        @visited_lines[line] = true
+
+        index = @unvisited_lines.bsearch_index { |l| line.indent_index <=> l.indent_index }
+        @unvisited_lines.delete_at(index)
+      end
       self
     end
 
@@ -117,11 +149,13 @@ module DeadEnd
       register_indent_block(block)
 
       # Make sure we don't double expand, if a code block fully engulfs another code block, keep the bigger one
-      @frontier.reject! { |b|
+      @frontier.to_a.reject! { |b|
         b.starts_at >= block.starts_at && b.ends_at <= block.ends_at
       }
+
+      @check_next = true if block.invalid?
       @frontier << block
-      @frontier.sort!
+      # @frontier.sort!
 
       self
     end
@@ -141,8 +175,8 @@ module DeadEnd
     # Given that we know our syntax error exists somewhere in our frontier, we want to find
     # the smallest possible set of blocks that contain all the syntax errors
     def detect_invalid_blocks
-      self.class.combination(@frontier.select(&:invalid?)).detect do |block_array|
-        holds_all_syntax_errors?(block_array)
+      self.class.combination(@frontier.to_a.select(&:invalid?)).detect do |block_array|
+        holds_all_syntax_errors?(block_array, can_cache: false)
       end || []
     end
   end

data/lib/dead_end/code_line.rb

@@ -26,9 +26,10 @@ module DeadEnd
 
     # Returns an array of CodeLine objects
     # from the source string
-    def self.from_source(source)
-      lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
-      source.lines.map.with_index do |line, index|
+    def self.from_source(source, lines: nil)
+      lines ||= source.lines
+      lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
+      lines.map.with_index do |line, index|
         CodeLine.new(
           line: line,
           index: index,
@@ -42,28 +43,20 @@ module DeadEnd
       @lex = lex
       @line = line
       @index = index
-      @original = line.freeze
+      @original = line
       @line_number = @index + 1
+      strip_line = line.dup
+      strip_line.lstrip!
 
-      if line.strip.empty?
+      if strip_line.empty?
         @empty = true
         @indent = 0
       else
         @empty = false
-        @indent = SpaceCount.indent(line)
+        @indent = line.length - strip_line.length
       end
 
-      kw_count = 0
-      end_count = 0
-      @lex.each do |lex|
-        kw_count += 1 if lex.is_kw?
-        end_count += 1 if lex.is_end?
-      end
-
-      kw_count -= oneliner_method_count
-
-      @is_kw = (kw_count - end_count) > 0
-      @is_end = (end_count - kw_count) > 0
+      set_kw_end
     end
 
     # Used for stable sort via indentation level
@@ -179,8 +172,7 @@ module DeadEnd
     #
     # For some reason this introduces `on_ignore_newline` but with BEG type
     def ignore_newline_not_beg?
-      lex_value = lex.detect { |l| l.type == :on_ignored_nl }
-      !!(lex_value && !lex_value.expr_beg?)
+      @ignore_newline_not_beg
     end
 
     # Determines if the given line has a trailing slash
@@ -206,11 +198,22 @@ module DeadEnd
     #
     #   ENDFN -> BEG (token = '=' ) -> END
     #
-    private def oneliner_method_count
+    private def set_kw_end
       oneliner_count = 0
       in_oneliner_def = nil
 
+      kw_count = 0
+      end_count = 0
+
+      @ignore_newline_not_beg = false
       @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
+
+        if lex.type == :on_ignored_nl
+          @ignore_newline_not_beg = !lex.expr_beg?
+        end
+
         if in_oneliner_def.nil?
           in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
         elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
@@ -227,7 +230,10 @@ module DeadEnd
         end
       end
 
-      oneliner_count
+      kw_count -= oneliner_count
+
+      @is_kw = (kw_count - end_count) > 0
+      @is_end = (end_count - kw_count) > 0
     end
   end
 end

data/lib/dead_end/code_search.rb

@@ -43,8 +43,7 @@ module DeadEnd
 
     def initialize(source, record_dir: ENV["DEAD_END_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
       if record_dir
-        @time = Time.now.strftime("%Y-%m-%d-%H-%M-%s-%N")
-        @record_dir = Pathname(record_dir).join(@time).tap { |p| p.mkpath }
+        @record_dir = DeadEnd.record_dir(record_dir)
         @write_count = 0
       end
 
@@ -73,12 +72,13 @@ module DeadEnd
         puts "  block indent:      #{block.current_indent}"
       end
       @record_dir.join(filename).open(mode: "a") do |f|
-        display = DisplayInvalidBlocks.new(
-          blocks: block,
+        document = DisplayCodeWithLineNumbers.new(
+          lines: @code_lines.select(&:visible?),
           terminal: false,
-          code_lines: @code_lines
-        )
-        f.write(display.indent(display.code_with_lines))
+          highlight_lines: block.lines
+        ).call
+
+        f.write(document)
       end
     end
 

data/lib/dead_end/display_invalid_blocks.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative "banner"
 require_relative "capture_code_context"
 require_relative "display_code_with_line_numbers"
 
@@ -9,18 +8,13 @@ module DeadEnd
   class DisplayInvalidBlocks
     attr_reader :filename
 
-    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: DEFAULT_VALUE, invalid_obj: WhoDisSyntaxError::Null.new)
-      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
-
-      @filename = filename
+    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: DEFAULT_VALUE)
       @io = io
-
       @blocks = Array(blocks)
-
-      @invalid_lines = @blocks.map(&:lines).flatten
+      @filename = filename
       @code_lines = code_lines
 
-      @invalid_obj = invalid_obj
+      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
     end
 
     def document_ok?
@@ -30,61 +24,58 @@ module DeadEnd
     def call
       if document_ok?
         @io.puts "Syntax OK"
-      else
-        found_invalid_blocks
+        return self
       end
-      self
-    end
 
-    private def no_invalid_blocks
-      @io.puts <<~EOM
-      EOM
-    end
-
-    private def found_invalid_blocks
-      @io.puts
-      if banner
-        @io.puts banner
+      if filename
+        @io.puts("--> #{filename}")
         @io.puts
       end
-      @io.puts("file: #{filename}") if filename
-      @io.puts <<~EOM
-        simplified:
-
-        #{indent(code_block)}
-      EOM
-    end
-
-    def banner
-      Banner.new(invalid_obj: @invalid_obj).call
-    end
+      @blocks.each do |block|
+        display_block(block)
+      end
 
-    def indent(string, with: "    ")
-      string.each_line.map { |l| with + l }.join
+      self
     end
 
-    def code_block
-      string = +""
-      string << code_with_context
-      string
-    end
+    private def display_block(block)
+      # Build explanation
+      explain = ExplainSyntax.new(
+        code_lines: block.lines
+      ).call
 
-    def code_with_context
+      # Enhance code output
+      # Also handles several ambiguious cases
       lines = CaptureCodeContext.new(
-        blocks: @blocks,
+        blocks: block,
         code_lines: @code_lines
       ).call
 
-      DisplayCodeWithLineNumbers.new(
+      # Build code output
+      document = DisplayCodeWithLineNumbers.new(
         lines: lines,
         terminal: @terminal,
-        highlight_lines: @invalid_lines
+        highlight_lines: block.lines
       ).call
+
+      # Output syntax error explanation
+      explain.errors.each do |e|
+        @io.puts e
+      end
+      @io.puts
+
+      # Output code
+      @io.puts(document)
     end
 
-    def code_with_lines
+    private def code_with_context
+      lines = CaptureCodeContext.new(
+        blocks: @blocks,
+        code_lines: @code_lines
+      ).call
+
       DisplayCodeWithLineNumbers.new(
-        lines: @code_lines.select(&:visible?),
+        lines: lines,
         terminal: @terminal,
         highlight_lines: @invalid_lines
       ).call

data/lib/dead_end/explain_syntax.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative "left_right_lex_count"
+
+module DeadEnd
+  # Explains syntax errors based on their source
+  #
+  # example:
+  #
+  #   source = "def foo; puts 'lol'" # Note missing end
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "Unmatched keyword, missing `end' ?"
+  #
+  # When the error cannot be determined by lexical counting
+  # then ripper is run against the input and the raw ripper
+  # errors returned.
+  #
+  # Example:
+  #
+  #   source = "1 * " # Note missing a second number
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "syntax error, unexpected end-of-input"
+  class ExplainSyntax
+    INVERSE = {
+      "{" => "}",
+      "}" => "{",
+      "[" => "]",
+      "]" => "[",
+      "(" => ")",
+      ")" => "(",
+      "|" => "|"
+    }.freeze
+
+    def initialize(code_lines:)
+      @code_lines = code_lines
+      @left_right = LeftRightLexCount.new
+      @missing = nil
+    end
+
+    def call
+      @code_lines.each do |line|
+        line.lex.each do |lex|
+          @left_right.count_lex(lex)
+        end
+      end
+
+      self
+    end
+
+    # Returns an array of missing elements
+    #
+    # For example this:
+    #
+    #   ExplainSyntax.new(code_lines: lines).missing
+    #   # => ["}"]
+    #
+    # Would indicate that the source is missing
+    # a `}` character in the source code
+    def missing
+      @missing ||= @left_right.missing
+    end
+
+    # Converts a missing string to
+    # an human understandable explanation.
+    #
+    # Example:
+    #
+    #   explain.why("}")
+    #   # => "Unmatched `{', missing `}' ?"
+    #
+    def why(miss)
+      case miss
+      when "keyword"
+        "Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ?"
+      when "end"
+        "Unmatched keyword, missing `end' ?"
+      else
+        inverse = INVERSE.fetch(miss) {
+          raise "Unknown explain syntax char or key: #{miss.inspect}"
+        }
+        "Unmatched `#{inverse}', missing `#{miss}' ?"
+      end
+    end
+
+    # Returns an array of syntax error messages
+    #
+    # If no missing pairs are found it falls back
+    # on the original ripper error messages
+    def errors
+      if missing.empty?
+        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+      end
+
+      missing.map { |miss| why(miss) }
+    end
+  end
+end

data/lib/dead_end/insertion_sort.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DeadEnd
+  # Sort elements on insert
+  #
+  # Instead of constantly calling `sort!`, put
+  # the element where it belongs the first time
+  # around
+  #
+  # Example:
+  #
+  #   sorted = InsertionSort.new
+  #   sorted << 33
+  #   sorted << 44
+  #   sorted << 1
+  #   puts sorted.to_a
+  #   # => [1, 44, 33]
+  #
+  class InsertionSort
+    def initialize
+      @array = []
+    end
+
+    def <<(value)
+      insert_in = @array.length
+      @array.each.with_index do |existing, index|
+        case value <=> existing
+        when -1
+          insert_in = index
+          break
+        when 0
+          insert_in = index
+          break
+        when 1
+          # Keep going
+        end
+      end
+
+      @array.insert(insert_in, value)
+    end
+
+    def to_a
+      @array
+    end
+  end
+end

data/lib/dead_end/left_right_lex_count.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module DeadEnd
+  # Find mis-matched syntax based on lexical count
+  #
+  # Used for detecting missing pairs of elements
+  # each keyword needs an end, each '{' needs a '}'
+  # etc.
+  #
+  # Example:
+  #
+  #   left_right = LeftRightLexCount.new
+  #   left_right.count_kw
+  #   left_right.missing.first
+  #   # => "end"
+  #
+  #   left_right = LeftRightLexCount.new
+  #   source = "{ a: b, c: d" # Note missing '}'
+  #   LexAll.new(source: source).each do |lex|
+  #     left_right.count_lex(lex)
+  #   end
+  #   left_right.missing.first
+  #   # => "}"
+  class LeftRightLexCount
+    def initialize
+      @kw_count = 0
+      @end_count = 0
+
+      @count_for_char = {
+        "{" => 0,
+        "}" => 0,
+        "[" => 0,
+        "]" => 0,
+        "(" => 0,
+        ")" => 0,
+        "|" => 0
+      }
+    end
+
+    def count_kw
+      @kw_count += 1
+    end
+
+    def count_end
+      @end_count += 1
+    end
+
+    # Count source code characters
+    #
+    # Example:
+    #
+    #   left_right = LeftRightLexCount.new
+    #   left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG))
+    #   left_right.count_for_char("{")
+    #   # => 1
+    #   left_right.count_for_char("}")
+    #   # => 0
+    def count_lex(lex)
+      case lex.type
+      when :on_tstring_content
+        # ^^^
+        # Means it's a string or a symbol `"{"` rather than being
+        # part of a data structure (like a hash) `{ a: b }`
+        # ignore it.
+      when :on_words_beg, :on_symbos_beg, :on_qwords_beg,
+           :on_qsymbols_beg, :on_regexp_beg, :on_tstring_beg
+        # ^^^
+        # Handle shorthand syntaxes like `%Q{ i am a string }`
+        #
+        # The start token will be the full thing `%Q{` but we
+        # need to count it as if it's a `{`. Any token
+        # can be used
+        char = lex.token[-1]
+        @count_for_char[char] += 1 if @count_for_char.key?(char)
+      when :on_embexpr_beg
+        # ^^^
+        # Embedded string expressions like `"#{foo} <-embed"`
+        # are parsed with chars:
+        #
+        # `#{` as :on_embexpr_beg
+        #  `}` as :on_embexpr_end
+        #
+        # We cannot ignore both :on_emb_expr_beg and :on_embexpr_end
+        # because sometimes the lexer thinks something is an embed
+        # string end, when it is not like `lol = }` (no clue why).
+        #
+        # When we see `#{` count it as a `{` or we will
+        # have a mis-match count.
+        #
+        case lex.token
+        when "\#{"
+          @count_for_char["{"] += 1
+        end
+      else
+        @end_count += 1 if lex.is_end?
+        @kw_count += 1 if lex.is_kw?
+        @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token)
+      end
+    end
+
+    def count_for_char(char)
+      @count_for_char[char]
+    end
+
+    # Returns an array of missing syntax characters
+    # or `"end"` or `"keyword"`
+    #
+    #   left_right.missing
+    #   # => ["}"]
+    def missing
+      out = missing_pairs
+      out << missing_pipe
+      out << missing_keyword_end
+      out.compact!
+      out
+    end
+
+    PAIRS = {
+      "{" => "}",
+      "[" => "]",
+      "(" => ")"
+    }.freeze
+
+    # Opening characters like `{` need closing characters # like `}`.
+    #
+    # When a mis-match count is detected, suggest the
+    # missing member.
+    #
+    # For example if there are 3 `}` and only two `{`
+    # return `"{"`
+    private def missing_pairs
+      PAIRS.map do |(left, right)|
+        case @count_for_char[left] <=> @count_for_char[right]
+        when 1
+          right
+        when 0
+          nil
+        when -1
+          left
+        end
+      end
+    end
+
+    # Keywords need ends and ends need keywords
+    #
+    # If we have more keywords, there's a missing `end`
+    # if we have more `end`-s, there's a missing keyword
+    private def missing_keyword_end
+      case @kw_count <=> @end_count
+      when 1
+        "end"
+      when 0
+        nil
+      when -1
+        "keyword"
+      end
+    end
+
+    # Pipes come in pairs.
+    # If there's an odd number of pipes then we
+    # are missing one
+    private def missing_pipe
+      if @count_for_char["|"].odd?
+        "|"
+      end
+    end
+  end
+end