dead_end 2.0.0 → 3.0.1

data/lib/dead_end/code_block.rb

@@ -70,8 +70,24 @@ module DeadEnd
     end
 
     def valid?
-      return @valid if @valid != UNSET
-      @valid = DeadEnd.valid?(to_s)
+      if @valid == UNSET
+        # Performance optimization
+        #
+        # If all the lines were previously hidden
+        # and we expand to capture additional empty
+        # lines then the result cannot be invalid
+        #
+        # That means there's no reason to re-check all
+        # lines with ripper (which is expensive).
+        # Benchmark in commit message
+        @valid = if lines.all? { |l| l.hidden? || l.empty? }
+          true
+        else
+          DeadEnd.valid?(lines.map(&:original).join)
+        end
+      else
+        @valid
+      end
     end
 
     def to_s

data/lib/dead_end/code_frontier.rb

@@ -52,20 +52,44 @@ module DeadEnd
   class CodeFrontier
     def initialize(code_lines:)
       @code_lines = code_lines
-      @frontier = []
+      @frontier = InsertionSort.new
       @unvisited_lines = @code_lines.sort_by(&:indent_index)
+      @has_run = false
+      @check_next = true
     end
 
     def count
-      @frontier.count
+      @frontier.to_a.length
+    end
+
+    # Performance optimization
+    #
+    # Parsing with ripper is expensive
+    # If we know we don't have any blocks with invalid
+    # syntax, then we know we cannot have found
+    # the incorrect syntax yet.
+    #
+    # When an invalid block is added onto the frontier
+    # check document state
+    private def can_skip_check?
+      check_next = @check_next
+      @check_next = false
+
+      if check_next
+        false
+      else
+        true
+      end
     end
 
     # Returns true if the document is valid with all lines
     # removed. By default it checks all blocks in present in
     # the frontier array, but can be used for arbitrary arrays
     # of codeblocks as well
-    def holds_all_syntax_errors?(block_array = @frontier)
-      without_lines = block_array.map do |block|
+    def holds_all_syntax_errors?(block_array = @frontier, can_cache: true)
+      return false if can_cache && can_skip_check?
+
+      without_lines = block_array.to_a.flat_map do |block|
         block.lines
       end
 
@@ -77,7 +101,7 @@ module DeadEnd
 
     # Returns a code block with the largest indentation possible
     def pop
-      @frontier.pop
+      @frontier.to_a.pop
     end
 
     def next_indent_line
@@ -85,15 +109,15 @@ module DeadEnd
     end
 
     def expand?
-      return false if @frontier.empty?
-      return true if @unvisited_lines.empty?
+      return false if @frontier.to_a.empty?
+      return true if @unvisited_lines.to_a.empty?
 
-      frontier_indent = @frontier.last.current_indent
+      frontier_indent = @frontier.to_a.last.current_indent
       unvisited_indent = next_indent_line.indent
 
       if ENV["DEBUG"]
         puts "```"
-        puts @frontier.last.to_s
+        puts @frontier.to_a.last.to_s
         puts "```"
         puts "  @frontier indent:  #{frontier_indent}"
         puts "  @unvisited indent: #{unvisited_indent}"
@@ -117,11 +141,13 @@ module DeadEnd
       register_indent_block(block)
 
       # Make sure we don't double expand, if a code block fully engulfs another code block, keep the bigger one
-      @frontier.reject! { |b|
+      @frontier.to_a.reject! { |b|
         b.starts_at >= block.starts_at && b.ends_at <= block.ends_at
       }
+
+      @check_next = true if block.invalid?
       @frontier << block
-      @frontier.sort!
+      # @frontier.sort!
 
       self
     end
@@ -141,8 +167,8 @@ module DeadEnd
     # Given that we know our syntax error exists somewhere in our frontier, we want to find
     # the smallest possible set of blocks that contain all the syntax errors
     def detect_invalid_blocks
-      self.class.combination(@frontier.select(&:invalid?)).detect do |block_array|
-        holds_all_syntax_errors?(block_array)
+      self.class.combination(@frontier.to_a.select(&:invalid?)).detect do |block_array|
+        holds_all_syntax_errors?(block_array, can_cache: false)
       end || []
     end
   end

data/lib/dead_end/code_search.rb

@@ -43,8 +43,7 @@ module DeadEnd
 
     def initialize(source, record_dir: ENV["DEAD_END_RECORD_DIR"] || ENV["DEBUG"] ? "tmp" : nil)
       if record_dir
-        @time = Time.now.strftime("%Y-%m-%d-%H-%M-%s-%N")
-        @record_dir = Pathname(record_dir).join(@time).tap { |p| p.mkpath }
+        @record_dir = DeadEnd.record_dir(record_dir)
         @write_count = 0
       end
 
@@ -73,12 +72,13 @@ module DeadEnd
         puts "  block indent:      #{block.current_indent}"
       end
       @record_dir.join(filename).open(mode: "a") do |f|
-        display = DisplayInvalidBlocks.new(
-          blocks: block,
+        document = DisplayCodeWithLineNumbers.new(
+          lines: @code_lines.select(&:visible?),
           terminal: false,
-          code_lines: @code_lines
-        )
-        f.write(display.indent(display.code_with_lines))
+          highlight_lines: block.lines
+        ).call
+
+        f.write(document)
       end
     end
 

data/lib/dead_end/display_invalid_blocks.rb

@@ -8,98 +8,67 @@ module DeadEnd
   class DisplayInvalidBlocks
     attr_reader :filename
 
-    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: false, invalid_obj: WhoDisSyntaxError::Null.new)
-      @terminal = terminal
-      @filename = filename
+    def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: DEFAULT_VALUE)
       @io = io
-
       @blocks = Array(blocks)
-
-      @invalid_lines = @blocks.map(&:lines).flatten
+      @filename = filename
       @code_lines = code_lines
 
-      @invalid_obj = invalid_obj
+      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
+    end
+
+    def document_ok?
+      @blocks.none? { |b| !b.hidden? }
     end
 
     def call
-      if @blocks.any? { |b| !b.hidden? }
-        found_invalid_blocks
-      else
+      if document_ok?
         @io.puts "Syntax OK"
+        return self
       end
-      self
-    end
 
-    private def no_invalid_blocks
-      @io.puts <<~EOM
-      EOM
-    end
-
-    private def found_invalid_blocks
-      @io.puts
-      if banner
-        @io.puts banner
+      if filename
+        @io.puts("--> #{filename}")
         @io.puts
       end
-      @io.puts("file: #{filename}") if filename
-      @io.puts <<~EOM
-        simplified:
+      @blocks.each do |block|
+        display_block(block)
+      end
 
-        #{indent(code_block)}
-      EOM
+      self
     end
 
-    def banner
-      case @invalid_obj.error_symbol
-      when :missing_end
-        <<~EOM
-          DeadEnd: Missing `end` detected
-
-          This code has a missing `end`. Ensure that all
-          syntax keywords (`def`, `do`, etc.) have a matching `end`.
-        EOM
-      when :unmatched_syntax
-        case @invalid_obj.unmatched_symbol
-        when :end
-          <<~EOM
-            DeadEnd: Unmatched `end` detected
-
-            This code has an unmatched `end`. Ensure that all `end` lines
-            in your code have a matching syntax keyword  (`def`,  `do`, etc.)
-            and that you don't have any extra `end` lines.
-          EOM
-        when :|
-          <<~EOM
-            DeadEnd: Unmatched `|` character detected
+    private def display_block(block)
+      # Build explanation
+      explain = ExplainSyntax.new(
+        code_lines: block.lines
+      ).call
 
-            Example:
+      # Enhance code output
+      # Also handles several ambiguious cases
+      lines = CaptureCodeContext.new(
+        blocks: block,
+        code_lines: @code_lines
+      ).call
 
-              `do |x` should be `do |x|`
-          EOM
-        when :"}"
-          <<~EOM
-            DeadEnd: Unmatched `}` character detected
+      # Build code output
+      document = DisplayCodeWithLineNumbers.new(
+        lines: lines,
+        terminal: @terminal,
+        highlight_lines: block.lines
+      ).call
 
-            This code has an unmatched `}`. Ensure that opening curly braces are
-            closed: `{ }`.
-          EOM
-        else
-          "DeadEnd: Unmatched `#{@invalid_obj.unmatched_symbol}` detected"
-        end
+      # Output syntax error explanation
+      explain.errors.each do |e|
+        @io.puts e
       end
-    end
-
-    def indent(string, with: "    ")
-      string.each_line.map { |l| with + l }.join
-    end
+      @io.puts
 
-    def code_block
-      string = +""
-      string << code_with_context
-      string
+      # Output code
+      @io.puts(document)
     end
 
-    def code_with_context
+    private def code_with_context
       lines = CaptureCodeContext.new(
         blocks: @blocks,
         code_lines: @code_lines
@@ -111,13 +80,5 @@ module DeadEnd
         highlight_lines: @invalid_lines
       ).call
     end
-
-    def code_with_lines
-      DisplayCodeWithLineNumbers.new(
-        lines: @code_lines.select(&:visible?),
-        terminal: @terminal,
-        highlight_lines: @invalid_lines
-      ).call
-    end
   end
 end

data/lib/dead_end/explain_syntax.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative "left_right_lex_count"
+
+module DeadEnd
+  # Explains syntax errors based on their source
+  #
+  # example:
+  #
+  #   source = "def foo; puts 'lol'" # Note missing end
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "Unmatched keyword, missing `end' ?"
+  #
+  # When the error cannot be determined by lexical counting
+  # then ripper is run against the input and the raw ripper
+  # errors returned.
+  #
+  # Example:
+  #
+  #   source = "1 * " # Note missing a second number
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "syntax error, unexpected end-of-input"
+  class ExplainSyntax
+    INVERSE = {
+      "{" => "}",
+      "}" => "{",
+      "[" => "]",
+      "]" => "[",
+      "(" => ")",
+      ")" => "(",
+      "|" => "|"
+    }.freeze
+
+    def initialize(code_lines:)
+      @code_lines = code_lines
+      @left_right = LeftRightLexCount.new
+      @missing = nil
+    end
+
+    def call
+      @code_lines.each do |line|
+        line.lex.each do |lex|
+          @left_right.count_lex(lex)
+        end
+      end
+
+      self
+    end
+
+    # Returns an array of missing elements
+    #
+    # For example this:
+    #
+    #   ExplainSyntax.new(code_lines: lines).missing
+    #   # => ["}"]
+    #
+    # Would indicate that the source is missing
+    # a `}` character in the source code
+    def missing
+      @missing ||= @left_right.missing
+    end
+
+    # Converts a missing string to
+    # an human understandable explanation.
+    #
+    # Example:
+    #
+    #   explain.why("}")
+    #   # => "Unmatched `{', missing `}' ?"
+    #
+    def why(miss)
+      case miss
+      when "keyword"
+        "Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ?"
+      when "end"
+        "Unmatched keyword, missing `end' ?"
+      else
+        inverse = INVERSE.fetch(miss) {
+          raise "Unknown explain syntax char or key: #{miss.inspect}"
+        }
+        "Unmatched `#{inverse}', missing `#{miss}' ?"
+      end
+    end
+
+    # Returns an array of syntax error messages
+    #
+    # If no missing pairs are found it falls back
+    # on the original ripper error messages
+    def errors
+      if missing.empty?
+        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+      end
+
+      missing.map { |miss| why(miss) }
+    end
+  end
+end

data/lib/dead_end/insertion_sort.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DeadEnd
+  # Sort elements on insert
+  #
+  # Instead of constantly calling `sort!`, put
+  # the element where it belongs the first time
+  # around
+  #
+  # Example:
+  #
+  #   sorted = InsertionSort.new
+  #   sorted << 33
+  #   sorted << 44
+  #   sorted << 1
+  #   puts sorted.to_a
+  #   # => [1, 44, 33]
+  #
+  class InsertionSort
+    def initialize
+      @array = []
+    end
+
+    def <<(value)
+      insert_in = @array.length
+      @array.each.with_index do |existing, index|
+        case value <=> existing
+        when -1
+          insert_in = index
+          break
+        when 0
+          insert_in = index
+          break
+        when 1
+          # Keep going
+        end
+      end
+
+      @array.insert(insert_in, value)
+    end
+
+    def to_a
+      @array
+    end
+  end
+end

data/lib/dead_end/left_right_lex_count.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module DeadEnd
+  # Find mis-matched syntax based on lexical count
+  #
+  # Used for detecting missing pairs of elements
+  # each keyword needs an end, each '{' needs a '}'
+  # etc.
+  #
+  # Example:
+  #
+  #   left_right = LeftRightLexCount.new
+  #   left_right.count_kw
+  #   left_right.missing.first
+  #   # => "end"
+  #
+  #   left_right = LeftRightLexCount.new
+  #   source = "{ a: b, c: d" # Note missing '}'
+  #   LexAll.new(source: source).each do |lex|
+  #     left_right.count_lex(lex)
+  #   end
+  #   left_right.missing.first
+  #   # => "}"
+  class LeftRightLexCount
+    def initialize
+      @kw_count = 0
+      @end_count = 0
+
+      @count_for_char = {
+        "{" => 0,
+        "}" => 0,
+        "[" => 0,
+        "]" => 0,
+        "(" => 0,
+        ")" => 0,
+        "|" => 0
+      }
+    end
+
+    def count_kw
+      @kw_count += 1
+    end
+
+    def count_end
+      @end_count += 1
+    end
+
+    # Count source code characters
+    #
+    # Example:
+    #
+    #   left_right = LeftRightLexCount.new
+    #   left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG))
+    #   left_right.count_for_char("{")
+    #   # => 1
+    #   left_right.count_for_char("}")
+    #   # => 0
+    def count_lex(lex)
+      case lex.type
+      when :on_tstring_content
+        # ^^^
+        # Means it's a string or a symbol `"{"` rather than being
+        # part of a data structure (like a hash) `{ a: b }`
+      when :on_embexpr_beg
+        # ^^^
+        # Embedded string expressions like `"#{foo} <-embed"`
+        # are parsed with chars:
+        #
+        # `#{` as :on_embexpr_beg
+        #  `}` as :on_embexpr_end
+        #
+        # We cannot ignore both :on_emb_expr_beg and :on_embexpr_end
+        # because sometimes the lexer thinks something is an embed
+        # string end, when it is not like `lol = }` (no clue why).
+        #
+        # When we see `#{` count it as a `{` or we will
+        # have a mis-match count.
+        #
+        case lex.token
+        when "\#{"
+          @count_for_char["{"] += 1
+        end
+      else
+        @end_count += 1 if lex.is_end?
+        @kw_count += 1 if lex.is_kw?
+        @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token)
+      end
+    end
+
+    def count_for_char(char)
+      @count_for_char[char]
+    end
+
+    # Returns an array of missing syntax characters
+    # or `"end"` or `"keyword"`
+    #
+    #   left_right.missing
+    #   # => ["}"]
+    def missing
+      out = missing_pairs
+      out << missing_pipe
+      out << missing_keyword_end
+      out.compact!
+      out
+    end
+
+    PAIRS = {
+      "{" => "}",
+      "[" => "]",
+      "(" => ")"
+    }.freeze
+
+    # Opening characters like `{` need closing characters # like `}`.
+    #
+    # When a mis-match count is detected, suggest the
+    # missing member.
+    #
+    # For example if there are 3 `}` and only two `{`
+    # return `"{"`
+    private def missing_pairs
+      PAIRS.map do |(left, right)|
+        case @count_for_char[left] <=> @count_for_char[right]
+        when 1
+          right
+        when 0
+          nil
+        when -1
+          left
+        end
+      end
+    end
+
+    # Keywords need ends and ends need keywords
+    #
+    # If we have more keywords, there's a missing `end`
+    # if we have more `end`-s, there's a missing keyword
+    private def missing_keyword_end
+      case @kw_count <=> @end_count
+      when 1
+        "end"
+      when 0
+        nil
+      when -1
+        "keyword"
+      end
+    end
+
+    # Pipes come in pairs.
+    # If there's an odd number of pipes then we
+    # are missing one
+    private def missing_pipe
+      if @count_for_char["|"].odd?
+        "|"
+      end
+    end
+  end
+end

data/lib/dead_end/lex_all.rb

@@ -15,7 +15,7 @@ module DeadEnd
       last_lineno = source_lines.count
 
       until lineno >= last_lineno
-        lines = source_lines[lineno..]
+        lines = source_lines[lineno..-1]
 
         @lex.concat(Ripper.lex(lines.join, "-", lineno + 1))
         lineno = @lex.last.first.first + 1

data/lib/dead_end/ripper_errors.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DeadEnd
+  # Capture parse errors from ripper
+  #
+  # Example:
+  #
+  #   puts RipperErrors.new(" def foo").call.errors
+  #   # => ["syntax error, unexpected end-of-input, expecting ';' or '\\n'"]
+  class RipperErrors < Ripper
+    attr_reader :errors
+
+    # Comes from ripper, called
+    # on every parse error, msg
+    # is a string
+    def on_parse_error(msg)
+      @errors ||= []
+      @errors << msg
+    end
+
+    def call
+      @run_once ||= begin
+        @errors = []
+        parse
+        true
+      end
+      self
+    end
+  end
+end

data/lib/dead_end/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DeadEnd
-  VERSION = "2.0.0"
+  VERSION = "3.0.1"
 end