syntax_suggest 0.0.1

data/lib/syntax_suggest/explain_syntax.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative "left_right_lex_count"
+
+module SyntaxSuggest
+  # Explains syntax errors based on their source
+  #
+  # example:
+  #
+  #   source = "def foo; puts 'lol'" # Note missing end
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "Unmatched keyword, missing `end' ?"
+  #
+  # When the error cannot be determined by lexical counting
+  # then ripper is run against the input and the raw ripper
+  # errors returned.
+  #
+  # Example:
+  #
+  #   source = "1 * " # Note missing a second number
+  #   explain ExplainSyntax.new(
+  #     code_lines: CodeLine.from_source(source)
+  #   ).call
+  #   explain.errors.first
+  #   # => "syntax error, unexpected end-of-input"
+  class ExplainSyntax
+    INVERSE = {
+      "{" => "}",
+      "}" => "{",
+      "[" => "]",
+      "]" => "[",
+      "(" => ")",
+      ")" => "(",
+      "|" => "|"
+    }.freeze
+
+    def initialize(code_lines:)
+      @code_lines = code_lines
+      @left_right = LeftRightLexCount.new
+      @missing = nil
+    end
+
+    def call
+      @code_lines.each do |line|
+        line.lex.each do |lex|
+          @left_right.count_lex(lex)
+        end
+      end
+
+      self
+    end
+
+    # Returns an array of missing elements
+    #
+    # For example this:
+    #
+    #   ExplainSyntax.new(code_lines: lines).missing
+    #   # => ["}"]
+    #
+    # Would indicate that the source is missing
+    # a `}` character in the source code
+    def missing
+      @missing ||= @left_right.missing
+    end
+
+    # Converts a missing string to
+    # an human understandable explanation.
+    #
+    # Example:
+    #
+    #   explain.why("}")
+    #   # => "Unmatched `{', missing `}' ?"
+    #
+    def why(miss)
+      case miss
+      when "keyword"
+        "Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ?"
+      when "end"
+        "Unmatched keyword, missing `end' ?"
+      else
+        inverse = INVERSE.fetch(miss) {
+          raise "Unknown explain syntax char or key: #{miss.inspect}"
+        }
+        "Unmatched `#{inverse}', missing `#{miss}' ?"
+      end
+    end
+
+    # Returns an array of syntax error messages
+    #
+    # If no missing pairs are found it falls back
+    # on the original ripper error messages
+    def errors
+      if missing.empty?
+        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+      end
+
+      missing.map { |miss| why(miss) }
+    end
+  end
+end

data/lib/syntax_suggest/left_right_lex_count.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Find mis-matched syntax based on lexical count
+  #
+  # Used for detecting missing pairs of elements
+  # each keyword needs an end, each '{' needs a '}'
+  # etc.
+  #
+  # Example:
+  #
+  #   left_right = LeftRightLexCount.new
+  #   left_right.count_kw
+  #   left_right.missing.first
+  #   # => "end"
+  #
+  #   left_right = LeftRightLexCount.new
+  #   source = "{ a: b, c: d" # Note missing '}'
+  #   LexAll.new(source: source).each do |lex|
+  #     left_right.count_lex(lex)
+  #   end
+  #   left_right.missing.first
+  #   # => "}"
+  class LeftRightLexCount
+    def initialize
+      @kw_count = 0
+      @end_count = 0
+
+      @count_for_char = {
+        "{" => 0,
+        "}" => 0,
+        "[" => 0,
+        "]" => 0,
+        "(" => 0,
+        ")" => 0,
+        "|" => 0
+      }
+    end
+
+    def count_kw
+      @kw_count += 1
+    end
+
+    def count_end
+      @end_count += 1
+    end
+
+    # Count source code characters
+    #
+    # Example:
+    #
+    #   left_right = LeftRightLexCount.new
+    #   left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG))
+    #   left_right.count_for_char("{")
+    #   # => 1
+    #   left_right.count_for_char("}")
+    #   # => 0
+    def count_lex(lex)
+      case lex.type
+      when :on_tstring_content
+        # ^^^
+        # Means it's a string or a symbol `"{"` rather than being
+        # part of a data structure (like a hash) `{ a: b }`
+        # ignore it.
+      when :on_words_beg, :on_symbos_beg, :on_qwords_beg,
+           :on_qsymbols_beg, :on_regexp_beg, :on_tstring_beg
+        # ^^^
+        # Handle shorthand syntaxes like `%Q{ i am a string }`
+        #
+        # The start token will be the full thing `%Q{` but we
+        # need to count it as if it's a `{`. Any token
+        # can be used
+        char = lex.token[-1]
+        @count_for_char[char] += 1 if @count_for_char.key?(char)
+      when :on_embexpr_beg
+        # ^^^
+        # Embedded string expressions like `"#{foo} <-embed"`
+        # are parsed with chars:
+        #
+        # `#{` as :on_embexpr_beg
+        #  `}` as :on_embexpr_end
+        #
+        # We cannot ignore both :on_emb_expr_beg and :on_embexpr_end
+        # because sometimes the lexer thinks something is an embed
+        # string end, when it is not like `lol = }` (no clue why).
+        #
+        # When we see `#{` count it as a `{` or we will
+        # have a mis-match count.
+        #
+        case lex.token
+        when "\#{"
+          @count_for_char["{"] += 1
+        end
+      else
+        @end_count += 1 if lex.is_end?
+        @kw_count += 1 if lex.is_kw?
+        @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token)
+      end
+    end
+
+    def count_for_char(char)
+      @count_for_char[char]
+    end
+
+    # Returns an array of missing syntax characters
+    # or `"end"` or `"keyword"`
+    #
+    #   left_right.missing
+    #   # => ["}"]
+    def missing
+      out = missing_pairs
+      out << missing_pipe
+      out << missing_keyword_end
+      out.compact!
+      out
+    end
+
+    PAIRS = {
+      "{" => "}",
+      "[" => "]",
+      "(" => ")"
+    }.freeze
+
+    # Opening characters like `{` need closing characters # like `}`.
+    #
+    # When a mis-match count is detected, suggest the
+    # missing member.
+    #
+    # For example if there are 3 `}` and only two `{`
+    # return `"{"`
+    private def missing_pairs
+      PAIRS.map do |(left, right)|
+        case @count_for_char[left] <=> @count_for_char[right]
+        when 1
+          right
+        when 0
+          nil
+        when -1
+          left
+        end
+      end
+    end
+
+    # Keywords need ends and ends need keywords
+    #
+    # If we have more keywords, there's a missing `end`
+    # if we have more `end`-s, there's a missing keyword
+    private def missing_keyword_end
+      case @kw_count <=> @end_count
+      when 1
+        "end"
+      when 0
+        nil
+      when -1
+        "keyword"
+      end
+    end
+
+    # Pipes come in pairs.
+    # If there's an odd number of pipes then we
+    # are missing one
+    private def missing_pipe
+      if @count_for_char["|"].odd?
+        "|"
+      end
+    end
+  end
+end

data/lib/syntax_suggest/lex_all.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Ripper.lex is not guaranteed to lex the entire source document
+  #
+  # lex = LexAll.new(source: source)
+  # lex.each do |value|
+  #   puts value.line
+  # end
+  class LexAll
+    include Enumerable
+
+    def initialize(source:, source_lines: nil)
+      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
+      lineno = @lex.last.pos.first + 1
+      source_lines ||= source.lines
+      last_lineno = source_lines.length
+
+      until lineno >= last_lineno
+        lines = source_lines[lineno..-1]
+
+        @lex.concat(
+          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+        )
+        lineno = @lex.last.pos.first + 1
+      end
+
+      last_lex = nil
+      @lex.map! { |elem|
+        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+      }
+    end
+
+    def to_a
+      @lex
+    end
+
+    def each
+      return @lex.each unless block_given?
+      @lex.each do |x|
+        yield x
+      end
+    end
+
+    def [](index)
+      @lex[index]
+    end
+
+    def last
+      @lex.last
+    end
+  end
+end
+
+require_relative "lex_value"

data/lib/syntax_suggest/lex_value.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Value object for accessing lex values
+  #
+  # This lex:
+  #
+  #   [1, 0], :on_ident, "describe", CMDARG
+  #
+  # Would translate into:
+  #
+  #  lex.line # => 1
+  #  lex.type # => :on_indent
+  #  lex.token # => "describe"
+  class LexValue
+    attr_reader :line, :type, :token, :state
+
+    def initialize(line, type, token, state, last_lex = nil)
+      @line = line
+      @type = type
+      @token = token
+      @state = state
+
+      set_kw_end(last_lex)
+    end
+
+    private def set_kw_end(last_lex)
+      @is_end = false
+      @is_kw = false
+      return if type != :on_kw
+      #
+      return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953
+
+      case token
+      when "if", "unless", "while", "until"
+        # Only count if/unless when it's not a "trailing" if/unless
+        # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375
+        @is_kw = true unless expr_label?
+      when "def", "case", "for", "begin", "class", "module", "do"
+        @is_kw = true
+      when "end"
+        @is_end = true
+      end
+    end
+
+    def fname?
+      state.allbits?(Ripper::EXPR_FNAME)
+    end
+
+    def ignore_newline?
+      type == :on_ignored_nl
+    end
+
+    def is_end?
+      @is_end
+    end
+
+    def is_kw?
+      @is_kw
+    end
+
+    def expr_beg?
+      state.anybits?(Ripper::EXPR_BEG)
+    end
+
+    def expr_label?
+      state.allbits?(Ripper::EXPR_LABEL)
+    end
+  end
+end

data/lib/syntax_suggest/parse_blocks_from_indent_line.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # This class is responsible for generating initial code blocks
+  # that will then later be expanded.
+  #
+  # The biggest concern when guessing code blocks, is accidentally
+  # grabbing one that contains only an "end". In this example:
+  #
+  #   def dog
+  #     begonn # mispelled `begin`
+  #     puts "bark"
+  #     end
+  #   end
+  #
+  # The following lines would be matched (from bottom to top):
+  #
+  #   1) end
+  #
+  #   2) puts "bark"
+  #      end
+  #
+  #   3) begonn
+  #      puts "bark"
+  #      end
+  #
+  # At this point it has no where else to expand, and it will yield this inner
+  # code as a block
+  class ParseBlocksFromIndentLine
+    attr_reader :code_lines
+
+    def initialize(code_lines:)
+      @code_lines = code_lines
+    end
+
+    # Builds blocks from bottom up
+    def each_neighbor_block(target_line)
+      scan = AroundBlockScan.new(code_lines: code_lines, block: CodeBlock.new(lines: target_line))
+        .skip(:empty?)
+        .skip(:hidden?)
+        .scan_while { |line| line.indent >= target_line.indent }
+
+      neighbors = scan.code_block.lines
+
+      block = CodeBlock.new(lines: neighbors)
+      if neighbors.length <= 2 || block.valid?
+        yield block
+      else
+        until neighbors.empty?
+          lines = [neighbors.pop]
+          while (block = CodeBlock.new(lines: lines)) && block.invalid? && neighbors.any?
+            lines.prepend neighbors.pop
+          end
+
+          yield block if block
+        end
+      end
+    end
+  end
+end

data/lib/syntax_suggest/pathname_from_message.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Converts a SyntaxError message to a path
+  #
+  # Handles the case where the filename has a colon in it
+  # such as on a windows file system: https://github.com/zombocom/syntax_suggest/issues/111
+  #
+  # Example:
+  #
+  #    message = "/tmp/scratch:2:in `require_relative': /private/tmp/bad.rb:1: syntax error, unexpected `end' (SyntaxError)"
+  #    puts PathnameFromMessage.new(message).call.name
+  #    # => "/tmp/scratch.rb"
+  #
+  class PathnameFromMessage
+    EVAL_RE = /^\(eval\):\d+/
+    STREAMING_RE = /^-:\d+/
+    attr_reader :name
+
+    def initialize(message, io: $stderr)
+      @line = message.lines.first
+      @parts = @line.split(":")
+      @guess = []
+      @name = nil
+      @io = io
+    end
+
+    def call
+      if skip_missing_file_name?
+        if ENV["SYNTAX_SUGGEST_DEBUG"]
+          @io.puts "SyntaxSuggest: Could not find filename from #{@line.inspect}"
+        end
+      else
+        until stop?
+          @guess << @parts.shift
+          @name = Pathname(@guess.join(":"))
+        end
+
+        if @parts.empty?
+          @io.puts "SyntaxSuggest: Could not find filename from #{@line.inspect}"
+          @name = nil
+        end
+      end
+
+      self
+    end
+
+    def stop?
+      return true if @parts.empty?
+      return false if @guess.empty?
+
+      @name&.exist?
+    end
+
+    def skip_missing_file_name?
+      @line.match?(EVAL_RE) || @line.match?(STREAMING_RE)
+    end
+  end
+end

data/lib/syntax_suggest/priority_engulf_queue.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Keeps track of what elements are in the queue in
+  # priority and also ensures that when one element
+  # engulfs/covers/eats another that the larger element
+  # evicts the smaller element
+  class PriorityEngulfQueue
+    def initialize
+      @queue = PriorityQueue.new
+    end
+
+    def to_a
+      @queue.to_a
+    end
+
+    def empty?
+      @queue.empty?
+    end
+
+    def length
+      @queue.length
+    end
+
+    def peek
+      @queue.peek
+    end
+
+    def pop
+      @queue.pop
+    end
+
+    def push(block)
+      prune_engulf(block)
+      @queue << block
+      flush_deleted
+
+      self
+    end
+
+    private def flush_deleted
+      while @queue&.peek&.deleted?
+        @queue.pop
+      end
+    end
+
+    private def prune_engulf(block)
+      # If we're about to pop off the same block, we can skip deleting
+      # things from the frontier this iteration since we'll get it
+      # on the next iteration
+      return if @queue.peek && (block <=> @queue.peek) == 1
+
+      if block.starts_at != block.ends_at # A block of size 1 cannot engulf another
+        @queue.to_a.each { |b|
+          if b.starts_at >= block.starts_at && b.ends_at <= block.ends_at
+            b.delete
+            true
+          end
+        }
+      end
+    end
+  end
+end

data/lib/syntax_suggest/priority_queue.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Holds elements in a priority heap on insert
+  #
+  # Instead of constantly calling `sort!`, put
+  # the element where it belongs the first time
+  # around
+  #
+  # Example:
+  #
+  #   queue = PriorityQueue.new
+  #   queue << 33
+  #   queue << 44
+  #   queue << 1
+  #
+  #   puts queue.peek # => 44
+  #
+  class PriorityQueue
+    attr_reader :elements
+
+    def initialize
+      @elements = []
+    end
+
+    def <<(element)
+      @elements << element
+      bubble_up(last_index, element)
+    end
+
+    def pop
+      exchange(0, last_index)
+      max = @elements.pop
+      bubble_down(0)
+      max
+    end
+
+    def length
+      @elements.length
+    end
+
+    def empty?
+      @elements.empty?
+    end
+
+    def peek
+      @elements.first
+    end
+
+    def to_a
+      @elements
+    end
+
+    # Used for testing, extremely not performant
+    def sorted
+      out = []
+      elements = @elements.dup
+      while (element = pop)
+        out << element
+      end
+      @elements = elements
+      out.reverse
+    end
+
+    private def last_index
+      @elements.size - 1
+    end
+
+    private def bubble_up(index, element)
+      return if index <= 0
+
+      parent_index = (index - 1) / 2
+      parent = @elements[parent_index]
+
+      return if (parent <=> element) >= 0
+
+      exchange(index, parent_index)
+      bubble_up(parent_index, element)
+    end
+
+    private def bubble_down(index)
+      child_index = (index * 2) + 1
+
+      return if child_index > last_index
+
+      not_the_last_element = child_index < last_index
+      left_element = @elements[child_index]
+      right_element = @elements[child_index + 1]
+
+      child_index += 1 if not_the_last_element && (right_element <=> left_element) == 1
+
+      return if (@elements[index] <=> @elements[child_index]) >= 0
+
+      exchange(index, child_index)
+      bubble_down(child_index)
+    end
+
+    def exchange(source, target)
+      a = @elements[source]
+      b = @elements[target]
+      @elements[source] = b
+      @elements[target] = a
+    end
+  end
+end