syntax_suggest 1.0.4 → 2.0.0

data/lib/syntax_suggest/block_expand.rb

@@ -61,11 +61,14 @@ module SyntaxSuggest
     # they can expand to capture more code up and down). It does this conservatively
     # as there's no undo (currently).
     def expand_indent(block)
-      AroundBlockScan.new(code_lines: @code_lines, block: block)
+      now = AroundBlockScan.new(code_lines: @code_lines, block: block)
         .force_add_hidden
         .stop_after_kw
         .scan_adjacent_indent
-        .code_block
+
+      now.lookahead_balance_one_line
+
+      now.code_block
     end
 
     # A neighbor is code that is at or above the current indent line.
@@ -125,17 +128,20 @@ module SyntaxSuggest
     #
     # We try to resolve this edge case with `lookahead_balance_one_line` below.
     def expand_neighbors(block)
-      neighbors = AroundBlockScan.new(code_lines: @code_lines, block: block)
+      now = AroundBlockScan.new(code_lines: @code_lines, block: block)
+
+      # Initial scan
+      now
         .force_add_hidden
         .stop_after_kw
         .scan_neighbors_not_empty
 
       # Slurp up empties
-      with_empties = neighbors
+      now
         .scan_while { |line| line.empty? }
 
       # If next line is kw and it will balance us, take it
-      expanded_lines = with_empties
+      expanded_lines = now
         .lookahead_balance_one_line
         .lines
 

data/lib/syntax_suggest/capture/before_after_keyword_ends.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  module Capture
+    # Shows surrounding kw/end pairs
+    #
+    # The purpose of showing these extra pairs is due to cases
+    # of ambiguity when only one visible line is matched.
+    #
+    # For example:
+    #
+    #     1  class Dog
+    #     2    def bark
+    #     4    def eat
+    #     5    end
+    #     6  end
+    #
+    # In this case either line 2 could be missing an `end` or
+    # line 4 was an extra line added by mistake (it happens).
+    #
+    # When we detect the above problem it shows the issue
+    # as only being on line 2
+    #
+    #     2    def bark
+    #
+    # Showing "neighbor" keyword pairs gives extra context:
+    #
+    #     2    def bark
+    #     4    def eat
+    #     5    end
+    #
+    #
+    # Example:
+    #
+    #   lines = BeforeAfterKeywordEnds.new(
+    #     block: block,
+    #     code_lines: code_lines
+    #   ).call()
+    #
+    class BeforeAfterKeywordEnds
+      def initialize(code_lines:, block:)
+        @scanner = ScanHistory.new(code_lines: code_lines, block: block)
+        @original_indent = block.current_indent
+      end
+
+      def call
+        lines = []
+
+        @scanner.scan(
+          up: ->(line, kw_count, end_count) {
+            next true if line.empty?
+            break if line.indent < @original_indent
+            next true if line.indent != @original_indent
+
+            # If we're going up and have one complete kw/end pair, stop
+            if kw_count != 0 && kw_count == end_count
+              lines << line
+              break
+            end
+
+            lines << line if line.is_kw? || line.is_end?
+            true
+          },
+          down: ->(line, kw_count, end_count) {
+            next true if line.empty?
+            break if line.indent < @original_indent
+            next true if line.indent != @original_indent
+
+            # if we're going down and have one complete kw/end pair,stop
+            if kw_count != 0 && kw_count == end_count
+              lines << line
+              break
+            end
+
+            lines << line if line.is_kw? || line.is_end?
+            true
+          }
+        )
+        @scanner.stash_changes
+
+        lines
+      end
+    end
+  end
+end

data/lib/syntax_suggest/capture/falling_indent_lines.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  module Capture
+    # Shows the context around code provided by "falling" indentation
+    #
+    # If this is the original code lines:
+    #
+    #   class OH
+    #     def hello
+    #       it "foo" do
+    #     end
+    #   end
+    #
+    # And this is the line that is captured
+    #
+    #       it "foo" do
+    #
+    # It will yield its surrounding context:
+    #
+    #   class OH
+    #     def hello
+    #     end
+    #   end
+    #
+    # Example:
+    #
+    #   FallingIndentLines.new(
+    #       block: block,
+    #       code_lines: @code_lines
+    #   ).call do |line|
+    #     @lines_to_output << line
+    #   end
+    #
+    class FallingIndentLines
+      def initialize(code_lines:, block:)
+        @lines = nil
+        @scanner = ScanHistory.new(code_lines: code_lines, block: block)
+        @original_indent = block.current_indent
+      end
+
+      def call(&yieldable)
+        last_indent_up = @original_indent
+        last_indent_down = @original_indent
+
+        @scanner.commit_if_changed
+        @scanner.scan(
+          up: ->(line, _, _) {
+            next true if line.empty?
+
+            if line.indent < last_indent_up
+              yieldable.call(line)
+              last_indent_up = line.indent
+            end
+            true
+          },
+          down: ->(line, _, _) {
+            next true if line.empty?
+
+            if line.indent < last_indent_down
+              yieldable.call(line)
+              last_indent_down = line.indent
+            end
+            true
+          }
+        )
+        @scanner.stash_changes
+      end
+    end
+  end
+end

data/lib/syntax_suggest/capture_code_context.rb

@@ -1,5 +1,13 @@
 # frozen_string_literal: true
 
+module SyntaxSuggest
+  module Capture
+  end
+end
+
+require_relative "capture/falling_indent_lines"
+require_relative "capture/before_after_keyword_ends"
+
 module SyntaxSuggest
   # Turns a "invalid block(s)" into useful context
   #
@@ -55,6 +63,10 @@ module SyntaxSuggest
         capture_falling_indent(block)
       end
 
+      sorted_lines
+    end
+
+    def sorted_lines
       @lines_to_output.select!(&:not_empty?)
       @lines_to_output.uniq!
       @lines_to_output.sort!
@@ -77,10 +89,10 @@ module SyntaxSuggest
     #   end
     #
     def capture_falling_indent(block)
-      AroundBlockScan.new(
+      Capture::FallingIndentLines.new(
         block: block,
         code_lines: @code_lines
-      ).on_falling_indent do |line|
+      ).call do |line|
         @lines_to_output << line
       end
     end
@@ -115,9 +127,10 @@ module SyntaxSuggest
     def capture_before_after_kws(block)
       return unless block.visible_lines.count == 1
 
-      around_lines = AroundBlockScan.new(code_lines: @code_lines, block: block)
-        .start_at_next_line
-        .capture_neighbor_context
+      around_lines = Capture::BeforeAfterKeywordEnds.new(
+        code_lines: @code_lines,
+        block: block
+      ).call
 
       around_lines -= block.lines
 

data/lib/syntax_suggest/clean_document.rb

@@ -47,9 +47,9 @@ module SyntaxSuggest
   # ## Heredocs
   #
   # A heredoc is an way of defining a multi-line string. They can cause many
-  # problems. If left as a single line, Ripper would try to parse the contents
+  # problems. If left as a single line, the parser would try to parse the contents
   # as ruby code rather than as a string. Even without this problem, we still
-  # hit an issue with indentation
+  # hit an issue with indentation:
   #
   #    1 foo = <<~HEREDOC
   #    2  "Be yourself; everyone else is already taken.""
@@ -224,7 +224,7 @@ module SyntaxSuggest
     #
     def join_consecutive!
       consecutive_groups = @document.select(&:ignore_newline_not_beg?).map do |code_line|
-        take_while_including(code_line.index..-1) do |line|
+        take_while_including(code_line.index..) do |line|
           line.ignore_newline_not_beg?
         end
       end
@@ -245,7 +245,7 @@ module SyntaxSuggest
     #     expect(lines[1].to_s).to eq("")
     def join_trailing_slash!
       trailing_groups = @document.select(&:trailing_slash?).map do |code_line|
-        take_while_including(code_line.index..-1) { |x| x.trailing_slash? }
+        take_while_including(code_line.index..) { |x| x.trailing_slash? }
       end
       join_groups(trailing_groups)
       self
@@ -279,7 +279,7 @@ module SyntaxSuggest
         )
 
         # Hide the rest of the lines
-        lines[1..-1].each do |line|
+        lines[1..].each do |line|
           # The above lines already have newlines in them, if add more
           # then there will be double newline, use an empty line instead
           @document[line.index] = CodeLine.new(line: "", index: line.index, lex: [])
@@ -293,7 +293,7 @@ module SyntaxSuggest
     # Like `take_while` except when it stops
     # iterating, it also returns the line
     # that caused it to stop
-    def take_while_including(range = 0..-1)
+    def take_while_including(range = 0..)
       take_next_and_stop = false
       @document[range].take_while do |line|
         next if take_next_and_stop

data/lib/syntax_suggest/code_block.rb

@@ -81,7 +81,7 @@ module SyntaxSuggest
         # lines then the result cannot be invalid
         #
         # That means there's no reason to re-check all
-        # lines with ripper (which is expensive).
+        # lines with the parser (which is expensive).
         # Benchmark in commit message
         @valid = if lines.all? { |l| l.hidden? || l.empty? }
           true

data/lib/syntax_suggest/code_frontier.rb

@@ -117,7 +117,7 @@ module SyntaxSuggest
 
       if ENV["SYNTAX_SUGGEST_DEBUG"]
         puts "```"
-        puts @queue.peek.to_s
+        puts @queue.peek
         puts "```"
         puts "  @frontier indent:  #{frontier_indent}"
         puts "  @unvisited indent: #{unvisited_indent}"

data/lib/syntax_suggest/code_line.rb

@@ -180,12 +180,19 @@ module SyntaxSuggest
     #     EOM
     #     expect(lines.first.trailing_slash?).to eq(true)
     #
-    def trailing_slash?
-      last = @lex.last
-      return false unless last
-      return false unless last.type == :on_sp
+    if SyntaxSuggest.use_prism_parser?
+      def trailing_slash?
+        last = @lex.last
+        last&.type == :on_tstring_end
+      end
+    else
+      def trailing_slash?
+        last = @lex.last
+        return false unless last
+        return false unless last.type == :on_sp
 
-      last.token == TRAILING_SLASH
+        last.token == TRAILING_SLASH
+      end
     end
 
     # Endless method detection

data/lib/syntax_suggest/code_search.rb

@@ -43,7 +43,7 @@ module SyntaxSuggest
 
     def initialize(source, record_dir: DEFAULT_VALUE)
       record_dir = if record_dir == DEFAULT_VALUE
-        ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"] ? "tmp" : nil
+        (ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"]) ? "tmp" : nil
       else
         record_dir
       end
@@ -73,7 +73,7 @@ module SyntaxSuggest
       if ENV["SYNTAX_SUGGEST_DEBUG"]
         puts "\n\n==== #{filename} ===="
         puts "\n```#{block.starts_at}..#{block.ends_at}"
-        puts block.to_s
+        puts block
         puts "```"
         puts "  block indent:      #{block.current_indent}"
       end

data/lib/syntax_suggest/core_ext.rb

@@ -21,7 +21,7 @@ if SyntaxError.method_defined?(:detailed_message)
       attr_reader :string
     end
 
-    # SyntaxSuggest.record_dir [Private]
+    # SyntaxSuggest.module_for_detailed_message [Private]
     #
     # Used to monkeypatch SyntaxError via Module.prepend
     def self.module_for_detailed_message

data/lib/syntax_suggest/display_invalid_blocks.rb

@@ -14,7 +14,7 @@ module SyntaxSuggest
       @filename = filename
       @code_lines = code_lines
 
-      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
+      @terminal = (terminal == DEFAULT_VALUE) ? io.isatty : terminal
     end
 
     def document_ok?

data/lib/syntax_suggest/explain_syntax.rb

@@ -2,7 +2,21 @@
 
 require_relative "left_right_lex_count"
 
+if !SyntaxSuggest.use_prism_parser?
+  require_relative "ripper_errors"
+end
+
 module SyntaxSuggest
+  class GetParseErrors
+    def self.errors(source)
+      if SyntaxSuggest.use_prism_parser?
+        Prism.parse(source).errors.map(&:message)
+      else
+        RipperErrors.new(source).call.errors
+      end
+    end
+  end
+
   # Explains syntax errors based on their source
   #
   # example:
@@ -15,8 +29,8 @@ module SyntaxSuggest
   #   # => "Unmatched keyword, missing `end' ?"
   #
   # When the error cannot be determined by lexical counting
-  # then ripper is run against the input and the raw ripper
-  # errors returned.
+  # then the parser is run against the input and the raw
+  # errors are returned.
   #
   # Example:
   #
@@ -91,10 +105,10 @@ module SyntaxSuggest
     # Returns an array of syntax error messages
     #
     # If no missing pairs are found it falls back
-    # on the original ripper error messages
+    # on the original error messages
     def errors
       if missing.empty?
-        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+        return GetParseErrors.errors(@code_lines.map(&:original).join).uniq
       end
 
       missing.map { |miss| why(miss) }

data/lib/syntax_suggest/lex_all.rb

@@ -3,34 +3,53 @@
 module SyntaxSuggest
   # Ripper.lex is not guaranteed to lex the entire source document
   #
-  # lex = LexAll.new(source: source)
-  # lex.each do |value|
-  #   puts value.line
-  # end
+  # This class guarantees the whole document is lex-ed by iteratively
+  # lexing the document where ripper stopped.
+  #
+  # Prism likely doesn't have the same problem. Once ripper support is removed
+  # we can likely reduce the complexity here if not remove the whole concept.
+  #
+  # Example usage:
+  #
+  #   lex = LexAll.new(source: source)
+  #   lex.each do |value|
+  #     puts value.line
+  #   end
   class LexAll
     include Enumerable
 
     def initialize(source:, source_lines: nil)
-      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
-      lineno = @lex.last.pos.first + 1
+      @lex = self.class.lex(source, 1)
+      lineno = @lex.last[0][0] + 1
       source_lines ||= source.lines
       last_lineno = source_lines.length
 
       until lineno >= last_lineno
-        lines = source_lines[lineno..-1]
+        lines = source_lines[lineno..]
 
         @lex.concat(
-          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+          self.class.lex(lines.join, lineno + 1)
         )
-        lineno = @lex.last.pos.first + 1
+
+        lineno = @lex.last[0].first + 1
       end
 
       last_lex = nil
       @lex.map! { |elem|
-        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
       }
     end
 
+    if SyntaxSuggest.use_prism_parser?
+      def self.lex(source, line_number)
+        Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] }
+      end
+    else
+      def self.lex(source, line_number)
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    end
+
     def to_a
       @lex
     end

data/lib/syntax_suggest/pathname_from_message.rb

@@ -13,7 +13,7 @@ module SyntaxSuggest
   #    # => "/tmp/scratch.rb"
   #
   class PathnameFromMessage
-    EVAL_RE = /^\(eval\):\d+/
+    EVAL_RE = /^\(eval.*\):\d+/
     STREAMING_RE = /^-:\d+/
     attr_reader :name
 

data/lib/syntax_suggest/ripper_errors.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 module SyntaxSuggest
-  # Capture parse errors from ripper
+  # Capture parse errors from Ripper
+  #
+  # Prism returns the errors with their messages, but Ripper
+  # does not. To get them we must make a custom subclass.
   #
   # Example:
   #

data/lib/syntax_suggest/scan_history.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module SyntaxSuggest
+  # Scans up/down from the given block
+  #
+  # You can try out a change, stash it, or commit it to save for later
+  #
+  # Example:
+  #
+  #   scanner = ScanHistory.new(code_lines: code_lines, block: block)
+  #   scanner.scan(
+  #     up: ->(_, _, _) { true },
+  #     down: ->(_, _, _) { true }
+  #   )
+  #   scanner.changed? # => true
+  #   expect(scanner.lines).to eq(code_lines)
+  #
+  #   scanner.stash_changes
+  #
+  #   expect(scanner.lines).to_not eq(code_lines)
+  class ScanHistory
+    attr_reader :before_index, :after_index
+
+    def initialize(code_lines:, block:)
+      @code_lines = code_lines
+      @history = [block]
+      refresh_index
+    end
+
+    def commit_if_changed
+      if changed?
+        @history << CodeBlock.new(lines: @code_lines[before_index..after_index])
+      end
+
+      self
+    end
+
+    # Discards any changes that have not been committed
+    def stash_changes
+      refresh_index
+      self
+    end
+
+    # Discard changes that have not been committed and revert the last commit
+    #
+    # Cannot revert the first commit
+    def revert_last_commit
+      if @history.length > 1
+        @history.pop
+        refresh_index
+      end
+
+      self
+    end
+
+    def changed?
+      @before_index != current.lines.first.index ||
+        @after_index != current.lines.last.index
+    end
+
+    # Iterates up and down
+    #
+    # Returns line, kw_count, end_count for each iteration
+    def scan(up:, down:)
+      kw_count = 0
+      end_count = 0
+
+      up_index = before_lines.reverse_each.take_while do |line|
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+        up.call(line, kw_count, end_count)
+      end.last&.index
+
+      kw_count = 0
+      end_count = 0
+
+      down_index = after_lines.each.take_while do |line|
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+        down.call(line, kw_count, end_count)
+      end.last&.index
+
+      @before_index = if up_index && up_index < @before_index
+        up_index
+      else
+        @before_index
+      end
+
+      @after_index = if down_index && down_index > @after_index
+        down_index
+      else
+        @after_index
+      end
+
+      self
+    end
+
+    def next_up
+      return nil if @before_index <= 0
+
+      @code_lines[@before_index - 1]
+    end
+
+    def next_down
+      return nil if @after_index >= @code_lines.length
+
+      @code_lines[@after_index + 1]
+    end
+
+    def lines
+      @code_lines[@before_index..@after_index]
+    end
+
+    private def before_lines
+      @code_lines[0...@before_index] || []
+    end
+
+    # Returns an array of all the CodeLines that exist after
+    # the currently scanned block
+    private def after_lines
+      @code_lines[@after_index.next..] || []
+    end
+
+    private def current
+      @history.last
+    end
+
+    private def refresh_index
+      @before_index = current.lines.first.index
+      @after_index = current.lines.last.index
+      self
+    end
+  end
+end

data/lib/syntax_suggest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxSuggest
-  VERSION = "1.0.4"
+  VERSION = "2.0.0"
 end

data/syntax_suggest.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it'
   spec.homepage = "https://github.com/ruby/syntax_suggest.git"
   spec.license = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git"
@@ -27,6 +27,6 @@ Gem::Specification.new do |spec|
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|assets)/}) }
   end
   spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables = ["syntax_suggest"]
   spec.require_paths = ["lib"]
 end