syntax_suggest 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0aa200b221c59254250acfa3343365ead0a7e8a0e699aec254e248a880bf4084
-  data.tar.gz: e343f13b8cfb3091b7e6f2ee97575fbe7e8e9b24d0f36168524a098e5d596098
+  metadata.gz: 7b3d83011e29784a7ea97dcae359b654ec8be6382c9fda7a5114353050ef5bf2
+  data.tar.gz: a2452866035aa4751948ac4f30cb7699352bc0778fb4b5bcae3ce3f421b901de
 SHA512:
-  metadata.gz: 5b5fe319b21f699251eb9d2a1580808fd8b454c8c5d04f1e0ac63619898c429726579596e2f182d98299f2850387ee34b245b6e4a9cfcfb17a1c76e160000a01
-  data.tar.gz: f1181078cb45c2c333fc96ff04efd0513b0da4088623ce9899eeda0ada7ba342d7a146bde872515678005cad6f200fe6116f4d33f600cd98f4aa520a13ccda21
+  metadata.gz: 52168116c672ee83619f59d1f158101adb64c650770424c3bd855fafb71131f72663c8afd060f8512bbe908fae2ecb8dea5a65c9384b980843c89901f70e202a
+  data.tar.gz: 5edfacbb66e7b37dc36f1b57de325ffea4656102b3a4638249577fb8c12460c2e05a3f7761675966e845214db891c83f5b51eef963d35da506478af7f0837874

data/.github/workflows/check_changelog.yml

@@ -13,7 +13,7 @@ jobs:
       !contains(github.event.pull_request.body, '[skip ci]') &&
       !contains(github.event.pull_request.labels.*.name, 'skip changelog')
     steps:
-      - uses: actions/checkout@v3.5.2
+      - uses: actions/checkout@v4.1.1
       - name: Check that CHANGELOG is touched
         run: |
           git fetch origin ${{ github.base_ref }} --depth 1 && \

data/.github/workflows/ci.yml

@@ -9,31 +9,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3.5.2
+        uses: actions/checkout@v4.1.1
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1
+          ruby-version: ruby
           bundler-cache: true
       - name: Linting
         run: bundle exec standardrb
+        env:
+          RUBYOPT: --disable=syntax_suggest
+
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
 
   test:
+    needs: ruby-versions
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        ruby:
-          - 2.5
-          - 2.6
-          - 2.7
-          - '3.0'
-          - 3.1
-          - 3.2
-          - head
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3.5.2
+        uses: actions/checkout@v4.1.1
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -42,3 +43,22 @@ jobs:
       - name: test
         run: bin/rake test
         continue-on-error: ${{ matrix.ruby == 'head' }}
+
+  test-disable-prism:
+    needs: ruby-versions
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: test
+        run: SYNTAX_SUGGEST_DISABLE_PRISM=1 bin/rake test
+        continue-on-error: ${{ matrix.ruby == 'head' }}

data/.standard.yml

@@ -1 +1 @@
-ruby_version: 2.5.9
+ruby_version: 3.0.0

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## HEAD (unreleased)
 
+## 2.0.0
+
+- Changed: No longer supports EOL versions of Ruby. (https://github.com/ruby/syntax_suggest/pull/210)
+- Added: Support prism parser (https://github.com/ruby/syntax_suggest/pull/208).
+- Added: Handle Ruby 3.3 new eval source location format (https://github.com/ruby/syntax_suggest/pull/200).
+
 ## 1.1.0
 
 - Handle if/else with comment or empty line in branch (https://github.com/ruby/syntax_suggest/pull/193)

data/Gemfile

@@ -12,3 +12,4 @@ gem "standard"
 gem "ruby-prof"
 
 gem "benchmark-ips"
+gem "prism"

data/Gemfile.lock

@@ -1,61 +1,79 @@
 PATH
   remote: .
   specs:
-    syntax_suggest (1.1.0)
+    syntax_suggest (2.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    benchmark-ips (2.9.2)
-    diff-lcs (1.4.4)
-    parallel (1.21.0)
-    parser (3.0.2.0)
+    benchmark-ips (2.12.0)
+    diff-lcs (1.5.0)
+    json (2.7.0)
+    language_server-protocol (3.17.0.3)
+    lint_roller (1.1.0)
+    parallel (1.23.0)
+    parser (3.2.2.4)
       ast (~> 2.4.1)
-    rainbow (3.0.0)
+      racc
+    prism (0.18.0)
+    racc (1.7.3)
+    rainbow (3.1.1)
     rake (12.3.3)
-    regexp_parser (2.1.1)
-    rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.0)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.0)
+    regexp_parser (2.8.3)
+    rexml (3.2.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.0)
-    rubocop (1.20.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    rubocop (1.57.2)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.9.1, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.12.0)
-      parser (>= 3.0.1.1)
-    rubocop-performance (1.11.5)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.30.0)
+      parser (>= 3.2.1.0)
+    rubocop-performance (1.19.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    ruby-prof (1.4.3)
-    ruby-progressbar (1.11.0)
-    stackprof (0.2.16)
-    standard (1.3.0)
-      rubocop (= 1.20.0)
-      rubocop-performance (= 1.11.5)
-    unicode-display_width (2.1.0)
+    ruby-prof (1.6.3)
+    ruby-progressbar (1.13.0)
+    stackprof (0.2.25)
+    standard (1.32.1)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.57.2)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.2)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.2.1)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.19.1)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   benchmark-ips
+  prism
   rake (~> 12.0)
   rspec (~> 3.0)
   ruby-prof
@@ -64,4 +82,4 @@ DEPENDENCIES
   syntax_suggest!
 
 BUNDLED WITH
-   2.3.14
+   2.4.21

data/README.md

@@ -122,7 +122,7 @@ Unmatched `(', missing `)' ?
   5  end
 ```
 
-- Any ambiguous or unknown errors will be annotated by the original ripper error output:
+- Any ambiguous or unknown errors will be annotated by the original parser error output:
 
 <!--
 class Dog
@@ -133,7 +133,7 @@ end
 -->
 
 ```
-syntax error, unexpected end-of-input
+Expected an expression after the operator
 
   1  class Dog
   2    def meals_last_month

data/lib/syntax_suggest/api.rb

@@ -5,9 +5,28 @@ require_relative "version"
 require "tmpdir"
 require "stringio"
 require "pathname"
-require "ripper"
 require "timeout"
 
+# We need Ripper loaded for `Prism.lex_compat` even if we're using Prism
+# for lexing and parsing
+require "ripper"
+
+# Prism is the new parser, replacing Ripper
+#
+# We need to "dual boot" both for now because syntax_suggest
+# supports older rubies that do not ship with syntax suggest.
+#
+# We also need the ability to control loading of this library
+# so we can test that both modes work correctly in CI.
+if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"])
+  warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}"
+else
+  begin
+    require "prism"
+  rescue LoadError
+  end
+end
+
 module SyntaxSuggest
   # Used to indicate a default value that cannot
   # be confused with another input.
@@ -16,6 +35,14 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
+  # SyntaxSuggest.use_prism_parser? [Private]
+  #
+  # Tells us if the prism parser is available for use
+  # or if we should fallback to `Ripper`
+  def self.use_prism_parser?
+    defined?(Prism)
+  end
+
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -129,11 +156,20 @@ module SyntaxSuggest
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  def self.invalid?(source)
-    source = source.join if source.is_a?(Array)
-    source = source.to_s
+  if defined?(Prism)
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
 
-    Ripper.new(source).tap(&:parse).error?
+      Prism.parse(source).failure?
+    end
+  else
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
+
+      Ripper.new(source).tap(&:parse).error?
+    end
   end
 
   # SyntaxSuggest.valid? [Private]
@@ -191,7 +227,6 @@ require_relative "lex_all"
 require_relative "code_line"
 require_relative "code_block"
 require_relative "block_expand"
-require_relative "ripper_errors"
 require_relative "priority_queue"
 require_relative "unvisited_lines"
 require_relative "around_block_scan"

data/lib/syntax_suggest/clean_document.rb

@@ -47,9 +47,9 @@ module SyntaxSuggest
   # ## Heredocs
   #
   # A heredoc is an way of defining a multi-line string. They can cause many
-  # problems. If left as a single line, Ripper would try to parse the contents
+  # problems. If left as a single line, the parser would try to parse the contents
   # as ruby code rather than as a string. Even without this problem, we still
-  # hit an issue with indentation
+  # hit an issue with indentation:
   #
   #    1 foo = <<~HEREDOC
   #    2  "Be yourself; everyone else is already taken.""
@@ -224,7 +224,7 @@ module SyntaxSuggest
     #
     def join_consecutive!
       consecutive_groups = @document.select(&:ignore_newline_not_beg?).map do |code_line|
-        take_while_including(code_line.index..-1) do |line|
+        take_while_including(code_line.index..) do |line|
           line.ignore_newline_not_beg?
         end
       end
@@ -245,7 +245,7 @@ module SyntaxSuggest
     #     expect(lines[1].to_s).to eq("")
     def join_trailing_slash!
       trailing_groups = @document.select(&:trailing_slash?).map do |code_line|
-        take_while_including(code_line.index..-1) { |x| x.trailing_slash? }
+        take_while_including(code_line.index..) { |x| x.trailing_slash? }
       end
       join_groups(trailing_groups)
       self
@@ -279,7 +279,7 @@ module SyntaxSuggest
         )
 
         # Hide the rest of the lines
-        lines[1..-1].each do |line|
+        lines[1..].each do |line|
           # The above lines already have newlines in them, if add more
           # then there will be double newline, use an empty line instead
           @document[line.index] = CodeLine.new(line: "", index: line.index, lex: [])
@@ -293,7 +293,7 @@ module SyntaxSuggest
     # Like `take_while` except when it stops
     # iterating, it also returns the line
     # that caused it to stop
-    def take_while_including(range = 0..-1)
+    def take_while_including(range = 0..)
       take_next_and_stop = false
       @document[range].take_while do |line|
         next if take_next_and_stop

data/lib/syntax_suggest/code_block.rb

@@ -81,7 +81,7 @@ module SyntaxSuggest
         # lines then the result cannot be invalid
         #
         # That means there's no reason to re-check all
-        # lines with ripper (which is expensive).
+        # lines with the parser (which is expensive).
         # Benchmark in commit message
         @valid = if lines.all? { |l| l.hidden? || l.empty? }
           true

data/lib/syntax_suggest/code_frontier.rb

@@ -117,7 +117,7 @@ module SyntaxSuggest
 
       if ENV["SYNTAX_SUGGEST_DEBUG"]
         puts "```"
-        puts @queue.peek.to_s
+        puts @queue.peek
         puts "```"
         puts "  @frontier indent:  #{frontier_indent}"
         puts "  @unvisited indent: #{unvisited_indent}"

data/lib/syntax_suggest/code_line.rb

@@ -180,12 +180,19 @@ module SyntaxSuggest
     #     EOM
     #     expect(lines.first.trailing_slash?).to eq(true)
     #
-    def trailing_slash?
-      last = @lex.last
-      return false unless last
-      return false unless last.type == :on_sp
+    if SyntaxSuggest.use_prism_parser?
+      def trailing_slash?
+        last = @lex.last
+        last&.type == :on_tstring_end
+      end
+    else
+      def trailing_slash?
+        last = @lex.last
+        return false unless last
+        return false unless last.type == :on_sp
 
-      last.token == TRAILING_SLASH
+        last.token == TRAILING_SLASH
+      end
     end
 
     # Endless method detection

data/lib/syntax_suggest/code_search.rb

@@ -43,7 +43,7 @@ module SyntaxSuggest
 
     def initialize(source, record_dir: DEFAULT_VALUE)
       record_dir = if record_dir == DEFAULT_VALUE
-        ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"] ? "tmp" : nil
+        (ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"]) ? "tmp" : nil
       else
         record_dir
       end
@@ -73,7 +73,7 @@ module SyntaxSuggest
       if ENV["SYNTAX_SUGGEST_DEBUG"]
         puts "\n\n==== #{filename} ===="
         puts "\n```#{block.starts_at}..#{block.ends_at}"
-        puts block.to_s
+        puts block
         puts "```"
         puts "  block indent:      #{block.current_indent}"
       end

data/lib/syntax_suggest/core_ext.rb

@@ -21,7 +21,7 @@ if SyntaxError.method_defined?(:detailed_message)
       attr_reader :string
     end
 
-    # SyntaxSuggest.record_dir [Private]
+    # SyntaxSuggest.module_for_detailed_message [Private]
     #
     # Used to monkeypatch SyntaxError via Module.prepend
     def self.module_for_detailed_message

data/lib/syntax_suggest/display_invalid_blocks.rb

@@ -14,7 +14,7 @@ module SyntaxSuggest
       @filename = filename
       @code_lines = code_lines
 
-      @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal
+      @terminal = (terminal == DEFAULT_VALUE) ? io.isatty : terminal
     end
 
     def document_ok?

data/lib/syntax_suggest/explain_syntax.rb

@@ -2,7 +2,21 @@
 
 require_relative "left_right_lex_count"
 
+if !SyntaxSuggest.use_prism_parser?
+  require_relative "ripper_errors"
+end
+
 module SyntaxSuggest
+  class GetParseErrors
+    def self.errors(source)
+      if SyntaxSuggest.use_prism_parser?
+        Prism.parse(source).errors.map(&:message)
+      else
+        RipperErrors.new(source).call.errors
+      end
+    end
+  end
+
   # Explains syntax errors based on their source
   #
   # example:
@@ -15,8 +29,8 @@ module SyntaxSuggest
   #   # => "Unmatched keyword, missing `end' ?"
   #
   # When the error cannot be determined by lexical counting
-  # then ripper is run against the input and the raw ripper
-  # errors returned.
+  # then the parser is run against the input and the raw
+  # errors are returned.
   #
   # Example:
   #
@@ -91,10 +105,10 @@ module SyntaxSuggest
     # Returns an array of syntax error messages
     #
     # If no missing pairs are found it falls back
-    # on the original ripper error messages
+    # on the original error messages
     def errors
       if missing.empty?
-        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+        return GetParseErrors.errors(@code_lines.map(&:original).join).uniq
       end
 
       missing.map { |miss| why(miss) }

data/lib/syntax_suggest/lex_all.rb

@@ -3,34 +3,53 @@
 module SyntaxSuggest
   # Ripper.lex is not guaranteed to lex the entire source document
   #
-  # lex = LexAll.new(source: source)
-  # lex.each do |value|
-  #   puts value.line
-  # end
+  # This class guarantees the whole document is lex-ed by iteratively
+  # lexing the document where ripper stopped.
+  #
+  # Prism likely doesn't have the same problem. Once ripper support is removed
+  # we can likely reduce the complexity here if not remove the whole concept.
+  #
+  # Example usage:
+  #
+  #   lex = LexAll.new(source: source)
+  #   lex.each do |value|
+  #     puts value.line
+  #   end
   class LexAll
     include Enumerable
 
     def initialize(source:, source_lines: nil)
-      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
-      lineno = @lex.last.pos.first + 1
+      @lex = self.class.lex(source, 1)
+      lineno = @lex.last[0][0] + 1
       source_lines ||= source.lines
       last_lineno = source_lines.length
 
       until lineno >= last_lineno
-        lines = source_lines[lineno..-1]
+        lines = source_lines[lineno..]
 
         @lex.concat(
-          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+          self.class.lex(lines.join, lineno + 1)
         )
-        lineno = @lex.last.pos.first + 1
+
+        lineno = @lex.last[0].first + 1
       end
 
       last_lex = nil
       @lex.map! { |elem|
-        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
       }
     end
 
+    if SyntaxSuggest.use_prism_parser?
+      def self.lex(source, line_number)
+        Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] }
+      end
+    else
+      def self.lex(source, line_number)
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    end
+
     def to_a
       @lex
     end

data/lib/syntax_suggest/pathname_from_message.rb

@@ -13,7 +13,7 @@ module SyntaxSuggest
   #    # => "/tmp/scratch.rb"
   #
   class PathnameFromMessage
-    EVAL_RE = /^\(eval\):\d+/
+    EVAL_RE = /^\(eval.*\):\d+/
     STREAMING_RE = /^-:\d+/
     attr_reader :name
 

data/lib/syntax_suggest/ripper_errors.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 module SyntaxSuggest
-  # Capture parse errors from ripper
+  # Capture parse errors from Ripper
+  #
+  # Prism returns the errors with their messages, but Ripper
+  # does not. To get them we must make a custom subclass.
   #
   # Example:
   #

data/lib/syntax_suggest/scan_history.rb

@@ -118,7 +118,7 @@ module SyntaxSuggest
     # Returns an array of all the CodeLines that exist after
     # the currently scanned block
     private def after_lines
-      @code_lines[@after_index.next..-1] || []
+      @code_lines[@after_index.next..] || []
     end
 
     private def current

data/lib/syntax_suggest/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxSuggest
-  VERSION = "1.1.0"
+  VERSION = "2.0.0"
 end

data/syntax_suggest.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it'
   spec.homepage = "https://github.com/ruby/syntax_suggest.git"
   spec.license = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git"
@@ -27,6 +27,6 @@ Gem::Specification.new do |spec|
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|assets)/}) }
   end
   spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables = ["syntax_suggest"]
   spec.require_paths = ["lib"]
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: syntax_suggest
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - schneems
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-08 00:00:00.000000000 Z
+date: 2023-12-08 00:00:00.000000000 Z
 dependencies: []
 description: When you get an "unexpected end" in your syntax this gem helps you find
   it
@@ -81,14 +81,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.22
 signing_key:
 specification_version: 4
 summary: Find syntax errors in your source in a snap