parser-prism 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 87ed9acec576c353ec93af84fbcdc73bebdfba153194e64fccad45be90a768e8
+  data.tar.gz: 4c7263d22aedd730a2859e7e52390f10c25b4f540baccedff8138fa478327cd3
+SHA512:
+  metadata.gz: 4451a6a3234c60e424b85b1c182a211c310846c73ed4abf6e18ee14ef673ccacfffa948268ce07d1b1ae9667c773a7535d8acb834beee85b7877eb37cafe0c3a
+  data.tar.gz: 596ff68ecba5813cd45ef83434a0b94e0146c6cb6ac40444992899f70481e8f181233bb5d6f96e1e63fe0dc290faeb89da667f44bd1e891d569f36a3886bc953

data/.github/workflows/main.yml

@@ -0,0 +1,18 @@
+name: Main
+
+on:
+- push
+- pull_request
+
+jobs:
+  ci:
+    name: CI
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.2'
+        bundler-cache: true
+    - name: Test
+      run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,3 @@
+pkg
+test.rb
+tmp

data/.rubocop.yml

@@ -0,0 +1,4 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 80_82_73_83_77.33

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "benchmark-ips"
+gem "prism"
+gem "rake"
+gem "rubocop"
+gem "test-unit"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    parser-prism (0.1.0)
+      parser
+      prism
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    benchmark-ips (2.12.0)
+    json (2.6.3)
+    language_server-protocol (3.17.0.3)
+    parallel (1.23.0)
+    parser (3.2.2.4)
+      ast (~> 2.4.1)
+      racc
+    power_assert (2.0.3)
+    prism (0.19.0)
+    racc (1.7.3)
+    rainbow (3.1.1)
+    rake (13.1.0)
+    regexp_parser (2.8.2)
+    rexml (3.2.6)
+    rubocop (1.57.2)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.2.2.4)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.30.0)
+      parser (>= 3.2.1.0)
+    ruby-progressbar (1.13.0)
+    test-unit (3.6.1)
+      power_assert
+    unicode-display_width (2.5.0)
+
+PLATFORMS
+  arm64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  benchmark-ips
+  parser-prism!
+  prism
+  rake
+  rubocop
+  test-unit
+
+BUNDLED WITH
+   2.4.13

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023-present Kevin Newton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# parser-prism
+
+This gem provides a new backend for the [whitequark/parser](https://github.com/whitequark/parser) gem's syntax tree that uses the [prism](https://github.com/ruby/prism) parser.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "parser-prism"
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install parser-prism
+
+## Usage
+
+The `parser` gem provides multiple parsers to support different versions of the Ruby grammar. This includes all of the Ruby versions going back to 1.8, as well as third-party parsers like MacRuby and RubyMotion. The `parser-prism` gem provides another parser that uses the `prism` parser to build the syntax tree.
+
+You can use the `prism` parser like you would any other. After requiring the parser, you should be able to call any of the regular `Parser::Base` APIs that you would normally use.
+
+```ruby
+require "parser/prism"
+
+Parser::Prism.parse_file("path/to/file.rb")
+```
+
+### RuboCop
+
+To run RuboCop using the `parser-prism` gem as the parser, you will need to require the `parser/prism/rubocop` file. This file injects `prism` into the known options for both `rubocop` and `rubocop-ast`, such that you can specify it in your `.rubocop.yml` file. Unfortunately `rubocop` doesn't support any direct way to do this, so we have to get a bit hacky.
+
+First, set the `TargetRubyVersion` in your RuboCop configuration file to `80_82_73_83_77.33`. This is the version of Ruby that `prism` reports itself as. (The leading numbers are the ASCII values for `PRISM`.)
+
+```yaml
+AllCops:
+  TargetRubyVersion: 80_82_73_83_77.33
+```
+
+Now when you run `rubocop` you will need to require the `parser/prism/rubocop` file before executing so that it can inject the `prism` parser into the known options.
+
+```
+bundle exec ruby -rparser/prism/rubocop $(bundle exec which rubocop)
+```
+
+This should run RuboCop using the `prism` parser.
+
+## Benchmarks
+
+As a whole, this parser should be significantly faster than the `parser` gem. The `bin/bench` script in this repository compares the performance of `Parser::CurrentRuby` and `Parser::Prism`. Running against a large file like `lib/parser/prism/compiler.rb` yields:
+
+```
+Warming up --------------------------------------
+ Parser::CurrentRuby     1.000  i/100ms
+       Parser::Prism     6.000  i/100ms
+Calculating -------------------------------------
+ Parser::CurrentRuby     16.642  (± 0.0%) i/s -     84.000  in   5.052021s
+       Parser::Prism     64.951  (± 3.1%) i/s -    330.000  in   5.088147s
+
+Comparison:
+       Parser::Prism:       65.0 i/s
+ Parser::CurrentRuby:       16.6 i/s - 3.90x  slower
+```
+
+When running with `--yjit`, the comparison is even more stark:
+
+```
+Warming up --------------------------------------
+ Parser::CurrentRuby     1.000  i/100ms
+       Parser::Prism     9.000  i/100ms
+Calculating -------------------------------------
+ Parser::CurrentRuby     20.062  (± 0.0%) i/s -    101.000  in   5.034389s
+       Parser::Prism    112.823  (± 9.7%) i/s -    558.000  in   5.009460s
+
+Comparison:
+       Parser::Prism:      112.8 i/s
+ Parser::CurrentRuby:       20.1 i/s - 5.62x  slower
+```
+
+These benchmarks were run on a single laptop without a lot of control for other processes, so take them with a grain of salt.
+
+## Development
+
+Run `bundle exec rake test` to run the tests. This runs tests exported from the `parser` gem into their own fixture files.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kddnewton/parser-prism.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: :test
+
+# This namespace's purpose is to extract the examples from the whitequark/parser
+# gem and generate a test file that we can use to ensure that our parser
+# generates equivalent syntax trees when translating. To do this, it runs the
+# parser's test suite but overrides the `assert_parses` method to collect the
+# examples into a hash. Then, it writes out the hash to a file that we can use
+# to generate our own tests.
+#
+# To run the test suite, it's important to note that we have to mirror both any
+# APIs provided to the test suite (for example the ParseHelper module below).
+# This is obviously relatively brittle, but it's effective for now.
+namespace :whitequark do
+  desc "Ensure there's a local copy of whitequark/parser"
+  file "tmp/whitequark" do
+    sh "git clone --depth=1 https://github.com/whitequark/parser tmp/whitequark"
+  end
+
+  desc "Ensure we have a fixtures directory for the whitequark/parser tests"
+  directory "test/fixtures"
+
+  desc "Import the whitequark/parser tests"
+  task import: ["tmp/whitequark", "test/fixtures"] do
+    cp "tmp/whitequark/LICENSE.txt", "test/fixtures/LICENSE"
+
+    mkdir_p "tmp/whitequark/scratch"
+    touch "tmp/whitequark/scratch/helper.rb"
+    touch "tmp/whitequark/scratch/parse_helper.rb"
+    $:.unshift("tmp/whitequark/scratch")
+
+    require "ast"
+    module ParseHelper
+      include AST::Sexp
+
+      # This object is going to collect all of the examples from the parser gem
+      # into a hash that we can use to generate our own tests.
+      COLLECTED = Hash.new { |hash, key| hash[key] = [] }
+      ALL_VERSIONS = %w[3.1 3.2]
+
+      private
+
+      def assert_context(*)
+      end
+
+      def assert_diagnoses(*)
+      end
+
+      def assert_diagnoses_many(*)
+      end
+
+      def refute_diagnoses(*)
+      end
+
+      def with_versions(*)
+      end
+
+      def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS)
+        # We're going to skip any examples that are for older Ruby versions
+        # that we do not support.
+        return if (versions & %w[3.1 3.2]).empty?
+
+        entry = caller.find { _1.include?("test_parser.rb") }
+        _, name = *entry.match(/\d+:in `(?:block in )?(?:test_|assert_parses_)?(.+)'/)
+
+        COLLECTED[name] << code
+      end
+    end
+
+    require "parser/current"
+    require "minitest/autorun"
+    require_relative "tmp/whitequark/test/test_parser"
+
+    Minitest.after_run do
+      ParseHelper::COLLECTED.each do |(name, codes)|
+        File.write("test/fixtures/#{name}.rb", "#{codes.sort.join("\n\n")}\n")
+      end
+    end
+  end
+
+  desc "Clean up tmp files related to whitequark/parser"
+  task :clean do
+    rm_rf "tmp/whitequark"
+    rm_rf "test/fixtures"
+  end
+end

data/bin/bench

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "benchmark/ips"
+require "parser/prism"
+require "parser/current"
+
+filepath = ARGV.fetch(0) { File.expand_path("../lib/parser/prism/compiler.rb", __dir__) }
+
+Benchmark.ips do |x|
+  x.report("Parser::CurrentRuby") { Parser::CurrentRuby.parse_file(filepath) }
+  x.report("Parser::Prism") { Parser::Prism.parse_file(filepath) }
+  x.compare!
+end

data/bin/parse

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+$:.unshift(File.expand_path("../lib", __dir__))
+
+require "parser/prism"
+require "parser/prism/compare"
+
+if ARGV[0].nil?
+  filepaths = Dir[File.expand_path("../test/fixtures/*.rb", __dir__)]
+  failures = 0
+
+  filepaths.each do |filepath|
+    failures += 1 unless Parser::Prism.compare(filepath)
+  end
+
+  puts "#{filepaths.size - failures}/#{filepaths.size} tests passed"
+elsif ARGV[0] == "-e"
+  puts "match!" if Parser::Prism.compare("-e", ARGV[1])
+else
+  puts "match!" if Parser::Prism.compare(ARGV[0])
+end

data/lib/parser/prism/compare.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "parser/current"
+
+# First, opt in to every AST feature.
+Parser::Builders::Default.modernize
+
+# Modify the source map == check so that it doesn't check against the node
+# itself so we don't get into a recursive loop.
+Parser::Source::Map.prepend(
+  Module.new {
+    def ==(other)
+      self.class == other.class &&
+        (instance_variables - %i[@node]).map do |ivar|
+          instance_variable_get(ivar) == other.instance_variable_get(ivar)
+        end.reduce(:&)
+    end
+  }
+)
+
+# Next, ensure that we're comparing the nodes and also comparing the source
+# ranges so that we're getting all of the necessary information.
+Parser::AST::Node.prepend(
+  Module.new {
+    def ==(other)
+      super && (location == other.location)
+    end
+  }
+)
+
+module Parser
+  class Prism
+    # Compare the ASTs between the translator and the whitequark/parser gem.
+    def self.compare(filepath, source = nil, compare_tokens: true)
+      buffer = Source::Buffer.new(filepath, 1)
+      buffer.source = source || File.read(filepath)
+
+      parser = CurrentRuby.default_parser
+      parser.diagnostics.consumer = ->(*) {}
+      parser.diagnostics.all_errors_are_fatal = true
+
+      expected_ast, expected_comments, expected_tokens =
+        begin
+          parser.tokenize(buffer)
+        rescue ArgumentError, SyntaxError
+          return true
+        end
+
+      actual_ast, actual_comments, actual_tokens = Prism.new.tokenize(buffer)
+
+      if expected_ast != actual_ast
+        puts filepath
+        queue = [[expected_ast, actual_ast]]
+
+        while (left, right = queue.shift)
+          if left.type != right.type
+            puts "expected:"
+            pp left
+
+            puts "actual:"
+            pp right
+
+            return false
+          end
+
+          if left.location != right.location
+            puts "expected:"
+            pp left
+            pp left.location
+
+            puts "actual:"
+            pp right
+            pp right.location
+
+            return false
+          end
+
+          if left.type == :str && left.children[0] != right.children[0]
+            puts "expected:"
+            pp left
+
+            puts "actual:"
+            pp right
+
+            return false
+          end
+
+          left.children.zip(right.children).each do |left_child, right_child|
+            queue << [left_child, right_child] if left_child.is_a?(Parser::AST::Node)
+          end
+        end
+
+        return false
+      end
+
+      if compare_tokens && expected_tokens != actual_tokens
+        expected_index = 0
+        actual_index = 0
+
+        while expected_index < expected_tokens.length
+          expected_token = expected_tokens[expected_index]
+          actual_token = actual_tokens[actual_index]
+
+          expected_index += 1
+          actual_index += 1
+
+          if expected_token[0] == :tSPACE && actual_token[0] == :tSTRING_END
+            expected_index += 1
+            next
+          end
+
+          case actual_token[0]
+          when :kDO
+            actual_token[0] = expected_token[0] if %i[kDO_BLOCK kDO_LAMBDA].include?(expected_token[0])
+          when :tLPAREN
+            actual_token[0] = expected_token[0] if expected_token[0] == :tLPAREN2
+          when :tLCURLY
+            actual_token[0] = expected_token[0] if %i[tLBRACE tLBRACE_ARG].include?(expected_token[0])
+          when :tPOW
+            actual_token[0] = expected_token[0] if expected_token[0] == :tDSTAR
+          end
+
+          if expected_token != actual_token
+            puts "expected:"
+            pp expected_token
+
+            puts "actual:"
+            pp actual_token
+
+            return false
+          end
+        end
+      end
+
+      if expected_comments != actual_comments
+        puts "expected:"
+        pp expected_comments
+
+        puts "actual:"
+        pp actual_comments
+
+        return false
+      end
+
+      true
+    end
+  end
+end