prism 2.0.0.rc1-x86_64-linux-gnu

data/lib/prism/translation/parser.rb

@@ -0,0 +1,376 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+begin
+  required_version = ">= 3.3.7.2"
+  gem "parser", required_version
+  require "parser"
+rescue LoadError
+  warn(<<~MSG)
+    Error: Unable to load parser #{required_version}. \
+    Add `gem "parser"` to your Gemfile or run `bundle update parser`.
+  MSG
+  exit(1)
+end
+
+module Prism
+  module Translation
+    # This class is the entry-point for converting a prism syntax tree into the
+    # whitequark/parser gem's syntax tree. It inherits from the base parser for
+    # the parser gem, and overrides the parse* methods to parse with prism and
+    # then translate.
+    #
+    # Note that this version of the parser always parses using the latest
+    # version of Ruby syntax supported by Prism. If you want specific version
+    # support, use one of the version-specific subclasses, such as
+    # `Prism::Translation::Parser34`. If you want to parse using the same
+    # version of Ruby syntax as the currently running version of Ruby, use
+    # `Prism::Translation::ParserCurrent`.
+    class Parser < ::Parser::Base
+      Diagnostic = ::Parser::Diagnostic # :nodoc:
+      private_constant :Diagnostic
+
+      # The parser gem has a list of diagnostics with a hard-coded set of error
+      # messages. We create our own diagnostic class in order to set our own
+      # error messages.
+      class PrismDiagnostic < Diagnostic # :nodoc:
+        # This is the cached message coming from prism.
+        attr_reader :message
+
+        # Initialize a new diagnostic with the given message and location.
+        def initialize(message, level, reason, location)
+          @message = message
+          super(level, reason, {}, location, [])
+        end
+      end
+
+      Racc_debug_parser = false # :nodoc:
+
+      # The `builder` argument is used to create the parser using our custom builder class by default.
+      #
+      # By using the `:parser` keyword argument, you can translate in a way that is compatible with
+      # the Parser gem using any parser.
+      #
+      # For example, in RuboCop for Ruby LSP, the following approach can be used to improve performance
+      # by reusing a pre-parsed `Prism::ParseLexResult`:
+      #
+      #   class PrismPreparsed
+      #     def initialize(prism_result)
+      #       @prism_result = prism_result
+      #     end
+      #
+      #     def parse_lex(source, **options)
+      #       @prism_result
+      #     end
+      #   end
+      #
+      #   prism_preparsed = PrismPreparsed.new(prism_result)
+      #
+      #   Prism::Translation::Ruby34.new(builder, parser: prism_preparsed)
+      #
+      # In an object passed to the `:parser` keyword argument, the `parse` and `parse_lex` methods
+      # should be implemented as needed.
+      #
+      def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism)
+        if !builder.is_a?(Prism::Translation::Parser::Builder)
+          warn(<<~MSG, uplevel: 1, category: :deprecated)
+            [deprecation]: The builder passed to `Prism::Translation::Parser.new` is not a \
+            `Prism::Translation::Parser::Builder` subclass. This will raise in the next major version.
+          MSG
+        end
+        @parser = parser
+
+        super(builder)
+      end
+
+      def version # :nodoc:
+        41
+      end
+
+      # The default encoding for Ruby files is UTF-8.
+      def default_encoding
+        Encoding::UTF_8
+      end
+
+      def yyerror # :nodoc:
+      end
+
+      # Parses a source buffer and returns the AST.
+      def parse(source_buffer)
+        @source_buffer = source_buffer
+        source = source_buffer.source
+
+        offset_cache = build_offset_cache(source)
+        result = unwrap(@parser.parse(source, **prism_options), offset_cache)
+
+        build_ast(result.value, offset_cache)
+      ensure
+        @source_buffer = nil
+      end
+
+      # Parses a source buffer and returns the AST and the source code comments.
+      def parse_with_comments(source_buffer)
+        @source_buffer = source_buffer
+        source = source_buffer.source
+
+        offset_cache = build_offset_cache(source)
+        result = unwrap(@parser.parse(source, **prism_options), offset_cache)
+
+        [
+          build_ast(result.value, offset_cache),
+          build_comments(result.comments, offset_cache)
+        ]
+      ensure
+        @source_buffer = nil
+      end
+
+      # Parses a source buffer and returns the AST, the source code comments,
+      # and the tokens emitted by the lexer.
+      def tokenize(source_buffer, recover = false)
+        @source_buffer = source_buffer
+        source = source_buffer.source
+
+        offset_cache = build_offset_cache(source)
+        result =
+          begin
+            unwrap(@parser.parse_lex(source, **prism_options), offset_cache)
+          rescue ::Parser::SyntaxError
+            raise if !recover
+          end
+
+        program, tokens = result.value
+        ast = build_ast(program, offset_cache) if result.success?
+
+        [
+          ast,
+          build_comments(result.comments, offset_cache),
+          build_tokens(tokens, offset_cache)
+        ]
+      ensure
+        @source_buffer = nil
+      end
+
+      # Since prism resolves num params for us, we don't need to support this
+      # kind of logic here.
+      def try_declare_numparam(node)
+        node.children[0].match?(/\A_[1-9]\z/)
+      end
+
+      private
+
+      # This is a hook to allow consumers to disable some errors if they don't
+      # want them to block creating the syntax tree.
+      def valid_error?(error)
+        true
+      end
+
+      # This is a hook to allow consumers to disable some warnings if they don't
+      # want them to block creating the syntax tree.
+      def valid_warning?(warning)
+        true
+      end
+
+      # Build a diagnostic from the given prism parse error.
+      def error_diagnostic(error, offset_cache)
+        location = error.location
+        diagnostic_location = build_range(location, offset_cache)
+
+        case error.type
+        when :argument_block_multi
+          Diagnostic.new(:error, :block_and_blockarg, {}, diagnostic_location, [])
+        when :argument_formal_constant
+          Diagnostic.new(:error, :argument_const, {}, diagnostic_location, [])
+        when :argument_formal_class
+          Diagnostic.new(:error, :argument_cvar, {}, diagnostic_location, [])
+        when :argument_formal_global
+          Diagnostic.new(:error, :argument_gvar, {}, diagnostic_location, [])
+        when :argument_formal_ivar
+          Diagnostic.new(:error, :argument_ivar, {}, diagnostic_location, [])
+        when :argument_no_forwarding_amp
+          Diagnostic.new(:error, :no_anonymous_blockarg, {}, diagnostic_location, [])
+        when :argument_no_forwarding_star
+          Diagnostic.new(:error, :no_anonymous_restarg, {}, diagnostic_location, [])
+        when :argument_no_forwarding_star_star
+          Diagnostic.new(:error, :no_anonymous_kwrestarg, {}, diagnostic_location, [])
+        when :begin_lonely_else
+          location = location.copy(length: 4)
+          diagnostic_location = build_range(location, offset_cache)
+          Diagnostic.new(:error, :useless_else, {}, diagnostic_location, [])
+        when :class_name, :module_name
+          Diagnostic.new(:error, :module_name_const, {}, diagnostic_location, [])
+        when :class_in_method
+          Diagnostic.new(:error, :class_in_def, {}, diagnostic_location, [])
+        when :def_endless_setter
+          Diagnostic.new(:error, :endless_setter, {}, diagnostic_location, [])
+        when :embdoc_term
+          Diagnostic.new(:error, :embedded_document, {}, diagnostic_location, [])
+        when :incomplete_variable_class, :incomplete_variable_class_3_3
+          location = location.copy(length: location.length + 1)
+          diagnostic_location = build_range(location, offset_cache)
+
+          Diagnostic.new(:error, :cvar_name, { name: location.slice }, diagnostic_location, [])
+        when :incomplete_variable_instance, :incomplete_variable_instance_3_3
+          location = location.copy(length: location.length + 1)
+          diagnostic_location = build_range(location, offset_cache)
+
+          Diagnostic.new(:error, :ivar_name, { name: location.slice }, diagnostic_location, [])
+        when :invalid_variable_global, :invalid_variable_global_3_3
+          Diagnostic.new(:error, :gvar_name, { name: location.slice }, diagnostic_location, [])
+        when :module_in_method
+          Diagnostic.new(:error, :module_in_def, {}, diagnostic_location, [])
+        when :numbered_parameter_ordinary
+          Diagnostic.new(:error, :ordinary_param_defined, {}, diagnostic_location, [])
+        when :numbered_parameter_outer_scope
+          Diagnostic.new(:error, :numparam_used_in_outer_scope, {}, diagnostic_location, [])
+        when :parameter_circular
+          Diagnostic.new(:error, :circular_argument_reference, { var_name: location.slice }, diagnostic_location, [])
+        when :parameter_name_repeat
+          Diagnostic.new(:error, :duplicate_argument, {}, diagnostic_location, [])
+        when :parameter_numbered_reserved
+          Diagnostic.new(:error, :reserved_for_numparam, { name: location.slice }, diagnostic_location, [])
+        when :regexp_unknown_options
+          Diagnostic.new(:error, :regexp_options, { options: location.slice[1..] }, diagnostic_location, [])
+        when :singleton_for_literals
+          Diagnostic.new(:error, :singleton_literal, {}, diagnostic_location, [])
+        when :string_literal_eof
+          Diagnostic.new(:error, :string_eof, {}, diagnostic_location, [])
+        when :unexpected_token_ignore
+          Diagnostic.new(:error, :unexpected_token, { token: location.slice }, diagnostic_location, [])
+        when :write_target_in_method
+          Diagnostic.new(:error, :dynamic_const, {}, diagnostic_location, [])
+        else
+          PrismDiagnostic.new(error.message, :error, error.type, diagnostic_location)
+        end
+      end
+
+      # Build a diagnostic from the given prism parse warning.
+      def warning_diagnostic(warning, offset_cache)
+        diagnostic_location = build_range(warning.location, offset_cache)
+
+        case warning.type
+        when :ambiguous_first_argument_plus
+          Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "+" }, diagnostic_location, [])
+        when :ambiguous_first_argument_minus
+          Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "-" }, diagnostic_location, [])
+        when :ambiguous_prefix_ampersand
+          Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "&" }, diagnostic_location, [])
+        when :ambiguous_prefix_star
+          Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "*" }, diagnostic_location, [])
+        when :ambiguous_prefix_star_star
+          Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "**" }, diagnostic_location, [])
+        when :ambiguous_slash
+          Diagnostic.new(:warning, :ambiguous_regexp, {}, diagnostic_location, [])
+        when :dot_dot_dot_eol
+          Diagnostic.new(:warning, :triple_dot_at_eol, {}, diagnostic_location, [])
+        when :duplicated_hash_key
+          # skip, parser does this on its own
+        else
+          PrismDiagnostic.new(warning.message, :warning, warning.type, diagnostic_location)
+        end
+      end
+
+      # If there was a error generated during the parse, then raise an
+      # appropriate syntax error. Otherwise return the result.
+      def unwrap(result, offset_cache)
+        result.errors.each do |error|
+          next unless valid_error?(error)
+          diagnostics.process(error_diagnostic(error, offset_cache))
+        end
+
+        result.warnings.each do |warning|
+          next unless valid_warning?(warning)
+          diagnostic = warning_diagnostic(warning, offset_cache)
+          diagnostics.process(diagnostic) if diagnostic
+        end
+
+        result
+      end
+
+      # Prism deals with offsets in bytes, while the parser gem deals with
+      # offsets in characters. We need to handle this conversion in order to
+      # build the parser gem AST.
+      #
+      # If the bytesize of the source is the same as the length, then we can
+      # just use the offset directly. Otherwise, we build an array where the
+      # index is the byte offset and the value is the character offset.
+      def build_offset_cache(source)
+        if source.bytesize == source.length
+          -> (offset) { offset }
+        else
+          offset_cache = []
+          offset = 0
+
+          source.each_char do |char|
+            char.bytesize.times { offset_cache << offset }
+            offset += 1
+          end
+
+          offset_cache << offset
+        end
+      end
+
+      # Build the parser gem AST from the prism AST.
+      def build_ast(program, offset_cache)
+        program.accept(Compiler.new(self, offset_cache))
+      end
+
+      # Build the parser gem comments from the prism comments.
+      def build_comments(comments, offset_cache)
+        comments.map do |comment|
+          ::Parser::Source::Comment.new(build_range(comment.location, offset_cache))
+        end
+      end
+
+      # Build the parser gem tokens from the prism tokens.
+      def build_tokens(tokens, offset_cache)
+        Lexer.new(source_buffer, tokens, offset_cache).to_a
+      end
+
+      # Build a range from a prism location.
+      def build_range(location, offset_cache)
+        ::Parser::Source::Range.new(
+          source_buffer,
+          offset_cache[location.start_offset],
+          offset_cache[location.end_offset]
+        )
+      end
+
+      # Options for how prism should parse/lex the source.
+      def prism_options
+        options = {
+          filepath: @source_buffer.name,
+          version: convert_for_prism(version),
+          partial_script: true,
+        }
+        # The parser gem always encodes to UTF-8, unless it is binary.
+        # https://github.com/whitequark/parser/blob/v3.3.6.0/lib/parser/source/buffer.rb#L80-L107
+        options[:encoding] = false if @source_buffer.source.encoding != Encoding::BINARY
+
+        options
+      end
+
+      # Converts the version format handled by Parser to the format handled by Prism.
+      def convert_for_prism(version)
+        case version
+        when 33
+          "3.3.1"
+        when 34
+          "3.4.0"
+        when 35, 40
+          "4.0.0"
+        when 41
+          "4.1.0"
+        else
+          "latest"
+        end
+      end
+
+      require_relative "parser/builder"
+      require_relative "parser/compiler"
+      require_relative "parser/lexer"
+
+      private_constant :Compiler
+      private_constant :Lexer
+    end
+  end
+end

data/lib/prism/translation/parser_current.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# :markup: markdown
+#--
+# typed: ignore
+
+module Prism
+  module Translation
+    case RUBY_VERSION
+    when /^3\.3\./
+      ParserCurrent = Parser33
+    when /^3\.4\./
+      ParserCurrent = Parser34
+    when /^3\.5\./, /^4\.0\./
+      ParserCurrent = Parser40
+    when /^4\.1\./
+      ParserCurrent = Parser41
+    else
+      # Keep this in sync with released Ruby.
+      parser = Parser40
+      major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
+      warn "warning: `Prism::Translation::Current` is loading #{parser.name}, " \
+           "but you are running #{major}.#{minor}."
+      ParserCurrent = parser
+    end
+  end
+end

data/lib/prism/translation/parser_versions.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+module Prism
+  module Translation
+    # This class is the entry-point for Ruby 3.3 of `Prism::Translation::Parser`.
+    class Parser33 < Parser
+      def version # :nodoc:
+        33
+      end
+    end
+
+    # This class is the entry-point for Ruby 3.4 of `Prism::Translation::Parser`.
+    class Parser34 < Parser
+      def version # :nodoc:
+        34
+      end
+    end
+
+    # This class is the entry-point for Ruby 4.0 of `Prism::Translation::Parser`.
+    class Parser40 < Parser
+      def version # :nodoc:
+        40
+      end
+    end
+
+    Parser35 = Parser40 # :nodoc:
+
+    # This class is the entry-point for Ruby 4.1 of `Prism::Translation::Parser`.
+    class Parser41 < Parser
+      def version # :nodoc:
+        41
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper/filter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Prism
+  module Translation
+    class Ripper
+      class Filter # :nodoc:
+        # :stopdoc:
+        def initialize(src, filename = '-', lineno = 1)
+          @__lexer = Lexer.new(src, filename, lineno)
+          @__line = nil
+          @__col = nil
+          @__state = nil
+        end
+
+        def filename
+          @__lexer.filename
+        end
+
+        def lineno
+          @__line
+        end
+
+        def column
+          @__col
+        end
+
+        def state
+          @__state
+        end
+
+        def parse(init = nil)
+          data = init
+          @__lexer.lex.each do |pos, event, tok, state|
+            @__line, @__col = *pos
+            @__state = state
+            data = if respond_to?(event, true)
+                  then __send__(event, tok, data)
+                  else on_default(event, tok, data)
+                  end
+          end
+          data
+        end
+
+        private
+
+        def on_default(event, token, data)
+          data
+        end
+        # :startdoc:
+      end
+    end
+  end
+end

data/lib/prism/translation/ripper/lexer.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+# :markup: markdown
+
+require_relative "../ripper"
+
+module Prism
+  module Translation
+    class Ripper
+      class Lexer < Ripper # :nodoc:
+        class State # :nodoc:
+          attr_reader :to_int, :to_s
+
+          def initialize(i)
+            @to_int = i
+            @to_s = Ripper.lex_state_name(i)
+            freeze
+          end
+
+          def [](index)
+            case index
+            when 0, :to_int
+              @to_int
+            when 1, :to_s
+              @to_s
+            else
+              nil
+            end
+          end
+
+          alias to_i to_int
+          alias inspect to_s
+          def pretty_print(q) q.text(to_s) end
+          def ==(i) super or to_int == i end
+          def &(i) self.class.new(to_int & i) end
+          def |(i) self.class.new(to_int | i) end
+          def allbits?(i) to_int.allbits?(i) end
+          def anybits?(i) to_int.anybits?(i) end
+          def nobits?(i) to_int.nobits?(i) end
+
+          # Instances are frozen and there are only a handful of them so we
+          # cache them here.
+          STATES = Hash.new { |hash, key| hash[key] = State.new(key) }
+          private_constant :STATES
+
+          def self.[](i)
+            STATES[i]
+          end
+        end
+
+        class Elem # :nodoc:
+          attr_accessor :pos, :event, :tok, :state, :message
+
+          def initialize(pos, event, tok, state, message = nil)
+            @pos = pos
+            @event = event
+            @tok = tok
+            @state = State[state]
+            @message = message
+          end
+
+          def [](index)
+            case index
+            when 0, :pos
+              @pos
+            when 1, :event
+              @event
+            when 2, :tok
+              @tok
+            when 3, :state
+              @state
+            when 4, :message
+              @message
+            else
+              nil
+            end
+          end
+
+          def inspect
+            "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
+          end
+
+          alias to_s inspect
+
+          def pretty_print(q)
+            q.group(2, "#<#{self.class}:", ">") {
+              q.breakable
+              q.text("#{event}@#{pos[0]}:#{pos[1]}")
+              q.breakable
+              state.pretty_print(q)
+              q.breakable
+              q.text("token: ")
+              tok.pretty_print(q)
+              if message
+                q.breakable
+                q.text("message: ")
+                q.text(message)
+              end
+            }
+          end
+
+          def to_a
+            if @message
+              [@pos, @event, @tok, @state, @message]
+            else
+              [@pos, @event, @tok, @state]
+            end
+          end
+        end
+
+        # Pretty much just the same as Prism.lex_compat.
+        def lex(raise_errors: false)
+          Ripper.lex(@source, filename, lineno, raise_errors: raise_errors)
+        end
+
+        # Returns the lex_compat result wrapped in `Elem`. Errors are omitted.
+        # Since ripper is a streaming parser, tokens are expected to be emitted in the order
+        # that the parser encounters them. This is not implemented.
+        def parse(...)
+          lex(...).map do |position, event, token, state|
+            Elem.new(position, event, token, state.to_int)
+          end
+        end
+
+        # Similar to parse but ripper sorts the elements by position in the source. Also
+        # includes errors. Since prism does error recovery, in cases of syntax errors
+        # the result may differ greatly compared to ripper.
+        def scan(...)
+          parse(...)
+        end
+      end
+    end
+  end
+end