prism 0.29.0 → 1.3.0

data/lib/prism/translation/parser/lexer.rb

@@ -134,7 +134,7 @@ module Prism
           MINUS_GREATER: :tLAMBDA,
           NEWLINE: :tNL,
           NUMBERED_REFERENCE: :tNTH_REF,
-          PARENTHESIS_LEFT: :tLPAREN,
+          PARENTHESIS_LEFT: :tLPAREN2,
           PARENTHESIS_LEFT_PARENTHESES: :tLPAREN_ARG,
           PARENTHESIS_RIGHT: :tRPAREN,
           PERCENT: :tPERCENT,
@@ -173,7 +173,7 @@ module Prism
           UMINUS_NUM: :tUNARY_NUM,
           UPLUS: :tUPLUS,
           USTAR: :tSTAR,
-          USTAR_STAR: :tPOW,
+          USTAR_STAR: :tDSTAR,
           WORDS_SEP: :tSPACE
         }
 
@@ -187,7 +187,20 @@ module Prism
         EXPR_BEG = 0x1 # :nodoc:
         EXPR_LABEL = 0x400 # :nodoc:
 
-        private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL
+        # It is used to determine whether `do` is of the token type `kDO` or `kDO_LAMBDA`.
+        #
+        # NOTE: In edge cases like `-> (foo = -> (bar) {}) do end`, please note that `kDO` is still returned
+        # instead of `kDO_LAMBDA`, which is expected: https://github.com/ruby/prism/pull/3046
+        LAMBDA_TOKEN_TYPES = [:kDO_LAMBDA, :tLAMBDA, :tLAMBEG]
+
+        # The `PARENTHESIS_LEFT` token in Prism is classified as either `tLPAREN` or `tLPAREN2` in the Parser gem.
+        # The following token types are listed as those classified as `tLPAREN`.
+        LPAREN_CONVERSION_TOKEN_TYPES = [
+          :kBREAK, :kCASE, :tDIVIDE, :kFOR, :kIF, :kNEXT, :kRETURN, :kUNTIL, :kWHILE, :tAMPER, :tANDOP, :tBANG, :tCOMMA, :tDOT2, :tDOT3,
+          :tEQL, :tLPAREN, :tLPAREN2, :tLSHFT, :tNL, :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
+        ]
+
+        private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL, :LAMBDA_TOKEN_TYPES, :LPAREN_CONVERSION_TOKEN_TYPES
 
         # The Parser::Source::Buffer that the tokens were lexed from.
         attr_reader :source_buffer
@@ -229,6 +242,13 @@ module Prism
             location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset])
 
             case type
+            when :kDO
+              types = tokens.map(&:first)
+              nearest_lambda_token_type = types.reverse.find { |type| LAMBDA_TOKEN_TYPES.include?(type) }
+
+              if nearest_lambda_token_type == :tLAMBDA
+                type = :kDO_LAMBDA
+              end
             when :tCHARACTER
               value.delete_prefix!("?")
             when :tCOMMENT
@@ -268,6 +288,8 @@ module Prism
               value.chomp!(":")
             when :tLCURLY
               type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
+            when :tLPAREN2
+              type = :tLPAREN if tokens.empty? || LPAREN_CONVERSION_TOKEN_TYPES.include?(tokens.dig(-1, 0))
             when :tNTH_REF
               value = parse_integer(value.delete_prefix("$"))
             when :tOP_ASGN
@@ -339,7 +361,7 @@ module Prism
                 location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
               end
             when :tSYMBEG
-              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
+              if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
                 next_location = token.location.join(next_token.location)
                 type = :tSYMBOL
                 value = next_token.value

data/lib/prism/translation/parser.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
-require "parser"
+begin
+  require "parser"
+rescue LoadError
+  warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.})
+  exit(1)
+end
 
 module Prism
   module Translation
@@ -46,7 +51,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
 
         build_ast(result.value, offset_cache)
       ensure
@@ -59,7 +64,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
 
         [
           build_ast(result.value, offset_cache),
@@ -78,7 +83,7 @@ module Prism
         offset_cache = build_offset_cache(source)
         result =
           begin
-            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache)
+            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
           rescue ::Parser::SyntaxError
             raise if !recover
           end

data/lib/prism/translation/ripper.rb

@@ -1273,8 +1273,8 @@ module Prism
       def visit_case_node(node)
         predicate = visit(node.predicate)
         clauses =
-          node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition|
-            on_when(*visit(condition), consequent)
+          node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition|
+            on_when(*visit(condition), current)
           end
 
         bounds(node.location)
@@ -1286,8 +1286,8 @@ module Prism
       def visit_case_match_node(node)
         predicate = visit(node.predicate)
         clauses =
-          node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition|
-            on_in(*visit(condition), consequent)
+          node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition|
+            on_in(*visit(condition), current)
           end
 
         bounds(node.location)
@@ -1908,7 +1908,7 @@ module Prism
         if node.then_keyword == "?"
           predicate = visit(node.predicate)
           truthy = visit(node.statements.body.first)
-          falsy = visit(node.consequent.statements.body.first)
+          falsy = visit(node.subsequent.statements.body.first)
 
           bounds(node.location)
           on_ifop(predicate, truthy, falsy)
@@ -1921,13 +1921,13 @@ module Prism
             else
               visit(node.statements)
             end
-          consequent = visit(node.consequent)
+          subsequent = visit(node.subsequent)
 
           bounds(node.location)
           if node.if_keyword == "if"
-            on_if(predicate, statements, consequent)
+            on_if(predicate, statements, subsequent)
           else
-            on_elsif(predicate, statements, consequent)
+            on_elsif(predicate, statements, subsequent)
           end
         else
           statements = visit(node.statements.body.first)
@@ -1960,7 +1960,7 @@ module Prism
       # ^^^^^^^^^^^^^^^^^^^^^
       def visit_in_node(node)
         # This is a special case where we're not going to call on_in directly
-        # because we don't have access to the consequent. Instead, we'll return
+        # because we don't have access to the subsequent. Instead, we'll return
         # the component parts and let the parent node handle it.
         pattern = visit_pattern_node(node.pattern)
         statements =
@@ -2217,6 +2217,13 @@ module Prism
         end
       end
 
+      # -> { it }
+      #      ^^
+      def visit_it_local_variable_read_node(node)
+        bounds(node.location)
+        on_vcall(on_ident(node.slice))
+      end
+
       # -> { it }
       # ^^^^^^^^^
       def visit_it_parameters_node(node)
@@ -2312,12 +2319,7 @@ module Prism
       # ^^^
       def visit_local_variable_read_node(node)
         bounds(node.location)
-
-        if node.name == :"0it"
-          on_vcall(on_ident(node.slice))
-        else
-          on_var_ref(on_ident(node.slice))
-        end
+        on_var_ref(on_ident(node.slice))
       end
 
       # foo = 1
@@ -2806,10 +2808,10 @@ module Prism
             visit(node.statements)
           end
 
-        consequent = visit(node.consequent)
+        subsequent = visit(node.subsequent)
 
         bounds(node.location)
-        on_rescue(exceptions, reference, statements, consequent)
+        on_rescue(exceptions, reference, statements, subsequent)
       end
 
       # def foo(*bar); end
@@ -3130,10 +3132,10 @@ module Prism
             else
               visit(node.statements)
             end
-          consequent = visit(node.consequent)
+          else_clause = visit(node.else_clause)
 
           bounds(node.location)
-          on_unless(predicate, statements, consequent)
+          on_unless(predicate, statements, else_clause)
         else
           statements = visit(node.statements.body.first)
           predicate = visit(node.predicate)
@@ -3174,7 +3176,7 @@ module Prism
       #           ^^^^^^^^^^^^^
       def visit_when_node(node)
         # This is a special case where we're not going to call on_when directly
-        # because we don't have access to the consequent. Instead, we'll return
+        # because we don't have access to the subsequent. Instead, we'll return
         # the component parts and let the parent node handle it.
         conditions = visit_arguments(node.conditions)
         statements =
@@ -3267,11 +3269,7 @@ module Prism
 
       # Lazily initialize the parse result.
       def result
-        @result ||=
-          begin
-            scopes = RUBY_VERSION >= "3.3.0" ? [] : [[]]
-            Prism.parse(source, scopes: scopes)
-          end
+        @result ||= Prism.parse(source, partial_script: true)
       end
 
       ##########################################################################

data/lib/prism/translation/ruby_parser.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
-require "ruby_parser"
+begin
+  require "ruby_parser"
+rescue LoadError
+  warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.})
+  exit(1)
+end
 
 module Prism
   module Translation
@@ -50,7 +55,19 @@ module Prism
         # a and b
         # ^^^^^^^
         def visit_and_node(node)
-          s(node, :and, visit(node.left), visit(node.right))
+          left = visit(node.left)
+
+          if left[0] == :and
+            # ruby_parser has the and keyword as right-associative as opposed to
+            # prism which has it as left-associative. We reverse that
+            # associativity here.
+            nest = left
+            nest = nest[2] while nest[2][0] == :and
+            nest[2] = s(node, :and, nest[2], visit(node.right))
+            left
+          else
+            s(node, :and, left, visit(node.right))
+          end
         end
 
         # []
@@ -130,7 +147,7 @@ module Prism
             end
 
             current = node.rescue_clause
-            until (current = current.consequent).nil?
+            until (current = current.subsequent).nil?
               result << visit(current)
             end
           end
@@ -246,6 +263,11 @@ module Prism
               when RegularExpressionNode, InterpolatedRegularExpressionNode
                 return s(node, :match2, visit(node.receiver), visit(node.arguments.arguments.first))
               end
+
+              case node.arguments.arguments.first
+              when RegularExpressionNode, InterpolatedRegularExpressionNode
+                return s(node, :match3, visit(node.arguments.arguments.first), visit(node.receiver))
+              end
             end
           end
 
@@ -325,13 +347,13 @@ module Prism
         # case foo; when bar; end
         # ^^^^^^^^^^^^^^^^^^^^^^^
         def visit_case_node(node)
-          s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent)
+          s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause)
         end
 
         # case foo; in bar; end
         # ^^^^^^^^^^^^^^^^^^^^^
         def visit_case_match_node(node)
-          s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent)
+          s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause)
         end
 
         # class Foo; end
@@ -480,9 +502,9 @@ module Prism
         def visit_constant_path_target_node(node)
           inner =
             if node.parent.nil?
-              s(node, :colon3, node.child.name)
+              s(node, :colon3, node.name)
             else
-              s(node, :colon2, visit(node.parent), node.child.name)
+              s(node, :colon2, visit(node.parent), node.name)
             end
 
           s(node, :const, inner)
@@ -678,7 +700,7 @@ module Prism
         # foo ? bar : baz
         # ^^^^^^^^^^^^^^^
         def visit_if_node(node)
-          s(node, :if, visit(node.predicate), visit(node.statements), visit(node.consequent))
+          s(node, :if, visit(node.predicate), visit(node.statements), visit(node.subsequent))
         end
 
         # 1i
@@ -859,6 +881,7 @@ module Prism
         # Visit the interpolated content of the string-like node.
         private def visit_interpolated_parts(parts)
           visited = []
+
           parts.each do |part|
             result = visit(part)
 
@@ -870,14 +893,25 @@ module Prism
               else
                 visited << result
               end
+              visited << :space
+            elsif result[0] == :dstr
+              if !visited.empty? && part.parts[0].is_a?(StringNode)
+                # If we are in the middle of an implicitly concatenated string,
+                # we should not have a bare string as the first part. In this
+                # case we need to visit just that first part and then we can
+                # push the rest of the parts onto the visited array.
+                result[1] = visit(part.parts[0])
+              end
+              visited.concat(result[1..-1])
             else
               visited << result
             end
           end
 
           state = :beginning #: :beginning | :string_content | :interpolated_content
+          results = []
 
-          visited.each_with_object([]) do |result, results|
+          visited.each_with_index do |result, index|
             case state
             when :beginning
               if result.is_a?(String)
@@ -892,7 +926,9 @@ module Prism
                 state = :interpolated_content
               end
             when :string_content
-              if result.is_a?(String)
+              if result == :space
+                # continue
+              elsif result.is_a?(String)
                 results[0] << result
               elsif result.is_a?(Array) && result[0] == :str
                 results[0] << result[1]
@@ -900,10 +936,25 @@ module Prism
                 results << result
                 state = :interpolated_content
               end
-            else
-              results << result
+            when :interpolated_content
+              if result == :space
+                # continue
+              elsif visited[index - 1] != :space && result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line)
+                results[-1][1] << result[1]
+                results[-1].line_max = result.line_max
+              else
+                results << result
+              end
             end
           end
+
+          results
+        end
+
+        # -> { it }
+        #      ^^
+        def visit_it_local_variable_read_node(node)
+          s(node, :call, nil, :it)
         end
 
         # foo(bar: baz)
@@ -1118,7 +1169,19 @@ module Prism
         # a or b
         # ^^^^^^
         def visit_or_node(node)
-          s(node, :or, visit(node.left), visit(node.right))
+          left = visit(node.left)
+
+          if left[0] == :or
+            # ruby_parser has the or keyword as right-associative as opposed to
+            # prism which has it as left-associative. We reverse that
+            # associativity here.
+            nest = left
+            nest = nest[2] while nest[2][0] == :or
+            nest[2] = s(node, :or, nest[2], visit(node.right))
+            left
+          else
+            s(node, :or, left, visit(node.right))
+          end
         end
 
         # def foo(bar, *baz); end
@@ -1374,7 +1437,13 @@ module Prism
         # "foo"
         # ^^^^^
         def visit_string_node(node)
-          s(node, :str, node.unescaped)
+          unescaped = node.unescaped
+
+          if node.forced_binary_encoding?
+            unescaped.force_encoding(Encoding::BINARY)
+          end
+
+          s(node, :str, unescaped)
         end
 
         # super(foo)
@@ -1416,7 +1485,7 @@ module Prism
         # bar unless foo
         # ^^^^^^^^^^^^^^
         def visit_unless_node(node)
-          s(node, :if, visit(node.predicate), visit(node.consequent), visit(node.statements))
+          s(node, :if, visit(node.predicate), visit(node.else_clause), visit(node.statements))
         end
 
         # until foo; bar end
@@ -1536,13 +1605,13 @@ module Prism
       # Parse the given source and translate it into the seattlerb/ruby_parser
       # gem's Sexp format.
       def parse(source, filepath = "(string)")
-        translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath)
+        translate(Prism.parse(source, filepath: filepath, partial_script: true), filepath)
       end
 
       # Parse the given file and translate it into the seattlerb/ruby_parser
       # gem's Sexp format.
       def parse_file(filepath)
-        translate(Prism.parse_file(filepath, scopes: [[]]), filepath)
+        translate(Prism.parse_file(filepath, partial_script: true), filepath)
       end
 
       class << self

data/lib/prism/visitor.rb

@@ -313,6 +313,9 @@ module Prism
     # Visit a InterpolatedXStringNode node
     alias visit_interpolated_x_string_node visit_child_nodes
 
+    # Visit a ItLocalVariableReadNode node
+    alias visit_it_local_variable_read_node visit_child_nodes
+
     # Visit a ItParametersNode node
     alias visit_it_parameters_node visit_child_nodes
 

data/lib/prism.rb

@@ -13,7 +13,6 @@ module Prism
 
   autoload :BasicVisitor, "prism/visitor"
   autoload :Compiler, "prism/compiler"
-  autoload :Debug, "prism/debug"
   autoload :DesugarCompiler, "prism/desugar_compiler"
   autoload :Dispatcher, "prism/dispatcher"
   autoload :DotVisitor, "prism/dot_visitor"
@@ -25,14 +24,15 @@ module Prism
   autoload :Pack, "prism/pack"
   autoload :Pattern, "prism/pattern"
   autoload :Reflection, "prism/reflection"
+  autoload :Relocation, "prism/relocation"
   autoload :Serialize, "prism/serialize"
+  autoload :StringQuery, "prism/string_query"
   autoload :Translation, "prism/translation"
   autoload :Visitor, "prism/visitor"
 
   # Some of these constants are not meant to be exposed, so marking them as
   # private here.
 
-  private_constant :Debug
   private_constant :LexCompat
   private_constant :LexRipper
 
@@ -71,21 +71,19 @@ require_relative "prism/polyfill/byteindex"
 require_relative "prism/node"
 require_relative "prism/node_ext"
 require_relative "prism/parse_result"
-require_relative "prism/parse_result/comments"
-require_relative "prism/parse_result/newlines"
 
 # This is a Ruby implementation of the prism parser. If we're running on CRuby
 # and we haven't explicitly set the PRISM_FFI_BACKEND environment variable, then
 # it's going to require the built library. Otherwise, it's going to require a
 # module that uses FFI to call into the library.
 if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
-  require "prism/prism"
-
   # The C extension is the default backend on CRuby.
   Prism::BACKEND = :CEXT
-else
-  require_relative "prism/ffi"
 
+  require "prism/prism"
+else
   # The FFI backend is used on other Ruby implementations.
   Prism::BACKEND = :FFI
+
+  require_relative "prism/ffi"
 end

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "0.29.0"
+  spec.version = "1.3.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
     "docs/parser_translation.md",
     "docs/parsing_rules.md",
     "docs/releasing.md",
+    "docs/relocation.md",
     "docs/ripper_translation.md",
     "docs/ruby_api.md",
     "docs/ruby_parser_translation.md",
@@ -65,12 +66,10 @@ Gem::Specification.new do |spec|
     "include/prism/util/pm_newline_list.h",
     "include/prism/util/pm_strncasecmp.h",
     "include/prism/util/pm_string.h",
-    "include/prism/util/pm_string_list.h",
     "include/prism/util/pm_strpbrk.h",
     "include/prism/version.h",
     "lib/prism.rb",
     "lib/prism/compiler.rb",
-    "lib/prism/debug.rb",
     "lib/prism/desugar_compiler.rb",
     "lib/prism/dispatcher.rb",
     "lib/prism/dot_visitor.rb",
@@ -84,19 +83,21 @@ Gem::Specification.new do |spec|
     "lib/prism/pack.rb",
     "lib/prism/parse_result.rb",
     "lib/prism/parse_result/comments.rb",
+    "lib/prism/parse_result/errors.rb",
     "lib/prism/parse_result/newlines.rb",
     "lib/prism/pattern.rb",
     "lib/prism/polyfill/byteindex.rb",
     "lib/prism/polyfill/unpack1.rb",
     "lib/prism/reflection.rb",
+    "lib/prism/relocation.rb",
     "lib/prism/serialize.rb",
+    "lib/prism/string_query.rb",
     "lib/prism/translation.rb",
     "lib/prism/translation/parser.rb",
     "lib/prism/translation/parser33.rb",
     "lib/prism/translation/parser34.rb",
     "lib/prism/translation/parser/compiler.rb",
     "lib/prism/translation/parser/lexer.rb",
-    "lib/prism/translation/parser/rubocop.rb",
     "lib/prism/translation/ripper.rb",
     "lib/prism/translation/ripper/sexp.rb",
     "lib/prism/translation/ripper/shim.rb",
@@ -105,11 +106,13 @@ Gem::Specification.new do |spec|
     "prism.gemspec",
     "rbi/prism.rbi",
     "rbi/prism/compiler.rbi",
+    "rbi/prism/dsl.rbi",
     "rbi/prism/inspect_visitor.rbi",
     "rbi/prism/node_ext.rbi",
     "rbi/prism/node.rbi",
     "rbi/prism/parse_result.rbi",
     "rbi/prism/reflection.rbi",
+    "rbi/prism/string_query.rbi",
     "rbi/prism/translation/parser.rbi",
     "rbi/prism/translation/parser33.rbi",
     "rbi/prism/translation/parser34.rbi",
@@ -129,7 +132,9 @@ Gem::Specification.new do |spec|
     "sig/prism/parse_result.rbs",
     "sig/prism/pattern.rbs",
     "sig/prism/reflection.rbs",
+    "sig/prism/relocation.rbs",
     "sig/prism/serialize.rbs",
+    "sig/prism/string_query.rbs",
     "sig/prism/visitor.rbs",
     "src/diagnostic.c",
     "src/encoding.c",
@@ -149,7 +154,6 @@ Gem::Specification.new do |spec|
     "src/util/pm_list.c",
     "src/util/pm_memchr.c",
     "src/util/pm_newline_list.c",
-    "src/util/pm_string_list.c",
     "src/util/pm_string.c",
     "src/util/pm_strncasecmp.c",
     "src/util/pm_strpbrk.c"