prism 1.1.0 → 1.3.0

data/lib/prism/translation/parser/compiler.rb

@@ -188,7 +188,7 @@ module Prism
                 rescue_clause.exceptions.any? ? builder.array(nil, visit_all(rescue_clause.exceptions), nil) : nil,
                 token(rescue_clause.operator_loc),
                 visit(rescue_clause.reference),
-                srange_find(find_start_offset, find_end_offset, [";"]),
+                srange_find(find_start_offset, find_end_offset, ";"),
                 visit(rescue_clause.statements)
               )
             end until (rescue_clause = rescue_clause.subsequent).nil?
@@ -294,7 +294,7 @@ module Prism
                       visit_all(arguments),
                       token(node.closing_loc),
                     ),
-                    srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, ["="]),
+                    srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, "="),
                     visit(node.arguments.arguments.last)
                   ),
                   block
@@ -311,7 +311,7 @@ module Prism
             if name.end_with?("=") && !message_loc.slice.end_with?("=") && node.arguments && block.nil?
               builder.assign(
                 builder.attr_asgn(visit(node.receiver), call_operator, token(message_loc)),
-                srange_find(message_loc.end_offset, node.arguments.location.start_offset, ["="]),
+                srange_find(message_loc.end_offset, node.arguments.location.start_offset, "="),
                 visit(node.arguments.arguments.last)
               )
             else
@@ -733,10 +733,10 @@ module Prism
             visit(node.index),
             token(node.in_keyword_loc),
             visit(node.collection),
-            if node.do_keyword_loc
-              token(node.do_keyword_loc)
+            if (do_keyword_loc = node.do_keyword_loc)
+              token(do_keyword_loc)
             else
-              srange_find(node.collection.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset, [";"])
+              srange_find(node.collection.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset, ";")
             end,
             visit(node.statements),
             token(node.end_keyword_loc)
@@ -865,10 +865,10 @@ module Prism
             builder.condition(
               token(node.if_keyword_loc),
               visit(node.predicate),
-              if node.then_keyword_loc
-                token(node.then_keyword_loc)
+              if (then_keyword_loc = node.then_keyword_loc)
+                token(then_keyword_loc)
               else
-                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.subsequent&.location || node.end_keyword_loc).start_offset, [";"])
+                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.subsequent&.location || node.end_keyword_loc).start_offset, ";")
               end,
               visit(node.statements),
               case node.subsequent
@@ -931,7 +931,11 @@ module Prism
             token(node.in_loc),
             pattern,
             guard,
-            srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset, [";", "then"]),
+            if (then_loc = node.then_loc)
+              token(then_loc)
+            else
+              srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset, ";")
+            end,
             visit(node.statements)
           )
         end
@@ -1781,10 +1785,10 @@ module Prism
             builder.condition(
               token(node.keyword_loc),
               visit(node.predicate),
-              if node.then_keyword_loc
-                token(node.then_keyword_loc)
+              if (then_keyword_loc = node.then_keyword_loc)
+                token(then_keyword_loc)
               else
-                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.else_clause&.location || node.end_keyword_loc).start_offset, [";"])
+                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.else_clause&.location || node.end_keyword_loc).start_offset, ";")
               end,
               visit(node.else_clause),
               token(node.else_clause&.else_keyword_loc),
@@ -1812,7 +1816,11 @@ module Prism
               :until,
               token(node.keyword_loc),
               visit(node.predicate),
-              srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, [";", "do"]),
+              if (do_keyword_loc = node.do_keyword_loc)
+                token(do_keyword_loc)
+              else
+                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, ";")
+              end,
               visit(node.statements),
               token(node.closing_loc)
             )
@@ -1832,10 +1840,10 @@ module Prism
           builder.when(
             token(node.keyword_loc),
             visit_all(node.conditions),
-            if node.then_keyword_loc
-              token(node.then_keyword_loc)
+            if (then_keyword_loc = node.then_keyword_loc)
+              token(then_keyword_loc)
             else
-              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset, [";"])
+              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset, ";")
             end,
             visit(node.statements)
           )
@@ -1852,7 +1860,11 @@ module Prism
               :while,
               token(node.keyword_loc),
               visit(node.predicate),
-              srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, [";", "do"]),
+              if (do_keyword_loc = node.do_keyword_loc)
+                token(do_keyword_loc)
+              else
+                srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, ";")
+              end,
               visit(node.statements),
               token(node.closing_loc)
             )
@@ -1985,18 +1997,16 @@ module Prism
           Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])
         end
 
-        # Constructs a new source range by finding the given tokens between the
-        # given start offset and end offset. If the needle is not found, it
+        # Constructs a new source range by finding the given character between
+        # the given start offset and end offset. If the needle is not found, it
         # returns nil. Importantly it does not search past newlines or comments.
         #
         # Note that end_offset is allowed to be nil, in which case this will
         # search until the end of the string.
-        def srange_find(start_offset, end_offset, tokens)
-          if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/\A(\s*)(#{tokens.join("|")})/))
-            _, whitespace, token = *match
-            token_offset = start_offset + whitespace.bytesize
-
-            [token, Range.new(source_buffer, offset_cache[token_offset], offset_cache[token_offset + token.bytesize])]
+        def srange_find(start_offset, end_offset, character)
+          if (match = source_buffer.source.byteslice(start_offset...end_offset)[/\A\s*#{character}/])
+            final_offset = start_offset + match.bytesize
+            [character, Range.new(source_buffer, offset_cache[final_offset - character.bytesize], offset_cache[final_offset])]
           end
         end
 

data/lib/prism/translation/parser.rb

@@ -51,7 +51,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]], encoding: false), offset_cache)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
 
         build_ast(result.value, offset_cache)
       ensure
@@ -64,7 +64,7 @@ module Prism
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]], encoding: false), offset_cache)
+        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
 
         [
           build_ast(result.value, offset_cache),
@@ -83,7 +83,7 @@ module Prism
         offset_cache = build_offset_cache(source)
         result =
           begin
-            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]], encoding: false), offset_cache)
+            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
           rescue ::Parser::SyntaxError
             raise if !recover
           end

data/lib/prism/translation/ripper.rb

@@ -3269,11 +3269,7 @@ module Prism
 
       # Lazily initialize the parse result.
       def result
-        @result ||=
-          begin
-            scopes = RUBY_VERSION >= "3.3.0" ? [] : [[]]
-            Prism.parse(source, scopes: scopes)
-          end
+        @result ||= Prism.parse(source, partial_script: true)
       end
 
       ##########################################################################

data/lib/prism/translation/ruby_parser.rb

@@ -881,6 +881,7 @@ module Prism
         # Visit the interpolated content of the string-like node.
         private def visit_interpolated_parts(parts)
           visited = []
+
           parts.each do |part|
             result = visit(part)
 
@@ -892,6 +893,7 @@ module Prism
               else
                 visited << result
               end
+              visited << :space
             elsif result[0] == :dstr
               if !visited.empty? && part.parts[0].is_a?(StringNode)
                 # If we are in the middle of an implicitly concatenated string,
@@ -907,8 +909,9 @@ module Prism
           end
 
           state = :beginning #: :beginning | :string_content | :interpolated_content
+          results = []
 
-          visited.each_with_object([]) do |result, results|
+          visited.each_with_index do |result, index|
             case state
             when :beginning
               if result.is_a?(String)
@@ -923,7 +926,9 @@ module Prism
                 state = :interpolated_content
               end
             when :string_content
-              if result.is_a?(String)
+              if result == :space
+                # continue
+              elsif result.is_a?(String)
                 results[0] << result
               elsif result.is_a?(Array) && result[0] == :str
                 results[0] << result[1]
@@ -932,7 +937,9 @@ module Prism
                 state = :interpolated_content
               end
             when :interpolated_content
-              if result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line)
+              if result == :space
+                # continue
+              elsif visited[index - 1] != :space && result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line)
                 results[-1][1] << result[1]
                 results[-1].line_max = result.line_max
               else
@@ -940,6 +947,8 @@ module Prism
               end
             end
           end
+
+          results
         end
 
         # -> { it }
@@ -1596,13 +1605,13 @@ module Prism
       # Parse the given source and translate it into the seattlerb/ruby_parser
       # gem's Sexp format.
       def parse(source, filepath = "(string)")
-        translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath)
+        translate(Prism.parse(source, filepath: filepath, partial_script: true), filepath)
       end
 
       # Parse the given file and translate it into the seattlerb/ruby_parser
       # gem's Sexp format.
       def parse_file(filepath)
-        translate(Prism.parse_file(filepath, scopes: [[]]), filepath)
+        translate(Prism.parse_file(filepath, partial_script: true), filepath)
       end
 
       class << self

data/lib/prism.rb

@@ -24,7 +24,9 @@ module Prism
   autoload :Pack, "prism/pack"
   autoload :Pattern, "prism/pattern"
   autoload :Reflection, "prism/reflection"
+  autoload :Relocation, "prism/relocation"
   autoload :Serialize, "prism/serialize"
+  autoload :StringQuery, "prism/string_query"
   autoload :Translation, "prism/translation"
   autoload :Visitor, "prism/visitor"
 
@@ -75,13 +77,13 @@ require_relative "prism/parse_result"
 # it's going to require the built library. Otherwise, it's going to require a
 # module that uses FFI to call into the library.
 if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
-  require "prism/prism"
-
   # The C extension is the default backend on CRuby.
   Prism::BACKEND = :CEXT
-else
-  require_relative "prism/ffi"
 
+  require "prism/prism"
+else
   # The FFI backend is used on other Ruby implementations.
   Prism::BACKEND = :FFI
+
+  require_relative "prism/ffi"
 end

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "1.1.0"
+  spec.version = "1.3.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
     "docs/parser_translation.md",
     "docs/parsing_rules.md",
     "docs/releasing.md",
+    "docs/relocation.md",
     "docs/ripper_translation.md",
     "docs/ruby_api.md",
     "docs/ruby_parser_translation.md",
@@ -88,7 +89,9 @@ Gem::Specification.new do |spec|
     "lib/prism/polyfill/byteindex.rb",
     "lib/prism/polyfill/unpack1.rb",
     "lib/prism/reflection.rb",
+    "lib/prism/relocation.rb",
     "lib/prism/serialize.rb",
+    "lib/prism/string_query.rb",
     "lib/prism/translation.rb",
     "lib/prism/translation/parser.rb",
     "lib/prism/translation/parser33.rb",
@@ -109,6 +112,7 @@ Gem::Specification.new do |spec|
     "rbi/prism/node.rbi",
     "rbi/prism/parse_result.rbi",
     "rbi/prism/reflection.rbi",
+    "rbi/prism/string_query.rbi",
     "rbi/prism/translation/parser.rbi",
     "rbi/prism/translation/parser33.rbi",
     "rbi/prism/translation/parser34.rbi",
@@ -128,7 +132,9 @@ Gem::Specification.new do |spec|
     "sig/prism/parse_result.rbs",
     "sig/prism/pattern.rbs",
     "sig/prism/reflection.rbs",
+    "sig/prism/relocation.rbs",
     "sig/prism/serialize.rbs",
+    "sig/prism/string_query.rbs",
     "sig/prism/visitor.rbs",
     "src/diagnostic.c",
     "src/encoding.c",

data/rbi/prism/dsl.rbi

@@ -451,14 +451,14 @@ module Prism::DSL
   sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, keyword_loc: Prism::Location, predicate: Prism::Node, then_keyword_loc: T.nilable(Prism::Location), statements: T.nilable(Prism::StatementsNode), else_clause: T.nilable(Prism::ElseNode), end_keyword_loc: T.nilable(Prism::Location)).returns(Prism::UnlessNode) }
   def unless_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, predicate: default_node(source, location), then_keyword_loc: nil, statements: nil, else_clause: nil, end_keyword_loc: nil); end
 
-  sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, keyword_loc: Prism::Location, closing_loc: T.nilable(Prism::Location), predicate: Prism::Node, statements: T.nilable(Prism::StatementsNode)).returns(Prism::UntilNode) }
-  def until_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, closing_loc: nil, predicate: default_node(source, location), statements: nil); end
+  sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, keyword_loc: Prism::Location, do_keyword_loc: T.nilable(Prism::Location), closing_loc: T.nilable(Prism::Location), predicate: Prism::Node, statements: T.nilable(Prism::StatementsNode)).returns(Prism::UntilNode) }
+  def until_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, do_keyword_loc: nil, closing_loc: nil, predicate: default_node(source, location), statements: nil); end
 
   sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, keyword_loc: Prism::Location, conditions: T::Array[Prism::Node], then_keyword_loc: T.nilable(Prism::Location), statements: T.nilable(Prism::StatementsNode)).returns(Prism::WhenNode) }
   def when_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, conditions: [], then_keyword_loc: nil, statements: nil); end
 
-  sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, keyword_loc: Prism::Location, closing_loc: T.nilable(Prism::Location), predicate: Prism::Node, statements: T.nilable(Prism::StatementsNode)).returns(Prism::WhileNode) }
-  def while_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, closing_loc: nil, predicate: default_node(source, location), statements: nil); end
+  sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, keyword_loc: Prism::Location, do_keyword_loc: T.nilable(Prism::Location), closing_loc: T.nilable(Prism::Location), predicate: Prism::Node, statements: T.nilable(Prism::StatementsNode)).returns(Prism::WhileNode) }
+  def while_node(source: default_source, node_id: 0, location: default_location, flags: 0, keyword_loc: location, do_keyword_loc: nil, closing_loc: nil, predicate: default_node(source, location), statements: nil); end
 
   sig { params(source: Prism::Source, node_id: Integer, location: Prism::Location, flags: Integer, opening_loc: Prism::Location, content_loc: Prism::Location, closing_loc: Prism::Location, unescaped: String).returns(Prism::XStringNode) }
   def x_string_node(source: default_source, node_id: 0, location: default_location, flags: 0, opening_loc: location, content_loc: location, closing_loc: location, unescaped: ""); end