prism 0.26.0 → 0.27.0

data/lib/prism/parse_result/newlines.rb

@@ -58,8 +58,6 @@ module Prism
 
     # Walk the tree and mark nodes that are on a new line.
     def mark_newlines!
-      value = self.value
-      raise "This method should only be called on a parse result that contains a node" unless Node === value
       value.accept(Newlines.new(Array.new(1 + source.offsets.size, false))) # steep:ignore
     end
   end

data/lib/prism/parse_result.rb

@@ -438,14 +438,9 @@ module Prism
   end
 
   # This represents the result of a call to ::parse or ::parse_file. It contains
-  # the AST, any comments that were encounters, and any errors that were
-  # encountered.
-  class ParseResult
-    # The value that was generated by parsing. Normally this holds the AST, but
-    # it can sometimes how a list of tokens or other results passed back from
-    # the parser.
-    attr_reader :value
-
+  # the requested structure, any comments that were encounters, and any errors
+  # that were encountered.
+  class Result
     # The list of comments that were encountered during parsing.
     attr_reader :comments
 
@@ -466,9 +461,8 @@ module Prism
     # A Source instance that represents the source code that was parsed.
     attr_reader :source
 
-    # Create a new parse result object with the given values.
-    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
-      @value = value
+    # Create a new result object with the given values.
+    def initialize(comments, magic_comments, data_loc, errors, warnings, source)
       @comments = comments
       @magic_comments = magic_comments
       @data_loc = data_loc
@@ -477,9 +471,9 @@ module Prism
       @source = source
     end
 
-    # Implement the hash pattern matching interface for ParseResult.
+    # Implement the hash pattern matching interface for Result.
     def deconstruct_keys(keys)
-      { value: value, comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
+      { comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings }
     end
 
     # Returns the encoding of the source code that was parsed.
@@ -500,6 +494,58 @@ module Prism
     end
   end
 
+  # This is a result specific to the `parse` and `parse_file` methods.
+  class ParseResult < Result
+    # The syntax tree that was parsed from the source code.
+    attr_reader :value
+
+    # Create a new parse result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for ParseResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
+  # This is a result specific to the `lex` and `lex_file` methods.
+  class LexResult < Result
+    # The list of tokens that were parsed from the source code.
+    attr_reader :value
+
+    # Create a new lex result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for LexResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
+  # This is a result specific to the `parse_lex` and `parse_lex_file` methods.
+  class ParseLexResult < Result
+    # A tuple of the syntax tree and the list of tokens that were parsed from
+    # the source code.
+    attr_reader :value
+
+    # Create a new parse lex result object with the given values.
+    def initialize(value, comments, magic_comments, data_loc, errors, warnings, source)
+      @value = value
+      super(comments, magic_comments, data_loc, errors, warnings, source)
+    end
+
+    # Implement the hash pattern matching interface for ParseLexResult.
+    def deconstruct_keys(keys)
+      super.merge!(value: value)
+    end
+  end
+
   # This represents a token from the Ruby source.
   class Token
     # The Source object that represents the source this token came from.

data/lib/prism/reflection.rb

@@ -73,14 +73,16 @@ module Prism
     class OptionalLocationField < Field
     end
 
-    # A uint8 field represents an unsigned 8-bit integer value on a node. It
-    # resolves to an Integer in Ruby.
-    class UInt8Field < Field
+    # An integer field represents an integer value. It is used to represent the
+    # value of an integer literal, the depth of local variables, and the number
+    # of a numbered reference. It resolves to an Integer in Ruby.
+    class IntegerField < Field
     end
 
-    # A uint32 field represents an unsigned 32-bit integer value on a node. It
-    # resolves to an Integer in Ruby.
-    class UInt32Field < Field
+    # A float field represents a double-precision floating point value. It is
+    # used exclusively to represent the value of a floating point literal. It
+    # resolves to a Float in Ruby.
+    class FloatField < Field
     end
 
     # A flags field represents a bitset of flags on a node. It resolves to an
@@ -98,18 +100,6 @@ module Prism
       end
     end
 
-    # An integer field represents an arbitrarily-sized integer value. It is used
-    # exclusively to represent the value of an integer literal. It resolves to
-    # an Integer in Ruby.
-    class IntegerField < Field
-    end
-
-    # A double field represents a double-precision floating point value. It is
-    # used exclusively to represent the value of a floating point literal. It
-    # resolves to a Float in Ruby.
-    class DoubleField < Field
-    end
-
     # Returns the fields for the given node.
     def self.fields_for(node)
       case node.type
@@ -220,7 +210,7 @@ module Prism
       when :flip_flop_node
         [FlagsField.new(:flags, [:exclude_end?]), OptionalNodeField.new(:left), OptionalNodeField.new(:right), LocationField.new(:operator_loc)]
       when :float_node
-        [DoubleField.new(:value)]
+        [FloatField.new(:value)]
       when :for_node
         [NodeField.new(:index), NodeField.new(:collection), OptionalNodeField.new(:statements), LocationField.new(:for_keyword_loc), LocationField.new(:in_keyword_loc), OptionalLocationField.new(:do_keyword_loc), LocationField.new(:end_keyword_loc)]
       when :forwarding_arguments_node
@@ -276,7 +266,7 @@ module Prism
       when :instance_variable_write_node
         [ConstantField.new(:name), LocationField.new(:name_loc), NodeField.new(:value), LocationField.new(:operator_loc)]
       when :integer_node
-        [FlagsField.new(:flags, [:binary?, :decimal?, :octal?, :hexadecimal?]), IntegerField.new(:value)]
+        [FlagsField.new(:flags, [:binary?, :decimal?, :octal?, :hexadecimal?]), Integer.new(:value)]
       when :interpolated_match_last_line_node
         [FlagsField.new(:flags, [:ignore_case?, :extended?, :multi_line?, :once?, :euc_jp?, :ascii_8bit?, :windows_31j?, :utf_8?, :forced_utf8_encoding?, :forced_binary_encoding?, :forced_us_ascii_encoding?]), LocationField.new(:opening_loc), NodeListField.new(:parts), LocationField.new(:closing_loc)]
       when :interpolated_regular_expression_node
@@ -296,17 +286,17 @@ module Prism
       when :lambda_node
         [ConstantListField.new(:locals), LocationField.new(:operator_loc), LocationField.new(:opening_loc), LocationField.new(:closing_loc), OptionalNodeField.new(:parameters), OptionalNodeField.new(:body)]
       when :local_variable_and_write_node
-        [LocationField.new(:name_loc), LocationField.new(:operator_loc), NodeField.new(:value), ConstantField.new(:name), UInt32Field.new(:depth)]
+        [LocationField.new(:name_loc), LocationField.new(:operator_loc), NodeField.new(:value), ConstantField.new(:name), Integer.new(:depth)]
       when :local_variable_operator_write_node
-        [LocationField.new(:name_loc), LocationField.new(:operator_loc), NodeField.new(:value), ConstantField.new(:name), ConstantField.new(:operator), UInt32Field.new(:depth)]
+        [LocationField.new(:name_loc), LocationField.new(:operator_loc), NodeField.new(:value), ConstantField.new(:name), ConstantField.new(:operator), Integer.new(:depth)]
       when :local_variable_or_write_node
-        [LocationField.new(:name_loc), LocationField.new(:operator_loc), NodeField.new(:value), ConstantField.new(:name), UInt32Field.new(:depth)]
+        [LocationField.new(:name_loc), LocationField.new(:operator_loc), NodeField.new(:value), ConstantField.new(:name), Integer.new(:depth)]
       when :local_variable_read_node
-        [ConstantField.new(:name), UInt32Field.new(:depth)]
+        [ConstantField.new(:name), Integer.new(:depth)]
       when :local_variable_target_node
-        [ConstantField.new(:name), UInt32Field.new(:depth)]
+        [ConstantField.new(:name), Integer.new(:depth)]
       when :local_variable_write_node
-        [ConstantField.new(:name), UInt32Field.new(:depth), LocationField.new(:name_loc), NodeField.new(:value), LocationField.new(:operator_loc)]
+        [ConstantField.new(:name), Integer.new(:depth), LocationField.new(:name_loc), NodeField.new(:value), LocationField.new(:operator_loc)]
       when :match_last_line_node
         [FlagsField.new(:flags, [:ignore_case?, :extended?, :multi_line?, :once?, :euc_jp?, :ascii_8bit?, :windows_31j?, :utf_8?, :forced_utf8_encoding?, :forced_binary_encoding?, :forced_us_ascii_encoding?]), LocationField.new(:opening_loc), LocationField.new(:content_loc), LocationField.new(:closing_loc), StringField.new(:unescaped)]
       when :match_predicate_node
@@ -330,9 +320,9 @@ module Prism
       when :no_keywords_parameter_node
         [LocationField.new(:operator_loc), LocationField.new(:keyword_loc)]
       when :numbered_parameters_node
-        [UInt8Field.new(:maximum)]
+        [Integer.new(:maximum)]
       when :numbered_reference_read_node
-        [UInt32Field.new(:number)]
+        [Integer.new(:number)]
       when :optional_keyword_parameter_node
         [FlagsField.new(:flags, [:repeated_parameter?]), ConstantField.new(:name), LocationField.new(:name_loc), NodeField.new(:value)]
       when :optional_parameter_node

data/lib/prism/serialize.rb

@@ -18,7 +18,7 @@ module Prism
 
     # The minor version of prism that we are expecting to find in the serialized
     # strings.
-    MINOR_VERSION = 26
+    MINOR_VERSION = 27
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.
@@ -260,6 +260,8 @@ module Prism
         :invalid_character,
         :invalid_encoding_magic_comment,
         :invalid_float_exponent,
+        :invalid_local_variable_read,
+        :invalid_local_variable_write,
         :invalid_multibyte_char,
         :invalid_multibyte_character,
         :invalid_multibyte_escape,
@@ -332,6 +334,7 @@ module Prism
         :pattern_hash_key,
         :pattern_hash_key_duplicate,
         :pattern_hash_key_label,
+        :pattern_hash_key_locals,
         :pattern_ident_after_hrocket,
         :pattern_label_after_comma,
         :pattern_rest,
@@ -426,7 +429,7 @@ module Prism
           length = load_varuint
           lex_state = load_varuint
           location = Location.new(@source, start, length)
-          tokens << [Prism::Token.new(source, type, location.slice, location), lex_state]
+          tokens << [Token.new(source, type, location.slice, location), lex_state]
         end
 
         tokens
@@ -441,7 +444,7 @@ module Prism
         tokens.each { |token,| token.value.force_encoding(encoding) }
 
         raise "Expected to consume all bytes while deserializing" unless @io.eof?
-        Prism::ParseResult.new(tokens, comments, magic_comments, data_loc, errors, warnings, @source)
+        LexResult.new(tokens, comments, magic_comments, data_loc, errors, warnings, @source)
       end
 
       def load_nodes
@@ -460,7 +463,7 @@ module Prism
 
       def load_result
         node, comments, magic_comments, data_loc, errors, warnings = load_nodes
-        Prism::ParseResult.new(node, comments, magic_comments, data_loc, errors, warnings, @source)
+        ParseResult.new(node, comments, magic_comments, data_loc, errors, warnings, @source)
       end
 
       private

data/lib/prism/translation/parser/compiler.rb

@@ -839,7 +839,7 @@ module Prism
             token(node.in_loc),
             pattern,
             guard,
-            srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset || node.location.end_offset, [";", "then"]),
+            srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset, [";", "then"]),
             visit(node.statements)
           )
         end
@@ -1030,6 +1030,12 @@ module Prism
           end
         end
 
+        # -> { it }
+        # ^^^^^^^^^
+        def visit_it_parameters_node(node)
+          builder.args(nil, [], nil, false)
+        end
+
         # foo(bar: baz)
         #     ^^^^^^^^
         def visit_keyword_hash_node(node)
@@ -1052,13 +1058,14 @@ module Prism
         # ^^^^^
         def visit_lambda_node(node)
           parameters = node.parameters
+          implicit_parameters = parameters.is_a?(NumberedParametersNode) || parameters.is_a?(ItParametersNode)
 
           builder.block(
             builder.call_lambda(token(node.operator_loc)),
             [node.opening, srange(node.opening_loc)],
             if parameters.nil?
               builder.args(nil, [], nil, false)
-            elsif node.parameters.is_a?(NumberedParametersNode)
+            elsif implicit_parameters
               visit(node.parameters)
             else
               builder.args(
@@ -1068,7 +1075,7 @@ module Prism
                 false
               )
             end,
-            node.body&.accept(copy_compiler(forwarding: parameters.is_a?(NumberedParametersNode) ? [] : find_forwarding(parameters&.parameters))),
+            node.body&.accept(copy_compiler(forwarding: implicit_parameters ? [] : find_forwarding(parameters&.parameters))),
             [node.closing, srange(node.closing_loc)]
           )
         end
@@ -1076,7 +1083,14 @@ module Prism
         # foo
         # ^^^
         def visit_local_variable_read_node(node)
-          builder.ident([node.name, srange(node.location)]).updated(:lvar)
+          name = node.name
+
+          # This is just a guess. parser doesn't have support for the implicit
+          # `it` variable yet, so we'll probably have to visit this once it
+          # does.
+          name = :it if name == :"0it"
+
+          builder.ident([name, srange(node.location)]).updated(:lvar)
         end
 
         # foo = 1
@@ -1665,7 +1679,7 @@ module Prism
         end
 
         # until foo; bar end
-        # ^^^^^^^^^^^^^^^^^
+        # ^^^^^^^^^^^^^^^^^^
         #
         # bar until foo
         # ^^^^^^^^^^^^^
@@ -1698,7 +1712,7 @@ module Prism
             if node.then_keyword_loc
               token(node.then_keyword_loc)
             else
-              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset || (node.conditions.last.location.end_offset + 1), [";"])
+              srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset, [";"])
             end,
             visit(node.statements)
           )
@@ -1857,12 +1871,16 @@ module Prism
 
         # Constructs a new source range by finding the given tokens between the
         # given start offset and end offset. If the needle is not found, it
-        # returns nil.
+        # returns nil. Importantly it does not search past newlines or comments.
+        #
+        # Note that end_offset is allowed to be nil, in which case this will
+        # search until the end of the string.
         def srange_find(start_offset, end_offset, tokens)
-          tokens.find do |token|
-            next unless (index = source_buffer.source.byteslice(start_offset...end_offset).index(token))
-            offset = start_offset + index
-            return [token, Range.new(source_buffer, offset_cache[offset], offset_cache[offset + token.length])]
+          if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/(\s*)(#{tokens.join("|")})/))
+            _, whitespace, token = *match
+            token_offset = start_offset + whitespace.bytesize
+
+            [token, Range.new(source_buffer, offset_cache[token_offset], offset_cache[token_offset + token.bytesize])]
           end
         end
 
@@ -1875,13 +1893,14 @@ module Prism
         def visit_block(call, block)
           if block
             parameters = block.parameters
+            implicit_parameters = parameters.is_a?(NumberedParametersNode) || parameters.is_a?(ItParametersNode)
 
             builder.block(
               call,
               token(block.opening_loc),
               if parameters.nil?
                 builder.args(nil, [], nil, false)
-              elsif parameters.is_a?(NumberedParametersNode)
+              elsif implicit_parameters
                 visit(parameters)
               else
                 builder.args(
@@ -1896,7 +1915,7 @@ module Prism
                   false
                 )
               end,
-              block.body&.accept(copy_compiler(forwarding: parameters.is_a?(NumberedParametersNode) ? [] : find_forwarding(parameters&.parameters))),
+              block.body&.accept(copy_compiler(forwarding: implicit_parameters ? [] : find_forwarding(parameters&.parameters))),
               token(block.closing_loc)
             )
           else

data/lib/prism/translation/ripper.rb

@@ -19,31 +19,31 @@ module Prism
     # The main known difference is that we may omit dispatching some events in
     # some cases. This impacts the following events:
     #
-    # * on_assign_error
-    # * on_comma
-    # * on_ignored_nl
-    # * on_ignored_sp
-    # * on_kw
-    # * on_label_end
-    # * on_lbrace
-    # * on_lbracket
-    # * on_lparen
-    # * on_nl
-    # * on_op
-    # * on_operator_ambiguous
-    # * on_rbrace
-    # * on_rbracket
-    # * on_rparen
-    # * on_semicolon
-    # * on_sp
-    # * on_symbeg
-    # * on_tstring_beg
-    # * on_tstring_end
+    # - on_assign_error
+    # - on_comma
+    # - on_ignored_nl
+    # - on_ignored_sp
+    # - on_kw
+    # - on_label_end
+    # - on_lbrace
+    # - on_lbracket
+    # - on_lparen
+    # - on_nl
+    # - on_op
+    # - on_operator_ambiguous
+    # - on_rbrace
+    # - on_rbracket
+    # - on_rparen
+    # - on_semicolon
+    # - on_sp
+    # - on_symbeg
+    # - on_tstring_beg
+    # - on_tstring_end
     #
     class Ripper < Compiler
       # Parses the given Ruby program read from +src+.
       # +src+ must be a String or an IO or a object with a #gets method.
-      def Ripper.parse(src, filename = "(ripper)", lineno = 1)
+      def self.parse(src, filename = "(ripper)", lineno = 1)
         new(src, filename, lineno).parse
       end
 
@@ -54,22 +54,22 @@ module Prism
       # By default, this method does not handle syntax errors in +src+,
       # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
       #
-      #   require 'ripper'
-      #   require 'pp'
+      #     require "ripper"
+      #     require "pp"
       #
-      #   pp Ripper.lex("def m(a) nil end")
-      #   #=> [[[1,  0], :on_kw,     "def", FNAME    ],
-      #        [[1,  3], :on_sp,     " ",   FNAME    ],
-      #        [[1,  4], :on_ident,  "m",   ENDFN    ],
-      #        [[1,  5], :on_lparen, "(",   BEG|LABEL],
-      #        [[1,  6], :on_ident,  "a",   ARG      ],
-      #        [[1,  7], :on_rparen, ")",   ENDFN    ],
-      #        [[1,  8], :on_sp,     " ",   BEG      ],
-      #        [[1,  9], :on_kw,     "nil", END      ],
-      #        [[1, 12], :on_sp,     " ",   END      ],
-      #        [[1, 13], :on_kw,     "end", END      ]]
+      #     pp Ripper.lex("def m(a) nil end")
+      #     #=> [[[1,  0], :on_kw,     "def", FNAME    ],
+      #          [[1,  3], :on_sp,     " ",   FNAME    ],
+      #          [[1,  4], :on_ident,  "m",   ENDFN    ],
+      #          [[1,  5], :on_lparen, "(",   BEG|LABEL],
+      #          [[1,  6], :on_ident,  "a",   ARG      ],
+      #          [[1,  7], :on_rparen, ")",   ENDFN    ],
+      #          [[1,  8], :on_sp,     " ",   BEG      ],
+      #          [[1,  9], :on_kw,     "nil", END      ],
+      #          [[1, 12], :on_sp,     " ",   END      ],
+      #          [[1, 13], :on_kw,     "end", END      ]]
       #
-      def Ripper.lex(src, filename = "-", lineno = 1, raise_errors: false)
+      def self.lex(src, filename = "-", lineno = 1, raise_errors: false)
         result = Prism.lex_compat(src, filepath: filename, line: lineno)
 
         if result.failure? && raise_errors
@@ -368,17 +368,17 @@ module Prism
       # returning +nil+ in such cases. Use the +raise_errors+ keyword
       # to raise a SyntaxError for an error in +src+.
       #
-      #   require "ripper"
-      #   require "pp"
+      #     require "ripper"
+      #     require "pp"
       #
-      #   pp Ripper.sexp("def m(a) nil end")
-      #     #=> [:program,
-      #          [[:def,
-      #           [:@ident, "m", [1, 4]],
-      #           [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil, nil, nil]],
-      #           [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]]
+      #     pp Ripper.sexp("def m(a) nil end")
+      #       #=> [:program,
+      #            [[:def,
+      #             [:@ident, "m", [1, 4]],
+      #             [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil, nil, nil]],
+      #             [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]]
       #
-      def Ripper.sexp(src, filename = "-", lineno = 1, raise_errors: false)
+      def self.sexp(src, filename = "-", lineno = 1, raise_errors: false)
         builder = SexpBuilderPP.new(src, filename, lineno)
         sexp = builder.parse
         if builder.error?
@@ -397,23 +397,23 @@ module Prism
       # returning +nil+ in such cases. Use the +raise_errors+ keyword
       # to raise a SyntaxError for an error in +src+.
       #
-      #   require 'ripper'
-      #   require 'pp'
+      #     require "ripper"
+      #     require "pp"
       #
-      #   pp Ripper.sexp_raw("def m(a) nil end")
-      #     #=> [:program,
-      #          [:stmts_add,
-      #           [:stmts_new],
-      #           [:def,
-      #            [:@ident, "m", [1, 4]],
-      #            [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
-      #            [:bodystmt,
-      #             [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
-      #             nil,
-      #             nil,
-      #             nil]]]]
+      #     pp Ripper.sexp_raw("def m(a) nil end")
+      #       #=> [:program,
+      #            [:stmts_add,
+      #             [:stmts_new],
+      #             [:def,
+      #              [:@ident, "m", [1, 4]],
+      #              [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
+      #              [:bodystmt,
+      #               [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
+      #               nil,
+      #               nil,
+      #               nil]]]]
       #
-      def Ripper.sexp_raw(src, filename = "-", lineno = 1, raise_errors: false)
+      def self.sexp_raw(src, filename = "-", lineno = 1, raise_errors: false)
         builder = SexpBuilder.new(src, filename, lineno)
         sexp = builder.parse
         if builder.error?