parser 2.5.1.0 → 3.0.1.1

data/lib/parser/runner.rb

@@ -14,9 +14,8 @@ module Parser
     end
 
     def initialize
-      Parser::Builders::Default.modernize
-
       @option_parser = OptionParser.new { |opts| setup_option_parsing(opts) }
+      @legacy = {}
       @parser_class  = nil
       @parser        = nil
       @files         = []
@@ -30,6 +29,7 @@ module Parser
 
     def execute(options)
       parse_options(options)
+      setup_builder_default
       prepare_parser
 
       process_all_input
@@ -37,6 +37,8 @@ module Parser
 
     private
 
+    LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0 forward_arg kwargs match_pattern].freeze
+
     def runner_name
       raise NotImplementedError, "implement #{self.class}##{__callee__}"
     end
@@ -106,6 +108,21 @@ module Parser
         @parser_class = Parser::Ruby26
       end
 
+      opts.on '--27', 'Parse as Ruby 2.7 would' do
+        require 'parser/ruby27'
+        @parser_class = Parser::Ruby27
+      end
+
+      opts.on '--30', 'Parse as Ruby 3.0 would' do
+        require 'parser/ruby30'
+        @parser_class = Parser::Ruby30
+      end
+
+      opts.on '--31', 'Parse as Ruby 3.1 would' do
+        require 'parser/ruby31'
+        @parser_class = Parser::Ruby31
+      end
+
       opts.on '--mac', 'Parse as MacRuby 0.12 would' do
         require 'parser/macruby'
         @parser_class = Parser::MacRuby
@@ -116,6 +133,17 @@ module Parser
         @parser_class = Parser::RubyMotion
       end
 
+      opts.on '--legacy', "Parse with all legacy modes" do
+        @legacy = Hash.new(true)
+      end
+
+      LEGACY_MODES.each do |mode|
+        opt_name = "--legacy-#{mode.to_s.gsub('_', '-')}"
+        opts.on opt_name, "Parse with legacy mode for emit_#{mode}" do
+          @legacy[mode] = true
+        end
+      end
+
       opts.on '-w', '--warnings', 'Enable warnings' do |w|
         @warnings = w
       end
@@ -154,6 +182,12 @@ module Parser
       end
     end
 
+    def setup_builder_default
+      LEGACY_MODES.each do |mode|
+        Parser::Builders::Default.send(:"emit_#{mode}=", !@legacy[mode])
+      end
+    end
+
     def prepare_parser
       @parser = @parser_class.new
 

data/lib/parser/runner/ruby_parse.rb

@@ -123,7 +123,7 @@ module Parser
       opts.on '--emit-ruby', 'Emit S-expressions as valid Ruby code' do
         @emit_ruby = true
       end
-      
+
       opts.on '--emit-json', 'Emit S-expressions as valid JSON' do
         @emit_json = true
       end
@@ -146,7 +146,7 @@ module Parser
         if @emit_ruby
           puts ast.inspect
         elsif @emit_json
-          puts JSON.generate(ast.to_sexp_array)
+          puts(ast ? JSON.generate(ast.to_sexp_array) : nil)
         else
           puts ast.to_s
         end

data/lib/parser/runner/ruby_rewrite.rb

@@ -55,8 +55,8 @@ module Parser
         new_source = rewriter.rewrite(buffer, ast)
 
         new_buffer = Source::Buffer.new(initial_buffer.name +
-                                    '|after ' + rewriter_class.name)
-        new_buffer.source = new_source
+                                    '|after ' + rewriter_class.name,
+                                    source: new_source)
 
         @parser.reset
         new_ast = @parser.parse(new_buffer)

data/lib/parser/source/buffer.rb

@@ -46,7 +46,7 @@ module Parser
       # magic encoding comment or UTF-8 BOM. `string` can be in any encoding.
       #
       # @param [String]  string
-      # @return [String|nil] encoding name, if recognized
+      # @return [String, nil] encoding name, if recognized
       #
       def self.recognize_encoding(string)
         return if string.empty?
@@ -102,7 +102,7 @@ module Parser
         end
       end
 
-      def initialize(name, first_line = 1)
+      def initialize(name, first_line = 1, source: nil)
         @name        = name.to_s
         @source      = nil
         @first_line  = first_line
@@ -114,8 +114,9 @@ module Parser
         @slice_source = nil
 
         # Cache for fast lookup
-        @line_for_position   = {}
-        @column_for_position = {}
+        @line_index_for_position = {}
+
+        self.source = source if source
       end
 
       ##
@@ -205,9 +206,10 @@ module Parser
       # @return [[Integer, Integer]] `[line, column]`
       #
       def decompose_position(position)
-        line_no, line_begin = line_for(position)
+        line_index = line_index_for_position(position)
+        line_begin = line_begins[line_index]
 
-        [ @first_line + line_no, position - line_begin ]
+        [ @first_line + line_index , position - line_begin ]
       end
 
       ##
@@ -218,10 +220,7 @@ module Parser
       # @api private
       #
       def line_for_position(position)
-        @line_for_position[position] ||= begin
-          line_no, _ = line_for(position)
-          @first_line + line_no
-        end
+        line_index_for_position(position) + @first_line
       end
 
       ##
@@ -232,10 +231,8 @@ module Parser
       # @api private
       #
       def column_for_position(position)
-        @column_for_position[position] ||= begin
-          _, line_begin = line_for(position)
-          position - line_begin
-        end
+        line_index = line_index_for_position(position)
+        position - line_begins[line_index]
       end
 
       ##
@@ -276,15 +273,13 @@ module Parser
       # @raise  [IndexError] if `lineno` is out of bounds
       #
       def line_range(lineno)
-        index = lineno - @first_line + 1
-        if index <= 0 || index > line_begins.size
+        index = lineno - @first_line
+        if index < 0 || index + 1 >= line_begins.size
           raise IndexError, 'Parser::Source::Buffer: range for line ' \
             "#{lineno} requested, valid line numbers are #{@first_line}.." \
-            "#{@first_line + line_begins.size - 1}"
-        elsif index == line_begins.size
-          Range.new(self, line_begins[-index][1], @source.size)
+            "#{@first_line + line_begins.size - 2}"
         else
-          Range.new(self, line_begins[-index][1], line_begins[-index - 1][1] - 1)
+          Range.new(self, line_begins[index], line_begins[index + 1] - 1)
         end
       end
 
@@ -301,27 +296,57 @@ module Parser
       # @return [Integer]
       #
       def last_line
-        line_begins.size + @first_line - 1
+        line_begins.size + @first_line - 2
+      end
+
+      # :nodoc:
+      def freeze
+        source_lines; line_begins; source_range # build cache
+        super
+      end
+
+      # :nodoc:
+      def inspect
+        "#<#{self.class} #{name}>"
       end
 
       private
 
+      # @returns [0, line_begin_of_line_1, ..., source.size + 1]
       def line_begins
-        unless @line_begins
-          @line_begins, index = [ [ 0, 0 ] ], 0
-
+        @line_begins ||= begin
+          begins = [0]
+          index = 0
           while index = @source.index("\n".freeze, index)
             index += 1
-            @line_begins.unshift [ @line_begins.length, index ]
+            begins << index
           end
+          begins << @source.size + 1
+          begins
         end
+      end
 
-        @line_begins
+      # @returns 0-based line index of position
+      def line_index_for_position(position)
+        @line_index_for_position[position] || begin
+          index = bsearch(line_begins, position) - 1
+          @line_index_for_position[position] = index unless @line_index_for_position.frozen?
+          index
+        end
       end
 
-      def line_for(position)
-        line_begins.bsearch do |line, line_begin|
-          line_begin <= position
+      if Array.method_defined?(:bsearch_index) # RUBY_VERSION >= 2.3
+        def bsearch(line_begins, position)
+          line_begins.bsearch_index do |line_begin|
+            position < line_begin
+          end || line_begins.size - 1 # || only for out of bound values
+        end
+      else
+        def bsearch(line_begins, position)
+          @line_range ||= 0...line_begins.size
+          @line_range.bsearch do |i|
+            position < line_begins[i]
+          end || line_begins.size - 1 # || only for out of bound values
         end
       end
     end

data/lib/parser/source/comment.rb

@@ -10,7 +10,7 @@ module Parser
     #  @return [String]
     #
     # @!attribute [r] location
-    #  @return [Parser::Source::Map]
+    #  @return [Parser::Source::Range]
     #
     # @api public
     #
@@ -24,8 +24,8 @@ module Parser
       # Associate `comments` with `ast` nodes by their corresponding node.
       #
       # @param [Parser::AST::Node] ast
-      # @param [Array(Comment)]    comments
-      # @return [Hash(Parser::AST::Node, Array(Comment))]
+      # @param [Array<Comment>]    comments
+      # @return [Hash<Parser::AST::Node, Array<Comment>>]
       # @see Parser::Source::Comment::Associator#associate
       # @deprecated Use {associate_locations}.
       #
@@ -39,8 +39,8 @@ module Parser
       # source.
       #
       # @param [Parser::AST::Node] ast
-      # @param [Array(Comment)]    comments
-      # @return [Hash(Parser::Source::Map, Array(Comment))]
+      # @param [Array<Comment>]    comments
+      # @return [Hash<Parser::Source::Map, Array<Comment>>]
       # @see Parser::Source::Comment::Associator#associate_locations
       #
       def self.associate_locations(ast, comments)
@@ -48,6 +48,19 @@ module Parser
         associator.associate_locations
       end
 
+      ##
+      # Associate `comments` with `ast` nodes using identity.
+      #
+      # @param [Parser::AST::Node] ast
+      # @param [Array<Comment>]    comments
+      # @return [Hash<Parser::Source::Node, Array<Comment>>]
+      # @see Parser::Source::Comment::Associator#associate_by_identity
+      #
+      def self.associate_by_identity(ast, comments)
+        associator = Associator.new(ast, comments)
+        associator.associate_by_identity
+      end
+
       ##
       # @param [Parser::Source::Range] range
       #

data/lib/parser/source/comment/associator.rb

@@ -47,7 +47,7 @@ module Parser
 
       ##
       # @param [Parser::AST::Node] ast
-      # @param [Array(Parser::Source::Comment)] comments
+      # @param [Array<Parser::Source::Comment>] comments
       def initialize(ast, comments)
         @ast         = ast
         @comments    = comments
@@ -84,12 +84,24 @@ module Parser
       #
       # Note that {associate} produces unexpected result for nodes which are
       # equal but have distinct locations; comments for these nodes are merged.
+      # You may prefer using {associate_by_identity} or {associate_locations}.
       #
-      # @return [Hash(Parser::AST::Node, Array(Parser::Source::Comment))]
+      # @return [Hash<Parser::AST::Node, Array<Parser::Source::Comment>>]
       # @deprecated Use {associate_locations}.
       #
       def associate
-        @map_using_locations = false
+        @map_using = :eql
+        do_associate
+      end
+
+      ##
+      # Same as {associate}, but compares by identity, thus producing an unambiguous
+      # result even in presence of equal nodes.
+      #
+      # @return [Hash<Parser::Source::Node, Array<Parser::Source::Comment>>]
+      #
+      def associate_locations
+        @map_using = :location
         do_associate
       end
 
@@ -98,17 +110,31 @@ module Parser
       # the hash key, thus producing an unambiguous result even in presence
       # of equal nodes.
       #
-      # @return [Hash(Parser::Source::Map, Array(Parser::Source::Comment))]
+      # @return [Hash<Parser::Source::Map, Array<Parser::Source::Comment>>]
       #
-      def associate_locations
-        @map_using_locations = true
+      def associate_by_identity
+        @map_using = :identity
         do_associate
       end
 
       private
 
+      POSTFIX_TYPES = Set[:if, :while, :while_post, :until, :until_post, :masgn].freeze
+      def children_in_source_order(node)
+        if POSTFIX_TYPES.include?(node.type)
+          # All these types have either nodes with expressions, or `nil`
+          # so a compact will do, but they need to be sorted.
+          node.children.compact.sort_by { |child| child.loc.expression.begin_pos }
+        else
+          node.children.select do |child|
+            child.is_a?(AST::Node) && child.loc && child.loc.expression
+          end
+        end
+      end
+
       def do_associate
         @mapping     = Hash.new { |h, k| h[k] = [] }
+        @mapping.compare_by_identity if @map_using == :identity
         @comment_num = -1
         advance_comment
 
@@ -131,10 +157,7 @@ module Parser
         node_loc = node.location
         if @current_comment.location.line <= node_loc.last_line ||
            node_loc.is_a?(Map::Heredoc)
-          node.children.each do |child|
-            next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
-            visit(child)
-          end
+          children_in_source_order(node).each { |child| visit(child) }
 
           process_trailing_comments(node)
         end
@@ -181,7 +204,7 @@ module Parser
       end
 
       def associate_and_advance_comment(node)
-        key = @map_using_locations ? node.location : node
+        key = @map_using == :location ? node.location : node
         @mapping[key] << @current_comment
         advance_comment
       end

data/lib/parser/source/map.rb

@@ -161,7 +161,7 @@ module Parser
       #  #   :expression => #<Source::Range (string) 0...6>
       #  # }
       #
-      # @return [Hash(Symbol, Parser::Source::Range)]
+      # @return [Hash<Symbol, Parser::Source::Range>]
       #
       def to_hash
         instance_variables.inject({}) do |hash, ivar|

data/lib/parser/source/map/method_definition.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Parser
+  module Source
+
+    class Map::MethodDefinition < Map
+      attr_reader :keyword
+      attr_reader :operator
+      attr_reader :name
+      attr_reader :end
+      attr_reader :assignment
+
+      def initialize(keyword_l, operator_l, name_l, end_l, assignment_l, body_l)
+        @keyword  = keyword_l
+        @operator = operator_l
+        @name     = name_l
+        @end      = end_l
+        @assignment = assignment_l
+
+        super(@keyword.join(end_l || body_l))
+      end
+    end
+
+  end
+end

data/lib/parser/source/range.rb

@@ -13,7 +13,7 @@ module Parser
     #        ^^
     #
     # @!attribute [r] source_buffer
-    #  @return [Parser::Diagnostic::Engine]
+    #  @return [Parser::Source::Buffer]
     #
     # @!attribute [r] begin_pos
     #  @return [Integer] index of the first character in the range
@@ -112,11 +112,11 @@ module Parser
       # @raise RangeError
       #
       def column_range
-        if self.begin.line != self.end.line
+        if line != last_line
           raise RangeError, "#{self.inspect} spans more than one line"
         end
 
-        self.begin.column...self.end.column
+        column...last_column
       end
 
       ##
@@ -143,12 +143,19 @@ module Parser
       end
 
       ##
-      # @return [Array(Integer)] a set of character indexes contained in this range.
+      # @return [Array<Integer>] a set of character indexes contained in this range.
       #
       def to_a
         (@begin_pos...@end_pos).to_a
       end
 
+      ##
+      # @return [Range] a Ruby range with the same `begin_pos` and `end_pos`
+      #
+      def to_range
+        self.begin_pos...self.end_pos
+      end
+
       ##
       # Composes a GNU/Clang-style string representation of the beginning of this
       # range.
@@ -298,6 +305,15 @@ module Parser
         (@end_pos <=> other.end_pos)
       end
 
+      alias_method :eql?, :==
+
+      ##
+      # Support for Ranges be used in as Hash indices and in Sets.
+      #
+      def hash
+        [@source_buffer, @begin_pos, @end_pos].hash
+      end
+
       ##
       # @return [String] a human-readable representation of this range.
       #