parser 2.2.0.3 → 2.2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32ce09c1a28d2549bf2f979555d9d961091df27b
-  data.tar.gz: e936c4eb15b204dd1aa89bb107e615f2cf9549dd
+  metadata.gz: 8228d5693ac3104ea947dac416025a20f29c32e1
+  data.tar.gz: b078eb868955e08112f252dde8a4957bb91c5e9b
 SHA512:
-  metadata.gz: 7abb6830a6804ed9b0debdb1ad1c985a33b947d1db7a66dfb36f7c0b3f1acafd74822984985756a67658f1b8a15da97fc7b62a9db2df76230ec4562c8d28bf84
-  data.tar.gz: ab9d08b75232d9fdd210e50265db49fa7703b5a05d03fdd5d50756f00b40a359f6da0660dc9c7436721fcf0bf958bcc900763b577e077f74bc0cc0e5742dd5b6
+  metadata.gz: 41b75cd456f815491a2bce7b24e8fc7cd584982895c39ea8167dd46e4ceeead8e287851056099d3405c4eccab687b77934fbb2e5c7fe079f555f22244a5a306f
+  data.tar.gz: 3c7bbc92c7152e2bc3740a9db8524c7f3712330edc493e0d5cefdba614855cefa7a5172406d0e6192d7561f4d6b1c885fa00cb3db6e7f31612e5a967515dbdc0

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 Changelog
 =========
 
+v2.2.0.4 (2015-04-14)
+---------------------
+
+Features implemented:
+ * Add Parser::Source::Map#node. (whitequark)
+ * Add Parser::Source::Comment.associate_locations. (kubicle)
+
 v2.2.0.1 (2014-12-27)
 ---------------------
 

data/lib/parser/ast/node.rb

@@ -27,6 +27,8 @@ module Parser
       #
       def assign_properties(properties)
         if (location = properties[:location])
+          location = location.dup if location.frozen?
+          location.node = self
           @location = location
         end
       end

data/lib/parser/builders/default.rb

@@ -172,6 +172,13 @@ module Parser
     end
 
     def regexp_compose(begin_t, parts, end_t, options)
+      begin
+        static_regexp(parts, options)
+      rescue RegexpError => e
+        diagnostic :error, :invalid_regexp, { :message => e.message },
+                   loc(begin_t).join(loc(end_t))
+      end
+
       n(:regexp, (parts << options),
         regexp_map(begin_t, end_t, options))
     end
@@ -325,7 +332,7 @@ module Parser
       when :__FILE__
         if @emit_file_line_as_literals
           n(:str, [ node.loc.expression.source_buffer.name ],
-            node.loc)
+            node.loc.dup)
         else
           node
         end
@@ -333,14 +340,14 @@ module Parser
       when :__LINE__
         if @emit_file_line_as_literals
           n(:int, [ node.loc.expression.line ],
-            node.loc)
+            node.loc.dup)
         else
           node
         end
 
       when :__ENCODING__
         n(:const, [ n(:const, [ nil, :Encoding], nil), :UTF_8 ],
-          node.loc)
+          node.loc.dup)
 
       when :ident
         name, = *node
@@ -587,7 +594,7 @@ module Parser
         expr.updated(:arg)
       else
         n(:arg_expr, [ expr ],
-          expr.loc)
+          expr.loc.dup)
       end
     end
 
@@ -598,7 +605,7 @@ module Parser
         expr.updated(:restarg)
       else
         n(:restarg_expr, [ expr ],
-          expr.loc)
+          expr.loc.dup)
       end
     end
 
@@ -607,7 +614,7 @@ module Parser
         expr.updated(:blockarg)
       else
         n(:blockarg_expr, [ expr ],
-          expr.loc)
+          expr.loc.dup)
       end
     end
 
@@ -708,30 +715,8 @@ module Parser
     def match_op(receiver, match_t, arg)
       source_map = send_binary_op_map(receiver, match_t, arg)
 
-      if receiver.type == :regexp &&
-            receiver.children.count == 2 &&
-            receiver.children.first.type == :str
-
-        str_node, opt_node = *receiver
-        regexp_body, = *str_node
-        *regexp_opt  = *opt_node
-
-        if defined?(Encoding)
-          regexp_body = case
-          when regexp_opt.include?(:u)
-            regexp_body.encode(Encoding::UTF_8)
-          when regexp_opt.include?(:e)
-            regexp_body.encode(Encoding::EUC_JP)
-          when regexp_opt.include?(:s)
-            regexp_body.encode(Encoding::WINDOWS_31J)
-          when regexp_opt.include?(:n)
-            regexp_body.encode(Encoding::BINARY)
-          else
-            regexp_body
-          end
-        end
-
-        Regexp.new(regexp_body).names.each do |name|
+      if (regexp = static_regexp_node(receiver))
+        regexp.names.each do |name|
           @parser.static_env.declare(name)
         end
 
@@ -1382,6 +1367,55 @@ module Parser
     # HELPERS
     #
 
+    # Extract a static string from e.g. a regular expression,
+    # honoring the fact that MRI expands interpolations like #{""}
+    # at parse time.
+    def static_string(nodes)
+      nodes.map do |node|
+        case node.type
+        when :str
+          node.children[0]
+        when :begin
+          if (string = static_string(node.children))
+            string
+          else
+            return nil
+          end
+        else
+          return nil
+        end
+      end.join
+    end
+
+    def static_regexp(parts, options)
+      source = static_string(parts)
+      return nil if source.nil?
+
+      if defined?(Encoding)
+        source = case
+        when options.children.include?(:u)
+          source.encode(Encoding::UTF_8)
+        when options.children.include?(:e)
+          source.encode(Encoding::EUC_JP)
+        when options.children.include?(:s)
+          source.encode(Encoding::WINDOWS_31J)
+        when options.children.include?(:n)
+          source.encode(Encoding::BINARY)
+        else
+          source
+        end
+      end
+
+      Regexp.new(source)
+    end
+
+    def static_regexp_node(node)
+      if node.type == :regexp
+        parts, options = node.children[0..-2], node.children[-1]
+        static_regexp(parts, options)
+      end
+    end
+
     def collapse_string_parts?(parts)
       parts.one? &&
           [:str, :dstr].include?(parts.first.type)

data/lib/parser/current.rb

@@ -29,8 +29,8 @@ module Parser
     CurrentRuby = Ruby20
 
   when /^2\.1\./
-    if RUBY_VERSION != '2.1.5'
-      warn_syntax_deviation 'parser/ruby21', '2.1.5'
+    if RUBY_VERSION != '2.1.6'
+      warn_syntax_deviation 'parser/ruby21', '2.1.6'
     end
 
     require 'parser/ruby21'

data/lib/parser/messages.rb

@@ -54,6 +54,7 @@ module Parser
     :block_and_blockarg      => 'both block argument and literal block are passed',
     :masgn_as_condition      => 'multiple assignment in conditional context',
     :block_given_to_yield    => 'block given to yield',
+    :invalid_regexp          => '%{message}',
 
     # Parser warnings
     :useless_else            => 'else without rescue is useless',

data/lib/parser/source/comment.rb

@@ -19,19 +19,33 @@ module Parser
       alias_method :loc, :location
 
       ##
-      # Associate `comments` with `ast` nodes by their location in the
-      # source.
+      # Associate `comments` with `ast` nodes by their corresponding node.
       #
       # @param [Parser::AST::Node] ast
       # @param [Array(Comment)]    comments
       # @return [Hash(Parser::AST::Node, Array(Comment))]
-      # @see Parser::Source::Comment::Associator
+      # @see Parser::Source::Comment::Associator#associate
+      # @deprecated Use {associate_locations}.
       #
       def self.associate(ast, comments)
         associator = Associator.new(ast, comments)
         associator.associate
       end
 
+      ##
+      # Associate `comments` with `ast` nodes by their location in the
+      # source.
+      #
+      # @param [Parser::AST::Node] ast
+      # @param [Array(Comment)]    comments
+      # @return [Hash(Parser::Source::Map, Array(Comment))]
+      # @see Parser::Source::Comment::Associator#associate_locations
+      #
+      def self.associate_locations(ast, comments)
+        associator = Associator.new(ast, comments)
+        associator.associate_locations
+      end
+
       ##
       # @param [Parser::Source::Range] range
       #

data/lib/parser/source/comment/associator.rb

@@ -27,7 +27,7 @@ module Parser
     #   #      #<Parser::Source::Comment (string):4:3 "# @see bar">]
     #   # }
     #
-    # @see #associate
+    # @see {associate}
     #
     # @!attribute skip_directives
     #  Skip file processing directives disguised as comments.
@@ -76,78 +76,125 @@ module Parser
       #         [#<Parser::Source::Comment (string):3:8 "# bar">]
       #     }
       #
+      # Note that {associate} produces unexpected result for nodes which are
+      # equal but have distinct locations; comments for these nodes are merged.
+      #
       # @return [Hash(Parser::AST::Node, Array(Parser::Source::Comment))]
+      # @deprecated Use {associate_locations}.
       #
       def associate
+        @map_using_locations = false
+        do_associate
+      end
+
+      ##
+      # Same as {associate}, but uses `node.loc` instead of `node` as
+      # the hash key, thus producing an unambiguous result even in presence
+      # of equal nodes.
+      #
+      # @return [Hash(Parser::Source::Map, Array(Parser::Source::Comment))]
+      #
+      def associate_locations
+        @map_using_locations = true
+        do_associate
+      end
+
+      private
+
+      def do_associate
         @mapping     = Hash.new { |h, k| h[k] = [] }
-        @comment_num = 0
+        @comment_num = -1
+        advance_comment
 
         advance_through_directives if @skip_directives
 
-        process(nil, @ast)
+        @prev_node = nil
+        visit(@ast)
 
         @mapping
       end
 
-      private
+      def visit(node)
+        process_node(node)
 
-      def process(upper_node, node)
-        if node.type == :begin
-          prev_node, next_node = nil, upper_node
-        else
-          while current_comment_between?(prev_node, node)
-            associate_and_advance_comment(node)
+        if node.children.length > 0
+          node.children.each do |child|
+            next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
+            visit(child)
           end
-
-          prev_node, next_node = nil, upper_node
+          process_trailing_comments(node)
+          @prev_node = node
         end
+      end
 
-        node.children.each do |child|
-          if child.is_a?(AST::Node) && child.loc && child.loc.expression
-            prev_node, next_node = next_node, child
-
-            process(prev_node, child)
-          end
+      def process_node(node)
+        return unless node.type != :begin
+        while current_comment_between?(@prev_node, node)
+          associate_and_advance_comment(@prev_node, node)
         end
+        @prev_node = node
       end
 
-      def current_comment
-        @comments[@comment_num]
+      def process_trailing_comments(parent)
+        while current_comment_decorates?(@prev_node)
+          associate_and_advance_comment(@prev_node, nil)
+        end
+        while current_comment_before_end?(parent)
+          associate_and_advance_comment(@prev_node, nil)
+        end
       end
 
       def advance_comment
         @comment_num += 1
+        @current_comment = @comments[@comment_num]
       end
 
       def current_comment_between?(prev_node, next_node)
-        return false if current_comment.nil?
+        return false if !@current_comment
+        comment_loc = @current_comment.location.expression
 
-        comment_loc = current_comment.location.expression
-        next_loc    = next_node.location.expression
+        if next_node
+          next_loc = next_node.location.expression
+          return false if comment_loc.end_pos > next_loc.begin_pos
+        end
+        if prev_node
+          prev_loc = prev_node.location.expression
+          return false if comment_loc.begin_pos < prev_loc.begin_pos
+        end
+        true
+      end
 
-        if prev_node.nil?
-          comment_loc.end_pos <= next_loc.begin_pos
-        else
-          prev_loc  = prev_node.location.expression
+      def current_comment_decorates?(prev_node)
+        return false if !@current_comment
+        @current_comment.location.line == prev_node.location.line
+      end
 
-          comment_loc.begin_pos >= prev_loc.end_pos &&
-                comment_loc.end_pos <= next_loc.begin_pos
-        end
+      def current_comment_before_end?(parent)
+        return false if !@current_comment
+        comment_loc = @current_comment.location.expression
+        parent_loc = parent.location.expression
+        comment_loc.end_pos <= parent_loc.end_pos
       end
 
-      def associate_and_advance_comment(node)
-        @mapping[node] << current_comment
+      def associate_and_advance_comment(prev_node, node)
+        if prev_node && node
+          owner_node = (@current_comment.location.line == prev_node.location.line) ? prev_node : node
+        else
+          owner_node = prev_node ? prev_node : node
+        end
+        key = @map_using_locations ? owner_node.location : owner_node
+        @mapping[key] << @current_comment
         advance_comment
       end
 
       def advance_through_directives
         # Skip shebang.
-        if current_comment && current_comment.text =~ /^#!/
+        if @current_comment && @current_comment.text =~ /^#!/
           advance_comment
         end
 
         # Skip encoding line.
-        if current_comment && current_comment.text =~ Buffer::ENCODING_RE
+        if @current_comment && @current_comment.text =~ Buffer::ENCODING_RE
           advance_comment
         end
       end

data/lib/parser/source/map.rb

@@ -56,20 +56,38 @@ module Parser
     #  #  @begin=#<Source::Range (string) 0...1>,
     #  #  @expression=#<Source::Range (string) 0...6>>
     #
+    # @!attribute [r] node
+    #  The node that is described by this map. Nodes and maps have 1:1 correspondence.
+    #  @return [Parser::AST::Node]
+    #
     # @!attribute [r] expression
     #  @return [Range]
     #
     # @api public
     #
     class Map
+      attr_reader :node
       attr_reader :expression
 
       ##
       # @param [Range] expression
       def initialize(expression)
         @expression = expression
+      end
+
+      ##
+      # @api private
+      def initialize_copy(other)
+        super
+        @node = nil
+      end
 
+      ##
+      # @api private
+      def node=(node)
+        @node = node
         freeze
+        @node
       end
 
       ##
@@ -134,7 +152,7 @@ module Parser
       protected
 
       def with(&block)
-        dup.tap(&block).freeze
+        dup.tap(&block)
       end
 
       def update_expression(expression_l)

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.2.0.3'
+  VERSION = '2.2.0.4'
 end

data/test/parse_helper.rb

@@ -95,6 +95,8 @@ module ParseHelper
           raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
         end
 
+        assert astlet.frozen?
+
         assert astlet.location.respond_to?(map_field),
                "(#{version}) #{astlet.location.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
 

data/test/test_base.rb

@@ -15,4 +15,15 @@ class TestBase < Minitest::Test
     assert_equal 1, comments.size
     assert_equal '# foo', comments.first.text
   end
+
+  def test_loc_to_node
+    ast = Parser::CurrentRuby.parse('1')
+    assert_equal ast.loc.node, ast
+  end
+
+  def test_loc_dup
+    ast = Parser::CurrentRuby.parse('1')
+    assert_equal nil, ast.loc.dup.node
+    Parser::AST::Node.new(:root, [], :location => ast.loc)
+  end
 end

data/test/test_parser.rb

@@ -360,6 +360,21 @@ class TestParser < Minitest::Test
         |~~~~~~~~~~~~~~ expression})
   end
 
+  def test_regex_error
+    # The tests work on 1.8, but with a different message.
+    assert_diagnoses(
+      [:error, :invalid_regexp, {:message => 'target of repeat operator is not specified: /?/'}],
+      %q[/?/],
+      %q(~~~ location),
+      ALL_VERSIONS - %w(1.8))
+
+    assert_diagnoses(
+      [:error, :invalid_regexp, {:message => 'target of repeat operator is not specified: /?/'}],
+      %q[/#{""}?/],
+      %q(~~~~~~~~ location),
+      ALL_VERSIONS - %w(1.8))
+  end
+
   # Arrays
 
   def test_array_plain
@@ -3152,11 +3167,12 @@ class TestParser < Minitest::Test
     assert_parses(
       s(:send,
         s(:regexp,
-          s(:begin, s(:str, '(?<match>bar)')),
+          s(:begin, s(:int, 1)),
+          s(:str, '(?<match>bar)'),
           s(:regopt)),
         :=~,
         s(:str, 'bar')),
-      %q{/#{'(?<match>bar)'}/ =~ 'bar'})
+      %q{/#{1}(?<match>bar)/ =~ 'bar'})
   end
 
   # To superclass

data/test/test_source_comment_associator.rb

@@ -2,18 +2,29 @@ require 'helper'
 require 'parser/ruby18'
 
 class TestSourceCommentAssociator < Minitest::Test
-  def associate(code)
+  def parse_with_comments(code)
     parser = Parser::Ruby18.new
 
     buffer = Parser::Source::Buffer.new('(comments)')
     buffer.source = code
 
-    ast, comments = parser.parse_with_comments(buffer)
+    parser.parse_with_comments(buffer)
+  end
+
+  def associate(code)
+    ast, comments = parse_with_comments(code)
     associations  = Parser::Source::Comment.associate(ast, comments)
 
     [ ast, associations ]
   end
 
+  def associate_locations(code)
+    ast, comments = parse_with_comments(code)
+    associations  = Parser::Source::Comment.associate_locations(ast, comments)
+
+    [ ast, associations ]
+  end
+
   def test_associate
     ast, associations = associate(<<-END)
 #!/usr/bin/env ruby
@@ -36,9 +47,9 @@ end
 
     klass_node         = ast
     attr_accessor_node = ast.children[2].children[0]
-    method_node        = ast.children[2].children[1]
-    expr_node          = method_node.children[2]
-    intermediate_node  = expr_node.children[2]
+    method_node        = ast.children[2].children[1] # def bar
+    expr_node          = method_node.children[2] # 1 + 2
+    intermediate_node  = expr_node.children[0] # 1
 
     assert_equal 5, associations.size
     assert_equal ['# Class comment', '# another class comment'],
@@ -47,12 +58,100 @@ end
                  associations[attr_accessor_node].map(&:text)
     assert_equal ['# method comment'],
                  associations[method_node].map(&:text)
-    assert_equal ['# expr comment'],
+    assert_equal ['# expr comment', "# stray comment"],
                  associations[expr_node].map(&:text)
     assert_equal ['# intermediate comment'],
                  associations[intermediate_node].map(&:text)
   end
 
+  # The bug below is fixed by using associate_locations
+  def test_associate_dupe_statement
+    ast, associations = associate(<<-END)
+class Foo
+  def bar
+    f1 # comment on 1st call to f1
+    f2
+    f1 # comment on 2nd call to f1
+  end
+end
+    END
+
+    klass_node         = ast
+    method_node        = ast.children[2]
+    body               = method_node.children[2]
+    f1_1_node          = body.children[0]
+    f1_2_node          = body.children[2]
+
+    assert_equal 1, associations.size
+    assert_equal ['# comment on 1st call to f1', '# comment on 2nd call to f1'],
+                 associations[f1_1_node].map(&:text)
+    assert_equal ['# comment on 1st call to f1', '# comment on 2nd call to f1'],
+                 associations[f1_2_node].map(&:text)
+  end
+
+  def test_associate_locations
+    ast, associations = associate_locations(<<-END)
+#!/usr/bin/env ruby
+# coding: utf-8
+# Class comment
+# another class comment
+class Foo
+  # attr_accessor comment
+  attr_accessor :foo
+
+  # method comment
+  def bar
+    # expr comment
+    1 + # intermediate comment
+      2
+    # stray comment
+  end
+end
+    END
+
+    klass_node         = ast
+    attr_accessor_node = ast.children[2].children[0]
+    method_node        = ast.children[2].children[1]
+    expr_node          = method_node.children[2]
+    intermediate_node  = expr_node.children[0]
+
+    assert_equal 5, associations.size
+    assert_equal ['# Class comment', '# another class comment'],
+                 associations[klass_node.loc].map(&:text)
+    assert_equal ['# attr_accessor comment'],
+                 associations[attr_accessor_node.loc].map(&:text)
+    assert_equal ['# method comment'],
+                 associations[method_node.loc].map(&:text)
+    assert_equal ['# expr comment', '# stray comment'],
+                 associations[expr_node.loc].map(&:text)
+    assert_equal ['# intermediate comment'],
+                 associations[intermediate_node.loc].map(&:text)
+  end
+
+  def test_associate_locations_dupe_statement
+    ast, associations = associate_locations(<<-END)
+class Foo
+  def bar
+    f1 # comment on 1st call to f1
+    f2
+    f1 # comment on 2nd call to f1
+  end
+end
+    END
+
+    klass_node         = ast
+    method_node        = ast.children[2]
+    body               = method_node.children[2]
+    f1_1_node          = body.children[0]
+    f1_2_node          = body.children[2]
+
+    assert_equal 2, associations.size
+    assert_equal ['# comment on 1st call to f1'],
+                 associations[f1_1_node.loc].map(&:text)
+    assert_equal ['# comment on 2nd call to f1'],
+                 associations[f1_2_node.loc].map(&:text)
+  end
+
   def test_associate_no_body
     ast, associations = associate(<<-END)
 # foo
@@ -91,7 +190,9 @@ def foo
 end
     END
 
-    assert_equal 0, associations.size
+    assert_equal 1, associations.size
+    assert_equal ['# foo'],
+                 associations[ast].map(&:text)
   end
 
   def test_associate___ENCODING__

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.2.0.3
+  version: 2.2.0.4
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-13 00:00:00.000000000 Z
+date: 2015-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast