parser 1.4.2 → 2.0.0.beta1

data/lib/parser/ruby20.y

@@ -1149,7 +1149,6 @@ rule
                         diagnostic(:error, :class_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1160,7 +1159,6 @@ rule
                                                   val[4], val[5])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kCLASS tLSHFT expr term
                     {
@@ -1175,7 +1173,6 @@ rule
                                                    val[5], val[6])
 
                       @static_env.unextend
-                      @lexer.clear_comments
 
                       @def_level = val[4]
                     }
@@ -1185,7 +1182,6 @@ rule
                         diagnostic(:error, :module_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1194,26 +1190,22 @@ rule
                                                    val[3], val[4])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kDEF fname
                     {
-                      @comments.push @lexer.clear_comments
                       @def_level += 1
                       @static_env.extend_static
                     }
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_method(val[0], val[1],
-                                  val[3], val[4], val[5], @comments.pop)
+                                  val[3], val[4], val[5])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kDEF singleton dot_or_colon
                     {
-                      @comments.push @lexer.clear_comments
                       @lexer.state = :expr_fname
                     }
                     fname
@@ -1224,11 +1216,10 @@ rule
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_singleton(val[0], val[1], val[2],
-                                  val[4], val[6], val[7], val[8], @comments.pop)
+                                  val[4], val[6], val[7], val[8])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kBREAK
                     {
@@ -2284,9 +2275,7 @@ keyword_variable: kNIL
                     }
                 | tLABEL arg_value
                     {
-                      # TODO: Extract colon
-                      key = @builder.symbol(val[0])
-                      result = @builder.pair(key, nil, val[1])
+                      result = @builder.pair_keyword(val[0], val[1])
                     }
                 | tDSTAR arg_value
                     {

data/lib/parser/ruby21.y

@@ -1150,7 +1150,6 @@ rule
                         diagnostic(:error, :class_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1161,7 +1160,6 @@ rule
                                                   val[4], val[5])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kCLASS tLSHFT expr term
                     {
@@ -1176,7 +1174,6 @@ rule
                                                    val[5], val[6])
 
                       @static_env.unextend
-                      @lexer.clear_comments
 
                       @def_level = val[4]
                     }
@@ -1186,7 +1183,6 @@ rule
                         diagnostic(:error, :module_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1195,26 +1191,22 @@ rule
                                                    val[3], val[4])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kDEF fname
                     {
-                      @comments.push @lexer.clear_comments
                       @def_level += 1
                       @static_env.extend_static
                     }
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_method(val[0], val[1],
-                                  val[3], val[4], val[5], @comments.pop)
+                                  val[3], val[4], val[5])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kDEF singleton dot_or_colon
                     {
-                      @comments.push @lexer.clear_comments
                       @lexer.state = :expr_fname
                     }
                     fname
@@ -1225,11 +1217,10 @@ rule
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_singleton(val[0], val[1], val[2],
-                                  val[4], val[6], val[7], val[8], @comments.pop)
+                                  val[4], val[6], val[7], val[8])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kBREAK
                     {
@@ -2301,9 +2292,7 @@ keyword_variable: kNIL
                     }
                 | tLABEL arg_value
                     {
-                      # TODO: Extract colon
-                      key = @builder.symbol(val[0])
-                      result = @builder.pair(key, nil, val[1])
+                      result = @builder.pair_keyword(val[0], val[1])
                     }
                 | tDSTAR arg_value
                     {

data/lib/parser/runner/ruby_parse.rb

@@ -107,7 +107,7 @@ module Parser
       @slop.on 'E', 'explain', 'Explain how the source is tokenized' do
         ENV['RACC_DEBUG'] = '1'
 
-        Lexer.send :include, Lexer::Explanation
+        Lexer.send :prepend, Lexer::Explanation
       end
     end
 

data/lib/parser/source/buffer.rb

@@ -7,12 +7,11 @@ module Parser
       attr_reader :name, :first_line
 
       def self.recognize_encoding(string)
-        if string.empty?
-          return Encoding::BINARY
-        end
+        return Encoding::BINARY if string.empty?
 
-        # TODO: Make this more efficient.
-        first_line, second_line = string.lines.first(2)
+        # extract the first two lines in an efficient way
+        string =~ /(.*)\n?(.*\n)?/
+        first_line, second_line = $1, $2
 
         [first_line, second_line].each do |line|
           line.force_encoding(Encoding::ASCII_8BIT) if line
@@ -95,15 +94,7 @@ module Parser
 
       def source_line(line)
         unless @lines
-          @lines = @source.lines.map do |source_line|
-            # Line endings will be commonly present for all lines
-            # except the last one. It does not make sense to keep them.
-            if source_line.end_with? "\n"
-              source_line.chomp
-            else
-              source_line
-            end
-          end
+          @lines = @source.lines.map(&:chomp)
         end
 
         @lines[line - @first_line].dup
@@ -117,13 +108,11 @@ module Parser
 
           @source.each_char do |char|
             if char == "\n"
-              @line_begins << [ @line_begins.length, index ]
+              @line_begins.unshift [ @line_begins.length, index ]
             end
 
             index += 1
           end
-
-          @line_begins = @line_begins.reverse
         end
 
         @line_begins
@@ -131,11 +120,12 @@ module Parser
 
       def line_for(position)
         if line_begins.respond_to? :bsearch
+          # Fast O(log n) variant for Ruby >=2.0.
           line_begins.bsearch do |line, line_begin|
             line_begin <= position
           end
         else
-          # Slower variant for <= Ruby 2.0.
+          # Slower O(n) variant for Ruby <2.0.
           line_begins.find do |line, line_begin|
             line_begin <= position
           end

data/lib/parser/source/comment.rb

@@ -0,0 +1,46 @@
+module Parser
+  module Source
+
+    class Comment
+      attr_reader  :text
+
+      attr_reader  :location
+      alias_method :loc, :location
+
+      def self.associate(ast, comments)
+        associator = Associator.new(comments, ast)
+        associator.associate
+      end
+
+      def initialize(location)
+        @location = location
+        @text     = location.source.freeze
+
+        freeze
+      end
+
+      def type
+        case text
+        when /^#/
+          :inline
+        when /^=begin/
+          :document
+        end
+      end
+
+      def inline?
+        type == :inline
+      end
+
+      def document?
+        type == :document
+      end
+
+      def ==(other)
+        other.is_a?(Source::Comment) &&
+          @location == other.location
+      end
+    end
+
+  end
+end

data/lib/parser/source/comment/associator.rb

@@ -0,0 +1,70 @@
+module Parser
+  module Source
+
+    class Comment::Associator
+      def initialize(comments, ast)
+        @comments    = comments
+        @ast         = ast
+      end
+
+      def associate
+        @mapping     = Hash.new { |h, k| h[k] = [] }
+        @comment_num = 0
+
+        process(nil, @ast)
+
+        @mapping
+      end
+
+      private
+
+      def process(upper_node, node)
+        if node.type == :begin
+          prev_node, next_node = nil, upper_node
+        else
+          while current_comment_between?(prev_node, node)
+            associate_and_advance_comment(node)
+          end
+
+          prev_node, next_node = nil, upper_node
+        end
+
+        node.children.each do |child|
+          if child.is_a?(AST::Node) && child.location.expression
+            prev_node, next_node = next_node, child
+
+            process(prev_node, child)
+          end
+        end
+      end
+
+      def current_comment
+        @comments[@comment_num]
+      end
+
+      def advance_comment
+        @comment_num += 1
+      end
+
+      def current_comment_between?(prev_node, next_node)
+        comment_loc = current_comment.location
+        next_loc    = next_node.location.expression
+
+        if prev_node.nil?
+          comment_loc.end_pos <= next_loc.begin_pos
+        else
+          prev_loc  = prev_node.location.expression
+
+          comment_loc.begin_pos >= prev_loc.end_pos &&
+                comment_loc.end_pos <= next_loc.begin_pos
+        end
+      end
+
+      def associate_and_advance_comment(node)
+        @mapping[node] << current_comment
+        advance_comment
+      end
+    end
+
+  end
+end

data/lib/parser/source/map.rb

@@ -19,6 +19,10 @@ module Parser
         @expression.line
       end
 
+      def column
+        @expression.column
+      end
+
       def with_expression(expression_l)
         with { |map| map.update_expression(expression_l) }
       end

data/lib/parser/source/range.rb

@@ -46,12 +46,12 @@ module Parser
         @source_buffer.source_line(line)
       end
 
-      def to_source
+      def source
         @source_buffer.source[self.begin_pos...self.end_pos]
       end
 
       def is?(*what)
-        what.include?(to_source)
+        what.include?(source)
       end
 
       def to_a

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '1.4.2'
+  VERSION = '2.0.0.beta1'
 end

data/parser.gemspec

@@ -1,4 +1,5 @@
 # encoding: utf-8
+
 require File.expand_path('../lib/parser/version', __FILE__)
 
 Gem::Specification.new do |spec|
@@ -6,20 +7,20 @@ Gem::Specification.new do |spec|
   spec.version       = Parser::VERSION
   spec.authors       = ['Peter Zotov']
   spec.email         = ['whitequark@whitequark.org']
-  spec.description   = %q{A Ruby parser written in pure Ruby.}
+  spec.description   = 'A Ruby parser written in pure Ruby.'
   spec.summary       = spec.description
   spec.homepage      = 'http://github.com/whitequark/parser'
   spec.license       = 'MIT'
   spec.has_rdoc      = 'yard'
 
-  spec.files         = `git ls-files`.split($/) + %w(
+  spec.files         = `git ls-files`.split + %w(
                           lib/parser/lexer.rb
                           lib/parser/ruby18.rb
                           lib/parser/ruby19.rb
                           lib/parser/ruby20.rb
                           lib/parser/ruby21.rb
                        )
-  spec.executables   = %w(ruby-parse ruby-rewrite)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 

data/test/parse_helper.rb

@@ -34,7 +34,7 @@ module ParseHelper
     parser
   end
 
-  def with_versions(code, versions)
+  def with_versions(versions)
     (versions & ALL_VERSIONS).each do |version|
       @diagnostics.clear
 
@@ -67,7 +67,7 @@ module ParseHelper
   # )
   # ~~~
   def assert_parses(ast, code, source_maps='', versions=ALL_VERSIONS)
-    with_versions(code, versions) do |version, parser|
+    with_versions(versions) do |version, parser|
       source_file = Parser::Source::Buffer.new('(assert_parses)')
       source_file.source = code
 
@@ -92,10 +92,10 @@ module ParseHelper
           raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
         end
 
-        assert astlet.source_map.respond_to?(map_field),
-               "(#{version}) #{astlet.source_map.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
+        assert astlet.location.respond_to?(map_field),
+               "(#{version}) #{astlet.location.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
 
-        range = astlet.source_map.send(map_field)
+        range = astlet.location.send(map_field)
 
         assert_source_range(begin_pos, end_pos, range, version, line.inspect)
       end
@@ -111,7 +111,7 @@ module ParseHelper
   #     |     ~~~ highlights (0)})
   # ~~~
   def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
-    with_versions(code, versions) do |version, parser|
+    with_versions(versions) do |version, parser|
       source_file = Parser::Source::Buffer.new('(assert_diagnoses)')
       source_file.source = code