parser 2.3.0.2 → 2.3.0.3

data/lib/parser/lexer/literal.rb

@@ -78,7 +78,7 @@ module Parser
                       !heredoc?)
 
       # Capture opening delimiter in percent-literals.
-      @str_type << delimiter if @str_type.start_with?('%')
+      @str_type << delimiter if @str_type.start_with?('%'.freeze)
 
       clear_buffer
 
@@ -103,7 +103,7 @@ module Parser
     end
 
     def backslash_delimited?
-      @end_delim == '\\'
+      @end_delim == '\\'.freeze
     end
 
     def type
@@ -116,7 +116,7 @@ module Parser
       if words? && character =~ /[ \t\v\r\f\n]/
         true
       else
-        ['\\', @start_delim, @end_delim].include?(character)
+        ['\\'.freeze, @start_delim, @end_delim].include?(character)
       end
     end
 
@@ -135,8 +135,8 @@ module Parser
           extend_space(ts, ts)
         end
 
-        if lookahead && lookahead[0] == ?: && lookahead[1] != ?: &&
-           @label_allowed && @start_tok == :tSTRING_BEG
+        if lookahead && @label_allowed && lookahead[0] == ?: &&
+           lookahead[1] != ?: && @start_tok == :tSTRING_BEG
           # This is a quoted label.
           flush_string
           emit(:tLABEL_END, @end_delim, ts, te + 1)

data/lib/parser/source/buffer.rb

@@ -40,8 +40,6 @@ module Parser
           )
         /x
 
-      NEW_LINE = "\n".freeze
-
       ##
       # Try to recognize encoding of `string` as Ruby would, i.e. by looking for
       # magic encoding comment or UTF-8 BOM. `string` can be in any encoding.
@@ -58,7 +56,7 @@ module Parser
 
         if first_line =~ /\A\xef\xbb\xbf/ # BOM
           return Encoding::UTF_8
-        elsif first_line[0, 2] == '#!'
+        elsif first_line[0, 2] == '#!'.freeze
           encoding_line = second_line
         else
           encoding_line = first_line
@@ -108,6 +106,10 @@ module Parser
 
         @lines       = nil
         @line_begins = nil
+
+        # Cache for fast lookup
+        @line_for_position = {}
+        @col_for_position  = {}
       end
 
       ##
@@ -175,7 +177,7 @@ module Parser
           raise ArgumentError, 'Source::Buffer is immutable'
         end
 
-        @source = input.gsub("\r\n", NEW_LINE).freeze
+        @source = input.gsub("\r\n".freeze, "\n".freeze).freeze
       end
 
       ##
@@ -190,6 +192,34 @@ module Parser
         [ @first_line + line_no, position - line_begin ]
       end
 
+      ##
+      # Convert a character index into the source to a line number.
+      #
+      # @param  [Integer] position
+      # @return [Integer] line
+      # @api private
+      #
+      def line_for_position(position)
+        @line_for_position[position] ||= begin
+          line_no, _ = line_for(position)
+          @first_line + line_no
+        end
+      end
+
+      ##
+      # Convert a character index into the source to a column number.
+      #
+      # @param  [Integer] position
+      # @return [Integer] column
+      # @api private
+      #
+      def column_for_position(position)
+        @col_for_position[position] ||= begin
+          _, line_begin = line_for(position)
+          position - line_begin
+        end
+      end
+
       ##
       # Return an `Array` of source code lines.
       #
@@ -198,10 +228,10 @@ module Parser
       def source_lines
         @lines ||= begin
           lines = @source.lines.to_a
-          lines << '' if @source.end_with?("\n")
+          lines << '' if @source.end_with?("\n".freeze)
 
           lines.each do |line|
-            line.chomp!(NEW_LINE)
+            line.chomp!("\n".freeze)
             line.freeze
           end
 
@@ -253,14 +283,11 @@ module Parser
 
       def line_begins
         unless @line_begins
-          @line_begins, index = [ [ 0, 0 ] ], 1
-
-          @source.each_char do |char|
-            if char == NEW_LINE
-              @line_begins.unshift [ @line_begins.length, index ]
-            end
+          @line_begins, index = [ [ 0, 0 ] ], 0
 
+          while index = @source.index("\n".freeze, index)
             index += 1
+            @line_begins.unshift [ @line_begins.length, index ]
           end
         end
 

data/lib/parser/source/comment/associator.rb

@@ -120,12 +120,22 @@ module Parser
       def visit(node)
         process_leading_comments(node)
 
-        node.children.each do |child|
-          next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
-          visit(child)
+        return unless @current_comment
+
+        # If the next comment is beyond the last line of this node, we don't
+        # need to iterate over its subnodes
+        # (Unless this node is a heredoc... there could be a comment in its body,
+        # inside an interpolation)
+        node_loc = node.location
+        if @current_comment.location.line <= node_loc.last_line ||
+           node_loc.is_a?(Map::Heredoc)
+          node.children.each do |child|
+            next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
+            visit(child)
+          end
+
+          process_trailing_comments(node)
         end
-
-        process_trailing_comments(node)
       end
 
       def process_leading_comments(node)
@@ -152,12 +162,8 @@ module Parser
       def current_comment_before?(node)
         return false if !@current_comment
         comment_loc = @current_comment.location.expression
-
-        if node
-          node_loc = node.location.expression
-          return false if comment_loc.end_pos > node_loc.begin_pos
-        end
-        true
+        node_loc = node.location.expression
+        comment_loc.end_pos <= node_loc.begin_pos
       end
 
       def current_comment_before_end?(node)

data/lib/parser/source/range.rb

@@ -34,6 +34,9 @@ module Parser
         if end_pos < begin_pos
           raise ArgumentError, 'Parser::Source::Range: end_pos must not be less than begin_pos'
         end
+        if source_buffer.nil?
+          raise ArgumentError, 'Parser::Source::Range: source_buffer must not be nil'
+        end
 
         @source_buffer       = source_buffer
         @begin_pos, @end_pos = begin_pos, end_pos
@@ -74,9 +77,7 @@ module Parser
       # @return [Integer] line number of the beginning of this range.
       #
       def line
-        line, _ = @source_buffer.decompose_position(@begin_pos)
-
-        line
+        @source_buffer.line_for_position(@begin_pos)
       end
 
       alias_method :first_line, :line
@@ -85,27 +86,21 @@ module Parser
       # @return [Integer] zero-based column number of the beginning of this range.
       #
       def column
-        _, column = @source_buffer.decompose_position(@begin_pos)
-
-        column
+        @source_buffer.column_for_position(@begin_pos)
       end
 
       ##
       # @return [Integer] line number of the end of this range.
       #
       def last_line
-        line, _ = @source_buffer.decompose_position(@end_pos)
-
-        line
+        @source_buffer.line_for_position(@end_pos)
       end
 
       ##
       # @return [Integer] zero-based column number of the end of this range.
       #
       def last_column
-        _, column = @source_buffer.decompose_position(@end_pos)
-
-        column
+        @source_buffer.column_for_position(@end_pos)
       end
 
       ##
@@ -209,6 +204,21 @@ module Parser
         @begin_pos >= other.end_pos || other.begin_pos >= @end_pos
       end
 
+      ##
+      # @param [Range] other
+      # @return [Boolean] `true` if this range and `other` overlap
+      #
+      def overlaps?(other)
+        @begin_pos < other.end_pos && other.begin_pos < @end_pos
+      end
+
+      ##
+      # Checks if a range is empty; if it contains no characters
+      # @return [Boolean]
+      def empty?
+        @begin_pos == @end_pos
+      end
+
       ##
       # Compares ranges.
       # @return [Boolean]

data/lib/parser/source/rewriter.rb

@@ -5,7 +5,8 @@ module Parser
     # {Rewriter} performs the heavy lifting in the source rewriting process.
     # It schedules code updates to be performed in the correct order and
     # verifies that no two updates _clobber_ each other, that is, attempt to
-    # modify the same part of code.
+    # modify the same section of code. (However, if two updates modify the
+    # same section in exactly the same way, they are merged.)
     #
     # If it is detected that one update clobbers another one, an `:error` and
     # a `:note` diagnostics describing both updates are generated and passed to
@@ -37,6 +38,7 @@ module Parser
         @source_buffer = source_buffer
         @queue         = []
         @clobber       = 0
+        @insertions    = 0 # clobbered zero-length positions; index 0 is the far left
       end
 
       ##
@@ -47,7 +49,7 @@ module Parser
       # @raise [ClobberingError] when clobbering is detected
       #
       def remove(range)
-        append Rewriter::Action.new(range, '')
+        append Rewriter::Action.new(range, ''.freeze)
       end
 
       ##
@@ -100,9 +102,7 @@ module Parser
         adjustment = 0
         source     = @source_buffer.source.dup
 
-        sorted_queue = @queue.sort_by.with_index do |action, index|
-          [action.range.begin_pos, index]
-        end
+        sorted_queue = @queue.sort_by { |action| action.range.begin_pos }
 
         sorted_queue.each do |action|
           begin_pos = action.range.begin_pos + adjustment
@@ -118,7 +118,7 @@ module Parser
 
       ##
       # Provides a protected block where a sequence of multiple rewrite actions
-      # are handled atomic. If any of the action failed by clobbering,
+      # are handled atomically. If any of the actions failed by clobbering,
       # all the actions are rolled back.
       #
       # @example
@@ -144,110 +144,249 @@ module Parser
 
         @pending_queue = @queue.dup
         @pending_clobber = @clobber
+        @pending_insertions = @insertions
 
         yield
 
         @queue = @pending_queue
         @clobber = @pending_clobber
+        @insertions = @pending_insertions
 
         self
       ensure
         @pending_queue = nil
         @pending_clobber = nil
+        @pending_insertions = nil
       end
 
       private
 
+      # Schedule a code update. If it overlaps with another update, check
+      # whether they conflict, and raise a clobbering error if they do.
+      # (As a special case, zero-length ranges at the same position are
+      # considered to "overlap".) Otherwise, merge them.
+      #
+      # Updates which are adjacent to each other, but do not overlap, are also
+      # merged.
+      #
+      # RULES:
+      #
+      # - Insertion ("replacing" a zero-length range):
+      #   - Two insertions at the same point conflict. This is true even
+      #     if the earlier insertion has already been merged with an adjacent
+      #     update, and even if they are both inserting the same text.
+      #   - An insertion never conflicts with a replace or remove operation
+      #     on its right or left side, which does not overlap it (in other
+      #     words, which does not update BOTH its right and left sides).
+      #   - An insertion always conflicts with a remove operation which spans
+      #     both its sides.
+      #   - An insertion conflicts with a replace operation which spans both its
+      #     sides, unless the replacement text is longer than the replaced text
+      #     by the size of the insertion (or more), and the portion of
+      #     replacement text immediately after the insertion position is
+      #     identical to the inserted text.
+      #
+      # - Removal operations never conflict with each other.
+      #
+      # - Replacement operations:
+      #   - Take the portion of each replacement text which falls within:
+      #     - The other operation's replaced region
+      #     - The other operation's replacement text, if it extends past the
+      #       end of its own replaced region (in other words, if the replacement
+      #       text is longer than the text it replaces)
+      #   - If and only if the taken texts are identical for both operations,
+      #     they do not conflict.
+      #
       def append(action)
-        if (clobber_actions = clobbered?(action.range))
-          handle_clobber(action, clobber_actions)
+        range = action.range
+
+        # Is this an insertion?
+        if range.empty?
+          # Replacing nothing with... nothing?
+          return self if action.replacement.empty?
+
+          if (conflicting = clobbered_insertion?(range))
+            raise_clobber_error(action, [conflicting])
+          end
+
+          record_insertion(range)
+
+          if (adjacent = adjacent_updates?(range))
+            conflicting = adjacent.find do |a|
+              a.range.overlaps?(range) &&
+                !replace_compatible_with_insertion?(a, action)
+            end
+            raise_clobber_error(action, [conflicting]) if conflicting
+
+            merge_actions!(action, adjacent)
+          else
+            active_queue << action
+          end
         else
-          clobber(action.range)
-          active_queue << action
+          # It's a replace or remove operation.
+          if (insertions = adjacent_insertions?(range))
+            insertions.each do |insertion|
+              if range.overlaps?(insertion.range) &&
+                 !replace_compatible_with_insertion?(action, insertion)
+                raise_clobber_error(action, [insertion])
+              else
+                action = merge_actions(action, [insertion])
+                active_queue.delete(insertion)
+              end
+            end
+          end
+
+          if (adjacent = adjacent_updates?(range))
+            if can_merge?(action, adjacent)
+              record_replace(range)
+              merge_actions!(action, adjacent)
+            else
+              raise_clobber_error(action, adjacent)
+            end
+          else
+            record_replace(range)
+            active_queue << action
+          end
         end
 
         self
       end
 
-      def clobber(range)
-        self.active_clobber = active_clobber | (2 ** range.size - 1) << range.begin_pos
+      def record_insertion(range)
+        self.active_insertions = active_insertions | (1 << range.begin_pos)
+      end
+
+      def record_replace(range)
+        self.active_clobber = active_clobber | clobbered_position_mask(range)
+      end
+
+      def clobbered_position_mask(range)
+        ((1 << range.size) - 1) << range.begin_pos
+      end
+
+      def adjacent_position_mask(range)
+        ((1 << (range.size + 2)) - 1) << (range.begin_pos - 1)
+      end
+
+      def adjacent_insertion_mask(range)
+        ((1 << (range.size + 1)) - 1) << range.begin_pos
       end
 
-      def clobbered?(range)
-        if active_clobber & ((2 ** range.size - 1) << range.begin_pos) != 0
-          active_queue.select do |action|
-            action.range.end_pos > range.begin_pos &&
-              range.end_pos > action.range.begin_pos
+      def clobbered_insertion?(insertion)
+        insertion_pos = insertion.begin_pos
+        if active_insertions & (1 << insertion_pos) != 0
+          # The clobbered insertion may have already been merged with other
+          # updates, so it won't necessarily have the same begin_pos.
+          active_queue.find do |a|
+            a.range.begin_pos <= insertion_pos && insertion_pos <= a.range.end_pos
           end
         end
       end
 
-      def handle_clobber(action, existing)
-        if can_merge?(action, existing)
-          merge_actions!(action, existing)
-        else
-          # cannot replace 3 characters with "foobar"
-          diagnostic = Diagnostic.new(:error,
-                                      :invalid_action,
-                                      { :action => action },
-                                      action.range)
-          @diagnostics.process(diagnostic)
-
-          # clobbered by: remove 3 characters
-          diagnostic = Diagnostic.new(:note,
-                                      :clobbered,
-                                      { :action => existing[0] },
-                                      existing[0].range)
-          @diagnostics.process(diagnostic)
-
-          raise ClobberingError, "Parser::Source::Rewriter detected clobbering"
+      def adjacent_insertions?(range)
+        # Just retrieve insertions which have not been merged with an adjacent
+        # remove or replace.
+        if active_insertions & adjacent_insertion_mask(range) != 0
+          result = active_queue.select do |a|
+            a.range.empty? && adjacent?(range, a.range)
+          end
+          result.empty? ? nil : result
+        end
+      end
+
+      def adjacent_updates?(range)
+        if active_clobber & adjacent_position_mask(range) != 0
+          active_queue.select { |a| adjacent?(range, a.range) }
         end
       end
 
+      def replace_compatible_with_insertion?(replace, insertion)
+        (replace.replacement.length - replace.range.size) >= insertion.range.size &&
+          (offset = insertion.range.begin_pos - replace.range.begin_pos) &&
+          replace.replacement[offset, insertion.replacement.length] == insertion.replacement
+      end
+
       def can_merge?(action, existing)
-        existing.all? do |other|
-          overlap       = action.range.intersect(other.range)
-          action_offset = overlap.begin_pos - action.range.begin_pos
-          other_offset  = overlap.begin_pos - other.range.begin_pos
+        # Compare 2 replace/remove operations (neither is an insertion)
+        range = action.range
 
-          replacement1 = action.replacement[action_offset, overlap.size] || ''
-          replacement2 = other.replacement[other_offset, overlap.size] || ''
+        existing.all? do |other|
+          overlap = range.intersect(other.range)
+          next true if overlap.nil? # adjacent, not overlapping
+
+          repl1_offset = overlap.begin_pos - range.begin_pos
+          repl2_offset = overlap.begin_pos - other.range.begin_pos
+          repl1_length = [other.range.length - repl2_offset,
+                          other.replacement.length  - repl2_offset].max
+          repl2_length = [range.length - repl1_offset,
+                          action.replacement.length - repl1_offset].max
+
+          replacement1 = action.replacement[repl1_offset, repl1_length] || ''.freeze
+          replacement2 = other.replacement[repl2_offset, repl2_length] || ''.freeze
           replacement1 == replacement2
         end
       end
 
-      def merge_actions!(action, existing)
-        actions      = existing.push(action).sort_by { |a| a.range.begin_pos }
-        merged_begin = actions.map { |act| act.range.begin_pos }.min
-        merged_end   = actions.map { |act| act.range.end_pos }.max
-        range        = Source::Range.new(@source_buffer,
-                                         merged_begin,
-                                         merged_end)
-        clobber(range)
-
-        replacement = merge_replacements(actions)
-        replace_actions(actions, Rewriter::Action.new(range, replacement))
+      def merge_actions(action, existing)
+        actions = existing.push(action).sort_by do |a|
+          [a.range.begin_pos, a.range.end_pos]
+        end
+        range = actions.first.range.join(actions.last.range)
+
+        Rewriter::Action.new(range, merge_replacements(actions))
       end
 
-      def replace_actions(old, updated)
-        old.each { |act| active_queue.delete(act) }
-        active_queue << updated
+      def merge_actions!(action, existing)
+        new_action = merge_actions(action, existing)
+        active_queue.delete(action)
+        replace_actions(existing, new_action)
       end
 
       def merge_replacements(actions)
         # `actions` must be sorted by beginning position
         begin_pos = actions.first.range.begin_pos
         result    = ''
+        prev_act  = nil
 
         actions.each do |act|
-          offset = result.size - act.range.begin_pos + begin_pos
-          next if offset < 0 || offset >= act.replacement.size
-          result << act.replacement[offset..-1]
+          if !prev_act || act.range.disjoint?(prev_act.range)
+            result << act.replacement
+          else
+            prev_end = [prev_act.range.begin_pos + prev_act.replacement.length,
+                        prev_act.range.end_pos].max
+            offset   = prev_end - act.range.begin_pos
+            result << act.replacement[offset..-1] if offset < act.replacement.size
+          end
+
+          prev_act = act
         end
 
         result
       end
 
+      def replace_actions(old, updated)
+        old.each { |act| active_queue.delete(act) }
+        active_queue << updated
+      end
+
+      def raise_clobber_error(action, existing)
+        # cannot replace 3 characters with "foobar"
+        diagnostic = Diagnostic.new(:error,
+                                    :invalid_action,
+                                    { :action => action },
+                                    action.range)
+        @diagnostics.process(diagnostic)
+
+        # clobbered by: remove 3 characters
+        diagnostic = Diagnostic.new(:note,
+                                    :clobbered,
+                                    { :action => existing[0] },
+                                    existing[0].range)
+        @diagnostics.process(diagnostic)
+
+        raise ClobberingError, "Parser::Source::Rewriter detected clobbering"
+      end
+
       def in_transaction?
         !@pending_queue.nil?
       end
@@ -260,6 +399,10 @@ module Parser
         @pending_clobber || @clobber
       end
 
+      def active_insertions
+        @pending_insertions || @insertions
+      end
+
       def active_clobber=(value)
         if @pending_clobber
           @pending_clobber = value
@@ -267,6 +410,18 @@ module Parser
           @clobber = value
         end
       end
+
+      def active_insertions=(value)
+        if @pending_insertions
+          @pending_insertions = value
+        else
+          @insertions = value
+        end
+      end
+
+      def adjacent?(range1, range2)
+        range1.begin_pos <= range2.end_pos && range2.begin_pos <= range1.end_pos
+      end
     end
 
   end