astel 0.2.0 → 0.3.0

data/lib/astel/rewriter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'rewriter/edit_index'
+
 module Astel
   class Rewriter
     class ConflictError < Error
@@ -29,85 +31,6 @@ module Astel
       end
     end
 
-    class EditIndex
-      include Enumerable
-
-      SPLIT_THRESHOLD = 256
-
-      def initialize
-        @blocks = []
-      end
-
-      def add(edit)
-        if @blocks.empty?
-          @blocks << [edit]
-          return
-        end
-
-        block_index, block, edit_index = insertion_position(edit)
-        conflict = conflict_at(block_index, block, edit_index) { |existing| yield existing }
-        return conflict if conflict
-
-        block.insert(edit_index, edit)
-        split(block_index, block) if block.length > SPLIT_THRESHOLD
-        nil
-      end
-
-      def find_conflict(edit)
-        return if @blocks.empty?
-
-        block_index, block, edit_index = insertion_position(edit)
-        conflict_at(block_index, block, edit_index) { |existing| yield existing }
-      end
-
-      def each
-        return enum_for(__method__) unless block_given?
-
-        @blocks.each do |block|
-          block.each { |edit| yield edit }
-        end
-      end
-
-      private
-
-      def insertion_position(edit)
-        block_index = block_index_for(edit)
-        block = @blocks.fetch(block_index)
-        edit_index = block.bsearch_index { |existing| ordered_after?(existing, edit) } || block.length
-        [block_index, block, edit_index]
-      end
-
-      def conflict_at(block_index, block, edit_index)
-        following = block[edit_index] || @blocks[block_index + 1]&.first
-        return following if following && yield(following)
-
-        preceding = if edit_index.positive?
-                      block[edit_index - 1]
-                    elsif block_index.positive?
-                      @blocks[block_index - 1].last
-                    end
-        return preceding if preceding && yield(preceding)
-
-        nil
-      end
-
-      def block_index_for(edit)
-        @blocks.bsearch_index { |block| ordered_after?(block.last, edit) } || @blocks.length - 1
-      end
-
-      def ordered_after?(existing, edit)
-        existing.start_offset > edit.start_offset ||
-          (existing.start_offset == edit.start_offset && existing.end_offset > edit.end_offset) ||
-          (existing.start_offset == edit.start_offset && existing.end_offset == edit.end_offset &&
-           existing.sequence > edit.sequence)
-      end
-
-      def split(block_index, block)
-        middle = block.length / 2
-        @blocks.insert(block_index + 1, block.slice!(middle, block.length - middle))
-      end
-    end
-
     attr_reader :source_file
 
     def initialize(source_file, duplicate_insertions: :accept)
@@ -158,27 +81,16 @@ module Astel
     end
 
     def merge!(other)
-      unless other.is_a?(Rewriter) && other.source_file.equal?(@source_file)
-        raise ArgumentError, 'cannot merge edits for a different source'
+      validated_edits_from(other).each do |edit|
+        add_edit(edit.start_offset, edit.end_offset, edit.replacement)
       end
-
-      validation_index = EditIndex.new
-      @edit_index.each { |edit| validation_index.add(edit) { false } }
-      other.edits.each do |edit|
-        conflict = validation_index.find_conflict(edit) { |existing| conflict?(edit, existing) }
-        raise ConflictError.new(edit, conflict) if conflict
-
-        validation_index.add(edit) { false }
-      end
-      other.edits.each { |edit| add_edit(edit.start_offset, edit.end_offset, edit.replacement) }
       self
     end
 
     def transaction(raise_on_conflict: false)
-      staging = Rewriter.new(@source_file, duplicate_insertions: @duplicate_insertions)
-      staging.instance_variable_set(:@transaction_staging, true)
+      staging = transaction_staging
       yield staging
-      return transaction_failed if staging.instance_variable_get(:@transaction_failed)
+      return transaction_failed if transaction_failed?(staging)
 
       merge!(staging)
       true
@@ -190,6 +102,14 @@ module Astel
     end
 
     def rewrite(validate: false)
+      result = apply_edits
+      validate_result(result, validate)
+      result
+    end
+
+    private
+
+    def apply_edits
       source = @source_file.source
       result = String.new(capacity: source.bytesize + @result_size_delta, encoding: source.encoding)
       cursor = 0
@@ -201,25 +121,40 @@ module Astel
       end
 
       result << source.byteslice(cursor, source.bytesize - cursor)
-      validate_result(result, validate)
       result
     end
 
-    private
-
     def add_edit(start_offset, end_offset, replacement)
+      edit = build_edit(start_offset, end_offset, replacement)
+      register_edit(edit)
+      self
+    end
+
+    def build_edit(start_offset, end_offset, replacement)
       validate_offsets(start_offset, end_offset)
-      replacement = replacement.to_s.dup
-      unless Encoding.compatible?(@source_file.source, replacement)
-        raise Encoding::CompatibilityError,
-              "#{replacement.encoding} replacement is incompatible with #{@source_file.source.encoding} source"
-      end
-      edit = Edit.new(
+      Edit.new(
         start_offset: start_offset,
         end_offset: end_offset,
-        replacement: replacement.freeze,
+        replacement: validated_replacement(replacement),
         sequence: @sequence
       )
+    end
+
+    def validated_replacement(replacement)
+      replacement = replacement.to_s
+      unless replacement.valid_encoding?
+        raise Encoding::CompatibilityError, "#{replacement.encoding} replacement contains invalid bytes"
+      end
+
+      begin
+        replacement.encode(@source_file.source.encoding).freeze
+      rescue EncodingError
+        raise Encoding::CompatibilityError,
+              "#{replacement.encoding} replacement is incompatible with #{@source_file.source.encoding} source"
+      end
+    end
+
+    def register_edit(edit)
       conflict = @edit_index.add(edit) { |existing| conflict?(edit, existing) }
       raise ConflictError.new(edit, conflict) if conflict
 
@@ -227,7 +162,34 @@ module Astel
       @result_size_delta += edit.replacement.bytesize - (edit.end_offset - edit.start_offset)
       @edits << edit
       @edits_snapshot = nil
-      self
+    end
+
+    def validated_edits_from(other)
+      raise ArgumentError, 'cannot merge edits for a different source' unless same_source_rewriter?(other)
+
+      validation_index = EditIndex.new
+      @edit_index.each { |edit| validation_index.add(edit) { false } }
+      other.edits.each do |edit|
+        conflict = validation_index.find_conflict(edit) { |existing| conflict?(edit, existing) }
+        raise ConflictError.new(edit, conflict) if conflict
+
+        validation_index.add(edit) { false }
+      end
+      other.edits
+    end
+
+    def same_source_rewriter?(other)
+      other.is_a?(Rewriter) && other.source_file.equal?(@source_file)
+    end
+
+    def transaction_staging
+      staging = Rewriter.new(@source_file, duplicate_insertions: @duplicate_insertions)
+      staging.instance_variable_set(:@transaction_staging, true)
+      staging
+    end
+
+    def transaction_failed?(staging)
+      staging.instance_variable_get(:@transaction_failed)
     end
 
     def offsets(location)
@@ -239,21 +201,31 @@ module Astel
     end
 
     def validate_offsets(start_offset, end_offset)
-      valid = start_offset.is_a?(Integer) && end_offset.is_a?(Integer) &&
-              start_offset >= 0 && start_offset <= end_offset && end_offset <= @source_file.source.bytesize
-      raise RangeError, 'edit range is outside the source' unless valid
+      raise RangeError, 'edit range is outside the source' unless valid_offsets?(start_offset, end_offset)
+    end
+
+    def valid_offsets?(start_offset, end_offset)
+      start_offset.is_a?(Integer) && end_offset.is_a?(Integer) &&
+        start_offset >= 0 && start_offset <= end_offset && end_offset <= @source_file.source.bytesize
     end
 
     def conflict?(left, right)
-      if @duplicate_insertions == :raise && left.insertion? && right.insertion?
-        return left.start_offset == right.start_offset
-      end
-      return left.start_offset > right.start_offset && left.start_offset < right.end_offset if left.insertion?
-      return right.start_offset > left.start_offset && right.start_offset < left.end_offset if right.insertion?
+      return true if duplicate_insertion_conflict?(left, right)
+      return insertion_inside?(left, right) if left.insertion?
+      return insertion_inside?(right, left) if right.insertion?
 
       left.start_offset < right.end_offset && right.start_offset < left.end_offset
     end
 
+    def duplicate_insertion_conflict?(left, right)
+      @duplicate_insertions == :raise && left.insertion? && right.insertion? &&
+        left.start_offset == right.start_offset
+    end
+
+    def insertion_inside?(insertion, edit)
+      insertion.start_offset > edit.start_offset && insertion.start_offset < edit.end_offset
+    end
+
     def transaction_failed
       @transaction_failed = true if @transaction_staging
       false

data/lib/astel/source_file.rb

@@ -17,10 +17,7 @@ module Astel
     def initialize(code, path:, version: nil)
       @path = path.to_s.freeze
       @version = normalize_version(version).freeze if version
-      raw_source = code.dup.freeze
-      options = { filepath: @path }
-      options[:version] = @version if @version
-      @parse_result = Prism.parse(raw_source, **options)
+      @parse_result = parse_source(code, path: @path, version: @version)
       @source = @parse_result.source.source.freeze
       @ast = @parse_result.value
       @comments = @parse_result.comments.freeze
@@ -53,10 +50,8 @@ module Astel
     end
 
     def magic_comment?(name)
-      pattern = /\A#\s*#{Regexp.escape(name.to_s)}\s*:\s*true\b/
-      lines.first(2).any? do |line|
-        candidate = line.start_with?("\uFEFF") ? line.delete_prefix("\uFEFF") : line
-        candidate.match?(pattern)
+      @parse_result.magic_comments.any? do |comment|
+        comment.key == name.to_s && comment.value == 'true' && effective_magic_comment?(comment)
       end
     end
 
@@ -85,9 +80,10 @@ module Astel
 
     def line_end(byte_offset)
       validate_offset!(byte_offset)
+      start = line_start(byte_offset)
       ending = @parse_result.source.line_end(byte_offset)
-      ending -= 1 if ending.positive? && source.getbyte(ending - 1) == 10
-      ending -= 1 if ending.positive? && source.getbyte(ending - 1) == 13
+      ending -= 1 if ending > start && source.getbyte(ending - 1) == 10
+      ending -= 1 if ending > start && source.getbyte(ending - 1) == 13
       ending
     end
 
@@ -105,10 +101,7 @@ module Astel
     end
 
     def newline
-      @newline ||= begin
-        crlf = source.b.scan(/\r\n/).length
-        crlf > source.count("\n") - crlf ? "\r\n" : "\n"
-      end
+      @newline ||= crlf_dominant? ? "\r\n" : "\n"
     end
 
     def indentation_at(byte_offset)
@@ -117,20 +110,40 @@ module Astel
     end
 
     def indent_unit
-      @indent_unit ||= begin
-        indentations = lines.filter_map { |line| line[/\A[ \t]+/] }
-        if indentations.any? { |indentation| indentation.include?("\t") }
-          "\t"
-        else
-          widths = [0, *indentations.map(&:bytesize)].uniq.sort
-          width = widths.each_cons(2).map { |left, right| right - left }.min
-          width ? ' ' * width : '  '
-        end
-      end
+      @indent_unit ||= detected_indent_unit
     end
 
     private
 
+    def effective_magic_comment?(comment)
+      line = @parse_result.source.line(comment.key_loc.start_offset)
+      line == 1 || (line == 2 && source.b.start_with?('#!'.b))
+    end
+
+    def parse_source(code, path:, version:)
+      options = { filepath: path }
+      options[:version] = version if version
+      Prism.parse(code.dup.freeze, **options)
+    end
+
+    def crlf_dominant?
+      crlf_count = source.b.scan(/\r\n/).length
+      crlf_count > source.count("\n") - crlf_count
+    end
+
+    def detected_indent_unit
+      indentations = lines.filter_map { |line| line[/\A[ \t]+/] }
+      return "\t" if indentations.any? { |indentation| indentation.include?("\t") }
+
+      space_indent_unit(indentations)
+    end
+
+    def space_indent_unit(indentations)
+      widths = [0, *indentations.map(&:bytesize)].uniq.sort
+      width = widths.each_cons(2).map { |left, right| right - left }.min
+      width ? ' ' * width : '  '
+    end
+
     def location(start_offset, end_offset)
       prism_source = @parse_result.source
       Location.new(

data/lib/astel/unified_diff.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require_relative '../astel'
+
+module Astel
+  module UnifiedDiff
+    MYERS_TRACE_DISTANCE_LIMIT = 256
+    private_constant :MYERS_TRACE_DISTANCE_LIMIT
+
+    module_function
+
+    def unified(before, after, path:, context: 3)
+      validate_options(path, context)
+      return '' if before == after
+
+      operations = operations(before.lines(chomp: false), after.lines(chomp: false))
+      annotated = annotate(operations)
+      hunks = hunk_ranges(annotated, context)
+      output = String.new(encoding: Encoding::BINARY)
+      output << '--- a/'.b << path.to_s.b << "\n+++ b/".b << path.to_s.b << "\n".b
+      hunks.each { |range| append_hunk(output, annotated[range]) }
+      output
+    end
+
+    def operations(before, after)
+      prefix = common_prefix_length(before, after)
+      suffix = common_suffix_length(before, after, prefix)
+
+      old_middle = before[prefix...(before.length - suffix)]
+      new_middle = after[prefix...(after.length - suffix)]
+      before.first(prefix).map { |line| [:equal, line] } +
+        myers_operations(old_middle, new_middle) +
+        (suffix.zero? ? [] : before.last(suffix).map { |line| [:equal, line] })
+    end
+    private_class_method :operations
+
+    def common_prefix_length(before, after)
+      length = 0
+      length += 1 while length < before.length && length < after.length && before[length] == after[length]
+      length
+    end
+    private_class_method :common_prefix_length
+
+    def common_suffix_length(before, after, prefix_length)
+      length = 0
+      while length < before.length - prefix_length && length < after.length - prefix_length &&
+            before[-length - 1] == after[-length - 1]
+        length += 1
+      end
+      length
+    end
+    private_class_method :common_suffix_length
+
+    def myers_operations(before, after)
+      frontier = Hash.new(0)
+      frontier[1] = 0
+      trace = []
+
+      (0..(before.length + after.length)).each do |distance|
+        # ponytail: use linear-space backtracking if minimal large rewrites become necessary.
+        return replacement_operations(before, after) if distance > MYERS_TRACE_DISTANCE_LIMIT
+
+        trace << frontier.dup
+        (-distance).step(distance, 2) do |diagonal|
+          old_index = next_old_index(frontier, diagonal, distance)
+          new_index = old_index - diagonal
+          while old_index < before.length && new_index < after.length && before[old_index] == after[new_index]
+            old_index += 1
+            new_index += 1
+          end
+          frontier[diagonal] = old_index
+          return backtrack_myers(before, after, trace, distance) if
+            old_index >= before.length && new_index >= after.length
+        end
+      end
+    end
+    private_class_method :myers_operations
+
+    def replacement_operations(before, after)
+      before.map { |line| [:delete, line] } + after.map { |line| [:insert, line] }
+    end
+    private_class_method :replacement_operations
+
+    def next_old_index(frontier, diagonal, distance)
+      if diagonal == -distance || (diagonal != distance && frontier[diagonal - 1] < frontier[diagonal + 1])
+        frontier[diagonal + 1]
+      else
+        frontier[diagonal - 1] + 1
+      end
+    end
+    private_class_method :next_old_index
+
+    def backtrack_myers(before, after, trace, distance)
+      old_index = before.length
+      new_index = after.length
+      operations = []
+
+      distance.downto(0) do |depth|
+        previous = trace[depth]
+        diagonal = old_index - new_index
+        previous_diagonal = if diagonal == -depth ||
+                               (diagonal != depth && previous[diagonal - 1] < previous[diagonal + 1])
+                              diagonal + 1
+                            else
+                              diagonal - 1
+                            end
+        previous_old_index = previous[previous_diagonal]
+        previous_new_index = previous_old_index - previous_diagonal
+
+        while old_index > previous_old_index && new_index > previous_new_index
+          operations << [:equal, before[old_index - 1]]
+          old_index -= 1
+          new_index -= 1
+        end
+        next if depth.zero?
+
+        if old_index == previous_old_index
+          operations << [:insert, after[new_index - 1]]
+          new_index -= 1
+        else
+          operations << [:delete, before[old_index - 1]]
+          old_index -= 1
+        end
+      end
+
+      operations.reverse
+    end
+    private_class_method :backtrack_myers
+
+    def annotate(operations)
+      old_line = 1
+      new_line = 1
+      operations.map do |type, line|
+        entry = { type: type, line: line, old_line: old_line, new_line: new_line }
+        old_line += 1 unless type == :insert
+        new_line += 1 unless type == :delete
+        entry
+      end
+    end
+    private_class_method :annotate
+
+    def hunk_ranges(operations, context)
+      changed = operations.each_index.reject { |index| operations[index][:type] == :equal }
+      ranges = changed.map do |index|
+        [index - context, 0].max..[index + context, operations.length - 1].min
+      end
+      merge_adjacent_ranges(ranges)
+    end
+    private_class_method :hunk_ranges
+
+    def merge_adjacent_ranges(ranges)
+      ranges.each_with_object([]) do |range, merged|
+        if merged.last && range.begin <= merged.last.end + 1
+          merged[-1] = merged.last.begin..[merged.last.end, range.end].max
+        else
+          merged << range
+        end
+      end
+    end
+    private_class_method :merge_adjacent_ranges
+
+    def append_hunk(output, entries)
+      output << hunk_header(entries)
+      entries.each { |entry| append_hunk_entry(output, entry) }
+    end
+    private_class_method :append_hunk
+
+    def append_hunk_entry(output, entry)
+      marker = { equal: ' ', delete: '-', insert: '+' }.fetch(entry[:type])
+      output << marker << entry[:line].b
+      output << "\n\\n" unless entry[:line].end_with?("\n")
+    end
+    private_class_method :append_hunk_entry
+
+    def hunk_header(entries)
+      old_count = entries.count { |entry| entry[:type] != :insert }
+      new_count = entries.count { |entry| entry[:type] != :delete }
+      old_start = entries.first[:old_line] - (old_count.zero? ? 1 : 0)
+      new_start = entries.first[:new_line] - (new_count.zero? ? 1 : 0)
+      "@@ -#{range(old_start, old_count)} +#{range(new_start, new_count)} @@\n"
+    end
+    private_class_method :hunk_header
+
+    def range(start_line, count)
+      count == 1 ? start_line.to_s : "#{start_line},#{count}"
+    end
+    private_class_method :range
+
+    def validate_options(path, context)
+      if path.to_s.empty? || path.to_s.match?(/[\t\r\n]/)
+        raise ArgumentError, 'path must not be empty or contain control separators'
+      end
+      raise ArgumentError, 'context must be a non-negative integer' unless context.is_a?(Integer) && context >= 0
+    end
+    private_class_method :validate_options
+  end
+
+  module Diffable
+    def to_diff(context: 3, path: @source_file.path)
+      UnifiedDiff.unified(@source_file.source, rewrite, path: path, context: context)
+    end
+  end
+end
+
+Astel::Rewriter.include(Astel::Diffable)

data/lib/astel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Astel
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/astel.rb

@@ -7,6 +7,7 @@ module Astel
 
   autoload :Location, File.expand_path('astel/location', __dir__)
   autoload :SourceFile, File.expand_path('astel/source_file', __dir__)
+  autoload :NodeSourceText, File.expand_path('astel/node_source_text', __dir__)
   autoload :NodeExt, File.expand_path('astel/node_ext', __dir__)
   autoload :NodeType, File.expand_path('astel/node_type', __dir__)
   autoload :Dispatcher, File.expand_path('astel/dispatcher', __dir__)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: astel
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yudai Takada
@@ -55,12 +55,18 @@ files:
 - lib/astel/node_pattern/matcher_compiler.rb
 - lib/astel/node_pattern/parser.rb
 - lib/astel/node_pattern/predicate_compiler.rb
+- lib/astel/node_source_text.rb
 - lib/astel/node_type.rb
 - lib/astel/refactor.rb
-- lib/astel/refactor/comments.rb
-- lib/astel/refactor/formatting.rb
 - lib/astel/rewriter.rb
+- lib/astel/rewriter/edit_index.rb
+- lib/astel/rewriter/structured.rb
+- lib/astel/rewriter/structured/collections.rb
+- lib/astel/rewriter/structured/comments.rb
+- lib/astel/rewriter/structured/formatting.rb
+- lib/astel/rewriter/structured/source_indenter.rb
 - lib/astel/source_file.rb
+- lib/astel/unified_diff.rb
 - lib/astel/version.rb
 homepage: https://rubygems.org/gems/astel
 licenses:

data/lib/astel/refactor/comments.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Astel
-  module Refactor
-    def leading_comments(node)
-      attach_comments
-      node.location.leading_comments
-    end
-
-    def trailing_comment(node)
-      attach_comments
-      node.location.trailing_comments.first
-    end
-
-    def remove_with_comments(node)
-      comments = leading_comments(node)
-      trailing = trailing_comment(node)
-      start_offset = comments.empty? ? node.location.start_offset : comments.first.location.start_offset
-      end_offset = trailing ? trailing.location.end_offset : node.location.end_offset
-      first = @source_file.line_location(start_offset)
-      last = @source_file.line_location(end_offset - 1)
-      before = @source_file.source.byteslice(first.start_offset, start_offset - first.start_offset)
-      after = @source_file.source.byteslice(end_offset, last.end_offset - end_offset)
-      return remove(start_offset...end_offset) unless before.strip.empty? && after.strip.empty?
-
-      ending = @source_file.line_location_with_newline(end_offset - 1).end_offset
-      remove(first.start_offset...ending)
-    end
-
-    private
-
-    def attach_comments
-      @source_file.attach_comments!
-    end
-  end
-end