astel 0.1.0 → 0.3.0

data/lib/astel/rewriter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'rewriter/edit_index'
+
 module Astel
   class Rewriter
     class ConflictError < Error
@@ -13,79 +15,38 @@ module Astel
       end
     end
 
+    class ValidationError < Error
+      attr_reader :result, :parse_errors
+
+      def initialize(result, parse_errors)
+        @result = result
+        @parse_errors = parse_errors.freeze
+        super("rewritten source contains #{parse_errors.length} syntax error(s)")
+      end
+    end
+
     Edit = Data.define(:start_offset, :end_offset, :replacement, :sequence) do
       def insertion?
         start_offset == end_offset
       end
     end
 
-    class EditIndex
-      include Enumerable
-
-      SPLIT_THRESHOLD = 256
-
-      def initialize
-        @blocks = []
-      end
-
-      def add(edit)
-        if @blocks.empty?
-          @blocks << [edit]
-          return
-        end
-
-        block_index = block_index_for(edit)
-        block = @blocks.fetch(block_index)
-        edit_index = block.bsearch_index { |existing| ordered_after?(existing, edit) } || block.length
-        following = block[edit_index] || @blocks[block_index + 1]&.first
-        return following if following && yield(following)
-
-        preceding = if edit_index.positive?
-                      block[edit_index - 1]
-                    elsif block_index.positive?
-                      @blocks[block_index - 1].last
-                    end
-        return preceding if preceding && yield(preceding)
-
-        block.insert(edit_index, edit)
-        split(block_index, block) if block.length > SPLIT_THRESHOLD
-        nil
-      end
-
-      def each
-        return enum_for(__method__) unless block_given?
-
-        @blocks.each do |block|
-          block.each { |edit| yield edit }
-        end
-      end
-
-      private
-
-      def block_index_for(edit)
-        @blocks.bsearch_index { |block| ordered_after?(block.last, edit) } || @blocks.length - 1
-      end
-
-      def ordered_after?(existing, edit)
-        existing.start_offset > edit.start_offset ||
-          (existing.start_offset == edit.start_offset && existing.end_offset > edit.end_offset) ||
-          (existing.start_offset == edit.start_offset && existing.end_offset == edit.end_offset &&
-           existing.sequence > edit.sequence)
-      end
+    attr_reader :source_file
 
-      def split(block_index, block)
-        middle = block.length / 2
-        @blocks.insert(block_index + 1, block.slice!(middle, block.length - middle))
+    def initialize(source_file, duplicate_insertions: :accept)
+      unless %i[accept raise].include?(duplicate_insertions)
+        raise ArgumentError, 'duplicate_insertions must be :accept or :raise'
       end
-    end
 
-    def initialize(source_file)
       @source_file = source_file
+      @duplicate_insertions = duplicate_insertions
       @edits = []
       @edits_snapshot = nil
       @edit_index = EditIndex.new
       @sequence = 0
       @result_size_delta = 0
+      @transaction_staging = false
+      @transaction_failed = false
     end
 
     def edits
@@ -111,7 +72,44 @@ module Astel
       replace(location, '')
     end
 
-    def rewrite
+    def wrap(location, before, after)
+      transaction(raise_on_conflict: true) do |rewriter|
+        rewriter.insert_before(location, before)
+        rewriter.insert_after(location, after)
+      end
+      self
+    end
+
+    def merge!(other)
+      validated_edits_from(other).each do |edit|
+        add_edit(edit.start_offset, edit.end_offset, edit.replacement)
+      end
+      self
+    end
+
+    def transaction(raise_on_conflict: false)
+      staging = transaction_staging
+      yield staging
+      return transaction_failed if transaction_failed?(staging)
+
+      merge!(staging)
+      true
+    rescue ConflictError
+      @transaction_failed = true if @transaction_staging
+      raise if raise_on_conflict
+
+      false
+    end
+
+    def rewrite(validate: false)
+      result = apply_edits
+      validate_result(result, validate)
+      result
+    end
+
+    private
+
+    def apply_edits
       source = @source_file.source
       result = String.new(capacity: source.bytesize + @result_size_delta, encoding: source.encoding)
       cursor = 0
@@ -126,16 +124,37 @@ module Astel
       result
     end
 
-    private
-
     def add_edit(start_offset, end_offset, replacement)
+      edit = build_edit(start_offset, end_offset, replacement)
+      register_edit(edit)
+      self
+    end
+
+    def build_edit(start_offset, end_offset, replacement)
       validate_offsets(start_offset, end_offset)
-      edit = Edit.new(
+      Edit.new(
         start_offset: start_offset,
         end_offset: end_offset,
-        replacement: replacement.to_s.dup.freeze,
+        replacement: validated_replacement(replacement),
         sequence: @sequence
       )
+    end
+
+    def validated_replacement(replacement)
+      replacement = replacement.to_s
+      unless replacement.valid_encoding?
+        raise Encoding::CompatibilityError, "#{replacement.encoding} replacement contains invalid bytes"
+      end
+
+      begin
+        replacement.encode(@source_file.source.encoding).freeze
+      rescue EncodingError
+        raise Encoding::CompatibilityError,
+              "#{replacement.encoding} replacement is incompatible with #{@source_file.source.encoding} source"
+      end
+    end
+
+    def register_edit(edit)
       conflict = @edit_index.add(edit) { |existing| conflict?(edit, existing) }
       raise ConflictError.new(edit, conflict) if conflict
 
@@ -143,7 +162,34 @@ module Astel
       @result_size_delta += edit.replacement.bytesize - (edit.end_offset - edit.start_offset)
       @edits << edit
       @edits_snapshot = nil
-      self
+    end
+
+    def validated_edits_from(other)
+      raise ArgumentError, 'cannot merge edits for a different source' unless same_source_rewriter?(other)
+
+      validation_index = EditIndex.new
+      @edit_index.each { |edit| validation_index.add(edit) { false } }
+      other.edits.each do |edit|
+        conflict = validation_index.find_conflict(edit) { |existing| conflict?(edit, existing) }
+        raise ConflictError.new(edit, conflict) if conflict
+
+        validation_index.add(edit) { false }
+      end
+      other.edits
+    end
+
+    def same_source_rewriter?(other)
+      other.is_a?(Rewriter) && other.source_file.equal?(@source_file)
+    end
+
+    def transaction_staging
+      staging = Rewriter.new(@source_file, duplicate_insertions: @duplicate_insertions)
+      staging.instance_variable_set(:@transaction_staging, true)
+      staging
+    end
+
+    def transaction_failed?(staging)
+      staging.instance_variable_get(:@transaction_failed)
     end
 
     def offsets(location)
@@ -155,17 +201,42 @@ module Astel
     end
 
     def validate_offsets(start_offset, end_offset)
-      valid = start_offset.is_a?(Integer) && end_offset.is_a?(Integer) &&
-              start_offset >= 0 && start_offset <= end_offset && end_offset <= @source_file.source.bytesize
-      raise RangeError, 'edit range is outside the source' unless valid
+      raise RangeError, 'edit range is outside the source' unless valid_offsets?(start_offset, end_offset)
+    end
+
+    def valid_offsets?(start_offset, end_offset)
+      start_offset.is_a?(Integer) && end_offset.is_a?(Integer) &&
+        start_offset >= 0 && start_offset <= end_offset && end_offset <= @source_file.source.bytesize
     end
 
     def conflict?(left, right)
-      return left.start_offset == right.start_offset if left.insertion? && right.insertion?
-      return left.start_offset > right.start_offset && left.start_offset < right.end_offset if left.insertion?
-      return right.start_offset > left.start_offset && right.start_offset < left.end_offset if right.insertion?
+      return true if duplicate_insertion_conflict?(left, right)
+      return insertion_inside?(left, right) if left.insertion?
+      return insertion_inside?(right, left) if right.insertion?
 
       left.start_offset < right.end_offset && right.start_offset < left.end_offset
     end
+
+    def duplicate_insertion_conflict?(left, right)
+      @duplicate_insertions == :raise && left.insertion? && right.insertion? &&
+        left.start_offset == right.start_offset
+    end
+
+    def insertion_inside?(insertion, edit)
+      insertion.start_offset > edit.start_offset && insertion.start_offset < edit.end_offset
+    end
+
+    def transaction_failed
+      @transaction_failed = true if @transaction_staging
+      false
+    end
+
+    def validate_result(result, validate)
+      return unless validate
+      raise ArgumentError, 'validate must be false or :parse' unless validate == :parse
+
+      parsed = SourceFile.from_string(result, path: @source_file.path, version: @source_file.version)
+      raise ValidationError.new(result, parsed.errors) unless parsed.valid?
+    end
   end
 end

data/lib/astel/source_file.rb

@@ -4,7 +4,7 @@ require 'prism'
 
 module Astel
   class SourceFile
-    attr_reader :path, :source, :ast, :comments, :errors, :parse_result
+    attr_reader :path, :source, :ast, :comments, :errors, :parse_result, :version
 
     def self.parse(path:, version: nil)
       from_string(File.binread(path), path: path, version: version)
@@ -16,10 +16,8 @@ module Astel
 
     def initialize(code, path:, version: nil)
       @path = path.to_s.freeze
-      raw_source = code.dup.freeze
-      options = { filepath: @path }
-      options[:version] = normalize_version(version) if version
-      @parse_result = Prism.parse(raw_source, **options)
+      @version = normalize_version(version).freeze if version
+      @parse_result = parse_source(code, path: @path, version: @version)
       @source = @parse_result.source.source.freeze
       @ast = @parse_result.value
       @comments = @parse_result.comments.freeze
@@ -47,20 +45,123 @@ module Astel
         start_line: 1,
         start_column: 0,
         end_line: 1,
-        end_column: line.delete_suffix("\n").delete_suffix("\r").length
+        end_column: line.delete_suffix("\n").delete_suffix("\r").bytesize
       )
     end
 
     def magic_comment?(name)
-      pattern = /\A#\s*#{Regexp.escape(name.to_s)}\s*:\s*true\b/
-      lines.first(2).any? do |line|
-        candidate = line.start_with?("\uFEFF") ? line.delete_prefix("\uFEFF") : line
-        candidate.match?(pattern)
+      @parse_result.magic_comments.any? do |comment|
+        comment.key == name.to_s && comment.value == 'true' && effective_magic_comment?(comment)
       end
     end
 
+    def attach_comments!
+      return self if @comments_attached
+
+      @parse_result.attach_comments!
+      @comments_attached = true
+      self
+    end
+
+    def line_at(byte_offset)
+      validate_offset!(byte_offset)
+      @parse_result.source.line(byte_offset)
+    end
+
+    def column_at(byte_offset)
+      validate_offset!(byte_offset)
+      @parse_result.source.column(byte_offset)
+    end
+
+    def line_start(byte_offset)
+      validate_offset!(byte_offset)
+      @parse_result.source.line_start(byte_offset)
+    end
+
+    def line_end(byte_offset)
+      validate_offset!(byte_offset)
+      start = line_start(byte_offset)
+      ending = @parse_result.source.line_end(byte_offset)
+      ending -= 1 if ending > start && source.getbyte(ending - 1) == 10
+      ending -= 1 if ending > start && source.getbyte(ending - 1) == 13
+      ending
+    end
+
+    def line_location(byte_offset)
+      location(line_start(byte_offset), line_end(byte_offset))
+    end
+
+    def line_location_with_newline(byte_offset)
+      validate_offset!(byte_offset)
+      location(line_start(byte_offset), @parse_result.source.line_end(byte_offset))
+    end
+
+    def slice(location)
+      source.byteslice(location.start_offset, location.end_offset - location.start_offset)
+    end
+
+    def newline
+      @newline ||= crlf_dominant? ? "\r\n" : "\n"
+    end
+
+    def indentation_at(byte_offset)
+      start_offset = line_start(byte_offset)
+      source.byteslice(start_offset, line_end(byte_offset) - start_offset)[/\A[ \t]*/]
+    end
+
+    def indent_unit
+      @indent_unit ||= detected_indent_unit
+    end
+
     private
 
+    def effective_magic_comment?(comment)
+      line = @parse_result.source.line(comment.key_loc.start_offset)
+      line == 1 || (line == 2 && source.b.start_with?('#!'.b))
+    end
+
+    def parse_source(code, path:, version:)
+      options = { filepath: path }
+      options[:version] = version if version
+      Prism.parse(code.dup.freeze, **options)
+    end
+
+    def crlf_dominant?
+      crlf_count = source.b.scan(/\r\n/).length
+      crlf_count > source.count("\n") - crlf_count
+    end
+
+    def detected_indent_unit
+      indentations = lines.filter_map { |line| line[/\A[ \t]+/] }
+      return "\t" if indentations.any? { |indentation| indentation.include?("\t") }
+
+      space_indent_unit(indentations)
+    end
+
+    def space_indent_unit(indentations)
+      widths = [0, *indentations.map(&:bytesize)].uniq.sort
+      width = widths.each_cons(2).map { |left, right| right - left }.min
+      width ? ' ' * width : '  '
+    end
+
+    def location(start_offset, end_offset)
+      prism_source = @parse_result.source
+      Location.new(
+        start_offset: start_offset,
+        end_offset: end_offset,
+        start_line: prism_source.line(start_offset),
+        start_column: prism_source.column(start_offset),
+        end_line: prism_source.line(end_offset),
+        end_column: prism_source.column(end_offset)
+      )
+    end
+
+    def validate_offset!(byte_offset)
+      unless byte_offset.is_a?(Integer) && byte_offset.between?(0, source.bytesize)
+        raise RangeError, 'offset is outside the source'
+      end
+    end
+
     def normalize_version(version)
       version.match?(/\A\d+\.\d+\z/) ? "#{version}.0" : version
     end

data/lib/astel/unified_diff.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require_relative '../astel'
+
+module Astel
+  module UnifiedDiff
+    MYERS_TRACE_DISTANCE_LIMIT = 256
+    private_constant :MYERS_TRACE_DISTANCE_LIMIT
+
+    module_function
+
+    def unified(before, after, path:, context: 3)
+      validate_options(path, context)
+      return '' if before == after
+
+      operations = operations(before.lines(chomp: false), after.lines(chomp: false))
+      annotated = annotate(operations)
+      hunks = hunk_ranges(annotated, context)
+      output = String.new(encoding: Encoding::BINARY)
+      output << '--- a/'.b << path.to_s.b << "\n+++ b/".b << path.to_s.b << "\n".b
+      hunks.each { |range| append_hunk(output, annotated[range]) }
+      output
+    end
+
+    def operations(before, after)
+      prefix = common_prefix_length(before, after)
+      suffix = common_suffix_length(before, after, prefix)
+
+      old_middle = before[prefix...(before.length - suffix)]
+      new_middle = after[prefix...(after.length - suffix)]
+      before.first(prefix).map { |line| [:equal, line] } +
+        myers_operations(old_middle, new_middle) +
+        (suffix.zero? ? [] : before.last(suffix).map { |line| [:equal, line] })
+    end
+    private_class_method :operations
+
+    def common_prefix_length(before, after)
+      length = 0
+      length += 1 while length < before.length && length < after.length && before[length] == after[length]
+      length
+    end
+    private_class_method :common_prefix_length
+
+    def common_suffix_length(before, after, prefix_length)
+      length = 0
+      while length < before.length - prefix_length && length < after.length - prefix_length &&
+            before[-length - 1] == after[-length - 1]
+        length += 1
+      end
+      length
+    end
+    private_class_method :common_suffix_length
+
+    def myers_operations(before, after)
+      frontier = Hash.new(0)
+      frontier[1] = 0
+      trace = []
+
+      (0..(before.length + after.length)).each do |distance|
+        # ponytail: use linear-space backtracking if minimal large rewrites become necessary.
+        return replacement_operations(before, after) if distance > MYERS_TRACE_DISTANCE_LIMIT
+
+        trace << frontier.dup
+        (-distance).step(distance, 2) do |diagonal|
+          old_index = next_old_index(frontier, diagonal, distance)
+          new_index = old_index - diagonal
+          while old_index < before.length && new_index < after.length && before[old_index] == after[new_index]
+            old_index += 1
+            new_index += 1
+          end
+          frontier[diagonal] = old_index
+          return backtrack_myers(before, after, trace, distance) if
+            old_index >= before.length && new_index >= after.length
+        end
+      end
+    end
+    private_class_method :myers_operations
+
+    def replacement_operations(before, after)
+      before.map { |line| [:delete, line] } + after.map { |line| [:insert, line] }
+    end
+    private_class_method :replacement_operations
+
+    def next_old_index(frontier, diagonal, distance)
+      if diagonal == -distance || (diagonal != distance && frontier[diagonal - 1] < frontier[diagonal + 1])
+        frontier[diagonal + 1]
+      else
+        frontier[diagonal - 1] + 1
+      end
+    end
+    private_class_method :next_old_index
+
+    def backtrack_myers(before, after, trace, distance)
+      old_index = before.length
+      new_index = after.length
+      operations = []
+
+      distance.downto(0) do |depth|
+        previous = trace[depth]
+        diagonal = old_index - new_index
+        previous_diagonal = if diagonal == -depth ||
+                               (diagonal != depth && previous[diagonal - 1] < previous[diagonal + 1])
+                              diagonal + 1
+                            else
+                              diagonal - 1
+                            end
+        previous_old_index = previous[previous_diagonal]
+        previous_new_index = previous_old_index - previous_diagonal
+
+        while old_index > previous_old_index && new_index > previous_new_index
+          operations << [:equal, before[old_index - 1]]
+          old_index -= 1
+          new_index -= 1
+        end
+        next if depth.zero?
+
+        if old_index == previous_old_index
+          operations << [:insert, after[new_index - 1]]
+          new_index -= 1
+        else
+          operations << [:delete, before[old_index - 1]]
+          old_index -= 1
+        end
+      end
+
+      operations.reverse
+    end
+    private_class_method :backtrack_myers
+
+    def annotate(operations)
+      old_line = 1
+      new_line = 1
+      operations.map do |type, line|
+        entry = { type: type, line: line, old_line: old_line, new_line: new_line }
+        old_line += 1 unless type == :insert
+        new_line += 1 unless type == :delete
+        entry
+      end
+    end
+    private_class_method :annotate
+
+    def hunk_ranges(operations, context)
+      changed = operations.each_index.reject { |index| operations[index][:type] == :equal }
+      ranges = changed.map do |index|
+        [index - context, 0].max..[index + context, operations.length - 1].min
+      end
+      merge_adjacent_ranges(ranges)
+    end
+    private_class_method :hunk_ranges
+
+    def merge_adjacent_ranges(ranges)
+      ranges.each_with_object([]) do |range, merged|
+        if merged.last && range.begin <= merged.last.end + 1
+          merged[-1] = merged.last.begin..[merged.last.end, range.end].max
+        else
+          merged << range
+        end
+      end
+    end
+    private_class_method :merge_adjacent_ranges
+
+    def append_hunk(output, entries)
+      output << hunk_header(entries)
+      entries.each { |entry| append_hunk_entry(output, entry) }
+    end
+    private_class_method :append_hunk
+
+    def append_hunk_entry(output, entry)
+      marker = { equal: ' ', delete: '-', insert: '+' }.fetch(entry[:type])
+      output << marker << entry[:line].b
+      output << "\n\\n" unless entry[:line].end_with?("\n")
+    end
+    private_class_method :append_hunk_entry
+
+    def hunk_header(entries)
+      old_count = entries.count { |entry| entry[:type] != :insert }
+      new_count = entries.count { |entry| entry[:type] != :delete }
+      old_start = entries.first[:old_line] - (old_count.zero? ? 1 : 0)
+      new_start = entries.first[:new_line] - (new_count.zero? ? 1 : 0)
+      "@@ -#{range(old_start, old_count)} +#{range(new_start, new_count)} @@\n"
+    end
+    private_class_method :hunk_header
+
+    def range(start_line, count)
+      count == 1 ? start_line.to_s : "#{start_line},#{count}"
+    end
+    private_class_method :range
+
+    def validate_options(path, context)
+      if path.to_s.empty? || path.to_s.match?(/[\t\r\n]/)
+        raise ArgumentError, 'path must not be empty or contain control separators'
+      end
+      raise ArgumentError, 'context must be a non-negative integer' unless context.is_a?(Integer) && context >= 0
+    end
+    private_class_method :validate_options
+  end
+
+  module Diffable
+    def to_diff(context: 3, path: @source_file.path)
+      UnifiedDiff.unified(@source_file.source, rewrite, path: path, context: context)
+    end
+  end
+end
+
+Astel::Rewriter.include(Astel::Diffable)

data/lib/astel/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Astel
-  VERSION = '0.1.0'
+  VERSION = '0.3.0'
 end

data/lib/astel.rb

@@ -7,6 +7,7 @@ module Astel
 
   autoload :Location, File.expand_path('astel/location', __dir__)
   autoload :SourceFile, File.expand_path('astel/source_file', __dir__)
+  autoload :NodeSourceText, File.expand_path('astel/node_source_text', __dir__)
   autoload :NodeExt, File.expand_path('astel/node_ext', __dir__)
   autoload :NodeType, File.expand_path('astel/node_type', __dir__)
   autoload :Dispatcher, File.expand_path('astel/dispatcher', __dir__)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: astel
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yudai Takada
@@ -43,7 +43,10 @@ files:
 - benchmark/dispatch_bench.rb
 - benchmark/node_pattern_bench.rb
 - benchmark/rewriter_bench.rb
+- docs/codemod-guide.md
+- docs/refactor.md
 - lib/astel.rb
+- lib/astel/diff.rb
 - lib/astel/dispatcher.rb
 - lib/astel/location.rb
 - lib/astel/node_ext.rb
@@ -52,9 +55,18 @@ files:
 - lib/astel/node_pattern/matcher_compiler.rb
 - lib/astel/node_pattern/parser.rb
 - lib/astel/node_pattern/predicate_compiler.rb
+- lib/astel/node_source_text.rb
 - lib/astel/node_type.rb
+- lib/astel/refactor.rb
 - lib/astel/rewriter.rb
+- lib/astel/rewriter/edit_index.rb
+- lib/astel/rewriter/structured.rb
+- lib/astel/rewriter/structured/collections.rb
+- lib/astel/rewriter/structured/comments.rb
+- lib/astel/rewriter/structured/formatting.rb
+- lib/astel/rewriter/structured/source_indenter.rb
 - lib/astel/source_file.rb
+- lib/astel/unified_diff.rb
 - lib/astel/version.rb
 homepage: https://rubygems.org/gems/astel
 licenses:
@@ -75,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Fast Prism AST traversal, matching, and source rewriting
 test_files: []