flexr 1.0.0 → 1.1.0

data/lib/flexr/runtime/interpreter.rb

@@ -13,25 +13,34 @@ module Flexr
     class Interpreter
       def initialize(lexer)
         @lexer = lexer
+        klass = lexer.class
+        @compiled = klass.__flexr_compiled
+        @rules = klass.__flexr_rules
+        @config = klass.__flexr_config
       end
 
       def scan
-        machine = @lexer.class.__flexr_compiled.machines.fetch(@lexer.state)
+        machine = @compiled.machines.fetch(@lexer.state)
         position = @lexer.byte_pos
         return nil unless @lexer.valid_utf8_at?(position)
-        return scan_firstmatch(position) if @lexer.class.__flexr_config.backend == :firstmatch
-        return scan_fast(machine, position) if fast_path?
+        return scan_firstmatch(position) if @config.backend == :firstmatch
+        return scan_fast(machine, position) if fast_path?(machine)
 
         buffer = @lexer.buffer
         state = machine.dfa.start
         best = reference_match(position, buffer)
         cursor = position
-        best = consider_acceptances(machine, state, cursor, position, buffer, best) if @lexer.class.__flexr_config.options[:allow_empty_match]
+        best = consider_acceptances(machine, state, cursor, position, buffer, best) if @config.options[:allow_empty_match]
         acceleration_regions = acceleration_enabled? ? acceleration_regions_for(machine) : nil
 
         while buffer.ensure_available?(cursor + 1)
-          if acceleration_regions && (region = acceleration_regions[state])
-            accelerated_end = accelerate(region, buffer, cursor)
+          if acceleration_regions && (region = acceleration_regions[state]) &&
+              !@disabled_auto_acceleration&.key?(region)
+            acceptance = machine.dfa.accepts[state].first
+            token_rule = acceptance&.rule_index
+            accelerated_end = accelerate(
+              region, buffer, cursor, token_rule: token_rule, token_start: position
+            )
             if accelerated_end && accelerated_end > cursor
               cursor = accelerated_end
               best = consider_acceptances(machine, state, cursor, position, buffer, best)
@@ -40,10 +49,13 @@ module Flexr
           end
 
           byte = buffer.getbyte(cursor)
+          @lexer.consume_step!
           state = transition(machine.dfa, state, byte)
           break unless state
 
           cursor += 1
+          acceptance = machine.dfa.accepts[state].first
+          ensure_token_size!(cursor, position, rule: acceptance&.rule_index)
           best = consider_acceptances(machine, state, cursor, position, buffer, best)
         end
         best
@@ -51,22 +63,57 @@ module Flexr
 
       def scan_fast(machine, position)
         buffer = @lexer.buffer
-        source = buffer.source
         dfa = machine.dfa
         accepts = dfa.accepts
-        rules = @lexer.class.__flexr_rules
+        rules = @rules
         transitions = dfa.transitions
         ec = dfa.ec
         direct = dfa.direct
+        direct_nxt = direct&.fetch(:nxt)
+        direct_classes = direct&.fetch(:classes)
+        source = buffer.stable_source
+        guarded_steps = @lexer.scan_steps_guarded?
+        text_start = @lexer.more_text_start || position
+        max_token_size = @lexer.max_token_size
+        token_limit_required = @lexer.token_limit_required?
         state = dfa.start
         cursor = position
         best = nil
 
-        while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
-          byte = source.getbyte(cursor)
-          state = if direct
+        if source && !guarded_steps && !token_limit_required
+          while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
+            byte = source.getbyte(cursor)
+            state = if direct_nxt
+              class_id = ec[byte]
+              value = direct_nxt[(state * direct_classes) + class_id]
+              value >= 0 ? value : nil
+            else
+              transitions[state][ec[byte]]
+            end
+            break unless state
+
+            cursor += 1
+            acceptance = accepts[state].first
+            next unless acceptance
+
+            rule = rules.fetch(acceptance.rule_index)
+            next if best && cursor == best.total_end_pos && rule.index > best.rule.index
+
+            best ||= (@match ||= Match.new)
+            best.rule = rule
+            best.start_pos = position
+            best.end_pos = cursor
+            best.total_end_pos = cursor
+          end
+          return best
+        end
+
+        while cursor < buffer.bytesize || buffer.ensure_available?(cursor + 1)
+          byte = buffer.getbyte(cursor)
+          @lexer.consume_step! if guarded_steps
+          state = if direct_nxt
             class_id = ec[byte]
-            value = direct[:nxt][(state * direct[:classes]) + class_id]
+            value = direct_nxt[(state * direct_classes) + class_id]
             value >= 0 ? value : nil
           else
             transitions[state][ec[byte]]
@@ -75,6 +122,10 @@ module Flexr
 
           cursor += 1
           acceptance = accepts[state].first
+          if token_limit_required
+            size = cursor - text_start
+            @lexer.defer_token_size_check!(size, rule: acceptance&.rule_index) if size > max_token_size
+          end
           next unless acceptance
 
           rule = rules.fetch(acceptance.rule_index)
@@ -94,7 +145,7 @@ module Flexr
       def reference_match(position, buffer)
         return unless reference_rules?
 
-        candidates = @lexer.class.__flexr_rules.filter_map do |rule|
+        candidates = @rules.filter_map do |rule|
           next unless rule_active?(rule)
           next unless rule.patterns.any? { |pattern| reference_pattern?(pattern) }
 
@@ -102,7 +153,10 @@ module Flexr
             condition = rule.pattern_conditions.fetch(pattern_index)
             next if condition.bol_only && !@lexer.beginning_of_line?
 
-            match = streamed_match(pattern, buffer, position, reference: reference_pattern?(pattern))
+            match = streamed_match(
+              pattern, buffer, position, reference: reference_pattern?(pattern),
+              limit: :token, rule: rule
+            )
             next unless match&.begin(0)&.zero?
 
             end_position = position + match[0].bytesize
@@ -120,7 +174,7 @@ module Flexr
           trailing = trailing_length(rule, buffer, end_position)
           next if rule.trailing && trailing.nil?
 
-          ensure_token_size!(end_position, position)
+          ensure_token_size!(end_position, position, rule: rule)
           Match.new(rule: rule, start_pos: position, end_pos: end_position,
                     total_end_pos: end_position + (trailing || 0))
         end
@@ -128,18 +182,20 @@ module Flexr
       end
 
       def reference_rules?
+        return false unless @config.backend == :firstmatch
         return @reference_rules unless @reference_rules.nil?
 
-        @reference_rules = @lexer.class.__flexr_rules.any? do |rule|
+        @reference_rules = @rules.any? do |rule|
           rule.patterns.any? { |pattern| reference_pattern?(pattern) }
         end
       end
 
-      def fast_path?
-        return @fast_path unless @fast_path.nil?
+      def fast_path?(machine)
+        @fast_paths ||= {}
+        return @fast_paths[machine.dfa] if @fast_paths.key?(machine.dfa)
 
-        rules = @lexer.class.__flexr_rules
-        @fast_path = !@lexer.class.__flexr_config.options[:allow_empty_match] &&
+        rules = machine.dfa.rule_ids.map { |rule_index| @rules.fetch(rule_index) }
+        @fast_paths[machine.dfa] = !@config.options[:allow_empty_match] &&
           !reference_rules? && rules.none?(&:trailing) &&
           rules.none? { |rule| rule.pattern_conditions.any? { |condition| condition&.bol_only || condition&.end_anchor } }
       end
@@ -148,13 +204,13 @@ module Flexr
         return false unless pattern.is_a?(::Regexp)
         return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
 
-        @lexer.class.__flexr_config.options[:unicode] == true && @lexer.utf8_input? &&
+        @config.options[:unicode] == true && @lexer.utf8_input? &&
           pattern.source.match?(/\\[dDwWsS]/)
       end
 
       def scan_firstmatch(position)
         buffer = @lexer.buffer
-        @lexer.class.__flexr_rules.sort_by(&:index).each do |rule|
+        @rules.sort_by(&:index).each do |rule|
           next unless rule_active?(rule)
 
           matches = rule.patterns.each_with_index.filter_map do |pattern, pattern_index|
@@ -162,10 +218,11 @@ module Flexr
             next if condition.bol_only && !@lexer.beginning_of_line?
 
             regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
-            streamed_match(regexp, buffer, position, reference: reference_pattern?(regexp))
+            streamed_match(
+              regexp, buffer, position, reference: reference_pattern?(regexp),
+              limit: :token, rule: rule
+            )
               &.then { |match| [match, condition] }
-          rescue ArgumentError, RegexpError
-            nil
           end
           match, condition = matches.select { |candidate| candidate[0].begin(0).zero? }
             .max_by { |candidate| candidate[0][0].bytesize }
@@ -177,7 +234,7 @@ module Flexr
           trailing = trailing_length(rule, buffer, end_position)
           next if rule.trailing && trailing.nil?
 
-          ensure_token_size!(end_position, position)
+          ensure_token_size!(end_position, position, rule: rule)
           return reusable_match(rule, position, end_position, end_position + (trailing || 0))
         end
         nil
@@ -187,27 +244,22 @@ module Flexr
         return 0 unless rule.trailing
 
         regexp = rule.trailing
-        match = streamed_match(regexp, buffer, position, reference: reference_pattern?(regexp))
+        match = streamed_match(
+          regexp, buffer, position, reference: reference_pattern?(regexp),
+          limit: :lookahead, rule: rule
+        )
         return nil unless match&.begin(0)&.zero?
 
-        match[0].bytesize
-      rescue ArgumentError
-        nil
+        match[0].bytesize.tap { |size| @lexer.ensure_lookahead_size!(size, rule: rule) }
       end
 
-      def streamed_match(pattern, buffer, position, reference: false)
+      def streamed_match(pattern, buffer, position, limit:, rule:, reference: false)
         pattern = regexp_pattern(pattern)
         minimum = minimum_match_bytes(pattern)
         loop do
           subject, tail = stream_subject(buffer, position)
-          match = if reference
-            Unicode::ReferenceRegexp.match(
-              pattern, subject, encoding: @lexer.class.__flexr_config.encoding,
-              options: pattern.options, unicode: @lexer.class.__flexr_config.options[:unicode] == true
-            )
-          else
-            pattern.match(subject, 0)
-          end
+          match = match_stream_pattern(pattern, subject, reference: reference)
+          ensure_stream_limit!(limit, position, match[0].bytesize, rule) if match
           return match if match && match[0].bytesize < subject.bytesize
           return match if match && %i[eof invalid].include?(tail)
           if !match && subject.bytesize >= minimum && tail != :incomplete
@@ -215,12 +267,25 @@ module Flexr
             return nil unless first_byte && possible_first_byte?(pattern, first_byte)
             return nil if %i[eof invalid].include?(tail)
           end
+          ensure_stream_limit!(limit, position, subject.bytesize, rule) unless match
+          @lexer.consume_step!
           return match unless can_refill_match?(buffer, position, subject.bytesize, tail)
-        rescue ArgumentError, RegexpError
-          return nil
         end
       end
 
+      def match_stream_pattern(pattern, subject, reference:)
+        if reference
+          Unicode::ReferenceRegexp.match(
+            pattern, subject, encoding: @config.encoding,
+            options: pattern.options, unicode: @config.options[:unicode] == true
+          )
+        else
+          pattern.match(subject, 0)
+        end
+      rescue ArgumentError, RegexpError
+        nil
+      end
+
       def stream_subject(buffer, position)
         return [buffer.byteslice(position...buffer.bytesize).to_s.b, buffer.eof_loaded? ? :eof : :end] unless @lexer.utf8_input?
 
@@ -245,7 +310,7 @@ module Flexr
       def utf8_status(buffer, position)
         return [:eof, 0] if position >= buffer.bytesize
 
-        first = buffer.source.getbyte(position)
+        first = buffer.getbyte(position)
         return [:complete, 1] if first <= 0x7f
         return [:invalid, 1] unless first.between?(0xc2, 0xf4)
 
@@ -289,7 +354,7 @@ module Flexr
 
       def minimum_ast_bytes(node, ignorecase: false)
         case node
-        when Regexp::AST::Empty, Regexp::AST::Anchor then 0
+        when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then 0
         when Regexp::AST::ByteRange then 1
         when Regexp::AST::CodepointRange then utf8_length(node.lo)
         when Regexp::AST::CharClass
@@ -300,6 +365,7 @@ module Flexr
           ranges.map { |lo, _hi| utf8_length(lo) }.min || 1
         when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }
         when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }.min || 0
+        when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child, ignorecase: ignorecase)
         else minimum_unknown_bytes(node)
         end
       end
@@ -323,7 +389,7 @@ module Flexr
 
       def first_byte_ranges(node, options, binary: false)
         case node
-        when Regexp::AST::Empty, Regexp::AST::Anchor then []
+        when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then []
         when Regexp::AST::ByteRange then [[node.lo, node.hi]]
         when Regexp::AST::CodepointRange
           ranges = options.anybits?(::Regexp::IGNORECASE) ? Unicode::CaseFold.ranges(node.lo, node.hi) : [[node.lo, node.hi]]
@@ -349,7 +415,7 @@ module Flexr
           ranges
         when Regexp::AST::Alt
           node.children.flat_map { |child| first_byte_ranges(child, options, binary: binary) }
-        when Regexp::AST::Star
+        when Regexp::AST::Star, Regexp::AST::Repeat
           first_byte_ranges(node.child, options, binary: binary)
         else
           first_byte_fallback
@@ -363,6 +429,7 @@ module Flexr
       def nullable?(node)
         case node
         when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Star then true
+        when Regexp::AST::Repeat then node.minimum.zero? || nullable?(node.child)
         when Regexp::AST::Seq then node.children.all? { |child| nullable?(child) }
         when Regexp::AST::Alt then node.children.any? { |child| nullable?(child) }
         else false
@@ -400,7 +467,7 @@ module Flexr
 
       def consider_acceptances(machine, state, cursor, position, buffer, best)
         machine.dfa.accepts[state].each do |acceptance|
-          candidate = @lexer.class.__flexr_rules.fetch(acceptance.rule_index)
+          candidate = @rules.fetch(acceptance.rule_index)
           next if acceptance.bol_only && !@lexer.beginning_of_line?
           next unless @lexer.utf8_boundary?(cursor)
           next unless !acceptance.end_anchor || end_anchor_match?(buffer, cursor)
@@ -412,7 +479,7 @@ module Flexr
           next if best && total_end < best.total_end_pos
           next if best && total_end == best.total_end_pos && acceptance.rule_index > best.rule.index
 
-          ensure_token_size!(cursor, position)
+          ensure_token_size!(cursor, position, rule: candidate)
           best = @match ||= Match.new
           best.rule = candidate
           best.start_pos = position
@@ -423,7 +490,8 @@ module Flexr
       end
 
       def acceleration_enabled?
-        @lexer.class.__flexr_config.options.fetch(:accel, :auto) != :none
+        mode = @config.options.fetch(:accel, :auto)
+        mode != :none && !(mode == :auto && @auto_acceleration_off)
       end
 
       def reusable_match(rule, start_pos, end_pos, total_end_pos)
@@ -435,32 +503,63 @@ module Flexr
         @match
       end
 
-      def accelerate(region, buffer, position)
-        mode = @lexer.class.__flexr_config.options.fetch(:accel, :auto)
-        binary = buffer.source.b
-        match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)
-          scanner = StringScanner.new(binary)
-          scanner.pos = position
-          length = scanner.skip(region.regexp)
-          length && scanner.pos
-        else
-          region.regexp.match(binary, position)&.then { |match| match.begin(0) == position ? match.end(0) : nil }
+      def accelerate(region, buffer, position, token_rule:, token_start:)
+        mode = @config.options.fetch(:accel, :auto)
+        cursor = position
+        matched = false
+        loop do
+          if cursor >= buffer.bytesize
+            return matched ? cursor : nil if buffer.eof_loaded?
+            return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
+          end
+
+          segment = buffer.source
+          segment_position = cursor - buffer.base_offset
+          length = acceleration_length(region, segment, segment_position, mode)
+          record_auto_acceleration_miss(region) if mode == :auto && length.to_i < 8
+          return matched ? cursor : nil unless length&.positive?
+
+          matched = true
+          cursor += length
+          @lexer.consume_step!(length)
+          ensure_token_size!(cursor, token_start, rule: token_rule)
+          return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
         end
-        return match_end if match_end && match_end < buffer.bytesize
-        return match_end if match_end && buffer.eof_loaded?
-        return unless buffer.ensure_available?(buffer.bytesize + 1)
+      end
 
-        accelerate(region, buffer, position)
+      def acceleration_length(region, segment, position, mode)
+        if segment.encoding == Encoding::UTF_8 && region.utf8_regexp
+          regexp = region.utf8_regexp
+        else
+          segment = segment.b unless segment.encoding == Encoding::BINARY
+          regexp = region.regexp
+        end
+        if %i[strscan auto].include?(mode) && defined?(::StringScanner)
+          @acceleration_scanner ||= ::StringScanner.new("".b)
+          @acceleration_scanner.string = segment
+          @acceleration_scanner.pos = position
+          @acceleration_scanner.skip(regexp)
+        else
+          match = regexp.match(segment, position)
+          match&.begin(0) == position ? match.end(0) - position : nil
+        end
       rescue ArgumentError
         nil
       end
 
+      def record_auto_acceleration_miss(region)
+        @auto_acceleration_misses ||= Hash.new(0)
+        @auto_acceleration_misses[region] += 1
+        @auto_acceleration_miss_total = @auto_acceleration_miss_total.to_i + 1
+        @auto_acceleration_off = true if @auto_acceleration_miss_total >= 9
+        return if @auto_acceleration_misses[region] < 3
+
+        @disabled_auto_acceleration ||= {}
+        @disabled_auto_acceleration[region] = true
+      end
+
       def transition(dfa, state, byte)
-        if @lexer.class.__flexr_config.backend == :direct &&
-            @lexer.class.respond_to?(:__flexr_generated_direct_transition)
-          return @lexer.class.__flexr_generated_direct_transition(@lexer.state, state, byte)
-        end
-        return dfa.transition_direct(state, byte) if @lexer.class.__flexr_config.backend == :direct
+        return dfa.transition_direct(state, byte) if @config.backend == :direct
 
         dfa.transition(state, byte)
       end
@@ -474,7 +573,7 @@ module Flexr
 
           accepting = machine.dfa.accepts[region.state]
           next if accepting.any? do |acceptance|
-            rule = @lexer.class.__flexr_rules.fetch(acceptance.rule_index)
+            rule = @rules.fetch(acceptance.rule_index)
             acceptance.bol_only || acceptance.end_anchor || rule.trailing
           end
 
@@ -486,15 +585,23 @@ module Flexr
         buffer.eof?(position) || buffer.getbyte(position) == 0x0a
       end
 
-      def ensure_token_size!(end_position, position)
+      def ensure_token_size!(end_position, position, rule: nil)
         text_start = @lexer.more_text_start || position
-        @lexer.defer_token_size_check!(end_position - text_start)
+        @lexer.defer_token_size_check!(end_position - text_start, rule: rule)
+      end
+
+      def ensure_stream_limit!(limit, position, size, rule)
+        if limit == :lookahead
+          @lexer.ensure_lookahead_size!(size, rule: rule)
+        else
+          ensure_token_size!(position + size, position, rule: rule)
+        end
       end
 
       def rule_active?(rule)
         return true if rule.states.include?(:initial) && @lexer.state == :initial
 
-        state = @lexer.class.__flexr_config.states.fetch(@lexer.state)
+        state = @config.states.fetch(@lexer.state)
         return true if state.inclusive && rule.states.include?(:initial)
 
         rule.states.include?(@lexer.state)

data/lib/flexr/runtime.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "json"
+require "monitor"
+require_relative "version"
+require_relative "errors"
+require_relative "diagnostics"
+require_relative "action_resolver"
+require_relative "configuration"
+require_relative "ir"
+require_relative "unicode/version"
+require_relative "regexp/ast"
+require_relative "regexp/parser"
+require_relative "regexp/normalizer"
+require_relative "regexp/unsupported"
+require_relative "regexp/char_class"
+require_relative "unicode/utf8_splitter"
+require_relative "unicode/data"
+require_relative "unicode/property"
+require_relative "unicode/reference_regexp"
+require_relative "unicode/case_fold"
+require_relative "automaton/types"
+require_relative "automaton/byte_class_set"
+require_relative "automaton/nfa"
+require_relative "automaton/dfa"
+require_relative "automaton/compiler"
+require_relative "automaton/analysis"
+require_relative "automaton/minimizer"
+require_relative "automaton/backend_cost_model"
+require_relative "automaton/accel"
+require_relative "runtime/location"
+require_relative "runtime/token"
+require_relative "runtime/buffer"
+require_relative "runtime/errors"
+require_relative "runtime/interpreter"
+require_relative "runtime/core"
+require_relative "dsl"
+require_relative "lexer"
+require_relative "generated"
+
+module Flexr
+  class << self
+    def compile_pattern(pattern, options: {})
+      regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
+      encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
+      rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
+      state = IR::State.new(name: :initial, inclusive: true, id: 0)
+      spec = IR::Spec.new(
+        class_name: "Pattern", backend: :table, token_kind: :array,
+        encoding: encoding, options: options, declared_tokens: [],
+        states: { initial: state }, rules: [rule], eof_rules: {}, verbatim: nil
+      )
+      Automaton::Compiler.new(spec).compile.machines.fetch(:initial).dfa
+    end
+
+    def reference_pattern?(regexp, unicode: false)
+      return true if regexp.source.match?(/\\[pP]\{/) || regexp.source.match?(/\[:(?:\^)?[a-z]+:\]/)
+
+      unicode && regexp.encoding != Encoding::BINARY && regexp.source.match?(/\\[dDwWsS]/)
+    end
+
+    def parse_pattern(pattern, options: {})
+      regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
+      encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
+      ast = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
+                               unicode: options[:unicode] == true).parse
+      Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
+    end
+  end
+end

data/lib/flexr/source/passthrough.rb

@@ -3,29 +3,82 @@
 module Flexr
   module Source
     module Passthrough
+      ENCODING_COMMENT = /\A[ \t]*#.*(?:coding|encoding)\s*[:=]\s*[-A-Za-z0-9_]+/i
+      FROZEN_COMMENT = /\A[ \t]*#\s*frozen_string_literal\s*:\s*(?:true|false)\b/i
+
       module_function
 
       def remove_spans(source, spans, insertion:, payload:)
+        rewrite(source, spans.map { |start_offset, end_offset| [start_offset, end_offset, ""] },
+                insertion: insertion, payload: payload)
+      end
+
+      def rewrite(source, edits, insertion:, payload:)
         result = source.dup
-        top_level = spans.reject do |span|
-          spans.any? do |outer|
-            next false if outer.equal?(span)
+        normalized_edits = edits.map do |start_offset, end_offset, replacement|
+          start_offset = line_start_offset(source, start_offset) if replacement.empty? && start_offset != insertion
+          [start_offset, end_offset, replacement]
+        end
+        validate_non_overlapping!(normalized_edits)
+        adjusted_insertion = insertion + normalized_edits.sum do |start_offset, end_offset, replacement|
+          next 0 if start_offset >= insertion
 
-            outer.first <= span.first && span.last <= outer.last &&
-              (outer.first < span.first || span.last < outer.last)
-          end
+          replaced_size = [end_offset, insertion].min - start_offset
+          replacement.bytesize - replaced_size
         end
-        top_level.sort_by(&:first).reverse_each do |start_offset, end_offset|
-          result.slice!(start_offset...end_offset)
+        normalized_edits.sort_by(&:first).reverse_each do |start_offset, end_offset, replacement|
+          result[start_offset...end_offset] = replacement
         end
-        result.insert(insertion, payload)
+        result.insert(adjusted_insertion, payload)
         result
       end
 
+      def insert_after_preamble(source, payload)
+        offset = 0
+        line, next_offset = next_line(source, offset)
+        if line&.start_with?("#!")
+          offset = next_offset
+          line, next_offset = next_line(source, offset)
+        end
+
+        while line && magic_comment?(line)
+          offset = next_offset
+          line, next_offset = next_line(source, offset)
+        end
+
+        source.dup.insert(offset, payload)
+      end
+
       def indentation(source, offset)
         line_start = source.rindex("\n", offset - 1)
         source[(line_start ? line_start + 1 : 0)...offset].to_s[/\A[ \t]*/].to_s
       end
+
+      def line_start_offset(source, offset)
+        line_start = source.rindex("\n", offset - 1) unless offset.zero?
+        line_start = line_start ? line_start + 1 : 0
+        prefix = source.byteslice(line_start...offset)
+        prefix&.match?(/\A[ \t]*\z/) ? line_start : offset
+      end
+
+      def next_line(source, offset)
+        return [nil, offset] if offset >= source.bytesize
+
+        ending = source.index("\n", offset)
+        ending ? [source.byteslice(offset..ending), ending + 1] : [source.byteslice(offset..), source.bytesize]
+      end
+
+      def magic_comment?(line)
+        ENCODING_COMMENT.match?(line) || FROZEN_COMMENT.match?(line)
+      end
+
+      def validate_non_overlapping!(edits)
+        edits.sort_by(&:first).each_cons(2) do |left, right|
+          next if left[1] <= right[0]
+
+          raise ArgumentError, "overlapping source edits"
+        end
+      end
     end
   end
 end