flexr 1.0.0 → 1.1.0

data/lib/flexr/regexp/tokenizer.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 # Generated by flexr. DO NOT EDIT.
 # source: lib/flexr/regexp/tokenizer.flexr.rb
-# spec-digest: sha256:3129b1c3e05f1c0680e3176ad96eb4ca7ad77fdd7168c47228351e47d79ad8f4
+# spec-digest: sha256:ef176b30fa1aec18f8474cdc44dfc9004802e14e4d7ed89e9a20890930811c63
+# artifact-schema: 1
+# runtime-abi: 1
 # unicode: 15.1.0
 # backend: table
 # compiled: true
@@ -15,48 +17,141 @@ module Flexr
     # This specification is intentionally small: it is a dogfood target for
     # the source lexer path, not the parser's implementation.
     class SourceLexer < Flexr::Lexer
-      Flexr::Generated.install_compiled!(self, { rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: proc{ emit :PROPERTY, text.byteslice(3...-1) }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: proc{ emit :ESCAPE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: proc{ emit :CHAR_CLASS, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: proc{ emit :QUANTIFIER, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: proc{ emit :ALTERNATION, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_OPEN, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_CLOSE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: proc{ emit :ANCHOR, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: proc{ emit :DOT, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: proc{ emit :LITERAL, text }, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: {  }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
+      Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 1, compiler_version: "1.1.0", runtime_abi_version: 1, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: proc{ emit :PROPERTY, text.byteslice(3...-1) }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: proc{ emit :ESCAPE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: proc{ emit :CHAR_CLASS, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: proc{ emit :QUANTIFIER, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: proc{ emit :ALTERNATION, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_OPEN, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_CLOSE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: proc{ emit :ANCHOR, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: proc{ emit :DOT, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: proc{ emit :LITERAL, text }, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: {  }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
       def scan_one
-        return Flexr::Runtime::Interpreter.new(self).scan unless __flexr_generated_fast_path?
-        machine = self.class.__flexr_compiled.machines.fetch(state)
-        dfa = machine.dfa
+        if @__flexr_generated_scan_state != @state
+          machine = self.class.__flexr_compiled.machines.fetch(@state)
+          @__flexr_generated_scan_state = @state
+          @__flexr_generated_scan_dfa = machine.dfa
+          @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
+          @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
+        end
+        dfa = @__flexr_generated_scan_dfa
+        return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
         position = byte_pos
         return nil unless valid_utf8_at?(position)
+        return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
         cursor = position
         current = dfa.start
         direct = dfa.direct
-        source = buffer.source
+        direct_nxt = direct&.fetch(:nxt)
+        direct_classes = direct&.fetch(:classes)
+        ec = dfa.ec
+        source = buffer.stable_source
+        guarded_steps = scan_steps_guarded?
+        token_start = more_text_start || position
+        token_limit = max_token_size
+        check_token_limit = token_limit_required?
         best = nil
-        accelerate = self.class.__flexr_config.options.fetch(:accel, :auto) != :none && !utf8_input?
-        while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
+        accel_mode = @accel_mode
+        accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
+        while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
           if accelerate
             region = __flexr_generated_acceleration_region(dfa, current)
-            accelerated_end = __flexr_generated_accelerate(region, cursor) if region
+            acceptance = dfa.accepts[current].first
+            token_rule = acceptance&.rule_index
+            accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
             if accelerated_end && accelerated_end > cursor
               cursor = accelerated_end
               best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
               next
             end
           end
-          byte = source.getbyte(cursor)
-          current = if direct
-            class_id = dfa.ec[byte]
-            value = direct[:nxt][(current * direct[:classes]) + class_id]
+          byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
+          consume_step! if guarded_steps
+          current = if direct_nxt
+            class_id = ec[byte]
+            value = direct_nxt[(current * direct_classes) + class_id]
             value >= 0 ? value : nil
           else
             dfa.transition(current, byte)
           end
           break unless current
           cursor += 1
-          acceptance = dfa.accepts[current].first
-          if acceptance
-            rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
+          resource_acceptance = dfa.accepts[current].first
+          resource_rule = resource_acceptance&.rule_index
+          if check_token_limit
+            size = cursor - token_start
+            defer_token_size_check!(size, rule: resource_rule) if size > token_limit
+          end
+          best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
+        end
+        best
+      end
+      def __flexr_generated_scan_simple(dfa, position)
+        current = dfa.start
+        direct = dfa.direct
+        direct_nxt = direct&.fetch(:nxt)
+        direct_classes = direct&.fetch(:classes)
+        ec = dfa.ec
+        accepts = dfa.accepts
+        rules = self.class.__flexr_rules
+        source = buffer.stable_source
+        guarded_steps = scan_steps_guarded?
+        token_start = more_text_start || position
+        token_limit = max_token_size
+        check_token_limit = token_limit_required?
+        cursor = position
+        best = nil
+        accel_mode = @accel_mode
+        accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
+        if source && !guarded_steps && !check_token_limit && !accelerate
+          while cursor < source.bytesize
+            byte = source.getbyte(cursor)
+            current = if direct_nxt
+              class_id = ec[byte]
+              value = direct_nxt[(current * direct_classes) + class_id]
+              value >= 0 ? value : nil
+            else
+              dfa.transition(current, byte)
+            end
+            break unless current
+            cursor += 1
+            acceptance = accepts[current].first
+            next unless acceptance
+            rule = rules.fetch(acceptance.rule_index)
             best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
             best.rule = rule
             best.start_pos = position
             best.end_pos = cursor
             best.total_end_pos = cursor
           end
+          return best
+        end
+        while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
+          if accelerate
+            region = __flexr_generated_acceleration_region(dfa, current)
+            acceptance = accepts[current].first
+            accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
+            if accelerated_end && accelerated_end > cursor
+              cursor = accelerated_end
+              best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
+              next
+            end
+          end
+          byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
+          consume_step! if guarded_steps
+          current = if direct_nxt
+            class_id = ec[byte]
+            value = direct_nxt[(current * direct_classes) + class_id]
+            value >= 0 ? value : nil
+          else
+            dfa.transition(current, byte)
+          end
+          break unless current
+          cursor += 1
+          acceptance = accepts[current].first
+          if check_token_limit
+            size = cursor - token_start
+            defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
+          end
+          next unless acceptance
+          rule = rules.fetch(acceptance.rule_index)
+          best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
+          best.rule = rule
+          best.start_pos = position
+          best.end_pos = cursor
+          best.total_end_pos = cursor
         end
         best
       end
@@ -65,44 +160,75 @@ module Flexr
         regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
         region = regions[state]
         return unless region
+        return if @__flexr_generated_disabled_auto_accel&.key?(region)
+        return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
         return if dfa.accepts[state].any? do |acceptance|
           rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
           acceptance.bol_only || acceptance.end_anchor || rule.trailing
         end
         region
       end
-      def __flexr_generated_accelerate(region, position)
-        binary = buffer.source.b
-        mode = self.class.__flexr_config.options.fetch(:accel, :auto)
-        match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)
-          scanner = StringScanner.new(binary)
-          scanner.pos = position
-          length = scanner.skip(region.regexp)
-          length && scanner.pos
-        else
-          match = region.regexp.match(binary, position)
-          match && match.begin(0) == position ? match.end(0) : nil
-        end
-        return match_end if match_end && match_end < buffer.bytesize
-        return match_end if match_end && buffer.eof_loaded?
-        return unless buffer.ensure_available?(buffer.bytesize + 1)
-        __flexr_generated_accelerate(region, position)
-      rescue ArgumentError
-        nil
-      end
-      def __flexr_generated_fast_path?
-        return @__flexr_generated_fast_path if defined?(@__flexr_generated_fast_path)
-        @__flexr_generated_fast_path = if self.class.__flexr_config.backend == :firstmatch ||
-                                        self.class.__flexr_config.options[:allow_empty_match] == true
-          false
-        else
-          self.class.__flexr_rules.none? do |rule|
-            !rule.trailing.nil? || rule.patterns.any? do |pattern|
-              next false unless pattern.is_a?(Regexp)
-            next true if pattern.source.match?(/\\[pP]\\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
-              self.class.__flexr_config.options[:unicode] == true && utf8_input? && pattern.source.match?(/\\[dDwWsS]/)
+      def __flexr_generated_accelerate(region, position, token_rule, token_start)
+        mode = @accel_mode
+        cursor = position
+        matched = false
+        loop do
+          if cursor >= buffer.bytesize
+            return matched ? cursor : nil if buffer.eof_loaded?
+            return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
+          end
+          segment = buffer.source
+          segment_position = cursor - buffer.base_offset
+          if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
+            regexp = region.utf8_regexp
+          else
+            segment = segment.b unless segment.encoding == ::Encoding::BINARY
+            regexp = region.regexp
+          end
+          length = begin
+            if %i[strscan auto].include?(mode) && defined?(::StringScanner)
+              @__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
+              @__flexr_generated_accel_scanner.string = segment
+              @__flexr_generated_accel_scanner.pos = segment_position
+              @__flexr_generated_accel_scanner.skip(regexp)
+            else
+              match = regexp.match(segment, segment_position)
+              match && match.begin(0) == segment_position ? match.end(0) - segment_position : nil
             end
+          rescue ArgumentError
+            nil
           end
+          __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
+          return matched ? cursor : nil unless length&.positive?
+          matched = true
+          cursor += length
+          consume_step!(length)
+          defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
+          return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
+        end
+      end
+      def __flexr_generated_record_auto_accel_miss(region)
+        @__flexr_generated_auto_accel_misses ||= Hash.new(0)
+        @__flexr_generated_auto_accel_misses[region] += 1
+        @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
+        @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
+        return if @__flexr_generated_auto_accel_misses[region] < 3
+        @__flexr_generated_disabled_auto_accel ||= {}
+        @__flexr_generated_disabled_auto_accel[region] = true
+      end
+      def __flexr_generated_fast_path?(dfa)
+        @__flexr_generated_fast_paths ||= {}
+        return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
+        rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
+        @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
+          self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
+      end
+      def __flexr_generated_simple_fast_path?(dfa)
+        @__flexr_generated_simple_fast_paths ||= {}
+        return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
+        rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
+        @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
+          rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
         end
       end
       def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
@@ -110,7 +236,6 @@ module Flexr
           next if acceptance.bol_only && !beginning_of_line?
           next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
           rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
-          defer_token_size_check!(cursor - start_position)
           next if best && cursor < best.total_end_pos
           next if best && cursor == best.total_end_pos &&
             rule.index > best.rule.index
@@ -126,25 +251,25 @@ module Flexr
         case rule.index
         when 0
         when 1
-          emit :PROPERTY, text.byteslice(3...-1)
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 2
-          emit :ESCAPE, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 3
-          emit :CHAR_CLASS, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 4
-          emit :QUANTIFIER, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 5
-          emit :ALTERNATION, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 6
-          emit :GROUP_OPEN, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 7
-          emit :GROUP_CLOSE, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 8
-          emit :ANCHOR, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 9
-          emit :DOT, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         when 10
-          emit :LITERAL, text
+          instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
         else
           instance_exec(&rule.action)
         end
@@ -153,16 +278,16 @@ module Flexr
 
 
 
-
-
-
-
-
-
-
-
-
-
+      __flexr_bind_generated_action(1, proc{ emit :PROPERTY, text.byteslice(3...-1) })
+      __flexr_bind_generated_action(2, proc{ emit :ESCAPE, text })
+      __flexr_bind_generated_action(3, proc{ emit :CHAR_CLASS, text })
+      __flexr_bind_generated_action(4, proc{ emit :QUANTIFIER, text })
+      __flexr_bind_generated_action(5, proc{ emit :ALTERNATION, text })
+      __flexr_bind_generated_action(6, proc{ emit :GROUP_OPEN, text })
+      __flexr_bind_generated_action(7, proc{ emit :GROUP_CLOSE, text })
+      __flexr_bind_generated_action(8, proc{ emit :ANCHOR, text })
+      __flexr_bind_generated_action(9, proc{ emit :DOT, text })
+      __flexr_bind_generated_action(10, proc{ emit :LITERAL, text })
     end
   end
 end

data/lib/flexr/runtime/buffer.rb

@@ -5,31 +5,65 @@ module Flexr
     class Buffer
       DEFAULT_CHUNK_SIZE = 64 * 1024
 
-      attr_reader :source
+      attr_reader :base_offset, :max_buffer_size
 
-      def initialize(input, chunk_size: DEFAULT_CHUNK_SIZE)
+      def initialize(input, chunk_size: DEFAULT_CHUNK_SIZE, max_buffer_size: 64 * 1024 * 1024,
+                     retain_input: true, filename: nil)
         raise ArgumentError, "chunk_size must be positive" unless chunk_size.to_i.positive?
+        raise ArgumentError, "max_buffer_size must be non-negative" if max_buffer_size.to_i.negative?
 
         @chunk_size = chunk_size.to_i
+        @max_buffer_size = max_buffer_size.to_i
+        @retain_input = retain_input
+        @filename = filename
+        @base_offset = 0
+        @window_start = 0
         @io = input.is_a?(String) ? nil : input
         raise ArgumentError, "input must be a String or IO" if @io && !@io.respond_to?(:read)
 
         @source = input.is_a?(String) ? input : String.new(encoding: Encoding::BINARY)
         @eof = @io.nil?
+        ensure_buffer_size!(@source.bytesize)
       end
 
       def bytesize
-        source.bytesize
+        return @source.bytesize if @window_start.zero? && base_offset.zero?
+
+        base_offset + retained_bytesize
+      end
+
+      def retained_bytesize
+        @source.bytesize - @window_start
+      end
+
+      def source
+        return @source if @window_start.zero?
+
+        @source.byteslice(@window_start..).to_s
+      end
+
+      def stable_source
+        @source if @retain_input && @window_start.zero? && base_offset.zero?
       end
 
       def getbyte(position)
+        return @source.getbyte(position) if @window_start.zero? && base_offset.zero? && position < @source.bytesize
+
         ensure_available?(position + 1)
-        source.getbyte(position)
+        @source.getbyte(storage_position(position))
       end
 
       def byteslice(range, length = nil)
         ensure_range(range, length)
-        length ? source.byteslice(range, length) : source.byteslice(range)
+        if @window_start.zero? && base_offset.zero?
+          return length ? @source.byteslice(range, length) : @source.byteslice(range)
+        end
+
+        if length
+          @source.byteslice(storage_position(range), length)
+        else
+          @source.byteslice(storage_range(range))
+        end
       end
 
       def ensure_available?(end_position)
@@ -37,7 +71,7 @@ module Flexr
         return false if @eof
 
         while bytesize < end_position && !@eof
-          chunk = @io.read(@chunk_size)
+          chunk = read_chunk
           if chunk.nil? || chunk.empty?
             @eof = true
             break
@@ -63,11 +97,11 @@ module Flexr
       end
 
       def utf8_boundary?(position)
-        return false if position.negative?
-        return true if position.zero?
+        return false if position < base_offset
+        return true if position == base_offset
         return true unless ensure_available?(position)
 
-        following = source.getbyte(position)
+        following = getbyte(position)
         following.nil? || (following & 0xc0) != 0x80
       end
 
@@ -89,6 +123,31 @@ module Flexr
         byteslice(position, length).dup.force_encoding(Encoding::UTF_8).valid_encoding?
       end
 
+      def utf8_character_length(position)
+        first = getbyte(position)
+        return 0 unless first
+        return 1 if first <= 0x7f
+
+        length = if first <= 0xdf
+          2
+        elsif first <= 0xef
+          3
+        else
+          4
+        end
+        valid_utf8_at?(position) ? length : 1
+      end
+
+      def discard_before(position)
+        return if @retain_input || position <= base_offset
+
+        ending = [position, bytesize].min
+        removed = ending - base_offset
+        @window_start += removed
+        @base_offset = ending
+        compact_storage!
+      end
+
       private
 
       def ensure_range(range, length)
@@ -107,6 +166,53 @@ module Flexr
         ending = range.end + (range.exclude_end? ? 0 : 1)
         ensure_available?(ending)
       end
+
+      def storage_position(position)
+        local = position - base_offset
+        return @window_start + local unless local.negative?
+
+        raise RangeError, "byte #{position} has been discarded from the streaming buffer"
+      end
+
+      def storage_range(range)
+        return storage_position(range) unless range.is_a?(Range)
+
+        ending = range.end
+        Range.new(storage_position(range.begin), ending.nil? ? nil : storage_position(ending), range.exclude_end?)
+      end
+
+      def ensure_buffer_size!(size)
+        return if size <= max_buffer_size
+
+        raise BufferTooLargeError.new(filename: @filename, byte_pos: bytesize)
+      end
+
+      def read_chunk
+        capacity = max_buffer_size - retained_bytesize
+        if capacity.zero?
+          probe = @io.read(1)
+          if probe.nil? || probe.empty?
+            @eof = true
+            return nil
+          end
+
+          raise BufferTooLargeError.new(filename: @filename, byte_pos: bytesize)
+        end
+
+        @io.read([@chunk_size, capacity].min)
+      end
+
+      def compact_storage!
+        return if @window_start.zero?
+
+        if @window_start == @source.bytesize
+          @source = String.new(encoding: @source.encoding)
+          @window_start = 0
+        elsif @window_start >= @chunk_size || @window_start * 2 >= @source.bytesize
+          @source = @source.byteslice(@window_start..)
+          @window_start = 0
+        end
+      end
     end
   end
 end