flexr 1.0.0

data/lib/flexr/regexp/tokenizer.flexr.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "flexr"
+
+module Flexr
+  module Regexp
+    # This specification is intentionally small: it is a dogfood target for
+    # the source lexer path, not the parser's implementation.
+    class SourceLexer < Flexr::Lexer
+      encoding Encoding::BINARY
+      emits :PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION,
+        :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL
+
+      rule(/[ \t\r\n]+/, skip: true)
+      rule(/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/) { emit :PROPERTY, text.byteslice(3...-1) }
+      rule(/\\./) { emit :ESCAPE, text }
+      rule(/\[[^\]\n]*\]/) { emit :CHAR_CLASS, text }
+      rule(/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/) { emit :QUANTIFIER, text }
+      rule(/\|/) { emit :ALTERNATION, text }
+      rule(/\(/) { emit :GROUP_OPEN, text }
+      rule(/\)/) { emit :GROUP_CLOSE, text }
+      rule(/\^|\$/) { emit :ANCHOR, text }
+      rule(/\./) { emit :DOT, text }
+      rule(/[^\\\[\]().|?*+{}^$ \t\r\n]/) { emit :LITERAL, text }
+    end
+  end
+end

data/lib/flexr/regexp/tokenizer.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+# Generated by flexr. DO NOT EDIT.
+# source: lib/flexr/regexp/tokenizer.flexr.rb
+# spec-digest: sha256:3129b1c3e05f1c0680e3176ad96eb4ca7ad77fdd7168c47228351e47d79ad8f4
+# unicode: 15.1.0
+# backend: table
+# compiled: true
+# eval: false
+# standalone: false
+
+require "flexr"
+
+module Flexr
+  module Regexp
+    # This specification is intentionally small: it is a dogfood target for
+    # the source lexer path, not the parser's implementation.
+    class SourceLexer < Flexr::Lexer
+      Flexr::Generated.install_compiled!(self, { rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: proc{ emit :PROPERTY, text.byteslice(3...-1) }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: proc{ emit :ESCAPE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: proc{ emit :CHAR_CLASS, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: proc{ emit :QUANTIFIER, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: proc{ emit :ALTERNATION, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_OPEN, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_CLOSE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: proc{ emit :ANCHOR, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: proc{ emit :DOT, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: proc{ emit :LITERAL, text }, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: {  }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
+      def scan_one
+        return Flexr::Runtime::Interpreter.new(self).scan unless __flexr_generated_fast_path?
+        machine = self.class.__flexr_compiled.machines.fetch(state)
+        dfa = machine.dfa
+        position = byte_pos
+        return nil unless valid_utf8_at?(position)
+        cursor = position
+        current = dfa.start
+        direct = dfa.direct
+        source = buffer.source
+        best = nil
+        accelerate = self.class.__flexr_config.options.fetch(:accel, :auto) != :none && !utf8_input?
+        while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
+          if accelerate
+            region = __flexr_generated_acceleration_region(dfa, current)
+            accelerated_end = __flexr_generated_accelerate(region, cursor) if region
+            if accelerated_end && accelerated_end > cursor
+              cursor = accelerated_end
+              best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
+              next
+            end
+          end
+          byte = source.getbyte(cursor)
+          current = if direct
+            class_id = dfa.ec[byte]
+            value = direct[:nxt][(current * direct[:classes]) + class_id]
+            value >= 0 ? value : nil
+          else
+            dfa.transition(current, byte)
+          end
+          break unless current
+          cursor += 1
+          acceptance = dfa.accepts[current].first
+          if acceptance
+            rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
+            best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
+            best.rule = rule
+            best.start_pos = position
+            best.end_pos = cursor
+            best.total_end_pos = cursor
+          end
+        end
+        best
+      end
+      def __flexr_generated_acceleration_region(dfa, state)
+        @__flexr_generated_accel_regions ||= {}
+        regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
+        region = regions[state]
+        return unless region
+        return if dfa.accepts[state].any? do |acceptance|
+          rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
+          acceptance.bol_only || acceptance.end_anchor || rule.trailing
+        end
+        region
+      end
+      def __flexr_generated_accelerate(region, position)
+        binary = buffer.source.b
+        mode = self.class.__flexr_config.options.fetch(:accel, :auto)
+        match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)
+          scanner = StringScanner.new(binary)
+          scanner.pos = position
+          length = scanner.skip(region.regexp)
+          length && scanner.pos
+        else
+          match = region.regexp.match(binary, position)
+          match && match.begin(0) == position ? match.end(0) : nil
+        end
+        return match_end if match_end && match_end < buffer.bytesize
+        return match_end if match_end && buffer.eof_loaded?
+        return unless buffer.ensure_available?(buffer.bytesize + 1)
+        __flexr_generated_accelerate(region, position)
+      rescue ArgumentError
+        nil
+      end
+      def __flexr_generated_fast_path?
+        return @__flexr_generated_fast_path if defined?(@__flexr_generated_fast_path)
+        @__flexr_generated_fast_path = if self.class.__flexr_config.backend == :firstmatch ||
+                                        self.class.__flexr_config.options[:allow_empty_match] == true
+          false
+        else
+          self.class.__flexr_rules.none? do |rule|
+            !rule.trailing.nil? || rule.patterns.any? do |pattern|
+              next false unless pattern.is_a?(Regexp)
+            next true if pattern.source.match?(/\\[pP]\\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
+              self.class.__flexr_config.options[:unicode] == true && utf8_input? && pattern.source.match?(/\\[dDwWsS]/)
+            end
+          end
+        end
+      end
+      def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
+        dfa.accepts[state].each do |acceptance|
+          next if acceptance.bol_only && !beginning_of_line?
+          next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
+          rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
+          defer_token_size_check!(cursor - start_position)
+          next if best && cursor < best.total_end_pos
+          next if best && cursor == best.total_end_pos &&
+            rule.index > best.rule.index
+          best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
+          best.rule = rule
+          best.start_pos = start_position
+          best.end_pos = cursor
+          best.total_end_pos = cursor
+        end
+        best
+      end
+      def __flexr_generated_execute(rule)
+        case rule.index
+        when 0
+        when 1
+          emit :PROPERTY, text.byteslice(3...-1)
+        when 2
+          emit :ESCAPE, text
+        when 3
+          emit :CHAR_CLASS, text
+        when 4
+          emit :QUANTIFIER, text
+        when 5
+          emit :ALTERNATION, text
+        when 6
+          emit :GROUP_OPEN, text
+        when 7
+          emit :GROUP_CLOSE, text
+        when 8
+          emit :ANCHOR, text
+        when 9
+          emit :DOT, text
+        when 10
+          emit :LITERAL, text
+        else
+          instance_exec(&rule.action)
+        end
+      end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    end
+  end
+end

data/lib/flexr/regexp/unsupported.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Regexp
+    Unsupported = Flexr::UnsupportedRegexpError
+  end
+end

data/lib/flexr/runtime/buffer.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Runtime
+    class Buffer
+      DEFAULT_CHUNK_SIZE = 64 * 1024
+
+      attr_reader :source
+
+      def initialize(input, chunk_size: DEFAULT_CHUNK_SIZE)
+        raise ArgumentError, "chunk_size must be positive" unless chunk_size.to_i.positive?
+
+        @chunk_size = chunk_size.to_i
+        @io = input.is_a?(String) ? nil : input
+        raise ArgumentError, "input must be a String or IO" if @io && !@io.respond_to?(:read)
+
+        @source = input.is_a?(String) ? input : String.new(encoding: Encoding::BINARY)
+        @eof = @io.nil?
+      end
+
+      def bytesize
+        source.bytesize
+      end
+
+      def getbyte(position)
+        ensure_available?(position + 1)
+        source.getbyte(position)
+      end
+
+      def byteslice(range, length = nil)
+        ensure_range(range, length)
+        length ? source.byteslice(range, length) : source.byteslice(range)
+      end
+
+      def ensure_available?(end_position)
+        return true if end_position <= bytesize
+        return false if @eof
+
+        while bytesize < end_position && !@eof
+          chunk = @io.read(@chunk_size)
+          if chunk.nil? || chunk.empty?
+            @eof = true
+            break
+          end
+
+          @source.force_encoding(chunk.encoding) if @source.empty?
+          @source << chunk
+        end
+        bytesize >= end_position
+      end
+
+      def read_to_end
+        ensure_available?(Float::INFINITY)
+        source
+      end
+
+      def eof_loaded?
+        @eof
+      end
+
+      def eof?(position)
+        !ensure_available?(position + 1)
+      end
+
+      def utf8_boundary?(position)
+        return false if position.negative?
+        return true if position.zero?
+        return true unless ensure_available?(position)
+
+        following = source.getbyte(position)
+        following.nil? || (following & 0xc0) != 0x80
+      end
+
+      def valid_utf8_at?(position)
+        first = getbyte(position)
+        return false unless first
+        return true if first <= 0x7f
+
+        length = if first.between?(0xc2, 0xdf)
+          2
+        elsif first.between?(0xe0, 0xef)
+          3
+        elsif first.between?(0xf0, 0xf4)
+          4
+        end
+        return false unless length
+        return false unless ensure_available?(position + length)
+
+        byteslice(position, length).dup.force_encoding(Encoding::UTF_8).valid_encoding?
+      end
+
+      private
+
+      def ensure_range(range, length)
+        if length
+          ensure_available?(range + length)
+          return
+        end
+
+        unless range.is_a?(Range)
+          ensure_available?(range + 1)
+          return
+        end
+
+        return read_to_end if range.end.nil?
+
+        ending = range.end + (range.exclude_end? ? 0 : 1)
+        ensure_available?(ending)
+      end
+    end
+  end
+end

data/lib/flexr/runtime/core.rb

@@ -0,0 +1,388 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Runtime
+    def initialize(input, filename: nil, error_mode: :raise, max_token_size: 16 * 1024 * 1024,
+                   max_state_stack: 1024, chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
+      raise ArgumentError, "max_token_size must be non-negative" if max_token_size.to_i.negative?
+      raise ArgumentError, "max_state_stack must be non-negative" if max_state_stack.to_i.negative?
+
+      self.class.compile!
+      @buffer = Runtime::Buffer.new(input, chunk_size: chunk_size)
+      @input = input.is_a?(String) ? input : nil
+      @line_tracking_needed = !@input || input.include?("\n")
+      @filename = filename
+      @error_mode = error_mode
+      @max_token_size = max_token_size.to_i
+      @max_state_stack = max_state_stack.to_i
+      @position = 0
+      @line = 1
+      @state = :initial
+      @state_stack = []
+      @pending = nil
+      @matched = nil
+      @match_start = 0
+      @match_end = 0
+      @text_start = 0
+      @text_line = 1
+      @bol = true
+      @more_start = nil
+      @more_line = nil
+      @more_requested = false
+      @candidate_token_size = 0
+      @eof_fired_states = {}
+      @on_error = nil
+      @halted = false
+      @interpreter = nil
+    end
+
+    attr_reader :filename, :error_mode, :buffer, :max_token_size
+
+    attr_writer :on_error
+
+    def input
+      @buffer.source
+    end
+
+    def next_token
+      return nil if @halted
+
+      loop do
+        return nil if @halted
+
+        if eof? && @pending.nil?
+          eof_action = self.class.__flexr_spec.eof_rules[@state]
+          if eof_action && !@eof_fired_states[@state]
+            @eof_fired_states[@state] = true
+            @match_start = @position
+            @match_end = @position
+            @text_start = @position
+            @text_line = @line
+            @matched = nil
+            instance_exec(&eof_action)
+            token = @pending
+            @pending = nil
+            return token if token
+            next
+          end
+          return nil
+        end
+
+        match = if generated_runtime? && respond_to?(:scan_one, true)
+          scan_one
+        else
+          (@interpreter ||= Runtime::Interpreter.new(self)).scan
+        end
+        unless match
+          unless eof?
+            token = handle_unmatched_byte
+            next unless token
+
+            return token
+          end
+          return nil
+        end
+        @match_start = match.start_pos
+        @match_end = match.end_pos
+        if @more_start
+          @text_start = @more_start
+          @text_line = @more_line
+        else
+          @text_start = @match_start
+          @text_line = @line
+        end
+        @matched = nil
+        @more_requested = false
+        @position = match.end_pos
+        empty_match = match.end_pos == match.start_pos
+        execute(match.rule)
+        finalize_more
+        force_empty_match_progress! if empty_match && self.class.__flexr_config.options[:allow_empty_match]
+        ensure_token_size!
+        update_position
+        token = @pending
+        @pending = nil
+        return token if token
+      end
+    end
+
+    def each_token
+      return enum_for(__method__) unless block_given?
+
+      loop do
+        token = next_token
+        break unless token
+
+        if self.class.__flexr_config.token_kind == :yield
+          yield(*token)
+        else
+          yield token
+        end
+      end
+      self
+    end
+
+    def tokens
+      result = []
+      while (token = next_token)
+        result << token
+      end
+      result
+    end
+
+    def racc_next_token
+      token = next_token
+      token ? [token[0], token[1]] : [false, "$end"]
+    end
+
+    def text
+      return @matched if @matched
+
+      @matched = @buffer.byteslice(@text_start...@match_end)
+    end
+
+    def text_bytesize
+      @match_end - @text_start
+    end
+
+    def byte_pos
+      @position
+    end
+
+    def more_text_start
+      @more_start
+    end
+
+    def defer_token_size_check!(size)
+      @candidate_token_size = size if size > @candidate_token_size
+    end
+
+    def utf8_input?
+      self.class.__flexr_config.encoding == Encoding::UTF_8
+    end
+
+    def valid_utf8_at?(position)
+      !utf8_input? || @buffer.valid_utf8_at?(position)
+    end
+
+    def utf8_boundary?(position)
+      !utf8_input? || @buffer.utf8_boundary?(position)
+    end
+
+    def lineno
+      @line
+    end
+
+    alias line lineno
+
+    def state
+      @state
+    end
+
+    def beginning_of_line?
+      @bol
+    end
+
+    def binary_input
+      @buffer.source.b
+    end
+
+    def emit(type, value = text)
+      @pending = case self.class.__flexr_config.token_kind
+      when :struct
+        Runtime::Token.new(type: type, value: value, location: last_location)
+      else
+        [type, value]
+      end
+    end
+
+    def skip
+      @pending = nil
+    end
+
+    def error!(message)
+      error = LexError.new(message, filename: @filename, byte_pos: @match_start, line: @line, text: text)
+      if @on_error
+        action = @on_error.call(error)
+        return @pending = nil if action == :skip
+        raise error if action == :raise
+        if action == :halt
+          @halted = true
+          return @pending = nil
+        end
+        return emit(:error, text) if action == :token
+      end
+      case @error_mode
+      when :token
+        emit(:error, text)
+      when :panic
+        @pending = nil
+      else
+        raise error
+      end
+    end
+
+    def echo
+      emit(nil, text)
+    end
+
+    def reject
+      diagnostic = Diagnostics.error(
+        "FLEXR-E013", "reject is not supported by flexr",
+        help: "use a state transition and less(n) to express the fallback"
+      )
+      raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+    end
+
+    def push(name)
+      ensure_state!(name)
+      if @state_stack.length >= @max_state_stack
+        raise Runtime::StateStackOverflowError.new(
+          "state stack exceeds max_state_stack (#{@max_state_stack})",
+          filename: @filename, byte_pos: @position, line: @line, text: text
+        )
+      end
+
+      @state_stack << @state
+      @state = name.to_sym
+    end
+
+    alias push_state push
+
+    def pop
+      @state = @state_stack.pop || :initial
+    end
+
+    alias pop_state pop
+
+    def begin_state(name)
+      ensure_state!(name)
+      @state = name.to_sym
+    end
+
+    alias state= begin_state
+
+    def less(count)
+      matched_bytes = @match_end - @match_start
+      raise ArgumentError, "less must not exceed matched bytes" if count.negative? || count > matched_bytes
+
+      @position = @match_start + count
+      @match_end = @position
+      @matched = nil
+    end
+
+    def more
+      @more_requested = true
+    end
+
+    def last_location
+      line_begin = @text_line || @line
+      Runtime::Location.new(
+        filename: @filename, byte_begin: @text_start, byte_end: @match_end,
+        line_begin: line_begin, line_end: line_begin + text.to_s.b.count("\n"),
+        column_values: [column_at(@text_start), column_at(@match_end)],
+        eager_columns: self.class.__flexr_config.options[:eager_columns] == true
+      )
+    end
+
+    private
+
+    def execute(rule)
+      return __flexr_generated_execute(rule) if generated_runtime? && respond_to?(:__flexr_generated_execute, true)
+
+      case rule.action
+      when :skip
+        nil
+      when Array
+        emit(rule.action[1], text)
+      else
+        instance_exec(&rule.action)
+      end
+    end
+
+    def finalize_more
+      if @more_requested
+        @more_start = @text_start
+        @more_line = @text_line
+      else
+        @more_start = nil
+        @more_line = nil
+      end
+      @more_requested = false
+    end
+
+    def generated_runtime?
+      self.class.respond_to?(:__flexr_generated?) && self.class.__flexr_generated?
+    end
+
+    def ensure_token_size!
+      actual_size = @match_end - @text_start
+      @candidate_token_size = 0
+      return if actual_size <= @max_token_size
+
+      raise Runtime::TokenTooLargeError
+    end
+
+    def force_empty_match_progress!
+      return if eof?
+
+      byte = @buffer.byteslice(@position, 1).to_s.b
+      @position += 1
+      @line += 1 if byte == "\n"
+      @bol = byte == "\n"
+    end
+
+    def update_position
+      if @match_end > @match_start
+        length = @match_end - @match_start
+        @line += if length == 1
+          @buffer.source.getbyte(@match_start) == 0x0a ? 1 : 0
+        elsif @line_tracking_needed && (newline = @buffer.source.index("\n", @match_start)) && newline < @match_end
+          @buffer.byteslice(@match_start, length).to_s.count("\n")
+        else
+          0
+        end
+      end
+      @bol = @match_end.zero? || @buffer.source.getbyte(@match_end - 1) == 0x0a
+    end
+
+    def handle_unmatched_byte
+      bad = @buffer.byteslice(@position, 1)
+      @match_start = @position
+      @match_end = @position + 1
+      @text_start = @match_start
+      @text_line = @line
+      @position += 1
+      @line += 1 if bad.to_s.b == "\n"
+      @bol = bad.to_s.b == "\n"
+      error!("unexpected byte #{bad.inspect}")
+      token = @pending
+      @pending = nil
+      token
+    end
+
+    def eof?
+      return @position >= @buffer.bytesize if @input
+
+      @buffer.eof?(@position)
+    end
+
+    def ensure_state!(name)
+      return if self.class.__flexr_config.states.key?(name.to_sym)
+
+      diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}")
+      raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+    end
+
+    def column_at(position)
+      prefix = @buffer.byteslice(0...position).to_s
+      last_newline = prefix.rindex("\n")
+      line_prefix = prefix.byteslice((last_newline ? last_newline + 1 : 0)..).to_s
+      if utf8_input? && line_prefix.dup.force_encoding(Encoding::UTF_8).valid_encoding?
+        line_prefix.force_encoding(Encoding::UTF_8).length + 1
+      else
+        line_prefix.bytesize + 1
+      end
+    end
+  end
+end