flexr 1.0.0 → 1.1.0

data/lib/flexr/codegen/table.rb

@@ -28,56 +28,139 @@ module Flexr
       def source(indent: "  ")
         lines = []
         lines << "#{indent}def scan_one"
-        lines << "#{indent}  return Flexr::Runtime::Interpreter.new(self).scan unless __flexr_generated_fast_path?"
-        lines << "#{indent}  machine = self.class.__flexr_compiled.machines.fetch(state)"
-        lines << "#{indent}  dfa = machine.dfa"
+        lines << "#{indent}  if @__flexr_generated_scan_state != @state"
+        lines << "#{indent}    machine = self.class.__flexr_compiled.machines.fetch(@state)"
+        lines << "#{indent}    @__flexr_generated_scan_state = @state"
+        lines << "#{indent}    @__flexr_generated_scan_dfa = machine.dfa"
+        lines << "#{indent}    @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)"
+        lines << "#{indent}    @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)"
+        lines << "#{indent}  end"
+        lines << "#{indent}  dfa = @__flexr_generated_scan_dfa"
+        lines << "#{indent}  return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast"
         lines << "#{indent}  position = byte_pos"
         lines << "#{indent}  return nil unless valid_utf8_at?(position)"
+        lines << "#{indent}  return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple"
         lines << "#{indent}  cursor = position"
         lines << "#{indent}  current = dfa.start"
         lines << "#{indent}  direct = dfa.direct"
-        lines << "#{indent}  source = buffer.source"
-        simple = if simple_fast_path?
-          lines << "#{indent}  best = nil"
-          true
-        else
-          lines << "#{indent}  best = __flexr_generated_acceptance(dfa, current, position, position, nil)"
-          false
-        end
-        lines << "#{indent}  accelerate = self.class.__flexr_config.options.fetch(:accel, :auto) != :none && !utf8_input?"
-        lines << "#{indent}  while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)"
+        lines << "#{indent}  direct_nxt = direct&.fetch(:nxt)"
+        lines << "#{indent}  direct_classes = direct&.fetch(:classes)"
+        lines << "#{indent}  ec = dfa.ec"
+        lines << "#{indent}  source = buffer.stable_source"
+        lines << "#{indent}  guarded_steps = scan_steps_guarded?"
+        lines << "#{indent}  token_start = more_text_start || position"
+        lines << "#{indent}  token_limit = max_token_size"
+        lines << "#{indent}  check_token_limit = token_limit_required?"
+        lines << "#{indent}  best = nil"
+        lines << "#{indent}  accel_mode = @accel_mode"
+        lines << "#{indent}  accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)"
+        lines << "#{indent}  while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)"
         lines << "#{indent}    if accelerate"
         lines << "#{indent}      region = __flexr_generated_acceleration_region(dfa, current)"
-        lines << "#{indent}      accelerated_end = __flexr_generated_accelerate(region, cursor) if region"
+        lines << "#{indent}      acceptance = dfa.accepts[current].first"
+        lines << "#{indent}      token_rule = acceptance&.rule_index"
+        lines << "#{indent}      accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region"
         lines << "#{indent}      if accelerated_end && accelerated_end > cursor"
         lines << "#{indent}        cursor = accelerated_end"
         lines << "#{indent}        best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
         lines << "#{indent}        next"
         lines << "#{indent}      end"
         lines << "#{indent}    end"
-        lines << "#{indent}    byte = source.getbyte(cursor)"
-        lines << "#{indent}    current = if direct"
-        lines << "#{indent}      class_id = dfa.ec[byte]"
-        lines << "#{indent}      value = direct[:nxt][(current * direct[:classes]) + class_id]"
+        lines << "#{indent}    byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)"
+        lines << "#{indent}    consume_step! if guarded_steps"
+        lines << "#{indent}    current = if direct_nxt"
+        lines << "#{indent}      class_id = ec[byte]"
+        lines << "#{indent}      value = direct_nxt[(current * direct_classes) + class_id]"
         lines << "#{indent}      value >= 0 ? value : nil"
         lines << "#{indent}    else"
         lines << "#{indent}      dfa.transition(current, byte)"
         lines << "#{indent}    end"
         lines << "#{indent}    break unless current"
         lines << "#{indent}    cursor += 1"
-        if simple
-          lines << "#{indent}    acceptance = dfa.accepts[current].first"
-          lines << "#{indent}    if acceptance"
-          lines << "#{indent}      rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
-          lines << "#{indent}      best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
-          lines << "#{indent}      best.rule = rule"
-          lines << "#{indent}      best.start_pos = position"
-          lines << "#{indent}      best.end_pos = cursor"
-          lines << "#{indent}      best.total_end_pos = cursor"
-          lines << "#{indent}    end"
-        else
-          lines << "#{indent}    best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
-        end
+        lines << "#{indent}    resource_acceptance = dfa.accepts[current].first"
+        lines << "#{indent}    resource_rule = resource_acceptance&.rule_index"
+        lines << "#{indent}    if check_token_limit"
+        lines << "#{indent}      size = cursor - token_start"
+        lines << "#{indent}      defer_token_size_check!(size, rule: resource_rule) if size > token_limit"
+        lines << "#{indent}    end"
+        lines << "#{indent}    best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
+        lines << "#{indent}  end"
+        lines << "#{indent}  best"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_scan_simple(dfa, position)"
+        lines << "#{indent}  current = dfa.start"
+        lines << "#{indent}  direct = dfa.direct"
+        lines << "#{indent}  direct_nxt = direct&.fetch(:nxt)"
+        lines << "#{indent}  direct_classes = direct&.fetch(:classes)"
+        lines << "#{indent}  ec = dfa.ec"
+        lines << "#{indent}  accepts = dfa.accepts"
+        lines << "#{indent}  rules = self.class.__flexr_rules"
+        lines << "#{indent}  source = buffer.stable_source"
+        lines << "#{indent}  guarded_steps = scan_steps_guarded?"
+        lines << "#{indent}  token_start = more_text_start || position"
+        lines << "#{indent}  token_limit = max_token_size"
+        lines << "#{indent}  check_token_limit = token_limit_required?"
+        lines << "#{indent}  cursor = position"
+        lines << "#{indent}  best = nil"
+        lines << "#{indent}  accel_mode = @accel_mode"
+        lines << "#{indent}  accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)"
+        lines << "#{indent}  if source && !guarded_steps && !check_token_limit && !accelerate"
+        lines << "#{indent}    while cursor < source.bytesize"
+        lines << "#{indent}      byte = source.getbyte(cursor)"
+        lines << "#{indent}      current = if direct_nxt"
+        lines << "#{indent}        class_id = ec[byte]"
+        lines << "#{indent}        value = direct_nxt[(current * direct_classes) + class_id]"
+        lines << "#{indent}        value >= 0 ? value : nil"
+        lines << "#{indent}      else"
+        lines << "#{indent}        dfa.transition(current, byte)"
+        lines << "#{indent}      end"
+        lines << "#{indent}      break unless current"
+        lines << "#{indent}      cursor += 1"
+        lines << "#{indent}      acceptance = accepts[current].first"
+        lines << "#{indent}      next unless acceptance"
+        lines << "#{indent}      rule = rules.fetch(acceptance.rule_index)"
+        lines << "#{indent}      best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
+        lines << "#{indent}      best.rule = rule"
+        lines << "#{indent}      best.start_pos = position"
+        lines << "#{indent}      best.end_pos = cursor"
+        lines << "#{indent}      best.total_end_pos = cursor"
+        lines << "#{indent}    end"
+        lines << "#{indent}    return best"
+        lines << "#{indent}  end"
+        lines << "#{indent}  while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)"
+        lines << "#{indent}    if accelerate"
+        lines << "#{indent}      region = __flexr_generated_acceleration_region(dfa, current)"
+        lines << "#{indent}      acceptance = accepts[current].first"
+        lines << "#{indent}      accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region"
+        lines << "#{indent}      if accelerated_end && accelerated_end > cursor"
+        lines << "#{indent}        cursor = accelerated_end"
+        lines << "#{indent}        best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
+        lines << "#{indent}        next"
+        lines << "#{indent}      end"
+        lines << "#{indent}    end"
+        lines << "#{indent}    byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)"
+        lines << "#{indent}    consume_step! if guarded_steps"
+        lines << "#{indent}    current = if direct_nxt"
+        lines << "#{indent}      class_id = ec[byte]"
+        lines << "#{indent}      value = direct_nxt[(current * direct_classes) + class_id]"
+        lines << "#{indent}      value >= 0 ? value : nil"
+        lines << "#{indent}    else"
+        lines << "#{indent}      dfa.transition(current, byte)"
+        lines << "#{indent}    end"
+        lines << "#{indent}    break unless current"
+        lines << "#{indent}    cursor += 1"
+        lines << "#{indent}    acceptance = accepts[current].first"
+        lines << "#{indent}    if check_token_limit"
+        lines << "#{indent}      size = cursor - token_start"
+        lines << "#{indent}      defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit"
+        lines << "#{indent}    end"
+        lines << "#{indent}    next unless acceptance"
+        lines << "#{indent}    rule = rules.fetch(acceptance.rule_index)"
+        lines << "#{indent}    best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
+        lines << "#{indent}    best.rule = rule"
+        lines << "#{indent}    best.start_pos = position"
+        lines << "#{indent}    best.end_pos = cursor"
+        lines << "#{indent}    best.total_end_pos = cursor"
         lines << "#{indent}  end"
         lines << "#{indent}  best"
         lines << "#{indent}end"
@@ -86,44 +169,75 @@ module Flexr
         lines << "#{indent}  regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })"
         lines << "#{indent}  region = regions[state]"
         lines << "#{indent}  return unless region"
+        lines << "#{indent}  return if @__flexr_generated_disabled_auto_accel&.key?(region)"
+        lines << "#{indent}  return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }"
         lines << "#{indent}  return if dfa.accepts[state].any? do |acceptance|"
         lines << "#{indent}    rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
         lines << "#{indent}    acceptance.bol_only || acceptance.end_anchor || rule.trailing"
         lines << "#{indent}  end"
         lines << "#{indent}  region"
         lines << "#{indent}end"
-        lines << "#{indent}def __flexr_generated_accelerate(region, position)"
-        lines << "#{indent}  binary = buffer.source.b"
-        lines << "#{indent}  mode = self.class.__flexr_config.options.fetch(:accel, :auto)"
-        lines << "#{indent}  match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)"
-        lines << "#{indent}    scanner = StringScanner.new(binary)"
-        lines << "#{indent}    scanner.pos = position"
-        lines << "#{indent}    length = scanner.skip(region.regexp)"
-        lines << "#{indent}    length && scanner.pos"
-        lines << "#{indent}  else"
-        lines << "#{indent}    match = region.regexp.match(binary, position)"
-        lines << "#{indent}    match && match.begin(0) == position ? match.end(0) : nil"
-        lines << "#{indent}  end"
-        lines << "#{indent}  return match_end if match_end && match_end < buffer.bytesize"
-        lines << "#{indent}  return match_end if match_end && buffer.eof_loaded?"
-        lines << "#{indent}  return unless buffer.ensure_available?(buffer.bytesize + 1)"
-        lines << "#{indent}  __flexr_generated_accelerate(region, position)"
-        lines << "#{indent}rescue ArgumentError"
-        lines << "#{indent}  nil"
-        lines << "#{indent}end"
-        lines << "#{indent}def __flexr_generated_fast_path?"
-        lines << "#{indent}  return @__flexr_generated_fast_path if defined?(@__flexr_generated_fast_path)"
-        lines << "#{indent}  @__flexr_generated_fast_path = if self.class.__flexr_config.backend == :firstmatch ||"
-        lines << "#{indent}                                  self.class.__flexr_config.options[:allow_empty_match] == true"
-        lines << "#{indent}    false"
-        lines << "#{indent}  else"
-        lines << "#{indent}    self.class.__flexr_rules.none? do |rule|"
-        lines << "#{indent}      !rule.trailing.nil? || rule.patterns.any? do |pattern|"
-        lines << "#{indent}        next false unless pattern.is_a?(Regexp)"
-        lines << "#{indent}      next true if pattern.source.match?(/\\\\[pP]\\\\{/) || pattern.source.match?(/\\[:(?:\\^)?[a-z]+:\\]/)"
-        lines << "#{indent}        self.class.__flexr_config.options[:unicode] == true && utf8_input? && pattern.source.match?(/\\\\[dDwWsS]/)"
+        lines << "#{indent}def __flexr_generated_accelerate(region, position, token_rule, token_start)"
+        lines << "#{indent}  mode = @accel_mode"
+        lines << "#{indent}  cursor = position"
+        lines << "#{indent}  matched = false"
+        lines << "#{indent}  loop do"
+        lines << "#{indent}    if cursor >= buffer.bytesize"
+        lines << "#{indent}      return matched ? cursor : nil if buffer.eof_loaded?"
+        lines << "#{indent}      return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)"
+        lines << "#{indent}    end"
+        lines << "#{indent}    segment = buffer.source"
+        lines << "#{indent}    segment_position = cursor - buffer.base_offset"
+        lines << "#{indent}    if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp"
+        lines << "#{indent}      regexp = region.utf8_regexp"
+        lines << "#{indent}    else"
+        lines << "#{indent}      segment = segment.b unless segment.encoding == ::Encoding::BINARY"
+        lines << "#{indent}      regexp = region.regexp"
+        lines << "#{indent}    end"
+        lines << "#{indent}    length = begin"
+        lines << "#{indent}      if %i[strscan auto].include?(mode) && defined?(::StringScanner)"
+        lines << "#{indent}        @__flexr_generated_accel_scanner ||= ::StringScanner.new(\"\".b)"
+        lines << "#{indent}        @__flexr_generated_accel_scanner.string = segment"
+        lines << "#{indent}        @__flexr_generated_accel_scanner.pos = segment_position"
+        lines << "#{indent}        @__flexr_generated_accel_scanner.skip(regexp)"
+        lines << "#{indent}      else"
+        lines << "#{indent}        match = regexp.match(segment, segment_position)"
+        lines << "#{indent}        match && match.begin(0) == segment_position ? match.end(0) - segment_position : nil"
         lines << "#{indent}      end"
+        lines << "#{indent}    rescue ArgumentError"
+        lines << "#{indent}      nil"
         lines << "#{indent}    end"
+        lines << "#{indent}    __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8"
+        lines << "#{indent}    return matched ? cursor : nil unless length&.positive?"
+        lines << "#{indent}    matched = true"
+        lines << "#{indent}    cursor += length"
+        lines << "#{indent}    consume_step!(length)"
+        lines << "#{indent}    defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)"
+        lines << "#{indent}    return cursor if cursor < buffer.bytesize || buffer.eof_loaded?"
+        lines << "#{indent}  end"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_record_auto_accel_miss(region)"
+        lines << "#{indent}  @__flexr_generated_auto_accel_misses ||= Hash.new(0)"
+        lines << "#{indent}  @__flexr_generated_auto_accel_misses[region] += 1"
+        lines << "#{indent}  @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1"
+        lines << "#{indent}  @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9"
+        lines << "#{indent}  return if @__flexr_generated_auto_accel_misses[region] < 3"
+        lines << "#{indent}  @__flexr_generated_disabled_auto_accel ||= {}"
+        lines << "#{indent}  @__flexr_generated_disabled_auto_accel[region] = true"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_fast_path?(dfa)"
+        lines << "#{indent}  @__flexr_generated_fast_paths ||= {}"
+        lines << "#{indent}  return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)"
+        lines << "#{indent}  rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }"
+        lines << "#{indent}  @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&"
+        lines << "#{indent}    self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_simple_fast_path?(dfa)"
+        lines << "#{indent}  @__flexr_generated_simple_fast_paths ||= {}"
+        lines << "#{indent}  return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)"
+        lines << "#{indent}  rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }"
+        lines << "#{indent}  @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|"
+        lines << "#{indent}    rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }"
         lines << "#{indent}  end"
         lines << "#{indent}end"
         lines << "#{indent}def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)"
@@ -131,7 +245,6 @@ module Flexr
         lines << "#{indent}    next if acceptance.bol_only && !beginning_of_line?"
         lines << "#{indent}    next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)"
         lines << "#{indent}    rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
-        lines << "#{indent}    defer_token_size_check!(cursor - start_position)"
         lines << "#{indent}    next if best && cursor < best.total_end_pos"
         lines << "#{indent}    next if best && cursor == best.total_end_pos &&"
         lines << "#{indent}      rule.index > best.rule.index"
@@ -146,13 +259,6 @@ module Flexr
         "#{lines.join("\n")}\n"
       end
 
-      def simple_fast_path?
-        compiled.rules.all? do |rule|
-          rule.trailing.nil? && rule.pattern_conditions.all? do |condition|
-            condition && !condition.bol_only && !condition.end_anchor
-          end
-        end
-      end
     end
   end
 end

data/lib/flexr/codegen.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "runtime" unless defined?(Flexr::Lexer)
+require_relative "codegen/base"
+require_relative "codegen/table"
+require_relative "codegen/direct"
+require_relative "codegen/firstmatch"
+require_relative "codegen/table_packer"

data/lib/flexr/configuration.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Configuration
+    BACKENDS = %i[table direct firstmatch auto].freeze
+    TOKEN_KINDS = %i[array struct yield].freeze
+    ACCELERATORS = %i[auto strscan regexp none].freeze
+    BOOLEAN_OPTIONS = %i[unicode eager_columns allow_empty_match experimental standalone].freeze
+
+    module_function
+
+    def backend!(value)
+      validated_symbol!(value, BACKENDS, :backend)
+    end
+
+    def token_kind!(value)
+      validated_symbol!(value, TOKEN_KINDS, :token_kind)
+    end
+
+    def accelerator!(value)
+      validated_symbol!(value, ACCELERATORS, :accel)
+    end
+
+    def option!(value)
+      validated_symbol!(value, BOOLEAN_OPTIONS, :option)
+    end
+
+    def validated_symbol!(value, allowed, name)
+      symbol = value.to_sym
+      return symbol if allowed.include?(symbol)
+
+      raise ArgumentError, "unsupported #{name}: #{value.inspect}"
+    rescue NoMethodError
+      raise ArgumentError, "unsupported #{name}: #{value.inspect}"
+    end
+  end
+end

data/lib/flexr/dsl.rb

@@ -3,8 +3,6 @@
 module Flexr
   module DSL
     DSL_METHODS = %i[rule state all_states on_eof emits backend token_kind encoding option accel].freeze
-    AUTO_DIRECT_CELL_THRESHOLD = 100_000
-
     def inherited(child)
       super
       child.__flexr_reset!
@@ -21,23 +19,23 @@ module Flexr
       )
       @__flexr_compiled = nil
       @__flexr_generated = false
-      @__flexr_compile_mutex = Mutex.new
+      @__flexr_generated_actions = []
+      @__flexr_specification_frozen = false
+      @__flexr_compile_mutex = Monitor.new
     end
 
     def rule(pattern, skip: false, emit: nil, followed_by: nil, &action)
-      patterns = normalize_patterns(pattern)
-      rule_action = if skip
-        :skip
-      elsif emit
-        [:emit, emit.to_sym]
-      else
-        action || proc { emit(nil, text) }
+      __flexr_mutate! do
+        patterns = normalize_patterns(pattern)
+        rule_action = ActionResolver.resolve(
+          skip: skip, emit: emit, block: action, default: proc { emit(nil, text) }
+        )
+        states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
+        @__flexr_rules << IR::Rule.new(
+          index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
+          action: rule_action, states: states, bol_only: false, end_anchor: nil
+        )
       end
-      states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
-      @__flexr_rules << IR::Rule.new(
-        index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
-        action: rule_action, states: states, bol_only: false, end_anchor: nil
-      )
       nil
     end
 
@@ -45,38 +43,56 @@ module Flexr
       raise ArgumentError, "state requires a block" unless block
       raise ArgumentError, "state requires a name" if names.empty?
 
-      names.each do |name|
-        symbol = name.to_sym
-        @__flexr_states[symbol] ||= IR::State.new(name: symbol, inclusive: inclusive, id: @__flexr_states.length)
+      __flexr_mutate! do
+        names.each do |name|
+          symbol = name.to_sym
+          existing = @__flexr_states[symbol]
+          raise ArgumentError, "state #{symbol.inspect} already declared with inclusive: #{existing.inclusive}" if
+            existing && existing.inclusive != inclusive
+          @__flexr_states[symbol] ||= IR::State.new(
+            name: symbol, inclusive: inclusive, id: @__flexr_states.length
+          )
+        end
+        @__flexr_state_stack.concat(names.map(&:to_sym))
+        begin
+          class_eval(&block)
+        ensure
+          names.length.times { @__flexr_state_stack.pop }
+        end
       end
-      @__flexr_state_stack.concat(names.map(&:to_sym))
-      class_eval(&block)
-    ensure
-      names&.length&.times { @__flexr_state_stack.pop }
     end
 
     def all_states(&)
-      state(*@__flexr_states.keys, &)
+      raise ArgumentError, "all_states requires a block" unless block_given?
+
+      __flexr_mutate! do
+        names = @__flexr_states.keys
+        @__flexr_state_stack.concat(names)
+        begin
+          class_eval(&)
+        ensure
+          names.length.times { @__flexr_state_stack.pop }
+        end
+      end
     end
 
     def on_eof(&action)
-      state_name = @__flexr_state_stack.last || :initial
-      @__flexr_eof_rules[state_name] = action
+      __flexr_mutate! do
+        state_name = @__flexr_state_stack.last || :initial
+        @__flexr_eof_rules[state_name] = action
+      end
     end
 
     def emits(*tokens)
-      @__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq!
+      __flexr_mutate! { @__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq! }
     end
 
     def backend(name)
-      @__flexr_config.backend = name.to_sym
+      __flexr_mutate! { @__flexr_config.backend = Configuration.backend!(name) }
     end
 
     def token_kind(name)
-      value = name.to_sym
-      raise ArgumentError, "unsupported token_kind: #{name}" unless %i[array struct yield].include?(value)
-
-      @__flexr_config.token_kind = value
+      __flexr_mutate! { @__flexr_config.token_kind = Configuration.token_kind!(name) }
     end
 
     def encoding(value)
@@ -86,15 +102,17 @@ module Flexr
         raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
       end
 
-      @__flexr_config.encoding = encoding
+      __flexr_mutate! { @__flexr_config.encoding = encoding }
     end
 
     def option(*values)
-      values.each { |value| @__flexr_config.options[value.to_sym] = true }
+      __flexr_mutate! do
+        values.each { |value| @__flexr_config.options[Configuration.option!(value)] = true }
+      end
     end
 
     def accel(value)
-      @__flexr_config.options[:accel] = value.to_sym
+      __flexr_mutate! { @__flexr_config.options[:accel] = Configuration.accelerator!(value) }
     end
 
     def compile!
@@ -104,6 +122,8 @@ module Flexr
         @__flexr_compiled ||= begin
           compiled = Automaton::Compiler.new(__flexr_spec).compile
           @__flexr_config.backend = auto_direct?(compiled) ? :direct : :table if @__flexr_config.backend == :auto
+          freeze_specification!
+          freeze_compiled!(compiled)
           compiled
         end
         # rubocop:enable Naming/MemoizedInstanceVariableName
@@ -131,26 +151,73 @@ module Flexr
       )
     end
 
-    attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled
+    attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions
 
     def __flexr_add_generated_eof(state, action)
-      @__flexr_eof_rules[state.to_sym] = action
+      __flexr_mutate! { @__flexr_eof_rules[state.to_sym] = action }
     end
 
     def __flexr_set_compiled!(compiled)
-      @__flexr_compiled = compiled
+      @__flexr_compile_mutex.synchronize do
+        @__flexr_compiled = freeze_compiled!(compiled)
+        freeze_specification!
+      end
     end
 
     def __flexr_mark_generated!
       @__flexr_generated = true
     end
 
+    def __flexr_bind_generated_action(index, action)
+      raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
+
+      @__flexr_generated_actions[index] = action
+    end
+
     def __flexr_generated?
       @__flexr_generated == true
     end
 
     private
 
+    def __flexr_mutate!
+      @__flexr_compile_mutex.synchronize do
+        raise FrozenSpecificationError, "lexer specification is immutable after compilation" if
+          @__flexr_specification_frozen
+
+        yield
+      end
+    end
+
+    def freeze_specification!
+      @__flexr_rules.each do |rule|
+        rule.patterns.freeze
+        rule.states.freeze
+        rule.pattern_conditions&.each(&:freeze)
+        rule.pattern_conditions&.freeze
+        rule.freeze
+      end
+      @__flexr_states.each_value(&:freeze)
+      @__flexr_eof_rules.freeze
+      @__flexr_rules.freeze
+      @__flexr_states.freeze
+      @__flexr_config.options.freeze
+      @__flexr_config.declared_tokens.freeze
+      @__flexr_config.freeze
+      @__flexr_specification_frozen = true
+    end
+
+    def freeze_compiled!(compiled)
+      compiled.machines.each_value(&:freeze)
+      compiled.machines.freeze
+      compiled.states.freeze
+      compiled.stats.each_value(&:freeze)
+      compiled.stats.freeze
+      Array(compiled.diagnostics).each(&:freeze)
+      compiled.diagnostics.freeze
+      compiled.freeze
+    end
+
     def normalize_patterns(pattern)
       values = pattern.is_a?(Array) ? pattern : [pattern]
       values.each do |value|
@@ -175,8 +242,7 @@ module Flexr
     end
 
     def auto_direct?(compiled)
-      cells = compiled.stats.values.map { |stats| stats[:states] * stats[:classes] }.max.to_i
-      cells > AUTO_DIRECT_CELL_THRESHOLD
+      Automaton::BackendCostModel.choose(compiled) == :direct
     end
   end
 end

data/lib/flexr/errors.rb

@@ -25,4 +25,5 @@ module Flexr
   class CompileError < Error; end
   class UnsupportedRegexpError < CompileError; end
   class StaticResolutionError < CompileError; end
+  class FrozenSpecificationError < Error; end
 end