flexr 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54c55f186a1f4aa048a7a46ca86cae52eb5f9566888fc9cdc09f527d4b6b341f
-  data.tar.gz: 7a5e9329e688889cfe665f3a4499deea7ca2e9de684f4fd5ad10847bd631fb4e
+  metadata.gz: edb2c916db82e07c56290ab6365d50bf9a0a148278ca811a100063031315bbf5
+  data.tar.gz: 13c08660e91681ab2143766d58072c4c4e3d081b5899a18568a772bfd80810b9
 SHA512:
-  metadata.gz: 7399ef9f672ff36623a8d54a6eed882ee43fed2c7e9d59b417038bd0d8a9a32b892a8556774531aceb41b0e1c3729a169a003b190b8f2a330c4b825960da32b2
-  data.tar.gz: 4ecdca40a21a0ff5af1146f20912297c96542bd2ec66412f3726461472c45e10ebc7e6d81a17ef942f3d0bfa036c8701f123fc15be72408dd62c5673c5636154
+  metadata.gz: aef18f76825b6544c03a3115a1dfeb8cd469560fb4bff23b2fa3e57cc9504e780a318ebfcc06d1ff7c8807c217b813a6a2768d154a8596cf7103347fd97e399e
+  data.tar.gz: 0355b004bc8ff95a00fd4be5fd4644f4cce1a0f2feaad49da48ef023fc594453de208600d542a26a6bc6fcb4a645864770816ed8d4b4411e8607860801d1c43c

data/Rakefile

@@ -86,6 +86,36 @@ module FlexrVerification
     codepoints.pack("U*")
   end
 
+  def merge_ranges(ranges)
+    ranges.sort_by(&:first).each_with_object([]) do |range, merged|
+      if merged.empty? || range.first > merged.last.last + 1
+        merged << range.dup
+      else
+        merged.last[1] = [merged.last.last, range.last].max
+      end
+    end
+  end
+
+  def lexer_outcome(lexer_class, input, **options)
+    { kind: :tokens, tokens: lexer_class.new(input.dup, **options).tokens }
+  rescue StandardError => e
+    {
+      kind: :error,
+      class: e.class.name,
+      message: e.message,
+      code: if e.respond_to?(:code)
+        e.code
+      elsif e.respond_to?(:diagnostic)
+        e.diagnostic&.code
+      end,
+      filename: e.respond_to?(:filename) ? e.filename : nil,
+      byte_pos: e.respond_to?(:byte_pos) ? e.byte_pos : nil,
+      line: e.respond_to?(:line) ? e.line : nil,
+      rule: e.respond_to?(:rule) ? e.rule : nil,
+      text: e.respond_to?(:text) ? e.text : nil
+    }
+  end
+
   def load_runtime(spec)
     class_name = runtime_class_name(spec)
     existing = constantize(class_name)
@@ -135,11 +165,11 @@ module FlexrVerification
     inputs = [input_for(spec)] + Array.new(32) do
       Array.new(random.rand(48..96)) { random.rand(32..126) }.pack("C*")
     end
-    options = lexer.__flexr_config.options
-    original_accel = options.fetch(:accel, :auto)
     accelerated = inputs.map { |input| lexer.new(input, error_mode: :panic).tokens }
-    options[:accel] = :none
-    reference = inputs.map { |input| lexer.new(input, error_mode: :panic).tokens }
+    remove_constant(runtime_class_name(spec))
+    reference_lexer = load_runtime(spec)
+    reference_lexer.accel(:none)
+    reference = inputs.map { |input| reference_lexer.new(input, error_mode: :panic).tokens }
     raise "acceleration token mismatch in #{spec}" unless accelerated == reference
 
     lexer.compile!.machines.each_value do |machine|
@@ -153,8 +183,6 @@ module FlexrVerification
         end
       end
     end
-  ensure
-    options[:accel] = original_accel if options && original_accel
   end
 
   def verify_dogfood(spec)
@@ -290,16 +318,12 @@ rescue Errno::ENOENT => e
 end
 
 task "direct:verify" do
-  unless defined?(RubyVM::InstructionSequence)
-    puts "direct: disassembly unavailable on #{RUBY_ENGINE}; skipped"
-    next
-  end
-
   spec = File.join(FlexrVerification::ROOT, "examples/json/lexer.flexr.rb")
-  disassembly = RubyVM::InstructionSequence.compile(FlexrVerification.generated_source(spec)).disasm
-  abort "direct dispatch did not compile to opt_case_dispatch" unless disassembly.include?("opt_case_dispatch")
+  source = Flexr::Generator.new(spec, options: { backend: :direct }).generate
+  abort "direct payload is missing its flattened transition representation" unless
+    source.include?("direct: {nxt:") && !source.include?("__flexr_generated_direct_transition")
 
-  puts "direct: opt_case_dispatch present"
+  puts "direct: single flattened representation present"
 end
 
 task "unicode:verify" do
@@ -315,6 +339,14 @@ task "unicode:verify" do
       previous = hi
     end
   end
+  expected_alnum = FlexrVerification.merge_ranges(
+    properties.fetch("L") + properties.fetch("N") + properties.fetch("Other_Alphabetic")
+  )
+  abort "vendored Alnum table is stale" unless properties.fetch("Alnum") == expected_alnum
+
+  expected_word = FlexrVerification.merge_ranges(expected_alnum + [[0x5f, 0x5f]])
+  abort "vendored Word table is stale" unless properties.fetch("Word") == expected_word
+
   scalar_count = 0
   (0..0x10_ffff).each do |codepoint|
     next if codepoint.between?(0xd800, 0xdfff)
@@ -365,6 +397,9 @@ namespace :test do
     random = Random.new(Integer(ENV.fetch("FLEXR_SEED", "17"), 10))
     unicode_inputs = ["", "a", "あ", "é", "ß", "Ω", "١", "　", "aあ", "éΩ", [0x18db8].pack("U")].freeze
     compiled = {}
+    host_patterns = {}
+    independent_cases = 0
+    version_mismatches = 0
     cases.times do
       pattern = patterns[random.rand(patterns.length)]
       input = if random.rand(3).zero?
@@ -372,26 +407,38 @@ namespace :test do
       else
         Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
       end
-      expected = if Flexr.reference_pattern?(pattern)
+      key = [pattern.source, pattern.options]
+      host_expected = begin
+        (host_patterns[key] ||= Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options)).match?(input)
+      rescue RegexpError, ArgumentError, EncodingError
+        false
+      end
+      if Flexr.reference_pattern?(pattern)
         reference = Flexr::Unicode::ReferenceRegexp.compiled(
           pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
         )
         match = reference.match(input, 0)
-        if match
+        vendored_expected = if match
           match.begin(0).zero? && match[0].bytesize == input.bytesize
         else
           false
         end
-      else
-        Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options).match?(input)
+        unless vendored_expected == host_expected
+          version_mismatches += 1
+          next
+        end
       end
-      key = [pattern.source, pattern.options]
+      independent_cases += 1
       actual = (compiled[key] ||= Flexr.compile_pattern(pattern)).accept?(input)
-      next if expected == actual
+      next if host_expected == actual
 
-      abort "differential mismatch: #{pattern.inspect} #{input.inspect} expected=#{expected} actual=#{actual}"
+      abort "differential mismatch: #{pattern.inspect} #{input.inspect} " \
+            "expected=#{host_expected} actual=#{actual}"
     end
-    puts "differential: #{cases} cases passed"
+    abort "differential did not exercise an independent host oracle" if independent_cases.zero?
+
+    puts "differential: #{independent_cases} independent cases passed " \
+         "(#{version_mismatches} host/vendored Unicode differences skipped)"
   end
 end
 
@@ -425,13 +472,12 @@ task :fuzz do
         (FlexrVerification.random_unicode_string(random, max_codepoints: 16) +
           Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
       end
-      runtime_tokens = runtime_lexer.new(input, error_mode: :panic).tokens
-      generated_tokens = generated_lexer.new(input, error_mode: :panic).tokens
-      next if runtime_tokens == generated_tokens
+      runtime_outcome = FlexrVerification.lexer_outcome(runtime_lexer, input, error_mode: :panic)
+      generated_outcome = FlexrVerification.lexer_outcome(generated_lexer, input, error_mode: :panic)
+      next if runtime_outcome == generated_outcome
 
-      abort "fuzz mode mismatch: #{spec} input=#{input.inspect} runtime=#{runtime_tokens.inspect} generated=#{generated_tokens.inspect}"
-    rescue Flexr::LexError, ArgumentError, EncodingError
-      # Invalid input and user-defined error actions are expected fuzz outcomes.
+      abort "fuzz mode mismatch: #{spec} input=#{input.inspect} " \
+            "runtime=#{runtime_outcome.inspect} generated=#{generated_outcome.inspect}"
     end
   ensure
     FileUtils.rm_f(generated_path) if generated_path

data/benchmark/golden/calculator_lexer.sha256

@@ -1 +1 @@
-6359976d9cdc72772eff8c72cf421d965e913febad3dbc0ca0f2057a0faea3b4
+d01c0b69f4f2fedd572e90000f4fc670524b01089a6942b94b1fc1c1cd27f95e

data/benchmark/golden/json_lexer.sha256

@@ -1 +1 @@
-913cf85ba7e1a3128eea0a4b06077486fead2d56357b669685ae6e26bdd55715
+f6d8418031b657a328d3ad62fcd780a43bbf4731e9d0e390120dd219894a515e

data/benchmark/golden/regexp_tokenizer.sha256

@@ -1 +1 @@
-7f20a1024eea1d674aee73a2baf7a32c52fa178f2d158d24e49f9c9b77a63960
+1f53aa34bf24a92fd03096648b3290bbfda76222078c0c58966e259fd3a13aca

data/benchmark/golden/ruby_subset_lexer.sha256

@@ -1 +1 @@
-0d99b92c658d56cbef7d9c33ac3e5770d83e379b94cf412f8b028f59d44910b8
+5bb597bfacaf3217d2852185a9c4e90833a3cb6da02e114817d00d00dd3a3aec

data/benchmark/golden/toy_lang_lexer.sha256

@@ -1 +1 @@
-16943c2d8a86999148dd6bcd2ed644671032d1e6b4acbe891d180ff0810cd97a
+0285b7b1ceff88a7eddcd1f7d3ac87b0a4bce1e4162f5e64205549aacd576dce

data/benchmark/golden/with_lrama_lexer.sha256

@@ -1 +1 @@
-7627b0fafbfdbdea850d02e67c3e9c24eab8512cdc74571a986323d397848071
+6abde13a982ee9f529cc2182dad6cefd1f8608af295981fdfe8671e6b46c7f99

data/benchmark/golden/with_racc_lexer.sha256

@@ -1 +1 @@
-037fa90c634117522c1e882d08c437c7b79653f59ca94e6cf6c115cd7897610f
+cb3836416ff7db7877efa68dc4f921755af9ebfb1e03aaa440892e12b6b7ac35

data/docs/README.md

@@ -61,4 +61,5 @@ pages explain how to use those specifications.
 
 The [performance log](perf-log.md), [architecture decisions](adr/),
 [internals notes](internals/README.md), and [release procedure](RELEASING.md)
-are maintainer-facing documents.
+are maintainer-facing documents. Release notes are kept under
+[releases](releases/v1.1.0.md).

data/docs/explanation/backends.md

@@ -8,26 +8,29 @@ target for `firstmatch`.
 
 ## `direct`
 
-The stable dense-dispatch backend. It emits a flattened dispatch structure for
-large machines and can reduce lookup overhead at the cost of artifact size.
-It preserves the table backend's longest-match behavior.
+The stable dense-dispatch backend. It emits one flattened transition structure
+and avoids retaining a second packed or row table. It can reduce lookup
+overhead at the cost of artifact size and preserves the table backend's
+leftmost-longest behavior.
 
 ## `auto`
 
-The stable selection policy. At compile time it chooses direct dispatch when the
-largest state-by-byte-class table exceeds the configured threshold, otherwise
-it uses table dispatch. The effective backend is recorded in generated headers.
+The stable selection policy. At compile time it compares estimated dense bytes,
+packed bytes, and lookup cost for each lexical-state machine. The effective
+backend is recorded in generated headers.
 
 ## `firstmatch`
 
 Experimental compatibility behavior. It requires `option :experimental` and
-can return a shorter match merely because its rule appears earlier. Generation
-compares a deterministic input corpus with the table matcher and aborts if the
-results differ.
+can return a shorter match merely because its rule appears earlier. Runtime and
+generated artifacts preserve that same behavior. Potential differences from
+the stable longest-match backends are reported with `FLEXR-W010`.
 
 ## Acceleration and packing
 
 Region acceleration and packed tables are representations, not semantic
 backends. `accel` can be disabled for debugging or equivalence tests. Table
 compression and packed Base64 output trade source size against loading cost;
-they do not change token streams.
+packed transitions are queried without creating a second dense table, and a
+dense row is materialized lazily only for inspection. These choices do not
+change token streams.

data/docs/explanation/matching-semantics.md

@@ -11,8 +11,10 @@ extends the selection end without extending the consumed token end.
 
 The matcher is byte-oriented so binary input and invalid UTF-8 have deterministic
 behavior. UTF-8 patterns still require valid codepoint boundaries before a
-match is accepted. The runtime and generated scanner share the compiled model;
-acceleration may skip self-loop bytes but cannot change the winner.
+match is accepted. Unicode properties and POSIX classes are expanded from the
+vendored UCD into this same model rather than matched by a separate host-regexp
+path. The runtime and generated scanner share the compiled model; acceleration
+may skip self-loop bytes but cannot change the winner.
 
 `firstmatch` is intentionally separate. It picks the first rule that has a
 matching alternative and therefore is useful only for compatibility migrations

data/docs/explanation/security-model.md

@@ -13,6 +13,13 @@ to discover dynamic patterns. Use it only with trusted files and a trusted
 environment. The same rule applies to generated output: review or sign the
 artifact before loading it in a process with sensitive access.
 
-The runtime's `max_token_size` and `max_state_stack` limits provide resource
-guards for input processing. They are operational controls, not a security
-sandbox and do not protect against arbitrary Ruby actions.
+The runtime's `max_token_size`, `max_lookahead_size`, `max_buffer_size`,
+`max_state_stack`, and optional `max_steps` limits provide resource guards for
+input processing. A `cancellation` callback can stop concurrent work
+cooperatively. Limits are checked while scanning, before an oversized token's
+action runs. They are operational controls, not a security sandbox and do not
+protect against arbitrary Ruby actions.
+
+Use `retain_input: false` for long-lived streaming lexers that do not need the
+complete input after consumption. This makes the buffer a sliding window;
+`input` and `binary_input` no longer contain discarded prefixes.

data/docs/explanation/unicode-and-encoding.md

@@ -8,7 +8,9 @@ allowing Unicode-aware patterns.
 The vendored Unicode Character Database snapshot, currently 15.1.0, is the
 compatibility oracle for `\p{...}`, POSIX properties where a Unicode property
 is defined, Unicode shorthand behavior with `option :unicode`, and simple case
-folding. Host Ruby Unicode tables are not used as a cross-version oracle.
+folding. Property ranges are compiled into the same DFA as ordinary rules, so
+they preserve leftmost-longest matching and streaming behavior. Host Ruby
+Unicode tables are not used as a cross-version oracle.
 
 Use `encoding Encoding::BINARY` for protocol or byte-oriented formats. Use the
 default UTF-8 encoding for text. A pattern with invalid UTF-8 or an input that

data/docs/how-to/deploy-a-standalone-lexer.md

@@ -6,9 +6,11 @@ Use `--standalone` to embed the flexr runtime into the generated file:
 flexr lexer.flexr.rb --standalone -o lexer_standalone.rb
 ```
 
-The generated file removes `require "flexr"` and includes the runtime sources
-needed to execute the compiled tables. It can be loaded by a Ruby process that
-does not have the flexr gem installed:
+The generated file removes `require "flexr"` and includes only the runtime
+sources needed to execute its compiled tables. Compiler, generator, and
+minimizer sources are omitted. The regexp reference matcher and Unicode tables
+are included only when the selected backend or trailing context needs them. It
+can be loaded by a Ruby process that does not have the flexr gem installed:
 
 ```sh
 ruby -e 'load ARGV.fetch(0); p Lexer.new("12").tokens' lexer_standalone.rb

data/docs/how-to/generate-a-lexer.md

@@ -13,9 +13,9 @@ Ruby constants, comments, requires, and non-DSL code remain in the file.
 
 ## Static expressions
 
-The default path accepts regexp and string literals, arrays of supported
-values, constants, ranges, interpolation whose values are static, `.freeze`,
-and `Regexp.union`. A call whose value depends on runtime state raises
+The default path accepts regexp and string literals, arrays and hashes of
+supported values, constants, ranges, interpolation whose values are static,
+`.freeze`, and `Regexp.union`. Constants follow Ruby source order. A call whose value depends on runtime state raises
 `FLEXR-E017`. See the [static-expression reference](../reference/generated-artifacts.md)
 for the exact boundary.
 

data/docs/how-to/handle-errors.md

@@ -29,4 +29,7 @@ instance with `lexer.on_error = ->(error) { :skip }`, `:raise`, `:token`, or
 byte position, line, and text.
 
 `error!` creates the same error from an action. `max_token_size` protects the
-runtime from unexpectedly large tokens and raises `FLEXR-E012` when exceeded.
+runtime from unexpectedly large tokens and raises `FLEXR-E012` before the
+action runs. Lookahead, buffer, step, state-stack, cancellation, and progress
+guards likewise raise structured runtime errors; their stable codes are listed
+in the [diagnostics catalog](../reference/diagnostics.md).

data/docs/how-to/run-a-lexer-at-runtime.md

@@ -20,6 +20,11 @@ in chunks; `chunk_size:` controls the buffer size. `next_token` returns one
 token or `nil` at EOF, `tokens` drains the lexer into an array, and
 `each_token` returns an enumerator when no block is given.
 
+For a long-lived stream that does not need old input, pass
+`retain_input: false`. Combine it with `max_buffer_size:`, `max_token_size:`,
+and `max_lookahead_size:` to keep memory bounded. Use `max_steps:` or a
+`cancellation:` callback when request-level work must also be bounded.
+
 Use runtime mode when the specification deliberately depends on ordinary Ruby
 execution. Static generation is a better fit when startup cost, deployment
 reproducibility, or a generator-free runtime matters.

data/docs/how-to/track-token-locations.md

@@ -21,7 +21,8 @@ Locations use byte offsets for `byte_begin` and `byte_end`, with an exclusive
 end offset. Lines and columns are one-based. UTF-8 columns count characters;
 binary columns count bytes. Column values are computed lazily unless
 `option :eager_columns` is set. `filename:` on the lexer is copied into each
-location and lexical error.
+location and lexical error. The lexer tracks line and column state
+incrementally; locations do not rescan all previously consumed input.
 
 `more` joins successive matches into one `text` value, so `last_location` spans
 the complete assembled token. See the [token and location contract](../reference/tokens-and-locations.md).

data/docs/how-to/tune-performance.md

@@ -9,9 +9,9 @@ bundle exec rake bench:regression
 
 Use `token_kind :yield` when the consumer streams tokens and does not need an
 intermediate token array. Use generated mode to move DFA construction out of
-application startup. `backend :auto` chooses `:direct` for large transition
-tables and `:table` otherwise; `:table` and `:direct` preserve the same
-longest-match semantics.
+application startup. `backend :auto` compares dense memory, packed memory, and
+lookup costs rather than using a fixed table-size threshold; `:table` and
+`:direct` preserve the same longest-match semantics.
 
 Acceleration is an optimization over the DFA, not a second matcher. `accel
 :auto` selects an available safe path, while `:strscan`, `:regexp`, and `:none`

data/docs/perf-log.md

@@ -54,3 +54,19 @@ the following local estimates (one iteration, `backend :direct`):
 
 The padded corpus can be reproduced with
 `FLEXR_JSON_PADDING_BYTES=100 ruby benchmark/corpora/generate_json.rb`.
+
+## 2026-08-12
+
+After adding bounded scanning and the compiler/generated-runtime
+optimizations, the same Ruby 4.0.0 host produced this three-iteration sample:
+
+| mode | MB/s | tokens/s | allocations/token | handwritten ratio |
+|---|---:|---:|---:|---:|
+| runtime | 1.603 | 572,508 | 5.2 | 0.203x |
+| generated | 1.607 | 573,857 | 5.21 | 0.203x |
+| handwritten | 7.902 | 2,822,254 | 3.4 | 1.000x |
+
+Both the portable baseline comparison and the ordinary 0.18x performance
+floor pass in this sample. `auto` acceleration now stops trying a region after
+repeated short runs, while explicit `strscan` and `regexp` modes remain fixed
+choices for workloads that have measured long runs.

data/docs/reference/actions.md

@@ -43,5 +43,8 @@ instance as `self`.
 | `less(count)` | Keep only the first `count` bytes of the current match |
 | `more` | Join the next match into the current token text |
 
-`less` cannot exceed the current matched byte count. `more` is finalized after
-the action; call `emit` only after the complete text has been assembled.
+`less` cannot exceed the current matched byte count. In UTF-8 mode its result
+must also be a codepoint boundary. Returning to the same byte and state is a
+non-progress error; state-changing rescan cycles are detected as well. `more`
+is finalized after the action; call `emit` only after the complete text has
+been assembled.

data/docs/reference/diagnostics.md

@@ -21,6 +21,14 @@ Use `flexr check SPEC --format json` to consume the structured form.
 | `FLEXR-E017` | Pattern is not statically resolvable | Rewrite as a static expression or use trusted `--eval` |
 | `FLEXR-E018` | Rule pattern or trailing context has an invalid type | Use a regexp, string, or valid array |
 | `FLEXR-E019` | Prism is unavailable for generation | Install the generator dependencies |
+| `FLEXR-E020` | Generated artifact metadata is missing or incompatible | Regenerate with the installed flexr version |
+| `FLEXR-E021` | An action or state cycle made no input progress | Consume input or break the state cycle |
+| `FLEXR-E022` | The retained streaming buffer is too large | Shorten tokens, raise the limit, or disable retention |
+| `FLEXR-E023` | Runtime work exceeded `max_steps` | Raise the limit or simplify the specification |
+| `FLEXR-E024` | Trailing context exceeded its lookahead limit | Bound the context or raise the limit |
+| `FLEXR-E025` | The lexical state stack overflowed | Remove recursive pushes or raise the limit |
+| `FLEXR-E026` | A cancellation callback stopped lexing | Resume with a new lexer when ready |
+| `FLEXR-E027` | An error callback returned an unknown action | Return `:skip`, `:raise`, `:token`, or `:halt` |
 
 ## Warnings
 

data/docs/reference/dsl.md

@@ -25,6 +25,11 @@ Creates one or more named states and registers rules in the block for those
 states. The block is required. Named states are exclusive by default. An
 inclusive state also sees rules registered for `:initial`.
 
+Nested state blocks register their rules for every enclosing state, matching
+Ruby runtime evaluation. Repeating a state declaration with a different
+`inclusive:` value raises immediately instead of silently retaining or replacing
+one declaration.
+
 ## `all_states(&block)`
 
 Registers the block for every state known when the call is evaluated, including
@@ -65,8 +70,9 @@ model is valid.
 
 Adds boolean options to the specification. Stable options include
 `:unicode`, `:eager_columns`, and `:allow_empty_match`. `:experimental` is a
-capability opt-in for experimental behavior such as `:firstmatch`. Unknown
-options are retained in generated configuration but may have no effect.
+capability opt-in for experimental behavior such as `:firstmatch`.
+`:standalone` is build metadata used by standalone generation. Unknown option
+names raise immediately so misspellings cannot silently change behavior.
 
 ## `accel(value)`
 
@@ -76,6 +82,14 @@ source of truth. Affected trailing-context rules cannot use region acceleration.
 
 ## Compilation helpers
 
-`compile!` and `dfa` exist for diagnostics and integration internals. They are
-not required for normal use and are not compatibility-stable; use the lexer
-constructor and token methods instead. See [public API](public-api.md).
+`compile!` and `dfa` exist for diagnostics and integration internals. The first
+compilation makes the specification immutable; later calls to `rule`, `state`,
+`backend`, `option`, and other declaration methods raise
+`Flexr::FrozenSpecificationError`. This keeps the compiled DFA and its DSL
+definition from diverging. These helpers are not required for normal use and
+are not compatibility-stable; use the lexer constructor and token methods
+instead. See [public API](public-api.md).
+
+Subclassing a custom lexer class starts a new, empty specification; rules and
+states are not inherited. This avoids silently changing rule order or state
+membership in the child class.

data/docs/reference/errors.md

@@ -11,13 +11,16 @@ Compile-time failures raise `Flexr::CompileError` or its
 
 `Flexr::LexError` records the message, filename, byte position, line, matched
 text, and optional diagnostic. `Flexr::Runtime::TokenTooLargeError` is a
-`LexError` with code `FLEXR-E012`. `StateStackOverflowError` is raised when a
-`push` would exceed `max_state_stack`.
+`LexError` with code `FLEXR-E012`. Other runtime guard failures use stable codes
+`FLEXR-E021` through `FLEXR-E027`; see the
+[diagnostics catalog](diagnostics.md). `StateStackOverflowError` is raised when
+a `push` would exceed `max_state_stack`.
 
 The default `error_mode: :raise` raises the lexical error. `:token` queues
 `[:error, text]`; `:panic` discards the unmatched byte and continues. An
 `on_error` callback can return `:skip`, `:raise`, `:token`, or `:halt` for
-instance-specific control.
+instance-specific control. Any other result raises
+`InvalidRecoveryActionError` instead of silently applying the default policy.
 
 ## Import failures
 

data/docs/reference/generated-artifacts.md

@@ -27,14 +27,24 @@ Methods with runtime-dependent results, `Time`, IO, environment reads, and
 arbitrary dynamic calls produce `FLEXR-E017`. `--eval` is the explicit escape
 hatch and executes the complete specification.
 
+Constants are resolved in source order, array splats are flattened with Ruby
+semantics, and regexp interpolation retains the embedded regexp's options.
+Receiver-qualified DSL calls, DSL hidden in conditionals or other unsupported
+control flow, unknown DSL keywords, and files containing multiple candidate
+lexer classes fail closed with `FLEXR-E017`. A file intended for static
+generation should contain one lexer class and place DSL statements directly in
+that class or a `state`/`all_states` body.
+
 ## Source transformation
 
 The generator removes DSL call spans, inserts `Flexr::Generated.install_compiled!`,
-and adds generated scanner methods. User actions remain Ruby source. A generated
+and adds generated scanner methods. Source edits use Prism spans, so heredocs and
+the `__END__` data section are not rewritten. User actions remain Ruby source. A generated
 header records:
 
 - source path;
 - SHA-256 digest of the generated payload;
+- artifact schema and runtime ABI versions;
 - vendored Unicode version;
 - effective backend;
 - whether compilation used eval; and
@@ -43,8 +53,25 @@ header records:
 Do not edit generated files by hand. Regenerate them when the specification,
 generator, backend, table format, or Unicode snapshot changes.
 
+Compiled payloads also record the artifact schema, compiler version, runtime ABI,
+and Unicode version. Loading fails with `FLEXR-E020` when those contracts are
+missing or incompatible. This prevents a stale artifact from silently running
+against a runtime with different table or Unicode semantics.
+
+Generation refuses to write through the input path, including aliases made with
+symbolic or hard links. Outputs are written to a file in the destination directory,
+synced, and atomically renamed. An unchanged output is left untouched so build
+timestamps remain stable. For input names other than `*.flexr.rb`, the default
+output is `*.generated.rb`.
+
 ## Table formats
 
 `--table-compression none|rows|full` controls row packing. `--table-format
 literal|packed` controls whether packed arrays are emitted as Ruby literals or
-Base64. These affect artifact size and loading cost, not matching semantics.
+Base64. Packed artifacts retain and query their packed representation; the
+dense row view is populated lazily only when an inspection API requests a row.
+These settings affect artifact size and loading cost, not matching semantics.
+
+Standalone output embeds a runtime core, not the compiler or generator. The
+regexp reference matcher and vendored Unicode tables are added only for a
+`firstmatch` backend or trailing-context rules that can use that matcher.

data/docs/reference/public-api.md

@@ -20,6 +20,10 @@ else may change between releases without a deprecation cycle.
 Method arguments, return shapes, and diagnostics for this surface are described
 in the reference pages and are covered by the test suite.
 
+`require "flexr"` keeps the build tools lazy while preserving all public
+constants through autoload. Applications that want an explicit load boundary
+can require `flexr/runtime`, `flexr/generator`, or `flexr/cli` directly.
+
 ## Experimental
 
 - `backend :firstmatch` and `--backend firstmatch`.
@@ -27,8 +31,9 @@ in the reference pages and are covered by the test suite.
 - Any future backend or option explicitly marked experimental in its reference.
 
 Experimental behavior may change semantics or output between minor releases.
-`firstmatch` requires the opt-in and generation performs a deterministic
-differential check against the table matcher.
+`firstmatch` requires the opt-in. Runtime and generated `firstmatch` modes use
+the same rule-order semantics; overlap with longest-match behavior is reported
+as `FLEXR-W010` rather than making generation fail.
 
 ## Internal; compatibility is not guaranteed
 

data/docs/reference/regexp.md

@@ -7,14 +7,14 @@ byte-oriented automaton. All matches start at the current lexer position.
 |---|---|---|
 | Literals, concatenation, alternation | Supported | |
 | Groups and `(?:...)` | Supported | Captures are treated as non-capturing and warn |
-| Character classes and ranges | Supported | POSIX classes are supported; see encoding notes |
+| Character classes and ranges | Supported | Includes trailing `-`, POSIX classes, and `&&` intersection |
 | Unicode `\p{...}` / `\P{...}` | Supported | Uses the vendored UCD snapshot |
 | `*`, `+`, `?` | Supported | Greedy forms only |
-| `{n}`, `{n,m}` | Supported | The upper bound is 1000 |
-| `{n,}` | Unsupported | Split the rule or use states |
+| `{n}`, `{n,m}`, `{n,}` | Supported | Explicit bounds may not exceed 1000 |
 | `^` / `$` | Restricted | Only at outermost pattern boundaries; otherwise `FLEXR-E009` |
 | `/i`, `/m`, `/x`, `/n` | Supported | Inline forms are supported where Ruby permits them |
 | `\d`, `\w`, `\s` | Supported | Unicode-aware with `option :unicode` in UTF-8 |
+| `\h`, `\H` | Supported | Hexadecimal digits and their complement |
 | Lookahead / lookbehind | Unsupported | Use `followed_by:` or a state; `FLEXR-E014` |
 | Backreferences | Unsupported | Split the language into states or actions |
 | `\b`, `\B`, `\A`, `\z`, `\G`, `\K` | Unsupported | Use rule boundaries or state/action logic |
@@ -23,14 +23,16 @@ byte-oriented automaton. All matches start at the current lexer position.
 
 Malformed syntax produces `FLEXR-E001`. Unsupported constructs produce
 `FLEXR-E014`; the diagnostic help normally names the supported replacement.
+Unknown alphabetic escapes are rejected rather than silently treated as
+literals; escaped regexp punctuation remains supported.
 Empty matches are rejected with `FLEXR-E005` unless `option :allow_empty_match`
 is explicitly enabled, in which case flexr advances to guarantee progress.
 
 ## Bounded repetition
 
-`{n,m}` expands into a finite automaton. `m` must be at least `n` and no greater
-than 1000. Open repetition is rejected because it is easy to express with `+`
-or a separate rule and otherwise gives a misleading compatibility promise.
+Repetition remains a symbolic AST node until NFA construction, avoiding large
+intermediate trees. In `{n,m}`, `m` must be at least `n`; explicit bounds may
+not exceed 1000. `{n,}` uses an unbounded loop after the required copies.
 
 ## Captures