flexr 1.0.0

data/benchmark/baselines/json.json

@@ -0,0 +1,34 @@
+{
+  "schema": 1,
+  "spec": "examples/json/lexer.flexr.rb",
+  "source_sha256": "909c8fbc07f27e5be871a6f7e2bb4d0c29996e4362dbb201dd90edd1ae55c7e3",
+  "source_bytes": 745,
+  "input_sha256": "f15489cc8fa286cd365fb167b3f99d3d216dc90c0de6279fe9f36c8d2215d7a7",
+  "input_bytes": 140000,
+  "tokens": 50000,
+  "iterations": 3,
+  "modes": {
+    "runtime": {
+      "tokens": 50000,
+      "mb_per_s": 0.889,
+      "tokens_per_s": 317556,
+      "allocations_per_token": 5.601
+    },
+    "generated": {
+      "tokens": 50000,
+      "mb_per_s": 0.936,
+      "tokens_per_s": 334309,
+      "allocations_per_token": 4.601
+    },
+    "handwritten": {
+      "tokens": 50000,
+      "mb_per_s": 8.772,
+      "tokens_per_s": 3132963,
+      "allocations_per_token": 3.4
+    }
+  },
+  "relative_to_handwritten": {
+    "runtime": 0.101,
+    "generated": 0.107
+  }
+}

data/benchmark/baselines/json_handwritten.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module FlexrBenchmark
+  class JsonHandwrittenLexer
+    WHITESPACE = /[ \t\r\n]+/
+    NUMBER = /-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/
+    STRING = /"(?:\\.|[^"\\])*"/
+
+    def initialize(input)
+      @scanner = StringScanner.new(input)
+    end
+
+    def tokens
+      result = []
+      token = nil
+      result << token while (token = next_token)
+      result
+    end
+
+    private
+
+    def next_token
+      return if @scanner.eos?
+      return next_token if @scanner.scan(WHITESPACE)
+
+      return [:LBRACE, @scanner.matched] if @scanner.scan("{")
+      return [:RBRACE, @scanner.matched] if @scanner.scan("}")
+      return [:LBRACKET, @scanner.matched] if @scanner.scan("[")
+      return [:RBRACKET, @scanner.matched] if @scanner.scan("]")
+      return [:COLON, @scanner.matched] if @scanner.scan(":")
+      return [:COMMA, @scanner.matched] if @scanner.scan(",")
+      return [:TRUE, true] if @scanner.scan("true")
+      return [:FALSE, false] if @scanner.scan("false")
+      return [:NULL, nil] if @scanner.scan("null")
+      return [:NUMBER, @scanner.matched.to_f] if @scanner.scan(NUMBER)
+      return [:STRING, @scanner.matched.byteslice(1...-1)] if @scanner.scan(STRING)
+
+      raise ArgumentError, "invalid JSON at byte #{@scanner.pos}"
+    end
+  end
+end

data/benchmark/baselines/json_rexical.rex

@@ -0,0 +1,25 @@
+# Optional Rexical comparison specification for the JSON corpus.
+# The action values intentionally mirror examples/json/lexer.flexr.rb.
+
+class JsonRexicalLexer
+  macro
+    digit [0-9]
+    number -?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?
+    string "(\\.|[^"\\])*"
+  end
+
+  rule
+    [ \t\r\n]+       { /* skip */ }
+    \{                { return [:LBRACE, yytext] }
+    \}                { return [:RBRACE, yytext] }
+    \[                { return [:LBRACKET, yytext] }
+    \]                { return [:RBRACKET, yytext] }
+    :                 { return [:COLON, yytext] }
+    ,                 { return [:COMMA, yytext] }
+    true              { return [:TRUE, true] }
+    false             { return [:FALSE, false] }
+    null              { return [:NULL, nil] }
+    {number}          { return [:NUMBER, yytext.to_f] }
+    {string}          { return [:STRING, yytext[1...-1]] }
+  end
+end

data/benchmark/corpora/README.md

@@ -0,0 +1,11 @@
+# Benchmark corpora
+
+The 10MB JSON corpus is deterministic and generated on demand so the repository
+does not need to carry a large derived blob:
+
+```sh
+ruby benchmark/corpora/generate_json.rb > benchmark/corpora/json_10mb.json
+```
+
+The default size is exactly 10,000,000 bytes. Set `FLEXR_JSON_BYTES` to produce
+a different deterministic size for local experiments.

data/benchmark/corpora/generate_json.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Generates the deterministic JSON corpus used by the benchmark plan.
+#
+#   ruby benchmark/corpora/generate_json.rb > benchmark/corpora/json_10mb.json
+
+TARGET_BYTES = Integer(ENV.fetch("FLEXR_JSON_BYTES", "10_000_000"), 10)
+PADDING_BYTES = Integer(ENV.fetch("FLEXR_JSON_PADDING_BYTES", "0"), 10)
+abort "FLEXR_JSON_PADDING_BYTES must not be negative" if PADDING_BYTES.negative?
+
+RECORD = %({"answer":42#{' ' * PADDING_BYTES},"name":"flexr"}).freeze
+
+abort "FLEXR_JSON_BYTES must be at least #{RECORD.bytesize + 3}" if RECORD.bytesize + 3 > TARGET_BYTES
+
+json = +"["
+first = true
+while json.bytesize + RECORD.bytesize + 1 < TARGET_BYTES
+  json << "," unless first
+  json << RECORD
+  first = false
+end
+json << (" " * (TARGET_BYTES - json.bytesize - 1))
+json << "]"
+abort "generated corpus has #{json.bytesize} bytes, expected #{TARGET_BYTES}" unless json.bytesize == TARGET_BYTES
+
+print json

data/benchmark/golden/calculator_lexer.sha256

@@ -0,0 +1 @@
+6359976d9cdc72772eff8c72cf421d965e913febad3dbc0ca0f2057a0faea3b4

data/benchmark/golden/json_lexer.sha256

@@ -0,0 +1 @@
+913cf85ba7e1a3128eea0a4b06077486fead2d56357b669685ae6e26bdd55715

data/benchmark/golden/regexp_tokenizer.sha256

@@ -0,0 +1 @@
+7f20a1024eea1d674aee73a2baf7a32c52fa178f2d158d24e49f9c9b77a63960

data/benchmark/golden/ruby_subset_lexer.sha256

@@ -0,0 +1 @@
+0d99b92c658d56cbef7d9c33ac3e5770d83e379b94cf412f8b028f59d44910b8

data/benchmark/golden/toy_lang_lexer.sha256

@@ -0,0 +1 @@
+16943c2d8a86999148dd6bcd2ed644671032d1e6b4acbe891d180ff0810cd97a

data/benchmark/golden/with_lrama_lexer.sha256

@@ -0,0 +1 @@
+7627b0fafbfdbdea850d02e67c3e9c24eab8512cdc74571a986323d397848071

data/benchmark/golden/with_racc_lexer.sha256

@@ -0,0 +1 @@
+037fa90c634117522c1e882d08c437c7b79653f59ca94e6cf6c115cd7897610f

data/benchmark/run.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "tempfile"
+
+require_relative "../lib/flexr"
+require_relative "baselines/json_handwritten"
+
+module Flexr
+  module Benchmarking
+    DEFAULT_SPEC = File.expand_path("../examples/json/lexer.flexr.rb", __dir__)
+    DEFAULT_INPUTS = {
+      %r{/examples/json/} => ('{"answer": 42}' * 10_000).freeze,
+      %r{/examples/toy_lang/} => ("answer + 12" * 10_000).freeze
+    }.freeze
+
+    module_function
+
+    def run(argv, out: $stdout, err: $stderr)
+      options = parse(argv.dup)
+      return usage(out) if options[:help]
+      if options[:baseline] && !File.file?(options[:baseline])
+        err.puts "benchmark error: baseline is missing or invalid"
+        return 2
+      end
+
+      result = Runner.new(options).run
+      Baseline.write(options[:write_baseline], result) if options[:write_baseline]
+      floor = ENV["FLEXR_BENCHMARK_PORTABLE"] == "1" ? 0.0 : Baseline::PERFORMANCE_FLOOR
+      status = Baseline.check(options[:baseline], result, threshold: options[:threshold], floor: floor)
+      emit(result, out, json: options[:json])
+      return status if status.zero?
+
+      err.puts status == 2 ? "benchmark error: baseline is missing or invalid" : "benchmark regression detected"
+      status
+    rescue ArgumentError => e
+      err.puts "benchmark error: #{e.message}"
+      usage(err, status: 2)
+    rescue Errno::ENOENT => e
+      err.puts "benchmark error: #{e.message}"
+      2
+    rescue StandardError => e
+      err.puts "benchmark error: #{e.class}: #{e.message}"
+      1
+    end
+
+    def parse(args)
+      options = {
+        spec: DEFAULT_SPEC, input_file: nil, baseline: nil, write_baseline: nil,
+        iterations: 3, threshold: 0.10, json: false, help: false
+      }
+      until args.empty?
+        case (argument = args.shift)
+        when "--help", "-h"
+          options[:help] = true
+        when "--spec"
+          options[:spec] = required_argument!(args, argument)
+        when "--input-file"
+          options[:input_file] = required_argument!(args, argument)
+        when "--baseline"
+          options[:baseline] = required_argument!(args, argument)
+        when "--write-baseline"
+          options[:write_baseline] = required_argument!(args, argument)
+        when "--iterations"
+          options[:iterations] = Integer(required_argument!(args, argument), 10)
+          raise ArgumentError, "iterations must be positive" unless options[:iterations].positive?
+        when "--threshold"
+          options[:threshold] = Float(required_argument!(args, argument))
+          raise ArgumentError, "threshold must be between 0 and 1" unless options[:threshold].between?(0, 1)
+        when "--json"
+          options[:json] = true
+        else
+          raise ArgumentError, "unknown option: #{argument}"
+        end
+      end
+      options
+    end
+
+    def required_argument!(args, option)
+      value = args.shift
+      raise ArgumentError, "#{option} requires a value" if value.nil? || value.start_with?("-")
+
+      value
+    end
+
+    def emit(result, out, json:)
+      if json
+        out.puts JSON.pretty_generate(result)
+        return
+      end
+
+      out.puts "spec: #{result.fetch('spec')}"
+      out.puts "input_bytes: #{result.fetch('input_bytes')}, tokens: #{result.fetch('tokens')}"
+      result.fetch("modes").each do |mode, metrics|
+        out.puts "#{mode}: #{metrics.fetch('mb_per_s')} MB/s, #{metrics.fetch('tokens_per_s')} tokens/s, " \
+          "#{metrics.fetch('allocations_per_token')} allocations/token"
+      end
+      if (ratios = result["relative_to_handwritten"])
+        out.puts "relative_to_handwritten: runtime=#{ratios.fetch('runtime')}x, generated=#{ratios.fetch('generated')}x"
+      end
+    end
+
+    def usage(out, status: 0)
+      out.puts "Usage: ruby benchmark/run.rb [--spec PATH] [--baseline PATH] [--write-baseline PATH] [--json]"
+      status
+    end
+
+    class Runner
+      def initialize(options)
+        @options = options
+      end
+
+      def run
+        source_path = File.expand_path(@options.fetch(:spec))
+        source = File.binread(source_path)
+        input = input_for(source_path)
+        runtime = measure(source_path, input, generated: false)
+        generated = measure(source_path, input, generated: true)
+        {
+          "schema" => 1,
+          "spec" => relative_path(source_path),
+          "source_sha256" => Digest::SHA256.hexdigest(source),
+          "source_bytes" => source.bytesize,
+          "input_sha256" => Digest::SHA256.hexdigest(input),
+          "input_bytes" => input.bytesize,
+          "tokens" => runtime.fetch("tokens"),
+          "iterations" => @options.fetch(:iterations),
+          "modes" => { "runtime" => runtime, "generated" => generated }
+        }.tap do |result|
+          next unless source_path.match?(%r{/examples/json/})
+
+          handwritten = measure_class(FlexrBenchmark::JsonHandwrittenLexer, input)
+          result["modes"]["handwritten"] = handwritten
+          result["relative_to_handwritten"] = {
+            "runtime" => (runtime.fetch("mb_per_s") / handwritten.fetch("mb_per_s")).round(3),
+            "generated" => (generated.fetch("mb_per_s") / handwritten.fetch("mb_per_s")).round(3)
+          }
+        end
+      end
+
+      private
+
+      def input_for(source_path)
+        return File.binread(@options.fetch(:input_file)) if @options[:input_file]
+
+        DEFAULT_INPUTS.each do |pattern, input|
+          return input if source_path.match?(pattern)
+        end
+        raise ArgumentError, "--input-file is required for #{source_path}"
+      end
+
+      def measure(source_path, input, generated:)
+        klass = generated ? load_generated(source_path) : load_runtime(source_path)
+        measure_class(klass, input)
+      end
+
+      def measure_class(klass, input)
+        iterations = @options.fetch(:iterations)
+        iterations.times { klass.new(input).tokens }
+        GC.start
+        allocated_before = GC.stat.fetch(:total_allocated_objects)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        tokens = nil
+        iterations.times { tokens = klass.new(input).tokens }
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+        allocated = GC.stat.fetch(:total_allocated_objects) - allocated_before
+        bytes_per_second = input.bytesize * iterations / [elapsed, Float::EPSILON].max
+        token_count = tokens.length
+        {
+          "tokens" => token_count,
+          "mb_per_s" => (bytes_per_second / 1_000_000).round(3),
+          "tokens_per_s" => (token_count * iterations / [elapsed, Float::EPSILON].max).round,
+          "allocations_per_token" => (allocated.to_f / [token_count * iterations, 1].max).round(3)
+        }
+      end
+
+      def load_runtime(source_path)
+        before = lexer_classes
+        load source_path
+        @runtime_class = lexer_classes.difference(before).last
+        @runtime_class || raise(ArgumentError, "no lexer class found in #{source_path}")
+      end
+
+      def load_generated(source_path)
+        generated = Flexr::Generator.new(source_path).generate
+        remove_constant(@runtime_class.name) if @runtime_class
+        before = lexer_classes
+        Tempfile.create(["flexr-benchmark-", ".rb"]) do |file|
+          file.write(generated)
+          file.flush
+          load file.path
+        end
+        (lexer_classes - before).last || raise(ArgumentError, "generated lexer class not found")
+      end
+
+      def lexer_classes
+        ObjectSpace.each_object(Class).select { |klass| klass.respond_to?(:__flexr_spec) }
+      end
+
+      def remove_constant(name)
+        parts = name.split("::")
+        parent = Object
+        parts[0...-1].each { |part| parent = parent.const_get(part) }
+        parent.send(:remove_const, parts.last) if parent.const_defined?(parts.last, false)
+      end
+
+      def relative_path(path)
+        root = File.expand_path("..", __dir__)
+        path.start_with?("#{root}/") ? path.delete_prefix("#{root}/") : path
+      end
+    end
+
+    module Baseline
+      PERFORMANCE_FLOOR = 0.18
+      FLOOR_MODES = %w[runtime generated].freeze
+
+      module_function
+
+      def check(path, result, threshold:, floor: PERFORMANCE_FLOOR)
+        return 0 unless path
+        return 2 unless File.file?(path)
+
+        baseline = JSON.parse(File.read(path))
+        return 2 unless baseline["schema"] == 1
+        return 1 unless identity_matches?(baseline, result)
+
+        baseline_ratios = baseline.fetch("relative_to_handwritten")
+        FLOOR_MODES.each do |mode|
+          actual = Float(result.fetch("relative_to_handwritten").fetch(mode))
+          expected = Float(baseline_ratios.fetch(mode)) * (1.0 - threshold)
+          return 1 if actual < [floor, expected].max
+        end
+        0
+      rescue JSON::ParserError, KeyError, TypeError, ArgumentError
+        2
+      end
+
+      def write(path, result)
+        FileUtils.mkdir_p(File.dirname(File.expand_path(path)))
+        File.write(path, "#{JSON.pretty_generate(result)}\n")
+      end
+
+      def identity_matches?(baseline, result)
+        %w[spec source_sha256 source_bytes input_sha256 input_bytes tokens].all? do |key|
+          baseline[key] == result[key]
+        end
+      end
+    end
+  end
+end
+
+exit Flexr::Benchmarking.run(ARGV) if $PROGRAM_NAME == __FILE__

data/docs/README.md

@@ -0,0 +1,64 @@
+# flexr documentation
+
+The documentation is organized by reader intent. The executable specifications
+under `examples/` are the source of truth for commands and token output; prose
+pages explain how to use those specifications.
+
+## Choose a path
+
+| You want to... | Read |
+|---|---|
+| Build your first lexer | [Build a calculator lexer](tutorial/build-a-calculator-lexer.md) |
+| Run a specification | [Run at runtime](how-to/run-a-lexer-at-runtime.md) |
+| Generate Ruby | [Generate a lexer](how-to/generate-a-lexer.md) |
+| Ship one file | [Deploy a standalone lexer](how-to/deploy-a-standalone-lexer.md) |
+| Add states or trailing context | [States](how-to/use-states.md), [trailing context](how-to/use-trailing-context.md) |
+| Integrate a parser | [Racc](how-to/integrate-with-racc.md), [Lrama](how-to/integrate-with-lrama.md) |
+| Migrate an existing lexer | [flex](how-to/migrate-from-flex.md), [Rexical](how-to/migrate-from-rexical.md) |
+| Look up exact behavior | [Reference](reference/README.md) |
+| Understand design trade-offs | [Explanation](explanation/) |
+
+## Tutorial
+
+- [Build a calculator lexer](tutorial/build-a-calculator-lexer.md)
+
+## How-to
+
+- [Run a lexer at runtime](how-to/run-a-lexer-at-runtime.md)
+- [Generate a lexer](how-to/generate-a-lexer.md)
+- [Deploy a standalone lexer](how-to/deploy-a-standalone-lexer.md)
+- [Use states](how-to/use-states.md)
+- [Use trailing context](how-to/use-trailing-context.md)
+- [Handle errors](how-to/handle-errors.md)
+- [Track token locations](how-to/track-token-locations.md)
+- [Integrate with Racc](how-to/integrate-with-racc.md)
+- [Integrate with Lrama](how-to/integrate-with-lrama.md)
+- [Migrate from flex](how-to/migrate-from-flex.md)
+- [Migrate from Rexical](how-to/migrate-from-rexical.md)
+- [Tune performance](how-to/tune-performance.md)
+
+## Reference
+
+- [DSL](reference/dsl.md)
+- [Runtime](reference/runtime.md)
+- [Actions](reference/actions.md)
+- [Tokens and locations](reference/tokens-and-locations.md)
+- [Errors](reference/errors.md)
+- [Regexp compatibility](reference/regexp.md)
+- [CLI](reference/cli.md)
+- [Diagnostics](reference/diagnostics.md)
+- [Generated artifacts](reference/generated-artifacts.md)
+- [Public API](reference/public-api.md)
+- [Compatibility](reference/compatibility.md)
+
+## Explanation
+
+- [Matching semantics](explanation/matching-semantics.md)
+- [Runtime versus generated](explanation/runtime-vs-generated.md)
+- [Backends](explanation/backends.md)
+- [Unicode and encoding](explanation/unicode-and-encoding.md)
+- [Security model](explanation/security-model.md)
+
+The [performance log](perf-log.md), [architecture decisions](adr/),
+[internals notes](internals/README.md), and [release procedure](RELEASING.md)
+are maintainer-facing documents.

data/docs/RELEASING.md

@@ -0,0 +1,30 @@
+# Releasing
+
+The release process validates both implementation and user-facing contracts.
+Run these commands from a clean checkout on the target Ruby versions:
+
+```sh
+bundle exec rake test
+bundle exec rubocop
+bundle exec rake docs:verify
+bundle exec rake modes:equivalence golden:verify accel:equivalence dogfood:verify
+bundle exec rake generated:verify unicode:verify examples:check
+bundle exec rake test:differential fuzz
+bundle exec rake bench:regression
+```
+
+`dot:verify` is required when Graphviz is available. It parses the generated DOT
+output as SVG and should be run before a release that changes CLI visualization.
+
+Then:
+
+1. Review the commits since the previous release and update `lib/flexr/version.rb`.
+2. Verify the gemspec metadata URLs.
+3. Build and inspect the package with `gem build flexr.gemspec`.
+4. Confirm generated golden files and the Unicode snapshot are intentional.
+5. Create the release tag only after the working tree and CI are clean.
+
+Unicode snapshot changes are minor-version compatibility changes because they
+can change generated output. Stable runtime and generated-file contracts are
+preserved within a major version unless this document and the release commits
+state a migration.

data/docs/adr/0001-byte-level-dfa.md

@@ -0,0 +1,5 @@
+# ADR 0001: Byte-level DFA
+
+The lexer automaton consumes bytes. This keeps the hot path independent of
+Ruby string encoding, gives invalid UTF-8 a deterministic one-byte fallback,
+and enables binary region acceleration.

data/docs/adr/0003-leftmost-longest.md

@@ -0,0 +1,4 @@
+# ADR 0003: Leftmost-longest matching
+
+At each input position the longest match wins. Source order breaks equal-length
+ties. This is the defining semantic difference from first-match lexers.

data/docs/adr/0006-accel-not-scanner.md

@@ -0,0 +1,4 @@
+# ADR 0006: Acceleration is an optimization, not a matcher
+
+Region acceleration may skip self-loop bytes, but the DFA remains the source
+of truth. Acceleration can be disabled for equivalence tests.

data/docs/adr/0008-what-pure-ruby-means.md

@@ -0,0 +1,5 @@
+# ADR 0008: Meaning of Pure Ruby
+
+Generated lexers and runtime mode use Ruby and its standard library only.
+Prism is a generator-side dependency and is never needed to run generated
+output.

data/docs/adr/0016-spec-is-plain-ruby.md

@@ -0,0 +1,4 @@
+# ADR 0016: Specifications are plain Ruby
+
+The specification format is an internal Ruby DSL. Constants, methods,
+requires, comments, and normal Ruby tooling remain available to the author.

data/docs/adr/0017-static-analysis-by-default.md

@@ -0,0 +1,5 @@
+# ADR 0017: Static generation by default
+
+The generator resolves only deterministic expressions. Dynamic construction is
+reported as `FLEXR-E017`; `--eval` is explicit because it executes user code
+and can make output depend on the build environment.

data/docs/adr/0018-prism-for-generator-only.md

@@ -0,0 +1,4 @@
+# ADR 0018: Prism is generator-only
+
+Prism reads source during AOT generation. Runtime mode receives real Ruby
+Regexp and Proc objects, so it remains dependency-free.

data/docs/adr/0019-measured-performance-floor.md

@@ -0,0 +1,26 @@
+# ADR 0019: Measured performance floor for the pure-Ruby backend
+
+## Context
+
+The original design used `0.7x` of the handwritten `StringScanner` lexer as a
+v1.0 acceptance target. The reproducible local benchmark on 2026-08-10
+measured `0.187x` for generated mode and `0.194x` for runtime mode. The
+runtime/generated ratio was `1.042x`. These figures are recorded in
+`docs/perf-log.md` and guarded by the benchmark regression baseline.
+
+## Decision
+
+The handwritten comparison is a stretch target, not a release claim. The v1.0
+performance floor is a measured, regression-tested `0.18x` handwritten ratio;
+the `0.7x` target remains open for a future optimization milestone. The
+runtime/generated relationship remains governed by the existing `0.2x` risk
+threshold.
+
+This revision preserves an honest acceptance criterion: the project must not
+claim to have reached `0.7x`, and benchmark output must continue to be recorded
+from an actual run.
+
+The absolute floor is enforced by the local release gate. CI uses a portable
+variant that keeps the 10% baseline-regression check but omits the absolute
+throughput floor because hosted runners are not a stable performance
+measurement environment.

data/docs/adr/0020-vendored-unicode-contract.md

@@ -0,0 +1,21 @@
+# ADR 0020: Vendored Unicode is the compatibility contract
+
+## Context
+
+Ruby's `Regexp` Unicode tables are supplied by the host runtime and can change
+between Ruby releases. A lexer generated with the same flexr source must not
+change merely because it was built on another Ruby version. The repository
+therefore vendors a Unicode Character Database snapshot.
+
+## Decision
+
+The vendored Unicode Character Database, currently version `15.1.0`, is the
+oracle for `\p{...}`, POSIX properties, Unicode shorthands under
+`option :unicode`, and case folding. Host Ruby `Regexp` is not used as a
+cross-version property oracle. Updating the snapshot is a minor-version
+compatibility change and must regenerate golden outputs.
+
+`rake unicode:verify` checks the snapshot version, range invariants, all scalar
+singleton UTF-8 expansions, and 100,000 deterministic random ranges. The
+source-of-truth tables can be regenerated with `tools/gen_unicode_tables.rb`
+from an explicitly supplied Unicode Character Database directory.

data/docs/explanation/backends.md

@@ -0,0 +1,33 @@
+# Backends
+
+## `table`
+
+The stable default. It stores DFA transitions by byte class and can apply row
+compression. It is the clearest reference for semantics and is the comparison
+target for `firstmatch`.
+
+## `direct`
+
+The stable dense-dispatch backend. It emits a flattened dispatch structure for
+large machines and can reduce lookup overhead at the cost of artifact size.
+It preserves the table backend's longest-match behavior.
+
+## `auto`
+
+The stable selection policy. At compile time it chooses direct dispatch when the
+largest state-by-byte-class table exceeds the configured threshold, otherwise
+it uses table dispatch. The effective backend is recorded in generated headers.
+
+## `firstmatch`
+
+Experimental compatibility behavior. It requires `option :experimental` and
+can return a shorter match merely because its rule appears earlier. Generation
+compares a deterministic input corpus with the table matcher and aborts if the
+results differ.
+
+## Acceleration and packing
+
+Region acceleration and packed tables are representations, not semantic
+backends. `accel` can be disabled for debugging or equivalence tests. Table
+compression and packed Base64 output trade source size against loading cost;
+they do not change token streams.

data/docs/explanation/matching-semantics.md

@@ -0,0 +1,20 @@
+# Matching semantics
+
+flexr implements leftmost-longest matching. At the current byte position, the
+automaton records every accepting rule reached while scanning forward. The
+last accepting position is the candidate token end. The rule with the smallest
+source index wins when several candidates end at that position.
+
+This is why `==` beats `=` regardless of which rule is written first, while two
+same-length alternatives use definition order. A rule's `followed_by:` context
+extends the selection end without extending the consumed token end.
+
+The matcher is byte-oriented so binary input and invalid UTF-8 have deterministic
+behavior. UTF-8 patterns still require valid codepoint boundaries before a
+match is accepted. The runtime and generated scanner share the compiled model;
+acceleration may skip self-loop bytes but cannot change the winner.
+
+`firstmatch` is intentionally separate. It picks the first rule that has a
+matching alternative and therefore is useful only for compatibility migrations
+that accept the semantic difference. It is experimental and guarded by a
+differential check during generation.