flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
data/docs/perf-log.md CHANGED
@@ -1,5 +1,58 @@
1
1
  # Performance log
2
2
 
3
+ ## State-specific dispatch experiment (2026-11-20)
4
+
5
+ `benchmark/state_dispatch.rb` compares generated table dispatch, the existing
6
+ flattened direct representation, and per-state Ruby `case` branches. It checks
7
+ token equality before timing. The transition test compares every state/byte pair.
8
+ It is a development experiment and does not add a public backend.
9
+
10
+ ```sh
11
+ bundle exec ruby benchmark/state_dispatch.rb --iterations 5
12
+ RUBYOPT=--yjit bundle exec ruby benchmark/state_dispatch.rb --iterations 5
13
+ RUBYOPT=--zjit bundle exec ruby benchmark/state_dispatch.rb --iterations 5
14
+ ```
15
+
16
+ Measured on Ruby 4.0.9, arm64-darwin25, built-in 140,000-byte JSON input:
17
+
18
+ | Engine | Table MB/s | Direct MB/s | State cases MB/s |
19
+ |---|---:|---:|---:|
20
+ | Interpreter | 1.170 | 1.729 | 1.709 |
21
+ | YJIT | 1.922 | 2.796 | 2.768 |
22
+ | ZJIT | 0.458 | 0.425 | 0.483 |
23
+
24
+ These short local samples are not release qualification or confidence intervals.
25
+ State cases do not improve on the existing direct representation in the
26
+ interpreter or YJIT samples, and ZJIT needs longer warm-up investigation.
27
+ Keep the existing direct backend and `auto` selection until repeatable gains
28
+ justify adding and maintaining another backend. No regression baseline changes
29
+ are warranted by this experiment.
30
+ ## Lrama standalone recognizer experiment (2026-11-20)
31
+
32
+ `benchmark/lrama_lexer.rb` builds a standalone grammar-token recognizer and
33
+ adapts Lrama's lexer to use it for normal grammar tokens. Existing trivia,
34
+ C-code scanning, identifier lookahead, token classes, locations, and parser
35
+ are retained. The adapter deliberately uses private scanner positions; this
36
+ is an experiment, not a supported Lrama integration or a production replacement.
37
+
38
+ ```sh
39
+ git clone https://github.com/ruby/lrama.git /tmp/lrama
40
+ git -C /tmp/lrama checkout 22de9d99f6402290a637627f3516a1a2db6c9aab
41
+ bundle exec ruby benchmark/lrama_lexer.rb /tmp/lrama
42
+ ```
43
+
44
+ The pinned upstream revision's 78 grammar fixtures produced identical token
45
+ kinds, values, locations, and parser errors: 77 parsed successfully, and 3,767
46
+ tokens were compared. Both paths include the same parser and event collection.
47
+ On Ruby 4.0.9, arm64-darwin25, three corpus passes took 0.0434 seconds with
48
+ Lrama's scanner and 11.9124 seconds with the standalone recognizer adapter.
49
+ These short local timings are illustrative, not a statistical benchmark.
50
+
51
+ This approach is substantially slower and adds adapter maintenance, so do not
52
+ propose it upstream. It adds no Lrama runtime dependency to flexr. The tool
53
+ checks the upstream scan layout and fails explicitly if it changes; a future
54
+ attempt should first remove per-token adapter and firstmatch overhead.
55
+
3
56
  The benchmark harness reports bytes/s, tokens/s, and allocations per token.
4
57
  Numbers are intentionally recorded only after running the harness on the
5
58
  target Ruby and hardware; no unmeasured figures are presented as guarantees.
@@ -10,6 +63,23 @@ Run:
10
63
  ruby -Ilib benchmark/run.rb
11
64
  ```
12
65
 
66
+ For independent Ruby interpreter, YJIT, and ZJIT processes, run
67
+ `bundle exec rake bench:matrix`. The JSON report checks that each engine is
68
+ actually enabled and marks unavailable builds explicitly. It does not replace
69
+ the baseline with a different engine's measurements. Use
70
+ `ruby benchmark/matrix.rb --input-file PATH --iterations 3` for a custom input.
71
+
72
+ `bundle exec rake bench:profile` provides an optional TracePoint profile without
73
+ an extra gem. It excludes class loading and compilation after a warm-up run and
74
+ reports non-overlapping time in scanning, actions, token handling, locations,
75
+ and other work for runtime and generated modes. Scanning includes the DFA byte
76
+ loop and matcher overhead; token handling includes `next_token`, `emit`, and
77
+ text materialization; locations include position updates and lazy location
78
+ construction. These are instrumented costs: TracePoint adds substantial
79
+ overhead and changes JIT behavior, so do not compare these times with normal
80
+ throughput. Use `ruby benchmark/profile.rb --input-file PATH --iterations 1`
81
+ to profile a custom input. `FLEXR_BENCHMARK_ITERATIONS` controls either rake task.
82
+
13
83
  The JSON result includes `mb_per_s`, `tokens_per_s`, and cumulative Ruby
14
84
  `allocations_per_token` for runtime, generated, and handwritten modes. JSON
15
85
  also records each Flexr mode as a ratio against the handwritten baseline.
@@ -70,3 +140,129 @@ Both the portable baseline comparison and the ordinary 0.18x performance
70
140
  floor pass in this sample. `auto` acceleration now stops trying a region after
71
141
  repeated short runs, while explicit `strscan` and `regexp` modes remain fixed
72
142
  choices for workloads that have measured long runs.
143
+
144
+ ## 2026-11-20
145
+
146
+ On Ruby 4.0.9 (`3e93b23afc`, arm64-darwin25), revision `0391a6e`, the default
147
+ 140,000-byte JSON input produced 50,000 tokens. `ruby benchmark/matrix.rb
148
+ --iterations 3` warmed each mode for three iterations, then measured three
149
+ iterations in a separate process per engine. All three engines were available.
150
+ This is one local sample from a development session, not an isolated performance
151
+ qualification or a reason to raise the regression baseline.
152
+
153
+ | Engine | Mode | MB/s | allocations/token | handwritten ratio |
154
+ |---|---|---:|---:|---:|
155
+ | Interpreter | runtime | 1.700 | 5.200 | 0.204x |
156
+ | Interpreter | generated | 1.668 | 5.210 | 0.200x |
157
+ | Interpreter | handwritten | 8.343 | 3.400 | 1.000x |
158
+ | YJIT | runtime | 4.789 | 5.200 | 0.464x |
159
+ | YJIT | generated | 2.894 | 5.210 | 0.281x |
160
+ | YJIT | handwritten | 10.316 | 3.400 | 1.000x |
161
+ | ZJIT | runtime | 1.958 | 5.200 | 0.203x |
162
+ | ZJIT | generated | 0.414 | 5.210 | 0.043x |
163
+ | ZJIT | handwritten | 9.639 | 3.400 | 1.000x |
164
+
165
+ The input SHA-256 is
166
+ `f15489cc8fa286cd365fb167b3f99d3d216dc90c0de6279fe9f36c8d2215d7a7`.
167
+ ZJIT's generated result is slow in this short run; it needs investigation with
168
+ longer warm-up before drawing conclusions about steady-state performance.
169
+
170
+ A separate `ruby benchmark/profile.rb --iterations 1` run reported the
171
+ following instrumented times:
172
+
173
+ | Mode | Scanning | Actions | Token handling | Locations | Other |
174
+ |---|---:|---:|---:|---:|---:|
175
+ | runtime | 2.495 s | 0.470 s | 1.479 s | 0.132 s | 0.002 s |
176
+ | generated | 2.669 s | 0.459 s | 1.466 s | 0.132 s | 0.001 s |
177
+
178
+ Scanning is the largest category in this instrumented sample, followed by token
179
+ handling. These categories identify work to investigate; they do not establish
180
+ uninstrumented phase percentages or prove a particular optimization will help.
181
+ ## 2026-11-20: whole-token acceleration
182
+
183
+ Ruby 4.0.9 on arm64-darwin-25, the default 140,000-byte JSON corpus, 50,000
184
+ emitted tokens, and five warmup plus five measured iterations produced these
185
+ sequential local samples. The control checkout was `0391a6e`. The explicit
186
+ StringScanner variant adds only `accel :strscan` to the JSON specification.
187
+
188
+ | accelerator | engine | runtime MB/s before → after | generated MB/s before → after | allocations/token after, runtime / generated |
189
+ |---|---|---:|---:|---:|
190
+ | auto | interpreter | 1.739 → 1.856 | 1.645 → 1.815 | 4.200 / 4.210 |
191
+ | auto | YJIT | 4.967 → 5.210 | 2.977 → 3.184 | 4.200 / 4.210 |
192
+ | auto | ZJIT | 1.301 → 2.119 | 0.373 → 0.441 | 4.200 / 4.210 |
193
+ | strscan | interpreter | 1.767 → 1.973 | 0.785 → 1.090 | 4.200 / 4.610 |
194
+ | strscan | YJIT | 4.983 → 5.182 | 1.224 → 1.633 | 4.200 / 4.610 |
195
+ | strscan | ZJIT | 2.029 → 1.838 | 0.875 → 0.791 | 4.200 / 4.610 |
196
+
197
+ The default mode's change removes temporary ranges when extracting text and
198
+ location spans. Its allocations fell from 5.200 / 5.210. Explicit StringScanner
199
+ mode also uses proven whole-token scans; its generated allocations fell from
200
+ 6.210 to 4.610. This does not meet the 4.0 or 3.4 allocation targets. Whole-token
201
+ scanning remains explicit because the ZJIT sample regressed; these samples do
202
+ not establish a universal speedup or a release performance qualification.
203
+
204
+ Reproduce the default sample with:
205
+
206
+ ```sh
207
+ ruby -Ilib benchmark/run.rb --iterations 5 --json
208
+ ruby --yjit -Ilib benchmark/run.rb --iterations 5 --json
209
+ ruby --zjit -Ilib benchmark/run.rb --iterations 5 --json
210
+ ```
211
+
212
+ To measure explicit StringScanner mode without changing the maintained example:
213
+
214
+ ```sh
215
+ mkdir -p tmp/whole-token/examples/json
216
+ ruby -e 'source = File.read("examples/json/lexer.flexr.rb"); File.write("tmp/whole-token/examples/json/lexer.flexr.rb", source.sub("backend :direct", "backend :direct\n accel :strscan"))'
217
+ ruby -Ilib benchmark/run.rb --spec tmp/whole-token/examples/json/lexer.flexr.rb --iterations 5 --json
218
+ ```
219
+
220
+ Repeat the last command with `--yjit` or `--zjit` immediately after `ruby`.
221
+
222
+ ### JSON punctuation with `emit:`
223
+
224
+ Replacing the JSON example's six punctuation action blocks with the existing
225
+ `emit:` option preserved token types and values across runtime, generated,
226
+ standalone, String, and one-byte IO input, including Unicode and escaped quotes.
227
+ The same Ruby 4.0.9 host, default corpus, and five warmup plus five measured
228
+ iterations produced this interpreter sample after the range-allocation change:
229
+
230
+ | mode | MB/s before → after | allocations/token before → after | handwritten ratio before → after |
231
+ |---|---:|---:|---:|
232
+ | runtime | 1.863 → 1.928 | 4.200 → 3.600 | 0.221x → 0.249x |
233
+ | generated | 1.794 → 1.963 | 4.210 → 3.610 | 0.213x → 0.254x |
234
+
235
+ The 4.0 allocations/token target is met for this JSON example; 3.4 and the 0.35x
236
+ throughput target remain unmet. This sample does not establish results for
237
+ other lexers or JIT engines. Conversion actions for strings, numbers, booleans,
238
+ and null remain blocks.
239
+
240
+ The JSON golden digest and regression baseline's source fingerprint were
241
+ updated for the equivalent shorthand specification. Input identity, captured
242
+ baseline metrics, ratio thresholds, and the 0.18x performance floor were kept.
243
+ Use the default benchmark commands above to reproduce the new example.
244
+
245
+
246
+ ## 2026-11-21: combined v2 readiness sample
247
+
248
+ The integration checkout at `f9c629c` combines the v2 Data records and generated
249
+ method actions with whole-token scanning support and the JSON `emit:` shorthand.
250
+ Ruby 4.0.9 on arm64-darwin-25, without a JIT, processed the default 140,000-byte
251
+ JSON corpus with 50,000 emitted tokens. Five warmup and five measured iterations
252
+ ran after the test and browser checks finished.
253
+
254
+ | mode | MB/s | allocations/token | throughput relative to handwritten |
255
+ |---|---:|---:|---:|
256
+ | runtime | 1.906 | 3.600 | 0.235x |
257
+ | generated | 1.933 | 3.210 | 0.238x |
258
+ | handwritten | 8.118 | 3.400 | 1.000x |
259
+
260
+ Both flexr modes meet the 4.0 allocations/token target for this corpus. The
261
+ 3.4 target is met by generated mode; runtime remains above it. Neither mode
262
+ meets the 0.35x handwritten throughput target. This local readiness sample is
263
+ not a release performance qualification and does not establish results for
264
+ other lexers or JIT engines.
265
+
266
+ `rake bench:regression` also passed with the existing baseline metrics, identity,
267
+ ratio thresholds, and 0.18x floor unchanged. Reproduce the five-iteration sample
268
+ with `ruby -Ilib benchmark/run.rb --iterations 5 --json`.
@@ -12,10 +12,12 @@ consuming tokens, or the [CLI](cli.md) when validating and generating files.
12
12
  - [Actions](actions.md) — methods available inside rule and EOF actions.
13
13
  - [Tokens and locations](tokens-and-locations.md) — exact return shapes and
14
14
  offsets.
15
+ - [Testing](testing.md) — RSpec and Minitest assertions and token snapshots.
15
16
  - [Errors](errors.md) — exception and input-recovery behavior.
16
17
  - [Regexp compatibility](regexp.md) — supported syntax and alternatives.
17
18
  - [CLI](cli.md) — commands, options, output, and exit statuses.
18
19
  - [Diagnostics](diagnostics.md) — diagnostic code catalog.
20
+ - [Ruby LSP](ruby-lsp.md) — editor diagnostics and rule hovers.
19
21
  - [Generated artifacts](generated-artifacts.md) — static evaluation, output
20
22
  headers, dependencies, and standalone files.
21
23
  - [Public API](public-api.md) — stable, experimental, and internal boundaries.
@@ -18,12 +18,13 @@ Without a command, `flexr SPEC.rb` generates Ruby. `SPEC.rb` is normally a
18
18
  | `stats SPEC` | Report DFA states, byte classes, and table sizes | JSON |
19
19
  | `tokens SPEC` | Print names declared by `emits` | Space-separated names |
20
20
  | `dot SPEC` | Render the DFA as Graphviz DOT | DOT on stdout |
21
- | `trace SPEC` | Print DFA states, accepts, and transitions | Text on stdout |
21
+ | `trace SPEC` | Print DFA states, or trace input when supplied | Human or JSON input trace |
22
22
  | `explain SPEC` | List parsed rules and patterns | Text on stdout |
23
23
  | `bench SPEC` | Run the benchmark harness | Benchmark output |
24
24
  | `import INPUT` | Translate flex/Rexical input | Ruby on stdout or `--output` |
25
25
 
26
- `trace` describes the automaton; it does not trace a particular input string.
26
+ `trace` without input describes the automaton. To execute a specification and
27
+ inspect rule selection, pass `--input STRING` or `--input-file PATH`.
27
28
  `dot` marks accepting states and accelerated regions for visual inspection.
28
29
 
29
30
  ## Options
@@ -39,11 +40,16 @@ Without a command, `flexr SPEC.rb` generates Ruby. `SPEC.rb` is normally a
39
40
  | `--table-compression VALUE` | `none`, `rows`, `full` | Generation |
40
41
  | `--table-format VALUE` | `literal`, `packed` | Generation |
41
42
  | `--max-dfa-states N` | Positive integer | Compilation |
42
- | `-W, --warn LEVEL` | `all`, `default`, `none` | Check |
43
+ | `-W, --warn LEVEL` | `all`, `default`, `none` | Check and generation |
43
44
  | `--warn-as-error` | Flag | Check and generation |
44
45
  | `--color WHEN` | `auto`, `always`, `never` | Human diagnostics |
45
- | `--format FMT` | `human`, `json` | Check and stats |
46
+ | `--format FMT` | `human`, `json`, `lrama` | `lrama` is for tokens only |
47
+ | `--grammar PATH` | Grammar path | Compare `%token` names with `emits` during check |
48
+ | `--target 2.0` | none | Static check migration requirements and structured-token declarations |
46
49
  | `--rule N` | Non-negative integer | Explain |
50
+ | `--input STRING` | Literal input, including an empty string | Trace |
51
+ | `--input-file PATH` | Input file | Trace and bench |
52
+ | `--generated` | Run the generated scanner | Input trace |
47
53
 
48
54
  The complete help output is kept as a checked snapshot:
49
55
 
@@ -64,11 +70,66 @@ Options:
64
70
  -W, --warn LEVEL all | default | none
65
71
  --warn-as-error
66
72
  --color WHEN auto | always | never
67
- --format FMT human | json
73
+ --format FMT human | json | lrama (tokens only)
74
+ --grammar PATH compare emits with grammar %token declarations (check only)
75
+ --input STRING trace input (executes SPEC actions)
76
+ --input-file PATH trace or benchmark input
77
+ --generated trace generated scanner
78
+ --target 2.0 report migration requirements (static check only)
68
79
  <!-- flexr-help:end -->
69
80
 
81
+ ## Input trace
82
+
83
+ ```sh
84
+ flexr trace lexer.flexr.rb --input 'if == total'
85
+ flexr trace lexer.flexr.rb --input-file input.txt --generated --format json
86
+ ```
87
+
88
+ Input tracing executes the specification's Ruby code and actions. Use trusted
89
+ specifications, as with running a lexer normally. Actions run once, including
90
+ actions that skip input or change state. `--generated` generates and evaluates
91
+ the scanner in memory; it does not write an output file.
92
+
93
+ Each step reports byte offsets, matched text, the lexical state before and
94
+ after the action, candidate rule numbers and lengths, the selected rule, and
95
+ the action result (`emit` or `skip`). Human candidate lengths use
96
+ `body/total`, with trailing context included only in the total. `rule_order`
97
+ means several candidates tied at the winning total length; `longest` means a
98
+ longer candidate won. `first_match` reports the firstmatch backend's rule
99
+ priority and preserves its regexp alternation behavior. `only_match` means
100
+ there was one candidate. EOF actions appear with reason `eof`; recovered
101
+ unmatched bytes appear with reason `unmatched` and no selected rule.
102
+
103
+ `--format json` returns an object with a `steps` array. Each candidate contains
104
+ `rule`, `patterns`, `length`, `trailing_length`, and `total_length`; lengths are
105
+ bytes. Steps include `byte_begin`, `byte_end`, `next_byte`, `state`,
106
+ `next_state`, `text`, `bytes`, `candidates`, `rule`, `patterns`, `reason`,
107
+ `action`, and `token`. `byte_end` is the scanner's matched end, while
108
+ `next_byte` reflects action changes such as `less`. `token` is the emitted
109
+ token's Ruby inspection string, or null when none was emitted. Invalid UTF-8
110
+ bytes are replaced in display text; `bytes` preserves the exact matched bytes.
111
+
112
+ Tracing adds candidate analysis and retains steps in memory; it is a debugging
113
+ tool, not a benchmark. Firstmatch candidate analysis reads the remaining
114
+ buffered input to inspect rules beyond the selected first match. It obeys the
115
+ lexer's buffer limit. Normal lexers do not load or run tracing code.
116
+
70
117
  ## Exit statuses
71
118
 
119
+ Compare a lexer with Lrama/Bison-style declarations:
120
+
121
+ ```sh
122
+ flexr check lexer.flexr.rb --grammar parse.y --warn-as-error
123
+ flexr tokens lexer.flexr.rb --format lrama
124
+ ```
125
+
126
+ The check compares named `%token` declarations before the grammar's first `%%`
127
+ with `emits`. It handles multiline declarations, numeric token IDs, type tags,
128
+ aliases, and comments. Literal character tokens are excluded. This checks token
129
+ names, not grammar validity or whether every action actually emits its declared
130
+ token. Exported names must be identifiers; numeric IDs and aliases remain the
131
+ grammar author's responsibility.
132
+
72
133
  | Status | Meaning |
73
134
  |---:|---|
74
135
  | `0` | Command completed; warnings do not fail by default |
@@ -78,3 +139,8 @@ Options:
78
139
  `check --format json` writes one JSON array to stdout. Human diagnostics go to
79
140
  stdout for `check` and errors go to stderr for command failures. Use
80
141
  `--warn-as-error` when a warning should fail CI.
142
+
143
+ `--warn all` also includes informational equal-length rule overlaps
144
+ (`FLEXR-I001`). These reports include an input example and do not fail
145
+ `--warn-as-error`. See [Diagnostics](diagnostics.md#informational-diagnostics)
146
+ for the supported backends and states.
@@ -4,13 +4,16 @@
4
4
 
5
5
  | Surface | Requirement |
6
6
  |---|---|
7
- | Runtime gem | Ruby 3.1+ |
8
- | Generator | Ruby 3.3+ with Prism available |
7
+ | Runtime gem | Ruby 3.4+ |
8
+ | Generator | Ruby 3.4+ with Prism available |
9
9
  | Standalone runtime | Compatible Ruby plus the generated file |
10
10
  | Optional acceleration | Ruby `strscan` when available; regexp fallback remains valid |
11
11
 
12
12
  Runtime mode does not require Prism. Generated mode uses Prism only while
13
13
  reading the source; normal generated output does not need the generator.
14
+ Alternate Ruby implementations must implement Ruby 3.4 before using this
15
+ major version. The former JRuby and TruffleRuby jobs targeting older language
16
+ versions are removed; this does not claim those engines have been verified.
14
17
 
15
18
  ## Matching and encoding
16
19
 
@@ -33,6 +36,9 @@ Runtime and generated-file compatibility is preserved within a major version
33
36
  unless a migration is explicitly documented. Generated artifacts include a
34
37
  payload digest and Unicode version so stale or cross-version output can be
35
38
  detected. Do not assume that internal table layouts are stable.
39
+ Version 2 uses artifact schema 2 and runtime ABI 2. Regenerate v1 artifacts
40
+ after upgrading; incompatible metadata produces `FLEXR-E020`.
36
41
 
37
- Stable backends are `table`, `direct`, and `auto`. `firstmatch` is experimental
38
- and does not share the longest-match promise.
42
+ All four backends are stable. `table`, `direct`, and `auto` use longest-match
43
+ selection; `firstmatch` follows rule and regexp alternative order. It does not
44
+ share the longest-match promise and must be selected explicitly.