flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -3,8 +3,132 @@
3
3
  Diagnostics have a code, severity, message, optional location, help, and note.
4
4
  Use `flexr check SPEC --format json` to consume the structured form.
5
5
 
6
+ ## Source context in human output
7
+
8
+ When a diagnostic has a source range, human output includes the file, line,
9
+ column, source line, and carets. For example, given `lexer.flexr.rb`:
10
+
11
+ ```ruby
12
+ class Lexer < Flexr::Lexer
13
+ rule(/[a-z]+/, skip: true)
14
+ rule(/if/, skip: true)
15
+ end
16
+ ```
17
+
18
+ `flexr check lexer.flexr.rb --color never` prints:
19
+
20
+ ```text
21
+ warning[FLEXR-W001]: rule /if/ is unreachable (shadowed by rule /[a-z]+/)
22
+ --> lexer.flexr.rb:3:3
23
+ |
24
+ 3 | rule(/if/, skip: true)
25
+ | ^^^^^^^^^^^^^^^^^^^^^^
26
+ help: remove it, reorder the rules, or make its language distinct
27
+ note: example: "if" selects rule /[a-z]+/ (same length, earlier rule wins)
28
+ ```
29
+
30
+ Line and column numbers are one-based. Columns count characters rather than
31
+ UTF-8 bytes. A multiline range shows its first source line; an empty range
32
+ gets one caret. Tabs are preserved, but wide or combining characters can make
33
+ caret alignment differ from terminal display widths.
34
+
35
+ Source ranges are available for:
36
+
37
+ - Ruby syntax errors (`FLEXR-E010`).
38
+ - Static DSL errors: unsupported encoding (`FLEXR-E011`), `reject` actions
39
+ (`FLEXR-E013`), unresolved expressions or unsupported DSL structure
40
+ (`FLEXR-E017`), and invalid pattern or trailing-context types (`FLEXR-E018`).
41
+ - Rule compilation errors, including empty matches (`FLEXR-E005`), repetition
42
+ limits (`FLEXR-E007`), misplaced anchors (`FLEXR-E009`), and unsupported regexp
43
+ constructs (`FLEXR-E014`), including errors in trailing context.
44
+ - Rule warnings: `FLEXR-W001`, `FLEXR-W003`, `FLEXR-W010`, `FLEXR-W012`,
45
+ `FLEXR-W013`, and `FLEXR-W014`.
46
+ - Rule overlap information: `FLEXR-I001`, pointing to the later rule.
47
+ - State and specification diagnostics: DFA state limits (`FLEXR-E006`), empty
48
+ states (`FLEXR-W002`), large transition tables (`FLEXR-W011`), and slow DFA
49
+ construction (`FLEXR-W016`).
50
+
51
+ Rule diagnostics highlight the complete rule declaration. `FLEXR-W010` points
52
+ to the later rule in the reported pair. `FLEXR-W014` points to the first rule
53
+ emitting the undeclared token, with one warning per token.
54
+
55
+ Compiler diagnostics identify rules by their patterns rather than numeric
56
+ indices. Regexp options, quoted strings, arrays of patterns, and `followed_by:`
57
+ context are preserved in these labels. For longest-match specifications without
58
+ trailing context, `FLEXR-W001` also names the patterns that shadow the unreachable
59
+ rule; otherwise, its example identifies the verified winner. `FLEXR-E006` lists
60
+ the patterns in the state being compiled. String trailing context is shown as
61
+ its compiled literal regexp. These labels also appear in JSON messages and when
62
+ source ranges are unavailable.
63
+
64
+ `FLEXR-W001` notes include a shortest nonempty input, measured in bytes, and
65
+ the rule selected for that input. Examples for named lexical states
66
+ include the state name. Control characters and binary bytes are escaped, and
67
+ the same note is preserved in JSON and generated artifacts.
68
+
69
+ For `firstmatch`, the candidate is checked with the scanner's regexp matcher
70
+ without running actions. The note identifies the first rule selected and its
71
+ matched byte count, which can be shorter than the example input. For example,
72
+ `"ab"` can select an earlier `/a/` rule with a one-byte match.
73
+
74
+ For states using `followed_by:`, the candidate includes the shortest trailing
75
+ input and is checked using the active rules. Longest-match backends compare
76
+ the combined body and lookahead length; `firstmatch` keeps rule-order priority.
77
+ The note separates consumed bytes from lookahead bytes. For example, two rules
78
+ with body `/a/` and trailing context `/b+/` produce this note with backend `table`:
79
+
80
+ ```text
81
+ example: "ab" selects rule /a/, followed_by: /b+/ (longest total match, then rule order; matched 1 byte, looked ahead 1 byte)
82
+ ```
83
+
84
+ If a checked candidate selects the supposedly unreachable rule in any lexical
85
+ state, W001 is omitted for that rule. No rule actions run during these checks.
86
+ An example is omitted when the candidate is empty, invalid in the specification's
87
+ encoding, or cannot be confirmed by the matcher. This includes trailing patterns
88
+ that the runtime cannot compile. Candidate generation does not search for an
89
+ alternative if anchor or encoding checks reject the shortest candidate.
90
+
91
+ Static DSL errors highlight the offending DSL call. Unresolved expressions
92
+ point to the expression instead; ambiguous lexer selection points to the
93
+ second lexer class containing DSL calls.
94
+
95
+ State diagnostics point to the first declaration of the affected state.
96
+ `FLEXR-E006` identifies the state being compiled, while `FLEXR-W011` identifies
97
+ the state with the largest transition table. The implicit `initial` state
98
+ and the total construction time (`FLEXR-W016`) point to the lexer class.
99
+
100
+ Rules and states created directly through the runtime DSL do not carry source
101
+ ranges. Diagnostics without ranges, or whose source cannot be read, retain
102
+ the message, help, and note without a source excerpt.
103
+
104
+ For direct rendering, pass the matching source text and optional file path to
105
+ `DiagnosticSet#render(source: source, path: path)`. Without a path, the label is
106
+ `(source)`. JSON output keeps `location` as a pair of byte offsets with an
107
+ exclusive end; source excerpts are not added to JSON.
108
+
109
+ Runtime input diagnostics (`FLEXR-E003`, `FLEXR-E012`, `FLEXR-E013`,
110
+ and `FLEXR-E021` through `FLEXR-E027`)
111
+ use `[byte_pos, byte_pos]` to mark their reported position in the original
112
+ input. Guard notes identify the affected rule by its pattern when available,
113
+ using the same labels as compiler diagnostics. Pass the complete input to
114
+ `DiagnosticSet#render` to show its line and caret; see
115
+ [errors and recovery](errors.md). These offsets remain absolute
116
+ when a streaming buffer discards consumed input. If the CLI reports a runtime
117
+ input failure, it reads the error's `filename` for context, preserving the
118
+ input's original line endings. To bound memory use while reporting input
119
+ failures, the CLI omits excerpts for input files larger than 64 KiB. An absent
120
+ or unreadable input file also produces the diagnostic without an excerpt.
121
+
6
122
  ## Errors
7
123
 
124
+ If Prism cannot be loaded (`FLEXR-E019`), the diagnostic points to the start of
125
+ the specification and suggests installing the dependency. This is a file-level
126
+ failure; no Ruby syntax has been parsed yet.
127
+
128
+ Incompatible generated artifacts (`FLEXR-E020`) identify the start of the file
129
+ that installs the artifact, when that file exists. Direct installation calls
130
+ without a readable caller file still report the regeneration guidance.
131
+
8
132
  | Code | Meaning | First fix to try |
9
133
  |---|---|---|
10
134
  | `FLEXR-E001` | Regexp syntax is malformed | Correct the regexp syntax |
@@ -30,6 +154,162 @@ Use `flexr check SPEC --format json` to consume the structured form.
30
154
  | `FLEXR-E026` | A cancellation callback stopped lexing | Resume with a new lexer when ready |
31
155
  | `FLEXR-E027` | An error callback returned an unknown action | Return `:skip`, `:raise`, `:token`, or `:halt` |
32
156
 
157
+ ### Regexp rewrite help
158
+
159
+ For positive lookahead, `FLEXR-E014` includes a concrete example:
160
+
161
+ ```text
162
+ error[FLEXR-E014]: look-around is not supported by flexr
163
+ help: for a trailing positive lookahead, rewrite rule(/a(?=b)/) as rule(/a/, followed_by: /b/); keep the action
164
+ ```
165
+
166
+ Keep the action or `emit:` / `skip:` options on the rewritten rule. The example
167
+ consumes `a` only when `b` follows, leaving `b` for the next token. The combined
168
+ body and trailing length participates in longest-match selection.
169
+
170
+ Adapt this example to positive lookahead at the end of a rule's pattern,
171
+ preserving the regexp options needed by each part. Lookahead inside a pattern
172
+ needs a separate rewrite. Negative lookahead and lookbehind receive separate
173
+ guidance to use rules or states because `followed_by:` cannot express those
174
+ assertions.
175
+
176
+ For lazy and possessive quantifiers, `FLEXR-E014` shows a negated character
177
+ class for unescaped, single-line quoted text:
178
+
179
+ ```text
180
+ error[FLEXR-E014]: lazy or possessive quantifier is not supported by flexr
181
+ help: for unescaped single-line quoted text, use a negated character class: rule(/"[^"\n]*"/); keep the action; removing lazy or possessive modifiers can change matches
182
+ ```
183
+
184
+ For this form of quoted text, `/".*?"/` and `/"[^"\n]*+"/` can both be
185
+ rewritten as `/"[^"\n]*"/`. Excluding the closing quote makes the match stop
186
+ at the first delimiter, so adjacent strings remain separate tokens. Excluding
187
+ `\n` preserves the line-feed restriction of `.` without `/m`. Escaped quotes
188
+ and multiline strings need a different pattern. Simply dropping a lazy or
189
+ possessive modifier can change which inputs match or how much text is consumed.
190
+
191
+ For atomic groups, the same quoted-text rule can replace `/"(?>[^"\n]*)"/`.
192
+ The repeated character class cannot consume the closing quote, so this example
193
+ does not need backtracking to find it. The diagnostic warns against removing
194
+ atomic groups in general: [Ruby's atomic grouping](https://docs.ruby-lang.org/en/4.0/Regexp.html)
195
+ can change whether the rest of a pattern matches. The example still excludes
196
+ escaped quotes and line feeds.
197
+
198
+ For numbered backreferences, the help uses a capture with only two possible
199
+ values. Rewrite `rule(/(a|b)\1/)` or `rule(/(a|b)\k<1>/)` as
200
+ `rule(/(?:aa|bb)/)` and keep the action. This accepts `aa` and `bb`, while
201
+ rejecting `ab` and `ba`. It reproduces the matched text; flexr does not expose
202
+ capture values to actions. If the repeated text cannot be enumerated, split
203
+ the rule into states and compare the matched text in actions instead.
204
+
205
+ For a non-recursive subexpression call, the help shows
206
+ `rule(/(a|b)\g<1>/)` becoming `rule(/[ab]{2}/)` with the same action.
207
+ A call reuses the expression, not the previously captured text: this example
208
+ accepts `aa`, `ab`, `ba`, and `bb`. Recursive calls need a parser or a state
209
+ stack; they cannot be expanded by this example.
210
+
211
+ For named captures used only for grouping, rewrite `rule(/(?<word>ab)/)` or
212
+ `rule(/(?'word'ab)/)` as `rule(/(?:ab)/)` and keep the action. Backreferences
213
+ and conditions that depend on the capture need a separate rewrite; removing
214
+ the name alone does not preserve them.
215
+
216
+ For the conditional example `rule(/(a)?(?(1)b|c)/)`, use `rule(/(?:ab|c)/)`
217
+ with the same action. The original matches `ab` when the optional `a` was
218
+ captured, or `c` otherwise. Other conditions need their own alternatives,
219
+ separate rules, or state checks.
220
+
221
+ For an absence group excluding a single literal, rewrite `rule(/x(?~a)/)`
222
+ as `rule(/x[^a]*/)` and keep the action. Both consume `x` and the following
223
+ characters up to the next `a` or end of input, including line feeds. The `x`
224
+ keeps the rule from matching empty input. Larger absence expressions need a
225
+ separate rewrite: `(?~ab)` does not mean `[^ab]*`.
226
+
227
+ For word boundaries, the help uses examples where the surrounding literals
228
+ already determine the assertion. Rewrite `rule(/foo\b:/)` as `rule(/foo:/)`:
229
+ `o` is a word character and `:` is not. Similarly, `rule(/foo\Bbar/)` becomes
230
+ `rule(/foobar/)` because `o` and `b` are both word characters. Keep the action.
231
+ These examples do not cover boundaries at token edges, such as `/if\b/`;
232
+ those need separate rules or state checks. Inside a character class, `[\b]`
233
+ remains a supported backspace escape. The escapes `\A`, `\B`, `\G`, `\z`,
234
+ `\Z`, `\K`, `\g`, and `\X` inside character classes receive literal-character examples,
235
+ such as rewriting `rule(/[\A]/)` as `rule(/[A]/)` with the same action.
236
+
237
+ For `\G` at the start of a rule, rewrite `rule(/\Ga/)` as `rule(/a/)` and
238
+ keep the action. Every flexr rule already starts matching at the current
239
+ lexer position, including after earlier tokens. A `\G` inside a larger
240
+ pattern needs a separate rewrite.
241
+
242
+ For input without line feeds, `rule(/\Aa/)` can become `rule(/^a/)`, and
243
+ `rule(/a\z/)` or `rule(/a\Z/)` can become `rule(/a$/)`. Keep the action.
244
+ This requires a guarantee about the entire input, not just the token pattern.
245
+ [Ruby's line and string anchors](https://docs.ruby-lang.org/en/4.0/Regexp.html)
246
+ differ when line feeds occur: `^` can match after a line feed, `$` can match
247
+ before one, and `\Z` also permits a final line feed. Do not apply these
248
+ examples to multiline input.
249
+
250
+ For the match-reset example `rule(/a\Kb/) { emit :B }` in `:initial`, the help
251
+ splits the prefix and suffix into separate rules:
252
+
253
+ ```ruby
254
+ rule(/a/, followed_by: /b/) { begin_state :suffix }
255
+ state(:suffix) do
256
+ rule(/b/) { begin_state :initial; emit :B }
257
+ end
258
+ ```
259
+
260
+ The prefix is consumed only when `b` follows. The suffix rule returns to
261
+ `:initial` before running the original action, so `text` and the token location
262
+ cover only `b`. Keep the original action in place of `emit :B`, choose an unused
263
+ state name, and return to the original state when adapting the example.
264
+ Larger patterns and competing rules need a separate review; this is not an
265
+ automatic replacement for every `\K`.
266
+
267
+ Escape diagnostics also include concrete examples. For an octal tab,
268
+ rewrite `rule(/\011/)` as `rule(/\x09/)`. For an unknown escape such as `\q`,
269
+ use `rule(/q/)` only if `q` is the intended literal character. Keep the original
270
+ action and regexp options when adapting either example.
271
+
272
+ Inside a character class, `\1` is a character code. Its diagnostic recommends
273
+ an explicit hexadecimal escape such as `rule(/[\x01]/)`, which matches the same
274
+ control character as `rule(/[\1]/)`.
275
+
276
+ When raw regexp sources contain control or meta escapes, their help gives
277
+ hexadecimal alternatives: `rule(/\cA/)` and `rule(/\C-A/)` become
278
+ `rule(/\x01/)`, while `rule(/\M-a/n)` becomes `rule(/\xE1/n)` for binary input.
279
+ Keep the action and use the corresponding hexadecimal value for other
280
+ characters. Ruby may already normalize these escapes in regexp literals;
281
+ the diagnostics apply when the escape reaches the parser unchanged.
282
+
283
+ For a standalone `\X` token with ASCII-only input, use:
284
+
285
+ ```ruby
286
+ rule(/\r\n|[\x00-\x7f]/) { emit :GRAPHEME }
287
+ ```
288
+
289
+ This keeps CRLF together and consumes other ASCII characters individually.
290
+ Keep the original action; the CRLF alternative must come first for `firstmatch`.
291
+ The example does not cover Unicode graphemes or `\X` inside larger patterns.
292
+ Unicode graphemes need separate segmentation, such as Ruby's
293
+ [`String#each_grapheme_cluster`](https://docs.ruby-lang.org/en/4.0/String.html#method-i-each_grapheme_cluster).
294
+
295
+ For `\R`, the help chooses a standalone line-break rule for the specification's
296
+ encoding. In UTF-8, use:
297
+
298
+ ```ruby
299
+ rule(/\r\n|[\n\v\f\r\u0085\u2028\u2029]/) { emit :NEWLINE }
300
+ ```
301
+
302
+ This covers the [line breaks recognized by Ruby's `\R`](https://docs.ruby-lang.org/en/4.0/Regexp.html#class-Regexp-label-Shorthand+Character+Classes).
303
+ With `encoding Encoding::BINARY`, the example uses only ASCII line breaks:
304
+
305
+ ```ruby
306
+ rule(/\r\n|[\n\v\f\r]/n) { emit :NEWLINE }
307
+ ```
308
+
309
+ CRLF comes first so that `firstmatch` also consumes the pair as one token.
310
+ Apply these examples as separate token rules. Replacing `\R` inside a larger
311
+ expression can change how the surrounding pattern matches.
312
+
33
313
  ## Warnings
34
314
 
35
315
  | Code | Meaning | First fix to try |
@@ -43,7 +323,65 @@ Use `flexr check SPEC --format json` to consume the structured form.
43
323
  | `FLEXR-W013` | Capturing group is treated as non-capturing | Use `(?:...)` |
44
324
  | `FLEXR-W014` | Emitted token is not declared | Add it to `emits` or accept the warning |
45
325
  | `FLEXR-W016` | DFA construction took more than 0.5 seconds | Use generated mode for startup |
326
+ | `FLEXR-W017` | Grammar token is missing from `emits` | Declare and emit it, or remove it from the grammar |
327
+ | `FLEXR-W018` | Token in `emits` is missing from the grammar | Add a `%token` declaration or remove it from `emits` |
328
+ | `FLEXR-W020` | Structured records become immutable in v2.0 | Review consumer mutations; use `record.with(...)` in v2 |
46
329
 
47
330
  Warnings are reported by `check` according to `--warn`. The default omits the
48
331
  wall-clock `FLEXR-W016`; `--warn all` includes it, and `--warn none` suppresses
49
332
  warnings. `--warn-as-error` turns selected warnings into a failed command.
333
+
334
+ ## Informational diagnostics
335
+
336
+ `flexr check --target 2.0` includes `FLEXR-I002` with Ruby and artifact
337
+ requirements. See the [v2 migration guide](../migration-v2.md) for the static
338
+ check's limits and opt-in runtime deprecation warnings.
339
+
340
+ `check --warn all` also reports `FLEXR-I001` when multiple rules match the same
341
+ input at the same length. This can happen even when both rules are reachable.
342
+ For example, `rule(%w[a c], emit: :FIRST)` followed by
343
+ `rule(%w[b c], emit: :SECOND)` produces:
344
+
345
+ ```text
346
+ info[FLEXR-I001]: rule ["b", "c"] overlaps rule ["a", "c"] at the same length
347
+ help: reorder the rules if the later rule should take priority
348
+ note: example: "c" selects rule ["a", "c"] (1 byte, earlier rule wins)
349
+ ```
350
+
351
+ Source context highlights the later rule. At most one shortest nonempty example
352
+ is reported for each later rule in each lexical state; this is not an exhaustive
353
+ list of overlapping rule pairs. Named states appear in the note, and lengths
354
+ are measured in bytes. Invalid input for the specification's encoding is omitted.
355
+
356
+ For `firstmatch`, the candidate is checked with the scanner's regexp matcher.
357
+ The diagnostic is emitted only if the first selected rule and the later rule
358
+ match the same number of bytes. For example, `/a|ab/` followed by `/ab/` does not
359
+ produce a tie for `"ab"`: the first rule matches one byte and the second matches
360
+ two. The note reports the actual matched byte count, including zero when empty
361
+ matches are allowed. No actions run during the check. If the shortest candidate
362
+ does not confirm a tie, no alternative candidate is searched.
363
+
364
+ For states using `followed_by:`, the candidate combines the shortest body with
365
+ the shortest trailing input. The check compares the total length, so rules can
366
+ overlap even when they consume different numbers of bytes. For example,
367
+ `rule(/a/, followed_by: /bc/, emit: :FIRST)` followed by
368
+ `rule(/ab/, followed_by: /c/, emit: :SECOND)` produces:
369
+
370
+ ```text
371
+ info[FLEXR-I001]: rule /ab/, followed_by: /c/ overlaps rule /a/, followed_by: /bc/ at the same total length
372
+ help: reorder the rules if the later rule should take priority
373
+ note: example: "abc" selects rule /a/, followed_by: /bc/ (same total length, earlier rule wins; matched 1 byte, looked ahead 2 bytes)
374
+ ```
375
+
376
+ The winner is checked against the full candidate, including trailing matches
377
+ and end anchors. The note separates the bytes consumed from those looked ahead.
378
+ For `firstmatch`, it says `first match wins` and requires the selected rule and
379
+ the later rule to have equal total lengths. Empty input, invalid character
380
+ boundaries, and candidates the runtime's trailing matcher cannot verify are
381
+ omitted. A rejected shortest candidate does not trigger a search for alternatives.
382
+
383
+ This analysis covers `table`, `direct`, `auto`, and `firstmatch` states.
384
+ `--warn default` and `--warn none` omit this information, and
385
+ `--warn-as-error` does not turn informational diagnostics into failures.
386
+ Generation with `--warn all` also preserves the information in generated and
387
+ standalone artifacts.
@@ -15,6 +15,10 @@ one rule and share its action.
15
15
  text as its value. Without either option, the block runs in the lexer instance;
16
16
  if no block is given, the default action emits `[nil, text]`.
17
17
 
18
+ Prefer `emit:` when the value is the unchanged matched text, such as
19
+ `rule(/\+/, emit: :PLUS)`. This avoids executing a Ruby action block for each
20
+ token. Use a block when converting a value or changing lexer state.
21
+
18
22
  `followed_by:` accepts a `Regexp` or `String`. It participates in rule
19
23
  selection but is not consumed. The method returns `nil`. Rules are numbered in
20
24
  source order starting at zero.
@@ -49,10 +53,13 @@ does not restrict `emit` at runtime.
49
53
 
50
54
  ## `backend(name)`
51
55
 
52
- Selects `:table`, `:direct`, `:auto`, or the experimental `:firstmatch`
56
+ Selects `:table`, `:direct`, `:auto`, or `:firstmatch`
53
57
  backend. `:table`, `:direct`, and `:auto` preserve the stable longest-match
54
- contract. `:firstmatch` requires `option :experimental` and can change the
55
- meaning of overlapping rules.
58
+ contract. `:firstmatch` selects the first matching rule in source order,
59
+ even when a later rule could consume more text. Within a rule, alternation
60
+ also follows Ruby regexp order. Choose it when rule priority is intentional;
61
+ the compiler reports possible differences from longest-match as `FLEXR-W010`.
62
+ The default remains `:table`.
56
63
 
57
64
  ## `token_kind(name)`
58
65
 
@@ -69,8 +76,8 @@ model is valid.
69
76
  ## `option(*values)`
70
77
 
71
78
  Adds boolean options to the specification. Stable options include
72
- `:unicode`, `:eager_columns`, and `:allow_empty_match`. `:experimental` is a
73
- capability opt-in for experimental behavior such as `:firstmatch`.
79
+ `:unicode`, `:eager_columns`, and `:allow_empty_match`. `:experimental` remains
80
+ accepted for compatibility; it is no longer required by `:firstmatch`.
74
81
  `:standalone` is build metadata used by standalone generation. Unknown option
75
82
  names raise immediately so misspellings cannot silently change behavior.
76
83
 
@@ -16,6 +16,34 @@ text, and optional diagnostic. `Flexr::Runtime::TokenTooLargeError` is a
16
16
  [diagnostics catalog](diagnostics.md). `StateStackOverflowError` is raised when
17
17
  a `push` would exceed `max_state_stack`.
18
18
 
19
+ Undefined state transitions (`FLEXR-E003`) and `reject` calls (`FLEXR-E013`)
20
+ at runtime continue to raise `Flexr::CompileError`. These errors also expose
21
+ the input `filename` and `byte_pos`, with a diagnostic point range at the
22
+ current cursor. During a rule action this is after the matched input, unless
23
+ the action moved the cursor with `less`. EOF actions point to the end of input.
24
+ Static `reject` diagnostics still highlight the rule in the specification.
25
+
26
+ Guard diagnostics include a point range at `byte_pos` when it is known. This
27
+ is the guard's reported input position, which may be the start of a token or
28
+ the end of the loaded buffer. In a rescue handler, render it with the complete
29
+ original input:
30
+
31
+ ```ruby
32
+ diagnostics = Flexr::DiagnosticSet.new << error.diagnostic
33
+ warn diagnostics.render(source: original_input, path: error.filename)
34
+ ```
35
+
36
+ With `retain_input: false`, `lexer.input` contains only the retained buffer;
37
+ keep the original input separately if a source excerpt is needed. JSON
38
+ diagnostics retain absolute byte offsets even after input has been discarded.
39
+
40
+ When a guard knows the affected rule, its diagnostic note shows the pattern,
41
+ including regexp options and trailing context, for example `rule /a+/i at
42
+ byte 0`. The error's `rule` attribute still returns the numeric rule index.
43
+ Errors constructed with only a numeric index retain a numeric label; guards
44
+ without an identified rule omit the note. The diagnostic location continues
45
+ to refer to the input, not the rule declaration.
46
+
19
47
  The default `error_mode: :raise` raises the lexical error. `:token` queues
20
48
  `[:error, text]`; `:panic` discards the unmatched byte and continues. An
21
49
  `on_error` callback can return `:skip`, `:raise`, `:token`, or `:halt` for
@@ -5,13 +5,40 @@
5
5
  | Use | Build-time requirement | Runtime requirement |
6
6
  |---|---|---|
7
7
  | Runtime mode | None beyond the flexr gem | `flexr` gem |
8
- | Normal generated mode | flexr gem and Prism on Ruby 3.3+ | `flexr` gem |
9
- | Standalone generated mode | flexr gem and Prism on Ruby 3.3+ | Generated file and Ruby standard library |
8
+ | Normal generated mode | flexr gem and Prism on Ruby 3.4+ | `flexr` gem and Ruby 3.4+ |
9
+ | Standalone generated mode | flexr gem and Prism on Ruby 3.4+ | Generated file and Ruby 3.4+ standard library |
10
10
 
11
- The runtime gem supports Ruby 3.1 and newer. The generator's Prism path is
12
- supported on Ruby 3.3 and newer. A generated file can be loaded by a compatible
11
+ The runtime gem and generator support Ruby 3.4 and newer. A generated file can be loaded by a compatible
13
12
  runtime without the generator installed.
14
13
 
14
+ Version 2 artifacts use schema 2 and runtime ABI 2. Regenerate v1 generated
15
+ files with the v2 generator. Old metadata is rejected with `FLEXR-E020` when
16
+ loading an artifact into the new runtime.
17
+
18
+ ## Ractor sharing
19
+
20
+ Load a generated class in the main Ractor, then explicitly prepare its data:
21
+
22
+ ```ruby
23
+ load "lexer.generated.rb"
24
+ Flexr::Generated.shareable!(Lexer)
25
+ worker = Ractor.new(Lexer) { |klass| klass.new("input").tokens }
26
+ tokens = worker.respond_to?(:value) ? worker.value : worker.take
27
+ ```
28
+
29
+ This eagerly materializes transition rows and calls `Ractor.make_shareable` on
30
+ the compiled data and configuration. Each Ractor creates its own lexer and
31
+ input. Simple `emit` blocks, `emit:` rules, and `skip:` rules use generated
32
+ methods. Captured closures, runtime specifications, and EOF blocks are rejected
33
+ by `shareable!`; arbitrary user methods must also obey Ruby's Ractor rules.
34
+ `firstmatch` patterns that need the Unicode reference matcher are also rejected
35
+ before preparation, because its shared caches are not Ractor-safe. Use `table`,
36
+ `direct`, or `auto` for those patterns. Local variables that shadow action
37
+ methods such as `text`, `line`, or `lineno` keep their closure bindings and are
38
+ not converted to shareable method actions.
39
+ Preparation may use more memory than lazy packed tables. Call it before
40
+ starting workers; it does not make an existing lexer instance shareable.
41
+
15
42
  ## Static evaluation boundary
16
43
 
17
44
  The default generator resolves:
@@ -75,3 +102,14 @@ These settings affect artifact size and loading cost, not matching semantics.
75
102
  Standalone output embeds a runtime core, not the compiler or generator. The
76
103
  regexp reference matcher and vendored Unicode tables are added only for a
77
104
  `firstmatch` backend or trailing-context rules that can use that matcher.
105
+
106
+ The generator compiles `followed_by:` expressions into read-only DFA tables
107
+ stored with their rules. These tables use the selected compression and table
108
+ format, so scanning trailing context does not require the compiler at runtime.
109
+ Any errors found while compiling them are reported at the rule declaration
110
+ before an existing output file is replaced.
111
+
112
+ The optional `trailing_dfa` rule field is compatible with schema 1. The runtime
113
+ continues to compile trailing context lazily for older generated files without
114
+ this field. Regenerate older standalone lexers that use `followed_by:` to fix
115
+ their missing `Flexr.compile_pattern` error.
@@ -24,16 +24,41 @@ in the reference pages and are covered by the test suite.
24
24
  constants through autoload. Applications that want an explicit load boundary
25
25
  can require `flexr/runtime`, `flexr/generator`, or `flexr/cli` directly.
26
26
 
27
- ## Experimental
27
+ Runtime and generated `firstmatch` modes are stable and use the same rule-order
28
+ semantics. The former `option :experimental` opt-in remains accepted but is
29
+ unnecessary. Differences from longest-match behavior are reported as
30
+ `FLEXR-W010` rather than making generation fail.
31
+ ## Type signatures
32
+
33
+ The gem includes RBS signatures in `sig/flexr.rbs` for the stable API above.
34
+ Steep can discover them through an application's RBS collection or an explicit
35
+ `library "flexr"` entry. User-defined token values and internal regexp ASTs
36
+ remain `untyped`; the signatures do not make internal tables stable APIs.
37
+ Token and location records inherit from `Data`; fields are read-only and `with`
38
+ returns a replacement. Their fields are nullable because their public constructors
39
+ allow omitted keywords. The token representation remains a union of arrays and
40
+ structured tokens because `token_kind` is selected dynamically.
41
+ Cancellation signatures accept Procs, Methods, and callable objects taking zero
42
+ arguments or the lexer. `on_error=` accepts a callable returning `:skip`,
43
+ `:raise`, `:token`, or `:halt`; assigning nil clears the handler.
44
+
45
+ Validate the signatures and type-check the consumer example with:
28
46
 
29
- - `backend :firstmatch` and `--backend firstmatch`.
30
- - `option :experimental` as the opt-in for experimental behavior.
31
- - Any future backend or option explicitly marked experimental in its reference.
47
+ ```sh
48
+ BUNDLE_GEMFILE=gemfiles/types.gemfile bundle install
49
+ BUNDLE_GEMFILE=gemfiles/types.gemfile bundle exec rbs -I sig validate
50
+ BUNDLE_GEMFILE=gemfiles/types.gemfile bundle exec steep check
51
+ ```
52
+
53
+ The type-checking tools run in a separate CI matrix for Ruby 3.4 and 4.0,
54
+ matching the runtime's supported Ruby versions. They add no runtime dependency to flexr.
55
+ This checks the public signatures and a typed consumer, not the entire internal
56
+ implementation.
57
+
58
+ ## Experimental
32
59
 
33
- Experimental behavior may change semantics or output between minor releases.
34
- `firstmatch` requires the opt-in. Runtime and generated `firstmatch` modes use
35
- the same rule-order semantics; overlap with longest-match behavior is reported
36
- as `FLEXR-W010` rather than making generation fail.
60
+ Future backends or options explicitly marked experimental in their reference
61
+ may change semantics or output between minor releases.
37
62
 
38
63
  ## Internal; compatibility is not guaranteed
39
64
 
@@ -6,7 +6,7 @@ byte-oriented automaton. All matches start at the current lexer position.
6
6
  | Ruby construct | Status | Notes and alternative |
7
7
  |---|---|---|
8
8
  | Literals, concatenation, alternation | Supported | |
9
- | Groups and `(?:...)` | Supported | Captures are treated as non-capturing and warn |
9
+ | Unnamed groups and `(?:...)` | Supported | Captures are treated as non-capturing and warn |
10
10
  | Character classes and ranges | Supported | Includes trailing `-`, POSIX classes, and `&&` intersection |
11
11
  | Unicode `\p{...}` / `\P{...}` | Supported | Uses the vendored UCD snapshot |
12
12
  | `*`, `+`, `?` | Supported | Greedy forms only |
@@ -15,14 +15,26 @@ byte-oriented automaton. All matches start at the current lexer position.
15
15
  | `/i`, `/m`, `/x`, `/n` | Supported | Inline forms are supported where Ruby permits them |
16
16
  | `\d`, `\w`, `\s` | Supported | Unicode-aware with `option :unicode` in UTF-8 |
17
17
  | `\h`, `\H` | Supported | Hexadecimal digits and their complement |
18
- | Lookahead / lookbehind | Unsupported | Use `followed_by:` or a state; `FLEXR-E014` |
19
- | Backreferences | Unsupported | Split the language into states or actions |
20
- | `\b`, `\B`, `\A`, `\z`, `\G`, `\K` | Unsupported | Use rule boundaries or state/action logic |
21
- | Lazy or possessive quantifiers | Unsupported | Use a negated character class or separate rule |
22
- | Conditional, atomic, and capture-dependent groups | Unsupported | Rewrite as DFA-compatible rules |
18
+ | Octal escapes such as `\011` | Unsupported | Use a hexadecimal escape such as `\x09` |
19
+ | `\R` | Unsupported | Use the standalone line-break rule shown by `FLEXR-E014` for the specification's encoding |
20
+ | Lookahead / lookbehind | Unsupported | Use `followed_by:` for trailing positive lookahead; other forms need rules or states; `FLEXR-E014` |
21
+ | Backreferences | Unsupported | Enumerate finite repeated strings, or use states and action checks |
22
+ | `\b`, `\B` | Unsupported | Use the literal-boundary examples in `FLEXR-E014` where applicable; token edges need rules or state checks |
23
+ | `\A`, `\z`, `\Z` | Unsupported | The `^` / `$` examples in `FLEXR-E014` apply only to input without line feeds |
24
+ | `\G` | Unsupported | At the start of a rule, omit it: rules already match at the current lexer position |
25
+ | `\K` | Unsupported | Use the prefix/suffix state example in `FLEXR-E014` where applicable to preserve token text and location |
26
+ | Subexpression calls `\g<...>` | Unsupported | Inline non-recursive expressions; recursive calls need a parser or state stack |
27
+ | `\X` | Unsupported | Use the ASCII-only standalone token example in `FLEXR-E014` where applicable; Unicode graphemes need separate segmentation |
28
+ | Lazy or possessive quantifiers | Unsupported | For unescaped single-line quoted text, use `/"[^"\n]*"/`; other patterns need a separate rewrite |
29
+ | Atomic groups | Unsupported | Use the delimited-text example in `FLEXR-E014` where applicable; removing atomic groups can change matches |
30
+ | Named captures | Unsupported | Use `(?:...)` when only grouping is needed; capture-dependent patterns need a separate rewrite |
31
+ | Conditional groups | Unsupported | Use the finite alternatives shown by `FLEXR-E014` where applicable, or separate rules and state checks |
32
+ | Absence groups `(?~...)` | Unsupported | For a single excluded literal, use the negated character class shown by `FLEXR-E014`; larger expressions need a separate rewrite |
23
33
 
24
34
  Malformed syntax produces `FLEXR-E001`. Unsupported constructs produce
25
35
  `FLEXR-E014`; the diagnostic help normally names the supported replacement.
36
+ Several `FLEXR-E014` diagnostics include concrete
37
+ [rule rewrite examples](diagnostics.md#regexp-rewrite-help).
26
38
  Unknown alphabetic escapes are rejected rather than silently treated as
27
39
  literals; escaped regexp punctuation remains supported.
28
40
  Empty matches are rejected with `FLEXR-E005` unless `option :allow_empty_match`
@@ -36,6 +48,6 @@ not exceed 1000. `{n,}` uses an unbounded loop after the required copies.
36
48
 
37
49
  ## Captures
38
50
 
39
- Capturing parentheses do not expose match groups to actions. They are treated
51
+ Unnamed capturing parentheses do not expose match groups to actions. They are treated
40
52
  as grouping syntax and produce `FLEXR-W013`; use `(?:...)` and inspect `text`
41
53
  in the action instead.
@@ -0,0 +1,34 @@
1
+ # Ruby LSP integration
2
+
3
+ flexr includes an optional add-on for [Ruby LSP](https://shopify.github.io/ruby-lsp/add-ons.html).
4
+ It supports Ruby LSP `~> 0.26.11`; incompatible server versions skip activation.
5
+ The lexer runtime does not depend on Ruby LSP.
6
+
7
+ Keep flexr in the project's bundle and enable its linter in the editor's Ruby
8
+ LSP configuration. For VS Code, add `flexr` to the existing list:
9
+
10
+ ```json
11
+ {
12
+ "rubyLsp.linters": ["flexr"]
13
+ }
14
+ ```
15
+
16
+ Keep any other linters you use in that array. Ruby LSP discovers the add-on
17
+ automatically. It displays flexr diagnostics for `.flexr.rb` files using the
18
+ current editor contents, including unsaved edits. Rule hovers show compiler
19
+ warnings, counterexamples, and overlap information. Diagnostics and hovers
20
+ use the static reader: they do not evaluate specifications or run actions.
21
+ Syntax errors are reported by Ruby LSP's Prism integration.
22
+
23
+ The add-on uses the same static specification subset as `flexr check`, so
24
+ dynamic Ruby DSL construction must be checked separately. Compilation is
25
+ cached for the most recent document contents. Timing-dependent `FLEXR-W016`
26
+ is omitted; other diagnostics include their help and notes. The add-on honors
27
+ the server's negotiated UTF-8, UTF-16, or UTF-32 position encoding.
28
+
29
+ To run the integration tests against the real Ruby LSP server:
30
+
31
+ ```sh
32
+ BUNDLE_GEMFILE=gemfiles/ruby_lsp.gemfile bundle install
33
+ BUNDLE_GEMFILE=gemfiles/ruby_lsp.gemfile bundle exec rspec -Ilib -Ispec test/ruby_lsp_addon_spec.rb
34
+ ```
@@ -21,8 +21,8 @@ Lexer.new(input,
21
21
 
22
22
  `input` must be a `String` or an object responding to `read`. `filename` is
23
23
  copied into locations and lexical errors. `error_mode` is `:raise`, `:token`,
24
- or `:panic`; unknown values fall back to raising when an unmatched byte is
25
- handled. `max_lookahead_size` defaults to `max_token_size`. `max_steps: nil`
24
+ or `:panic`; unknown values raise `ArgumentError` at initialization.
25
+ `max_lookahead_size` defaults to `max_token_size`. `max_steps: nil`
26
26
  leaves the step count unbounded; `cancellation` may be a callable accepting
27
27
  zero arguments or the lexer. Size limits are non-negative, and `chunk_size`
28
28
  must be positive.