flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -6,34 +6,48 @@ byte-oriented automaton. All matches start at the current lexer position.
6
6
  | Ruby construct | Status | Notes and alternative |
7
7
  |---|---|---|
8
8
  | Literals, concatenation, alternation | Supported | |
9
- | Groups and `(?:...)` | Supported | Captures are treated as non-capturing and warn |
10
- | Character classes and ranges | Supported | POSIX classes are supported; see encoding notes |
9
+ | Unnamed groups and `(?:...)` | Supported | Captures are treated as non-capturing and warn |
10
+ | Character classes and ranges | Supported | Includes trailing `-`, POSIX classes, and `&&` intersection |
11
11
  | Unicode `\p{...}` / `\P{...}` | Supported | Uses the vendored UCD snapshot |
12
12
  | `*`, `+`, `?` | Supported | Greedy forms only |
13
- | `{n}`, `{n,m}` | Supported | The upper bound is 1000 |
14
- | `{n,}` | Unsupported | Split the rule or use states |
13
+ | `{n}`, `{n,m}`, `{n,}` | Supported | Explicit bounds may not exceed 1000 |
15
14
  | `^` / `$` | Restricted | Only at outermost pattern boundaries; otherwise `FLEXR-E009` |
16
15
  | `/i`, `/m`, `/x`, `/n` | Supported | Inline forms are supported where Ruby permits them |
17
16
  | `\d`, `\w`, `\s` | Supported | Unicode-aware with `option :unicode` in UTF-8 |
18
- | Lookahead / lookbehind | Unsupported | Use `followed_by:` or a state; `FLEXR-E014` |
19
- | Backreferences | Unsupported | Split the language into states or actions |
20
- | `\b`, `\B`, `\A`, `\z`, `\G`, `\K` | Unsupported | Use rule boundaries or state/action logic |
21
- | Lazy or possessive quantifiers | Unsupported | Use a negated character class or separate rule |
22
- | Conditional, atomic, and capture-dependent groups | Unsupported | Rewrite as DFA-compatible rules |
17
+ | `\h`, `\H` | Supported | Hexadecimal digits and their complement |
18
+ | Octal escapes such as `\011` | Unsupported | Use a hexadecimal escape such as `\x09` |
19
+ | `\R` | Unsupported | Use the standalone line-break rule shown by `FLEXR-E014` for the specification's encoding |
20
+ | Lookahead / lookbehind | Unsupported | Use `followed_by:` for trailing positive lookahead; other forms need rules or states; `FLEXR-E014` |
21
+ | Backreferences | Unsupported | Enumerate finite repeated strings, or use states and action checks |
22
+ | `\b`, `\B` | Unsupported | Use the literal-boundary examples in `FLEXR-E014` where applicable; token edges need rules or state checks |
23
+ | `\A`, `\z`, `\Z` | Unsupported | The `^` / `$` examples in `FLEXR-E014` apply only to input without line feeds |
24
+ | `\G` | Unsupported | At the start of a rule, omit it: rules already match at the current lexer position |
25
+ | `\K` | Unsupported | Use the prefix/suffix state example in `FLEXR-E014` where applicable to preserve token text and location |
26
+ | Subexpression calls `\g<...>` | Unsupported | Inline non-recursive expressions; recursive calls need a parser or state stack |
27
+ | `\X` | Unsupported | Use the ASCII-only standalone token example in `FLEXR-E014` where applicable; Unicode graphemes need separate segmentation |
28
+ | Lazy or possessive quantifiers | Unsupported | For unescaped single-line quoted text, use `/"[^"\n]*"/`; other patterns need a separate rewrite |
29
+ | Atomic groups | Unsupported | Use the delimited-text example in `FLEXR-E014` where applicable; removing atomic groups can change matches |
30
+ | Named captures | Unsupported | Use `(?:...)` when only grouping is needed; capture-dependent patterns need a separate rewrite |
31
+ | Conditional groups | Unsupported | Use the finite alternatives shown by `FLEXR-E014` where applicable, or separate rules and state checks |
32
+ | Absence groups `(?~...)` | Unsupported | For a single excluded literal, use the negated character class shown by `FLEXR-E014`; larger expressions need a separate rewrite |
23
33
 
24
34
  Malformed syntax produces `FLEXR-E001`. Unsupported constructs produce
25
35
  `FLEXR-E014`; the diagnostic help normally names the supported replacement.
36
+ Several `FLEXR-E014` diagnostics include concrete
37
+ [rule rewrite examples](diagnostics.md#regexp-rewrite-help).
38
+ Unknown alphabetic escapes are rejected rather than silently treated as
39
+ literals; escaped regexp punctuation remains supported.
26
40
  Empty matches are rejected with `FLEXR-E005` unless `option :allow_empty_match`
27
41
  is explicitly enabled, in which case flexr advances to guarantee progress.
28
42
 
29
43
  ## Bounded repetition
30
44
 
31
- `{n,m}` expands into a finite automaton. `m` must be at least `n` and no greater
32
- than 1000. Open repetition is rejected because it is easy to express with `+`
33
- or a separate rule and otherwise gives a misleading compatibility promise.
45
+ Repetition remains a symbolic AST node until NFA construction, avoiding large
46
+ intermediate trees. In `{n,m}`, `m` must be at least `n`; explicit bounds may
47
+ not exceed 1000. `{n,}` uses an unbounded loop after the required copies.
34
48
 
35
49
  ## Captures
36
50
 
37
- Capturing parentheses do not expose match groups to actions. They are treated
51
+ Unnamed capturing parentheses do not expose match groups to actions. They are treated
38
52
  as grouping syntax and produce `FLEXR-W013`; use `(?:...)` and inspect `text`
39
53
  in the action instead.
@@ -0,0 +1,34 @@
1
+ # Ruby LSP integration
2
+
3
+ flexr includes an optional add-on for [Ruby LSP](https://shopify.github.io/ruby-lsp/add-ons.html).
4
+ It supports Ruby LSP `~> 0.26.11`; incompatible server versions skip activation.
5
+ The lexer runtime does not depend on Ruby LSP.
6
+
7
+ Keep flexr in the project's bundle and enable its linter in the editor's Ruby
8
+ LSP configuration. For VS Code, add `flexr` to the existing list:
9
+
10
+ ```json
11
+ {
12
+ "rubyLsp.linters": ["flexr"]
13
+ }
14
+ ```
15
+
16
+ Keep any other linters you use in that array. Ruby LSP discovers the add-on
17
+ automatically. It displays flexr diagnostics for `.flexr.rb` files using the
18
+ current editor contents, including unsaved edits. Rule hovers show compiler
19
+ warnings, counterexamples, and overlap information. Diagnostics and hovers
20
+ use the static reader: they do not evaluate specifications or run actions.
21
+ Syntax errors are reported by Ruby LSP's Prism integration.
22
+
23
+ The add-on uses the same static specification subset as `flexr check`, so
24
+ dynamic Ruby DSL construction must be checked separately. Compilation is
25
+ cached for the most recent document contents. Timing-dependent `FLEXR-W016`
26
+ is omitted; other diagnostics include their help and notes. The add-on honors
27
+ the server's negotiated UTF-8, UTF-16, or UTF-32 position encoding.
28
+
29
+ To run the integration tests against the real Ruby LSP server:
30
+
31
+ ```sh
32
+ BUNDLE_GEMFILE=gemfiles/ruby_lsp.gemfile bundle install
33
+ BUNDLE_GEMFILE=gemfiles/ruby_lsp.gemfile bundle exec rspec -Ilib -Ispec test/ruby_lsp_addon_spec.rb
34
+ ```
@@ -10,14 +10,22 @@ Lexer.new(input,
10
10
  filename: nil,
11
11
  error_mode: :raise,
12
12
  max_token_size: 16 * 1024 * 1024,
13
+ max_lookahead_size: nil,
14
+ max_buffer_size: 64 * 1024 * 1024,
13
15
  max_state_stack: 1024,
16
+ max_steps: nil,
17
+ cancellation: nil,
18
+ retain_input: true,
14
19
  chunk_size: 64 * 1024)
15
20
  ```
16
21
 
17
22
  `input` must be a `String` or an object responding to `read`. `filename` is
18
23
  copied into locations and lexical errors. `error_mode` is `:raise`, `:token`,
19
- or `:panic`; unknown values fall back to raising when an unmatched byte is
20
- handled. Size limits are non-negative, and `chunk_size` must be positive.
24
+ or `:panic`; unknown values raise `ArgumentError` at initialization.
25
+ `max_lookahead_size` defaults to `max_token_size`. `max_steps: nil`
26
+ leaves the step count unbounded; `cancellation` may be a callable accepting
27
+ zero arguments or the lexer. Size limits are non-negative, and `chunk_size`
28
+ must be positive.
21
29
 
22
30
  ## Consumption
23
31
 
@@ -27,11 +35,13 @@ handled. Size limits are non-negative, and `chunk_size` must be positive.
27
35
  | `tokens` | Array containing every token until EOF |
28
36
  | `each_token` | Enumerator without a block; otherwise yields tokens and returns `self` |
29
37
  | `racc_next_token` | `[type, value]`, or `[false, "$end"]` at EOF |
30
- | `input` | The buffered input string |
38
+ | `input` | The currently retained input string |
31
39
  | `buffer` | Internal buffered input object; use only for integrations that need it |
32
40
  | `filename` | The configured filename |
33
41
  | `error_mode` | The configured default input error mode |
34
42
  | `max_token_size` | The configured token-size limit |
43
+ | `max_lookahead_size` | The configured trailing-context limit |
44
+ | `steps` | Work units consumed by scanning |
35
45
 
36
46
  `each_token` yields two arguments instead of one when `token_kind :yield` is
37
47
  selected. `tokens` always returns an array of token values.
@@ -46,4 +56,11 @@ class or another lexer instance.
46
56
 
47
57
  IO input is read lazily in chunks. UTF-8 input is checked at codepoint
48
58
  boundaries, while binary input is consumed byte by byte. Invalid UTF-8 becomes
49
- an unmatched-byte error rather than being silently normalized.
59
+ an unmatched-byte error rather than being silently normalized. With
60
+ `retain_input: false`, bytes before the current token are released after each
61
+ match; `input` and `binary_input` then expose only the retained window.
62
+
63
+ `max_token_size` is checked before an action can shorten or materialize an
64
+ oversized match. `max_lookahead_size` separately bounds trailing context, and
65
+ `max_buffer_size` bounds bytes retained at one time. `max_steps` and
66
+ `cancellation` bound CPU work cooperatively.
@@ -0,0 +1,57 @@
1
+ # Testing lexers
2
+
3
+ The helpers consume the complete input and compare the returned tokens. They
4
+ work with runtime and generated lexer classes, and forward keyword options to
5
+ the lexer constructor. Install the test framework separately; requiring `flexr`
6
+ does not load either framework.
7
+
8
+ ## RSpec
9
+
10
+ ```ruby
11
+ require "flexr/rspec"
12
+
13
+ expect(Lexer).to tokenize("12 + 3").as([
14
+ [:INTEGER, 12], [:PLUS, "+"], [:INTEGER, 3]
15
+ ])
16
+ ```
17
+
18
+ Use `.as` to compare token values exactly, including symbol/string distinctions.
19
+ Lexer errors propagate so that tests can use the usual `raise_error` matcher.
20
+
21
+ ## Minitest
22
+
23
+ ```ruby
24
+ require "minitest/autorun"
25
+ require "flexr/minitest"
26
+
27
+ class LexerTest < Minitest::Test
28
+ include Flexr::Minitest::Assertions
29
+
30
+ def test_numbers
31
+ assert_tokenize Lexer, "12", [[:INTEGER, 12]], chunk_size: 1
32
+ end
33
+ end
34
+ ```
35
+
36
+ ## Token snapshots
37
+
38
+ Store snapshots in a directory owned by the test suite. Snapshot assertions
39
+ fail if the file is missing or differs. They write files only when the call
40
+ explicitly passes `update: true`; review the resulting diff before committing.
41
+
42
+ ```ruby
43
+ expect(Lexer).to tokenize("12 + 3").matching_snapshot("spec/snapshots/sum.json")
44
+ assert_token_snapshot Lexer, "12 + 3", "test/snapshots/sum.json"
45
+ ```
46
+
47
+ To create or replace a snapshot, temporarily add `update: true` to
48
+ `matching_snapshot` or `assert_token_snapshot`, then remove it for normal tests.
49
+ Create the parent directory first.
50
+
51
+ Snapshots use deterministic JSON, preserving symbols and non-UTF-8 bytes with
52
+ tags. Hash entries are sorted by their serialized keys, so insertion order
53
+ does not affect the snapshot. Supported values are nil, booleans, finite
54
+ numbers, strings, symbols, arrays, hashes, `Flexr::Runtime::Token`, and
55
+ `Flexr::Runtime::Location`. Convert custom token values to these types before
56
+ snapshot testing. Snapshot strings compare content rather than Ruby encoding
57
+ tags, allowing ASCII strings from String and IO input to use the same file.
@@ -12,6 +12,14 @@ For `:struct`, `Token` has `type`, `value`, and `location` members. `emit(nil,
12
12
  value)` is valid and produces a token whose type is `nil`. `skip` produces no
13
13
  token.
14
14
 
15
+ In v2, `Token` and `Location` are immutable `Data` records. The `:struct`
16
+ option name is retained. Attribute readers and read-only indexing by member
17
+ name or index remain available; field writers and `[]=` are removed. Replace
18
+ values with `token.with(value: replacement)` or `location.with(...)`.
19
+ Immutability is shallow: custom token values can still be mutable.
20
+ Lazy columns use a frozen pair without mutating the record or retaining the
21
+ lexer; `to_h` returns the resolved columns.
22
+
15
23
  ## Locations
16
24
 
17
25
  `Flexr::Runtime::Location` has these public members:
@@ -24,10 +32,12 @@ token.
24
32
  | `line_begin` / `line_end` | One-based line range |
25
33
  | `column_begin` / `column_end` | One-based character columns for UTF-8, byte columns for binary input |
26
34
 
27
- Columns are computed lazily by default. `option :eager_columns` fills them while
28
- the location is created. The end line and column identify the position after
29
- the token's last byte, so a token ending at the start of the next line may have
30
- an end column of 1.
35
+ Line and column positions are tracked incrementally while input is consumed,
36
+ so creating locations does not rescan the input prefix. Column members are
37
+ filled lazily by default; `option :eager_columns` fills them while the location
38
+ is created. The end line and column identify the position after the token's
39
+ last byte, so a token ending at the start of the next line may have an end
40
+ column of 1.
31
41
 
32
42
  `more` extends the location from the first match to the final match. Trailing
33
43
  context is not part of the location because it is not consumed.
@@ -0,0 +1,14 @@
1
+ # GraphQL subset lexer
2
+
3
+ Tokenizes names, numbers, single-line strings, comments, ignored commas, spread,
4
+ variables, directives, and punctuation. Keywords remain `NAME` tokens so the
5
+ parser can interpret them by context. Values remain source text.
6
+ Block strings, escape validation, and grammar validation are outside this example.
7
+
8
+ ```sh
9
+ ruby -Ilib -e 'load "examples/graphql/lexer.flexr.rb"; p GraphqlExample::Lexer.new("query { user(id: 12) { name } }").tokens'
10
+ bundle exec ruby benchmark/run.rb --spec examples/graphql/lexer.flexr.rb --iterations 1 --json
11
+ ```
12
+
13
+ The benchmark uses a repeated built-in query and compares runtime with generated
14
+ mode. Pass `--input-file PATH` to measure a different corpus.
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "flexr"
4
+
5
+ module GraphqlExample
6
+ class Lexer < Flexr::Lexer
7
+ emits :NAME, :STRING, :NUMBER, :SPREAD, :LBRACE, :RBRACE, :LPAREN,
8
+ :RPAREN, :LBRACKET, :RBRACKET, :COLON, :EQUAL, :BANG, :DOLLAR, :AT, :PIPE, :AMP
9
+ rule(/[ \t\r\n,]+/, skip: true)
10
+ rule(/#[^\n]*/, skip: true)
11
+ rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
12
+ rule(/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
13
+ rule(/"(?:\\.|[^"\\\r\n])*"/, emit: :STRING)
14
+ rule(/\.\.\./, emit: :SPREAD)
15
+ rule(/\{/, emit: :LBRACE)
16
+ rule(/\}/, emit: :RBRACE)
17
+ rule(/\(/, emit: :LPAREN)
18
+ rule(/\)/, emit: :RPAREN)
19
+ rule(/\[/, emit: :LBRACKET)
20
+ rule(/\]/, emit: :RBRACKET)
21
+ rule(/:/, emit: :COLON)
22
+ rule(/=/, emit: :EQUAL)
23
+ rule(/!/, emit: :BANG)
24
+ rule(/\$/, emit: :DOLLAR)
25
+ rule(/@/, emit: :AT)
26
+ rule(/\|/, emit: :PIPE)
27
+ rule(/&/, emit: :AMP)
28
+ end
29
+ end
@@ -9,12 +9,12 @@ module JsonExample
9
9
  :STRING, :NUMBER, :TRUE, :FALSE, :NULL
10
10
 
11
11
  rule(/[ \t\r\n]+/, skip: true)
12
- rule(/\{/) { emit :LBRACE }
13
- rule(/\}/) { emit :RBRACE }
14
- rule(/\[/) { emit :LBRACKET }
15
- rule(/\]/) { emit :RBRACKET }
16
- rule(/:/) { emit :COLON }
17
- rule(/,/) { emit :COMMA }
12
+ rule(/\{/, emit: :LBRACE)
13
+ rule(/\}/, emit: :RBRACE)
14
+ rule(/\[/, emit: :LBRACKET)
15
+ rule(/\]/, emit: :RBRACKET)
16
+ rule(/:/, emit: :COLON)
17
+ rule(/,/, emit: :COMMA)
18
18
  rule(/true/) { emit :TRUE, true }
19
19
  rule(/false/) { emit :FALSE, false }
20
20
  rule(/null/) { emit :NULL, nil }
@@ -0,0 +1,16 @@
1
+ # SQL subset lexer
2
+
3
+ Tokenizes a small set of case-insensitive SQL keywords, unquoted names, decimal
4
+ numbers, single-quoted strings (including doubled quotes), line comments, and
5
+ operators. Keywords are uppercased; other values remain source text. Longest
6
+ match keeps `select_name` as one name and `>=` as one operator.
7
+ Quoted identifiers, block comments, vendor-specific syntax, and SQL grammar
8
+ validation are outside this example. Never use this lexer to sanitize SQL.
9
+
10
+ ```sh
11
+ ruby -Ilib -e 'load "examples/sql/lexer.flexr.rb"; p SqlExample::Lexer.new("select name from users where id >= 12;").tokens'
12
+ bundle exec ruby benchmark/run.rb --spec examples/sql/lexer.flexr.rb --iterations 1 --json
13
+ ```
14
+
15
+ The benchmark uses a repeated built-in query and compares runtime with generated
16
+ mode. Pass `--input-file PATH` to measure a different corpus.
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "flexr"
4
+
5
+ module SqlExample
6
+ class Lexer < Flexr::Lexer
7
+ emits :KEYWORD, :NAME, :STRING, :NUMBER, :OPERATOR, :COMMA, :DOT, :SEMICOLON, :LPAREN, :RPAREN
8
+ rule(/[ \t\r\n]+/, skip: true)
9
+ rule(/--[^\n]*/, skip: true)
10
+ rule(/SELECT|FROM|WHERE|AND|OR|NOT|NULL|AS|JOIN|ON|ORDER|BY|GROUP|HAVING|LIMIT|INSERT|INTO|VALUES|UPDATE|SET|DELETE/i) do
11
+ emit :KEYWORD, text.upcase
12
+ end
13
+ rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
14
+ rule(/'(?:''|[^'])*'/, emit: :STRING)
15
+ rule(/[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
16
+ rule(%r{>=|<=|<>|!=|[=<>+*/-]}, emit: :OPERATOR)
17
+ rule(/,/, emit: :COMMA)
18
+ rule(/\./, emit: :DOT)
19
+ rule(/;/, emit: :SEMICOLON)
20
+ rule(/\(/, emit: :LPAREN)
21
+ rule(/\)/, emit: :RPAREN)
22
+ end
23
+ end
@@ -0,0 +1,14 @@
1
+ # TOML subset lexer
2
+
3
+ Tokenizes bare names, single-line quoted strings, decimal numbers, booleans,
4
+ tables, inline tables, arrays, and comments. Values remain source text.
5
+ This is a lexer example, not a TOML validator: multiline strings, date/time
6
+ values, non-decimal numbers, and escape decoding need additional rules or a parser.
7
+
8
+ ```sh
9
+ ruby -Ilib -e 'load "examples/toml/lexer.flexr.rb"; p TomlExample::Lexer.new(%q(title = "flexr")).tokens'
10
+ bundle exec ruby benchmark/run.rb --spec examples/toml/lexer.flexr.rb --iterations 1 --json
11
+ ```
12
+
13
+ The benchmark uses a repeated built-in document fragment and compares runtime
14
+ with generated mode. Pass `--input-file PATH` to measure a different corpus.
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "flexr"
4
+
5
+ module TomlExample
6
+ class Lexer < Flexr::Lexer
7
+ emits :NAME, :STRING, :NUMBER, :BOOLEAN, :EQUAL, :DOT, :COMMA,
8
+ :LBRACKET, :RBRACKET, :LBRACE, :RBRACE
9
+ rule(/[ \t\r\n]+/, skip: true)
10
+ rule(/#[^\n]*/, skip: true)
11
+ rule(/"(?:\\.|[^"\\\r\n])*"|'[^'\r\n]*'/, emit: :STRING)
12
+ rule(/true|false/, emit: :BOOLEAN)
13
+ rule(/[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
14
+ rule(/[A-Za-z_][A-Za-z_0-9-]*/, emit: :NAME)
15
+ rule(/=/, emit: :EQUAL)
16
+ rule(/\./, emit: :DOT)
17
+ rule(/,/, emit: :COMMA)
18
+ rule(/\[/, emit: :LBRACKET)
19
+ rule(/\]/, emit: :RBRACKET)
20
+ rule(/\{/, emit: :LBRACE)
21
+ rule(/\}/, emit: :RBRACE)
22
+ end
23
+ end
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ eval_gemfile "../Gemfile"
4
+ gem "ruby-lsp", "~> 0.26.11"
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gemspec path: ".."
6
+ gem "rbs", "~> 4.0"
7
+ gem "steep", "~> 2.1"
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module ActionResolver
5
+ module_function
6
+
7
+ def resolve(skip:, emit:, block:, default:)
8
+ return :skip if skip
9
+ return [:emit, emit.to_sym] if emit
10
+ return block if block
11
+
12
+ default
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tempfile"
4
+
5
+ module Flexr
6
+ module ArtifactWriter
7
+ module_function
8
+
9
+ def default_generated_path(source_path)
10
+ return source_path.sub(/\.flexr\.rb\z/, ".rb") if source_path.end_with?(".flexr.rb")
11
+ return source_path.sub(/\.rb\z/, ".generated.rb") if source_path.end_with?(".rb")
12
+
13
+ "#{source_path}.generated.rb"
14
+ end
15
+
16
+ def write!(path, content, source_path: nil)
17
+ ensure_distinct!(source_path, path) if source_path
18
+ return :unchanged if File.file?(path) && File.binread(path) == content.b
19
+
20
+ atomic_replace(path, content)
21
+ :written
22
+ end
23
+
24
+ def ensure_distinct!(source_path, output_path)
25
+ same_path = File.expand_path(source_path) == File.expand_path(output_path)
26
+ same_file = File.exist?(source_path) && File.exist?(output_path) && File.identical?(source_path, output_path)
27
+ return unless same_path || same_file
28
+
29
+ raise ArgumentError, "refusing to overwrite input file: #{source_path}"
30
+ end
31
+
32
+ def atomic_replace(path, content)
33
+ expanded = File.expand_path(path)
34
+ directory = File.dirname(expanded)
35
+ basename = File.basename(expanded)
36
+ mode = File.exist?(expanded) ? File.stat(expanded).mode & 0o7777 : 0o666 & ~File.umask
37
+ temporary_path = nil
38
+
39
+ Tempfile.create([".#{basename}.", ".tmp"], directory, binmode: true) do |temporary|
40
+ temporary_path = temporary.path
41
+ temporary.write(content)
42
+ temporary.flush
43
+ temporary.fsync
44
+ temporary.chmod(mode)
45
+ temporary.close
46
+ File.rename(temporary_path, expanded)
47
+ temporary_path = nil
48
+ end
49
+ sync_directory(directory)
50
+ ensure
51
+ File.unlink(temporary_path) if temporary_path && File.exist?(temporary_path)
52
+ end
53
+
54
+ def sync_directory(directory)
55
+ File.open(directory, File::RDONLY, &:fsync)
56
+ rescue SystemCallError, IOError
57
+ # Some supported filesystems do not allow fsync on directories. The file
58
+ # itself has already been synced before the atomic rename.
59
+ end
60
+ end
61
+ end
@@ -2,23 +2,96 @@
2
2
 
3
3
  module Flexr
4
4
  module Automaton
5
- Region = Struct.new(:state, :bytes, :regexp, keyword_init: true)
5
+ Region = Struct.new(:state, :bytes, :regexp, :utf8_regexp, keyword_init: true)
6
+ WholeToken = Struct.new(:regexp, :rule_index, keyword_init: true)
6
7
 
7
8
  module Accel
8
9
  module_function
9
10
 
10
11
  def extract(dfa)
11
12
  dfa.transitions.each_index.filter_map do |state|
12
- bytes = Analysis.self_loop_set(dfa, state)
13
+ bytes = self_loop_set(dfa, state)
13
14
  next if bytes.empty?
14
15
 
15
- Region.new(state: state, bytes: bytes.freeze, regexp: regexp_for(bytes))
16
+ Region.new(
17
+ state: state, bytes: bytes.freeze,
18
+ regexp: regexp_for(bytes, binary: true),
19
+ utf8_regexp: bytes.all? { |byte| byte < 128 } ? regexp_for(bytes, binary: false) : nil
20
+ )
16
21
  end
17
22
  end
18
23
 
19
- def regexp_for(bytes)
24
+ def whole_token_plans(dfa, rules)
25
+ return if rules.any?(&:trailing) || !dfa.accepts[dfa.start].empty?
26
+ return if dfa.accepts.any? { |acceptances| acceptances.any? { |item| item.bol_only || item.end_anchor } }
27
+
28
+ plans = Array.new(256)
29
+ memo = {}
30
+ token_edges(dfa, dfa.start).each do |destination, bytes|
31
+ next if bytes.any? { |byte| byte >= 128 }
32
+ expression, winners = token_expression(dfa, destination, memo, [])
33
+ next unless expression && winners.one?
34
+
35
+ source = "[#{bytes_to_source(bytes)}]#{expression}"
36
+ next if source.bytesize > 4096
37
+
38
+ regexp = ::Regexp.new(source).freeze
39
+ plan = WholeToken.new(regexp: regexp, rule_index: winners.first).freeze
40
+ bytes.each { |byte| plans[byte] = plan }
41
+ end
42
+ plans.freeze unless plans.compact.empty?
43
+ end
44
+
45
+ def token_expression(dfa, state, memo, ancestors)
46
+ return memo[state] if memo.key?(state)
47
+ # shortcut: cap expansion at 32 states/4 KiB, revisit for measured DFA fallback bottlenecks.
48
+ return [nil, []] if ancestors.include?(state) || ancestors.length >= 32
49
+
50
+ edges = token_edges(dfa, state)
51
+ return memo[state] = [nil, []] if edges.values.any? { |bytes| bytes.any? { |byte| byte >= 128 } }
52
+
53
+ loops = edges.delete(state)
54
+ winner = dfa.accepts[state].first&.rule_index
55
+ winners = winner ? [winner] : []
56
+ alternatives = edges.map do |destination, bytes|
57
+ tail, accepted = token_expression(dfa, destination, memo, ancestors + [state])
58
+ return memo[state] = [nil, []] unless tail
59
+
60
+ winners |= accepted
61
+ return memo[state] = [nil, []] if winners.length > 1
62
+
63
+ "[#{bytes_to_source(bytes)}]#{tail}"
64
+ end
65
+ return memo[state] = [nil, []] if alternatives.empty? && winner.nil?
66
+
67
+ tail = alternatives.empty? ? "" : "(?:#{alternatives.join('|')})#{'?' unless winner.nil?}"
68
+ expression = "#{"[#{bytes_to_source(loops)}]*" if loops}#{tail}"
69
+ return memo[state] = [nil, []] if expression.bytesize > 4096
70
+
71
+ memo[state] = [expression, winners]
72
+ end
73
+
74
+ def token_edges(dfa, state)
75
+ (0..255).each_with_object({}) do |byte, edges|
76
+ destination = dfa.transition(state, byte)
77
+ (edges[destination] ||= []) << byte if destination
78
+ end
79
+ end
80
+
81
+ def self_loop_set(dfa, state)
82
+ representatives = Array.new(dfa.class_count)
83
+ dfa.ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
84
+ self_loops = representatives.each_index.map do |class_id|
85
+ byte = representatives.fetch(class_id)
86
+ byte && dfa.transition(state, byte) == state
87
+ end
88
+ dfa.ec.each_index.select { |byte| self_loops.fetch(dfa.ec.fetch(byte)) }
89
+ end
90
+
91
+ def regexp_for(bytes, binary: true)
20
92
  source = bytes_to_source(bytes)
21
- ::Regexp.new("(?:[#{source}])+", ::Regexp::NOENCODING)
93
+ options = binary ? ::Regexp::NOENCODING : 0
94
+ ::Regexp.new("(?:[#{source}])+", options)
22
95
  end
23
96
 
24
97
  def bytes_to_source(bytes)