flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 54c55f186a1f4aa048a7a46ca86cae52eb5f9566888fc9cdc09f527d4b6b341f
4
- data.tar.gz: 7a5e9329e688889cfe665f3a4499deea7ca2e9de684f4fd5ad10847bd631fb4e
3
+ metadata.gz: 9ddc29026c24ee0e7fbb30d382589d810faa73e671324ea589c689744849b94b
4
+ data.tar.gz: ddda688781086eb9bd6707b4f92eeb4b1f7d24ecfdbd370942f4db0a06d42659
5
5
  SHA512:
6
- metadata.gz: 7399ef9f672ff36623a8d54a6eed882ee43fed2c7e9d59b417038bd0d8a9a32b892a8556774531aceb41b0e1c3729a169a003b190b8f2a330c4b825960da32b2
7
- data.tar.gz: 4ecdca40a21a0ff5af1146f20912297c96542bd2ec66412f3726461472c45e10ebc7e6d81a17ef942f3d0bfa036c8701f123fc15be72408dd62c5673c5636154
6
+ metadata.gz: 6e55f1ba6d9130a3eb2869866040f5b35c7305788589898cdd1c949030f18b5a4a2855acc0e914d0c287472370e8688b5fdd5e755e8aa00a93b0729ae8c10ebf
7
+ data.tar.gz: a92711bf693c04cbc0ac9f8983165041de096e5f6b5ac0a2f8997113208dd70224d165f018ac2a065c78a56023934570a5f10284a2e3c9d1c8d774483f166890
data/.rubocop.yml CHANGED
@@ -1,7 +1,8 @@
1
1
  AllCops:
2
- TargetRubyVersion: 3.1
2
+ TargetRubyVersion: 3.4
3
3
  NewCops: enable
4
4
  Exclude:
5
+ - "benchmark/golden/**/*"
5
6
  - "lib/flexr/codegen/**/*"
6
7
  - "lib/flexr/unicode/data/**/*"
7
8
  - "lib/flexr/regexp/tokenizer.rb"
data/CHANGELOG.md ADDED
@@ -0,0 +1,116 @@
1
+ # Changelog
2
+
3
+ Notable changes to flexr, newest first. Release dates use Japan Standard Time
4
+ (UTC+09:00).
5
+
6
+ ## [2.0.0](https://github.com/ydah/flexr/compare/v1.1.0...v2.0.0) - 2026-11-22
7
+
8
+ ### Added
9
+
10
+ - A real Ruby WASM Playground with editable specifications, diagnostics, DFA
11
+ diagrams, trace steps, generated code, presets, and share links.
12
+ - Input tracing with candidate lengths, rule priority, actions, state changes,
13
+ and human or JSON output for runtime and generated scanners.
14
+ - Optional phase profiling and isolated interpreter, YJIT, and ZJIT comparisons.
15
+ - RSpec and Minitest token assertions with explicit-update token snapshots.
16
+ - Grammar token validation with `check --grammar` and Lrama declaration export.
17
+ - Ruby LSP diagnostics and rule hovers for unsaved lexer specifications.
18
+ - Public API RBS signatures and typed consumer checks.
19
+ - TOML, GraphQL, and SQL subset examples with benchmarks and parity checks.
20
+ - A migration guide and static `check --target 2.0` impact reporting.
21
+ - Explicit `accel :strscan` whole-token acceleration for proven ASCII paths.
22
+ - Opt-in Ractor sharing of generated method-action tables; unsupported closures,
23
+ EOF blocks, and firstmatch reference caches are rejected explicitly.
24
+
25
+ ### Changed
26
+
27
+ - Require Ruby 3.4+ for runtime, generation, and standalone artifacts.
28
+ - Make structured tokens and locations immutable `Data` records while retaining
29
+ attribute readers and read-only indexing. Replace field mutation with `with`.
30
+ - Reject unknown `error_mode` values during initialization.
31
+ - Use artifact schema 2 and runtime ABI 2; regenerate v1 lexers after upgrading.
32
+ - Stabilize explicit firstmatch selection while retaining longest-match defaults.
33
+ - Reduce temporary Range allocations and use direct emit declarations in the
34
+ JSON example without changing its tokens.
35
+
36
+ ### Fixed
37
+
38
+ - Preserve UTF-8 source when generating lexers with Unicode action literals and surrounding constants.
39
+ - Show source context and installation guidance when Prism cannot load.
40
+ - Identify the installing file for incompatible generated artifacts when available.
41
+
42
+ ## [1.1.0](https://github.com/ydah/flexr/compare/v1.0.0...v1.1.0) - 2026-08-13
43
+
44
+ ### Added
45
+
46
+ - Runtime limits for lookahead, retained buffer size, and scanning steps, plus a
47
+ cancellation callback. `retain_input: false` discards consumed input so
48
+ streaming lexers can keep a sliding buffer.
49
+ - Regexp support for open repetition (`{n,}`), character-class intersection
50
+ (`&&`), trailing hyphens in classes, and hexadecimal classes (`\h` and `\H`).
51
+ - Explicit `flexr/runtime`, `flexr/generator`, and `flexr/cli` entry points.
52
+ `require "flexr"` loads build tools lazily.
53
+
54
+ ### Changed
55
+
56
+ - Unicode properties and POSIX classes now compile into the DFA, using the
57
+ same vendored Unicode 15.1.0 data for longest-match selection.
58
+ - Compiler construction and minimization avoid repeated work. Generated
59
+ scanners use state-specific fast paths, and token locations track positions
60
+ incrementally instead of rescanning consumed input.
61
+ - Packed tables and the `direct` backend avoid retaining duplicate transition
62
+ tables. The `auto` backend selects a representation using estimated table
63
+ size and lookup cost.
64
+
65
+ ### Fixed
66
+
67
+ - Aligned action precedence, nested state membership, and structured errors
68
+ across runtime, generated, and standalone modes.
69
+ - Static generation preserves heredocs and `__END__` data, resolves constants
70
+ in source order, and retains interpolated regexp options. Unsupported DSL
71
+ control flow and ambiguous lexer classes are rejected with `FLEXR-E017`.
72
+ - Generation refuses to overwrite its input, including symlink and hard-link
73
+ aliases. Output writes are atomic and leave unchanged files untouched;
74
+ non-`.flexr.rb` inputs default to a separate `.generated.rb` output.
75
+ - UTF-8 `less` and empty-match handling respect codepoint boundaries.
76
+ Token limits apply before actions run, and non-progressing scan cycles raise
77
+ a diagnostic instead of looping indefinitely.
78
+
79
+ ### Upgrade notes
80
+
81
+ - Regenerate lexer artifacts when upgrading from v1.0.0. Generated files now
82
+ record schema, runtime ABI, compiler, and Unicode versions; missing or
83
+ incompatible metadata raises `FLEXR-E020`.
84
+ - The retained input buffer now defaults to a 64 MiB limit. Adjust
85
+ `max_buffer_size` when retaining larger inputs, or use `retain_input: false`
86
+ for streaming lexers that can discard consumed prefixes.
87
+ - Specifications become immutable after compilation. Later DSL changes raise
88
+ `Flexr::FrozenSpecificationError`; unknown option names and conflicting state
89
+ declarations are rejected immediately.
90
+ - Unknown alphabetic regexp escapes are rejected instead of treated as
91
+ literals. Unknown `on_error` callback results raise
92
+ `InvalidRecoveryActionError` instead of falling back to the default policy.
93
+ - `firstmatch` remains experimental. Runtime and generated modes share its
94
+ rule-order behavior; differences from longest-match behavior produce
95
+ `FLEXR-W010` instead of aborting generation.
96
+
97
+ ## [1.0.0](https://github.com/ydah/flexr/tree/v1.0.0) - 2026-08-11
98
+
99
+ Initial release.
100
+
101
+ ### Added
102
+
103
+ - A pure Ruby lexer DSL with deterministic longest-match selection and source
104
+ order as the tie-breaker, available in runtime and generated modes.
105
+ - Static source generation with Prism, standalone output that does not require
106
+ the flexr gem, and a Rake task for generating lexers during builds.
107
+ - Lexical states, trailing context, EOF actions, token locations, streaming IO,
108
+ UTF-8 and binary input, and Unicode 15.1.0 property data.
109
+ - Stable `table`, `direct`, and `auto` backends, optional scanner acceleration,
110
+ packed tables, and an experimental `firstmatch` backend.
111
+ - CLI diagnostics, rule and DFA inspection, Graphviz output, benchmarks, and
112
+ import tooling for flex and Rexical specifications.
113
+ - Calculator, JSON, toy-language, Ruby-subset, Racc, and Lrama examples.
114
+
115
+ Runtime requires Ruby 3.1 or newer. Generation requires Ruby 3.3 or newer with
116
+ Prism available.
data/CONTRIBUTING.md CHANGED
@@ -8,8 +8,8 @@ bundle exec rake test
8
8
  bundle exec rubocop
9
9
  ```
10
10
 
11
- The runtime supports Ruby 3.1 and newer. Generator changes need Ruby 3.3 or
12
- newer because Prism is used for source analysis.
11
+ The runtime and generator require Ruby 3.4 or newer.
12
+ The generator uses Prism for source analysis.
13
13
 
14
14
  ## Validation layers
15
15
 
@@ -22,12 +22,55 @@ bundle exec rake modes:equivalence generated:verify golden:verify
22
22
  bundle exec rake test:differential fuzz
23
23
  ```
24
24
 
25
- The full CI workflow also checks Unicode invariants, acceleration equivalence,
25
+ The CI workflows also check Unicode invariants, acceleration equivalence,
26
26
  generated-only loading, Graphviz output, coverage, and benchmark regressions.
27
27
  Do not update a generated golden file to hide a semantic change; inspect the
28
28
  generated diff and update the compatibility documentation when the change is
29
29
  intentional.
30
30
 
31
+ Golden snapshots in `benchmark/golden/*.rb` contain generated Ruby with only
32
+ the compiler version and its derived digest normalized. Scanner code, user
33
+ literals, artifact schema, runtime ABI, and Unicode metadata remain part of
34
+ the comparison. A release version bump alone does not change these snapshots.
35
+ Generator safety tests separately check the real compiler version, digest,
36
+ and exact reproduction of committed generated artifacts.
37
+
38
+ On a mismatch, `golden:verify` prints a source diff and saves the actual snapshot
39
+ under `tmp/golden/`. Review that diff before accepting an intentional generator
40
+ change with `bundle exec rake golden:update`, then review the committed snapshot
41
+ diffs. Verification never updates the expected snapshots.
42
+
43
+ Markdown-only pushes and pull requests run documentation verification without
44
+ the test matrices or other Ruby checks. Changes limited to `site/` or its Pages
45
+ workflow skip Ruby CI and follow the Pages workflow's triggers. Changes that
46
+ also include code or Ruby configuration still run the full Ruby checks. The
47
+ daily scheduled runs retain all validation layers.
48
+
49
+ ## Replay a verification mismatch
50
+
51
+ `test:differential` and `fuzz` save the first mismatch in
52
+ `tmp/verification-failures/` and exit with status 1.
53
+ The differential-fuzz workflow attaches these files as the
54
+ `verification-failures` artifact when a job fails.
55
+ Replay one case from the repository root.
56
+
57
+ ```sh
58
+ FLEXR_FAILURE_CASE=path/to/failure.json bundle exec rake verification:replay
59
+ ```
60
+
61
+ The JSON records the commit, a fingerprint of the tracked diff and non-ignored
62
+ untracked files, Ruby and gem versions, seed, case number, comparison settings,
63
+ and both outcomes. For fuzz cases it also records the lexer path and its SHA-256.
64
+ The input is stored as hex with its original encoding, including invalid UTF-8.
65
+
66
+ Replay uses that exact input and runs the same comparison on the current
67
+ checkout. It reports drift in the fuzz lexer specification and exits with
68
+ status 1 if a mismatch persists or the comparison cannot run.
69
+ Before fixing a mismatch, record the base commit, expected behavior, and
70
+ validation commands in the task notes. Turn the mismatch into a focused
71
+ regression test that fails before the fix. Run it again after the fix with
72
+ the relevant validation layers and record the results.
73
+
31
74
  ## Documentation changes
32
75
 
33
76
  README is an entry point. Put task-oriented instructions in `docs/how-to/`,
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # flexr
2
2
 
3
- [![CI](https://github.com/ydah/flexr/actions/workflows/main.yml/badge.svg)](https://github.com/ydah/flexr/actions/workflows/main.yml)
3
+ [![Runtime](https://github.com/ydah/flexr/actions/workflows/runtime.yml/badge.svg)](https://github.com/ydah/flexr/actions/workflows/runtime.yml)
4
4
  [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.txt)
5
5
 
6
6
  [Product site](https://ydah.github.io/flexr/) · [Playground](https://ydah.github.io/flexr/playground/) · [Documentation](docs/README.md)
@@ -60,8 +60,8 @@ backreferences, or other non-regular constructs.
60
60
  | Mode | Build requirement | Runtime requirement | Best for |
61
61
  |---|---|---|---|
62
62
  | Runtime | `flexr` gem | `flexr` gem | Development, tests, and dynamic Ruby specs |
63
- | Generated | `flexr` plus Prism on Ruby 3.3+ | `flexr` gem | Reproducible deployment artifacts |
64
- | Standalone generated | `flexr` plus Prism on Ruby 3.3+ | Generated file and Ruby standard library | Distribution without the gem |
63
+ | Generated | `flexr` plus Prism on Ruby 3.4+ | `flexr` gem and Ruby 3.4+ | Reproducible deployment artifacts |
64
+ | Standalone generated | `flexr` plus Prism on Ruby 3.4+ | Generated file and Ruby 3.4+ standard library | Distribution without the gem |
65
65
 
66
66
  Static generation is the default. Use `--eval` only for trusted specifications;
67
67
  it executes the specification during the build. See the
@@ -89,10 +89,11 @@ matches your task:
89
89
  - [Explanation](docs/explanation/) — understand matching, backends, Unicode,
90
90
  generation, and security decisions.
91
91
  - [Examples](examples/) — executable specifications and parser integrations.
92
+ - [Changelog](CHANGELOG.md) — release highlights and upgrade notes.
92
93
 
93
94
  ## Compatibility and stability
94
95
 
95
- Runtime Ruby support starts at 3.1. The generator requires Ruby 3.3 or newer
96
+ Runtime Ruby support starts at 3.4. The generator requires Ruby 3.4 or newer
96
97
  because it uses Prism. Stable and experimental APIs are listed in the
97
98
  [public API contract](docs/reference/public-api.md). The vendored Unicode
98
99
  snapshot and generated-artifact policy are described in the
data/Rakefile CHANGED
@@ -10,6 +10,8 @@ require "rspec/core/rake_task"
10
10
  require "tmpdir"
11
11
  require "flexr"
12
12
  require_relative "tools/regexp_tokenizer_reference"
13
+ require_relative "tools/verification_failure"
14
+ require_relative "tools/golden"
13
15
 
14
16
  RSpec::Core::RakeTask.new(:spec)
15
17
  task test: :spec
@@ -20,10 +22,14 @@ module FlexrVerification
20
22
  TOKENIZER_SPEC = File.join(ROOT, "lib/flexr/regexp/tokenizer.flexr.rb").freeze
21
23
  TOKENIZER_GENERATED = File.join(ROOT, "lib/flexr/regexp/tokenizer.rb").freeze
22
24
  VERIFICATION_SPECS = (EXAMPLES + [TOKENIZER_SPEC]).freeze
25
+ FUZZ_OPTIONS = { error_mode: :panic }.freeze
23
26
  EXPECTED_INPUTS = {
24
27
  %r{/examples/calculator/} => "if ifx == = 12 + 3",
25
28
  %r{/examples/json/} => '{"answer": 42}',
26
29
  %r{/examples/toy_lang/} => "answer + 12",
30
+ %r{/examples/toml/} => 'title = "flexr"',
31
+ %r{/examples/graphql/} => 'query { user(id: 12) { name } }',
32
+ %r{/examples/sql/} => 'select name from users where id >= 12;',
27
33
  %r{/examples/ruby_subset/} => 'class Foo "ok" end',
28
34
  %r{/examples/with_racc/} => "12 + 3",
29
35
  %r{/examples/with_lrama/} => "12 - 3",
@@ -40,6 +46,9 @@ module FlexrVerification
40
46
  %r{/examples/calculator/} => "CalculatorExample::Lexer",
41
47
  %r{/examples/json/} => "JsonExample::Lexer",
42
48
  %r{/examples/toy_lang/} => "ToyLang::Lexer",
49
+ %r{/examples/toml/} => "TomlExample::Lexer",
50
+ %r{/examples/graphql/} => "GraphqlExample::Lexer",
51
+ %r{/examples/sql/} => "SqlExample::Lexer",
43
52
  %r{/examples/ruby_subset/} => "RubySubset::Lexer",
44
53
  %r{/examples/with_racc/} => "WithRacc::RaccLexer",
45
54
  %r{/examples/with_lrama/} => "WithLrama::LramaLexer",
@@ -53,7 +62,7 @@ module FlexrVerification
53
62
  end
54
63
 
55
64
  def golden_path(spec)
56
- name = "#{File.basename(File.dirname(spec))}_#{File.basename(spec, '.flexr.rb')}.sha256"
65
+ name = "#{File.basename(File.dirname(spec))}_#{File.basename(spec, '.flexr.rb')}.rb"
57
66
  File.join(ROOT, "benchmark/golden", name)
58
67
  end
59
68
 
@@ -86,6 +95,121 @@ module FlexrVerification
86
95
  codepoints.pack("U*")
87
96
  end
88
97
 
98
+ def merge_ranges(ranges)
99
+ ranges.sort_by(&:first).each_with_object([]) do |range, merged|
100
+ if merged.empty? || range.first > merged.last.last + 1
101
+ merged << range.dup
102
+ else
103
+ merged.last[1] = [merged.last.last, range.last].max
104
+ end
105
+ end
106
+ end
107
+
108
+ def lexer_outcome(lexer_class, input, **)
109
+ { kind: :tokens, tokens: lexer_class.new(input.dup, **).tokens }
110
+ rescue StandardError => e
111
+ {
112
+ kind: :error,
113
+ class: e.class.name,
114
+ message: e.message,
115
+ code: if e.respond_to?(:code)
116
+ e.code
117
+ elsif e.respond_to?(:diagnostic)
118
+ e.diagnostic&.code
119
+ end,
120
+ filename: e.respond_to?(:filename) ? e.filename : nil,
121
+ byte_pos: e.respond_to?(:byte_pos) ? e.byte_pos : nil,
122
+ line: e.respond_to?(:line) ? e.line : nil,
123
+ rule: e.respond_to?(:rule) ? e.rule : nil,
124
+ text: e.respond_to?(:text) ? e.text : nil
125
+ }
126
+ end
127
+
128
+ def differential_outcomes(pattern, input, compiled, host_patterns)
129
+ key = [pattern.source, pattern.options]
130
+ host_expected = begin
131
+ (host_patterns[key] ||= Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options)).match?(input)
132
+ rescue RegexpError, ArgumentError, EncodingError
133
+ false
134
+ end
135
+ if Flexr.reference_pattern?(pattern)
136
+ reference = Flexr::Unicode::ReferenceRegexp.compiled(
137
+ pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
138
+ )
139
+ match = reference.match(input, 0)
140
+ vendored_expected = !match.nil? && match.begin(0).zero? && match[0].bytesize == input.bytesize
141
+ return unless vendored_expected == host_expected
142
+ end
143
+
144
+ [host_expected, (compiled[key] ||= Flexr.compile_pattern(pattern)).accept?(input)]
145
+ end
146
+
147
+ def differential_mismatch(pattern, input, expected, actual)
148
+ "differential mismatch: #{pattern.inspect} #{input.inspect} expected=#{expected} actual=#{actual}"
149
+ end
150
+
151
+ def with_fuzz_lexers(spec)
152
+ runtime_lexer = load_runtime(spec)
153
+ class_name = runtime_lexer.name
154
+ remove_constant(class_name)
155
+ Dir.mktmpdir("flexr-fuzz-") do |directory|
156
+ generated_path = File.join(directory, File.basename(spec))
157
+ File.binwrite(generated_path, generated_source(spec))
158
+ load generated_path
159
+ generated_lexer = constantize(class_name)
160
+ raise "no generated lexer found for #{spec}" unless generated_lexer&.__flexr_generated?
161
+
162
+ yield runtime_lexer, generated_lexer
163
+ end
164
+ ensure
165
+ if class_name
166
+ remove_constant(class_name)
167
+ parent, _, name = class_name.rpartition("::")
168
+ constantize(parent).const_set(name, runtime_lexer)
169
+ end
170
+ end
171
+
172
+ def fuzz_mismatch(spec, input, expected, actual)
173
+ "fuzz mode mismatch: #{relative_spec(spec)} input=#{input.inspect} " \
174
+ "runtime=#{expected.inspect} generated=#{actual.inspect}"
175
+ end
176
+
177
+ def replay_failure(record)
178
+ input = FailureCase.decode_bytes(FailureCase.section(record, "input"))
179
+ task = record.fetch("task")
180
+ case task
181
+ when "test:differential"
182
+ source = FailureCase.section(record, "pattern")
183
+ options = source.fetch("options")
184
+ raise ArgumentError, "invalid regexp options" unless options.is_a?(Integer)
185
+
186
+ pattern = Regexp.new(FailureCase.decode_bytes(FailureCase.section(source, "source")), options)
187
+ raise ArgumentError, "regexp encoding differs from the recorded encoding" unless pattern.encoding.name == source.fetch("encoding")
188
+
189
+ outcomes = differential_outcomes(pattern, input, {}, {})
190
+ abort "differential replay skipped: host and vendored Unicode results differ" unless outcomes
191
+ expected, actual = outcomes
192
+ abort differential_mismatch(pattern, input, expected, actual) unless expected == actual
193
+ when "fuzz"
194
+ recorded_spec = FailureCase.section(record, "spec")
195
+ path = recorded_spec.fetch("path")
196
+ spec = VERIFICATION_SPECS.find { |candidate| relative_spec(candidate) == path }
197
+ raise ArgumentError, "unknown fuzz lexer specification: #{path.inspect}" unless spec
198
+ options = FUZZ_OPTIONS.transform_keys(&:to_s).transform_values(&:to_s)
199
+ raise ArgumentError, "unsupported fuzz options" unless record.fetch("options") == options
200
+
201
+ warn "Fuzz lexer specification has changed: #{path}" unless Digest::SHA256.file(spec).hexdigest == recorded_spec.fetch("sha256")
202
+ with_fuzz_lexers(spec) do |runtime_lexer, generated_lexer|
203
+ expected = lexer_outcome(runtime_lexer, input, **FUZZ_OPTIONS)
204
+ actual = lexer_outcome(generated_lexer, input, **FUZZ_OPTIONS)
205
+ abort fuzz_mismatch(spec, input, expected, actual) unless expected == actual
206
+ end
207
+ else
208
+ raise ArgumentError, "unknown verification task: #{task.inspect}"
209
+ end
210
+ puts "verification: #{task} case #{record.fetch('case')} passed"
211
+ end
212
+
89
213
  def load_runtime(spec)
90
214
  class_name = runtime_class_name(spec)
91
215
  existing = constantize(class_name)
@@ -135,13 +259,19 @@ module FlexrVerification
135
259
  inputs = [input_for(spec)] + Array.new(32) do
136
260
  Array.new(random.rand(48..96)) { random.rand(32..126) }.pack("C*")
137
261
  end
138
- options = lexer.__flexr_config.options
139
- original_accel = options.fetch(:accel, :auto)
140
262
  accelerated = inputs.map { |input| lexer.new(input, error_mode: :panic).tokens }
141
- options[:accel] = :none
142
- reference = inputs.map { |input| lexer.new(input, error_mode: :panic).tokens }
263
+ remove_constant(runtime_class_name(spec))
264
+ reference_lexer = load_runtime(spec)
265
+ reference_lexer.accel(:none)
266
+ reference = inputs.map { |input| reference_lexer.new(input, error_mode: :panic).tokens }
143
267
  raise "acceleration token mismatch in #{spec}" unless accelerated == reference
144
268
 
269
+ remove_constant(runtime_class_name(spec))
270
+ whole_token_lexer = load_runtime(spec)
271
+ whole_token_lexer.accel(:strscan)
272
+ whole_tokens = inputs.map { |input| whole_token_lexer.new(input.dup.freeze, error_mode: :panic).tokens }
273
+ raise "whole-token acceleration mismatch in #{spec}" unless whole_tokens == reference
274
+
145
275
  lexer.compile!.machines.each_value do |machine|
146
276
  Flexr::Automaton::Accel.extract(machine.dfa).each do |region|
147
277
  256.times do |byte|
@@ -153,8 +283,6 @@ module FlexrVerification
153
283
  end
154
284
  end
155
285
  end
156
- ensure
157
- options[:accel] = original_accel if options && original_accel
158
286
  end
159
287
 
160
288
  def verify_dogfood(spec)
@@ -247,11 +375,30 @@ end
247
375
  task "golden:verify" do
248
376
  FlexrVerification::VERIFICATION_SPECS.each do |spec|
249
377
  golden = FlexrVerification.golden_path(spec)
250
- abort "missing golden file: #{golden}" unless File.file?(golden)
378
+ abort "missing golden snapshot: #{golden}" unless File.file?(golden)
379
+
380
+ expected = FlexrVerification.normalized_source(File.read(golden, encoding: Encoding::UTF_8))
381
+ actual = FlexrVerification::Golden.normalize(FlexrVerification.generated_source(spec))
382
+ next if expected == actual
383
+
384
+ actual_path = File.join(FlexrVerification::ROOT, "tmp/golden", File.basename(golden))
385
+ FileUtils.mkdir_p(File.dirname(actual_path))
386
+ File.binwrite(actual_path, actual)
387
+ begin
388
+ diff, = Open3.capture3("git", "--no-pager", "diff", "--no-index", "--color=never", "--", golden, actual_path)
389
+ warn diff unless diff.empty?
390
+ rescue Errno::ENOENT => e
391
+ warn "git diff unavailable: #{e.message}"
392
+ end
393
+ abort "golden mismatch: #{spec}\ncompare with: git diff --no-index -- #{golden.dump} #{actual_path.dump}"
394
+ end
395
+ end
251
396
 
252
- expected = File.read(golden).strip
253
- actual = Digest::SHA256.hexdigest(FlexrVerification.generated_source(spec))
254
- abort "golden mismatch: #{spec} (expected #{expected}, got #{actual})" unless expected == actual
397
+ task "golden:update" do
398
+ FlexrVerification::VERIFICATION_SPECS.each do |spec|
399
+ golden = FlexrVerification.golden_path(spec)
400
+ FileUtils.mkdir_p(File.dirname(golden))
401
+ File.binwrite(golden, FlexrVerification::Golden.normalize(FlexrVerification.generated_source(spec)))
255
402
  end
256
403
  end
257
404
 
@@ -290,16 +437,12 @@ rescue Errno::ENOENT => e
290
437
  end
291
438
 
292
439
  task "direct:verify" do
293
- unless defined?(RubyVM::InstructionSequence)
294
- puts "direct: disassembly unavailable on #{RUBY_ENGINE}; skipped"
295
- next
296
- end
297
-
298
440
  spec = File.join(FlexrVerification::ROOT, "examples/json/lexer.flexr.rb")
299
- disassembly = RubyVM::InstructionSequence.compile(FlexrVerification.generated_source(spec)).disasm
300
- abort "direct dispatch did not compile to opt_case_dispatch" unless disassembly.include?("opt_case_dispatch")
441
+ source = Flexr::Generator.new(spec, options: { backend: :direct }).generate
442
+ abort "direct payload is missing its flattened transition representation" unless
443
+ source.include?("direct: {nxt:") && !source.include?("__flexr_generated_direct_transition")
301
444
 
302
- puts "direct: opt_case_dispatch present"
445
+ puts "direct: single flattened representation present"
303
446
  end
304
447
 
305
448
  task "unicode:verify" do
@@ -315,6 +458,14 @@ task "unicode:verify" do
315
458
  previous = hi
316
459
  end
317
460
  end
461
+ expected_alnum = FlexrVerification.merge_ranges(
462
+ properties.fetch("L") + properties.fetch("N") + properties.fetch("Other_Alphabetic")
463
+ )
464
+ abort "vendored Alnum table is stale" unless properties.fetch("Alnum") == expected_alnum
465
+
466
+ expected_word = FlexrVerification.merge_ranges(expected_alnum + [[0x5f, 0x5f]])
467
+ abort "vendored Word table is stale" unless properties.fetch("Word") == expected_word
468
+
318
469
  scalar_count = 0
319
470
  (0..0x10_ffff).each do |codepoint|
320
471
  next if codepoint.between?(0xd800, 0xdfff)
@@ -357,88 +508,106 @@ task "bench:regression" do
357
508
  puts stdout
358
509
  end
359
510
 
511
+ desc "Profile runtime and generated scanning phases with TracePoint"
512
+ task "bench:profile" do
513
+ ruby "benchmark/profile.rb", "--iterations", ENV.fetch("FLEXR_BENCHMARK_ITERATIONS", "1")
514
+ end
515
+
516
+ desc "Measure interpreter, YJIT, and ZJIT where supported by this Ruby"
517
+ task "bench:matrix" do
518
+ ruby "benchmark/matrix.rb", "--iterations", ENV.fetch("FLEXR_BENCHMARK_ITERATIONS", "3")
519
+ end
520
+
360
521
  namespace :test do
522
+ desc "Compare Flexr pattern acceptance with Ruby's regexp engine"
361
523
  task :differential do
362
524
  patterns = [/[a-z]+/, /a(?:b|c)?/, /[0-9]{1,3}/, /[^\n]+/, /foo/,
363
525
  /[[:alpha:]]+/, /[[:alnum:]]+/, /\p{L}+/, /\p{Nd}+/]
364
526
  cases = Integer(ENV.fetch("FLEXR_DIFFERENTIAL_CASES", "1000000"), 10)
365
- random = Random.new(Integer(ENV.fetch("FLEXR_SEED", "17"), 10))
527
+ seed = Integer(ENV.fetch("FLEXR_SEED", "17"), 10)
528
+ random = Random.new(seed)
366
529
  unicode_inputs = ["", "a", "あ", "é", "ß", "Ω", "١", " ", "aあ", "éΩ", [0x18db8].pack("U")].freeze
367
530
  compiled = {}
368
- cases.times do
531
+ host_patterns = {}
532
+ independent_cases = 0
533
+ version_mismatches = 0
534
+ cases.times do |index|
369
535
  pattern = patterns[random.rand(patterns.length)]
370
536
  input = if random.rand(3).zero?
371
537
  random.rand(2).zero? ? unicode_inputs.sample(random: random) : FlexrVerification.random_unicode_string(random)
372
538
  else
373
539
  Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
374
540
  end
375
- expected = if Flexr.reference_pattern?(pattern)
376
- reference = Flexr::Unicode::ReferenceRegexp.compiled(
377
- pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
378
- )
379
- match = reference.match(input, 0)
380
- if match
381
- match.begin(0).zero? && match[0].bytesize == input.bytesize
382
- else
383
- false
384
- end
385
- else
386
- Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options).match?(input)
541
+ outcomes = FlexrVerification.differential_outcomes(pattern, input, compiled, host_patterns)
542
+ unless outcomes
543
+ version_mismatches += 1
544
+ next
387
545
  end
388
- key = [pattern.source, pattern.options]
389
- actual = (compiled[key] ||= Flexr.compile_pattern(pattern)).accept?(input)
546
+ independent_cases += 1
547
+ expected, actual = outcomes
390
548
  next if expected == actual
391
549
 
392
- abort "differential mismatch: #{pattern.inspect} #{input.inspect} expected=#{expected} actual=#{actual}"
550
+ FlexrVerification::FailureCase.abort_with_record(
551
+ FlexrVerification.differential_mismatch(pattern, input, expected, actual),
552
+ root: FlexrVerification::ROOT, task: "test:differential", seed: seed, case: index + 1, input: input,
553
+ pattern: { source: FlexrVerification::FailureCase.encode_bytes(pattern.source), options: pattern.options,
554
+ encoding: pattern.encoding.name },
555
+ backends: { expected: "ruby_regexp", actual: "table" }, expected: expected, actual: actual
556
+ )
393
557
  end
394
- puts "differential: #{cases} cases passed"
558
+ abort "differential did not exercise an independent host oracle" if independent_cases.zero?
559
+
560
+ puts "differential: #{independent_cases} independent cases passed " \
561
+ "(#{version_mismatches} host/vendored Unicode differences skipped)"
395
562
  end
396
563
  end
397
564
 
565
+ desc "Compare runtime and generated lexers on randomized inputs"
398
566
  task :fuzz do
399
567
  cases = Integer(ENV.fetch("FLEXR_FUZZ_CASES", "10000"), 10)
400
- random = Random.new(Integer(ENV.fetch("FLEXR_SEED", "17"), 10))
568
+ seed = Integer(ENV.fetch("FLEXR_SEED", "17"), 10)
569
+ random = Random.new(seed)
401
570
  FlexrVerification::VERIFICATION_SPECS.each do |spec|
402
- runtime_lexer = FlexrVerification.load_runtime(spec)
403
- parts = runtime_lexer.name.split("::")
404
- parent = Object
405
- parts[0...-1].each { |part| parent = parent.const_get(part) }
406
- parent.send(:remove_const, parts.last) if parent.const_defined?(parts.last, false)
407
- generated_path = File.join(Dir.tmpdir, "flexr-fuzz-#{Process.pid}-#{File.basename(spec)}")
408
- before = ObjectSpace.each_object(Class).to_a
409
- File.binwrite(generated_path, FlexrVerification.generated_source(spec))
410
- load generated_path
411
- generated_lexer = (ObjectSpace.each_object(Class).to_a - before).find do |klass|
412
- klass.respond_to?(:__flexr_spec)
413
- end
414
- raise "no generated lexer found for #{spec}" unless generated_lexer
415
-
416
- cases.times do
417
- input = case random.rand(4)
418
- when 0
419
- Array.new(random.rand(128)) { random.rand(0..127) }.pack("C*").force_encoding(Encoding::UTF_8)
420
- when 1
421
- FlexrVerification.random_unicode_string(random, max_codepoints: 32)
422
- when 2
423
- Array.new(random.rand(128)) { random.rand(0..255) }.pack("C*").force_encoding(Encoding::UTF_8)
424
- else
425
- (FlexrVerification.random_unicode_string(random, max_codepoints: 16) +
426
- Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
571
+ FlexrVerification.with_fuzz_lexers(spec) do |runtime_lexer, generated_lexer|
572
+ cases.times do |index|
573
+ input = case random.rand(4)
574
+ when 0
575
+ Array.new(random.rand(128)) { random.rand(0..127) }.pack("C*").force_encoding(Encoding::UTF_8)
576
+ when 1
577
+ FlexrVerification.random_unicode_string(random, max_codepoints: 32)
578
+ when 2
579
+ Array.new(random.rand(128)) { random.rand(0..255) }.pack("C*").force_encoding(Encoding::UTF_8)
580
+ else
581
+ (FlexrVerification.random_unicode_string(random, max_codepoints: 16) +
582
+ Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
583
+ end
584
+ expected = FlexrVerification.lexer_outcome(runtime_lexer, input, **FlexrVerification::FUZZ_OPTIONS)
585
+ actual = FlexrVerification.lexer_outcome(generated_lexer, input, **FlexrVerification::FUZZ_OPTIONS)
586
+ next if expected == actual
587
+
588
+ FlexrVerification::FailureCase.abort_with_record(
589
+ FlexrVerification.fuzz_mismatch(spec, input, expected, actual),
590
+ root: FlexrVerification::ROOT, task: "fuzz", seed: seed, case: index + 1, input: input,
591
+ spec: { path: FlexrVerification.relative_spec(spec), sha256: Digest::SHA256.file(spec).hexdigest },
592
+ backends: { runtime: runtime_lexer.__flexr_config.backend, generated: generated_lexer.__flexr_config.backend },
593
+ options: FlexrVerification::FUZZ_OPTIONS, expected: expected, actual: actual
594
+ )
427
595
  end
428
- runtime_tokens = runtime_lexer.new(input, error_mode: :panic).tokens
429
- generated_tokens = generated_lexer.new(input, error_mode: :panic).tokens
430
- next if runtime_tokens == generated_tokens
431
-
432
- abort "fuzz mode mismatch: #{spec} input=#{input.inspect} runtime=#{runtime_tokens.inspect} generated=#{generated_tokens.inspect}"
433
- rescue Flexr::LexError, ArgumentError, EncodingError
434
- # Invalid input and user-defined error actions are expected fuzz outcomes.
435
596
  end
436
- ensure
437
- FileUtils.rm_f(generated_path) if generated_path
438
597
  end
439
598
  puts "fuzz: #{cases} inputs per example passed"
440
599
  end
441
600
 
601
+ namespace :verification do
602
+ desc "Replay one saved mismatch (FLEXR_FAILURE_CASE=path/to/case.json)"
603
+ task :replay do
604
+ path = ENV.fetch("FLEXR_FAILURE_CASE")
605
+ FlexrVerification.replay_failure(FlexrVerification::FailureCase.read(path))
606
+ rescue ArgumentError, KeyError, JSON::ParserError, SystemCallError, EncodingError, RegexpError, TypeError => e
607
+ abort "Cannot replay verification failure: #{e.message}"
608
+ end
609
+ end
610
+
442
611
  namespace :examples do
443
612
  task :check do
444
613
  FlexrVerification::EXAMPLES.each do |spec|