flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: edb2c916db82e07c56290ab6365d50bf9a0a148278ca811a100063031315bbf5
4
- data.tar.gz: 13c08660e91681ab2143766d58072c4c4e3d081b5899a18568a772bfd80810b9
3
+ metadata.gz: 9ddc29026c24ee0e7fbb30d382589d810faa73e671324ea589c689744849b94b
4
+ data.tar.gz: ddda688781086eb9bd6707b4f92eeb4b1f7d24ecfdbd370942f4db0a06d42659
5
5
  SHA512:
6
- metadata.gz: aef18f76825b6544c03a3115a1dfeb8cd469560fb4bff23b2fa3e57cc9504e780a318ebfcc06d1ff7c8807c217b813a6a2768d154a8596cf7103347fd97e399e
7
- data.tar.gz: 0355b004bc8ff95a00fd4be5fd4644f4cce1a0f2feaad49da48ef023fc594453de208600d542a26a6bc6fcb4a645864770816ed8d4b4411e8607860801d1c43c
6
+ metadata.gz: 6e55f1ba6d9130a3eb2869866040f5b35c7305788589898cdd1c949030f18b5a4a2855acc0e914d0c287472370e8688b5fdd5e755e8aa00a93b0729ae8c10ebf
7
+ data.tar.gz: a92711bf693c04cbc0ac9f8983165041de096e5f6b5ac0a2f8997113208dd70224d165f018ac2a065c78a56023934570a5f10284a2e3c9d1c8d774483f166890
data/.rubocop.yml CHANGED
@@ -1,7 +1,8 @@
1
1
  AllCops:
2
- TargetRubyVersion: 3.1
2
+ TargetRubyVersion: 3.4
3
3
  NewCops: enable
4
4
  Exclude:
5
+ - "benchmark/golden/**/*"
5
6
  - "lib/flexr/codegen/**/*"
6
7
  - "lib/flexr/unicode/data/**/*"
7
8
  - "lib/flexr/regexp/tokenizer.rb"
data/CHANGELOG.md ADDED
@@ -0,0 +1,116 @@
1
+ # Changelog
2
+
3
+ Notable changes to flexr, newest first. Release dates use Japan Standard Time
4
+ (UTC+09:00).
5
+
6
+ ## [2.0.0](https://github.com/ydah/flexr/compare/v1.1.0...v2.0.0) - 2026-11-22
7
+
8
+ ### Added
9
+
10
+ - A real Ruby WASM Playground with editable specifications, diagnostics, DFA
11
+ diagrams, trace steps, generated code, presets, and share links.
12
+ - Input tracing with candidate lengths, rule priority, actions, state changes,
13
+ and human or JSON output for runtime and generated scanners.
14
+ - Optional phase profiling and isolated interpreter, YJIT, and ZJIT comparisons.
15
+ - RSpec and Minitest token assertions with explicit-update token snapshots.
16
+ - Grammar token validation with `check --grammar` and Lrama declaration export.
17
+ - Ruby LSP diagnostics and rule hovers for unsaved lexer specifications.
18
+ - Public API RBS signatures and typed consumer checks.
19
+ - TOML, GraphQL, and SQL subset examples with benchmarks and parity checks.
20
+ - A migration guide and static `check --target 2.0` impact reporting.
21
+ - Explicit `accel :strscan` whole-token acceleration for proven ASCII paths.
22
+ - Opt-in Ractor sharing of generated method-action tables; unsupported closures,
23
+ EOF blocks, and firstmatch reference caches are rejected explicitly.
24
+
25
+ ### Changed
26
+
27
+ - Require Ruby 3.4+ for runtime, generation, and standalone artifacts.
28
+ - Make structured tokens and locations immutable `Data` records while retaining
29
+ attribute readers and read-only indexing. Replace field mutation with `with`.
30
+ - Reject unknown `error_mode` values during initialization.
31
+ - Use artifact schema 2 and runtime ABI 2; regenerate v1 lexers after upgrading.
32
+ - Stabilize explicit firstmatch selection while retaining longest-match defaults.
33
+ - Reduce temporary Range allocations and use direct emit declarations in the
34
+ JSON example without changing its tokens.
35
+
36
+ ### Fixed
37
+
38
+ - Preserve UTF-8 source when generating lexers with Unicode action literals and surrounding constants.
39
+ - Show source context and installation guidance when Prism cannot load.
40
+ - Identify the installing file for incompatible generated artifacts when available.
41
+
42
+ ## [1.1.0](https://github.com/ydah/flexr/compare/v1.0.0...v1.1.0) - 2026-08-13
43
+
44
+ ### Added
45
+
46
+ - Runtime limits for lookahead, retained buffer size, and scanning steps, plus a
47
+ cancellation callback. `retain_input: false` discards consumed input so
48
+ streaming lexers can keep a sliding buffer.
49
+ - Regexp support for open repetition (`{n,}`), character-class intersection
50
+ (`&&`), trailing hyphens in classes, and hexadecimal classes (`\h` and `\H`).
51
+ - Explicit `flexr/runtime`, `flexr/generator`, and `flexr/cli` entry points.
52
+ `require "flexr"` loads build tools lazily.
53
+
54
+ ### Changed
55
+
56
+ - Unicode properties and POSIX classes now compile into the DFA, using the
57
+ same vendored Unicode 15.1.0 data for longest-match selection.
58
+ - Compiler construction and minimization avoid repeated work. Generated
59
+ scanners use state-specific fast paths, and token locations track positions
60
+ incrementally instead of rescanning consumed input.
61
+ - Packed tables and the `direct` backend avoid retaining duplicate transition
62
+ tables. The `auto` backend selects a representation using estimated table
63
+ size and lookup cost.
64
+
65
+ ### Fixed
66
+
67
+ - Aligned action precedence, nested state membership, and structured errors
68
+ across runtime, generated, and standalone modes.
69
+ - Static generation preserves heredocs and `__END__` data, resolves constants
70
+ in source order, and retains interpolated regexp options. Unsupported DSL
71
+ control flow and ambiguous lexer classes are rejected with `FLEXR-E017`.
72
+ - Generation refuses to overwrite its input, including symlink and hard-link
73
+ aliases. Output writes are atomic and leave unchanged files untouched;
74
+ non-`.flexr.rb` inputs default to a separate `.generated.rb` output.
75
+ - UTF-8 `less` and empty-match handling respect codepoint boundaries.
76
+ Token limits apply before actions run, and non-progressing scan cycles raise
77
+ a diagnostic instead of looping indefinitely.
78
+
79
+ ### Upgrade notes
80
+
81
+ - Regenerate lexer artifacts when upgrading from v1.0.0. Generated files now
82
+ record schema, runtime ABI, compiler, and Unicode versions; missing or
83
+ incompatible metadata raises `FLEXR-E020`.
84
+ - The retained input buffer now defaults to a 64 MiB limit. Adjust
85
+ `max_buffer_size` when retaining larger inputs, or use `retain_input: false`
86
+ for streaming lexers that can discard consumed prefixes.
87
+ - Specifications become immutable after compilation. Later DSL changes raise
88
+ `Flexr::FrozenSpecificationError`; unknown option names and conflicting state
89
+ declarations are rejected immediately.
90
+ - Unknown alphabetic regexp escapes are rejected instead of treated as
91
+ literals. Unknown `on_error` callback results raise
92
+ `InvalidRecoveryActionError` instead of falling back to the default policy.
93
+ - `firstmatch` remains experimental. Runtime and generated modes share its
94
+ rule-order behavior; differences from longest-match behavior produce
95
+ `FLEXR-W010` instead of aborting generation.
96
+
97
+ ## [1.0.0](https://github.com/ydah/flexr/tree/v1.0.0) - 2026-08-11
98
+
99
+ Initial release.
100
+
101
+ ### Added
102
+
103
+ - A pure Ruby lexer DSL with deterministic longest-match selection and source
104
+ order as the tie-breaker, available in runtime and generated modes.
105
+ - Static source generation with Prism, standalone output that does not require
106
+ the flexr gem, and a Rake task for generating lexers during builds.
107
+ - Lexical states, trailing context, EOF actions, token locations, streaming IO,
108
+ UTF-8 and binary input, and Unicode 15.1.0 property data.
109
+ - Stable `table`, `direct`, and `auto` backends, optional scanner acceleration,
110
+ packed tables, and an experimental `firstmatch` backend.
111
+ - CLI diagnostics, rule and DFA inspection, Graphviz output, benchmarks, and
112
+ import tooling for flex and Rexical specifications.
113
+ - Calculator, JSON, toy-language, Ruby-subset, Racc, and Lrama examples.
114
+
115
+ Runtime requires Ruby 3.1 or newer. Generation requires Ruby 3.3 or newer with
116
+ Prism available.
data/CONTRIBUTING.md CHANGED
@@ -8,8 +8,8 @@ bundle exec rake test
8
8
  bundle exec rubocop
9
9
  ```
10
10
 
11
- The runtime supports Ruby 3.1 and newer. Generator changes need Ruby 3.3 or
12
- newer because Prism is used for source analysis.
11
+ The runtime and generator require Ruby 3.4 or newer.
12
+ The generator uses Prism for source analysis.
13
13
 
14
14
  ## Validation layers
15
15
 
@@ -22,12 +22,55 @@ bundle exec rake modes:equivalence generated:verify golden:verify
22
22
  bundle exec rake test:differential fuzz
23
23
  ```
24
24
 
25
- The full CI workflow also checks Unicode invariants, acceleration equivalence,
25
+ The CI workflows also check Unicode invariants, acceleration equivalence,
26
26
  generated-only loading, Graphviz output, coverage, and benchmark regressions.
27
27
  Do not update a generated golden file to hide a semantic change; inspect the
28
28
  generated diff and update the compatibility documentation when the change is
29
29
  intentional.
30
30
 
31
+ Golden snapshots in `benchmark/golden/*.rb` contain generated Ruby with only
32
+ the compiler version and its derived digest normalized. Scanner code, user
33
+ literals, artifact schema, runtime ABI, and Unicode metadata remain part of
34
+ the comparison. A release version bump alone does not change these snapshots.
35
+ Generator safety tests separately check the real compiler version, digest,
36
+ and exact reproduction of committed generated artifacts.
37
+
38
+ On a mismatch, `golden:verify` prints a source diff and saves the actual snapshot
39
+ under `tmp/golden/`. Review that diff before accepting an intentional generator
40
+ change with `bundle exec rake golden:update`, then review the committed snapshot
41
+ diffs. Verification never updates the expected snapshots.
42
+
43
+ Markdown-only pushes and pull requests run documentation verification without
44
+ the test matrices or other Ruby checks. Changes limited to `site/` or its Pages
45
+ workflow skip Ruby CI and follow the Pages workflow's triggers. Changes that
46
+ also include code or Ruby configuration still run the full Ruby checks. The
47
+ daily scheduled runs retain all validation layers.
48
+
49
+ ## Replay a verification mismatch
50
+
51
+ `test:differential` and `fuzz` save the first mismatch in
52
+ `tmp/verification-failures/` and exit with status 1.
53
+ The differential-fuzz workflow attaches these files as the
54
+ `verification-failures` artifact when a job fails.
55
+ Replay one case from the repository root.
56
+
57
+ ```sh
58
+ FLEXR_FAILURE_CASE=path/to/failure.json bundle exec rake verification:replay
59
+ ```
60
+
61
+ The JSON records the commit, a fingerprint of the tracked diff and non-ignored
62
+ untracked files, Ruby and gem versions, seed, case number, comparison settings,
63
+ and both outcomes. For fuzz cases it also records the lexer path and its SHA-256.
64
+ The input is stored as hex with its original encoding, including invalid UTF-8.
65
+
66
+ Replay uses that exact input and runs the same comparison on the current
67
+ checkout. It reports drift in the fuzz lexer specification and exits with
68
+ status 1 if a mismatch persists or the comparison cannot run.
69
+ Before fixing a mismatch, record the base commit, expected behavior, and
70
+ validation commands in the task notes. Turn the mismatch into a focused
71
+ regression test that fails before the fix. Run it again after the fix with
72
+ the relevant validation layers and record the results.
73
+
31
74
  ## Documentation changes
32
75
 
33
76
  README is an entry point. Put task-oriented instructions in `docs/how-to/`,
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # flexr
2
2
 
3
- [![CI](https://github.com/ydah/flexr/actions/workflows/main.yml/badge.svg)](https://github.com/ydah/flexr/actions/workflows/main.yml)
3
+ [![Runtime](https://github.com/ydah/flexr/actions/workflows/runtime.yml/badge.svg)](https://github.com/ydah/flexr/actions/workflows/runtime.yml)
4
4
  [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.txt)
5
5
 
6
6
  [Product site](https://ydah.github.io/flexr/) · [Playground](https://ydah.github.io/flexr/playground/) · [Documentation](docs/README.md)
@@ -60,8 +60,8 @@ backreferences, or other non-regular constructs.
60
60
  | Mode | Build requirement | Runtime requirement | Best for |
61
61
  |---|---|---|---|
62
62
  | Runtime | `flexr` gem | `flexr` gem | Development, tests, and dynamic Ruby specs |
63
- | Generated | `flexr` plus Prism on Ruby 3.3+ | `flexr` gem | Reproducible deployment artifacts |
64
- | Standalone generated | `flexr` plus Prism on Ruby 3.3+ | Generated file and Ruby standard library | Distribution without the gem |
63
+ | Generated | `flexr` plus Prism on Ruby 3.4+ | `flexr` gem and Ruby 3.4+ | Reproducible deployment artifacts |
64
+ | Standalone generated | `flexr` plus Prism on Ruby 3.4+ | Generated file and Ruby 3.4+ standard library | Distribution without the gem |
65
65
 
66
66
  Static generation is the default. Use `--eval` only for trusted specifications;
67
67
  it executes the specification during the build. See the
@@ -89,10 +89,11 @@ matches your task:
89
89
  - [Explanation](docs/explanation/) — understand matching, backends, Unicode,
90
90
  generation, and security decisions.
91
91
  - [Examples](examples/) — executable specifications and parser integrations.
92
+ - [Changelog](CHANGELOG.md) — release highlights and upgrade notes.
92
93
 
93
94
  ## Compatibility and stability
94
95
 
95
- Runtime Ruby support starts at 3.1. The generator requires Ruby 3.3 or newer
96
+ Runtime Ruby support starts at 3.4. The generator requires Ruby 3.4 or newer
96
97
  because it uses Prism. Stable and experimental APIs are listed in the
97
98
  [public API contract](docs/reference/public-api.md). The vendored Unicode
98
99
  snapshot and generated-artifact policy are described in the
data/Rakefile CHANGED
@@ -10,6 +10,8 @@ require "rspec/core/rake_task"
10
10
  require "tmpdir"
11
11
  require "flexr"
12
12
  require_relative "tools/regexp_tokenizer_reference"
13
+ require_relative "tools/verification_failure"
14
+ require_relative "tools/golden"
13
15
 
14
16
  RSpec::Core::RakeTask.new(:spec)
15
17
  task test: :spec
@@ -20,10 +22,14 @@ module FlexrVerification
20
22
  TOKENIZER_SPEC = File.join(ROOT, "lib/flexr/regexp/tokenizer.flexr.rb").freeze
21
23
  TOKENIZER_GENERATED = File.join(ROOT, "lib/flexr/regexp/tokenizer.rb").freeze
22
24
  VERIFICATION_SPECS = (EXAMPLES + [TOKENIZER_SPEC]).freeze
25
+ FUZZ_OPTIONS = { error_mode: :panic }.freeze
23
26
  EXPECTED_INPUTS = {
24
27
  %r{/examples/calculator/} => "if ifx == = 12 + 3",
25
28
  %r{/examples/json/} => '{"answer": 42}',
26
29
  %r{/examples/toy_lang/} => "answer + 12",
30
+ %r{/examples/toml/} => 'title = "flexr"',
31
+ %r{/examples/graphql/} => 'query { user(id: 12) { name } }',
32
+ %r{/examples/sql/} => 'select name from users where id >= 12;',
27
33
  %r{/examples/ruby_subset/} => 'class Foo "ok" end',
28
34
  %r{/examples/with_racc/} => "12 + 3",
29
35
  %r{/examples/with_lrama/} => "12 - 3",
@@ -40,6 +46,9 @@ module FlexrVerification
40
46
  %r{/examples/calculator/} => "CalculatorExample::Lexer",
41
47
  %r{/examples/json/} => "JsonExample::Lexer",
42
48
  %r{/examples/toy_lang/} => "ToyLang::Lexer",
49
+ %r{/examples/toml/} => "TomlExample::Lexer",
50
+ %r{/examples/graphql/} => "GraphqlExample::Lexer",
51
+ %r{/examples/sql/} => "SqlExample::Lexer",
43
52
  %r{/examples/ruby_subset/} => "RubySubset::Lexer",
44
53
  %r{/examples/with_racc/} => "WithRacc::RaccLexer",
45
54
  %r{/examples/with_lrama/} => "WithLrama::LramaLexer",
@@ -53,7 +62,7 @@ module FlexrVerification
53
62
  end
54
63
 
55
64
  def golden_path(spec)
56
- name = "#{File.basename(File.dirname(spec))}_#{File.basename(spec, '.flexr.rb')}.sha256"
65
+ name = "#{File.basename(File.dirname(spec))}_#{File.basename(spec, '.flexr.rb')}.rb"
57
66
  File.join(ROOT, "benchmark/golden", name)
58
67
  end
59
68
 
@@ -96,8 +105,8 @@ module FlexrVerification
96
105
  end
97
106
  end
98
107
 
99
- def lexer_outcome(lexer_class, input, **options)
100
- { kind: :tokens, tokens: lexer_class.new(input.dup, **options).tokens }
108
+ def lexer_outcome(lexer_class, input, **)
109
+ { kind: :tokens, tokens: lexer_class.new(input.dup, **).tokens }
101
110
  rescue StandardError => e
102
111
  {
103
112
  kind: :error,
@@ -116,6 +125,91 @@ module FlexrVerification
116
125
  }
117
126
  end
118
127
 
128
+ def differential_outcomes(pattern, input, compiled, host_patterns)
129
+ key = [pattern.source, pattern.options]
130
+ host_expected = begin
131
+ (host_patterns[key] ||= Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options)).match?(input)
132
+ rescue RegexpError, ArgumentError, EncodingError
133
+ false
134
+ end
135
+ if Flexr.reference_pattern?(pattern)
136
+ reference = Flexr::Unicode::ReferenceRegexp.compiled(
137
+ pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
138
+ )
139
+ match = reference.match(input, 0)
140
+ vendored_expected = !match.nil? && match.begin(0).zero? && match[0].bytesize == input.bytesize
141
+ return unless vendored_expected == host_expected
142
+ end
143
+
144
+ [host_expected, (compiled[key] ||= Flexr.compile_pattern(pattern)).accept?(input)]
145
+ end
146
+
147
+ def differential_mismatch(pattern, input, expected, actual)
148
+ "differential mismatch: #{pattern.inspect} #{input.inspect} expected=#{expected} actual=#{actual}"
149
+ end
150
+
151
+ def with_fuzz_lexers(spec)
152
+ runtime_lexer = load_runtime(spec)
153
+ class_name = runtime_lexer.name
154
+ remove_constant(class_name)
155
+ Dir.mktmpdir("flexr-fuzz-") do |directory|
156
+ generated_path = File.join(directory, File.basename(spec))
157
+ File.binwrite(generated_path, generated_source(spec))
158
+ load generated_path
159
+ generated_lexer = constantize(class_name)
160
+ raise "no generated lexer found for #{spec}" unless generated_lexer&.__flexr_generated?
161
+
162
+ yield runtime_lexer, generated_lexer
163
+ end
164
+ ensure
165
+ if class_name
166
+ remove_constant(class_name)
167
+ parent, _, name = class_name.rpartition("::")
168
+ constantize(parent).const_set(name, runtime_lexer)
169
+ end
170
+ end
171
+
172
+ def fuzz_mismatch(spec, input, expected, actual)
173
+ "fuzz mode mismatch: #{relative_spec(spec)} input=#{input.inspect} " \
174
+ "runtime=#{expected.inspect} generated=#{actual.inspect}"
175
+ end
176
+
177
+ def replay_failure(record)
178
+ input = FailureCase.decode_bytes(FailureCase.section(record, "input"))
179
+ task = record.fetch("task")
180
+ case task
181
+ when "test:differential"
182
+ source = FailureCase.section(record, "pattern")
183
+ options = source.fetch("options")
184
+ raise ArgumentError, "invalid regexp options" unless options.is_a?(Integer)
185
+
186
+ pattern = Regexp.new(FailureCase.decode_bytes(FailureCase.section(source, "source")), options)
187
+ raise ArgumentError, "regexp encoding differs from the recorded encoding" unless pattern.encoding.name == source.fetch("encoding")
188
+
189
+ outcomes = differential_outcomes(pattern, input, {}, {})
190
+ abort "differential replay skipped: host and vendored Unicode results differ" unless outcomes
191
+ expected, actual = outcomes
192
+ abort differential_mismatch(pattern, input, expected, actual) unless expected == actual
193
+ when "fuzz"
194
+ recorded_spec = FailureCase.section(record, "spec")
195
+ path = recorded_spec.fetch("path")
196
+ spec = VERIFICATION_SPECS.find { |candidate| relative_spec(candidate) == path }
197
+ raise ArgumentError, "unknown fuzz lexer specification: #{path.inspect}" unless spec
198
+ options = FUZZ_OPTIONS.transform_keys(&:to_s).transform_values(&:to_s)
199
+ raise ArgumentError, "unsupported fuzz options" unless record.fetch("options") == options
200
+
201
+ warn "Fuzz lexer specification has changed: #{path}" unless Digest::SHA256.file(spec).hexdigest == recorded_spec.fetch("sha256")
202
+ with_fuzz_lexers(spec) do |runtime_lexer, generated_lexer|
203
+ expected = lexer_outcome(runtime_lexer, input, **FUZZ_OPTIONS)
204
+ actual = lexer_outcome(generated_lexer, input, **FUZZ_OPTIONS)
205
+ abort fuzz_mismatch(spec, input, expected, actual) unless expected == actual
206
+ end
207
+ else
208
+ raise ArgumentError, "unknown verification task: #{task.inspect}"
209
+ end
210
+ puts "verification: #{task} case #{record.fetch('case')} passed"
211
+ end
212
+
119
213
  def load_runtime(spec)
120
214
  class_name = runtime_class_name(spec)
121
215
  existing = constantize(class_name)
@@ -172,6 +266,12 @@ module FlexrVerification
172
266
  reference = inputs.map { |input| reference_lexer.new(input, error_mode: :panic).tokens }
173
267
  raise "acceleration token mismatch in #{spec}" unless accelerated == reference
174
268
 
269
+ remove_constant(runtime_class_name(spec))
270
+ whole_token_lexer = load_runtime(spec)
271
+ whole_token_lexer.accel(:strscan)
272
+ whole_tokens = inputs.map { |input| whole_token_lexer.new(input.dup.freeze, error_mode: :panic).tokens }
273
+ raise "whole-token acceleration mismatch in #{spec}" unless whole_tokens == reference
274
+
175
275
  lexer.compile!.machines.each_value do |machine|
176
276
  Flexr::Automaton::Accel.extract(machine.dfa).each do |region|
177
277
  256.times do |byte|
@@ -275,11 +375,30 @@ end
275
375
  task "golden:verify" do
276
376
  FlexrVerification::VERIFICATION_SPECS.each do |spec|
277
377
  golden = FlexrVerification.golden_path(spec)
278
- abort "missing golden file: #{golden}" unless File.file?(golden)
378
+ abort "missing golden snapshot: #{golden}" unless File.file?(golden)
379
+
380
+ expected = FlexrVerification.normalized_source(File.read(golden, encoding: Encoding::UTF_8))
381
+ actual = FlexrVerification::Golden.normalize(FlexrVerification.generated_source(spec))
382
+ next if expected == actual
383
+
384
+ actual_path = File.join(FlexrVerification::ROOT, "tmp/golden", File.basename(golden))
385
+ FileUtils.mkdir_p(File.dirname(actual_path))
386
+ File.binwrite(actual_path, actual)
387
+ begin
388
+ diff, = Open3.capture3("git", "--no-pager", "diff", "--no-index", "--color=never", "--", golden, actual_path)
389
+ warn diff unless diff.empty?
390
+ rescue Errno::ENOENT => e
391
+ warn "git diff unavailable: #{e.message}"
392
+ end
393
+ abort "golden mismatch: #{spec}\ncompare with: git diff --no-index -- #{golden.dump} #{actual_path.dump}"
394
+ end
395
+ end
279
396
 
280
- expected = File.read(golden).strip
281
- actual = Digest::SHA256.hexdigest(FlexrVerification.generated_source(spec))
282
- abort "golden mismatch: #{spec} (expected #{expected}, got #{actual})" unless expected == actual
397
+ task "golden:update" do
398
+ FlexrVerification::VERIFICATION_SPECS.each do |spec|
399
+ golden = FlexrVerification.golden_path(spec)
400
+ FileUtils.mkdir_p(File.dirname(golden))
401
+ File.binwrite(golden, FlexrVerification::Golden.normalize(FlexrVerification.generated_source(spec)))
283
402
  end
284
403
  end
285
404
 
@@ -389,51 +508,52 @@ task "bench:regression" do
389
508
  puts stdout
390
509
  end
391
510
 
511
+ desc "Profile runtime and generated scanning phases with TracePoint"
512
+ task "bench:profile" do
513
+ ruby "benchmark/profile.rb", "--iterations", ENV.fetch("FLEXR_BENCHMARK_ITERATIONS", "1")
514
+ end
515
+
516
+ desc "Measure interpreter, YJIT, and ZJIT where supported by this Ruby"
517
+ task "bench:matrix" do
518
+ ruby "benchmark/matrix.rb", "--iterations", ENV.fetch("FLEXR_BENCHMARK_ITERATIONS", "3")
519
+ end
520
+
392
521
  namespace :test do
522
+ desc "Compare Flexr pattern acceptance with Ruby's regexp engine"
393
523
  task :differential do
394
524
  patterns = [/[a-z]+/, /a(?:b|c)?/, /[0-9]{1,3}/, /[^\n]+/, /foo/,
395
525
  /[[:alpha:]]+/, /[[:alnum:]]+/, /\p{L}+/, /\p{Nd}+/]
396
526
  cases = Integer(ENV.fetch("FLEXR_DIFFERENTIAL_CASES", "1000000"), 10)
397
- random = Random.new(Integer(ENV.fetch("FLEXR_SEED", "17"), 10))
527
+ seed = Integer(ENV.fetch("FLEXR_SEED", "17"), 10)
528
+ random = Random.new(seed)
398
529
  unicode_inputs = ["", "a", "あ", "é", "ß", "Ω", "١", " ", "aあ", "éΩ", [0x18db8].pack("U")].freeze
399
530
  compiled = {}
400
531
  host_patterns = {}
401
532
  independent_cases = 0
402
533
  version_mismatches = 0
403
- cases.times do
534
+ cases.times do |index|
404
535
  pattern = patterns[random.rand(patterns.length)]
405
536
  input = if random.rand(3).zero?
406
537
  random.rand(2).zero? ? unicode_inputs.sample(random: random) : FlexrVerification.random_unicode_string(random)
407
538
  else
408
539
  Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
409
540
  end
410
- key = [pattern.source, pattern.options]
411
- host_expected = begin
412
- (host_patterns[key] ||= Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options)).match?(input)
413
- rescue RegexpError, ArgumentError, EncodingError
414
- false
415
- end
416
- if Flexr.reference_pattern?(pattern)
417
- reference = Flexr::Unicode::ReferenceRegexp.compiled(
418
- pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
419
- )
420
- match = reference.match(input, 0)
421
- vendored_expected = if match
422
- match.begin(0).zero? && match[0].bytesize == input.bytesize
423
- else
424
- false
425
- end
426
- unless vendored_expected == host_expected
427
- version_mismatches += 1
428
- next
429
- end
541
+ outcomes = FlexrVerification.differential_outcomes(pattern, input, compiled, host_patterns)
542
+ unless outcomes
543
+ version_mismatches += 1
544
+ next
430
545
  end
431
546
  independent_cases += 1
432
- actual = (compiled[key] ||= Flexr.compile_pattern(pattern)).accept?(input)
433
- next if host_expected == actual
434
-
435
- abort "differential mismatch: #{pattern.inspect} #{input.inspect} " \
436
- "expected=#{host_expected} actual=#{actual}"
547
+ expected, actual = outcomes
548
+ next if expected == actual
549
+
550
+ FlexrVerification::FailureCase.abort_with_record(
551
+ FlexrVerification.differential_mismatch(pattern, input, expected, actual),
552
+ root: FlexrVerification::ROOT, task: "test:differential", seed: seed, case: index + 1, input: input,
553
+ pattern: { source: FlexrVerification::FailureCase.encode_bytes(pattern.source), options: pattern.options,
554
+ encoding: pattern.encoding.name },
555
+ backends: { expected: "ruby_regexp", actual: "table" }, expected: expected, actual: actual
556
+ )
437
557
  end
438
558
  abort "differential did not exercise an independent host oracle" if independent_cases.zero?
439
559
 
@@ -442,49 +562,52 @@ namespace :test do
442
562
  end
443
563
  end
444
564
 
565
+ desc "Compare runtime and generated lexers on randomized inputs"
445
566
  task :fuzz do
446
567
  cases = Integer(ENV.fetch("FLEXR_FUZZ_CASES", "10000"), 10)
447
- random = Random.new(Integer(ENV.fetch("FLEXR_SEED", "17"), 10))
568
+ seed = Integer(ENV.fetch("FLEXR_SEED", "17"), 10)
569
+ random = Random.new(seed)
448
570
  FlexrVerification::VERIFICATION_SPECS.each do |spec|
449
- runtime_lexer = FlexrVerification.load_runtime(spec)
450
- parts = runtime_lexer.name.split("::")
451
- parent = Object
452
- parts[0...-1].each { |part| parent = parent.const_get(part) }
453
- parent.send(:remove_const, parts.last) if parent.const_defined?(parts.last, false)
454
- generated_path = File.join(Dir.tmpdir, "flexr-fuzz-#{Process.pid}-#{File.basename(spec)}")
455
- before = ObjectSpace.each_object(Class).to_a
456
- File.binwrite(generated_path, FlexrVerification.generated_source(spec))
457
- load generated_path
458
- generated_lexer = (ObjectSpace.each_object(Class).to_a - before).find do |klass|
459
- klass.respond_to?(:__flexr_spec)
460
- end
461
- raise "no generated lexer found for #{spec}" unless generated_lexer
462
-
463
- cases.times do
464
- input = case random.rand(4)
465
- when 0
466
- Array.new(random.rand(128)) { random.rand(0..127) }.pack("C*").force_encoding(Encoding::UTF_8)
467
- when 1
468
- FlexrVerification.random_unicode_string(random, max_codepoints: 32)
469
- when 2
470
- Array.new(random.rand(128)) { random.rand(0..255) }.pack("C*").force_encoding(Encoding::UTF_8)
471
- else
472
- (FlexrVerification.random_unicode_string(random, max_codepoints: 16) +
473
- Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
571
+ FlexrVerification.with_fuzz_lexers(spec) do |runtime_lexer, generated_lexer|
572
+ cases.times do |index|
573
+ input = case random.rand(4)
574
+ when 0
575
+ Array.new(random.rand(128)) { random.rand(0..127) }.pack("C*").force_encoding(Encoding::UTF_8)
576
+ when 1
577
+ FlexrVerification.random_unicode_string(random, max_codepoints: 32)
578
+ when 2
579
+ Array.new(random.rand(128)) { random.rand(0..255) }.pack("C*").force_encoding(Encoding::UTF_8)
580
+ else
581
+ (FlexrVerification.random_unicode_string(random, max_codepoints: 16) +
582
+ Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
583
+ end
584
+ expected = FlexrVerification.lexer_outcome(runtime_lexer, input, **FlexrVerification::FUZZ_OPTIONS)
585
+ actual = FlexrVerification.lexer_outcome(generated_lexer, input, **FlexrVerification::FUZZ_OPTIONS)
586
+ next if expected == actual
587
+
588
+ FlexrVerification::FailureCase.abort_with_record(
589
+ FlexrVerification.fuzz_mismatch(spec, input, expected, actual),
590
+ root: FlexrVerification::ROOT, task: "fuzz", seed: seed, case: index + 1, input: input,
591
+ spec: { path: FlexrVerification.relative_spec(spec), sha256: Digest::SHA256.file(spec).hexdigest },
592
+ backends: { runtime: runtime_lexer.__flexr_config.backend, generated: generated_lexer.__flexr_config.backend },
593
+ options: FlexrVerification::FUZZ_OPTIONS, expected: expected, actual: actual
594
+ )
474
595
  end
475
- runtime_outcome = FlexrVerification.lexer_outcome(runtime_lexer, input, error_mode: :panic)
476
- generated_outcome = FlexrVerification.lexer_outcome(generated_lexer, input, error_mode: :panic)
477
- next if runtime_outcome == generated_outcome
478
-
479
- abort "fuzz mode mismatch: #{spec} input=#{input.inspect} " \
480
- "runtime=#{runtime_outcome.inspect} generated=#{generated_outcome.inspect}"
481
596
  end
482
- ensure
483
- FileUtils.rm_f(generated_path) if generated_path
484
597
  end
485
598
  puts "fuzz: #{cases} inputs per example passed"
486
599
  end
487
600
 
601
+ namespace :verification do
602
+ desc "Replay one saved mismatch (FLEXR_FAILURE_CASE=path/to/case.json)"
603
+ task :replay do
604
+ path = ENV.fetch("FLEXR_FAILURE_CASE")
605
+ FlexrVerification.replay_failure(FlexrVerification::FailureCase.read(path))
606
+ rescue ArgumentError, KeyError, JSON::ParserError, SystemCallError, EncodingError, RegexpError, TypeError => e
607
+ abort "Cannot replay verification failure: #{e.message}"
608
+ end
609
+ end
610
+
488
611
  namespace :examples do
489
612
  task :check do
490
613
  FlexrVerification::EXAMPLES.each do |spec|
data/Steepfile ADDED
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ target :public_api do
4
+ signature "sig"
5
+ signature "spec/types"
6
+ check "spec/types/public_api.rb"
7
+ end
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "schema": 1,
3
3
  "spec": "examples/json/lexer.flexr.rb",
4
- "source_sha256": "909c8fbc07f27e5be871a6f7e2bb4d0c29996e4362dbb201dd90edd1ae55c7e3",
5
- "source_bytes": 745,
4
+ "source_sha256": "74d57259c5f67f4f6bf64cb870d4165095864fb61c58c9b8bff2d2d88da8b186",
5
+ "source_bytes": 733,
6
6
  "input_sha256": "f15489cc8fa286cd365fb167b3f99d3d216dc90c0de6279fe9f36c8d2215d7a7",
7
7
  "input_bytes": 140000,
8
8
  "tokens": 50000,