flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9ddc29026c24ee0e7fbb30d382589d810faa73e671324ea589c689744849b94b
|
|
4
|
+
data.tar.gz: ddda688781086eb9bd6707b4f92eeb4b1f7d24ecfdbd370942f4db0a06d42659
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6e55f1ba6d9130a3eb2869866040f5b35c7305788589898cdd1c949030f18b5a4a2855acc0e914d0c287472370e8688b5fdd5e755e8aa00a93b0729ae8c10ebf
|
|
7
|
+
data.tar.gz: a92711bf693c04cbc0ac9f8983165041de096e5f6b5ac0a2f8997113208dd70224d165f018ac2a065c78a56023934570a5f10284a2e3c9d1c8d774483f166890
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
Notable changes to flexr, newest first. Release dates use Japan Standard Time
|
|
4
|
+
(UTC+09:00).
|
|
5
|
+
|
|
6
|
+
## [2.0.0](https://github.com/ydah/flexr/compare/v1.1.0...v2.0.0) - 2026-11-22
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
|
|
10
|
+
- A real Ruby WASM Playground with editable specifications, diagnostics, DFA
|
|
11
|
+
diagrams, trace steps, generated code, presets, and share links.
|
|
12
|
+
- Input tracing with candidate lengths, rule priority, actions, state changes,
|
|
13
|
+
and human or JSON output for runtime and generated scanners.
|
|
14
|
+
- Optional phase profiling and isolated interpreter, YJIT, and ZJIT comparisons.
|
|
15
|
+
- RSpec and Minitest token assertions with explicit-update token snapshots.
|
|
16
|
+
- Grammar token validation with `check --grammar` and Lrama declaration export.
|
|
17
|
+
- Ruby LSP diagnostics and rule hovers for unsaved lexer specifications.
|
|
18
|
+
- Public API RBS signatures and typed consumer checks.
|
|
19
|
+
- TOML, GraphQL, and SQL subset examples with benchmarks and parity checks.
|
|
20
|
+
- A migration guide and static `check --target 2.0` impact reporting.
|
|
21
|
+
- Explicit `accel :strscan` whole-token acceleration for proven ASCII paths.
|
|
22
|
+
- Opt-in Ractor sharing of generated method-action tables; unsupported closures,
|
|
23
|
+
EOF blocks, and firstmatch reference caches are rejected explicitly.
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
|
|
27
|
+
- Require Ruby 3.4+ for runtime, generation, and standalone artifacts.
|
|
28
|
+
- Make structured tokens and locations immutable `Data` records while retaining
|
|
29
|
+
attribute readers and read-only indexing. Replace field mutation with `with`.
|
|
30
|
+
- Reject unknown `error_mode` values during initialization.
|
|
31
|
+
- Use artifact schema 2 and runtime ABI 2; regenerate v1 lexers after upgrading.
|
|
32
|
+
- Stabilize explicit firstmatch selection while retaining longest-match defaults.
|
|
33
|
+
- Reduce temporary Range allocations and use direct emit declarations in the
|
|
34
|
+
JSON example without changing its tokens.
|
|
35
|
+
|
|
36
|
+
### Fixed
|
|
37
|
+
|
|
38
|
+
- Preserve UTF-8 source when generating lexers with Unicode action literals and surrounding constants.
|
|
39
|
+
- Show source context and installation guidance when Prism cannot load.
|
|
40
|
+
- Identify the installing file for incompatible generated artifacts when available.
|
|
41
|
+
|
|
42
|
+
## [1.1.0](https://github.com/ydah/flexr/compare/v1.0.0...v1.1.0) - 2026-08-13
|
|
43
|
+
|
|
44
|
+
### Added
|
|
45
|
+
|
|
46
|
+
- Runtime limits for lookahead, retained buffer size, and scanning steps, plus a
|
|
47
|
+
cancellation callback. `retain_input: false` discards consumed input so
|
|
48
|
+
streaming lexers can keep a sliding buffer.
|
|
49
|
+
- Regexp support for open repetition (`{n,}`), character-class intersection
|
|
50
|
+
(`&&`), trailing hyphens in classes, and hexadecimal classes (`\h` and `\H`).
|
|
51
|
+
- Explicit `flexr/runtime`, `flexr/generator`, and `flexr/cli` entry points.
|
|
52
|
+
`require "flexr"` loads build tools lazily.
|
|
53
|
+
|
|
54
|
+
### Changed
|
|
55
|
+
|
|
56
|
+
- Unicode properties and POSIX classes now compile into the DFA, using the
|
|
57
|
+
same vendored Unicode 15.1.0 data for longest-match selection.
|
|
58
|
+
- Compiler construction and minimization avoid repeated work. Generated
|
|
59
|
+
scanners use state-specific fast paths, and token locations track positions
|
|
60
|
+
incrementally instead of rescanning consumed input.
|
|
61
|
+
- Packed tables and the `direct` backend avoid retaining duplicate transition
|
|
62
|
+
tables. The `auto` backend selects a representation using estimated table
|
|
63
|
+
size and lookup cost.
|
|
64
|
+
|
|
65
|
+
### Fixed
|
|
66
|
+
|
|
67
|
+
- Aligned action precedence, nested state membership, and structured errors
|
|
68
|
+
across runtime, generated, and standalone modes.
|
|
69
|
+
- Static generation preserves heredocs and `__END__` data, resolves constants
|
|
70
|
+
in source order, and retains interpolated regexp options. Unsupported DSL
|
|
71
|
+
control flow and ambiguous lexer classes are rejected with `FLEXR-E017`.
|
|
72
|
+
- Generation refuses to overwrite its input, including symlink and hard-link
|
|
73
|
+
aliases. Output writes are atomic and leave unchanged files untouched;
|
|
74
|
+
non-`.flexr.rb` inputs default to a separate `.generated.rb` output.
|
|
75
|
+
- UTF-8 `less` and empty-match handling respect codepoint boundaries.
|
|
76
|
+
Token limits apply before actions run, and non-progressing scan cycles raise
|
|
77
|
+
a diagnostic instead of looping indefinitely.
|
|
78
|
+
|
|
79
|
+
### Upgrade notes
|
|
80
|
+
|
|
81
|
+
- Regenerate lexer artifacts when upgrading from v1.0.0. Generated files now
|
|
82
|
+
record schema, runtime ABI, compiler, and Unicode versions; missing or
|
|
83
|
+
incompatible metadata raises `FLEXR-E020`.
|
|
84
|
+
- The retained input buffer now defaults to a 64 MiB limit. Adjust
|
|
85
|
+
`max_buffer_size` when retaining larger inputs, or use `retain_input: false`
|
|
86
|
+
for streaming lexers that can discard consumed prefixes.
|
|
87
|
+
- Specifications become immutable after compilation. Later DSL changes raise
|
|
88
|
+
`Flexr::FrozenSpecificationError`; unknown option names and conflicting state
|
|
89
|
+
declarations are rejected immediately.
|
|
90
|
+
- Unknown alphabetic regexp escapes are rejected instead of treated as
|
|
91
|
+
literals. Unknown `on_error` callback results raise
|
|
92
|
+
`InvalidRecoveryActionError` instead of falling back to the default policy.
|
|
93
|
+
- `firstmatch` remains experimental. Runtime and generated modes share its
|
|
94
|
+
rule-order behavior; differences from longest-match behavior produce
|
|
95
|
+
`FLEXR-W010` instead of aborting generation.
|
|
96
|
+
|
|
97
|
+
## [1.0.0](https://github.com/ydah/flexr/tree/v1.0.0) - 2026-08-11
|
|
98
|
+
|
|
99
|
+
Initial release.
|
|
100
|
+
|
|
101
|
+
### Added
|
|
102
|
+
|
|
103
|
+
- A pure Ruby lexer DSL with deterministic longest-match selection and source
|
|
104
|
+
order as the tie-breaker, available in runtime and generated modes.
|
|
105
|
+
- Static source generation with Prism, standalone output that does not require
|
|
106
|
+
the flexr gem, and a Rake task for generating lexers during builds.
|
|
107
|
+
- Lexical states, trailing context, EOF actions, token locations, streaming IO,
|
|
108
|
+
UTF-8 and binary input, and Unicode 15.1.0 property data.
|
|
109
|
+
- Stable `table`, `direct`, and `auto` backends, optional scanner acceleration,
|
|
110
|
+
packed tables, and an experimental `firstmatch` backend.
|
|
111
|
+
- CLI diagnostics, rule and DFA inspection, Graphviz output, benchmarks, and
|
|
112
|
+
import tooling for flex and Rexical specifications.
|
|
113
|
+
- Calculator, JSON, toy-language, Ruby-subset, Racc, and Lrama examples.
|
|
114
|
+
|
|
115
|
+
Runtime requires Ruby 3.1 or newer. Generation requires Ruby 3.3 or newer with
|
|
116
|
+
Prism available.
|
data/CONTRIBUTING.md
CHANGED
|
@@ -8,8 +8,8 @@ bundle exec rake test
|
|
|
8
8
|
bundle exec rubocop
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
The runtime
|
|
12
|
-
|
|
11
|
+
The runtime and generator require Ruby 3.4 or newer.
|
|
12
|
+
The generator uses Prism for source analysis.
|
|
13
13
|
|
|
14
14
|
## Validation layers
|
|
15
15
|
|
|
@@ -22,12 +22,55 @@ bundle exec rake modes:equivalence generated:verify golden:verify
|
|
|
22
22
|
bundle exec rake test:differential fuzz
|
|
23
23
|
```
|
|
24
24
|
|
|
25
|
-
The
|
|
25
|
+
The CI workflows also check Unicode invariants, acceleration equivalence,
|
|
26
26
|
generated-only loading, Graphviz output, coverage, and benchmark regressions.
|
|
27
27
|
Do not update a generated golden file to hide a semantic change; inspect the
|
|
28
28
|
generated diff and update the compatibility documentation when the change is
|
|
29
29
|
intentional.
|
|
30
30
|
|
|
31
|
+
Golden snapshots in `benchmark/golden/*.rb` contain generated Ruby with only
|
|
32
|
+
the compiler version and its derived digest normalized. Scanner code, user
|
|
33
|
+
literals, artifact schema, runtime ABI, and Unicode metadata remain part of
|
|
34
|
+
the comparison. A release version bump alone does not change these snapshots.
|
|
35
|
+
Generator safety tests separately check the real compiler version, digest,
|
|
36
|
+
and exact reproduction of committed generated artifacts.
|
|
37
|
+
|
|
38
|
+
On a mismatch, `golden:verify` prints a source diff and saves the actual snapshot
|
|
39
|
+
under `tmp/golden/`. Review that diff before accepting an intentional generator
|
|
40
|
+
change with `bundle exec rake golden:update`, then review the committed snapshot
|
|
41
|
+
diffs. Verification never updates the expected snapshots.
|
|
42
|
+
|
|
43
|
+
Markdown-only pushes and pull requests run documentation verification without
|
|
44
|
+
the test matrices or other Ruby checks. Changes limited to `site/` or its Pages
|
|
45
|
+
workflow skip Ruby CI and follow the Pages workflow's triggers. Changes that
|
|
46
|
+
also include code or Ruby configuration still run the full Ruby checks. The
|
|
47
|
+
daily scheduled runs retain all validation layers.
|
|
48
|
+
|
|
49
|
+
## Replay a verification mismatch
|
|
50
|
+
|
|
51
|
+
`test:differential` and `fuzz` save the first mismatch in
|
|
52
|
+
`tmp/verification-failures/` and exit with status 1.
|
|
53
|
+
The differential-fuzz workflow attaches these files as the
|
|
54
|
+
`verification-failures` artifact when a job fails.
|
|
55
|
+
Replay one case from the repository root.
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
FLEXR_FAILURE_CASE=path/to/failure.json bundle exec rake verification:replay
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The JSON records the commit, a fingerprint of the tracked diff and non-ignored
|
|
62
|
+
untracked files, Ruby and gem versions, seed, case number, comparison settings,
|
|
63
|
+
and both outcomes. For fuzz cases it also records the lexer path and its SHA-256.
|
|
64
|
+
The input is stored as hex with its original encoding, including invalid UTF-8.
|
|
65
|
+
|
|
66
|
+
Replay uses that exact input and runs the same comparison on the current
|
|
67
|
+
checkout. It reports drift in the fuzz lexer specification and exits with
|
|
68
|
+
status 1 if a mismatch persists or the comparison cannot run.
|
|
69
|
+
Before fixing a mismatch, record the base commit, expected behavior, and
|
|
70
|
+
validation commands in the task notes. Turn the mismatch into a focused
|
|
71
|
+
regression test that fails before the fix. Run it again after the fix with
|
|
72
|
+
the relevant validation layers and record the results.
|
|
73
|
+
|
|
31
74
|
## Documentation changes
|
|
32
75
|
|
|
33
76
|
README is an entry point. Put task-oriented instructions in `docs/how-to/`,
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# flexr
|
|
2
2
|
|
|
3
|
-
[](https://github.com/ydah/flexr/actions/workflows/runtime.yml)
|
|
4
4
|
[](LICENSE.txt)
|
|
5
5
|
|
|
6
6
|
[Product site](https://ydah.github.io/flexr/) · [Playground](https://ydah.github.io/flexr/playground/) · [Documentation](docs/README.md)
|
|
@@ -60,8 +60,8 @@ backreferences, or other non-regular constructs.
|
|
|
60
60
|
| Mode | Build requirement | Runtime requirement | Best for |
|
|
61
61
|
|---|---|---|---|
|
|
62
62
|
| Runtime | `flexr` gem | `flexr` gem | Development, tests, and dynamic Ruby specs |
|
|
63
|
-
| Generated | `flexr` plus Prism on Ruby 3.
|
|
64
|
-
| Standalone generated | `flexr` plus Prism on Ruby 3.
|
|
63
|
+
| Generated | `flexr` plus Prism on Ruby 3.4+ | `flexr` gem and Ruby 3.4+ | Reproducible deployment artifacts |
|
|
64
|
+
| Standalone generated | `flexr` plus Prism on Ruby 3.4+ | Generated file and Ruby 3.4+ standard library | Distribution without the gem |
|
|
65
65
|
|
|
66
66
|
Static generation is the default. Use `--eval` only for trusted specifications;
|
|
67
67
|
it executes the specification during the build. See the
|
|
@@ -89,10 +89,11 @@ matches your task:
|
|
|
89
89
|
- [Explanation](docs/explanation/) — understand matching, backends, Unicode,
|
|
90
90
|
generation, and security decisions.
|
|
91
91
|
- [Examples](examples/) — executable specifications and parser integrations.
|
|
92
|
+
- [Changelog](CHANGELOG.md) — release highlights and upgrade notes.
|
|
92
93
|
|
|
93
94
|
## Compatibility and stability
|
|
94
95
|
|
|
95
|
-
Runtime Ruby support starts at 3.
|
|
96
|
+
Runtime Ruby support starts at 3.4. The generator requires Ruby 3.4 or newer
|
|
96
97
|
because it uses Prism. Stable and experimental APIs are listed in the
|
|
97
98
|
[public API contract](docs/reference/public-api.md). The vendored Unicode
|
|
98
99
|
snapshot and generated-artifact policy are described in the
|
data/Rakefile
CHANGED
|
@@ -10,6 +10,8 @@ require "rspec/core/rake_task"
|
|
|
10
10
|
require "tmpdir"
|
|
11
11
|
require "flexr"
|
|
12
12
|
require_relative "tools/regexp_tokenizer_reference"
|
|
13
|
+
require_relative "tools/verification_failure"
|
|
14
|
+
require_relative "tools/golden"
|
|
13
15
|
|
|
14
16
|
RSpec::Core::RakeTask.new(:spec)
|
|
15
17
|
task test: :spec
|
|
@@ -20,10 +22,14 @@ module FlexrVerification
|
|
|
20
22
|
TOKENIZER_SPEC = File.join(ROOT, "lib/flexr/regexp/tokenizer.flexr.rb").freeze
|
|
21
23
|
TOKENIZER_GENERATED = File.join(ROOT, "lib/flexr/regexp/tokenizer.rb").freeze
|
|
22
24
|
VERIFICATION_SPECS = (EXAMPLES + [TOKENIZER_SPEC]).freeze
|
|
25
|
+
FUZZ_OPTIONS = { error_mode: :panic }.freeze
|
|
23
26
|
EXPECTED_INPUTS = {
|
|
24
27
|
%r{/examples/calculator/} => "if ifx == = 12 + 3",
|
|
25
28
|
%r{/examples/json/} => '{"answer": 42}',
|
|
26
29
|
%r{/examples/toy_lang/} => "answer + 12",
|
|
30
|
+
%r{/examples/toml/} => 'title = "flexr"',
|
|
31
|
+
%r{/examples/graphql/} => 'query { user(id: 12) { name } }',
|
|
32
|
+
%r{/examples/sql/} => 'select name from users where id >= 12;',
|
|
27
33
|
%r{/examples/ruby_subset/} => 'class Foo "ok" end',
|
|
28
34
|
%r{/examples/with_racc/} => "12 + 3",
|
|
29
35
|
%r{/examples/with_lrama/} => "12 - 3",
|
|
@@ -40,6 +46,9 @@ module FlexrVerification
|
|
|
40
46
|
%r{/examples/calculator/} => "CalculatorExample::Lexer",
|
|
41
47
|
%r{/examples/json/} => "JsonExample::Lexer",
|
|
42
48
|
%r{/examples/toy_lang/} => "ToyLang::Lexer",
|
|
49
|
+
%r{/examples/toml/} => "TomlExample::Lexer",
|
|
50
|
+
%r{/examples/graphql/} => "GraphqlExample::Lexer",
|
|
51
|
+
%r{/examples/sql/} => "SqlExample::Lexer",
|
|
43
52
|
%r{/examples/ruby_subset/} => "RubySubset::Lexer",
|
|
44
53
|
%r{/examples/with_racc/} => "WithRacc::RaccLexer",
|
|
45
54
|
%r{/examples/with_lrama/} => "WithLrama::LramaLexer",
|
|
@@ -53,7 +62,7 @@ module FlexrVerification
|
|
|
53
62
|
end
|
|
54
63
|
|
|
55
64
|
def golden_path(spec)
|
|
56
|
-
name = "#{File.basename(File.dirname(spec))}_#{File.basename(spec, '.flexr.rb')}.
|
|
65
|
+
name = "#{File.basename(File.dirname(spec))}_#{File.basename(spec, '.flexr.rb')}.rb"
|
|
57
66
|
File.join(ROOT, "benchmark/golden", name)
|
|
58
67
|
end
|
|
59
68
|
|
|
@@ -96,8 +105,8 @@ module FlexrVerification
|
|
|
96
105
|
end
|
|
97
106
|
end
|
|
98
107
|
|
|
99
|
-
def lexer_outcome(lexer_class, input, **
|
|
100
|
-
{ kind: :tokens, tokens: lexer_class.new(input.dup, **
|
|
108
|
+
def lexer_outcome(lexer_class, input, **)
|
|
109
|
+
{ kind: :tokens, tokens: lexer_class.new(input.dup, **).tokens }
|
|
101
110
|
rescue StandardError => e
|
|
102
111
|
{
|
|
103
112
|
kind: :error,
|
|
@@ -116,6 +125,91 @@ module FlexrVerification
|
|
|
116
125
|
}
|
|
117
126
|
end
|
|
118
127
|
|
|
128
|
+
def differential_outcomes(pattern, input, compiled, host_patterns)
|
|
129
|
+
key = [pattern.source, pattern.options]
|
|
130
|
+
host_expected = begin
|
|
131
|
+
(host_patterns[key] ||= Regexp.new("\\A(?:#{pattern.source})\\z", pattern.options)).match?(input)
|
|
132
|
+
rescue RegexpError, ArgumentError, EncodingError
|
|
133
|
+
false
|
|
134
|
+
end
|
|
135
|
+
if Flexr.reference_pattern?(pattern)
|
|
136
|
+
reference = Flexr::Unicode::ReferenceRegexp.compiled(
|
|
137
|
+
pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
|
|
138
|
+
)
|
|
139
|
+
match = reference.match(input, 0)
|
|
140
|
+
vendored_expected = !match.nil? && match.begin(0).zero? && match[0].bytesize == input.bytesize
|
|
141
|
+
return unless vendored_expected == host_expected
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
[host_expected, (compiled[key] ||= Flexr.compile_pattern(pattern)).accept?(input)]
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def differential_mismatch(pattern, input, expected, actual)
|
|
148
|
+
"differential mismatch: #{pattern.inspect} #{input.inspect} expected=#{expected} actual=#{actual}"
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def with_fuzz_lexers(spec)
|
|
152
|
+
runtime_lexer = load_runtime(spec)
|
|
153
|
+
class_name = runtime_lexer.name
|
|
154
|
+
remove_constant(class_name)
|
|
155
|
+
Dir.mktmpdir("flexr-fuzz-") do |directory|
|
|
156
|
+
generated_path = File.join(directory, File.basename(spec))
|
|
157
|
+
File.binwrite(generated_path, generated_source(spec))
|
|
158
|
+
load generated_path
|
|
159
|
+
generated_lexer = constantize(class_name)
|
|
160
|
+
raise "no generated lexer found for #{spec}" unless generated_lexer&.__flexr_generated?
|
|
161
|
+
|
|
162
|
+
yield runtime_lexer, generated_lexer
|
|
163
|
+
end
|
|
164
|
+
ensure
|
|
165
|
+
if class_name
|
|
166
|
+
remove_constant(class_name)
|
|
167
|
+
parent, _, name = class_name.rpartition("::")
|
|
168
|
+
constantize(parent).const_set(name, runtime_lexer)
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def fuzz_mismatch(spec, input, expected, actual)
|
|
173
|
+
"fuzz mode mismatch: #{relative_spec(spec)} input=#{input.inspect} " \
|
|
174
|
+
"runtime=#{expected.inspect} generated=#{actual.inspect}"
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def replay_failure(record)
|
|
178
|
+
input = FailureCase.decode_bytes(FailureCase.section(record, "input"))
|
|
179
|
+
task = record.fetch("task")
|
|
180
|
+
case task
|
|
181
|
+
when "test:differential"
|
|
182
|
+
source = FailureCase.section(record, "pattern")
|
|
183
|
+
options = source.fetch("options")
|
|
184
|
+
raise ArgumentError, "invalid regexp options" unless options.is_a?(Integer)
|
|
185
|
+
|
|
186
|
+
pattern = Regexp.new(FailureCase.decode_bytes(FailureCase.section(source, "source")), options)
|
|
187
|
+
raise ArgumentError, "regexp encoding differs from the recorded encoding" unless pattern.encoding.name == source.fetch("encoding")
|
|
188
|
+
|
|
189
|
+
outcomes = differential_outcomes(pattern, input, {}, {})
|
|
190
|
+
abort "differential replay skipped: host and vendored Unicode results differ" unless outcomes
|
|
191
|
+
expected, actual = outcomes
|
|
192
|
+
abort differential_mismatch(pattern, input, expected, actual) unless expected == actual
|
|
193
|
+
when "fuzz"
|
|
194
|
+
recorded_spec = FailureCase.section(record, "spec")
|
|
195
|
+
path = recorded_spec.fetch("path")
|
|
196
|
+
spec = VERIFICATION_SPECS.find { |candidate| relative_spec(candidate) == path }
|
|
197
|
+
raise ArgumentError, "unknown fuzz lexer specification: #{path.inspect}" unless spec
|
|
198
|
+
options = FUZZ_OPTIONS.transform_keys(&:to_s).transform_values(&:to_s)
|
|
199
|
+
raise ArgumentError, "unsupported fuzz options" unless record.fetch("options") == options
|
|
200
|
+
|
|
201
|
+
warn "Fuzz lexer specification has changed: #{path}" unless Digest::SHA256.file(spec).hexdigest == recorded_spec.fetch("sha256")
|
|
202
|
+
with_fuzz_lexers(spec) do |runtime_lexer, generated_lexer|
|
|
203
|
+
expected = lexer_outcome(runtime_lexer, input, **FUZZ_OPTIONS)
|
|
204
|
+
actual = lexer_outcome(generated_lexer, input, **FUZZ_OPTIONS)
|
|
205
|
+
abort fuzz_mismatch(spec, input, expected, actual) unless expected == actual
|
|
206
|
+
end
|
|
207
|
+
else
|
|
208
|
+
raise ArgumentError, "unknown verification task: #{task.inspect}"
|
|
209
|
+
end
|
|
210
|
+
puts "verification: #{task} case #{record.fetch('case')} passed"
|
|
211
|
+
end
|
|
212
|
+
|
|
119
213
|
def load_runtime(spec)
|
|
120
214
|
class_name = runtime_class_name(spec)
|
|
121
215
|
existing = constantize(class_name)
|
|
@@ -172,6 +266,12 @@ module FlexrVerification
|
|
|
172
266
|
reference = inputs.map { |input| reference_lexer.new(input, error_mode: :panic).tokens }
|
|
173
267
|
raise "acceleration token mismatch in #{spec}" unless accelerated == reference
|
|
174
268
|
|
|
269
|
+
remove_constant(runtime_class_name(spec))
|
|
270
|
+
whole_token_lexer = load_runtime(spec)
|
|
271
|
+
whole_token_lexer.accel(:strscan)
|
|
272
|
+
whole_tokens = inputs.map { |input| whole_token_lexer.new(input.dup.freeze, error_mode: :panic).tokens }
|
|
273
|
+
raise "whole-token acceleration mismatch in #{spec}" unless whole_tokens == reference
|
|
274
|
+
|
|
175
275
|
lexer.compile!.machines.each_value do |machine|
|
|
176
276
|
Flexr::Automaton::Accel.extract(machine.dfa).each do |region|
|
|
177
277
|
256.times do |byte|
|
|
@@ -275,11 +375,30 @@ end
|
|
|
275
375
|
task "golden:verify" do
|
|
276
376
|
FlexrVerification::VERIFICATION_SPECS.each do |spec|
|
|
277
377
|
golden = FlexrVerification.golden_path(spec)
|
|
278
|
-
abort "missing golden
|
|
378
|
+
abort "missing golden snapshot: #{golden}" unless File.file?(golden)
|
|
379
|
+
|
|
380
|
+
expected = FlexrVerification.normalized_source(File.read(golden, encoding: Encoding::UTF_8))
|
|
381
|
+
actual = FlexrVerification::Golden.normalize(FlexrVerification.generated_source(spec))
|
|
382
|
+
next if expected == actual
|
|
383
|
+
|
|
384
|
+
actual_path = File.join(FlexrVerification::ROOT, "tmp/golden", File.basename(golden))
|
|
385
|
+
FileUtils.mkdir_p(File.dirname(actual_path))
|
|
386
|
+
File.binwrite(actual_path, actual)
|
|
387
|
+
begin
|
|
388
|
+
diff, = Open3.capture3("git", "--no-pager", "diff", "--no-index", "--color=never", "--", golden, actual_path)
|
|
389
|
+
warn diff unless diff.empty?
|
|
390
|
+
rescue Errno::ENOENT => e
|
|
391
|
+
warn "git diff unavailable: #{e.message}"
|
|
392
|
+
end
|
|
393
|
+
abort "golden mismatch: #{spec}\ncompare with: git diff --no-index -- #{golden.dump} #{actual_path.dump}"
|
|
394
|
+
end
|
|
395
|
+
end
|
|
279
396
|
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
397
|
+
task "golden:update" do
|
|
398
|
+
FlexrVerification::VERIFICATION_SPECS.each do |spec|
|
|
399
|
+
golden = FlexrVerification.golden_path(spec)
|
|
400
|
+
FileUtils.mkdir_p(File.dirname(golden))
|
|
401
|
+
File.binwrite(golden, FlexrVerification::Golden.normalize(FlexrVerification.generated_source(spec)))
|
|
283
402
|
end
|
|
284
403
|
end
|
|
285
404
|
|
|
@@ -389,51 +508,52 @@ task "bench:regression" do
|
|
|
389
508
|
puts stdout
|
|
390
509
|
end
|
|
391
510
|
|
|
511
|
+
desc "Profile runtime and generated scanning phases with TracePoint"
|
|
512
|
+
task "bench:profile" do
|
|
513
|
+
ruby "benchmark/profile.rb", "--iterations", ENV.fetch("FLEXR_BENCHMARK_ITERATIONS", "1")
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
desc "Measure interpreter, YJIT, and ZJIT where supported by this Ruby"
|
|
517
|
+
task "bench:matrix" do
|
|
518
|
+
ruby "benchmark/matrix.rb", "--iterations", ENV.fetch("FLEXR_BENCHMARK_ITERATIONS", "3")
|
|
519
|
+
end
|
|
520
|
+
|
|
392
521
|
namespace :test do
|
|
522
|
+
desc "Compare Flexr pattern acceptance with Ruby's regexp engine"
|
|
393
523
|
task :differential do
|
|
394
524
|
patterns = [/[a-z]+/, /a(?:b|c)?/, /[0-9]{1,3}/, /[^\n]+/, /foo/,
|
|
395
525
|
/[[:alpha:]]+/, /[[:alnum:]]+/, /\p{L}+/, /\p{Nd}+/]
|
|
396
526
|
cases = Integer(ENV.fetch("FLEXR_DIFFERENTIAL_CASES", "1000000"), 10)
|
|
397
|
-
|
|
527
|
+
seed = Integer(ENV.fetch("FLEXR_SEED", "17"), 10)
|
|
528
|
+
random = Random.new(seed)
|
|
398
529
|
unicode_inputs = ["", "a", "あ", "é", "ß", "Ω", "١", " ", "aあ", "éΩ", [0x18db8].pack("U")].freeze
|
|
399
530
|
compiled = {}
|
|
400
531
|
host_patterns = {}
|
|
401
532
|
independent_cases = 0
|
|
402
533
|
version_mismatches = 0
|
|
403
|
-
cases.times do
|
|
534
|
+
cases.times do |index|
|
|
404
535
|
pattern = patterns[random.rand(patterns.length)]
|
|
405
536
|
input = if random.rand(3).zero?
|
|
406
537
|
random.rand(2).zero? ? unicode_inputs.sample(random: random) : FlexrVerification.random_unicode_string(random)
|
|
407
538
|
else
|
|
408
539
|
Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
|
|
409
540
|
end
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
false
|
|
415
|
-
end
|
|
416
|
-
if Flexr.reference_pattern?(pattern)
|
|
417
|
-
reference = Flexr::Unicode::ReferenceRegexp.compiled(
|
|
418
|
-
pattern, encoding: pattern.encoding, options: pattern.options, unicode: false
|
|
419
|
-
)
|
|
420
|
-
match = reference.match(input, 0)
|
|
421
|
-
vendored_expected = if match
|
|
422
|
-
match.begin(0).zero? && match[0].bytesize == input.bytesize
|
|
423
|
-
else
|
|
424
|
-
false
|
|
425
|
-
end
|
|
426
|
-
unless vendored_expected == host_expected
|
|
427
|
-
version_mismatches += 1
|
|
428
|
-
next
|
|
429
|
-
end
|
|
541
|
+
outcomes = FlexrVerification.differential_outcomes(pattern, input, compiled, host_patterns)
|
|
542
|
+
unless outcomes
|
|
543
|
+
version_mismatches += 1
|
|
544
|
+
next
|
|
430
545
|
end
|
|
431
546
|
independent_cases += 1
|
|
432
|
-
actual =
|
|
433
|
-
next if
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
547
|
+
expected, actual = outcomes
|
|
548
|
+
next if expected == actual
|
|
549
|
+
|
|
550
|
+
FlexrVerification::FailureCase.abort_with_record(
|
|
551
|
+
FlexrVerification.differential_mismatch(pattern, input, expected, actual),
|
|
552
|
+
root: FlexrVerification::ROOT, task: "test:differential", seed: seed, case: index + 1, input: input,
|
|
553
|
+
pattern: { source: FlexrVerification::FailureCase.encode_bytes(pattern.source), options: pattern.options,
|
|
554
|
+
encoding: pattern.encoding.name },
|
|
555
|
+
backends: { expected: "ruby_regexp", actual: "table" }, expected: expected, actual: actual
|
|
556
|
+
)
|
|
437
557
|
end
|
|
438
558
|
abort "differential did not exercise an independent host oracle" if independent_cases.zero?
|
|
439
559
|
|
|
@@ -442,49 +562,52 @@ namespace :test do
|
|
|
442
562
|
end
|
|
443
563
|
end
|
|
444
564
|
|
|
565
|
+
desc "Compare runtime and generated lexers on randomized inputs"
|
|
445
566
|
task :fuzz do
|
|
446
567
|
cases = Integer(ENV.fetch("FLEXR_FUZZ_CASES", "10000"), 10)
|
|
447
|
-
|
|
568
|
+
seed = Integer(ENV.fetch("FLEXR_SEED", "17"), 10)
|
|
569
|
+
random = Random.new(seed)
|
|
448
570
|
FlexrVerification::VERIFICATION_SPECS.each do |spec|
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
|
|
571
|
+
FlexrVerification.with_fuzz_lexers(spec) do |runtime_lexer, generated_lexer|
|
|
572
|
+
cases.times do |index|
|
|
573
|
+
input = case random.rand(4)
|
|
574
|
+
when 0
|
|
575
|
+
Array.new(random.rand(128)) { random.rand(0..127) }.pack("C*").force_encoding(Encoding::UTF_8)
|
|
576
|
+
when 1
|
|
577
|
+
FlexrVerification.random_unicode_string(random, max_codepoints: 32)
|
|
578
|
+
when 2
|
|
579
|
+
Array.new(random.rand(128)) { random.rand(0..255) }.pack("C*").force_encoding(Encoding::UTF_8)
|
|
580
|
+
else
|
|
581
|
+
(FlexrVerification.random_unicode_string(random, max_codepoints: 16) +
|
|
582
|
+
Array.new(random.rand(64)) { random.rand(32..126) }.pack("C*")).force_encoding(Encoding::UTF_8)
|
|
583
|
+
end
|
|
584
|
+
expected = FlexrVerification.lexer_outcome(runtime_lexer, input, **FlexrVerification::FUZZ_OPTIONS)
|
|
585
|
+
actual = FlexrVerification.lexer_outcome(generated_lexer, input, **FlexrVerification::FUZZ_OPTIONS)
|
|
586
|
+
next if expected == actual
|
|
587
|
+
|
|
588
|
+
FlexrVerification::FailureCase.abort_with_record(
|
|
589
|
+
FlexrVerification.fuzz_mismatch(spec, input, expected, actual),
|
|
590
|
+
root: FlexrVerification::ROOT, task: "fuzz", seed: seed, case: index + 1, input: input,
|
|
591
|
+
spec: { path: FlexrVerification.relative_spec(spec), sha256: Digest::SHA256.file(spec).hexdigest },
|
|
592
|
+
backends: { runtime: runtime_lexer.__flexr_config.backend, generated: generated_lexer.__flexr_config.backend },
|
|
593
|
+
options: FlexrVerification::FUZZ_OPTIONS, expected: expected, actual: actual
|
|
594
|
+
)
|
|
474
595
|
end
|
|
475
|
-
runtime_outcome = FlexrVerification.lexer_outcome(runtime_lexer, input, error_mode: :panic)
|
|
476
|
-
generated_outcome = FlexrVerification.lexer_outcome(generated_lexer, input, error_mode: :panic)
|
|
477
|
-
next if runtime_outcome == generated_outcome
|
|
478
|
-
|
|
479
|
-
abort "fuzz mode mismatch: #{spec} input=#{input.inspect} " \
|
|
480
|
-
"runtime=#{runtime_outcome.inspect} generated=#{generated_outcome.inspect}"
|
|
481
596
|
end
|
|
482
|
-
ensure
|
|
483
|
-
FileUtils.rm_f(generated_path) if generated_path
|
|
484
597
|
end
|
|
485
598
|
puts "fuzz: #{cases} inputs per example passed"
|
|
486
599
|
end
|
|
487
600
|
|
|
601
|
+
namespace :verification do
|
|
602
|
+
desc "Replay one saved mismatch (FLEXR_FAILURE_CASE=path/to/case.json)"
|
|
603
|
+
task :replay do
|
|
604
|
+
path = ENV.fetch("FLEXR_FAILURE_CASE")
|
|
605
|
+
FlexrVerification.replay_failure(FlexrVerification::FailureCase.read(path))
|
|
606
|
+
rescue ArgumentError, KeyError, JSON::ParserError, SystemCallError, EncodingError, RegexpError, TypeError => e
|
|
607
|
+
abort "Cannot replay verification failure: #{e.message}"
|
|
608
|
+
end
|
|
609
|
+
end
|
|
610
|
+
|
|
488
611
|
namespace :examples do
|
|
489
612
|
task :check do
|
|
490
613
|
FlexrVerification::EXAMPLES.each do |spec|
|
data/Steepfile
ADDED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema": 1,
|
|
3
3
|
"spec": "examples/json/lexer.flexr.rb",
|
|
4
|
-
"source_sha256": "
|
|
5
|
-
"source_bytes":
|
|
4
|
+
"source_sha256": "74d57259c5f67f4f6bf64cb870d4165095864fb61c58c9b8bff2d2d88da8b186",
|
|
5
|
+
"source_bytes": 733,
|
|
6
6
|
"input_sha256": "f15489cc8fa286cd365fb167b3f99d3d216dc90c0de6279fe9f36c8d2215d7a7",
|
|
7
7
|
"input_bytes": 140000,
|
|
8
8
|
"tokens": 50000,
|