flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
data/docs/reference/regexp.md
CHANGED
|
@@ -6,34 +6,48 @@ byte-oriented automaton. All matches start at the current lexer position.
|
|
|
6
6
|
| Ruby construct | Status | Notes and alternative |
|
|
7
7
|
|---|---|---|
|
|
8
8
|
| Literals, concatenation, alternation | Supported | |
|
|
9
|
-
|
|
|
10
|
-
| Character classes and ranges | Supported | POSIX classes
|
|
9
|
+
| Unnamed groups and `(?:...)` | Supported | Captures are treated as non-capturing and warn |
|
|
10
|
+
| Character classes and ranges | Supported | Includes trailing `-`, POSIX classes, and `&&` intersection |
|
|
11
11
|
| Unicode `\p{...}` / `\P{...}` | Supported | Uses the vendored UCD snapshot |
|
|
12
12
|
| `*`, `+`, `?` | Supported | Greedy forms only |
|
|
13
|
-
| `{n}`, `{n,m}` | Supported |
|
|
14
|
-
| `{n,}` | Unsupported | Split the rule or use states |
|
|
13
|
+
| `{n}`, `{n,m}`, `{n,}` | Supported | Explicit bounds may not exceed 1000 |
|
|
15
14
|
| `^` / `$` | Restricted | Only at outermost pattern boundaries; otherwise `FLEXR-E009` |
|
|
16
15
|
| `/i`, `/m`, `/x`, `/n` | Supported | Inline forms are supported where Ruby permits them |
|
|
17
16
|
| `\d`, `\w`, `\s` | Supported | Unicode-aware with `option :unicode` in UTF-8 |
|
|
18
|
-
|
|
|
19
|
-
|
|
|
20
|
-
| `\
|
|
21
|
-
|
|
|
22
|
-
|
|
|
17
|
+
| `\h`, `\H` | Supported | Hexadecimal digits and their complement |
|
|
18
|
+
| Octal escapes such as `\011` | Unsupported | Use a hexadecimal escape such as `\x09` |
|
|
19
|
+
| `\R` | Unsupported | Use the standalone line-break rule shown by `FLEXR-E014` for the specification's encoding |
|
|
20
|
+
| Lookahead / lookbehind | Unsupported | Use `followed_by:` for trailing positive lookahead; other forms need rules or states; `FLEXR-E014` |
|
|
21
|
+
| Backreferences | Unsupported | Enumerate finite repeated strings, or use states and action checks |
|
|
22
|
+
| `\b`, `\B` | Unsupported | Use the literal-boundary examples in `FLEXR-E014` where applicable; token edges need rules or state checks |
|
|
23
|
+
| `\A`, `\z`, `\Z` | Unsupported | The `^` / `$` examples in `FLEXR-E014` apply only to input without line feeds |
|
|
24
|
+
| `\G` | Unsupported | At the start of a rule, omit it: rules already match at the current lexer position |
|
|
25
|
+
| `\K` | Unsupported | Use the prefix/suffix state example in `FLEXR-E014` where applicable to preserve token text and location |
|
|
26
|
+
| Subexpression calls `\g<...>` | Unsupported | Inline non-recursive expressions; recursive calls need a parser or state stack |
|
|
27
|
+
| `\X` | Unsupported | Use the ASCII-only standalone token example in `FLEXR-E014` where applicable; Unicode graphemes need separate segmentation |
|
|
28
|
+
| Lazy or possessive quantifiers | Unsupported | For unescaped single-line quoted text, use `/"[^"\n]*"/`; other patterns need a separate rewrite |
|
|
29
|
+
| Atomic groups | Unsupported | Use the delimited-text example in `FLEXR-E014` where applicable; removing atomic groups can change matches |
|
|
30
|
+
| Named captures | Unsupported | Use `(?:...)` when only grouping is needed; capture-dependent patterns need a separate rewrite |
|
|
31
|
+
| Conditional groups | Unsupported | Use the finite alternatives shown by `FLEXR-E014` where applicable, or separate rules and state checks |
|
|
32
|
+
| Absence groups `(?~...)` | Unsupported | For a single excluded literal, use the negated character class shown by `FLEXR-E014`; larger expressions need a separate rewrite |
|
|
23
33
|
|
|
24
34
|
Malformed syntax produces `FLEXR-E001`. Unsupported constructs produce
|
|
25
35
|
`FLEXR-E014`; the diagnostic help normally names the supported replacement.
|
|
36
|
+
Several `FLEXR-E014` diagnostics include concrete
|
|
37
|
+
[rule rewrite examples](diagnostics.md#regexp-rewrite-help).
|
|
38
|
+
Unknown alphabetic escapes are rejected rather than silently treated as
|
|
39
|
+
literals; escaped regexp punctuation remains supported.
|
|
26
40
|
Empty matches are rejected with `FLEXR-E005` unless `option :allow_empty_match`
|
|
27
41
|
is explicitly enabled, in which case flexr advances to guarantee progress.
|
|
28
42
|
|
|
29
43
|
## Bounded repetition
|
|
30
44
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
45
|
+
Repetition remains a symbolic AST node until NFA construction, avoiding large
|
|
46
|
+
intermediate trees. In `{n,m}`, `m` must be at least `n`; explicit bounds may
|
|
47
|
+
not exceed 1000. `{n,}` uses an unbounded loop after the required copies.
|
|
34
48
|
|
|
35
49
|
## Captures
|
|
36
50
|
|
|
37
|
-
|
|
51
|
+
Unnamed capturing parentheses do not expose match groups to actions. They are treated
|
|
38
52
|
as grouping syntax and produce `FLEXR-W013`; use `(?:...)` and inspect `text`
|
|
39
53
|
in the action instead.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Ruby LSP integration
|
|
2
|
+
|
|
3
|
+
flexr includes an optional add-on for [Ruby LSP](https://shopify.github.io/ruby-lsp/add-ons.html).
|
|
4
|
+
It supports Ruby LSP `~> 0.26.11`; incompatible server versions skip activation.
|
|
5
|
+
The lexer runtime does not depend on Ruby LSP.
|
|
6
|
+
|
|
7
|
+
Keep flexr in the project's bundle and enable its linter in the editor's Ruby
|
|
8
|
+
LSP configuration. For VS Code, add `flexr` to the existing list:
|
|
9
|
+
|
|
10
|
+
```json
|
|
11
|
+
{
|
|
12
|
+
"rubyLsp.linters": ["flexr"]
|
|
13
|
+
}
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Keep any other linters you use in that array. Ruby LSP discovers the add-on
|
|
17
|
+
automatically. It displays flexr diagnostics for `.flexr.rb` files using the
|
|
18
|
+
current editor contents, including unsaved edits. Rule hovers show compiler
|
|
19
|
+
warnings, counterexamples, and overlap information. Diagnostics and hovers
|
|
20
|
+
use the static reader: they do not evaluate specifications or run actions.
|
|
21
|
+
Syntax errors are reported by Ruby LSP's Prism integration.
|
|
22
|
+
|
|
23
|
+
The add-on uses the same static specification subset as `flexr check`, so
|
|
24
|
+
dynamic Ruby DSL construction must be checked separately. Compilation is
|
|
25
|
+
cached for the most recent document contents. Timing-dependent `FLEXR-W016`
|
|
26
|
+
is omitted; other diagnostics include their help and notes. The add-on honors
|
|
27
|
+
the server's negotiated UTF-8, UTF-16, or UTF-32 position encoding.
|
|
28
|
+
|
|
29
|
+
To run the integration tests against the real Ruby LSP server:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
BUNDLE_GEMFILE=gemfiles/ruby_lsp.gemfile bundle install
|
|
33
|
+
BUNDLE_GEMFILE=gemfiles/ruby_lsp.gemfile bundle exec rspec -Ilib -Ispec test/ruby_lsp_addon_spec.rb
|
|
34
|
+
```
|
data/docs/reference/runtime.md
CHANGED
|
@@ -10,14 +10,22 @@ Lexer.new(input,
|
|
|
10
10
|
filename: nil,
|
|
11
11
|
error_mode: :raise,
|
|
12
12
|
max_token_size: 16 * 1024 * 1024,
|
|
13
|
+
max_lookahead_size: nil,
|
|
14
|
+
max_buffer_size: 64 * 1024 * 1024,
|
|
13
15
|
max_state_stack: 1024,
|
|
16
|
+
max_steps: nil,
|
|
17
|
+
cancellation: nil,
|
|
18
|
+
retain_input: true,
|
|
14
19
|
chunk_size: 64 * 1024)
|
|
15
20
|
```
|
|
16
21
|
|
|
17
22
|
`input` must be a `String` or an object responding to `read`. `filename` is
|
|
18
23
|
copied into locations and lexical errors. `error_mode` is `:raise`, `:token`,
|
|
19
|
-
or `:panic`; unknown values
|
|
20
|
-
|
|
24
|
+
or `:panic`; unknown values raise `ArgumentError` at initialization.
|
|
25
|
+
`max_lookahead_size` defaults to `max_token_size`. `max_steps: nil`
|
|
26
|
+
leaves the step count unbounded; `cancellation` may be a callable accepting
|
|
27
|
+
zero arguments or the lexer. Size limits are non-negative, and `chunk_size`
|
|
28
|
+
must be positive.
|
|
21
29
|
|
|
22
30
|
## Consumption
|
|
23
31
|
|
|
@@ -27,11 +35,13 @@ handled. Size limits are non-negative, and `chunk_size` must be positive.
|
|
|
27
35
|
| `tokens` | Array containing every token until EOF |
|
|
28
36
|
| `each_token` | Enumerator without a block; otherwise yields tokens and returns `self` |
|
|
29
37
|
| `racc_next_token` | `[type, value]`, or `[false, "$end"]` at EOF |
|
|
30
|
-
| `input` | The
|
|
38
|
+
| `input` | The currently retained input string |
|
|
31
39
|
| `buffer` | Internal buffered input object; use only for integrations that need it |
|
|
32
40
|
| `filename` | The configured filename |
|
|
33
41
|
| `error_mode` | The configured default input error mode |
|
|
34
42
|
| `max_token_size` | The configured token-size limit |
|
|
43
|
+
| `max_lookahead_size` | The configured trailing-context limit |
|
|
44
|
+
| `steps` | Work units consumed by scanning |
|
|
35
45
|
|
|
36
46
|
`each_token` yields two arguments instead of one when `token_kind :yield` is
|
|
37
47
|
selected. `tokens` always returns an array of token values.
|
|
@@ -46,4 +56,11 @@ class or another lexer instance.
|
|
|
46
56
|
|
|
47
57
|
IO input is read lazily in chunks. UTF-8 input is checked at codepoint
|
|
48
58
|
boundaries, while binary input is consumed byte by byte. Invalid UTF-8 becomes
|
|
49
|
-
an unmatched-byte error rather than being silently normalized.
|
|
59
|
+
an unmatched-byte error rather than being silently normalized. With
|
|
60
|
+
`retain_input: false`, bytes before the current token are released after each
|
|
61
|
+
match; `input` and `binary_input` then expose only the retained window.
|
|
62
|
+
|
|
63
|
+
`max_token_size` is checked before an action can shorten or materialize an
|
|
64
|
+
oversized match. `max_lookahead_size` separately bounds trailing context, and
|
|
65
|
+
`max_buffer_size` bounds bytes retained at one time. `max_steps` and
|
|
66
|
+
`cancellation` bound CPU work cooperatively.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Testing lexers
|
|
2
|
+
|
|
3
|
+
The helpers consume the complete input and compare the returned tokens. They
|
|
4
|
+
work with runtime and generated lexer classes, and forward keyword options to
|
|
5
|
+
the lexer constructor. Install the test framework separately; requiring `flexr`
|
|
6
|
+
does not load either framework.
|
|
7
|
+
|
|
8
|
+
## RSpec
|
|
9
|
+
|
|
10
|
+
```ruby
|
|
11
|
+
require "flexr/rspec"
|
|
12
|
+
|
|
13
|
+
expect(Lexer).to tokenize("12 + 3").as([
|
|
14
|
+
[:INTEGER, 12], [:PLUS, "+"], [:INTEGER, 3]
|
|
15
|
+
])
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Use `.as` to compare token values exactly, including symbol/string distinctions.
|
|
19
|
+
Lexer errors propagate so that tests can use the usual `raise_error` matcher.
|
|
20
|
+
|
|
21
|
+
## Minitest
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
require "minitest/autorun"
|
|
25
|
+
require "flexr/minitest"
|
|
26
|
+
|
|
27
|
+
class LexerTest < Minitest::Test
|
|
28
|
+
include Flexr::Minitest::Assertions
|
|
29
|
+
|
|
30
|
+
def test_numbers
|
|
31
|
+
assert_tokenize Lexer, "12", [[:INTEGER, 12]], chunk_size: 1
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Token snapshots
|
|
37
|
+
|
|
38
|
+
Store snapshots in a directory owned by the test suite. Snapshot assertions
|
|
39
|
+
fail if the file is missing or differs. They write files only when the call
|
|
40
|
+
explicitly passes `update: true`; review the resulting diff before committing.
|
|
41
|
+
|
|
42
|
+
```ruby
|
|
43
|
+
expect(Lexer).to tokenize("12 + 3").matching_snapshot("spec/snapshots/sum.json")
|
|
44
|
+
assert_token_snapshot Lexer, "12 + 3", "test/snapshots/sum.json"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
To create or replace a snapshot, temporarily add `update: true` to
|
|
48
|
+
`matching_snapshot` or `assert_token_snapshot`, then remove it for normal tests.
|
|
49
|
+
Create the parent directory first.
|
|
50
|
+
|
|
51
|
+
Snapshots use deterministic JSON, preserving symbols and non-UTF-8 bytes with
|
|
52
|
+
tags. Hash entries are sorted by their serialized keys, so insertion order
|
|
53
|
+
does not affect the snapshot. Supported values are nil, booleans, finite
|
|
54
|
+
numbers, strings, symbols, arrays, hashes, `Flexr::Runtime::Token`, and
|
|
55
|
+
`Flexr::Runtime::Location`. Convert custom token values to these types before
|
|
56
|
+
snapshot testing. Snapshot strings compare content rather than Ruby encoding
|
|
57
|
+
tags, allowing ASCII strings from String and IO input to use the same file.
|
|
@@ -12,6 +12,14 @@ For `:struct`, `Token` has `type`, `value`, and `location` members. `emit(nil,
|
|
|
12
12
|
value)` is valid and produces a token whose type is `nil`. `skip` produces no
|
|
13
13
|
token.
|
|
14
14
|
|
|
15
|
+
In v2, `Token` and `Location` are immutable `Data` records. The `:struct`
|
|
16
|
+
option name is retained. Attribute readers and read-only indexing by member
|
|
17
|
+
name or index remain available; field writers and `[]=` are removed. Replace
|
|
18
|
+
values with `token.with(value: replacement)` or `location.with(...)`.
|
|
19
|
+
Immutability is shallow: custom token values can still be mutable.
|
|
20
|
+
Lazy columns use a frozen pair without mutating the record or retaining the
|
|
21
|
+
lexer; `to_h` returns the resolved columns.
|
|
22
|
+
|
|
15
23
|
## Locations
|
|
16
24
|
|
|
17
25
|
`Flexr::Runtime::Location` has these public members:
|
|
@@ -24,10 +32,12 @@ token.
|
|
|
24
32
|
| `line_begin` / `line_end` | One-based line range |
|
|
25
33
|
| `column_begin` / `column_end` | One-based character columns for UTF-8, byte columns for binary input |
|
|
26
34
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
35
|
+
Line and column positions are tracked incrementally while input is consumed,
|
|
36
|
+
so creating locations does not rescan the input prefix. Column members are
|
|
37
|
+
filled lazily by default; `option :eager_columns` fills them while the location
|
|
38
|
+
is created. The end line and column identify the position after the token's
|
|
39
|
+
last byte, so a token ending at the start of the next line may have an end
|
|
40
|
+
column of 1.
|
|
31
41
|
|
|
32
42
|
`more` extends the location from the first match to the final match. Trailing
|
|
33
43
|
context is not part of the location because it is not consumed.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# GraphQL subset lexer
|
|
2
|
+
|
|
3
|
+
Tokenizes names, numbers, single-line strings, comments, ignored commas, spread,
|
|
4
|
+
variables, directives, and punctuation. Keywords remain `NAME` tokens so the
|
|
5
|
+
parser can interpret them by context. Values remain source text.
|
|
6
|
+
Block strings, escape validation, and grammar validation are outside this example.
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
ruby -Ilib -e 'load "examples/graphql/lexer.flexr.rb"; p GraphqlExample::Lexer.new("query { user(id: 12) { name } }").tokens'
|
|
10
|
+
bundle exec ruby benchmark/run.rb --spec examples/graphql/lexer.flexr.rb --iterations 1 --json
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The benchmark uses a repeated built-in query and compares runtime with generated
|
|
14
|
+
mode. Pass `--input-file PATH` to measure a different corpus.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "flexr"
|
|
4
|
+
|
|
5
|
+
module GraphqlExample
|
|
6
|
+
class Lexer < Flexr::Lexer
|
|
7
|
+
emits :NAME, :STRING, :NUMBER, :SPREAD, :LBRACE, :RBRACE, :LPAREN,
|
|
8
|
+
:RPAREN, :LBRACKET, :RBRACKET, :COLON, :EQUAL, :BANG, :DOLLAR, :AT, :PIPE, :AMP
|
|
9
|
+
rule(/[ \t\r\n,]+/, skip: true)
|
|
10
|
+
rule(/#[^\n]*/, skip: true)
|
|
11
|
+
rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
|
|
12
|
+
rule(/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
|
|
13
|
+
rule(/"(?:\\.|[^"\\\r\n])*"/, emit: :STRING)
|
|
14
|
+
rule(/\.\.\./, emit: :SPREAD)
|
|
15
|
+
rule(/\{/, emit: :LBRACE)
|
|
16
|
+
rule(/\}/, emit: :RBRACE)
|
|
17
|
+
rule(/\(/, emit: :LPAREN)
|
|
18
|
+
rule(/\)/, emit: :RPAREN)
|
|
19
|
+
rule(/\[/, emit: :LBRACKET)
|
|
20
|
+
rule(/\]/, emit: :RBRACKET)
|
|
21
|
+
rule(/:/, emit: :COLON)
|
|
22
|
+
rule(/=/, emit: :EQUAL)
|
|
23
|
+
rule(/!/, emit: :BANG)
|
|
24
|
+
rule(/\$/, emit: :DOLLAR)
|
|
25
|
+
rule(/@/, emit: :AT)
|
|
26
|
+
rule(/\|/, emit: :PIPE)
|
|
27
|
+
rule(/&/, emit: :AMP)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -9,12 +9,12 @@ module JsonExample
|
|
|
9
9
|
:STRING, :NUMBER, :TRUE, :FALSE, :NULL
|
|
10
10
|
|
|
11
11
|
rule(/[ \t\r\n]+/, skip: true)
|
|
12
|
-
rule(/\{
|
|
13
|
-
rule(/\}
|
|
14
|
-
rule(/\[
|
|
15
|
-
rule(/\]
|
|
16
|
-
rule(
|
|
17
|
-
rule(
|
|
12
|
+
rule(/\{/, emit: :LBRACE)
|
|
13
|
+
rule(/\}/, emit: :RBRACE)
|
|
14
|
+
rule(/\[/, emit: :LBRACKET)
|
|
15
|
+
rule(/\]/, emit: :RBRACKET)
|
|
16
|
+
rule(/:/, emit: :COLON)
|
|
17
|
+
rule(/,/, emit: :COMMA)
|
|
18
18
|
rule(/true/) { emit :TRUE, true }
|
|
19
19
|
rule(/false/) { emit :FALSE, false }
|
|
20
20
|
rule(/null/) { emit :NULL, nil }
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# SQL subset lexer
|
|
2
|
+
|
|
3
|
+
Tokenizes a small set of case-insensitive SQL keywords, unquoted names, decimal
|
|
4
|
+
numbers, single-quoted strings (including doubled quotes), line comments, and
|
|
5
|
+
operators. Keywords are uppercased; other values remain source text. Longest
|
|
6
|
+
match keeps `select_name` as one name and `>=` as one operator.
|
|
7
|
+
Quoted identifiers, block comments, vendor-specific syntax, and SQL grammar
|
|
8
|
+
validation are outside this example. Never use this lexer to sanitize SQL.
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
ruby -Ilib -e 'load "examples/sql/lexer.flexr.rb"; p SqlExample::Lexer.new("select name from users where id >= 12;").tokens'
|
|
12
|
+
bundle exec ruby benchmark/run.rb --spec examples/sql/lexer.flexr.rb --iterations 1 --json
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The benchmark uses a repeated built-in query and compares runtime with generated
|
|
16
|
+
mode. Pass `--input-file PATH` to measure a different corpus.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "flexr"
|
|
4
|
+
|
|
5
|
+
module SqlExample
|
|
6
|
+
class Lexer < Flexr::Lexer
|
|
7
|
+
emits :KEYWORD, :NAME, :STRING, :NUMBER, :OPERATOR, :COMMA, :DOT, :SEMICOLON, :LPAREN, :RPAREN
|
|
8
|
+
rule(/[ \t\r\n]+/, skip: true)
|
|
9
|
+
rule(/--[^\n]*/, skip: true)
|
|
10
|
+
rule(/SELECT|FROM|WHERE|AND|OR|NOT|NULL|AS|JOIN|ON|ORDER|BY|GROUP|HAVING|LIMIT|INSERT|INTO|VALUES|UPDATE|SET|DELETE/i) do
|
|
11
|
+
emit :KEYWORD, text.upcase
|
|
12
|
+
end
|
|
13
|
+
rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
|
|
14
|
+
rule(/'(?:''|[^'])*'/, emit: :STRING)
|
|
15
|
+
rule(/[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
|
|
16
|
+
rule(%r{>=|<=|<>|!=|[=<>+*/-]}, emit: :OPERATOR)
|
|
17
|
+
rule(/,/, emit: :COMMA)
|
|
18
|
+
rule(/\./, emit: :DOT)
|
|
19
|
+
rule(/;/, emit: :SEMICOLON)
|
|
20
|
+
rule(/\(/, emit: :LPAREN)
|
|
21
|
+
rule(/\)/, emit: :RPAREN)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# TOML subset lexer
|
|
2
|
+
|
|
3
|
+
Tokenizes bare names, single-line quoted strings, decimal numbers, booleans,
|
|
4
|
+
tables, inline tables, arrays, and comments. Values remain source text.
|
|
5
|
+
This is a lexer example, not a TOML validator: multiline strings, date/time
|
|
6
|
+
values, non-decimal numbers, and escape decoding need additional rules or a parser.
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
ruby -Ilib -e 'load "examples/toml/lexer.flexr.rb"; p TomlExample::Lexer.new(%q(title = "flexr")).tokens'
|
|
10
|
+
bundle exec ruby benchmark/run.rb --spec examples/toml/lexer.flexr.rb --iterations 1 --json
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The benchmark uses a repeated built-in document fragment and compares runtime
|
|
14
|
+
with generated mode. Pass `--input-file PATH` to measure a different corpus.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "flexr"
|
|
4
|
+
|
|
5
|
+
module TomlExample
|
|
6
|
+
class Lexer < Flexr::Lexer
|
|
7
|
+
emits :NAME, :STRING, :NUMBER, :BOOLEAN, :EQUAL, :DOT, :COMMA,
|
|
8
|
+
:LBRACKET, :RBRACKET, :LBRACE, :RBRACE
|
|
9
|
+
rule(/[ \t\r\n]+/, skip: true)
|
|
10
|
+
rule(/#[^\n]*/, skip: true)
|
|
11
|
+
rule(/"(?:\\.|[^"\\\r\n])*"|'[^'\r\n]*'/, emit: :STRING)
|
|
12
|
+
rule(/true|false/, emit: :BOOLEAN)
|
|
13
|
+
rule(/[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
|
|
14
|
+
rule(/[A-Za-z_][A-Za-z_0-9-]*/, emit: :NAME)
|
|
15
|
+
rule(/=/, emit: :EQUAL)
|
|
16
|
+
rule(/\./, emit: :DOT)
|
|
17
|
+
rule(/,/, emit: :COMMA)
|
|
18
|
+
rule(/\[/, emit: :LBRACKET)
|
|
19
|
+
rule(/\]/, emit: :RBRACKET)
|
|
20
|
+
rule(/\{/, emit: :LBRACE)
|
|
21
|
+
rule(/\}/, emit: :RBRACE)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module ActionResolver
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
def resolve(skip:, emit:, block:, default:)
|
|
8
|
+
return :skip if skip
|
|
9
|
+
return [:emit, emit.to_sym] if emit
|
|
10
|
+
return block if block
|
|
11
|
+
|
|
12
|
+
default
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tempfile"
|
|
4
|
+
|
|
5
|
+
module Flexr
|
|
6
|
+
module ArtifactWriter
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def default_generated_path(source_path)
|
|
10
|
+
return source_path.sub(/\.flexr\.rb\z/, ".rb") if source_path.end_with?(".flexr.rb")
|
|
11
|
+
return source_path.sub(/\.rb\z/, ".generated.rb") if source_path.end_with?(".rb")
|
|
12
|
+
|
|
13
|
+
"#{source_path}.generated.rb"
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def write!(path, content, source_path: nil)
|
|
17
|
+
ensure_distinct!(source_path, path) if source_path
|
|
18
|
+
return :unchanged if File.file?(path) && File.binread(path) == content.b
|
|
19
|
+
|
|
20
|
+
atomic_replace(path, content)
|
|
21
|
+
:written
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def ensure_distinct!(source_path, output_path)
|
|
25
|
+
same_path = File.expand_path(source_path) == File.expand_path(output_path)
|
|
26
|
+
same_file = File.exist?(source_path) && File.exist?(output_path) && File.identical?(source_path, output_path)
|
|
27
|
+
return unless same_path || same_file
|
|
28
|
+
|
|
29
|
+
raise ArgumentError, "refusing to overwrite input file: #{source_path}"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def atomic_replace(path, content)
|
|
33
|
+
expanded = File.expand_path(path)
|
|
34
|
+
directory = File.dirname(expanded)
|
|
35
|
+
basename = File.basename(expanded)
|
|
36
|
+
mode = File.exist?(expanded) ? File.stat(expanded).mode & 0o7777 : 0o666 & ~File.umask
|
|
37
|
+
temporary_path = nil
|
|
38
|
+
|
|
39
|
+
Tempfile.create([".#{basename}.", ".tmp"], directory, binmode: true) do |temporary|
|
|
40
|
+
temporary_path = temporary.path
|
|
41
|
+
temporary.write(content)
|
|
42
|
+
temporary.flush
|
|
43
|
+
temporary.fsync
|
|
44
|
+
temporary.chmod(mode)
|
|
45
|
+
temporary.close
|
|
46
|
+
File.rename(temporary_path, expanded)
|
|
47
|
+
temporary_path = nil
|
|
48
|
+
end
|
|
49
|
+
sync_directory(directory)
|
|
50
|
+
ensure
|
|
51
|
+
File.unlink(temporary_path) if temporary_path && File.exist?(temporary_path)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def sync_directory(directory)
|
|
55
|
+
File.open(directory, File::RDONLY, &:fsync)
|
|
56
|
+
rescue SystemCallError, IOError
|
|
57
|
+
# Some supported filesystems do not allow fsync on directories. The file
|
|
58
|
+
# itself has already been synced before the atomic rename.
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -2,23 +2,96 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Automaton
|
|
5
|
-
Region = Struct.new(:state, :bytes, :regexp, keyword_init: true)
|
|
5
|
+
Region = Struct.new(:state, :bytes, :regexp, :utf8_regexp, keyword_init: true)
|
|
6
|
+
WholeToken = Struct.new(:regexp, :rule_index, keyword_init: true)
|
|
6
7
|
|
|
7
8
|
module Accel
|
|
8
9
|
module_function
|
|
9
10
|
|
|
10
11
|
def extract(dfa)
|
|
11
12
|
dfa.transitions.each_index.filter_map do |state|
|
|
12
|
-
bytes =
|
|
13
|
+
bytes = self_loop_set(dfa, state)
|
|
13
14
|
next if bytes.empty?
|
|
14
15
|
|
|
15
|
-
Region.new(
|
|
16
|
+
Region.new(
|
|
17
|
+
state: state, bytes: bytes.freeze,
|
|
18
|
+
regexp: regexp_for(bytes, binary: true),
|
|
19
|
+
utf8_regexp: bytes.all? { |byte| byte < 128 } ? regexp_for(bytes, binary: false) : nil
|
|
20
|
+
)
|
|
16
21
|
end
|
|
17
22
|
end
|
|
18
23
|
|
|
19
|
-
def
|
|
24
|
+
def whole_token_plans(dfa, rules)
|
|
25
|
+
return if rules.any?(&:trailing) || !dfa.accepts[dfa.start].empty?
|
|
26
|
+
return if dfa.accepts.any? { |acceptances| acceptances.any? { |item| item.bol_only || item.end_anchor } }
|
|
27
|
+
|
|
28
|
+
plans = Array.new(256)
|
|
29
|
+
memo = {}
|
|
30
|
+
token_edges(dfa, dfa.start).each do |destination, bytes|
|
|
31
|
+
next if bytes.any? { |byte| byte >= 128 }
|
|
32
|
+
expression, winners = token_expression(dfa, destination, memo, [])
|
|
33
|
+
next unless expression && winners.one?
|
|
34
|
+
|
|
35
|
+
source = "[#{bytes_to_source(bytes)}]#{expression}"
|
|
36
|
+
next if source.bytesize > 4096
|
|
37
|
+
|
|
38
|
+
regexp = ::Regexp.new(source).freeze
|
|
39
|
+
plan = WholeToken.new(regexp: regexp, rule_index: winners.first).freeze
|
|
40
|
+
bytes.each { |byte| plans[byte] = plan }
|
|
41
|
+
end
|
|
42
|
+
plans.freeze unless plans.compact.empty?
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def token_expression(dfa, state, memo, ancestors)
|
|
46
|
+
return memo[state] if memo.key?(state)
|
|
47
|
+
# shortcut: cap expansion at 32 states/4 KiB, revisit for measured DFA fallback bottlenecks.
|
|
48
|
+
return [nil, []] if ancestors.include?(state) || ancestors.length >= 32
|
|
49
|
+
|
|
50
|
+
edges = token_edges(dfa, state)
|
|
51
|
+
return memo[state] = [nil, []] if edges.values.any? { |bytes| bytes.any? { |byte| byte >= 128 } }
|
|
52
|
+
|
|
53
|
+
loops = edges.delete(state)
|
|
54
|
+
winner = dfa.accepts[state].first&.rule_index
|
|
55
|
+
winners = winner ? [winner] : []
|
|
56
|
+
alternatives = edges.map do |destination, bytes|
|
|
57
|
+
tail, accepted = token_expression(dfa, destination, memo, ancestors + [state])
|
|
58
|
+
return memo[state] = [nil, []] unless tail
|
|
59
|
+
|
|
60
|
+
winners |= accepted
|
|
61
|
+
return memo[state] = [nil, []] if winners.length > 1
|
|
62
|
+
|
|
63
|
+
"[#{bytes_to_source(bytes)}]#{tail}"
|
|
64
|
+
end
|
|
65
|
+
return memo[state] = [nil, []] if alternatives.empty? && winner.nil?
|
|
66
|
+
|
|
67
|
+
tail = alternatives.empty? ? "" : "(?:#{alternatives.join('|')})#{'?' unless winner.nil?}"
|
|
68
|
+
expression = "#{"[#{bytes_to_source(loops)}]*" if loops}#{tail}"
|
|
69
|
+
return memo[state] = [nil, []] if expression.bytesize > 4096
|
|
70
|
+
|
|
71
|
+
memo[state] = [expression, winners]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def token_edges(dfa, state)
|
|
75
|
+
(0..255).each_with_object({}) do |byte, edges|
|
|
76
|
+
destination = dfa.transition(state, byte)
|
|
77
|
+
(edges[destination] ||= []) << byte if destination
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def self_loop_set(dfa, state)
|
|
82
|
+
representatives = Array.new(dfa.class_count)
|
|
83
|
+
dfa.ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
|
|
84
|
+
self_loops = representatives.each_index.map do |class_id|
|
|
85
|
+
byte = representatives.fetch(class_id)
|
|
86
|
+
byte && dfa.transition(state, byte) == state
|
|
87
|
+
end
|
|
88
|
+
dfa.ec.each_index.select { |byte| self_loops.fetch(dfa.ec.fetch(byte)) }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def regexp_for(bytes, binary: true)
|
|
20
92
|
source = bytes_to_source(bytes)
|
|
21
|
-
|
|
93
|
+
options = binary ? ::Regexp::NOENCODING : 0
|
|
94
|
+
::Regexp.new("(?:[#{source}])+", options)
|
|
22
95
|
end
|
|
23
96
|
|
|
24
97
|
def bytes_to_source(bytes)
|