flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Testing lexers
|
|
2
|
+
|
|
3
|
+
The helpers consume the complete input and compare the returned tokens. They
|
|
4
|
+
work with runtime and generated lexer classes, and forward keyword options to
|
|
5
|
+
the lexer constructor. Install the test framework separately; requiring `flexr`
|
|
6
|
+
does not load either framework.
|
|
7
|
+
|
|
8
|
+
## RSpec
|
|
9
|
+
|
|
10
|
+
```ruby
|
|
11
|
+
require "flexr/rspec"
|
|
12
|
+
|
|
13
|
+
expect(Lexer).to tokenize("12 + 3").as([
|
|
14
|
+
[:INTEGER, 12], [:PLUS, "+"], [:INTEGER, 3]
|
|
15
|
+
])
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Use `.as` to compare token values exactly, including symbol/string distinctions.
|
|
19
|
+
Lexer errors propagate so that tests can use the usual `raise_error` matcher.
|
|
20
|
+
|
|
21
|
+
## Minitest
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
require "minitest/autorun"
|
|
25
|
+
require "flexr/minitest"
|
|
26
|
+
|
|
27
|
+
class LexerTest < Minitest::Test
|
|
28
|
+
include Flexr::Minitest::Assertions
|
|
29
|
+
|
|
30
|
+
def test_numbers
|
|
31
|
+
assert_tokenize Lexer, "12", [[:INTEGER, 12]], chunk_size: 1
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Token snapshots
|
|
37
|
+
|
|
38
|
+
Store snapshots in a directory owned by the test suite. Snapshot assertions
|
|
39
|
+
fail if the file is missing or differs. They write files only when the call
|
|
40
|
+
explicitly passes `update: true`; review the resulting diff before committing.
|
|
41
|
+
|
|
42
|
+
```ruby
|
|
43
|
+
expect(Lexer).to tokenize("12 + 3").matching_snapshot("spec/snapshots/sum.json")
|
|
44
|
+
assert_token_snapshot Lexer, "12 + 3", "test/snapshots/sum.json"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
To create or replace a snapshot, temporarily add `update: true` to
|
|
48
|
+
`matching_snapshot` or `assert_token_snapshot`, then remove it for normal tests.
|
|
49
|
+
Create the parent directory first.
|
|
50
|
+
|
|
51
|
+
Snapshots use deterministic JSON, preserving symbols and non-UTF-8 bytes with
|
|
52
|
+
tags. Hash entries are sorted by their serialized keys, so insertion order
|
|
53
|
+
does not affect the snapshot. Supported values are nil, booleans, finite
|
|
54
|
+
numbers, strings, symbols, arrays, hashes, `Flexr::Runtime::Token`, and
|
|
55
|
+
`Flexr::Runtime::Location`. Convert custom token values to these types before
|
|
56
|
+
snapshot testing. Snapshot strings compare content rather than Ruby encoding
|
|
57
|
+
tags, allowing ASCII strings from String and IO input to use the same file.
|
|
@@ -12,6 +12,14 @@ For `:struct`, `Token` has `type`, `value`, and `location` members. `emit(nil,
|
|
|
12
12
|
value)` is valid and produces a token whose type is `nil`. `skip` produces no
|
|
13
13
|
token.
|
|
14
14
|
|
|
15
|
+
In v2, `Token` and `Location` are immutable `Data` records. The `:struct`
|
|
16
|
+
option name is retained. Attribute readers and read-only indexing by member
|
|
17
|
+
name or index remain available; field writers and `[]=` are removed. Replace
|
|
18
|
+
values with `token.with(value: replacement)` or `location.with(...)`.
|
|
19
|
+
Immutability is shallow: custom token values can still be mutable.
|
|
20
|
+
Lazy columns use a frozen pair without mutating the record or retaining the
|
|
21
|
+
lexer; `to_h` returns the resolved columns.
|
|
22
|
+
|
|
15
23
|
## Locations
|
|
16
24
|
|
|
17
25
|
`Flexr::Runtime::Location` has these public members:
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# GraphQL subset lexer
|
|
2
|
+
|
|
3
|
+
Tokenizes names, numbers, single-line strings, comments, ignored commas, spread,
|
|
4
|
+
variables, directives, and punctuation. Keywords remain `NAME` tokens so the
|
|
5
|
+
parser can interpret them by context. Values remain source text.
|
|
6
|
+
Block strings, escape validation, and grammar validation are outside this example.
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
ruby -Ilib -e 'load "examples/graphql/lexer.flexr.rb"; p GraphqlExample::Lexer.new("query { user(id: 12) { name } }").tokens'
|
|
10
|
+
bundle exec ruby benchmark/run.rb --spec examples/graphql/lexer.flexr.rb --iterations 1 --json
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The benchmark uses a repeated built-in query and compares runtime with generated
|
|
14
|
+
mode. Pass `--input-file PATH` to measure a different corpus.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "flexr"
|
|
4
|
+
|
|
5
|
+
module GraphqlExample
|
|
6
|
+
class Lexer < Flexr::Lexer
|
|
7
|
+
emits :NAME, :STRING, :NUMBER, :SPREAD, :LBRACE, :RBRACE, :LPAREN,
|
|
8
|
+
:RPAREN, :LBRACKET, :RBRACKET, :COLON, :EQUAL, :BANG, :DOLLAR, :AT, :PIPE, :AMP
|
|
9
|
+
rule(/[ \t\r\n,]+/, skip: true)
|
|
10
|
+
rule(/#[^\n]*/, skip: true)
|
|
11
|
+
rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
|
|
12
|
+
rule(/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
|
|
13
|
+
rule(/"(?:\\.|[^"\\\r\n])*"/, emit: :STRING)
|
|
14
|
+
rule(/\.\.\./, emit: :SPREAD)
|
|
15
|
+
rule(/\{/, emit: :LBRACE)
|
|
16
|
+
rule(/\}/, emit: :RBRACE)
|
|
17
|
+
rule(/\(/, emit: :LPAREN)
|
|
18
|
+
rule(/\)/, emit: :RPAREN)
|
|
19
|
+
rule(/\[/, emit: :LBRACKET)
|
|
20
|
+
rule(/\]/, emit: :RBRACKET)
|
|
21
|
+
rule(/:/, emit: :COLON)
|
|
22
|
+
rule(/=/, emit: :EQUAL)
|
|
23
|
+
rule(/!/, emit: :BANG)
|
|
24
|
+
rule(/\$/, emit: :DOLLAR)
|
|
25
|
+
rule(/@/, emit: :AT)
|
|
26
|
+
rule(/\|/, emit: :PIPE)
|
|
27
|
+
rule(/&/, emit: :AMP)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -9,12 +9,12 @@ module JsonExample
|
|
|
9
9
|
:STRING, :NUMBER, :TRUE, :FALSE, :NULL
|
|
10
10
|
|
|
11
11
|
rule(/[ \t\r\n]+/, skip: true)
|
|
12
|
-
rule(/\{
|
|
13
|
-
rule(/\}
|
|
14
|
-
rule(/\[
|
|
15
|
-
rule(/\]
|
|
16
|
-
rule(
|
|
17
|
-
rule(
|
|
12
|
+
rule(/\{/, emit: :LBRACE)
|
|
13
|
+
rule(/\}/, emit: :RBRACE)
|
|
14
|
+
rule(/\[/, emit: :LBRACKET)
|
|
15
|
+
rule(/\]/, emit: :RBRACKET)
|
|
16
|
+
rule(/:/, emit: :COLON)
|
|
17
|
+
rule(/,/, emit: :COMMA)
|
|
18
18
|
rule(/true/) { emit :TRUE, true }
|
|
19
19
|
rule(/false/) { emit :FALSE, false }
|
|
20
20
|
rule(/null/) { emit :NULL, nil }
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# SQL subset lexer
|
|
2
|
+
|
|
3
|
+
Tokenizes a small set of case-insensitive SQL keywords, unquoted names, decimal
|
|
4
|
+
numbers, single-quoted strings (including doubled quotes), line comments, and
|
|
5
|
+
operators. Keywords are uppercased; other values remain source text. Longest
|
|
6
|
+
match keeps `select_name` as one name and `>=` as one operator.
|
|
7
|
+
Quoted identifiers, block comments, vendor-specific syntax, and SQL grammar
|
|
8
|
+
validation are outside this example. Never use this lexer to sanitize SQL.
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
ruby -Ilib -e 'load "examples/sql/lexer.flexr.rb"; p SqlExample::Lexer.new("select name from users where id >= 12;").tokens'
|
|
12
|
+
bundle exec ruby benchmark/run.rb --spec examples/sql/lexer.flexr.rb --iterations 1 --json
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The benchmark uses a repeated built-in query and compares runtime with generated
|
|
16
|
+
mode. Pass `--input-file PATH` to measure a different corpus.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "flexr"
|
|
4
|
+
|
|
5
|
+
module SqlExample
|
|
6
|
+
class Lexer < Flexr::Lexer
|
|
7
|
+
emits :KEYWORD, :NAME, :STRING, :NUMBER, :OPERATOR, :COMMA, :DOT, :SEMICOLON, :LPAREN, :RPAREN
|
|
8
|
+
rule(/[ \t\r\n]+/, skip: true)
|
|
9
|
+
rule(/--[^\n]*/, skip: true)
|
|
10
|
+
rule(/SELECT|FROM|WHERE|AND|OR|NOT|NULL|AS|JOIN|ON|ORDER|BY|GROUP|HAVING|LIMIT|INSERT|INTO|VALUES|UPDATE|SET|DELETE/i) do
|
|
11
|
+
emit :KEYWORD, text.upcase
|
|
12
|
+
end
|
|
13
|
+
rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
|
|
14
|
+
rule(/'(?:''|[^'])*'/, emit: :STRING)
|
|
15
|
+
rule(/[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
|
|
16
|
+
rule(%r{>=|<=|<>|!=|[=<>+*/-]}, emit: :OPERATOR)
|
|
17
|
+
rule(/,/, emit: :COMMA)
|
|
18
|
+
rule(/\./, emit: :DOT)
|
|
19
|
+
rule(/;/, emit: :SEMICOLON)
|
|
20
|
+
rule(/\(/, emit: :LPAREN)
|
|
21
|
+
rule(/\)/, emit: :RPAREN)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# TOML subset lexer
|
|
2
|
+
|
|
3
|
+
Tokenizes bare names, single-line quoted strings, decimal numbers, booleans,
|
|
4
|
+
tables, inline tables, arrays, and comments. Values remain source text.
|
|
5
|
+
This is a lexer example, not a TOML validator: multiline strings, date/time
|
|
6
|
+
values, non-decimal numbers, and escape decoding need additional rules or a parser.
|
|
7
|
+
|
|
8
|
+
```sh
|
|
9
|
+
ruby -Ilib -e 'load "examples/toml/lexer.flexr.rb"; p TomlExample::Lexer.new(%q(title = "flexr")).tokens'
|
|
10
|
+
bundle exec ruby benchmark/run.rb --spec examples/toml/lexer.flexr.rb --iterations 1 --json
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The benchmark uses a repeated built-in document fragment and compares runtime
|
|
14
|
+
with generated mode. Pass `--input-file PATH` to measure a different corpus.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "flexr"
|
|
4
|
+
|
|
5
|
+
module TomlExample
|
|
6
|
+
class Lexer < Flexr::Lexer
|
|
7
|
+
emits :NAME, :STRING, :NUMBER, :BOOLEAN, :EQUAL, :DOT, :COMMA,
|
|
8
|
+
:LBRACKET, :RBRACKET, :LBRACE, :RBRACE
|
|
9
|
+
rule(/[ \t\r\n]+/, skip: true)
|
|
10
|
+
rule(/#[^\n]*/, skip: true)
|
|
11
|
+
rule(/"(?:\\.|[^"\\\r\n])*"|'[^'\r\n]*'/, emit: :STRING)
|
|
12
|
+
rule(/true|false/, emit: :BOOLEAN)
|
|
13
|
+
rule(/[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
|
|
14
|
+
rule(/[A-Za-z_][A-Za-z_0-9-]*/, emit: :NAME)
|
|
15
|
+
rule(/=/, emit: :EQUAL)
|
|
16
|
+
rule(/\./, emit: :DOT)
|
|
17
|
+
rule(/,/, emit: :COMMA)
|
|
18
|
+
rule(/\[/, emit: :LBRACKET)
|
|
19
|
+
rule(/\]/, emit: :RBRACKET)
|
|
20
|
+
rule(/\{/, emit: :LBRACE)
|
|
21
|
+
rule(/\}/, emit: :RBRACE)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -3,13 +3,14 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Automaton
|
|
5
5
|
Region = Struct.new(:state, :bytes, :regexp, :utf8_regexp, keyword_init: true)
|
|
6
|
+
WholeToken = Struct.new(:regexp, :rule_index, keyword_init: true)
|
|
6
7
|
|
|
7
8
|
module Accel
|
|
8
9
|
module_function
|
|
9
10
|
|
|
10
11
|
def extract(dfa)
|
|
11
12
|
dfa.transitions.each_index.filter_map do |state|
|
|
12
|
-
bytes =
|
|
13
|
+
bytes = self_loop_set(dfa, state)
|
|
13
14
|
next if bytes.empty?
|
|
14
15
|
|
|
15
16
|
Region.new(
|
|
@@ -20,6 +21,73 @@ module Flexr
|
|
|
20
21
|
end
|
|
21
22
|
end
|
|
22
23
|
|
|
24
|
+
def whole_token_plans(dfa, rules)
|
|
25
|
+
return if rules.any?(&:trailing) || !dfa.accepts[dfa.start].empty?
|
|
26
|
+
return if dfa.accepts.any? { |acceptances| acceptances.any? { |item| item.bol_only || item.end_anchor } }
|
|
27
|
+
|
|
28
|
+
plans = Array.new(256)
|
|
29
|
+
memo = {}
|
|
30
|
+
token_edges(dfa, dfa.start).each do |destination, bytes|
|
|
31
|
+
next if bytes.any? { |byte| byte >= 128 }
|
|
32
|
+
expression, winners = token_expression(dfa, destination, memo, [])
|
|
33
|
+
next unless expression && winners.one?
|
|
34
|
+
|
|
35
|
+
source = "[#{bytes_to_source(bytes)}]#{expression}"
|
|
36
|
+
next if source.bytesize > 4096
|
|
37
|
+
|
|
38
|
+
regexp = ::Regexp.new(source).freeze
|
|
39
|
+
plan = WholeToken.new(regexp: regexp, rule_index: winners.first).freeze
|
|
40
|
+
bytes.each { |byte| plans[byte] = plan }
|
|
41
|
+
end
|
|
42
|
+
plans.freeze unless plans.compact.empty?
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def token_expression(dfa, state, memo, ancestors)
|
|
46
|
+
return memo[state] if memo.key?(state)
|
|
47
|
+
# shortcut: cap expansion at 32 states/4 KiB, revisit for measured DFA fallback bottlenecks.
|
|
48
|
+
return [nil, []] if ancestors.include?(state) || ancestors.length >= 32
|
|
49
|
+
|
|
50
|
+
edges = token_edges(dfa, state)
|
|
51
|
+
return memo[state] = [nil, []] if edges.values.any? { |bytes| bytes.any? { |byte| byte >= 128 } }
|
|
52
|
+
|
|
53
|
+
loops = edges.delete(state)
|
|
54
|
+
winner = dfa.accepts[state].first&.rule_index
|
|
55
|
+
winners = winner ? [winner] : []
|
|
56
|
+
alternatives = edges.map do |destination, bytes|
|
|
57
|
+
tail, accepted = token_expression(dfa, destination, memo, ancestors + [state])
|
|
58
|
+
return memo[state] = [nil, []] unless tail
|
|
59
|
+
|
|
60
|
+
winners |= accepted
|
|
61
|
+
return memo[state] = [nil, []] if winners.length > 1
|
|
62
|
+
|
|
63
|
+
"[#{bytes_to_source(bytes)}]#{tail}"
|
|
64
|
+
end
|
|
65
|
+
return memo[state] = [nil, []] if alternatives.empty? && winner.nil?
|
|
66
|
+
|
|
67
|
+
tail = alternatives.empty? ? "" : "(?:#{alternatives.join('|')})#{'?' unless winner.nil?}"
|
|
68
|
+
expression = "#{"[#{bytes_to_source(loops)}]*" if loops}#{tail}"
|
|
69
|
+
return memo[state] = [nil, []] if expression.bytesize > 4096
|
|
70
|
+
|
|
71
|
+
memo[state] = [expression, winners]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def token_edges(dfa, state)
|
|
75
|
+
(0..255).each_with_object({}) do |byte, edges|
|
|
76
|
+
destination = dfa.transition(state, byte)
|
|
77
|
+
(edges[destination] ||= []) << byte if destination
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def self_loop_set(dfa, state)
|
|
82
|
+
representatives = Array.new(dfa.class_count)
|
|
83
|
+
dfa.ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
|
|
84
|
+
self_loops = representatives.each_index.map do |class_id|
|
|
85
|
+
byte = representatives.fetch(class_id)
|
|
86
|
+
byte && dfa.transition(state, byte) == state
|
|
87
|
+
end
|
|
88
|
+
dfa.ec.each_index.select { |byte| self_loops.fetch(dfa.ec.fetch(byte)) }
|
|
89
|
+
end
|
|
90
|
+
|
|
23
91
|
def regexp_for(bytes, binary: true)
|
|
24
92
|
source = bytes_to_source(bytes)
|
|
25
93
|
options = binary ? ::Regexp::NOENCODING : 0
|
|
@@ -19,13 +19,7 @@ module Flexr
|
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
def self_loop_set(dfa, state)
|
|
22
|
-
|
|
23
|
-
dfa.ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
|
|
24
|
-
self_loops = representatives.each_index.map do |class_id|
|
|
25
|
-
byte = representatives.fetch(class_id)
|
|
26
|
-
byte && dfa.transition(state, byte) == state
|
|
27
|
-
end
|
|
28
|
-
dfa.ec.each_index.select { |byte| self_loops.fetch(dfa.ec.fetch(byte)) }
|
|
22
|
+
Accel.self_loop_set(dfa, state)
|
|
29
23
|
end
|
|
30
24
|
|
|
31
25
|
def dead_states(dfa)
|
|
@@ -71,6 +65,166 @@ module Flexr
|
|
|
71
65
|
representatives.values.sort_by { |byte| [byte.between?(32, 126) ? 0 : 1, byte] }
|
|
72
66
|
end
|
|
73
67
|
|
|
68
|
+
def shadowing_examples(dfa, rule_indices, include_empty: false, include_unshadowed: false)
|
|
69
|
+
remaining = (rule_indices & dfa.rule_ids).to_h { |index| [index, true] }
|
|
70
|
+
return {} if remaining.empty?
|
|
71
|
+
|
|
72
|
+
examples = {}
|
|
73
|
+
queue = [[dfa.start, nil, nil]]
|
|
74
|
+
# Visiting the start again permits a nonempty example when it accepts and loops.
|
|
75
|
+
visited = {}
|
|
76
|
+
bytes = joint_byte_representatives(dfa, dfa)
|
|
77
|
+
cursor = 0
|
|
78
|
+
while cursor < queue.length
|
|
79
|
+
state = queue.fetch(cursor).first
|
|
80
|
+
accepts = dfa.accepts.fetch(state)
|
|
81
|
+
winner = accepts.min_by(&:rule_index)&.rule_index
|
|
82
|
+
if (include_empty || cursor.positive?) && winner
|
|
83
|
+
shadowed = accepts.map(&:rule_index).uniq.select do |index|
|
|
84
|
+
(include_unshadowed || index > winner) && remaining.key?(index)
|
|
85
|
+
end
|
|
86
|
+
unless shadowed.empty?
|
|
87
|
+
input = example_input(queue, cursor)
|
|
88
|
+
shadowed.each do |index|
|
|
89
|
+
examples[index] = [input, winner]
|
|
90
|
+
remaining.delete(index)
|
|
91
|
+
end
|
|
92
|
+
return examples if remaining.empty?
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
bytes.each do |byte|
|
|
97
|
+
target = dfa.transition(state, byte)
|
|
98
|
+
next if target.nil? || visited[target]
|
|
99
|
+
|
|
100
|
+
visited[target] = true
|
|
101
|
+
queue << [target, cursor, byte]
|
|
102
|
+
end
|
|
103
|
+
cursor += 1
|
|
104
|
+
end
|
|
105
|
+
examples
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def example_input(paths, cursor)
|
|
109
|
+
bytes = []
|
|
110
|
+
while (parent = paths.fetch(cursor)[1])
|
|
111
|
+
bytes << paths.fetch(cursor)[2]
|
|
112
|
+
cursor = parent
|
|
113
|
+
end
|
|
114
|
+
bytes.reverse.pack("C*")
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def shortest_input(dfa)
|
|
118
|
+
queue = [[dfa.start, nil, nil]]
|
|
119
|
+
visited = { dfa.start => true }
|
|
120
|
+
bytes = joint_byte_representatives(dfa, dfa)
|
|
121
|
+
cursor = 0
|
|
122
|
+
while cursor < queue.length
|
|
123
|
+
state = queue.fetch(cursor).first
|
|
124
|
+
return example_input(queue, cursor) if accepting?(dfa, state)
|
|
125
|
+
|
|
126
|
+
bytes.each do |byte|
|
|
127
|
+
target = dfa.transition(state, byte)
|
|
128
|
+
next if target.nil? || visited[target]
|
|
129
|
+
|
|
130
|
+
visited[target] = true
|
|
131
|
+
queue << [target, cursor, byte]
|
|
132
|
+
end
|
|
133
|
+
cursor += 1
|
|
134
|
+
end
|
|
135
|
+
nil
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def trailing_example(dfa, rules, target, input, trailing_dfas, encoding:, unicode:, firstmatch:, same_length: false)
|
|
139
|
+
lengths = if firstmatch
|
|
140
|
+
rules.filter_map do |rule|
|
|
141
|
+
next if rule.index > target.index
|
|
142
|
+
|
|
143
|
+
length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
|
|
144
|
+
[rule.index, length] if length
|
|
145
|
+
end
|
|
146
|
+
else
|
|
147
|
+
dfa_match_lengths(dfa, input, encoding: encoding)
|
|
148
|
+
end
|
|
149
|
+
matches = {}
|
|
150
|
+
lengths.each do |index, length|
|
|
151
|
+
trailing = trailing_dfas[index]
|
|
152
|
+
lookahead = trailing ? dfa_match_lengths(trailing, input, offset: length).map(&:last).max : 0
|
|
153
|
+
next unless lookahead
|
|
154
|
+
|
|
155
|
+
total = length + lookahead
|
|
156
|
+
next if matches[index] && matches[index].last > total
|
|
157
|
+
|
|
158
|
+
matches[index] = [length, total]
|
|
159
|
+
end
|
|
160
|
+
return unless matches.key?(target.index)
|
|
161
|
+
|
|
162
|
+
winner, (length, total) = matches.max_by { |index, (_, size)| firstmatch ? -index : [size, -index] }
|
|
163
|
+
return if same_length && total != matches.fetch(target.index).last
|
|
164
|
+
|
|
165
|
+
[winner, length, total - length]
|
|
166
|
+
rescue EncodingError
|
|
167
|
+
nil
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def dfa_match_lengths(dfa, input, offset: 0, encoding: Encoding::BINARY)
|
|
171
|
+
state = dfa.start
|
|
172
|
+
cursor = offset
|
|
173
|
+
matches = []
|
|
174
|
+
loop do
|
|
175
|
+
if encoding != Encoding::UTF_8 || input.byteslice(0, cursor).valid_encoding?
|
|
176
|
+
dfa.accepts.fetch(state).each do |acceptance|
|
|
177
|
+
next if acceptance.end_anchor && cursor < input.bytesize && input.getbyte(cursor) != 0x0a
|
|
178
|
+
|
|
179
|
+
matches << [acceptance.rule_index, cursor - offset]
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
break if cursor == input.bytesize
|
|
183
|
+
|
|
184
|
+
state = dfa.transition(state, input.getbyte(cursor))
|
|
185
|
+
break unless state
|
|
186
|
+
|
|
187
|
+
cursor += 1
|
|
188
|
+
end
|
|
189
|
+
matches
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def firstmatch_example(rules, target, input, encoding:, unicode: false, same_length: false)
|
|
193
|
+
target_length = firstmatch_length(target, input, encoding: encoding, unicode: unicode)
|
|
194
|
+
return unless target_length
|
|
195
|
+
|
|
196
|
+
rules.sort_by(&:index).each do |rule|
|
|
197
|
+
break if rule.index >= target.index
|
|
198
|
+
|
|
199
|
+
length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
|
|
200
|
+
next unless length
|
|
201
|
+
|
|
202
|
+
return nil if same_length && length != target_length
|
|
203
|
+
|
|
204
|
+
return [rule.index, length]
|
|
205
|
+
end
|
|
206
|
+
[target.index, target_length]
|
|
207
|
+
rescue EncodingError
|
|
208
|
+
nil
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def firstmatch_length(rule, input, encoding:, unicode:)
|
|
212
|
+
matches = rule.patterns.each_with_index.filter_map do |pattern, index|
|
|
213
|
+
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
214
|
+
reference = Regexp::Matcher.reference?(regexp, encoding: encoding, unicode: unicode)
|
|
215
|
+
match = Regexp::Matcher.match(regexp, input, encoding: encoding, unicode: unicode, reference: reference)
|
|
216
|
+
next unless match&.begin(0)&.zero?
|
|
217
|
+
|
|
218
|
+
[match[0].bytesize, rule.pattern_conditions.fetch(index)]
|
|
219
|
+
end
|
|
220
|
+
length, condition = matches.max_by(&:first)
|
|
221
|
+
return unless length
|
|
222
|
+
return if encoding == Encoding::UTF_8 && !input.byteslice(0, length).valid_encoding?
|
|
223
|
+
return if condition.end_anchor && length < input.bytesize && input.getbyte(length) != 0x0a
|
|
224
|
+
|
|
225
|
+
length
|
|
226
|
+
end
|
|
227
|
+
|
|
74
228
|
def accepting?(dfa, state)
|
|
75
229
|
state && !dfa.accepts.fetch(state).empty?
|
|
76
230
|
end
|