flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -0,0 +1,57 @@
1
+ # Testing lexers
2
+
3
+ The helpers consume the complete input and compare the returned tokens. They
4
+ work with runtime and generated lexer classes, and forward keyword options to
5
+ the lexer constructor. Install the test framework separately; requiring `flexr`
6
+ does not load either framework.
7
+
8
+ ## RSpec
9
+
10
+ ```ruby
11
+ require "flexr/rspec"
12
+
13
+ expect(Lexer).to tokenize("12 + 3").as([
14
+ [:INTEGER, 12], [:PLUS, "+"], [:INTEGER, 3]
15
+ ])
16
+ ```
17
+
18
+ Use `.as` to compare token values exactly, including symbol/string distinctions.
19
+ Lexer errors propagate so that tests can use the usual `raise_error` matcher.
20
+
21
+ ## Minitest
22
+
23
+ ```ruby
24
+ require "minitest/autorun"
25
+ require "flexr/minitest"
26
+
27
+ class LexerTest < Minitest::Test
28
+ include Flexr::Minitest::Assertions
29
+
30
+ def test_numbers
31
+ assert_tokenize Lexer, "12", [[:INTEGER, 12]], chunk_size: 1
32
+ end
33
+ end
34
+ ```
35
+
36
+ ## Token snapshots
37
+
38
+ Store snapshots in a directory owned by the test suite. Snapshot assertions
39
+ fail if the file is missing or differs. They write files only when the call
40
+ explicitly passes `update: true`; review the resulting diff before committing.
41
+
42
+ ```ruby
43
+ expect(Lexer).to tokenize("12 + 3").matching_snapshot("spec/snapshots/sum.json")
44
+ assert_token_snapshot Lexer, "12 + 3", "test/snapshots/sum.json"
45
+ ```
46
+
47
+ To create or replace a snapshot, temporarily add `update: true` to
48
+ `matching_snapshot` or `assert_token_snapshot`, then remove it for normal tests.
49
+ Create the parent directory first.
50
+
51
+ Snapshots use deterministic JSON, preserving symbols and non-UTF-8 bytes with
52
+ tags. Hash entries are sorted by their serialized keys, so insertion order
53
+ does not affect the snapshot. Supported values are nil, booleans, finite
54
+ numbers, strings, symbols, arrays, hashes, `Flexr::Runtime::Token`, and
55
+ `Flexr::Runtime::Location`. Convert custom token values to these types before
56
+ snapshot testing. Snapshot strings compare content rather than Ruby encoding
57
+ tags, allowing ASCII strings from String and IO input to use the same file.
@@ -12,6 +12,14 @@ For `:struct`, `Token` has `type`, `value`, and `location` members. `emit(nil,
12
12
  value)` is valid and produces a token whose type is `nil`. `skip` produces no
13
13
  token.
14
14
 
15
+ In v2, `Token` and `Location` are immutable `Data` records. The `:struct`
16
+ option name is retained. Attribute readers and read-only indexing by member
17
+ name or index remain available; field writers and `[]=` are removed. Replace
18
+ values with `token.with(value: replacement)` or `location.with(...)`.
19
+ Immutability is shallow: custom token values can still be mutable.
20
+ Lazy columns use a frozen pair without mutating the record or retaining the
21
+ lexer; `to_h` returns the resolved columns.
22
+
15
23
  ## Locations
16
24
 
17
25
  `Flexr::Runtime::Location` has these public members:
@@ -0,0 +1,14 @@
1
+ # GraphQL subset lexer
2
+
3
+ Tokenizes names, numbers, single-line strings, comments, ignored commas, spread,
4
+ variables, directives, and punctuation. Keywords remain `NAME` tokens so the
5
+ parser can interpret them by context. Values remain source text.
6
+ Block strings, escape validation, and grammar validation are outside this example.
7
+
8
+ ```sh
9
+ ruby -Ilib -e 'load "examples/graphql/lexer.flexr.rb"; p GraphqlExample::Lexer.new("query { user(id: 12) { name } }").tokens'
10
+ bundle exec ruby benchmark/run.rb --spec examples/graphql/lexer.flexr.rb --iterations 1 --json
11
+ ```
12
+
13
+ The benchmark uses a repeated built-in query and compares runtime with generated
14
+ mode. Pass `--input-file PATH` to measure a different corpus.
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "flexr"
4
+
5
+ module GraphqlExample
6
+ class Lexer < Flexr::Lexer
7
+ emits :NAME, :STRING, :NUMBER, :SPREAD, :LBRACE, :RBRACE, :LPAREN,
8
+ :RPAREN, :LBRACKET, :RBRACKET, :COLON, :EQUAL, :BANG, :DOLLAR, :AT, :PIPE, :AMP
9
+ rule(/[ \t\r\n,]+/, skip: true)
10
+ rule(/#[^\n]*/, skip: true)
11
+ rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
12
+ rule(/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
13
+ rule(/"(?:\\.|[^"\\\r\n])*"/, emit: :STRING)
14
+ rule(/\.\.\./, emit: :SPREAD)
15
+ rule(/\{/, emit: :LBRACE)
16
+ rule(/\}/, emit: :RBRACE)
17
+ rule(/\(/, emit: :LPAREN)
18
+ rule(/\)/, emit: :RPAREN)
19
+ rule(/\[/, emit: :LBRACKET)
20
+ rule(/\]/, emit: :RBRACKET)
21
+ rule(/:/, emit: :COLON)
22
+ rule(/=/, emit: :EQUAL)
23
+ rule(/!/, emit: :BANG)
24
+ rule(/\$/, emit: :DOLLAR)
25
+ rule(/@/, emit: :AT)
26
+ rule(/\|/, emit: :PIPE)
27
+ rule(/&/, emit: :AMP)
28
+ end
29
+ end
@@ -9,12 +9,12 @@ module JsonExample
9
9
  :STRING, :NUMBER, :TRUE, :FALSE, :NULL
10
10
 
11
11
  rule(/[ \t\r\n]+/, skip: true)
12
- rule(/\{/) { emit :LBRACE }
13
- rule(/\}/) { emit :RBRACE }
14
- rule(/\[/) { emit :LBRACKET }
15
- rule(/\]/) { emit :RBRACKET }
16
- rule(/:/) { emit :COLON }
17
- rule(/,/) { emit :COMMA }
12
+ rule(/\{/, emit: :LBRACE)
13
+ rule(/\}/, emit: :RBRACE)
14
+ rule(/\[/, emit: :LBRACKET)
15
+ rule(/\]/, emit: :RBRACKET)
16
+ rule(/:/, emit: :COLON)
17
+ rule(/,/, emit: :COMMA)
18
18
  rule(/true/) { emit :TRUE, true }
19
19
  rule(/false/) { emit :FALSE, false }
20
20
  rule(/null/) { emit :NULL, nil }
@@ -0,0 +1,16 @@
1
+ # SQL subset lexer
2
+
3
+ Tokenizes a small set of case-insensitive SQL keywords, unquoted names, decimal
4
+ numbers, single-quoted strings (including doubled quotes), line comments, and
5
+ operators. Keywords are uppercased; other values remain source text. Longest
6
+ match keeps `select_name` as one name and `>=` as one operator.
7
+ Quoted identifiers, block comments, vendor-specific syntax, and SQL grammar
8
+ validation are outside this example. Never use this lexer to sanitize SQL.
9
+
10
+ ```sh
11
+ ruby -Ilib -e 'load "examples/sql/lexer.flexr.rb"; p SqlExample::Lexer.new("select name from users where id >= 12;").tokens'
12
+ bundle exec ruby benchmark/run.rb --spec examples/sql/lexer.flexr.rb --iterations 1 --json
13
+ ```
14
+
15
+ The benchmark uses a repeated built-in query and compares runtime with generated
16
+ mode. Pass `--input-file PATH` to measure a different corpus.
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "flexr"
4
+
5
+ module SqlExample
6
+ class Lexer < Flexr::Lexer
7
+ emits :KEYWORD, :NAME, :STRING, :NUMBER, :OPERATOR, :COMMA, :DOT, :SEMICOLON, :LPAREN, :RPAREN
8
+ rule(/[ \t\r\n]+/, skip: true)
9
+ rule(/--[^\n]*/, skip: true)
10
+ rule(/SELECT|FROM|WHERE|AND|OR|NOT|NULL|AS|JOIN|ON|ORDER|BY|GROUP|HAVING|LIMIT|INSERT|INTO|VALUES|UPDATE|SET|DELETE/i) do
11
+ emit :KEYWORD, text.upcase
12
+ end
13
+ rule(/[A-Za-z_][A-Za-z_0-9]*/, emit: :NAME)
14
+ rule(/'(?:''|[^'])*'/, emit: :STRING)
15
+ rule(/[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
16
+ rule(%r{>=|<=|<>|!=|[=<>+*/-]}, emit: :OPERATOR)
17
+ rule(/,/, emit: :COMMA)
18
+ rule(/\./, emit: :DOT)
19
+ rule(/;/, emit: :SEMICOLON)
20
+ rule(/\(/, emit: :LPAREN)
21
+ rule(/\)/, emit: :RPAREN)
22
+ end
23
+ end
@@ -0,0 +1,14 @@
1
+ # TOML subset lexer
2
+
3
+ Tokenizes bare names, single-line quoted strings, decimal numbers, booleans,
4
+ tables, inline tables, arrays, and comments. Values remain source text.
5
+ This is a lexer example, not a TOML validator: multiline strings, date/time
6
+ values, non-decimal numbers, and escape decoding need additional rules or a parser.
7
+
8
+ ```sh
9
+ ruby -Ilib -e 'load "examples/toml/lexer.flexr.rb"; p TomlExample::Lexer.new(%q(title = "flexr")).tokens'
10
+ bundle exec ruby benchmark/run.rb --spec examples/toml/lexer.flexr.rb --iterations 1 --json
11
+ ```
12
+
13
+ The benchmark uses a repeated built-in document fragment and compares runtime
14
+ with generated mode. Pass `--input-file PATH` to measure a different corpus.
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "flexr"
4
+
5
+ module TomlExample
6
+ class Lexer < Flexr::Lexer
7
+ emits :NAME, :STRING, :NUMBER, :BOOLEAN, :EQUAL, :DOT, :COMMA,
8
+ :LBRACKET, :RBRACKET, :LBRACE, :RBRACE
9
+ rule(/[ \t\r\n]+/, skip: true)
10
+ rule(/#[^\n]*/, skip: true)
11
+ rule(/"(?:\\.|[^"\\\r\n])*"|'[^'\r\n]*'/, emit: :STRING)
12
+ rule(/true|false/, emit: :BOOLEAN)
13
+ rule(/[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/, emit: :NUMBER)
14
+ rule(/[A-Za-z_][A-Za-z_0-9-]*/, emit: :NAME)
15
+ rule(/=/, emit: :EQUAL)
16
+ rule(/\./, emit: :DOT)
17
+ rule(/,/, emit: :COMMA)
18
+ rule(/\[/, emit: :LBRACKET)
19
+ rule(/\]/, emit: :RBRACKET)
20
+ rule(/\{/, emit: :LBRACE)
21
+ rule(/\}/, emit: :RBRACE)
22
+ end
23
+ end
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ eval_gemfile "../Gemfile"
4
+ gem "ruby-lsp", "~> 0.26.11"
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gemspec path: ".."
6
+ gem "rbs", "~> 4.0"
7
+ gem "steep", "~> 2.1"
@@ -3,13 +3,14 @@
3
3
  module Flexr
4
4
  module Automaton
5
5
  Region = Struct.new(:state, :bytes, :regexp, :utf8_regexp, keyword_init: true)
6
+ WholeToken = Struct.new(:regexp, :rule_index, keyword_init: true)
6
7
 
7
8
  module Accel
8
9
  module_function
9
10
 
10
11
  def extract(dfa)
11
12
  dfa.transitions.each_index.filter_map do |state|
12
- bytes = Analysis.self_loop_set(dfa, state)
13
+ bytes = self_loop_set(dfa, state)
13
14
  next if bytes.empty?
14
15
 
15
16
  Region.new(
@@ -20,6 +21,73 @@ module Flexr
20
21
  end
21
22
  end
22
23
 
24
+ def whole_token_plans(dfa, rules)
25
+ return if rules.any?(&:trailing) || !dfa.accepts[dfa.start].empty?
26
+ return if dfa.accepts.any? { |acceptances| acceptances.any? { |item| item.bol_only || item.end_anchor } }
27
+
28
+ plans = Array.new(256)
29
+ memo = {}
30
+ token_edges(dfa, dfa.start).each do |destination, bytes|
31
+ next if bytes.any? { |byte| byte >= 128 }
32
+ expression, winners = token_expression(dfa, destination, memo, [])
33
+ next unless expression && winners.one?
34
+
35
+ source = "[#{bytes_to_source(bytes)}]#{expression}"
36
+ next if source.bytesize > 4096
37
+
38
+ regexp = ::Regexp.new(source).freeze
39
+ plan = WholeToken.new(regexp: regexp, rule_index: winners.first).freeze
40
+ bytes.each { |byte| plans[byte] = plan }
41
+ end
42
+ plans.freeze unless plans.compact.empty?
43
+ end
44
+
45
+ def token_expression(dfa, state, memo, ancestors)
46
+ return memo[state] if memo.key?(state)
47
+ # shortcut: cap expansion at 32 states/4 KiB, revisit for measured DFA fallback bottlenecks.
48
+ return [nil, []] if ancestors.include?(state) || ancestors.length >= 32
49
+
50
+ edges = token_edges(dfa, state)
51
+ return memo[state] = [nil, []] if edges.values.any? { |bytes| bytes.any? { |byte| byte >= 128 } }
52
+
53
+ loops = edges.delete(state)
54
+ winner = dfa.accepts[state].first&.rule_index
55
+ winners = winner ? [winner] : []
56
+ alternatives = edges.map do |destination, bytes|
57
+ tail, accepted = token_expression(dfa, destination, memo, ancestors + [state])
58
+ return memo[state] = [nil, []] unless tail
59
+
60
+ winners |= accepted
61
+ return memo[state] = [nil, []] if winners.length > 1
62
+
63
+ "[#{bytes_to_source(bytes)}]#{tail}"
64
+ end
65
+ return memo[state] = [nil, []] if alternatives.empty? && winner.nil?
66
+
67
+ tail = alternatives.empty? ? "" : "(?:#{alternatives.join('|')})#{'?' unless winner.nil?}"
68
+ expression = "#{"[#{bytes_to_source(loops)}]*" if loops}#{tail}"
69
+ return memo[state] = [nil, []] if expression.bytesize > 4096
70
+
71
+ memo[state] = [expression, winners]
72
+ end
73
+
74
+ def token_edges(dfa, state)
75
+ (0..255).each_with_object({}) do |byte, edges|
76
+ destination = dfa.transition(state, byte)
77
+ (edges[destination] ||= []) << byte if destination
78
+ end
79
+ end
80
+
81
+ def self_loop_set(dfa, state)
82
+ representatives = Array.new(dfa.class_count)
83
+ dfa.ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
84
+ self_loops = representatives.each_index.map do |class_id|
85
+ byte = representatives.fetch(class_id)
86
+ byte && dfa.transition(state, byte) == state
87
+ end
88
+ dfa.ec.each_index.select { |byte| self_loops.fetch(dfa.ec.fetch(byte)) }
89
+ end
90
+
23
91
  def regexp_for(bytes, binary: true)
24
92
  source = bytes_to_source(bytes)
25
93
  options = binary ? ::Regexp::NOENCODING : 0
@@ -19,13 +19,7 @@ module Flexr
19
19
  end
20
20
 
21
21
  def self_loop_set(dfa, state)
22
- representatives = Array.new(dfa.class_count)
23
- dfa.ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
24
- self_loops = representatives.each_index.map do |class_id|
25
- byte = representatives.fetch(class_id)
26
- byte && dfa.transition(state, byte) == state
27
- end
28
- dfa.ec.each_index.select { |byte| self_loops.fetch(dfa.ec.fetch(byte)) }
22
+ Accel.self_loop_set(dfa, state)
29
23
  end
30
24
 
31
25
  def dead_states(dfa)
@@ -71,6 +65,166 @@ module Flexr
71
65
  representatives.values.sort_by { |byte| [byte.between?(32, 126) ? 0 : 1, byte] }
72
66
  end
73
67
 
68
+ def shadowing_examples(dfa, rule_indices, include_empty: false, include_unshadowed: false)
69
+ remaining = (rule_indices & dfa.rule_ids).to_h { |index| [index, true] }
70
+ return {} if remaining.empty?
71
+
72
+ examples = {}
73
+ queue = [[dfa.start, nil, nil]]
74
+ # Visiting the start again permits a nonempty example when it accepts and loops.
75
+ visited = {}
76
+ bytes = joint_byte_representatives(dfa, dfa)
77
+ cursor = 0
78
+ while cursor < queue.length
79
+ state = queue.fetch(cursor).first
80
+ accepts = dfa.accepts.fetch(state)
81
+ winner = accepts.min_by(&:rule_index)&.rule_index
82
+ if (include_empty || cursor.positive?) && winner
83
+ shadowed = accepts.map(&:rule_index).uniq.select do |index|
84
+ (include_unshadowed || index > winner) && remaining.key?(index)
85
+ end
86
+ unless shadowed.empty?
87
+ input = example_input(queue, cursor)
88
+ shadowed.each do |index|
89
+ examples[index] = [input, winner]
90
+ remaining.delete(index)
91
+ end
92
+ return examples if remaining.empty?
93
+ end
94
+ end
95
+
96
+ bytes.each do |byte|
97
+ target = dfa.transition(state, byte)
98
+ next if target.nil? || visited[target]
99
+
100
+ visited[target] = true
101
+ queue << [target, cursor, byte]
102
+ end
103
+ cursor += 1
104
+ end
105
+ examples
106
+ end
107
+
108
+ def example_input(paths, cursor)
109
+ bytes = []
110
+ while (parent = paths.fetch(cursor)[1])
111
+ bytes << paths.fetch(cursor)[2]
112
+ cursor = parent
113
+ end
114
+ bytes.reverse.pack("C*")
115
+ end
116
+
117
+ def shortest_input(dfa)
118
+ queue = [[dfa.start, nil, nil]]
119
+ visited = { dfa.start => true }
120
+ bytes = joint_byte_representatives(dfa, dfa)
121
+ cursor = 0
122
+ while cursor < queue.length
123
+ state = queue.fetch(cursor).first
124
+ return example_input(queue, cursor) if accepting?(dfa, state)
125
+
126
+ bytes.each do |byte|
127
+ target = dfa.transition(state, byte)
128
+ next if target.nil? || visited[target]
129
+
130
+ visited[target] = true
131
+ queue << [target, cursor, byte]
132
+ end
133
+ cursor += 1
134
+ end
135
+ nil
136
+ end
137
+
138
+ def trailing_example(dfa, rules, target, input, trailing_dfas, encoding:, unicode:, firstmatch:, same_length: false)
139
+ lengths = if firstmatch
140
+ rules.filter_map do |rule|
141
+ next if rule.index > target.index
142
+
143
+ length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
144
+ [rule.index, length] if length
145
+ end
146
+ else
147
+ dfa_match_lengths(dfa, input, encoding: encoding)
148
+ end
149
+ matches = {}
150
+ lengths.each do |index, length|
151
+ trailing = trailing_dfas[index]
152
+ lookahead = trailing ? dfa_match_lengths(trailing, input, offset: length).map(&:last).max : 0
153
+ next unless lookahead
154
+
155
+ total = length + lookahead
156
+ next if matches[index] && matches[index].last > total
157
+
158
+ matches[index] = [length, total]
159
+ end
160
+ return unless matches.key?(target.index)
161
+
162
+ winner, (length, total) = matches.max_by { |index, (_, size)| firstmatch ? -index : [size, -index] }
163
+ return if same_length && total != matches.fetch(target.index).last
164
+
165
+ [winner, length, total - length]
166
+ rescue EncodingError
167
+ nil
168
+ end
169
+
170
+ def dfa_match_lengths(dfa, input, offset: 0, encoding: Encoding::BINARY)
171
+ state = dfa.start
172
+ cursor = offset
173
+ matches = []
174
+ loop do
175
+ if encoding != Encoding::UTF_8 || input.byteslice(0, cursor).valid_encoding?
176
+ dfa.accepts.fetch(state).each do |acceptance|
177
+ next if acceptance.end_anchor && cursor < input.bytesize && input.getbyte(cursor) != 0x0a
178
+
179
+ matches << [acceptance.rule_index, cursor - offset]
180
+ end
181
+ end
182
+ break if cursor == input.bytesize
183
+
184
+ state = dfa.transition(state, input.getbyte(cursor))
185
+ break unless state
186
+
187
+ cursor += 1
188
+ end
189
+ matches
190
+ end
191
+
192
+ def firstmatch_example(rules, target, input, encoding:, unicode: false, same_length: false)
193
+ target_length = firstmatch_length(target, input, encoding: encoding, unicode: unicode)
194
+ return unless target_length
195
+
196
+ rules.sort_by(&:index).each do |rule|
197
+ break if rule.index >= target.index
198
+
199
+ length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
200
+ next unless length
201
+
202
+ return nil if same_length && length != target_length
203
+
204
+ return [rule.index, length]
205
+ end
206
+ [target.index, target_length]
207
+ rescue EncodingError
208
+ nil
209
+ end
210
+
211
+ def firstmatch_length(rule, input, encoding:, unicode:)
212
+ matches = rule.patterns.each_with_index.filter_map do |pattern, index|
213
+ regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
214
+ reference = Regexp::Matcher.reference?(regexp, encoding: encoding, unicode: unicode)
215
+ match = Regexp::Matcher.match(regexp, input, encoding: encoding, unicode: unicode, reference: reference)
216
+ next unless match&.begin(0)&.zero?
217
+
218
+ [match[0].bytesize, rule.pattern_conditions.fetch(index)]
219
+ end
220
+ length, condition = matches.max_by(&:first)
221
+ return unless length
222
+ return if encoding == Encoding::UTF_8 && !input.byteslice(0, length).valid_encoding?
223
+ return if condition.end_anchor && length < input.bytesize && input.getbyte(length) != 0x0a
224
+
225
+ length
226
+ end
227
+
74
228
  def accepting?(dfa, state)
75
229
  state && !dfa.accepts.fetch(state).empty?
76
230
  end