flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -2,9 +2,6 @@
2
2
 
3
3
  module Flexr
4
4
  module Automaton
5
- CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
6
- Machine = Struct.new(:dfa, :state_name, keyword_init: true)
7
-
8
5
  class Compiler
9
6
  def initialize(spec)
10
7
  @spec = spec
@@ -16,7 +13,13 @@ module Flexr
16
13
  state_names = effective_states
17
14
  machines = state_names.to_h do |state_name|
18
15
  rules = rules_for(state_name)
19
- [state_name, Machine.new(dfa: compile_machine(rules), state_name: state_name)]
16
+ dfa = compile_machine(rules)
17
+ plans = Accel.whole_token_plans(dfa, rules) if
18
+ @spec.backend != :firstmatch && @spec.options[:accel] == :strscan
19
+ [state_name, Machine.new(dfa: dfa, state_name: state_name, whole_tokens: plans)]
20
+ rescue CompileError => e
21
+ e.diagnostic.location ||= @spec.states.fetch(state_name).location if e.diagnostic
22
+ raise
20
23
  end
21
24
  stats = machines.transform_values { |machine| machine.dfa.stats }
22
25
  compiled = CompiledSpec.new(machines: machines, rules: @spec.rules, states: state_names, stats: stats)
@@ -44,15 +47,10 @@ module Flexr
44
47
  end
45
48
 
46
49
  def compile_machine(rules)
47
- @active_rule_ids = rules.map(&:index)
50
+ @active_rules = rules
48
51
  normalized = []
49
52
  rules.each do |rule|
50
53
  rule.pattern_conditions = []
51
- if reference_rule?(rule)
52
- validate_reference_patterns(rule)
53
- next
54
- end
55
-
56
54
  rule.patterns.each_with_index do |pattern, pattern_index|
57
55
  regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
58
56
  encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : @spec.encoding
@@ -66,6 +64,9 @@ module Flexr
66
64
  normalized_ast = Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
67
65
  normalized << [normalized_ast, condition]
68
66
  end
67
+ rescue CompileError => e
68
+ e.diagnostic.location ||= rule.location if e.diagnostic
69
+ raise
69
70
  end
70
71
  return empty_dfa if normalized.empty?
71
72
 
@@ -78,32 +79,6 @@ module Flexr
78
79
  DFA.new(transitions: [[nil]], accepts: [[]], ec: Array.new(256, 0), class_count: 1, start: 0, rule_ids: [])
79
80
  end
80
81
 
81
- def reference_rule?(rule)
82
- rule.patterns.any? { |pattern| reference_pattern?(pattern) }
83
- end
84
-
85
- def reference_pattern?(pattern)
86
- return false unless pattern.is_a?(::Regexp)
87
- return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
88
-
89
- @spec.options[:unicode] == true && @spec.encoding != Encoding::BINARY &&
90
- pattern.source.match?(/\\[dDwWsS]/)
91
- end
92
-
93
- def validate_reference_patterns(rule)
94
- rule.patterns.each_with_index do |pattern, pattern_index|
95
- regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
96
- encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : @spec.encoding
97
- parser = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
98
- unicode: @spec.options[:unicode] == true)
99
- ast = parser.parse
100
- _body, bol_only, end_anchor = strip_anchors(ast)
101
- rule.pattern_conditions[pattern_index] = Acceptance.new(
102
- rule_index: rule.index, pattern_index: pattern_index, bol_only: bol_only, end_anchor: end_anchor
103
- )
104
- end
105
- end
106
-
107
82
  def strip_anchors(ast)
108
83
  children = ast.is_a?(Regexp::AST::Seq) ? ast.children.dup : [ast]
109
84
  children.shift while children.first.is_a?(Regexp::AST::Empty)
@@ -131,37 +106,40 @@ module Flexr
131
106
 
132
107
  def contains_anchor?(node)
133
108
  return true if node.is_a?(Regexp::AST::Anchor)
109
+ return contains_anchor?(node.child) if node.is_a?(Regexp::AST::Repeat) || node.is_a?(Regexp::AST::Star)
134
110
  return false unless node.respond_to?(:children)
135
111
 
136
112
  node.children.any? { |child| contains_anchor?(child) }
137
113
  end
138
114
 
139
115
  def subset_construction(nfa, ec, class_count)
140
- representatives = Array.new(class_count)
141
- ec.each_with_index { |class_id, byte| representatives[class_id] ||= byte }
142
- start_set = epsilon_closure(nfa, 1 << nfa.start)
116
+ closures = {}
117
+ closure_for = lambda do |state|
118
+ closures[state] ||= epsilon_closure_for(nfa, state)
119
+ end
120
+ transition_closures = indexed_transition_closures(nfa, ec, closure_for)
121
+ start_set = closure_for.call(nfa.start)
143
122
  sets = [start_set]
144
123
  ids = { start_set => 0 }
145
124
  transitions = []
146
125
  accepts = []
147
126
  queue = [start_set]
127
+ queue_index = 0
128
+ limit = @spec.options.fetch(:max_dfa_states, 100_000)
129
+ limit = 100_000 unless limit.is_a?(Integer) && limit.positive?
148
130
 
149
- until queue.empty?
150
- set = queue.shift
131
+ while queue_index < queue.length
132
+ set = queue.fetch(queue_index)
133
+ queue_index += 1
151
134
  state_id = ids.fetch(set)
152
135
  transitions[state_id] ||= Array.new(class_count)
153
136
  accepts[state_id] = accepting_rules(nfa, set)
154
- class_count.times do |class_id|
155
- moved = move(nfa, set, representatives[class_id])
156
- next if moved.zero?
157
- closure = epsilon_closure(nfa, moved)
137
+ move_closures(set, transition_closures).each do |class_id, closure|
158
138
  destination = ids[closure]
159
139
  unless destination
160
140
  destination = sets.length
161
- limit = @spec.options.fetch(:max_dfa_states, 100_000)
162
- limit = 100_000 unless limit.is_a?(Integer) && limit.positive?
163
141
  if destination >= limit
164
- message = "DFA state limit exceeded while compiling rules #{@active_rule_ids.join(', ')}"
142
+ message = "DFA state limit exceeded while compiling #{@active_rules.map { |rule| Diagnostics.rule_label(rule) }.join(', ')}"
165
143
  diagnostic = Diagnostics.error("FLEXR-E006", message,
166
144
  help: "raise max_dfa_states or split the listed rules")
167
145
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
@@ -180,13 +158,12 @@ module Flexr
180
158
  Minimizer.minimize(dfa)
181
159
  end
182
160
 
183
- def epsilon_closure(nfa, set)
184
- closure = set
185
- stack = []
186
- nfa.states.each_index { |id| stack << id if set.anybits?(1 << id) }
161
+ def epsilon_closure_for(nfa, state)
162
+ closure = 1 << state
163
+ stack = [state]
187
164
  until stack.empty?
188
- state = stack.pop
189
- nfa.states[state].epsilon.each do |target|
165
+ current = stack.pop
166
+ nfa.states[current].epsilon.each do |target|
190
167
  next if closure.anybits?(1 << target)
191
168
 
192
169
  closure |= 1 << target
@@ -196,35 +173,50 @@ module Flexr
196
173
  closure
197
174
  end
198
175
 
199
- def move(nfa, set, byte)
200
- moved = 0
201
- nfa.states.each_index do |state|
202
- next if set.nobits?(1 << state)
203
-
204
- nfa.states[state].transitions.each do |transition|
205
- next unless byte.between?(transition.lo, transition.hi)
176
+ def indexed_transition_closures(nfa, ec, closure_for)
177
+ nfa.states.map do |state|
178
+ raw = {}
179
+ state.transitions.each do |transition|
180
+ transition.lo.upto(transition.hi).map { |byte| ec.fetch(byte) }.uniq.each do |class_id|
181
+ raw[class_id] = raw.fetch(class_id, 0) | (1 << transition.to)
182
+ end
183
+ end
184
+ raw.transform_values do |targets|
185
+ each_state_id(targets).reduce(0) { |closure, target| closure | closure_for.call(target) }
186
+ end
187
+ end
188
+ end
206
189
 
207
- moved |= 1 << transition.to
190
+ def move_closures(set, transition_closures)
191
+ each_state_id(set).with_object({}) do |state, moves|
192
+ transition_closures.fetch(state).each do |class_id, closure|
193
+ moves[class_id] = moves.fetch(class_id, 0) | closure
208
194
  end
209
195
  end
210
- moved
211
196
  end
212
197
 
213
198
  def accepting_rules(nfa, set)
214
- rules = []
215
- nfa.states.each_index do |state|
216
- next if set.nobits?(1 << state)
217
-
218
- rules.concat(nfa.states[state].accepts)
199
+ rules = each_state_id(set).with_object([]) do |state, result|
200
+ result.concat(nfa.states[state].accepts)
219
201
  end
220
202
  rules.uniq.sort_by { |acceptance| [acceptance.rule_index, acceptance.pattern_index] }
221
203
  end
222
204
 
223
- def validate_rules
224
- raise CompileError, "firstmatch requires option :experimental" if @spec.backend == :firstmatch && !@spec.options[:experimental]
205
+ def each_state_id(set)
206
+ return enum_for(__method__, set) unless block_given?
207
+
208
+ remaining = set
209
+ until remaining.zero?
210
+ bit = remaining & -remaining
211
+ yield bit.bit_length - 1
212
+ remaining ^= bit
213
+ end
214
+ end
225
215
 
216
+ def validate_rules
226
217
  @spec.rules.each do |rule|
227
218
  raise CompileError, "rule #{rule.index} has no pattern" if rule.patterns.empty?
219
+ parse_regexp(rule.trailing) if rule.trailing.is_a?(::Regexp)
228
220
  next if @spec.options[:allow_empty_match]
229
221
 
230
222
  rule.patterns.each do |pattern|
@@ -232,9 +224,12 @@ module Flexr
232
224
  parse_regexp(regexp) if regexp.is_a?(::Regexp)
233
225
  next unless regexp.is_a?(::Regexp) && regexp.match?("")
234
226
 
235
- diagnostic = Diagnostics.error("FLEXR-E005", "rule #{rule.index} can match an empty string")
227
+ diagnostic = Diagnostics.error("FLEXR-E005", "#{Diagnostics.rule_label(rule)} can match an empty string")
236
228
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
237
229
  end
230
+ rescue CompileError => e
231
+ e.diagnostic.location ||= rule.location if e.diagnostic
232
+ raise
238
233
  end
239
234
  end
240
235
 
@@ -249,36 +244,46 @@ module Flexr
249
244
  present = compiled.machines.values.flat_map do |machine|
250
245
  machine.dfa.accepts.filter_map { |acceptances| acceptances.min_by(&:rule_index)&.rule_index }
251
246
  end
252
- reference_rules = @spec.rules.select { |rule| reference_rule?(rule) && rule_active_anywhere?(rule) }
253
- present.concat(reference_rules.map(&:index))
254
247
  shadowers = shadowed_rules(compiled)
255
- diagnostics.concat(@spec.rules.reject { |rule| present.include?(rule.index) }.map do |rule|
256
- winners = shadowers.fetch(rule.index, []).uniq.sort
257
- suffix = winners.empty? ? "" : " (shadowed by rule #{winners.join(', ')})"
258
- Diagnostics.warning("FLEXR-W001", "rule #{rule.index} is unreachable#{suffix}", location: rule.location,
259
- help: "remove it, reorder the rules, or make its language distinct")
248
+ rules_by_index = @spec.rules.to_h { |rule| [rule.index, rule] }
249
+ unreachable = @spec.rules.reject { |rule| present.include?(rule.index) }
250
+ examples = unreachable_examples(compiled, unreachable, rules_by_index)
251
+ diagnostics.concat(unreachable.filter_map do |rule|
252
+ example = examples[rule.index]
253
+ next if example && example.fetch(:winner) == rule.index
254
+
255
+ winner_indices = @spec.backend == :firstmatch || @spec.rules.any?(&:trailing) ? [] : shadowers.fetch(rule.index, [])
256
+ winners = winner_indices.uniq.sort.map { |index| Diagnostics.rule_label(rules_by_index.fetch(index)) }
257
+ suffix = winners.empty? ? "" : " (shadowed by #{winners.join(', ')})"
258
+ Diagnostics.warning("FLEXR-W001", "#{Diagnostics.rule_label(rule)} is unreachable#{suffix}", location: rule.location,
259
+ help: "remove it, reorder the rules, or make its language distinct",
260
+ note: example&.fetch(:note))
260
261
  end)
262
+ diagnostics.concat(overlap_diagnostics(compiled, rules_by_index)) if @spec.options[:warn_level] == :all
261
263
 
262
- @spec.states.each_key do |state_name|
264
+ @spec.states.each do |state_name, state|
263
265
  next if state_name == :initial
264
266
  next unless rules_for(state_name).empty?
265
267
 
266
268
  diagnostics << Diagnostics.warning("FLEXR-W002", "state #{state_name.inspect} has no rules",
269
+ location: state.location,
267
270
  help: "add a rule to the state or remove the unused state")
268
271
  end
269
272
 
270
273
  firstmatch_conflicts(@spec.rules).each do |left, right|
271
274
  diagnostics << Diagnostics.warning(
272
- "FLEXR-W010", "firstmatch rules #{left.index} and #{right.index} may change longest-match semantics",
275
+ "FLEXR-W010", "firstmatch #{Diagnostics.rule_label(left)} and #{Diagnostics.rule_label(right)} may change longest-match semantics",
276
+ location: right.location,
273
277
  help: "use backend :table unless first-match compatibility is required"
274
278
  )
275
279
  end
276
280
 
277
- max_cells = compiled.stats.values.map { |stat| stat[:states] * stat[:classes] }.max.to_i
278
- if max_cells > 1_000_000
281
+ largest_state, max_cells = compiled.stats.map { |name, stat| [name, stat[:states] * stat[:classes]] }.max_by(&:last)
282
+ if max_cells && max_cells > 1_000_000
279
283
  diagnostics << Diagnostics.warning(
280
284
  "FLEXR-W011",
281
285
  "generated transition table is large",
286
+ location: @spec.states.fetch(largest_state).location,
282
287
  help: "use backend :direct, table compression, or split the specification"
283
288
  )
284
289
  end
@@ -286,21 +291,24 @@ module Flexr
286
291
  capture_rules.each do |rule|
287
292
  diagnostics << Diagnostics.warning(
288
293
  "FLEXR-W013",
289
- "rule #{rule.index} uses a capturing group; flexr treats it as non-capturing",
294
+ "#{Diagnostics.rule_label(rule)} uses a capturing group; flexr treats it as non-capturing",
295
+ location: rule.location,
290
296
  help: "rewrite capturing groups as (?:...) and extract text in the action"
291
297
  )
292
298
  end
293
299
 
294
- undeclared_tokens.each do |token|
300
+ undeclared_token_rules.each do |token, rule|
295
301
  diagnostics << Diagnostics.warning(
296
302
  "FLEXR-W014",
297
303
  "token #{token.inspect} is not declared by emits",
304
+ location: rule.location,
298
305
  help: "add the token to emits or remove the declaration if it is intentionally private"
299
306
  )
300
307
  end
301
308
 
302
309
  diagnostics.concat(variable_trailing_rules.map do |rule|
303
- Diagnostics.warning("FLEXR-W003", "rule #{rule.index} uses variable-length trailing context",
310
+ Diagnostics.warning("FLEXR-W003", "#{Diagnostics.rule_label(rule)} uses variable-length trailing context",
311
+ location: rule.location,
304
312
  help: "make the body or followed_by expression fixed length when possible")
305
313
  end)
306
314
 
@@ -308,7 +316,8 @@ module Flexr
308
316
  diagnostics.concat(@spec.rules.select(&:trailing).map do |rule|
309
317
  Diagnostics.warning(
310
318
  "FLEXR-W012",
311
- "rule #{rule.index} cannot use region acceleration with trailing context",
319
+ "#{Diagnostics.rule_label(rule)} cannot use region acceleration with trailing context",
320
+ location: rule.location,
312
321
  help: "remove trailing context or set accel: :none when the trade-off is intentional"
313
322
  )
314
323
  end)
@@ -316,11 +325,138 @@ module Flexr
316
325
 
317
326
  if elapsed > 0.5
318
327
  diagnostics << Diagnostics.warning("FLEXR-W016", format("DFA construction took %.3fs", elapsed),
328
+ location: @spec.states.fetch(:initial).location,
319
329
  help: "use generated mode for production startup")
320
330
  end
321
331
  diagnostics
322
332
  end
323
333
 
334
+ def overlap_diagnostics(compiled, rules_by_index)
335
+ # shortcut: verify one shortest body and trailing candidate per rule; search alternatives if rejected examples need coverage.
336
+ compiled.machines.flat_map do |state_name, machine|
337
+ active_rules = rules_for(state_name)
338
+ trailing_dfas = diagnostic_trailing_dfas(active_rules)
339
+ next [] unless trailing_dfas
340
+
341
+ has_trailing = !trailing_dfas.empty?
342
+ candidates = Analysis.shadowing_examples(machine.dfa, machine.dfa.rule_ids,
343
+ include_empty: has_trailing, include_unshadowed: has_trailing)
344
+ candidates.filter_map do |index, (bytes, winner)|
345
+ if (trailing = trailing_dfas[index])
346
+ suffix = Analysis.shortest_input(trailing)
347
+ next unless suffix
348
+
349
+ bytes += suffix
350
+ end
351
+ input = bytes.dup.force_encoding(@spec.encoding)
352
+ next if input.empty? || !input.valid_encoding?
353
+
354
+ rule = rules_by_index.fetch(index)
355
+ length = input.bytesize
356
+ if has_trailing
357
+ result = Analysis.trailing_example(machine.dfa, active_rules, rule, input, trailing_dfas,
358
+ encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
359
+ firstmatch: @spec.backend == :firstmatch, same_length: true)
360
+ next unless result && result.first != index
361
+
362
+ winner, length, lookahead = result
363
+ elsif @spec.backend == :firstmatch
364
+ result = Analysis.firstmatch_example(active_rules, rule, input, encoding: @spec.encoding,
365
+ unicode: @spec.options[:unicode] == true, same_length: true)
366
+ next unless result && result.first != index
367
+
368
+ winner, length = result
369
+ end
370
+ selected = Diagnostics.rule_label(rules_by_index.fetch(winner))
371
+ state = state_name == :initial ? "" : " in state #{state_name.inspect}"
372
+ size = "#{length} #{length == 1 ? 'byte' : 'bytes'}"
373
+ reason = if has_trailing
374
+ priority = @spec.backend == :firstmatch ? "first match wins" : "same total length, earlier rule wins"
375
+ "#{priority}; matched #{size}, looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}"
376
+ elsif @spec.backend == :firstmatch
377
+ "first match wins, matched #{size}"
378
+ else
379
+ "#{size}, earlier rule wins"
380
+ end
381
+ match_size = has_trailing ? "total length" : "length"
382
+ Diagnostics.info("FLEXR-I001", "#{Diagnostics.rule_label(rule)} overlaps #{selected} at the same #{match_size}",
383
+ location: rule.location, help: "reorder the rules if the later rule should take priority",
384
+ note: "example: #{input.inspect} selects #{selected}#{state} (#{reason})")
385
+ end
386
+ end
387
+ end
388
+
389
+ def unreachable_examples(compiled, rules, rules_by_index)
390
+ return {} if rules.empty?
391
+
392
+ indices = rules.map(&:index)
393
+ examples = {}
394
+ compiled.machines.each do |state_name, machine|
395
+ active_rules = rules_for(state_name)
396
+ # shortcut: verify only the shortest candidate; search alternatives if rejected examples need coverage.
397
+ candidates = Analysis.shadowing_examples(machine.dfa, indices, include_empty: active_rules.any?(&:trailing))
398
+ next if candidates.empty?
399
+
400
+ trailing_dfas = diagnostic_trailing_dfas(active_rules)
401
+ next unless trailing_dfas
402
+
403
+ candidates.each do |index, (bytes, winner)|
404
+ if (trailing = trailing_dfas[index])
405
+ suffix = Analysis.shortest_input(trailing)
406
+ next unless suffix
407
+
408
+ bytes += suffix
409
+ end
410
+ input = bytes.dup.force_encoding(@spec.encoding)
411
+ next if input.empty? || !input.valid_encoding?
412
+ next if examples[index] && examples[index][1] == index
413
+
414
+ length = lookahead = nil
415
+ if !trailing_dfas.empty?
416
+ result = Analysis.trailing_example(machine.dfa, active_rules, rules_by_index.fetch(index), input, trailing_dfas,
417
+ encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
418
+ firstmatch: @spec.backend == :firstmatch)
419
+ next unless result
420
+
421
+ winner, length, lookahead = result
422
+ elsif @spec.backend == :firstmatch
423
+ result = Analysis.firstmatch_example(active_rules, rules_by_index.fetch(index), input,
424
+ encoding: @spec.encoding, unicode: @spec.options[:unicode] == true)
425
+ next unless result
426
+
427
+ winner, length = result
428
+ end
429
+ next if winner != index && examples[index] && examples[index].first.bytesize <= input.bytesize
430
+
431
+ examples[index] = [input, winner, state_name, length, lookahead]
432
+ end
433
+ end
434
+ examples.transform_values do |input, winner, state_name, length, lookahead|
435
+ state = state_name == :initial ? "" : " in state #{state_name.inspect}"
436
+ reason = if @spec.backend == :firstmatch
437
+ "first match wins, matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
438
+ elsif lookahead
439
+ "longest total match, then rule order; matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
440
+ else
441
+ "same length, earlier rule wins"
442
+ end
443
+ reason += ", looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}" if lookahead
444
+ note = "example: #{input.inspect} selects #{Diagnostics.rule_label(rules_by_index.fetch(winner))}#{state} (#{reason})"
445
+ { note: note, winner: winner }
446
+ end
447
+ end
448
+
449
+ def diagnostic_trailing_dfas(rules)
450
+ rules.select(&:trailing).to_h do |rule|
451
+ dfa = rule.trailing_dfa || Flexr.compile_pattern(rule.trailing, encoding: @spec.encoding,
452
+ options: { unicode: @spec.options[:unicode] == true })
453
+ [rule.index, dfa]
454
+ end
455
+ rescue CompileError
456
+ # shortcut: omit examples for trailing patterns the runtime cannot compile, until that matcher supports them.
457
+ nil
458
+ end
459
+
324
460
  def shadowed_rules(compiled)
325
461
  shadowers = Hash.new { |hash, key| hash[key] = [] }
326
462
  compiled.machines.each_value do |machine|
@@ -344,10 +480,6 @@ module Flexr
344
480
  rules.combination(2).to_a
345
481
  end
346
482
 
347
- def rule_active_anywhere?(rule)
348
- @spec.states.keys.any? { |state_name| rules_for(state_name).include?(rule) }
349
- end
350
-
351
483
  def capture_rules
352
484
  @spec.rules.select do |rule|
353
485
  rule.patterns.any? do |pattern|
@@ -378,21 +510,22 @@ module Flexr
378
510
  false
379
511
  end
380
512
 
381
- def undeclared_tokens
382
- return [] if Array(@spec.declared_tokens).empty?
513
+ def undeclared_token_rules
514
+ declared = Array(@spec.declared_tokens)
515
+ return {} if declared.empty?
383
516
 
384
- emitted = @spec.rules.flat_map do |rule|
517
+ emitted = @spec.rules.each_with_object({}) do |rule, rules_by_token|
385
518
  action = rule.action
386
- if action.is_a?(Array) && action.first == :emit
519
+ tokens = if action.is_a?(Array) && action.first == :emit
387
520
  [action.last]
388
521
  elsif action.is_a?(String)
389
522
  action.scan(/\bemit\s*\(?\s*:([A-Za-z_]\w*)/).flatten.map(&:to_sym)
390
523
  else
391
524
  []
392
525
  end
526
+ tokens.each { |token| rules_by_token[token] ||= rule }
393
527
  end
394
- declared = Array(@spec.declared_tokens)
395
- emitted.uniq.reject { |token| declared.include?(token) }
528
+ emitted.except(*declared)
396
529
  end
397
530
 
398
531
  def variable_trailing_rules
@@ -2,42 +2,81 @@
2
2
 
3
3
  module Flexr
4
4
  module Automaton
5
+ class TransitionRows
6
+ include Enumerable
7
+
8
+ attr_reader :length
9
+
10
+ def initialize(length, &loader)
11
+ @length = length
12
+ @loader = loader
13
+ @rows = Array.new(length)
14
+ freeze
15
+ end
16
+
17
+ alias size length
18
+
19
+ def [](state)
20
+ return unless state&.between?(0, length - 1)
21
+
22
+ @rows[state] ||= @loader.call(state).freeze
23
+ end
24
+
25
+ def fetch(state)
26
+ return self[state] if state.between?(0, length - 1)
27
+
28
+ raise IndexError, "transition row #{state} is out of bounds"
29
+ end
30
+
31
+ def each
32
+ return enum_for(__method__) unless block_given?
33
+
34
+ length.times { |state| yield self[state] }
35
+ end
36
+
37
+ def each_index(&)
38
+ return enum_for(__method__) unless block_given?
39
+
40
+ length.times(&)
41
+ end
42
+
43
+ def materialized_count
44
+ @rows.count { |row| !row.nil? }
45
+ end
46
+ end
47
+
5
48
  class DFA
6
49
  attr_reader :transitions, :accepts, :ec, :class_count, :start, :states, :rule_ids, :packed, :direct
7
50
 
8
- def initialize(transitions:, accepts:, ec:, class_count:, start:, rule_ids:, packed: nil, direct: nil)
9
- @transitions = transitions.freeze
51
+ def initialize(accepts:, ec:, class_count:, start:, rule_ids:, transitions: nil, packed: nil, direct: nil,
52
+ state_count: nil)
53
+ @packed = freeze_representation(packed)
54
+ @direct = freeze_representation(direct)
55
+ @states = state_count || transitions&.length || direct_state_count
56
+ raise ArgumentError, "state_count is required without dense transitions" unless @states
57
+
58
+ @transitions = transition_rows(transitions)
10
59
  @accepts = accepts.map do |rules|
11
60
  rules.map do |acceptance|
12
- next acceptance if acceptance.is_a?(Acceptance)
61
+ next acceptance.freeze if acceptance.is_a?(Acceptance)
13
62
 
14
63
  Acceptance.new(rule_index: acceptance[0], pattern_index: acceptance[1],
15
- bol_only: acceptance[2], end_anchor: acceptance[3])
64
+ bol_only: acceptance[2], end_anchor: acceptance[3]).freeze
16
65
  end.freeze
17
66
  end.freeze
18
67
  @ec = ec.freeze
19
68
  @class_count = class_count
20
69
  @start = start
21
- @states = transitions.length
22
70
  @rule_ids = rule_ids.map { |id| id.respond_to?(:rule_index) ? id.rule_index : id }.uniq.sort.freeze
23
- @packed = packed
24
- @direct = direct
71
+ freeze
25
72
  end
26
73
 
27
74
  def transition(state, byte)
28
75
  class_id = @ec[byte]
76
+ return direct_transition(state, class_id) if @direct
29
77
  return @transitions[state][class_id] unless @packed
30
78
 
31
- cursor = state
32
- loop do
33
- index = @packed.fetch(:base).fetch(cursor) + class_id
34
- return @packed.fetch(:next).fetch(index) if @packed.fetch(:check)[index] == cursor
35
-
36
- fallback = @packed[:fallback]&.fetch(cursor)
37
- return @packed.fetch(:default).fetch(cursor) unless fallback
38
-
39
- cursor = fallback
40
- end
79
+ packed_transition(state, class_id)
41
80
  end
42
81
 
43
82
  # Generated direct lexers use a flattened dispatch representation. The
@@ -45,8 +84,7 @@ module Flexr
45
84
  def transition_direct(state, byte)
46
85
  if @direct
47
86
  class_id = @ec[byte]
48
- value = @direct.fetch(:nxt).fetch((state * @direct.fetch(:classes)) + class_id)
49
- return value >= 0 ? value : nil
87
+ return direct_transition(state, class_id)
50
88
  end
51
89
 
52
90
  @transitions[state][@ec[byte]]
@@ -65,6 +103,55 @@ module Flexr
65
103
  def stats
66
104
  { states: states, classes: class_count, accepting_states: accepts.count { |rules| !rules.empty? } }
67
105
  end
106
+
107
+ private
108
+
109
+ def transition_rows(transitions)
110
+ return transitions.map(&:freeze).freeze if transitions
111
+ if @packed
112
+ return TransitionRows.new(states) do |state|
113
+ Array.new(class_count) { |class_id| packed_transition(state, class_id) }
114
+ end
115
+ end
116
+ if @direct
117
+ return TransitionRows.new(states) do |state|
118
+ Array.new(class_count) { |class_id| direct_transition(state, class_id) }
119
+ end
120
+ end
121
+
122
+ raise ArgumentError, "a transition representation is required"
123
+ end
124
+
125
+ def packed_transition(state, class_id)
126
+ cursor = state
127
+ loop do
128
+ index = @packed.fetch(:base).fetch(cursor) + class_id
129
+ return @packed.fetch(:next).fetch(index) if @packed.fetch(:check)[index] == cursor
130
+
131
+ fallback = @packed[:fallback]&.fetch(cursor)
132
+ return @packed.fetch(:default).fetch(cursor) unless fallback
133
+
134
+ cursor = fallback
135
+ end
136
+ end
137
+
138
+ def direct_transition(state, class_id)
139
+ value = @direct.fetch(:nxt).fetch((state * @direct.fetch(:classes)) + class_id)
140
+ value >= 0 ? value : nil
141
+ end
142
+
143
+ def direct_state_count
144
+ return unless @direct
145
+
146
+ @direct.fetch(:nxt).length / @direct.fetch(:classes)
147
+ end
148
+
149
+ def freeze_representation(representation)
150
+ return unless representation
151
+
152
+ representation.each_value { |value| value.freeze if value.respond_to?(:freeze) }
153
+ representation.freeze
154
+ end
68
155
  end
69
156
 
70
157
  class ReferenceDFA