flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -13,7 +13,13 @@ module Flexr
13
13
  state_names = effective_states
14
14
  machines = state_names.to_h do |state_name|
15
15
  rules = rules_for(state_name)
16
- [state_name, Machine.new(dfa: compile_machine(rules), state_name: state_name)]
16
+ dfa = compile_machine(rules)
17
+ plans = Accel.whole_token_plans(dfa, rules) if
18
+ @spec.backend != :firstmatch && @spec.options[:accel] == :strscan
19
+ [state_name, Machine.new(dfa: dfa, state_name: state_name, whole_tokens: plans)]
20
+ rescue CompileError => e
21
+ e.diagnostic.location ||= @spec.states.fetch(state_name).location if e.diagnostic
22
+ raise
17
23
  end
18
24
  stats = machines.transform_values { |machine| machine.dfa.stats }
19
25
  compiled = CompiledSpec.new(machines: machines, rules: @spec.rules, states: state_names, stats: stats)
@@ -41,7 +47,7 @@ module Flexr
41
47
  end
42
48
 
43
49
  def compile_machine(rules)
44
- @active_rule_ids = rules.map(&:index)
50
+ @active_rules = rules
45
51
  normalized = []
46
52
  rules.each do |rule|
47
53
  rule.pattern_conditions = []
@@ -58,6 +64,9 @@ module Flexr
58
64
  normalized_ast = Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
59
65
  normalized << [normalized_ast, condition]
60
66
  end
67
+ rescue CompileError => e
68
+ e.diagnostic.location ||= rule.location if e.diagnostic
69
+ raise
61
70
  end
62
71
  return empty_dfa if normalized.empty?
63
72
 
@@ -130,7 +139,7 @@ module Flexr
130
139
  unless destination
131
140
  destination = sets.length
132
141
  if destination >= limit
133
- message = "DFA state limit exceeded while compiling rules #{@active_rule_ids.join(', ')}"
142
+ message = "DFA state limit exceeded while compiling #{@active_rules.map { |rule| Diagnostics.rule_label(rule) }.join(', ')}"
134
143
  diagnostic = Diagnostics.error("FLEXR-E006", message,
135
144
  help: "raise max_dfa_states or split the listed rules")
136
145
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
@@ -205,8 +214,6 @@ module Flexr
205
214
  end
206
215
 
207
216
  def validate_rules
208
- raise CompileError, "firstmatch requires option :experimental" if @spec.backend == :firstmatch && !@spec.options[:experimental]
209
-
210
217
  @spec.rules.each do |rule|
211
218
  raise CompileError, "rule #{rule.index} has no pattern" if rule.patterns.empty?
212
219
  parse_regexp(rule.trailing) if rule.trailing.is_a?(::Regexp)
@@ -217,9 +224,12 @@ module Flexr
217
224
  parse_regexp(regexp) if regexp.is_a?(::Regexp)
218
225
  next unless regexp.is_a?(::Regexp) && regexp.match?("")
219
226
 
220
- diagnostic = Diagnostics.error("FLEXR-E005", "rule #{rule.index} can match an empty string")
227
+ diagnostic = Diagnostics.error("FLEXR-E005", "#{Diagnostics.rule_label(rule)} can match an empty string")
221
228
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
222
229
  end
230
+ rescue CompileError => e
231
+ e.diagnostic.location ||= rule.location if e.diagnostic
232
+ raise
223
233
  end
224
234
  end
225
235
 
@@ -235,33 +245,45 @@ module Flexr
235
245
  machine.dfa.accepts.filter_map { |acceptances| acceptances.min_by(&:rule_index)&.rule_index }
236
246
  end
237
247
  shadowers = shadowed_rules(compiled)
238
- diagnostics.concat(@spec.rules.reject { |rule| present.include?(rule.index) }.map do |rule|
239
- winners = shadowers.fetch(rule.index, []).uniq.sort
240
- suffix = winners.empty? ? "" : " (shadowed by rule #{winners.join(', ')})"
241
- Diagnostics.warning("FLEXR-W001", "rule #{rule.index} is unreachable#{suffix}", location: rule.location,
242
- help: "remove it, reorder the rules, or make its language distinct")
248
+ rules_by_index = @spec.rules.to_h { |rule| [rule.index, rule] }
249
+ unreachable = @spec.rules.reject { |rule| present.include?(rule.index) }
250
+ examples = unreachable_examples(compiled, unreachable, rules_by_index)
251
+ diagnostics.concat(unreachable.filter_map do |rule|
252
+ example = examples[rule.index]
253
+ next if example && example.fetch(:winner) == rule.index
254
+
255
+ winner_indices = @spec.backend == :firstmatch || @spec.rules.any?(&:trailing) ? [] : shadowers.fetch(rule.index, [])
256
+ winners = winner_indices.uniq.sort.map { |index| Diagnostics.rule_label(rules_by_index.fetch(index)) }
257
+ suffix = winners.empty? ? "" : " (shadowed by #{winners.join(', ')})"
258
+ Diagnostics.warning("FLEXR-W001", "#{Diagnostics.rule_label(rule)} is unreachable#{suffix}", location: rule.location,
259
+ help: "remove it, reorder the rules, or make its language distinct",
260
+ note: example&.fetch(:note))
243
261
  end)
262
+ diagnostics.concat(overlap_diagnostics(compiled, rules_by_index)) if @spec.options[:warn_level] == :all
244
263
 
245
- @spec.states.each_key do |state_name|
264
+ @spec.states.each do |state_name, state|
246
265
  next if state_name == :initial
247
266
  next unless rules_for(state_name).empty?
248
267
 
249
268
  diagnostics << Diagnostics.warning("FLEXR-W002", "state #{state_name.inspect} has no rules",
269
+ location: state.location,
250
270
  help: "add a rule to the state or remove the unused state")
251
271
  end
252
272
 
253
273
  firstmatch_conflicts(@spec.rules).each do |left, right|
254
274
  diagnostics << Diagnostics.warning(
255
- "FLEXR-W010", "firstmatch rules #{left.index} and #{right.index} may change longest-match semantics",
275
+ "FLEXR-W010", "firstmatch #{Diagnostics.rule_label(left)} and #{Diagnostics.rule_label(right)} may change longest-match semantics",
276
+ location: right.location,
256
277
  help: "use backend :table unless first-match compatibility is required"
257
278
  )
258
279
  end
259
280
 
260
- max_cells = compiled.stats.values.map { |stat| stat[:states] * stat[:classes] }.max.to_i
261
- if max_cells > 1_000_000
281
+ largest_state, max_cells = compiled.stats.map { |name, stat| [name, stat[:states] * stat[:classes]] }.max_by(&:last)
282
+ if max_cells && max_cells > 1_000_000
262
283
  diagnostics << Diagnostics.warning(
263
284
  "FLEXR-W011",
264
285
  "generated transition table is large",
286
+ location: @spec.states.fetch(largest_state).location,
265
287
  help: "use backend :direct, table compression, or split the specification"
266
288
  )
267
289
  end
@@ -269,21 +291,24 @@ module Flexr
269
291
  capture_rules.each do |rule|
270
292
  diagnostics << Diagnostics.warning(
271
293
  "FLEXR-W013",
272
- "rule #{rule.index} uses a capturing group; flexr treats it as non-capturing",
294
+ "#{Diagnostics.rule_label(rule)} uses a capturing group; flexr treats it as non-capturing",
295
+ location: rule.location,
273
296
  help: "rewrite capturing groups as (?:...) and extract text in the action"
274
297
  )
275
298
  end
276
299
 
277
- undeclared_tokens.each do |token|
300
+ undeclared_token_rules.each do |token, rule|
278
301
  diagnostics << Diagnostics.warning(
279
302
  "FLEXR-W014",
280
303
  "token #{token.inspect} is not declared by emits",
304
+ location: rule.location,
281
305
  help: "add the token to emits or remove the declaration if it is intentionally private"
282
306
  )
283
307
  end
284
308
 
285
309
  diagnostics.concat(variable_trailing_rules.map do |rule|
286
- Diagnostics.warning("FLEXR-W003", "rule #{rule.index} uses variable-length trailing context",
310
+ Diagnostics.warning("FLEXR-W003", "#{Diagnostics.rule_label(rule)} uses variable-length trailing context",
311
+ location: rule.location,
287
312
  help: "make the body or followed_by expression fixed length when possible")
288
313
  end)
289
314
 
@@ -291,7 +316,8 @@ module Flexr
291
316
  diagnostics.concat(@spec.rules.select(&:trailing).map do |rule|
292
317
  Diagnostics.warning(
293
318
  "FLEXR-W012",
294
- "rule #{rule.index} cannot use region acceleration with trailing context",
319
+ "#{Diagnostics.rule_label(rule)} cannot use region acceleration with trailing context",
320
+ location: rule.location,
295
321
  help: "remove trailing context or set accel: :none when the trade-off is intentional"
296
322
  )
297
323
  end)
@@ -299,11 +325,138 @@ module Flexr
299
325
 
300
326
  if elapsed > 0.5
301
327
  diagnostics << Diagnostics.warning("FLEXR-W016", format("DFA construction took %.3fs", elapsed),
328
+ location: @spec.states.fetch(:initial).location,
302
329
  help: "use generated mode for production startup")
303
330
  end
304
331
  diagnostics
305
332
  end
306
333
 
334
+ def overlap_diagnostics(compiled, rules_by_index)
335
+ # shortcut: verify one shortest body and trailing candidate per rule; search alternatives if rejected examples need coverage.
336
+ compiled.machines.flat_map do |state_name, machine|
337
+ active_rules = rules_for(state_name)
338
+ trailing_dfas = diagnostic_trailing_dfas(active_rules)
339
+ next [] unless trailing_dfas
340
+
341
+ has_trailing = !trailing_dfas.empty?
342
+ candidates = Analysis.shadowing_examples(machine.dfa, machine.dfa.rule_ids,
343
+ include_empty: has_trailing, include_unshadowed: has_trailing)
344
+ candidates.filter_map do |index, (bytes, winner)|
345
+ if (trailing = trailing_dfas[index])
346
+ suffix = Analysis.shortest_input(trailing)
347
+ next unless suffix
348
+
349
+ bytes += suffix
350
+ end
351
+ input = bytes.dup.force_encoding(@spec.encoding)
352
+ next if input.empty? || !input.valid_encoding?
353
+
354
+ rule = rules_by_index.fetch(index)
355
+ length = input.bytesize
356
+ if has_trailing
357
+ result = Analysis.trailing_example(machine.dfa, active_rules, rule, input, trailing_dfas,
358
+ encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
359
+ firstmatch: @spec.backend == :firstmatch, same_length: true)
360
+ next unless result && result.first != index
361
+
362
+ winner, length, lookahead = result
363
+ elsif @spec.backend == :firstmatch
364
+ result = Analysis.firstmatch_example(active_rules, rule, input, encoding: @spec.encoding,
365
+ unicode: @spec.options[:unicode] == true, same_length: true)
366
+ next unless result && result.first != index
367
+
368
+ winner, length = result
369
+ end
370
+ selected = Diagnostics.rule_label(rules_by_index.fetch(winner))
371
+ state = state_name == :initial ? "" : " in state #{state_name.inspect}"
372
+ size = "#{length} #{length == 1 ? 'byte' : 'bytes'}"
373
+ reason = if has_trailing
374
+ priority = @spec.backend == :firstmatch ? "first match wins" : "same total length, earlier rule wins"
375
+ "#{priority}; matched #{size}, looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}"
376
+ elsif @spec.backend == :firstmatch
377
+ "first match wins, matched #{size}"
378
+ else
379
+ "#{size}, earlier rule wins"
380
+ end
381
+ match_size = has_trailing ? "total length" : "length"
382
+ Diagnostics.info("FLEXR-I001", "#{Diagnostics.rule_label(rule)} overlaps #{selected} at the same #{match_size}",
383
+ location: rule.location, help: "reorder the rules if the later rule should take priority",
384
+ note: "example: #{input.inspect} selects #{selected}#{state} (#{reason})")
385
+ end
386
+ end
387
+ end
388
+
389
+ def unreachable_examples(compiled, rules, rules_by_index)
390
+ return {} if rules.empty?
391
+
392
+ indices = rules.map(&:index)
393
+ examples = {}
394
+ compiled.machines.each do |state_name, machine|
395
+ active_rules = rules_for(state_name)
396
+ # shortcut: verify only the shortest candidate; search alternatives if rejected examples need coverage.
397
+ candidates = Analysis.shadowing_examples(machine.dfa, indices, include_empty: active_rules.any?(&:trailing))
398
+ next if candidates.empty?
399
+
400
+ trailing_dfas = diagnostic_trailing_dfas(active_rules)
401
+ next unless trailing_dfas
402
+
403
+ candidates.each do |index, (bytes, winner)|
404
+ if (trailing = trailing_dfas[index])
405
+ suffix = Analysis.shortest_input(trailing)
406
+ next unless suffix
407
+
408
+ bytes += suffix
409
+ end
410
+ input = bytes.dup.force_encoding(@spec.encoding)
411
+ next if input.empty? || !input.valid_encoding?
412
+ next if examples[index] && examples[index][1] == index
413
+
414
+ length = lookahead = nil
415
+ if !trailing_dfas.empty?
416
+ result = Analysis.trailing_example(machine.dfa, active_rules, rules_by_index.fetch(index), input, trailing_dfas,
417
+ encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
418
+ firstmatch: @spec.backend == :firstmatch)
419
+ next unless result
420
+
421
+ winner, length, lookahead = result
422
+ elsif @spec.backend == :firstmatch
423
+ result = Analysis.firstmatch_example(active_rules, rules_by_index.fetch(index), input,
424
+ encoding: @spec.encoding, unicode: @spec.options[:unicode] == true)
425
+ next unless result
426
+
427
+ winner, length = result
428
+ end
429
+ next if winner != index && examples[index] && examples[index].first.bytesize <= input.bytesize
430
+
431
+ examples[index] = [input, winner, state_name, length, lookahead]
432
+ end
433
+ end
434
+ examples.transform_values do |input, winner, state_name, length, lookahead|
435
+ state = state_name == :initial ? "" : " in state #{state_name.inspect}"
436
+ reason = if @spec.backend == :firstmatch
437
+ "first match wins, matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
438
+ elsif lookahead
439
+ "longest total match, then rule order; matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
440
+ else
441
+ "same length, earlier rule wins"
442
+ end
443
+ reason += ", looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}" if lookahead
444
+ note = "example: #{input.inspect} selects #{Diagnostics.rule_label(rules_by_index.fetch(winner))}#{state} (#{reason})"
445
+ { note: note, winner: winner }
446
+ end
447
+ end
448
+
449
+ def diagnostic_trailing_dfas(rules)
450
+ rules.select(&:trailing).to_h do |rule|
451
+ dfa = rule.trailing_dfa || Flexr.compile_pattern(rule.trailing, encoding: @spec.encoding,
452
+ options: { unicode: @spec.options[:unicode] == true })
453
+ [rule.index, dfa]
454
+ end
455
+ rescue CompileError
456
+ # shortcut: omit examples for trailing patterns the runtime cannot compile, until that matcher supports them.
457
+ nil
458
+ end
459
+
307
460
  def shadowed_rules(compiled)
308
461
  shadowers = Hash.new { |hash, key| hash[key] = [] }
309
462
  compiled.machines.each_value do |machine|
@@ -357,21 +510,22 @@ module Flexr
357
510
  false
358
511
  end
359
512
 
360
- def undeclared_tokens
361
- return [] if Array(@spec.declared_tokens).empty?
513
+ def undeclared_token_rules
514
+ declared = Array(@spec.declared_tokens)
515
+ return {} if declared.empty?
362
516
 
363
- emitted = @spec.rules.flat_map do |rule|
517
+ emitted = @spec.rules.each_with_object({}) do |rule, rules_by_token|
364
518
  action = rule.action
365
- if action.is_a?(Array) && action.first == :emit
519
+ tokens = if action.is_a?(Array) && action.first == :emit
366
520
  [action.last]
367
521
  elsif action.is_a?(String)
368
522
  action.scan(/\bemit\s*\(?\s*:([A-Za-z_]\w*)/).flatten.map(&:to_sym)
369
523
  else
370
524
  []
371
525
  end
526
+ tokens.each { |token| rules_by_token[token] ||= rule }
372
527
  end
373
- declared = Array(@spec.declared_tokens)
374
- emitted.uniq.reject { |token| declared.include?(token) }
528
+ emitted.except(*declared)
375
529
  end
376
530
 
377
531
  def variable_trailing_rules
@@ -9,6 +9,6 @@ module Flexr
9
9
  end
10
10
 
11
11
  CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
12
- Machine = Struct.new(:dfa, :state_name, keyword_init: true)
12
+ Machine = Struct.new(:dfa, :state_name, :whole_tokens, keyword_init: true)
13
13
  end
14
14
  end
data/lib/flexr/cli.rb CHANGED
@@ -5,6 +5,7 @@ require_relative "artifact_writer"
5
5
  require_relative "generator"
6
6
  require_relative "importer"
7
7
  require_relative "options"
8
+ require_relative "migration"
8
9
 
9
10
  module Flexr
10
11
  module CLI
@@ -21,17 +22,19 @@ module Flexr
21
22
  return version(out) if args.delete("--version")
22
23
 
23
24
  command = COMMANDS.include?(args.first) ? args.shift.to_sym : :generate
24
- options, output, rule_number, benchmark_args, positionals = parse(args, command)
25
+ options, output, rule_number, benchmark_args, positionals, trace_args = parse(args, command)
26
+ raise ArgumentError, "lrama format is only valid for tokens" if options.format == :lrama && command != :tokens
25
27
  spec = positionals.shift
26
28
  raise ArgumentError, "a SPEC.rb path is required" unless spec
27
29
  raise ArgumentError, "unexpected argument: #{positionals.first}" unless positionals.empty?
28
30
 
29
- execute(command, spec, options, output, rule_number, benchmark_args, out, err)
31
+ execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: trace_args)
30
32
  rescue ArgumentError => e
31
33
  err.puts "error: #{e.message}"
32
34
  usage(err, status: EXIT_USAGE)
33
35
  rescue Flexr::Error, Errno::ENOENT, Errno::EACCES, SyntaxError => e
34
- err.puts render_error(e, options: options)
36
+ err.puts render_error(e, options: options, path: spec,
37
+ normalize_source: command == :generate || (command == :check && options&.eval_mode))
35
38
  EXIT_FAILURE
36
39
  rescue StandardError => e
37
40
  err.puts "error: #{e.class}: #{e.message}"
@@ -44,6 +47,7 @@ module Flexr
44
47
  rule_number = nil
45
48
  benchmark_args = []
46
49
  positionals = []
50
+ trace_args = {}
47
51
  parsing_options = true
48
52
 
49
53
  until args.empty?
@@ -84,10 +88,32 @@ module Flexr
84
88
  options.color = required_argument!(args, argument).to_sym
85
89
  when "--format"
86
90
  options.format = required_argument!(args, argument).to_sym
91
+ when "--target"
92
+ raise ArgumentError, "--target is only valid for the check command" unless command == :check
93
+ options.target = required_argument!(args, argument)
87
94
  when "--rule"
88
95
  rule_number = Integer(required_argument!(args, argument), 10)
89
96
  raise ArgumentError, "--rule must be non-negative" if rule_number.negative?
90
- when "--input-file", "--baseline", "--iterations"
97
+ when "--grammar"
98
+ raise ArgumentError, "--grammar is only valid for check" unless command == :check
99
+ options.grammar = required_argument!(args, argument)
100
+ when "--input", "--generated"
101
+ raise ArgumentError, "#{argument} is only valid for the trace command" unless command == :trace
102
+ if argument == "--generated"
103
+ trace_args[:generated] = true
104
+ else
105
+ trace_args[:input] = args.shift || raise(ArgumentError, "--input requires a value")
106
+ end
107
+ when "--input-file"
108
+ value = required_argument!(args, argument)
109
+ if command == :trace
110
+ trace_args[:input_file] = value
111
+ elsif command == :bench
112
+ benchmark_args.push(argument, value)
113
+ else
114
+ raise ArgumentError, "#{argument} is only valid for the trace or bench command"
115
+ end
116
+ when "--baseline", "--iterations"
91
117
  raise ArgumentError, "#{argument} is only valid for the bench command" unless command == :bench
92
118
  benchmark_args.push(argument, required_argument!(args, argument))
93
119
  else
@@ -96,14 +122,17 @@ module Flexr
96
122
  end
97
123
 
98
124
  options.validate!
99
- [options, output, rule_number, benchmark_args, positionals]
125
+ raise ArgumentError, "choose --input or --input-file" if trace_args.key?(:input) && trace_args[:input_file]
126
+ raise ArgumentError, "--generated requires --input or --input-file" if
127
+ trace_args[:generated] && !trace_args.key?(:input) && !trace_args[:input_file]
128
+ [options, output, rule_number, benchmark_args, positionals, trace_args]
100
129
  rescue ArgumentError => e
101
130
  raise e if e.message.start_with?("unsupported ", "--rule")
102
131
 
103
132
  raise ArgumentError, "invalid option value: #{e.message}"
104
133
  end
105
134
 
106
- def execute(command, spec, options, output, rule_number, benchmark_args, out, err)
135
+ def execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: {})
107
136
  case command
108
137
  when :check
109
138
  check(spec, options, out)
@@ -111,14 +140,24 @@ module Flexr
111
140
  print_stats(spec, options, out)
112
141
  when :tokens
113
142
  parsed = read_spec(spec)
114
- out.puts Array(parsed.config[:declared_tokens]).join(" ")
143
+ tokens = Array(parsed.config[:declared_tokens])
144
+ if options.format == :lrama
145
+ require_relative "grammar"
146
+ out.puts Grammar.declarations(tokens)
147
+ else
148
+ out.puts tokens.join(" ")
149
+ end
115
150
  EXIT_OK
116
151
  when :dot
117
152
  print_dot(spec, options, out)
118
153
  when :explain
119
154
  print_explanation(spec, rule_number, out)
120
155
  when :trace
121
- print_trace(spec, options, out)
156
+ if trace_args.key?(:input) || trace_args[:input_file]
157
+ print_input_trace(spec, options, trace_args, out)
158
+ else
159
+ print_trace(spec, options, out)
160
+ end
122
161
  when :bench
123
162
  run_benchmark(spec, options, benchmark_args, out)
124
163
  when :import
@@ -146,18 +185,26 @@ module Flexr
146
185
  generator.generate
147
186
  generator.diagnostics
148
187
  else
149
- compile(read_spec(spec), overrides: options.overrides)
188
+ parsed = read_spec(spec)
189
+ compile(parsed, overrides: options.overrides)
150
190
  end
151
191
  diagnostics = (compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)).dup
192
+ if options.grammar
193
+ require_relative "grammar"
194
+ diagnostics.concat(Grammar.diagnostics(read_spec(spec), options.grammar))
195
+ end
152
196
  diagnostics.select! do |diagnostic|
153
- options.warn_level == :all || (options.warn_level == :default && diagnostic.code != "FLEXR-W016")
197
+ options.warn_level == :all ||
198
+ (options.warn_level == :default && diagnostic.severity != :info && diagnostic.code != "FLEXR-W016")
154
199
  end
200
+ diagnostics.concat(Migration.diagnostics(parsed)) if options.target
155
201
  set = DiagnosticSet.new
156
202
  diagnostics.each { |diagnostic| set << diagnostic }
157
203
  if options.format == :json
158
204
  out.puts set.render(format: :json)
159
205
  elsif !diagnostics.empty?
160
- out.puts set.render(format: :human, color: options.color)
206
+ out.puts set.render(format: :human, color: options.color, path: spec,
207
+ source: diagnostic_source(spec, normalize: options.eval_mode))
161
208
  end
162
209
  return EXIT_FAILURE if options.warn_as_error && diagnostics.any?(&:warning?)
163
210
 
@@ -257,6 +304,34 @@ module Flexr
257
304
  EXIT_OK
258
305
  end
259
306
 
307
+ def print_input_trace(spec, options, trace_args, out)
308
+ require_relative "trace"
309
+ parsed = read_spec(spec)
310
+ source = trace_args[:generated] ? generate(spec, options) : File.read(spec)
311
+ scope = Module.new
312
+ scope.module_eval(source, spec, 1)
313
+ namespace = parsed.class_name.start_with?("::") ? Object : scope
314
+ klass = parsed.class_name.split("::").reject(&:empty?).inject(namespace) { |parent, name| parent.const_get(name, false) }
315
+ unless trace_args[:generated]
316
+ options.overrides.each do |name, value|
317
+ klass.public_send(name, value) if %i[backend token_kind accel].include?(name)
318
+ end
319
+ klass.__flexr_config.options.merge!(options.overrides.slice(:max_dfa_states, :warn_level))
320
+ compiled = klass.compile!
321
+ if options.warn_as_error
322
+ diagnostic = Array(compiled.diagnostics).find do |item|
323
+ item.warning? && (options.warn_level == :all || item.code != "FLEXR-W016")
324
+ end
325
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic) if diagnostic
326
+ end
327
+ end
328
+ input = trace_args[:input_file] ? File.binread(trace_args[:input_file]) : trace_args.fetch(:input)
329
+ trace = Trace.new(klass.new(input, filename: trace_args[:input_file]))
330
+ trace.run
331
+ out.puts(options.format == :json ? JSON.pretty_generate(steps: trace.steps) : trace.render)
332
+ EXIT_OK
333
+ end
334
+
260
335
  def run_benchmark(spec, options, benchmark_args, out)
261
336
  require_relative "../../benchmark/run"
262
337
  args = ["--spec", spec, *benchmark_args]
@@ -285,12 +360,12 @@ module Flexr
285
360
  klass.option(name) if value
286
361
  end
287
362
  klass.__flexr_config.options[:max_dfa_states] = overrides[:max_dfa_states] if overrides[:max_dfa_states]
363
+ klass.__flexr_config.options[:warn_level] = overrides[:warn_level] if overrides[:warn_level]
288
364
  accel = overrides.fetch(:accel, config_options[:accel])
289
365
  klass.accel(accel) if accel
290
366
  parsed.states.each do |name, value|
291
- next if name.to_sym == :initial
292
-
293
- klass.state(name, inclusive: value[:inclusive]) { nil }
367
+ klass.state(name, inclusive: value[:inclusive]) { nil } unless name.to_sym == :initial
368
+ klass.__flexr_states.fetch(name.to_sym).location = value[:location]
294
369
  end
295
370
  parsed.rules.each { |rule| klass.__flexr_add_generated_rule(rule.to_h) }
296
371
  klass.compile!
@@ -303,10 +378,30 @@ module Flexr
303
378
  value
304
379
  end
305
380
 
306
- def render_error(error, options: nil)
381
+ def diagnostic_source(path, normalize: false, max_bytes: nil)
382
+ return unless path
383
+
384
+ source = File.binread(path, max_bytes && (max_bytes + 1))
385
+ return if max_bytes && source.bytesize > max_bytes
386
+
387
+ source.force_encoding(Encoding::UTF_8)
388
+ normalize ? source.gsub(/\r\n?/, "\n") : source
389
+ rescue SystemCallError, IOError
390
+ nil
391
+ end
392
+
393
+ def render_error(error, options: nil, path: nil, normalize_source: false)
307
394
  if error.respond_to?(:diagnostic) && error.diagnostic
395
+ if error.is_a?(LexError) || (error.is_a?(Error) && error.byte_pos)
396
+ path = error.filename
397
+ normalize_source = false
398
+ # shortcut: omit input excerpts above 64 KiB; use bounded line reads if larger inputs need excerpts.
399
+ max_bytes = 64 * 1024
400
+ end
401
+ source = diagnostic_source(path, normalize: normalize_source, max_bytes: max_bytes) if
402
+ error.diagnostic.location && options&.format != :json
308
403
  return DiagnosticSet.new.tap { |set| set << error.diagnostic }.render(
309
- format: options&.format || :human, color: options&.color || :auto
404
+ format: options&.format || :human, color: options&.color || :auto, source: source, path: path
310
405
  )
311
406
  end
312
407
 
@@ -340,7 +435,12 @@ module Flexr
340
435
  -W, --warn LEVEL all | default | none
341
436
  --warn-as-error
342
437
  --color WHEN auto | always | never
343
- --format FMT human | json
438
+ --format FMT human | json | lrama (tokens only)
439
+ --grammar PATH compare emits with grammar %token declarations (check only)
440
+ --input STRING trace input (executes SPEC actions)
441
+ --input-file PATH trace or benchmark input
442
+ --generated trace generated scanner
443
+ --target 2.0 report migration requirements (static check only)
344
444
  USAGE
345
445
  status
346
446
  end
@@ -3,14 +3,8 @@
3
3
  module Flexr
4
4
  module Codegen
5
5
  class Firstmatch < Base
6
- def initialize(compiled, experimental: false)
7
- raise CompileError, "firstmatch requires option :experimental" unless experimental
8
-
9
- super(compiled)
10
- end
11
-
12
6
  def generate
13
- compiled.rules.map { |rule| rule.patterns }.flatten.freeze
7
+ compiled.rules.flat_map(&:patterns).freeze
14
8
  end
15
9
  end
16
10
  end
@@ -27,11 +27,13 @@ module Flexr
27
27
 
28
28
  def source(indent: " ")
29
29
  lines = []
30
+ whole_tokens = compiled.machines.values.any?(&:whole_tokens)
30
31
  lines << "#{indent}def scan_one"
31
32
  lines << "#{indent} if @__flexr_generated_scan_state != @state"
32
33
  lines << "#{indent} machine = self.class.__flexr_compiled.machines.fetch(@state)"
33
34
  lines << "#{indent} @__flexr_generated_scan_state = @state"
34
35
  lines << "#{indent} @__flexr_generated_scan_dfa = machine.dfa"
36
+ lines << "#{indent} @__flexr_generated_whole_tokens = machine.respond_to?(:whole_tokens) ? machine.whole_tokens : nil" if whole_tokens
35
37
  lines << "#{indent} @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)"
36
38
  lines << "#{indent} @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)"
37
39
  lines << "#{indent} end"
@@ -39,6 +41,10 @@ module Flexr
39
41
  lines << "#{indent} return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast"
40
42
  lines << "#{indent} position = byte_pos"
41
43
  lines << "#{indent} return nil unless valid_utf8_at?(position)"
44
+ if whole_tokens
45
+ lines << "#{indent} match = __flexr_whole_token_match(@__flexr_generated_whole_tokens, position) if @__flexr_generated_whole_tokens"
46
+ lines << "#{indent} return match if match"
47
+ end
42
48
  lines << "#{indent} return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple"
43
49
  lines << "#{indent} cursor = position"
44
50
  lines << "#{indent} current = dfa.start"
@@ -201,8 +207,8 @@ module Flexr
201
207
  lines << "#{indent} @__flexr_generated_accel_scanner.pos = segment_position"
202
208
  lines << "#{indent} @__flexr_generated_accel_scanner.skip(regexp)"
203
209
  lines << "#{indent} else"
204
- lines << "#{indent} match = regexp.match(segment, segment_position)"
205
- lines << "#{indent} match && match.begin(0) == segment_position ? match.end(0) - segment_position : nil"
210
+ lines << "#{indent} match = regexp.match(segment.byteslice(segment_position..).to_s, 0)"
211
+ lines << "#{indent} match && match.begin(0).zero? ? match[0].bytesize : nil"
206
212
  lines << "#{indent} end"
207
213
  lines << "#{indent} rescue ArgumentError"
208
214
  lines << "#{indent} nil"
@@ -21,6 +21,14 @@ module Flexr
21
21
  validated_symbol!(value, ACCELERATORS, :accel)
22
22
  end
23
23
 
24
+ def encoding!(value)
25
+ encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
26
+ return encoding if [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
27
+
28
+ diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
29
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
30
+ end
31
+
24
32
  def option!(value)
25
33
  validated_symbol!(value, BOOLEAN_OPTIONS, :option)
26
34
  end