flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -39,7 +39,7 @@ module Flexr
39
39
  def normalize_leaf(node)
40
40
  case node
41
41
  when AST::CodepointRange
42
- byte_sequences_for_ranges(casefold_ranges(node.lo, node.hi), loc: node.loc)
42
+ byte_sequences_for_ranges([[node.lo, node.hi]], loc: node.loc)
43
43
  else
44
44
  node
45
45
  end
@@ -48,11 +48,11 @@ module Flexr
48
48
  def char_class(node)
49
49
  ranges = node.ranges.flat_map do |range|
50
50
  if range.first == AST::Property
51
- property_ranges = Unicode::Property.ranges(range.last)
52
- property_ranges = property_ranges.flat_map { |lo, hi| casefold_ranges(lo, hi) }
51
+ property_ranges = Unicode::Property.ranges(range[2])
52
+ property_ranges = property_ranges.flat_map { |lo, hi| Unicode::CaseFold.ranges(lo, hi) } if range[3]
53
53
  range[1] ? complement(property_ranges) : property_ranges
54
54
  else
55
- casefold_ranges(*range)
55
+ [range]
56
56
  end
57
57
  end
58
58
  ranges = complement(ranges) if node.negated
@@ -154,10 +154,6 @@ module Flexr
154
154
  result
155
155
  end
156
156
 
157
- def casefold_ranges(lo, hi)
158
- return [[lo, hi]] if @options.nobits?(::Regexp::IGNORECASE)
159
- Unicode::CaseFold.ranges(lo, hi)
160
- end
161
157
  end
162
158
  end
163
159
  end
@@ -104,6 +104,7 @@ module Flexr
104
104
  def parse_atom
105
105
  starting = @index
106
106
  @escaped_value = false
107
+ @escaped_byte = false
107
108
  @last_ranges = nil
108
109
  return parse_group if consume?("(")
109
110
  return parse_class if consume?("[")
@@ -119,6 +120,7 @@ module Flexr
119
120
  if consume?("\\")
120
121
  parse_escape
121
122
  return AST::CharClass.new(ranges: @last_ranges, negated: false, loc: span(starting)) if @last_ranges
123
+ return AST::ByteRange.new(lo: @last_byte, hi: @last_byte, loc: span(starting)) if @escaped_byte
122
124
  return codepoint_node(read_codepoint, span(starting)) if @escaped_value
123
125
  end
124
126
 
@@ -169,8 +171,30 @@ module Flexr
169
171
  end
170
172
  raise_syntax("invalid inline option group")
171
173
  end
172
- raise unsupported("look-around", "use followed_by: or a state instead") if peek_prefix?("=") || peek_prefix?("!") || peek_prefix?("<=") || peek_prefix?("<!")
173
- raise unsupported("atomic groups", "rewrite the expression as a DFA-compatible expression") if consume?(">")
174
+ if peek_prefix?("=")
175
+ raise unsupported("look-around",
176
+ "for a trailing positive lookahead, rewrite rule(/a(?=b)/) as rule(/a/, followed_by: /b/); keep the action")
177
+ end
178
+ if peek_prefix?("!") || peek_prefix?("<=") || peek_prefix?("<!")
179
+ raise unsupported("look-around",
180
+ "use separate rules or states; followed_by: cannot express negative lookahead or lookbehind")
181
+ end
182
+ if consume?(">")
183
+ raise unsupported("atomic groups",
184
+ 'for unescaped single-line quoted text, a DFA-compatible rewrite is rule(/"[^"\n]*"/); keep the action; removing atomic groups can change matches')
185
+ end
186
+ if peek_prefix?("<") || peek_prefix?("'")
187
+ raise unsupported("named captures",
188
+ "when only grouping is needed, rewrite rule(/(?<word>ab)/) as rule(/(?:ab)/); keep the action; capture-dependent patterns need a separate rewrite")
189
+ end
190
+ if peek_prefix?("(")
191
+ raise unsupported("conditional groups",
192
+ "for this optional-prefix conditional, rewrite rule(/(a)?(?(1)b|c)/) as rule(/(?:ab|c)/); keep the action; other conditions need separate rules or state checks")
193
+ end
194
+ if peek_prefix?("~")
195
+ raise unsupported("absence groups",
196
+ "when excluding a single literal, rewrite rule(/x(?~a)/) as rule(/x[^a]*/); keep the action; larger absence expressions need a separate rewrite")
197
+ end
174
198
  raise unsupported("unsupported group syntax", "use a non-capturing group (?:...)")
175
199
  end
176
200
 
@@ -248,7 +272,7 @@ module Flexr
248
272
  raw_name = read_until(":]")
249
273
  inner_negated = raw_name.start_with?("^")
250
274
  name = raw_name.delete_prefix("^")
251
- return [[AST::Property, inner_negated, "POSIX_#{name}"]] if
275
+ return [[AST::Property, inner_negated, "POSIX_#{name}", ignorecase?]] if
252
276
  @encoding != Encoding::BINARY && POSIX_CLASSES.include?(name)
253
277
 
254
278
  ranges = case name
@@ -273,6 +297,7 @@ module Flexr
273
297
  def parse_class_atom
274
298
  if consume?("\\")
275
299
  parse_escape
300
+ return [[@last_byte, @last_byte]] if @escaped_byte
276
301
  return [[@last_codepoint, @last_codepoint]] if @escaped_value
277
302
  return @last_ranges
278
303
  end
@@ -292,7 +317,9 @@ module Flexr
292
317
  def materialize_class_ranges(ranges, negated:)
293
318
  concrete = ranges.flat_map do |range|
294
319
  if range.first == AST::Property
295
- Unicode::Property.ranges(range[2], negate: range[1])
320
+ property = Unicode::Property.ranges(range[2])
321
+ property = fold_ranges(property) if range[3]
322
+ range[1] ? complement_ranges(property) : property
296
323
  else
297
324
  [range]
298
325
  end
@@ -322,17 +349,24 @@ module Flexr
322
349
 
323
350
  def parse_escape
324
351
  @escaped_value = false
352
+ @escaped_byte = false
325
353
  @last_ranges = nil
326
354
  char = advance_raw
327
355
  raise_syntax("trailing backslash") unless char
328
356
  if ESCAPES.key?(char)
329
- raise unsupported("octal escapes", "use an explicit hexadecimal escape") if
330
- char == "0" && current&.match?(/[0-9]/)
357
+ if char == "0" && current&.match?(/[0-9]/)
358
+ raise unsupported("octal escapes",
359
+ 'use an explicit hexadecimal escape: rewrite rule(/\011/) as rule(/\x09/) for a tab; keep the action')
360
+ end
331
361
 
332
362
  @escaped_value = true
333
363
  @last_codepoint = ESCAPES.fetch(char)
334
364
  return
335
365
  end
366
+ if @class_depth.positive? && %w[A B G z Z K g X].include?(char)
367
+ raise unsupported("\\#{char}",
368
+ "inside a character class this is a literal character: rewrite rule(/[\\#{char}]/) as rule(/[#{char}]/); keep the action")
369
+ end
336
370
  case char
337
371
  when "d", "D", "w", "W", "s", "S", "h", "H"
338
372
  ranges = shorthand_ranges(char)
@@ -341,16 +375,15 @@ module Flexr
341
375
  when "p", "P"
342
376
  expect("{")
343
377
  name = read_until("}")
344
- ranges = [[AST::Property, char == "P", name]]
378
+ ranges = [[AST::Property, char == "P", name, ignorecase?]]
345
379
  @last_ranges = ranges
346
380
  nil
347
381
  when "x"
348
- digits = if consume?("{")
349
- read_until("}")
382
+ if consume?("{")
383
+ assign_codepoint(read_until("}"))
350
384
  else
351
- read_exact(2)
385
+ assign_hex_byte(read_exact(2))
352
386
  end
353
- assign_codepoint(digits)
354
387
  nil
355
388
  when "u"
356
389
  digits = if consume?("{")
@@ -362,28 +395,69 @@ module Flexr
362
395
  assign_codepoint(digits)
363
396
  nil
364
397
  when "b"
365
- raise unsupported("\\b", "use an explicit token boundary rule") unless @class_depth.positive?
398
+ unless @class_depth.positive?
399
+ raise unsupported("\\b",
400
+ 'when surrounding literals already establish the boundary, rewrite rule(/foo\b:/) as rule(/foo:/); keep the action; token edges need separate rules or state checks')
401
+ end
366
402
 
367
403
  @escaped_value = true
368
404
  @last_codepoint = 0x08
369
- when "G", "K", "B", "A", "z", "Z", "1", "2", "3", "4", "5", "6", "7", "8", "9",
370
- "g", "R", "X", "c", "C", "M"
371
- raise unsupported("\\#{char}", "use a state or followed_by: instead")
372
- when "k"
373
- read_until(">") if consume?("<")
374
- raise unsupported("backreferences", "split the rule into DFA-compatible states")
405
+ when "B"
406
+ raise unsupported("\\B",
407
+ 'when surrounding literals already establish the non-boundary, rewrite rule(/foo\Bbar/) as rule(/foobar/); keep the action; token edges need separate rules or state checks')
408
+ when "G"
409
+ raise unsupported("\\G",
410
+ 'at the start of a rule, rewrite rule(/\Ga/) as rule(/a/); keep the action; rules already match at the current lexer position; interior \G needs a separate rewrite')
411
+ when "A"
412
+ raise unsupported("\\A",
413
+ 'for input without line feeds, rewrite rule(/\Aa/) as rule(/^a/); keep the action; line boundaries differ from string boundaries')
414
+ when "z", "Z"
415
+ raise unsupported("\\#{char}",
416
+ "for input without line feeds, rewrite rule(/a\\#{char}/) as rule(/a$/); keep the action; line boundaries differ from string boundaries")
417
+ when "R"
418
+ example = if @encoding == Encoding::BINARY
419
+ 'rule(/\r\n|[\n\v\f\r]/n)'
420
+ else
421
+ 'rule(/\r\n|[\n\v\f\r\u0085\u2028\u2029]/)'
422
+ end
423
+ raise unsupported("\\R", "for a standalone line-break token, use #{example}; keep the action")
424
+ when "K"
425
+ raise unsupported("\\K",
426
+ 'for /a\Kb/ in :initial, use rule(/a/, followed_by: /b/) { begin_state :suffix }; state(:suffix) { rule(/b/) { begin_state :initial; emit :B } }; keep the action on the suffix rule after returning to :initial; choose an unused state name')
427
+ when "g"
428
+ raise unsupported("\\g",
429
+ 'for this non-recursive call, rewrite rule(/(a|b)\g<1>/) as rule(/[ab]{2}/); keep the action; recursive calls need a parser or state stack')
430
+ when "X"
431
+ raise unsupported("\\X",
432
+ 'for a standalone grapheme token with ASCII-only input, rewrite rule(/\X/) as rule(/\r\n|[\x00-\x7f]/); keep the action; Unicode graphemes need separate segmentation')
433
+ when "c", "C"
434
+ raise unsupported("\\#{char}",
435
+ 'for control-A, rewrite rule(/\cA/) or rule(/\C-A/) as rule(/\x01/); keep the action; other control characters need their own hexadecimal code')
436
+ when "M"
437
+ raise unsupported("\\M",
438
+ 'for binary input, rewrite rule(/\M-a/n) as rule(/\xE1/n); keep the action; other meta characters need their own byte value')
439
+ when "1".."9", "k"
440
+ read_until(">") if char == "k" && consume?("<")
441
+ feature = char == "k" ? "backreferences" : "\\#{char}"
442
+ help = if @class_depth.positive? && char != "k"
443
+ 'for a character code, use an explicit hexadecimal escape such as rule(/[\x01]/); keep the action'
444
+ else
445
+ 'for a capture limited to a or b, rewrite rule(/(a|b)\1/) as rule(/(?:aa|bb)/) and keep the action; otherwise split the rule into states and compare matched text in actions'
446
+ end
447
+ raise unsupported(feature, help)
375
448
  when *LITERAL_ESCAPES
376
449
  @escaped_value = true
377
450
  @last_codepoint = char.ord
378
451
  else
379
- raise unsupported("unknown escape \\#{char}", "remove the backslash or use a supported escape")
452
+ raise unsupported("unknown escape \\#{char}",
453
+ 'if q is literal, rewrite rule(/\q/) as rule(/q/); otherwise use a supported escape; keep the action')
380
454
  end
381
455
  end
382
456
 
383
457
  def shorthand_ranges(char)
384
458
  if @unicode && @encoding != Encoding::BINARY
385
459
  property = { "d" => "Nd", "w" => "Word", "s" => "Space" }.fetch(char.downcase, nil)
386
- return [[AST::Property, char == char.upcase, property]] if property
460
+ return [[AST::Property, char == char.upcase, property, ignorecase?]] if property
387
461
  end
388
462
 
389
463
  base = {
@@ -468,7 +542,8 @@ module Flexr
468
542
  def reject_postfix_quantifier
469
543
  return unless ["?", "+"].include?(current)
470
544
 
471
- raise unsupported("lazy or possessive quantifier", "use a negated character class")
545
+ raise unsupported("lazy or possessive quantifier",
546
+ 'for unescaped single-line quoted text, use a negated character class: rule(/"[^"\n]*"/); keep the action; removing lazy or possessive modifiers can change matches')
472
547
  end
473
548
 
474
549
  def merge_ranges(ranges)
@@ -574,6 +649,37 @@ module Flexr
574
649
  @last_codepoint = codepoint
575
650
  end
576
651
 
652
+ def assign_hex_byte(digits)
653
+ byte = digits.to_i(16)
654
+ if @encoding == Encoding::BINARY
655
+ @escaped_byte = true
656
+ @last_byte = byte
657
+ return
658
+ end
659
+
660
+ length = case byte
661
+ when 0x00..0x7f then 1
662
+ when 0xc2..0xdf then 2
663
+ when 0xe0..0xef then 3
664
+ when 0xf0..0xf4 then 4
665
+ else 0
666
+ end
667
+ bytes = [byte]
668
+ (length - 1).times do
669
+ expect("\\x")
670
+ bytes << read_exact(2).to_i(16)
671
+ end
672
+ value = bytes.pack("C*").force_encoding(Encoding::UTF_8)
673
+ raise_syntax("invalid UTF-8 byte escape") unless length.positive? && value.valid_encoding?
674
+
675
+ @escaped_value = true
676
+ @last_codepoint = value.ord
677
+ end
678
+
679
+ def ignorecase?
680
+ @options.anybits?(::Regexp::IGNORECASE)
681
+ end
682
+
577
683
  def skip_extended_space
578
684
  loop do
579
685
  @index += 1 while @source[@index]&.match?(/\s/)
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
  # Generated by flexr. DO NOT EDIT.
3
3
  # source: lib/flexr/regexp/tokenizer.flexr.rb
4
- # spec-digest: sha256:ef176b30fa1aec18f8474cdc44dfc9004802e14e4d7ed89e9a20890930811c63
5
- # artifact-schema: 1
6
- # runtime-abi: 1
4
+ # spec-digest: sha256:a9c3e840a6b33c0cb45b25200a372b0ce4f578497f96cc58a591ccd751954f16
5
+ # artifact-schema: 2
6
+ # runtime-abi: 2
7
7
  # unicode: 15.1.0
8
8
  # backend: table
9
9
  # compiled: true
@@ -17,7 +17,7 @@ module Flexr
17
17
  # This specification is intentionally small: it is a dogfood target for
18
18
  # the source lexer path, not the parser's implementation.
19
19
  class SourceLexer < Flexr::Lexer
20
- Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 1, compiler_version: "1.1.0", runtime_abi_version: 1, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: proc{ emit :PROPERTY, text.byteslice(3...-1) }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: proc{ emit :ESCAPE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: proc{ emit :CHAR_CLASS, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: proc{ emit :QUANTIFIER, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: proc{ emit :ALTERNATION, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_OPEN, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: proc{ emit :GROUP_CLOSE, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: proc{ emit :ANCHOR, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: proc{ emit :DOT, text }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: proc{ emit :LITERAL, text }, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
20
+ Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "2.0.0", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
21
21
  def scan_one
22
22
  if @__flexr_generated_scan_state != @state
23
23
  machine = self.class.__flexr_compiled.machines.fetch(@state)
@@ -192,8 +192,8 @@ module Flexr
192
192
  @__flexr_generated_accel_scanner.pos = segment_position
193
193
  @__flexr_generated_accel_scanner.skip(regexp)
194
194
  else
195
- match = regexp.match(segment, segment_position)
196
- match && match.begin(0) == segment_position ? match.end(0) - segment_position : nil
195
+ match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
196
+ match && match.begin(0).zero? ? match[0].bytesize : nil
197
197
  end
198
198
  rescue ArgumentError
199
199
  nil
@@ -251,25 +251,25 @@ module Flexr
251
251
  case rule.index
252
252
  when 0
253
253
  when 1
254
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
254
+ emit :PROPERTY, text.byteslice(3...-1)
255
255
  when 2
256
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
256
+ emit :ESCAPE, text
257
257
  when 3
258
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
258
+ emit :CHAR_CLASS, text
259
259
  when 4
260
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
260
+ emit :QUANTIFIER, text
261
261
  when 5
262
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
262
+ emit :ALTERNATION, text
263
263
  when 6
264
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
264
+ emit :GROUP_OPEN, text
265
265
  when 7
266
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
266
+ emit :GROUP_CLOSE, text
267
267
  when 8
268
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
268
+ emit :ANCHOR, text
269
269
  when 9
270
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
270
+ emit :DOT, text
271
271
  when 10
272
- instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
272
+ emit :LITERAL, text
273
273
  else
274
274
  instance_exec(&rule.action)
275
275
  end
@@ -278,16 +278,16 @@ module Flexr
278
278
 
279
279
 
280
280
 
281
- __flexr_bind_generated_action(1, proc{ emit :PROPERTY, text.byteslice(3...-1) })
282
- __flexr_bind_generated_action(2, proc{ emit :ESCAPE, text })
283
- __flexr_bind_generated_action(3, proc{ emit :CHAR_CLASS, text })
284
- __flexr_bind_generated_action(4, proc{ emit :QUANTIFIER, text })
285
- __flexr_bind_generated_action(5, proc{ emit :ALTERNATION, text })
286
- __flexr_bind_generated_action(6, proc{ emit :GROUP_OPEN, text })
287
- __flexr_bind_generated_action(7, proc{ emit :GROUP_CLOSE, text })
288
- __flexr_bind_generated_action(8, proc{ emit :ANCHOR, text })
289
- __flexr_bind_generated_action(9, proc{ emit :DOT, text })
290
- __flexr_bind_generated_action(10, proc{ emit :LITERAL, text })
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
291
  end
292
292
  end
293
293
  end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rspec/expectations"
4
+ require_relative "testing"
5
+
6
+ RSpec::Matchers.define :tokenize do |input, **options|
7
+ chain :as do |tokens|
8
+ @expected = tokens
9
+ @comparison = :tokens
10
+ end
11
+
12
+ chain :matching_snapshot do |path, snapshot_options = {}|
13
+ @snapshot_path = path
14
+ @update_snapshot = snapshot_options.fetch(:update, false)
15
+ @comparison = :snapshot
16
+ end
17
+
18
+ match do |lexer|
19
+ raise ArgumentError, "use .as(tokens) or .matching_snapshot(path)" unless @comparison
20
+
21
+ @actual = lexer.new(input, **options).tokens
22
+ if @comparison == :snapshot
23
+ @actual = Flexr::Testing::Snapshot.render(@actual)
24
+ begin
25
+ @expected = Flexr::Testing::Snapshot.expected(@snapshot_path, @actual, update: @update_snapshot)
26
+ rescue Errno::ENOENT
27
+ @missing_snapshot = true
28
+ next false
29
+ end
30
+ end
31
+ @actual == @expected
32
+ end
33
+
34
+ failure_message do
35
+ if @missing_snapshot
36
+ "missing token snapshot #{@snapshot_path}; create it with update: true"
37
+ elsif @comparison == :snapshot
38
+ "expected tokens to match snapshot #{@snapshot_path}"
39
+ else
40
+ "expected tokens #{@expected.inspect}, got #{@actual.inspect}"
41
+ end
42
+ end
43
+
44
+ failure_message_when_negated do
45
+ "expected tokens to differ from #{@comparison == :snapshot ? @snapshot_path : @expected.inspect}"
46
+ end
47
+
48
+ diffable
49
+ define_method(:expected) { @expected }
50
+ define_method(:actual) { @actual }
51
+ end
@@ -6,6 +6,7 @@ module Flexr
6
6
  max_lookahead_size: nil, max_buffer_size: 64 * 1024 * 1024,
7
7
  max_state_stack: 1024, max_steps: nil, cancellation: nil, retain_input: true,
8
8
  chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
9
+ raise ArgumentError, "unsupported error_mode: #{error_mode.inspect}" unless %i[raise token panic].include?(error_mode)
9
10
  raise ArgumentError, "max_token_size must be non-negative" if max_token_size.to_i.negative?
10
11
  raise ArgumentError, "max_lookahead_size must be non-negative" if max_lookahead_size&.to_i&.negative?
11
12
  raise ArgumentError, "max_buffer_size must be non-negative" if max_buffer_size.to_i.negative?
@@ -37,7 +38,7 @@ module Flexr
37
38
  @max_lookahead_size = (max_lookahead_size || max_token_size).to_i
38
39
  @max_state_stack = max_state_stack.to_i
39
40
  @max_steps = max_steps&.to_i
40
- @cancellation = cancellation
41
+ @cancellation = cancellation.respond_to?(:arity) ? cancellation : cancellation&.method(:call)
41
42
  @steps = 0
42
43
  @position = 0
43
44
  @line = 1
@@ -176,7 +177,7 @@ module Flexr
176
177
  def text
177
178
  return @matched if @matched
178
179
 
179
- @matched = @buffer.byteslice(@text_start...@match_end)
180
+ @matched = @buffer.byteslice(@text_start, @match_end - @text_start)
180
181
  end
181
182
 
182
183
  def text_bytesize
@@ -196,7 +197,7 @@ module Flexr
196
197
 
197
198
  raise Runtime::TokenTooLargeError.new(
198
199
  filename: @filename, byte_pos: @more_start || @position, line: @line,
199
- rule: rule_index(rule || @active_rule)
200
+ rule: diagnostic_rule(rule || @active_rule)
200
201
  )
201
202
  end
202
203
 
@@ -205,7 +206,7 @@ module Flexr
205
206
 
206
207
  raise Runtime::LookaheadTooLargeError.new(
207
208
  filename: @filename, byte_pos: @position, line: @line,
208
- rule: rule_index(rule || @active_rule)
209
+ rule: diagnostic_rule(rule || @active_rule)
209
210
  )
210
211
  end
211
212
 
@@ -215,13 +216,13 @@ module Flexr
215
216
  @steps += count
216
217
  if @max_steps && @steps > @max_steps
217
218
  raise Runtime::StepLimitError.new(
218
- filename: @filename, byte_pos: @position, line: @line, rule: rule_index(@active_rule)
219
+ filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
219
220
  )
220
221
  end
221
222
  return unless cancelled?
222
223
 
223
224
  raise Runtime::CancelledError.new(
224
- filename: @filename, byte_pos: @position, line: @line, rule: rule_index(@active_rule)
225
+ filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
225
226
  )
226
227
  end
227
228
 
@@ -233,6 +234,31 @@ module Flexr
233
234
  @token_limit_required
234
235
  end
235
236
 
237
+ def __flexr_whole_token_match(plans, position)
238
+ return unless plans && defined?(::StringScanner) && !scan_steps_guarded? && !token_limit_required?
239
+
240
+ source = @buffer.stable_source
241
+ return unless source&.frozen?
242
+
243
+ byte = source.getbyte(position)
244
+ plan = plans[byte] if byte
245
+ return unless plan
246
+
247
+ scanner = @__flexr_whole_scanner ||= ::StringScanner.new(source)
248
+ scanner.pos = position
249
+ length = scanner.skip(plan.regexp)
250
+ return unless length
251
+
252
+ match = @__flexr_whole_match ||= Runtime::Match.new
253
+ match.rule = self.class.__flexr_rules.fetch(plan.rule_index)
254
+ match.start_pos = position
255
+ match.end_pos = position + length
256
+ match.total_end_pos = match.end_pos
257
+ match
258
+ rescue ArgumentError
259
+ nil
260
+ end
261
+
236
262
  def utf8_input?
237
263
  self.class.__flexr_config.encoding == Encoding::UTF_8
238
264
  end
@@ -291,7 +317,7 @@ module Flexr
291
317
 
292
318
  raise Runtime::InvalidRecoveryActionError.new(
293
319
  action: action, filename: @filename, byte_pos: @match_start, line: @text_line,
294
- text: text, rule: rule_index(@active_rule)
320
+ text: text, rule: @active_rule
295
321
  )
296
322
  end
297
323
  case @error_mode
@@ -311,9 +337,10 @@ module Flexr
311
337
  def reject
312
338
  diagnostic = Diagnostics.error(
313
339
  "FLEXR-E013", "reject is not supported by flexr",
340
+ location: [@position, @position],
314
341
  help: "use a state transition and less(n) to express the fallback"
315
342
  )
316
- raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
343
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
317
344
  end
318
345
 
319
346
  def push(name)
@@ -322,7 +349,7 @@ module Flexr
322
349
  raise Runtime::StateStackOverflowError.new(
323
350
  "state stack exceeds max_state_stack (#{@max_state_stack})",
324
351
  filename: @filename, byte_pos: @position, line: @line, text: text,
325
- rule: rule_index(@active_rule)
352
+ rule: @active_rule
326
353
  )
327
354
  end
328
355
 
@@ -443,6 +470,7 @@ module Flexr
443
470
  @text_start = @match_start
444
471
  @text_line = @line
445
472
  @text_column = @column
473
+ @matched = nil
446
474
  @position += 1
447
475
  @non_progress_signatures.clear
448
476
  advance_location!(@match_start, @match_end)
@@ -464,8 +492,8 @@ module Flexr
464
492
  def ensure_state!(name)
465
493
  return if self.class.__flexr_config.states.key?(name.to_sym)
466
494
 
467
- diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}")
468
- raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
495
+ diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}", location: [@position, @position])
496
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
469
497
  end
470
498
 
471
499
  def ensure_progress!(match, scan_state, empty_match:)
@@ -488,7 +516,7 @@ module Flexr
488
516
 
489
517
  raise Runtime::NonProgressError.new(
490
518
  filename: @filename, byte_pos: match.start_pos, line: @line,
491
- text: text, rule: match.rule.index
519
+ text: text, rule: match.rule
492
520
  )
493
521
  end
494
522
 
@@ -501,7 +529,7 @@ module Flexr
501
529
  end
502
530
 
503
531
  value = location_value(starting, ending)
504
- location_bytes = value.valid_encoding? ? value : value.b
532
+ location_bytes = value.b
505
533
  newline_count = location_bytes.count("\n")
506
534
  if newline_count.zero?
507
535
  @column += display_length(value)
@@ -516,7 +544,7 @@ module Flexr
516
544
  return [line, column] if ending <= starting
517
545
 
518
546
  value = location_value(starting, ending)
519
- location_bytes = value.valid_encoding? ? value : value.b
547
+ location_bytes = value.b
520
548
  newline_count = location_bytes.count("\n")
521
549
  return [line, column + display_length(value)] if newline_count.zero?
522
550
 
@@ -535,7 +563,7 @@ module Flexr
535
563
  def location_value(starting, ending)
536
564
  return @matched if @matched && starting == @text_start && ending == @match_end
537
565
 
538
- @buffer.byteslice(starting...ending).to_s
566
+ @buffer.byteslice(starting, ending - starting).to_s
539
567
  end
540
568
 
541
569
  def discard_consumed_input!
@@ -548,8 +576,10 @@ module Flexr
548
576
  @cancellation.arity.zero? ? @cancellation.call : @cancellation.call(self)
549
577
  end
550
578
 
551
- def rule_index(rule)
552
- rule.respond_to?(:index) ? rule.index : rule
579
+ def diagnostic_rule(rule)
580
+ return rule unless rule.is_a?(Integer) && rule >= 0
581
+
582
+ self.class.__flexr_rules.fetch(rule, rule)
553
583
  end
554
584
  end
555
585
  end
@@ -10,12 +10,13 @@ module Flexr
10
10
  def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil,
11
11
  rule: nil)
12
12
  @code = CODE
13
- @rule = rule
13
+ @rule = rule.is_a?(IR::Rule) ? rule.index : rule
14
14
  diagnostic = Diagnostics.error(
15
15
  CODE,
16
16
  message,
17
+ location: byte_pos && [byte_pos, byte_pos],
17
18
  help: "increase max_token_size or split the input token",
18
- note: rule.nil? ? nil : "rule #{rule} at byte #{byte_pos}"
19
+ note: rule.nil? ? nil : "#{Diagnostics.rule_label(rule)} at byte #{byte_pos}"
19
20
  )
20
21
  super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
21
22
  end
@@ -26,8 +27,9 @@ module Flexr
26
27
 
27
28
  def initialize(code, message, help:, filename: nil, byte_pos: nil, line: nil, text: nil, rule: nil)
28
29
  @code = code
29
- @rule = rule
30
- diagnostic = Diagnostics.error(code, message, help: help, note: rule.nil? ? nil : "rule #{rule}")
30
+ @rule = rule.is_a?(IR::Rule) ? rule.index : rule
31
+ diagnostic = Diagnostics.error(code, message, location: byte_pos && [byte_pos, byte_pos],
32
+ help: help, note: Diagnostics.rule_label(rule))
31
33
  super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
32
34
  end
33
35
  end