flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -39,7 +39,7 @@ module Flexr
|
|
|
39
39
|
def normalize_leaf(node)
|
|
40
40
|
case node
|
|
41
41
|
when AST::CodepointRange
|
|
42
|
-
byte_sequences_for_ranges(
|
|
42
|
+
byte_sequences_for_ranges([[node.lo, node.hi]], loc: node.loc)
|
|
43
43
|
else
|
|
44
44
|
node
|
|
45
45
|
end
|
|
@@ -48,11 +48,11 @@ module Flexr
|
|
|
48
48
|
def char_class(node)
|
|
49
49
|
ranges = node.ranges.flat_map do |range|
|
|
50
50
|
if range.first == AST::Property
|
|
51
|
-
property_ranges = Unicode::Property.ranges(range
|
|
52
|
-
property_ranges = property_ranges.flat_map { |lo, hi|
|
|
51
|
+
property_ranges = Unicode::Property.ranges(range[2])
|
|
52
|
+
property_ranges = property_ranges.flat_map { |lo, hi| Unicode::CaseFold.ranges(lo, hi) } if range[3]
|
|
53
53
|
range[1] ? complement(property_ranges) : property_ranges
|
|
54
54
|
else
|
|
55
|
-
|
|
55
|
+
[range]
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
ranges = complement(ranges) if node.negated
|
|
@@ -154,10 +154,6 @@ module Flexr
|
|
|
154
154
|
result
|
|
155
155
|
end
|
|
156
156
|
|
|
157
|
-
def casefold_ranges(lo, hi)
|
|
158
|
-
return [[lo, hi]] if @options.nobits?(::Regexp::IGNORECASE)
|
|
159
|
-
Unicode::CaseFold.ranges(lo, hi)
|
|
160
|
-
end
|
|
161
157
|
end
|
|
162
158
|
end
|
|
163
159
|
end
|
data/lib/flexr/regexp/parser.rb
CHANGED
|
@@ -104,6 +104,7 @@ module Flexr
|
|
|
104
104
|
def parse_atom
|
|
105
105
|
starting = @index
|
|
106
106
|
@escaped_value = false
|
|
107
|
+
@escaped_byte = false
|
|
107
108
|
@last_ranges = nil
|
|
108
109
|
return parse_group if consume?("(")
|
|
109
110
|
return parse_class if consume?("[")
|
|
@@ -119,6 +120,7 @@ module Flexr
|
|
|
119
120
|
if consume?("\\")
|
|
120
121
|
parse_escape
|
|
121
122
|
return AST::CharClass.new(ranges: @last_ranges, negated: false, loc: span(starting)) if @last_ranges
|
|
123
|
+
return AST::ByteRange.new(lo: @last_byte, hi: @last_byte, loc: span(starting)) if @escaped_byte
|
|
122
124
|
return codepoint_node(read_codepoint, span(starting)) if @escaped_value
|
|
123
125
|
end
|
|
124
126
|
|
|
@@ -169,8 +171,30 @@ module Flexr
|
|
|
169
171
|
end
|
|
170
172
|
raise_syntax("invalid inline option group")
|
|
171
173
|
end
|
|
172
|
-
|
|
173
|
-
|
|
174
|
+
if peek_prefix?("=")
|
|
175
|
+
raise unsupported("look-around",
|
|
176
|
+
"for a trailing positive lookahead, rewrite rule(/a(?=b)/) as rule(/a/, followed_by: /b/); keep the action")
|
|
177
|
+
end
|
|
178
|
+
if peek_prefix?("!") || peek_prefix?("<=") || peek_prefix?("<!")
|
|
179
|
+
raise unsupported("look-around",
|
|
180
|
+
"use separate rules or states; followed_by: cannot express negative lookahead or lookbehind")
|
|
181
|
+
end
|
|
182
|
+
if consume?(">")
|
|
183
|
+
raise unsupported("atomic groups",
|
|
184
|
+
'for unescaped single-line quoted text, a DFA-compatible rewrite is rule(/"[^"\n]*"/); keep the action; removing atomic groups can change matches')
|
|
185
|
+
end
|
|
186
|
+
if peek_prefix?("<") || peek_prefix?("'")
|
|
187
|
+
raise unsupported("named captures",
|
|
188
|
+
"when only grouping is needed, rewrite rule(/(?<word>ab)/) as rule(/(?:ab)/); keep the action; capture-dependent patterns need a separate rewrite")
|
|
189
|
+
end
|
|
190
|
+
if peek_prefix?("(")
|
|
191
|
+
raise unsupported("conditional groups",
|
|
192
|
+
"for this optional-prefix conditional, rewrite rule(/(a)?(?(1)b|c)/) as rule(/(?:ab|c)/); keep the action; other conditions need separate rules or state checks")
|
|
193
|
+
end
|
|
194
|
+
if peek_prefix?("~")
|
|
195
|
+
raise unsupported("absence groups",
|
|
196
|
+
"when excluding a single literal, rewrite rule(/x(?~a)/) as rule(/x[^a]*/); keep the action; larger absence expressions need a separate rewrite")
|
|
197
|
+
end
|
|
174
198
|
raise unsupported("unsupported group syntax", "use a non-capturing group (?:...)")
|
|
175
199
|
end
|
|
176
200
|
|
|
@@ -248,7 +272,7 @@ module Flexr
|
|
|
248
272
|
raw_name = read_until(":]")
|
|
249
273
|
inner_negated = raw_name.start_with?("^")
|
|
250
274
|
name = raw_name.delete_prefix("^")
|
|
251
|
-
return [[AST::Property, inner_negated, "POSIX_#{name}"]] if
|
|
275
|
+
return [[AST::Property, inner_negated, "POSIX_#{name}", ignorecase?]] if
|
|
252
276
|
@encoding != Encoding::BINARY && POSIX_CLASSES.include?(name)
|
|
253
277
|
|
|
254
278
|
ranges = case name
|
|
@@ -273,6 +297,7 @@ module Flexr
|
|
|
273
297
|
def parse_class_atom
|
|
274
298
|
if consume?("\\")
|
|
275
299
|
parse_escape
|
|
300
|
+
return [[@last_byte, @last_byte]] if @escaped_byte
|
|
276
301
|
return [[@last_codepoint, @last_codepoint]] if @escaped_value
|
|
277
302
|
return @last_ranges
|
|
278
303
|
end
|
|
@@ -292,7 +317,9 @@ module Flexr
|
|
|
292
317
|
def materialize_class_ranges(ranges, negated:)
|
|
293
318
|
concrete = ranges.flat_map do |range|
|
|
294
319
|
if range.first == AST::Property
|
|
295
|
-
Unicode::Property.ranges(range[2]
|
|
320
|
+
property = Unicode::Property.ranges(range[2])
|
|
321
|
+
property = fold_ranges(property) if range[3]
|
|
322
|
+
range[1] ? complement_ranges(property) : property
|
|
296
323
|
else
|
|
297
324
|
[range]
|
|
298
325
|
end
|
|
@@ -322,17 +349,24 @@ module Flexr
|
|
|
322
349
|
|
|
323
350
|
def parse_escape
|
|
324
351
|
@escaped_value = false
|
|
352
|
+
@escaped_byte = false
|
|
325
353
|
@last_ranges = nil
|
|
326
354
|
char = advance_raw
|
|
327
355
|
raise_syntax("trailing backslash") unless char
|
|
328
356
|
if ESCAPES.key?(char)
|
|
329
|
-
|
|
330
|
-
|
|
357
|
+
if char == "0" && current&.match?(/[0-9]/)
|
|
358
|
+
raise unsupported("octal escapes",
|
|
359
|
+
'use an explicit hexadecimal escape: rewrite rule(/\011/) as rule(/\x09/) for a tab; keep the action')
|
|
360
|
+
end
|
|
331
361
|
|
|
332
362
|
@escaped_value = true
|
|
333
363
|
@last_codepoint = ESCAPES.fetch(char)
|
|
334
364
|
return
|
|
335
365
|
end
|
|
366
|
+
if @class_depth.positive? && %w[A B G z Z K g X].include?(char)
|
|
367
|
+
raise unsupported("\\#{char}",
|
|
368
|
+
"inside a character class this is a literal character: rewrite rule(/[\\#{char}]/) as rule(/[#{char}]/); keep the action")
|
|
369
|
+
end
|
|
336
370
|
case char
|
|
337
371
|
when "d", "D", "w", "W", "s", "S", "h", "H"
|
|
338
372
|
ranges = shorthand_ranges(char)
|
|
@@ -341,16 +375,15 @@ module Flexr
|
|
|
341
375
|
when "p", "P"
|
|
342
376
|
expect("{")
|
|
343
377
|
name = read_until("}")
|
|
344
|
-
ranges = [[AST::Property, char == "P", name]]
|
|
378
|
+
ranges = [[AST::Property, char == "P", name, ignorecase?]]
|
|
345
379
|
@last_ranges = ranges
|
|
346
380
|
nil
|
|
347
381
|
when "x"
|
|
348
|
-
|
|
349
|
-
read_until("}")
|
|
382
|
+
if consume?("{")
|
|
383
|
+
assign_codepoint(read_until("}"))
|
|
350
384
|
else
|
|
351
|
-
read_exact(2)
|
|
385
|
+
assign_hex_byte(read_exact(2))
|
|
352
386
|
end
|
|
353
|
-
assign_codepoint(digits)
|
|
354
387
|
nil
|
|
355
388
|
when "u"
|
|
356
389
|
digits = if consume?("{")
|
|
@@ -362,28 +395,69 @@ module Flexr
|
|
|
362
395
|
assign_codepoint(digits)
|
|
363
396
|
nil
|
|
364
397
|
when "b"
|
|
365
|
-
|
|
398
|
+
unless @class_depth.positive?
|
|
399
|
+
raise unsupported("\\b",
|
|
400
|
+
'when surrounding literals already establish the boundary, rewrite rule(/foo\b:/) as rule(/foo:/); keep the action; token edges need separate rules or state checks')
|
|
401
|
+
end
|
|
366
402
|
|
|
367
403
|
@escaped_value = true
|
|
368
404
|
@last_codepoint = 0x08
|
|
369
|
-
when "
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
when "
|
|
373
|
-
|
|
374
|
-
|
|
405
|
+
when "B"
|
|
406
|
+
raise unsupported("\\B",
|
|
407
|
+
'when surrounding literals already establish the non-boundary, rewrite rule(/foo\Bbar/) as rule(/foobar/); keep the action; token edges need separate rules or state checks')
|
|
408
|
+
when "G"
|
|
409
|
+
raise unsupported("\\G",
|
|
410
|
+
'at the start of a rule, rewrite rule(/\Ga/) as rule(/a/); keep the action; rules already match at the current lexer position; interior \G needs a separate rewrite')
|
|
411
|
+
when "A"
|
|
412
|
+
raise unsupported("\\A",
|
|
413
|
+
'for input without line feeds, rewrite rule(/\Aa/) as rule(/^a/); keep the action; line boundaries differ from string boundaries')
|
|
414
|
+
when "z", "Z"
|
|
415
|
+
raise unsupported("\\#{char}",
|
|
416
|
+
"for input without line feeds, rewrite rule(/a\\#{char}/) as rule(/a$/); keep the action; line boundaries differ from string boundaries")
|
|
417
|
+
when "R"
|
|
418
|
+
example = if @encoding == Encoding::BINARY
|
|
419
|
+
'rule(/\r\n|[\n\v\f\r]/n)'
|
|
420
|
+
else
|
|
421
|
+
'rule(/\r\n|[\n\v\f\r\u0085\u2028\u2029]/)'
|
|
422
|
+
end
|
|
423
|
+
raise unsupported("\\R", "for a standalone line-break token, use #{example}; keep the action")
|
|
424
|
+
when "K"
|
|
425
|
+
raise unsupported("\\K",
|
|
426
|
+
'for /a\Kb/ in :initial, use rule(/a/, followed_by: /b/) { begin_state :suffix }; state(:suffix) { rule(/b/) { begin_state :initial; emit :B } }; keep the action on the suffix rule after returning to :initial; choose an unused state name')
|
|
427
|
+
when "g"
|
|
428
|
+
raise unsupported("\\g",
|
|
429
|
+
'for this non-recursive call, rewrite rule(/(a|b)\g<1>/) as rule(/[ab]{2}/); keep the action; recursive calls need a parser or state stack')
|
|
430
|
+
when "X"
|
|
431
|
+
raise unsupported("\\X",
|
|
432
|
+
'for a standalone grapheme token with ASCII-only input, rewrite rule(/\X/) as rule(/\r\n|[\x00-\x7f]/); keep the action; Unicode graphemes need separate segmentation')
|
|
433
|
+
when "c", "C"
|
|
434
|
+
raise unsupported("\\#{char}",
|
|
435
|
+
'for control-A, rewrite rule(/\cA/) or rule(/\C-A/) as rule(/\x01/); keep the action; other control characters need their own hexadecimal code')
|
|
436
|
+
when "M"
|
|
437
|
+
raise unsupported("\\M",
|
|
438
|
+
'for binary input, rewrite rule(/\M-a/n) as rule(/\xE1/n); keep the action; other meta characters need their own byte value')
|
|
439
|
+
when "1".."9", "k"
|
|
440
|
+
read_until(">") if char == "k" && consume?("<")
|
|
441
|
+
feature = char == "k" ? "backreferences" : "\\#{char}"
|
|
442
|
+
help = if @class_depth.positive? && char != "k"
|
|
443
|
+
'for a character code, use an explicit hexadecimal escape such as rule(/[\x01]/); keep the action'
|
|
444
|
+
else
|
|
445
|
+
'for a capture limited to a or b, rewrite rule(/(a|b)\1/) as rule(/(?:aa|bb)/) and keep the action; otherwise split the rule into states and compare matched text in actions'
|
|
446
|
+
end
|
|
447
|
+
raise unsupported(feature, help)
|
|
375
448
|
when *LITERAL_ESCAPES
|
|
376
449
|
@escaped_value = true
|
|
377
450
|
@last_codepoint = char.ord
|
|
378
451
|
else
|
|
379
|
-
raise unsupported("unknown escape \\#{char}",
|
|
452
|
+
raise unsupported("unknown escape \\#{char}",
|
|
453
|
+
'if q is literal, rewrite rule(/\q/) as rule(/q/); otherwise use a supported escape; keep the action')
|
|
380
454
|
end
|
|
381
455
|
end
|
|
382
456
|
|
|
383
457
|
def shorthand_ranges(char)
|
|
384
458
|
if @unicode && @encoding != Encoding::BINARY
|
|
385
459
|
property = { "d" => "Nd", "w" => "Word", "s" => "Space" }.fetch(char.downcase, nil)
|
|
386
|
-
return [[AST::Property, char == char.upcase, property]] if property
|
|
460
|
+
return [[AST::Property, char == char.upcase, property, ignorecase?]] if property
|
|
387
461
|
end
|
|
388
462
|
|
|
389
463
|
base = {
|
|
@@ -468,7 +542,8 @@ module Flexr
|
|
|
468
542
|
def reject_postfix_quantifier
|
|
469
543
|
return unless ["?", "+"].include?(current)
|
|
470
544
|
|
|
471
|
-
raise unsupported("lazy or possessive quantifier",
|
|
545
|
+
raise unsupported("lazy or possessive quantifier",
|
|
546
|
+
'for unescaped single-line quoted text, use a negated character class: rule(/"[^"\n]*"/); keep the action; removing lazy or possessive modifiers can change matches')
|
|
472
547
|
end
|
|
473
548
|
|
|
474
549
|
def merge_ranges(ranges)
|
|
@@ -574,6 +649,37 @@ module Flexr
|
|
|
574
649
|
@last_codepoint = codepoint
|
|
575
650
|
end
|
|
576
651
|
|
|
652
|
+
def assign_hex_byte(digits)
|
|
653
|
+
byte = digits.to_i(16)
|
|
654
|
+
if @encoding == Encoding::BINARY
|
|
655
|
+
@escaped_byte = true
|
|
656
|
+
@last_byte = byte
|
|
657
|
+
return
|
|
658
|
+
end
|
|
659
|
+
|
|
660
|
+
length = case byte
|
|
661
|
+
when 0x00..0x7f then 1
|
|
662
|
+
when 0xc2..0xdf then 2
|
|
663
|
+
when 0xe0..0xef then 3
|
|
664
|
+
when 0xf0..0xf4 then 4
|
|
665
|
+
else 0
|
|
666
|
+
end
|
|
667
|
+
bytes = [byte]
|
|
668
|
+
(length - 1).times do
|
|
669
|
+
expect("\\x")
|
|
670
|
+
bytes << read_exact(2).to_i(16)
|
|
671
|
+
end
|
|
672
|
+
value = bytes.pack("C*").force_encoding(Encoding::UTF_8)
|
|
673
|
+
raise_syntax("invalid UTF-8 byte escape") unless length.positive? && value.valid_encoding?
|
|
674
|
+
|
|
675
|
+
@escaped_value = true
|
|
676
|
+
@last_codepoint = value.ord
|
|
677
|
+
end
|
|
678
|
+
|
|
679
|
+
def ignorecase?
|
|
680
|
+
@options.anybits?(::Regexp::IGNORECASE)
|
|
681
|
+
end
|
|
682
|
+
|
|
577
683
|
def skip_extended_space
|
|
578
684
|
loop do
|
|
579
685
|
@index += 1 while @source[@index]&.match?(/\s/)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
# Generated by flexr. DO NOT EDIT.
|
|
3
3
|
# source: lib/flexr/regexp/tokenizer.flexr.rb
|
|
4
|
-
# spec-digest: sha256:
|
|
5
|
-
# artifact-schema:
|
|
6
|
-
# runtime-abi:
|
|
4
|
+
# spec-digest: sha256:a9c3e840a6b33c0cb45b25200a372b0ce4f578497f96cc58a591ccd751954f16
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
7
|
# unicode: 15.1.0
|
|
8
8
|
# backend: table
|
|
9
9
|
# compiled: true
|
|
@@ -17,7 +17,7 @@ module Flexr
|
|
|
17
17
|
# This specification is intentionally small: it is a dogfood target for
|
|
18
18
|
# the source lexer path, not the parser's implementation.
|
|
19
19
|
class SourceLexer < Flexr::Lexer
|
|
20
|
-
Flexr::Generated.install_compiled!(self, { artifact: {schema_version:
|
|
20
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "2.0.0", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
|
|
21
21
|
def scan_one
|
|
22
22
|
if @__flexr_generated_scan_state != @state
|
|
23
23
|
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
@@ -192,8 +192,8 @@ module Flexr
|
|
|
192
192
|
@__flexr_generated_accel_scanner.pos = segment_position
|
|
193
193
|
@__flexr_generated_accel_scanner.skip(regexp)
|
|
194
194
|
else
|
|
195
|
-
match = regexp.match(segment,
|
|
196
|
-
match && match.begin(0)
|
|
195
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
196
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
197
197
|
end
|
|
198
198
|
rescue ArgumentError
|
|
199
199
|
nil
|
|
@@ -251,25 +251,25 @@ module Flexr
|
|
|
251
251
|
case rule.index
|
|
252
252
|
when 0
|
|
253
253
|
when 1
|
|
254
|
-
|
|
254
|
+
emit :PROPERTY, text.byteslice(3...-1)
|
|
255
255
|
when 2
|
|
256
|
-
|
|
256
|
+
emit :ESCAPE, text
|
|
257
257
|
when 3
|
|
258
|
-
|
|
258
|
+
emit :CHAR_CLASS, text
|
|
259
259
|
when 4
|
|
260
|
-
|
|
260
|
+
emit :QUANTIFIER, text
|
|
261
261
|
when 5
|
|
262
|
-
|
|
262
|
+
emit :ALTERNATION, text
|
|
263
263
|
when 6
|
|
264
|
-
|
|
264
|
+
emit :GROUP_OPEN, text
|
|
265
265
|
when 7
|
|
266
|
-
|
|
266
|
+
emit :GROUP_CLOSE, text
|
|
267
267
|
when 8
|
|
268
|
-
|
|
268
|
+
emit :ANCHOR, text
|
|
269
269
|
when 9
|
|
270
|
-
|
|
270
|
+
emit :DOT, text
|
|
271
271
|
when 10
|
|
272
|
-
|
|
272
|
+
emit :LITERAL, text
|
|
273
273
|
else
|
|
274
274
|
instance_exec(&rule.action)
|
|
275
275
|
end
|
|
@@ -278,16 +278,16 @@ module Flexr
|
|
|
278
278
|
|
|
279
279
|
|
|
280
280
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
|
|
291
291
|
end
|
|
292
292
|
end
|
|
293
293
|
end
|
data/lib/flexr/rspec.rb
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rspec/expectations"
|
|
4
|
+
require_relative "testing"
|
|
5
|
+
|
|
6
|
+
RSpec::Matchers.define :tokenize do |input, **options|
|
|
7
|
+
chain :as do |tokens|
|
|
8
|
+
@expected = tokens
|
|
9
|
+
@comparison = :tokens
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
chain :matching_snapshot do |path, snapshot_options = {}|
|
|
13
|
+
@snapshot_path = path
|
|
14
|
+
@update_snapshot = snapshot_options.fetch(:update, false)
|
|
15
|
+
@comparison = :snapshot
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
match do |lexer|
|
|
19
|
+
raise ArgumentError, "use .as(tokens) or .matching_snapshot(path)" unless @comparison
|
|
20
|
+
|
|
21
|
+
@actual = lexer.new(input, **options).tokens
|
|
22
|
+
if @comparison == :snapshot
|
|
23
|
+
@actual = Flexr::Testing::Snapshot.render(@actual)
|
|
24
|
+
begin
|
|
25
|
+
@expected = Flexr::Testing::Snapshot.expected(@snapshot_path, @actual, update: @update_snapshot)
|
|
26
|
+
rescue Errno::ENOENT
|
|
27
|
+
@missing_snapshot = true
|
|
28
|
+
next false
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
@actual == @expected
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
failure_message do
|
|
35
|
+
if @missing_snapshot
|
|
36
|
+
"missing token snapshot #{@snapshot_path}; create it with update: true"
|
|
37
|
+
elsif @comparison == :snapshot
|
|
38
|
+
"expected tokens to match snapshot #{@snapshot_path}"
|
|
39
|
+
else
|
|
40
|
+
"expected tokens #{@expected.inspect}, got #{@actual.inspect}"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
failure_message_when_negated do
|
|
45
|
+
"expected tokens to differ from #{@comparison == :snapshot ? @snapshot_path : @expected.inspect}"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
diffable
|
|
49
|
+
define_method(:expected) { @expected }
|
|
50
|
+
define_method(:actual) { @actual }
|
|
51
|
+
end
|
data/lib/flexr/runtime/core.rb
CHANGED
|
@@ -6,6 +6,7 @@ module Flexr
|
|
|
6
6
|
max_lookahead_size: nil, max_buffer_size: 64 * 1024 * 1024,
|
|
7
7
|
max_state_stack: 1024, max_steps: nil, cancellation: nil, retain_input: true,
|
|
8
8
|
chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
|
|
9
|
+
raise ArgumentError, "unsupported error_mode: #{error_mode.inspect}" unless %i[raise token panic].include?(error_mode)
|
|
9
10
|
raise ArgumentError, "max_token_size must be non-negative" if max_token_size.to_i.negative?
|
|
10
11
|
raise ArgumentError, "max_lookahead_size must be non-negative" if max_lookahead_size&.to_i&.negative?
|
|
11
12
|
raise ArgumentError, "max_buffer_size must be non-negative" if max_buffer_size.to_i.negative?
|
|
@@ -37,7 +38,7 @@ module Flexr
|
|
|
37
38
|
@max_lookahead_size = (max_lookahead_size || max_token_size).to_i
|
|
38
39
|
@max_state_stack = max_state_stack.to_i
|
|
39
40
|
@max_steps = max_steps&.to_i
|
|
40
|
-
@cancellation = cancellation
|
|
41
|
+
@cancellation = cancellation.respond_to?(:arity) ? cancellation : cancellation&.method(:call)
|
|
41
42
|
@steps = 0
|
|
42
43
|
@position = 0
|
|
43
44
|
@line = 1
|
|
@@ -176,7 +177,7 @@ module Flexr
|
|
|
176
177
|
def text
|
|
177
178
|
return @matched if @matched
|
|
178
179
|
|
|
179
|
-
@matched = @buffer.byteslice(@text_start
|
|
180
|
+
@matched = @buffer.byteslice(@text_start, @match_end - @text_start)
|
|
180
181
|
end
|
|
181
182
|
|
|
182
183
|
def text_bytesize
|
|
@@ -196,7 +197,7 @@ module Flexr
|
|
|
196
197
|
|
|
197
198
|
raise Runtime::TokenTooLargeError.new(
|
|
198
199
|
filename: @filename, byte_pos: @more_start || @position, line: @line,
|
|
199
|
-
rule:
|
|
200
|
+
rule: diagnostic_rule(rule || @active_rule)
|
|
200
201
|
)
|
|
201
202
|
end
|
|
202
203
|
|
|
@@ -205,7 +206,7 @@ module Flexr
|
|
|
205
206
|
|
|
206
207
|
raise Runtime::LookaheadTooLargeError.new(
|
|
207
208
|
filename: @filename, byte_pos: @position, line: @line,
|
|
208
|
-
rule:
|
|
209
|
+
rule: diagnostic_rule(rule || @active_rule)
|
|
209
210
|
)
|
|
210
211
|
end
|
|
211
212
|
|
|
@@ -215,13 +216,13 @@ module Flexr
|
|
|
215
216
|
@steps += count
|
|
216
217
|
if @max_steps && @steps > @max_steps
|
|
217
218
|
raise Runtime::StepLimitError.new(
|
|
218
|
-
filename: @filename, byte_pos: @position, line: @line, rule:
|
|
219
|
+
filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
|
|
219
220
|
)
|
|
220
221
|
end
|
|
221
222
|
return unless cancelled?
|
|
222
223
|
|
|
223
224
|
raise Runtime::CancelledError.new(
|
|
224
|
-
filename: @filename, byte_pos: @position, line: @line, rule:
|
|
225
|
+
filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
|
|
225
226
|
)
|
|
226
227
|
end
|
|
227
228
|
|
|
@@ -233,6 +234,31 @@ module Flexr
|
|
|
233
234
|
@token_limit_required
|
|
234
235
|
end
|
|
235
236
|
|
|
237
|
+
def __flexr_whole_token_match(plans, position)
|
|
238
|
+
return unless plans && defined?(::StringScanner) && !scan_steps_guarded? && !token_limit_required?
|
|
239
|
+
|
|
240
|
+
source = @buffer.stable_source
|
|
241
|
+
return unless source&.frozen?
|
|
242
|
+
|
|
243
|
+
byte = source.getbyte(position)
|
|
244
|
+
plan = plans[byte] if byte
|
|
245
|
+
return unless plan
|
|
246
|
+
|
|
247
|
+
scanner = @__flexr_whole_scanner ||= ::StringScanner.new(source)
|
|
248
|
+
scanner.pos = position
|
|
249
|
+
length = scanner.skip(plan.regexp)
|
|
250
|
+
return unless length
|
|
251
|
+
|
|
252
|
+
match = @__flexr_whole_match ||= Runtime::Match.new
|
|
253
|
+
match.rule = self.class.__flexr_rules.fetch(plan.rule_index)
|
|
254
|
+
match.start_pos = position
|
|
255
|
+
match.end_pos = position + length
|
|
256
|
+
match.total_end_pos = match.end_pos
|
|
257
|
+
match
|
|
258
|
+
rescue ArgumentError
|
|
259
|
+
nil
|
|
260
|
+
end
|
|
261
|
+
|
|
236
262
|
def utf8_input?
|
|
237
263
|
self.class.__flexr_config.encoding == Encoding::UTF_8
|
|
238
264
|
end
|
|
@@ -291,7 +317,7 @@ module Flexr
|
|
|
291
317
|
|
|
292
318
|
raise Runtime::InvalidRecoveryActionError.new(
|
|
293
319
|
action: action, filename: @filename, byte_pos: @match_start, line: @text_line,
|
|
294
|
-
text: text, rule:
|
|
320
|
+
text: text, rule: @active_rule
|
|
295
321
|
)
|
|
296
322
|
end
|
|
297
323
|
case @error_mode
|
|
@@ -311,9 +337,10 @@ module Flexr
|
|
|
311
337
|
def reject
|
|
312
338
|
diagnostic = Diagnostics.error(
|
|
313
339
|
"FLEXR-E013", "reject is not supported by flexr",
|
|
340
|
+
location: [@position, @position],
|
|
314
341
|
help: "use a state transition and less(n) to express the fallback"
|
|
315
342
|
)
|
|
316
|
-
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
343
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
|
|
317
344
|
end
|
|
318
345
|
|
|
319
346
|
def push(name)
|
|
@@ -322,7 +349,7 @@ module Flexr
|
|
|
322
349
|
raise Runtime::StateStackOverflowError.new(
|
|
323
350
|
"state stack exceeds max_state_stack (#{@max_state_stack})",
|
|
324
351
|
filename: @filename, byte_pos: @position, line: @line, text: text,
|
|
325
|
-
rule:
|
|
352
|
+
rule: @active_rule
|
|
326
353
|
)
|
|
327
354
|
end
|
|
328
355
|
|
|
@@ -443,6 +470,7 @@ module Flexr
|
|
|
443
470
|
@text_start = @match_start
|
|
444
471
|
@text_line = @line
|
|
445
472
|
@text_column = @column
|
|
473
|
+
@matched = nil
|
|
446
474
|
@position += 1
|
|
447
475
|
@non_progress_signatures.clear
|
|
448
476
|
advance_location!(@match_start, @match_end)
|
|
@@ -464,8 +492,8 @@ module Flexr
|
|
|
464
492
|
def ensure_state!(name)
|
|
465
493
|
return if self.class.__flexr_config.states.key?(name.to_sym)
|
|
466
494
|
|
|
467
|
-
diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}")
|
|
468
|
-
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
495
|
+
diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}", location: [@position, @position])
|
|
496
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
|
|
469
497
|
end
|
|
470
498
|
|
|
471
499
|
def ensure_progress!(match, scan_state, empty_match:)
|
|
@@ -488,7 +516,7 @@ module Flexr
|
|
|
488
516
|
|
|
489
517
|
raise Runtime::NonProgressError.new(
|
|
490
518
|
filename: @filename, byte_pos: match.start_pos, line: @line,
|
|
491
|
-
text: text, rule: match.rule
|
|
519
|
+
text: text, rule: match.rule
|
|
492
520
|
)
|
|
493
521
|
end
|
|
494
522
|
|
|
@@ -501,7 +529,7 @@ module Flexr
|
|
|
501
529
|
end
|
|
502
530
|
|
|
503
531
|
value = location_value(starting, ending)
|
|
504
|
-
location_bytes = value.
|
|
532
|
+
location_bytes = value.b
|
|
505
533
|
newline_count = location_bytes.count("\n")
|
|
506
534
|
if newline_count.zero?
|
|
507
535
|
@column += display_length(value)
|
|
@@ -516,7 +544,7 @@ module Flexr
|
|
|
516
544
|
return [line, column] if ending <= starting
|
|
517
545
|
|
|
518
546
|
value = location_value(starting, ending)
|
|
519
|
-
location_bytes = value.
|
|
547
|
+
location_bytes = value.b
|
|
520
548
|
newline_count = location_bytes.count("\n")
|
|
521
549
|
return [line, column + display_length(value)] if newline_count.zero?
|
|
522
550
|
|
|
@@ -535,7 +563,7 @@ module Flexr
|
|
|
535
563
|
def location_value(starting, ending)
|
|
536
564
|
return @matched if @matched && starting == @text_start && ending == @match_end
|
|
537
565
|
|
|
538
|
-
@buffer.byteslice(starting
|
|
566
|
+
@buffer.byteslice(starting, ending - starting).to_s
|
|
539
567
|
end
|
|
540
568
|
|
|
541
569
|
def discard_consumed_input!
|
|
@@ -548,8 +576,10 @@ module Flexr
|
|
|
548
576
|
@cancellation.arity.zero? ? @cancellation.call : @cancellation.call(self)
|
|
549
577
|
end
|
|
550
578
|
|
|
551
|
-
def
|
|
552
|
-
rule.
|
|
579
|
+
def diagnostic_rule(rule)
|
|
580
|
+
return rule unless rule.is_a?(Integer) && rule >= 0
|
|
581
|
+
|
|
582
|
+
self.class.__flexr_rules.fetch(rule, rule)
|
|
553
583
|
end
|
|
554
584
|
end
|
|
555
585
|
end
|
data/lib/flexr/runtime/errors.rb
CHANGED
|
@@ -10,12 +10,13 @@ module Flexr
|
|
|
10
10
|
def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil,
|
|
11
11
|
rule: nil)
|
|
12
12
|
@code = CODE
|
|
13
|
-
@rule = rule
|
|
13
|
+
@rule = rule.is_a?(IR::Rule) ? rule.index : rule
|
|
14
14
|
diagnostic = Diagnostics.error(
|
|
15
15
|
CODE,
|
|
16
16
|
message,
|
|
17
|
+
location: byte_pos && [byte_pos, byte_pos],
|
|
17
18
|
help: "increase max_token_size or split the input token",
|
|
18
|
-
note: rule.nil? ? nil : "
|
|
19
|
+
note: rule.nil? ? nil : "#{Diagnostics.rule_label(rule)} at byte #{byte_pos}"
|
|
19
20
|
)
|
|
20
21
|
super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
|
|
21
22
|
end
|
|
@@ -26,8 +27,9 @@ module Flexr
|
|
|
26
27
|
|
|
27
28
|
def initialize(code, message, help:, filename: nil, byte_pos: nil, line: nil, text: nil, rule: nil)
|
|
28
29
|
@code = code
|
|
29
|
-
@rule = rule
|
|
30
|
-
diagnostic = Diagnostics.error(code, message,
|
|
30
|
+
@rule = rule.is_a?(IR::Rule) ? rule.index : rule
|
|
31
|
+
diagnostic = Diagnostics.error(code, message, location: byte_pos && [byte_pos, byte_pos],
|
|
32
|
+
help: help, note: Diagnostics.rule_label(rule))
|
|
31
33
|
super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
|
|
32
34
|
end
|
|
33
35
|
end
|