flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -13,7 +13,13 @@ module Flexr
|
|
|
13
13
|
state_names = effective_states
|
|
14
14
|
machines = state_names.to_h do |state_name|
|
|
15
15
|
rules = rules_for(state_name)
|
|
16
|
-
|
|
16
|
+
dfa = compile_machine(rules)
|
|
17
|
+
plans = Accel.whole_token_plans(dfa, rules) if
|
|
18
|
+
@spec.backend != :firstmatch && @spec.options[:accel] == :strscan
|
|
19
|
+
[state_name, Machine.new(dfa: dfa, state_name: state_name, whole_tokens: plans)]
|
|
20
|
+
rescue CompileError => e
|
|
21
|
+
e.diagnostic.location ||= @spec.states.fetch(state_name).location if e.diagnostic
|
|
22
|
+
raise
|
|
17
23
|
end
|
|
18
24
|
stats = machines.transform_values { |machine| machine.dfa.stats }
|
|
19
25
|
compiled = CompiledSpec.new(machines: machines, rules: @spec.rules, states: state_names, stats: stats)
|
|
@@ -41,7 +47,7 @@ module Flexr
|
|
|
41
47
|
end
|
|
42
48
|
|
|
43
49
|
def compile_machine(rules)
|
|
44
|
-
@
|
|
50
|
+
@active_rules = rules
|
|
45
51
|
normalized = []
|
|
46
52
|
rules.each do |rule|
|
|
47
53
|
rule.pattern_conditions = []
|
|
@@ -58,6 +64,9 @@ module Flexr
|
|
|
58
64
|
normalized_ast = Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
|
|
59
65
|
normalized << [normalized_ast, condition]
|
|
60
66
|
end
|
|
67
|
+
rescue CompileError => e
|
|
68
|
+
e.diagnostic.location ||= rule.location if e.diagnostic
|
|
69
|
+
raise
|
|
61
70
|
end
|
|
62
71
|
return empty_dfa if normalized.empty?
|
|
63
72
|
|
|
@@ -130,7 +139,7 @@ module Flexr
|
|
|
130
139
|
unless destination
|
|
131
140
|
destination = sets.length
|
|
132
141
|
if destination >= limit
|
|
133
|
-
message = "DFA state limit exceeded while compiling
|
|
142
|
+
message = "DFA state limit exceeded while compiling #{@active_rules.map { |rule| Diagnostics.rule_label(rule) }.join(', ')}"
|
|
134
143
|
diagnostic = Diagnostics.error("FLEXR-E006", message,
|
|
135
144
|
help: "raise max_dfa_states or split the listed rules")
|
|
136
145
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
@@ -205,8 +214,6 @@ module Flexr
|
|
|
205
214
|
end
|
|
206
215
|
|
|
207
216
|
def validate_rules
|
|
208
|
-
raise CompileError, "firstmatch requires option :experimental" if @spec.backend == :firstmatch && !@spec.options[:experimental]
|
|
209
|
-
|
|
210
217
|
@spec.rules.each do |rule|
|
|
211
218
|
raise CompileError, "rule #{rule.index} has no pattern" if rule.patterns.empty?
|
|
212
219
|
parse_regexp(rule.trailing) if rule.trailing.is_a?(::Regexp)
|
|
@@ -217,9 +224,12 @@ module Flexr
|
|
|
217
224
|
parse_regexp(regexp) if regexp.is_a?(::Regexp)
|
|
218
225
|
next unless regexp.is_a?(::Regexp) && regexp.match?("")
|
|
219
226
|
|
|
220
|
-
diagnostic = Diagnostics.error("FLEXR-E005", "
|
|
227
|
+
diagnostic = Diagnostics.error("FLEXR-E005", "#{Diagnostics.rule_label(rule)} can match an empty string")
|
|
221
228
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
222
229
|
end
|
|
230
|
+
rescue CompileError => e
|
|
231
|
+
e.diagnostic.location ||= rule.location if e.diagnostic
|
|
232
|
+
raise
|
|
223
233
|
end
|
|
224
234
|
end
|
|
225
235
|
|
|
@@ -235,33 +245,45 @@ module Flexr
|
|
|
235
245
|
machine.dfa.accepts.filter_map { |acceptances| acceptances.min_by(&:rule_index)&.rule_index }
|
|
236
246
|
end
|
|
237
247
|
shadowers = shadowed_rules(compiled)
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
248
|
+
rules_by_index = @spec.rules.to_h { |rule| [rule.index, rule] }
|
|
249
|
+
unreachable = @spec.rules.reject { |rule| present.include?(rule.index) }
|
|
250
|
+
examples = unreachable_examples(compiled, unreachable, rules_by_index)
|
|
251
|
+
diagnostics.concat(unreachable.filter_map do |rule|
|
|
252
|
+
example = examples[rule.index]
|
|
253
|
+
next if example && example.fetch(:winner) == rule.index
|
|
254
|
+
|
|
255
|
+
winner_indices = @spec.backend == :firstmatch || @spec.rules.any?(&:trailing) ? [] : shadowers.fetch(rule.index, [])
|
|
256
|
+
winners = winner_indices.uniq.sort.map { |index| Diagnostics.rule_label(rules_by_index.fetch(index)) }
|
|
257
|
+
suffix = winners.empty? ? "" : " (shadowed by #{winners.join(', ')})"
|
|
258
|
+
Diagnostics.warning("FLEXR-W001", "#{Diagnostics.rule_label(rule)} is unreachable#{suffix}", location: rule.location,
|
|
259
|
+
help: "remove it, reorder the rules, or make its language distinct",
|
|
260
|
+
note: example&.fetch(:note))
|
|
243
261
|
end)
|
|
262
|
+
diagnostics.concat(overlap_diagnostics(compiled, rules_by_index)) if @spec.options[:warn_level] == :all
|
|
244
263
|
|
|
245
|
-
@spec.states.
|
|
264
|
+
@spec.states.each do |state_name, state|
|
|
246
265
|
next if state_name == :initial
|
|
247
266
|
next unless rules_for(state_name).empty?
|
|
248
267
|
|
|
249
268
|
diagnostics << Diagnostics.warning("FLEXR-W002", "state #{state_name.inspect} has no rules",
|
|
269
|
+
location: state.location,
|
|
250
270
|
help: "add a rule to the state or remove the unused state")
|
|
251
271
|
end
|
|
252
272
|
|
|
253
273
|
firstmatch_conflicts(@spec.rules).each do |left, right|
|
|
254
274
|
diagnostics << Diagnostics.warning(
|
|
255
|
-
"FLEXR-W010", "firstmatch
|
|
275
|
+
"FLEXR-W010", "firstmatch #{Diagnostics.rule_label(left)} and #{Diagnostics.rule_label(right)} may change longest-match semantics",
|
|
276
|
+
location: right.location,
|
|
256
277
|
help: "use backend :table unless first-match compatibility is required"
|
|
257
278
|
)
|
|
258
279
|
end
|
|
259
280
|
|
|
260
|
-
max_cells = compiled.stats.
|
|
261
|
-
if max_cells > 1_000_000
|
|
281
|
+
largest_state, max_cells = compiled.stats.map { |name, stat| [name, stat[:states] * stat[:classes]] }.max_by(&:last)
|
|
282
|
+
if max_cells && max_cells > 1_000_000
|
|
262
283
|
diagnostics << Diagnostics.warning(
|
|
263
284
|
"FLEXR-W011",
|
|
264
285
|
"generated transition table is large",
|
|
286
|
+
location: @spec.states.fetch(largest_state).location,
|
|
265
287
|
help: "use backend :direct, table compression, or split the specification"
|
|
266
288
|
)
|
|
267
289
|
end
|
|
@@ -269,21 +291,24 @@ module Flexr
|
|
|
269
291
|
capture_rules.each do |rule|
|
|
270
292
|
diagnostics << Diagnostics.warning(
|
|
271
293
|
"FLEXR-W013",
|
|
272
|
-
"
|
|
294
|
+
"#{Diagnostics.rule_label(rule)} uses a capturing group; flexr treats it as non-capturing",
|
|
295
|
+
location: rule.location,
|
|
273
296
|
help: "rewrite capturing groups as (?:...) and extract text in the action"
|
|
274
297
|
)
|
|
275
298
|
end
|
|
276
299
|
|
|
277
|
-
|
|
300
|
+
undeclared_token_rules.each do |token, rule|
|
|
278
301
|
diagnostics << Diagnostics.warning(
|
|
279
302
|
"FLEXR-W014",
|
|
280
303
|
"token #{token.inspect} is not declared by emits",
|
|
304
|
+
location: rule.location,
|
|
281
305
|
help: "add the token to emits or remove the declaration if it is intentionally private"
|
|
282
306
|
)
|
|
283
307
|
end
|
|
284
308
|
|
|
285
309
|
diagnostics.concat(variable_trailing_rules.map do |rule|
|
|
286
|
-
Diagnostics.warning("FLEXR-W003", "
|
|
310
|
+
Diagnostics.warning("FLEXR-W003", "#{Diagnostics.rule_label(rule)} uses variable-length trailing context",
|
|
311
|
+
location: rule.location,
|
|
287
312
|
help: "make the body or followed_by expression fixed length when possible")
|
|
288
313
|
end)
|
|
289
314
|
|
|
@@ -291,7 +316,8 @@ module Flexr
|
|
|
291
316
|
diagnostics.concat(@spec.rules.select(&:trailing).map do |rule|
|
|
292
317
|
Diagnostics.warning(
|
|
293
318
|
"FLEXR-W012",
|
|
294
|
-
"
|
|
319
|
+
"#{Diagnostics.rule_label(rule)} cannot use region acceleration with trailing context",
|
|
320
|
+
location: rule.location,
|
|
295
321
|
help: "remove trailing context or set accel: :none when the trade-off is intentional"
|
|
296
322
|
)
|
|
297
323
|
end)
|
|
@@ -299,11 +325,138 @@ module Flexr
|
|
|
299
325
|
|
|
300
326
|
if elapsed > 0.5
|
|
301
327
|
diagnostics << Diagnostics.warning("FLEXR-W016", format("DFA construction took %.3fs", elapsed),
|
|
328
|
+
location: @spec.states.fetch(:initial).location,
|
|
302
329
|
help: "use generated mode for production startup")
|
|
303
330
|
end
|
|
304
331
|
diagnostics
|
|
305
332
|
end
|
|
306
333
|
|
|
334
|
+
def overlap_diagnostics(compiled, rules_by_index)
|
|
335
|
+
# shortcut: verify one shortest body and trailing candidate per rule; search alternatives if rejected examples need coverage.
|
|
336
|
+
compiled.machines.flat_map do |state_name, machine|
|
|
337
|
+
active_rules = rules_for(state_name)
|
|
338
|
+
trailing_dfas = diagnostic_trailing_dfas(active_rules)
|
|
339
|
+
next [] unless trailing_dfas
|
|
340
|
+
|
|
341
|
+
has_trailing = !trailing_dfas.empty?
|
|
342
|
+
candidates = Analysis.shadowing_examples(machine.dfa, machine.dfa.rule_ids,
|
|
343
|
+
include_empty: has_trailing, include_unshadowed: has_trailing)
|
|
344
|
+
candidates.filter_map do |index, (bytes, winner)|
|
|
345
|
+
if (trailing = trailing_dfas[index])
|
|
346
|
+
suffix = Analysis.shortest_input(trailing)
|
|
347
|
+
next unless suffix
|
|
348
|
+
|
|
349
|
+
bytes += suffix
|
|
350
|
+
end
|
|
351
|
+
input = bytes.dup.force_encoding(@spec.encoding)
|
|
352
|
+
next if input.empty? || !input.valid_encoding?
|
|
353
|
+
|
|
354
|
+
rule = rules_by_index.fetch(index)
|
|
355
|
+
length = input.bytesize
|
|
356
|
+
if has_trailing
|
|
357
|
+
result = Analysis.trailing_example(machine.dfa, active_rules, rule, input, trailing_dfas,
|
|
358
|
+
encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
|
|
359
|
+
firstmatch: @spec.backend == :firstmatch, same_length: true)
|
|
360
|
+
next unless result && result.first != index
|
|
361
|
+
|
|
362
|
+
winner, length, lookahead = result
|
|
363
|
+
elsif @spec.backend == :firstmatch
|
|
364
|
+
result = Analysis.firstmatch_example(active_rules, rule, input, encoding: @spec.encoding,
|
|
365
|
+
unicode: @spec.options[:unicode] == true, same_length: true)
|
|
366
|
+
next unless result && result.first != index
|
|
367
|
+
|
|
368
|
+
winner, length = result
|
|
369
|
+
end
|
|
370
|
+
selected = Diagnostics.rule_label(rules_by_index.fetch(winner))
|
|
371
|
+
state = state_name == :initial ? "" : " in state #{state_name.inspect}"
|
|
372
|
+
size = "#{length} #{length == 1 ? 'byte' : 'bytes'}"
|
|
373
|
+
reason = if has_trailing
|
|
374
|
+
priority = @spec.backend == :firstmatch ? "first match wins" : "same total length, earlier rule wins"
|
|
375
|
+
"#{priority}; matched #{size}, looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}"
|
|
376
|
+
elsif @spec.backend == :firstmatch
|
|
377
|
+
"first match wins, matched #{size}"
|
|
378
|
+
else
|
|
379
|
+
"#{size}, earlier rule wins"
|
|
380
|
+
end
|
|
381
|
+
match_size = has_trailing ? "total length" : "length"
|
|
382
|
+
Diagnostics.info("FLEXR-I001", "#{Diagnostics.rule_label(rule)} overlaps #{selected} at the same #{match_size}",
|
|
383
|
+
location: rule.location, help: "reorder the rules if the later rule should take priority",
|
|
384
|
+
note: "example: #{input.inspect} selects #{selected}#{state} (#{reason})")
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
def unreachable_examples(compiled, rules, rules_by_index)
|
|
390
|
+
return {} if rules.empty?
|
|
391
|
+
|
|
392
|
+
indices = rules.map(&:index)
|
|
393
|
+
examples = {}
|
|
394
|
+
compiled.machines.each do |state_name, machine|
|
|
395
|
+
active_rules = rules_for(state_name)
|
|
396
|
+
# shortcut: verify only the shortest candidate; search alternatives if rejected examples need coverage.
|
|
397
|
+
candidates = Analysis.shadowing_examples(machine.dfa, indices, include_empty: active_rules.any?(&:trailing))
|
|
398
|
+
next if candidates.empty?
|
|
399
|
+
|
|
400
|
+
trailing_dfas = diagnostic_trailing_dfas(active_rules)
|
|
401
|
+
next unless trailing_dfas
|
|
402
|
+
|
|
403
|
+
candidates.each do |index, (bytes, winner)|
|
|
404
|
+
if (trailing = trailing_dfas[index])
|
|
405
|
+
suffix = Analysis.shortest_input(trailing)
|
|
406
|
+
next unless suffix
|
|
407
|
+
|
|
408
|
+
bytes += suffix
|
|
409
|
+
end
|
|
410
|
+
input = bytes.dup.force_encoding(@spec.encoding)
|
|
411
|
+
next if input.empty? || !input.valid_encoding?
|
|
412
|
+
next if examples[index] && examples[index][1] == index
|
|
413
|
+
|
|
414
|
+
length = lookahead = nil
|
|
415
|
+
if !trailing_dfas.empty?
|
|
416
|
+
result = Analysis.trailing_example(machine.dfa, active_rules, rules_by_index.fetch(index), input, trailing_dfas,
|
|
417
|
+
encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
|
|
418
|
+
firstmatch: @spec.backend == :firstmatch)
|
|
419
|
+
next unless result
|
|
420
|
+
|
|
421
|
+
winner, length, lookahead = result
|
|
422
|
+
elsif @spec.backend == :firstmatch
|
|
423
|
+
result = Analysis.firstmatch_example(active_rules, rules_by_index.fetch(index), input,
|
|
424
|
+
encoding: @spec.encoding, unicode: @spec.options[:unicode] == true)
|
|
425
|
+
next unless result
|
|
426
|
+
|
|
427
|
+
winner, length = result
|
|
428
|
+
end
|
|
429
|
+
next if winner != index && examples[index] && examples[index].first.bytesize <= input.bytesize
|
|
430
|
+
|
|
431
|
+
examples[index] = [input, winner, state_name, length, lookahead]
|
|
432
|
+
end
|
|
433
|
+
end
|
|
434
|
+
examples.transform_values do |input, winner, state_name, length, lookahead|
|
|
435
|
+
state = state_name == :initial ? "" : " in state #{state_name.inspect}"
|
|
436
|
+
reason = if @spec.backend == :firstmatch
|
|
437
|
+
"first match wins, matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
|
|
438
|
+
elsif lookahead
|
|
439
|
+
"longest total match, then rule order; matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
|
|
440
|
+
else
|
|
441
|
+
"same length, earlier rule wins"
|
|
442
|
+
end
|
|
443
|
+
reason += ", looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}" if lookahead
|
|
444
|
+
note = "example: #{input.inspect} selects #{Diagnostics.rule_label(rules_by_index.fetch(winner))}#{state} (#{reason})"
|
|
445
|
+
{ note: note, winner: winner }
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
def diagnostic_trailing_dfas(rules)
|
|
450
|
+
rules.select(&:trailing).to_h do |rule|
|
|
451
|
+
dfa = rule.trailing_dfa || Flexr.compile_pattern(rule.trailing, encoding: @spec.encoding,
|
|
452
|
+
options: { unicode: @spec.options[:unicode] == true })
|
|
453
|
+
[rule.index, dfa]
|
|
454
|
+
end
|
|
455
|
+
rescue CompileError
|
|
456
|
+
# shortcut: omit examples for trailing patterns the runtime cannot compile, until that matcher supports them.
|
|
457
|
+
nil
|
|
458
|
+
end
|
|
459
|
+
|
|
307
460
|
def shadowed_rules(compiled)
|
|
308
461
|
shadowers = Hash.new { |hash, key| hash[key] = [] }
|
|
309
462
|
compiled.machines.each_value do |machine|
|
|
@@ -357,21 +510,22 @@ module Flexr
|
|
|
357
510
|
false
|
|
358
511
|
end
|
|
359
512
|
|
|
360
|
-
def
|
|
361
|
-
|
|
513
|
+
def undeclared_token_rules
|
|
514
|
+
declared = Array(@spec.declared_tokens)
|
|
515
|
+
return {} if declared.empty?
|
|
362
516
|
|
|
363
|
-
emitted = @spec.rules.
|
|
517
|
+
emitted = @spec.rules.each_with_object({}) do |rule, rules_by_token|
|
|
364
518
|
action = rule.action
|
|
365
|
-
if action.is_a?(Array) && action.first == :emit
|
|
519
|
+
tokens = if action.is_a?(Array) && action.first == :emit
|
|
366
520
|
[action.last]
|
|
367
521
|
elsif action.is_a?(String)
|
|
368
522
|
action.scan(/\bemit\s*\(?\s*:([A-Za-z_]\w*)/).flatten.map(&:to_sym)
|
|
369
523
|
else
|
|
370
524
|
[]
|
|
371
525
|
end
|
|
526
|
+
tokens.each { |token| rules_by_token[token] ||= rule }
|
|
372
527
|
end
|
|
373
|
-
declared
|
|
374
|
-
emitted.uniq.reject { |token| declared.include?(token) }
|
|
528
|
+
emitted.except(*declared)
|
|
375
529
|
end
|
|
376
530
|
|
|
377
531
|
def variable_trailing_rules
|
|
@@ -9,6 +9,6 @@ module Flexr
|
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
|
|
12
|
-
Machine = Struct.new(:dfa, :state_name, keyword_init: true)
|
|
12
|
+
Machine = Struct.new(:dfa, :state_name, :whole_tokens, keyword_init: true)
|
|
13
13
|
end
|
|
14
14
|
end
|
data/lib/flexr/cli.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "artifact_writer"
|
|
|
5
5
|
require_relative "generator"
|
|
6
6
|
require_relative "importer"
|
|
7
7
|
require_relative "options"
|
|
8
|
+
require_relative "migration"
|
|
8
9
|
|
|
9
10
|
module Flexr
|
|
10
11
|
module CLI
|
|
@@ -21,17 +22,19 @@ module Flexr
|
|
|
21
22
|
return version(out) if args.delete("--version")
|
|
22
23
|
|
|
23
24
|
command = COMMANDS.include?(args.first) ? args.shift.to_sym : :generate
|
|
24
|
-
options, output, rule_number, benchmark_args, positionals = parse(args, command)
|
|
25
|
+
options, output, rule_number, benchmark_args, positionals, trace_args = parse(args, command)
|
|
26
|
+
raise ArgumentError, "lrama format is only valid for tokens" if options.format == :lrama && command != :tokens
|
|
25
27
|
spec = positionals.shift
|
|
26
28
|
raise ArgumentError, "a SPEC.rb path is required" unless spec
|
|
27
29
|
raise ArgumentError, "unexpected argument: #{positionals.first}" unless positionals.empty?
|
|
28
30
|
|
|
29
|
-
execute(command, spec, options, output, rule_number, benchmark_args, out, err)
|
|
31
|
+
execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: trace_args)
|
|
30
32
|
rescue ArgumentError => e
|
|
31
33
|
err.puts "error: #{e.message}"
|
|
32
34
|
usage(err, status: EXIT_USAGE)
|
|
33
35
|
rescue Flexr::Error, Errno::ENOENT, Errno::EACCES, SyntaxError => e
|
|
34
|
-
err.puts render_error(e, options: options
|
|
36
|
+
err.puts render_error(e, options: options, path: spec,
|
|
37
|
+
normalize_source: command == :generate || (command == :check && options&.eval_mode))
|
|
35
38
|
EXIT_FAILURE
|
|
36
39
|
rescue StandardError => e
|
|
37
40
|
err.puts "error: #{e.class}: #{e.message}"
|
|
@@ -44,6 +47,7 @@ module Flexr
|
|
|
44
47
|
rule_number = nil
|
|
45
48
|
benchmark_args = []
|
|
46
49
|
positionals = []
|
|
50
|
+
trace_args = {}
|
|
47
51
|
parsing_options = true
|
|
48
52
|
|
|
49
53
|
until args.empty?
|
|
@@ -84,10 +88,32 @@ module Flexr
|
|
|
84
88
|
options.color = required_argument!(args, argument).to_sym
|
|
85
89
|
when "--format"
|
|
86
90
|
options.format = required_argument!(args, argument).to_sym
|
|
91
|
+
when "--target"
|
|
92
|
+
raise ArgumentError, "--target is only valid for the check command" unless command == :check
|
|
93
|
+
options.target = required_argument!(args, argument)
|
|
87
94
|
when "--rule"
|
|
88
95
|
rule_number = Integer(required_argument!(args, argument), 10)
|
|
89
96
|
raise ArgumentError, "--rule must be non-negative" if rule_number.negative?
|
|
90
|
-
when "--
|
|
97
|
+
when "--grammar"
|
|
98
|
+
raise ArgumentError, "--grammar is only valid for check" unless command == :check
|
|
99
|
+
options.grammar = required_argument!(args, argument)
|
|
100
|
+
when "--input", "--generated"
|
|
101
|
+
raise ArgumentError, "#{argument} is only valid for the trace command" unless command == :trace
|
|
102
|
+
if argument == "--generated"
|
|
103
|
+
trace_args[:generated] = true
|
|
104
|
+
else
|
|
105
|
+
trace_args[:input] = args.shift || raise(ArgumentError, "--input requires a value")
|
|
106
|
+
end
|
|
107
|
+
when "--input-file"
|
|
108
|
+
value = required_argument!(args, argument)
|
|
109
|
+
if command == :trace
|
|
110
|
+
trace_args[:input_file] = value
|
|
111
|
+
elsif command == :bench
|
|
112
|
+
benchmark_args.push(argument, value)
|
|
113
|
+
else
|
|
114
|
+
raise ArgumentError, "#{argument} is only valid for the trace or bench command"
|
|
115
|
+
end
|
|
116
|
+
when "--baseline", "--iterations"
|
|
91
117
|
raise ArgumentError, "#{argument} is only valid for the bench command" unless command == :bench
|
|
92
118
|
benchmark_args.push(argument, required_argument!(args, argument))
|
|
93
119
|
else
|
|
@@ -96,14 +122,17 @@ module Flexr
|
|
|
96
122
|
end
|
|
97
123
|
|
|
98
124
|
options.validate!
|
|
99
|
-
|
|
125
|
+
raise ArgumentError, "choose --input or --input-file" if trace_args.key?(:input) && trace_args[:input_file]
|
|
126
|
+
raise ArgumentError, "--generated requires --input or --input-file" if
|
|
127
|
+
trace_args[:generated] && !trace_args.key?(:input) && !trace_args[:input_file]
|
|
128
|
+
[options, output, rule_number, benchmark_args, positionals, trace_args]
|
|
100
129
|
rescue ArgumentError => e
|
|
101
130
|
raise e if e.message.start_with?("unsupported ", "--rule")
|
|
102
131
|
|
|
103
132
|
raise ArgumentError, "invalid option value: #{e.message}"
|
|
104
133
|
end
|
|
105
134
|
|
|
106
|
-
def execute(command, spec, options, output, rule_number, benchmark_args, out, err)
|
|
135
|
+
def execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: {})
|
|
107
136
|
case command
|
|
108
137
|
when :check
|
|
109
138
|
check(spec, options, out)
|
|
@@ -111,14 +140,24 @@ module Flexr
|
|
|
111
140
|
print_stats(spec, options, out)
|
|
112
141
|
when :tokens
|
|
113
142
|
parsed = read_spec(spec)
|
|
114
|
-
|
|
143
|
+
tokens = Array(parsed.config[:declared_tokens])
|
|
144
|
+
if options.format == :lrama
|
|
145
|
+
require_relative "grammar"
|
|
146
|
+
out.puts Grammar.declarations(tokens)
|
|
147
|
+
else
|
|
148
|
+
out.puts tokens.join(" ")
|
|
149
|
+
end
|
|
115
150
|
EXIT_OK
|
|
116
151
|
when :dot
|
|
117
152
|
print_dot(spec, options, out)
|
|
118
153
|
when :explain
|
|
119
154
|
print_explanation(spec, rule_number, out)
|
|
120
155
|
when :trace
|
|
121
|
-
|
|
156
|
+
if trace_args.key?(:input) || trace_args[:input_file]
|
|
157
|
+
print_input_trace(spec, options, trace_args, out)
|
|
158
|
+
else
|
|
159
|
+
print_trace(spec, options, out)
|
|
160
|
+
end
|
|
122
161
|
when :bench
|
|
123
162
|
run_benchmark(spec, options, benchmark_args, out)
|
|
124
163
|
when :import
|
|
@@ -146,18 +185,26 @@ module Flexr
|
|
|
146
185
|
generator.generate
|
|
147
186
|
generator.diagnostics
|
|
148
187
|
else
|
|
149
|
-
|
|
188
|
+
parsed = read_spec(spec)
|
|
189
|
+
compile(parsed, overrides: options.overrides)
|
|
150
190
|
end
|
|
151
191
|
diagnostics = (compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)).dup
|
|
192
|
+
if options.grammar
|
|
193
|
+
require_relative "grammar"
|
|
194
|
+
diagnostics.concat(Grammar.diagnostics(read_spec(spec), options.grammar))
|
|
195
|
+
end
|
|
152
196
|
diagnostics.select! do |diagnostic|
|
|
153
|
-
options.warn_level == :all ||
|
|
197
|
+
options.warn_level == :all ||
|
|
198
|
+
(options.warn_level == :default && diagnostic.severity != :info && diagnostic.code != "FLEXR-W016")
|
|
154
199
|
end
|
|
200
|
+
diagnostics.concat(Migration.diagnostics(parsed)) if options.target
|
|
155
201
|
set = DiagnosticSet.new
|
|
156
202
|
diagnostics.each { |diagnostic| set << diagnostic }
|
|
157
203
|
if options.format == :json
|
|
158
204
|
out.puts set.render(format: :json)
|
|
159
205
|
elsif !diagnostics.empty?
|
|
160
|
-
out.puts set.render(format: :human, color: options.color
|
|
206
|
+
out.puts set.render(format: :human, color: options.color, path: spec,
|
|
207
|
+
source: diagnostic_source(spec, normalize: options.eval_mode))
|
|
161
208
|
end
|
|
162
209
|
return EXIT_FAILURE if options.warn_as_error && diagnostics.any?(&:warning?)
|
|
163
210
|
|
|
@@ -257,6 +304,34 @@ module Flexr
|
|
|
257
304
|
EXIT_OK
|
|
258
305
|
end
|
|
259
306
|
|
|
307
|
+
def print_input_trace(spec, options, trace_args, out)
|
|
308
|
+
require_relative "trace"
|
|
309
|
+
parsed = read_spec(spec)
|
|
310
|
+
source = trace_args[:generated] ? generate(spec, options) : File.read(spec)
|
|
311
|
+
scope = Module.new
|
|
312
|
+
scope.module_eval(source, spec, 1)
|
|
313
|
+
namespace = parsed.class_name.start_with?("::") ? Object : scope
|
|
314
|
+
klass = parsed.class_name.split("::").reject(&:empty?).inject(namespace) { |parent, name| parent.const_get(name, false) }
|
|
315
|
+
unless trace_args[:generated]
|
|
316
|
+
options.overrides.each do |name, value|
|
|
317
|
+
klass.public_send(name, value) if %i[backend token_kind accel].include?(name)
|
|
318
|
+
end
|
|
319
|
+
klass.__flexr_config.options.merge!(options.overrides.slice(:max_dfa_states, :warn_level))
|
|
320
|
+
compiled = klass.compile!
|
|
321
|
+
if options.warn_as_error
|
|
322
|
+
diagnostic = Array(compiled.diagnostics).find do |item|
|
|
323
|
+
item.warning? && (options.warn_level == :all || item.code != "FLEXR-W016")
|
|
324
|
+
end
|
|
325
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic) if diagnostic
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
input = trace_args[:input_file] ? File.binread(trace_args[:input_file]) : trace_args.fetch(:input)
|
|
329
|
+
trace = Trace.new(klass.new(input, filename: trace_args[:input_file]))
|
|
330
|
+
trace.run
|
|
331
|
+
out.puts(options.format == :json ? JSON.pretty_generate(steps: trace.steps) : trace.render)
|
|
332
|
+
EXIT_OK
|
|
333
|
+
end
|
|
334
|
+
|
|
260
335
|
def run_benchmark(spec, options, benchmark_args, out)
|
|
261
336
|
require_relative "../../benchmark/run"
|
|
262
337
|
args = ["--spec", spec, *benchmark_args]
|
|
@@ -285,12 +360,12 @@ module Flexr
|
|
|
285
360
|
klass.option(name) if value
|
|
286
361
|
end
|
|
287
362
|
klass.__flexr_config.options[:max_dfa_states] = overrides[:max_dfa_states] if overrides[:max_dfa_states]
|
|
363
|
+
klass.__flexr_config.options[:warn_level] = overrides[:warn_level] if overrides[:warn_level]
|
|
288
364
|
accel = overrides.fetch(:accel, config_options[:accel])
|
|
289
365
|
klass.accel(accel) if accel
|
|
290
366
|
parsed.states.each do |name, value|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
klass.state(name, inclusive: value[:inclusive]) { nil }
|
|
367
|
+
klass.state(name, inclusive: value[:inclusive]) { nil } unless name.to_sym == :initial
|
|
368
|
+
klass.__flexr_states.fetch(name.to_sym).location = value[:location]
|
|
294
369
|
end
|
|
295
370
|
parsed.rules.each { |rule| klass.__flexr_add_generated_rule(rule.to_h) }
|
|
296
371
|
klass.compile!
|
|
@@ -303,10 +378,30 @@ module Flexr
|
|
|
303
378
|
value
|
|
304
379
|
end
|
|
305
380
|
|
|
306
|
-
def
|
|
381
|
+
def diagnostic_source(path, normalize: false, max_bytes: nil)
|
|
382
|
+
return unless path
|
|
383
|
+
|
|
384
|
+
source = File.binread(path, max_bytes && (max_bytes + 1))
|
|
385
|
+
return if max_bytes && source.bytesize > max_bytes
|
|
386
|
+
|
|
387
|
+
source.force_encoding(Encoding::UTF_8)
|
|
388
|
+
normalize ? source.gsub(/\r\n?/, "\n") : source
|
|
389
|
+
rescue SystemCallError, IOError
|
|
390
|
+
nil
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def render_error(error, options: nil, path: nil, normalize_source: false)
|
|
307
394
|
if error.respond_to?(:diagnostic) && error.diagnostic
|
|
395
|
+
if error.is_a?(LexError) || (error.is_a?(Error) && error.byte_pos)
|
|
396
|
+
path = error.filename
|
|
397
|
+
normalize_source = false
|
|
398
|
+
# shortcut: omit input excerpts above 64 KiB; use bounded line reads if larger inputs need excerpts.
|
|
399
|
+
max_bytes = 64 * 1024
|
|
400
|
+
end
|
|
401
|
+
source = diagnostic_source(path, normalize: normalize_source, max_bytes: max_bytes) if
|
|
402
|
+
error.diagnostic.location && options&.format != :json
|
|
308
403
|
return DiagnosticSet.new.tap { |set| set << error.diagnostic }.render(
|
|
309
|
-
format: options&.format || :human, color: options&.color || :auto
|
|
404
|
+
format: options&.format || :human, color: options&.color || :auto, source: source, path: path
|
|
310
405
|
)
|
|
311
406
|
end
|
|
312
407
|
|
|
@@ -340,7 +435,12 @@ module Flexr
|
|
|
340
435
|
-W, --warn LEVEL all | default | none
|
|
341
436
|
--warn-as-error
|
|
342
437
|
--color WHEN auto | always | never
|
|
343
|
-
--format FMT human | json
|
|
438
|
+
--format FMT human | json | lrama (tokens only)
|
|
439
|
+
--grammar PATH compare emits with grammar %token declarations (check only)
|
|
440
|
+
--input STRING trace input (executes SPEC actions)
|
|
441
|
+
--input-file PATH trace or benchmark input
|
|
442
|
+
--generated trace generated scanner
|
|
443
|
+
--target 2.0 report migration requirements (static check only)
|
|
344
444
|
USAGE
|
|
345
445
|
status
|
|
346
446
|
end
|
|
@@ -3,14 +3,8 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Codegen
|
|
5
5
|
class Firstmatch < Base
|
|
6
|
-
def initialize(compiled, experimental: false)
|
|
7
|
-
raise CompileError, "firstmatch requires option :experimental" unless experimental
|
|
8
|
-
|
|
9
|
-
super(compiled)
|
|
10
|
-
end
|
|
11
|
-
|
|
12
6
|
def generate
|
|
13
|
-
compiled.rules.
|
|
7
|
+
compiled.rules.flat_map(&:patterns).freeze
|
|
14
8
|
end
|
|
15
9
|
end
|
|
16
10
|
end
|
data/lib/flexr/codegen/table.rb
CHANGED
|
@@ -27,11 +27,13 @@ module Flexr
|
|
|
27
27
|
|
|
28
28
|
def source(indent: " ")
|
|
29
29
|
lines = []
|
|
30
|
+
whole_tokens = compiled.machines.values.any?(&:whole_tokens)
|
|
30
31
|
lines << "#{indent}def scan_one"
|
|
31
32
|
lines << "#{indent} if @__flexr_generated_scan_state != @state"
|
|
32
33
|
lines << "#{indent} machine = self.class.__flexr_compiled.machines.fetch(@state)"
|
|
33
34
|
lines << "#{indent} @__flexr_generated_scan_state = @state"
|
|
34
35
|
lines << "#{indent} @__flexr_generated_scan_dfa = machine.dfa"
|
|
36
|
+
lines << "#{indent} @__flexr_generated_whole_tokens = machine.respond_to?(:whole_tokens) ? machine.whole_tokens : nil" if whole_tokens
|
|
35
37
|
lines << "#{indent} @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)"
|
|
36
38
|
lines << "#{indent} @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)"
|
|
37
39
|
lines << "#{indent} end"
|
|
@@ -39,6 +41,10 @@ module Flexr
|
|
|
39
41
|
lines << "#{indent} return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast"
|
|
40
42
|
lines << "#{indent} position = byte_pos"
|
|
41
43
|
lines << "#{indent} return nil unless valid_utf8_at?(position)"
|
|
44
|
+
if whole_tokens
|
|
45
|
+
lines << "#{indent} match = __flexr_whole_token_match(@__flexr_generated_whole_tokens, position) if @__flexr_generated_whole_tokens"
|
|
46
|
+
lines << "#{indent} return match if match"
|
|
47
|
+
end
|
|
42
48
|
lines << "#{indent} return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple"
|
|
43
49
|
lines << "#{indent} cursor = position"
|
|
44
50
|
lines << "#{indent} current = dfa.start"
|
|
@@ -201,8 +207,8 @@ module Flexr
|
|
|
201
207
|
lines << "#{indent} @__flexr_generated_accel_scanner.pos = segment_position"
|
|
202
208
|
lines << "#{indent} @__flexr_generated_accel_scanner.skip(regexp)"
|
|
203
209
|
lines << "#{indent} else"
|
|
204
|
-
lines << "#{indent} match = regexp.match(segment,
|
|
205
|
-
lines << "#{indent} match && match.begin(0)
|
|
210
|
+
lines << "#{indent} match = regexp.match(segment.byteslice(segment_position..).to_s, 0)"
|
|
211
|
+
lines << "#{indent} match && match.begin(0).zero? ? match[0].bytesize : nil"
|
|
206
212
|
lines << "#{indent} end"
|
|
207
213
|
lines << "#{indent} rescue ArgumentError"
|
|
208
214
|
lines << "#{indent} nil"
|
data/lib/flexr/configuration.rb
CHANGED
|
@@ -21,6 +21,14 @@ module Flexr
|
|
|
21
21
|
validated_symbol!(value, ACCELERATORS, :accel)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
def encoding!(value)
|
|
25
|
+
encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
|
|
26
|
+
return encoding if [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
|
|
27
|
+
|
|
28
|
+
diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
|
|
29
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
30
|
+
end
|
|
31
|
+
|
|
24
32
|
def option!(value)
|
|
25
33
|
validated_symbol!(value, BOOLEAN_OPTIONS, :option)
|
|
26
34
|
end
|