flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -6,65 +6,113 @@ module Flexr
|
|
|
6
6
|
module_function
|
|
7
7
|
|
|
8
8
|
def minimize(dfa)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
dead = dfa.states
|
|
10
|
+
rows = dfa.transitions.map { |row| row.map { |destination| destination.nil? ? dead : destination } }
|
|
11
|
+
rows << Array.new(dfa.class_count, dead)
|
|
12
|
+
partitions = initial_partitions(dfa, dead)
|
|
13
|
+
predecessors = predecessor_index(rows, dfa.class_count)
|
|
14
|
+
partitions = refine(partitions, predecessors, dfa.class_count)
|
|
15
|
+
rebuild(dfa, partitions, dead)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def initial_partitions(dfa, dead)
|
|
19
|
+
(0..dead).group_by { |state| state == dead ? [] : dfa.accepts[state] }.values
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def predecessor_index(rows, class_count)
|
|
23
|
+
Array.new(class_count) { Array.new(rows.length) { [] } }.tap do |index|
|
|
24
|
+
rows.each_with_index do |row, state|
|
|
25
|
+
row.each_with_index { |destination, class_id| index[class_id][destination] << state }
|
|
16
26
|
end
|
|
17
|
-
|
|
27
|
+
end
|
|
28
|
+
end
|
|
18
29
|
|
|
19
|
-
|
|
30
|
+
def refine(partitions, predecessors, class_count)
|
|
31
|
+
worklist = partitions.dup
|
|
32
|
+
pending = {}.compare_by_identity
|
|
33
|
+
partitions.each { |partition| pending[partition] = true }
|
|
34
|
+
cursor = 0
|
|
35
|
+
while cursor < worklist.length
|
|
36
|
+
splitter = worklist.fetch(cursor)
|
|
37
|
+
cursor += 1
|
|
38
|
+
next unless pending.delete(splitter)
|
|
39
|
+
|
|
40
|
+
class_count.times do |class_id|
|
|
41
|
+
marked = splitter.each_with_object({}) do |state, result|
|
|
42
|
+
predecessors[class_id][state].each { |predecessor| result[predecessor] = true }
|
|
43
|
+
end
|
|
44
|
+
next if marked.empty?
|
|
45
|
+
|
|
46
|
+
partitions = split_partitions(partitions, marked, worklist, pending)
|
|
47
|
+
end
|
|
20
48
|
end
|
|
21
|
-
|
|
49
|
+
partitions
|
|
22
50
|
end
|
|
23
51
|
|
|
24
|
-
def
|
|
25
|
-
|
|
52
|
+
def split_partitions(partitions, marked, worklist, pending)
|
|
53
|
+
partitions.flat_map do |partition|
|
|
54
|
+
inside, outside = partition.partition { |state| marked[state] }
|
|
55
|
+
next [partition] if inside.empty? || outside.empty?
|
|
56
|
+
|
|
57
|
+
if pending.delete(partition)
|
|
58
|
+
enqueue(inside, worklist, pending)
|
|
59
|
+
enqueue(outside, worklist, pending)
|
|
60
|
+
else
|
|
61
|
+
enqueue(inside.length <= outside.length ? inside : outside, worklist, pending)
|
|
62
|
+
end
|
|
63
|
+
[inside, outside]
|
|
64
|
+
end
|
|
26
65
|
end
|
|
27
66
|
|
|
28
|
-
def
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
ids
|
|
67
|
+
def enqueue(partition, worklist, pending)
|
|
68
|
+
worklist << partition
|
|
69
|
+
pending[partition] = true
|
|
32
70
|
end
|
|
33
71
|
|
|
34
|
-
def rebuild(dfa, partitions)
|
|
72
|
+
def rebuild(dfa, partitions, dead)
|
|
35
73
|
groups = group_ids(partitions)
|
|
74
|
+
dead_group = groups.fetch(dead)
|
|
36
75
|
start_group = groups.fetch(dfa.start)
|
|
37
|
-
|
|
76
|
+
representatives = partitions.map { |partition| partition.find { |state| state != dead } }
|
|
77
|
+
order = reachable_groups(dfa, groups, representatives, start_group, dead_group)
|
|
38
78
|
index = order.each_with_index.to_h
|
|
39
79
|
transitions = order.map do |group|
|
|
40
|
-
representative =
|
|
41
|
-
dfa.transitions
|
|
80
|
+
representative = representatives.fetch(group)
|
|
81
|
+
dfa.transitions.fetch(representative).map do |destination|
|
|
82
|
+
next nil if destination.nil? || groups.fetch(destination) == dead_group
|
|
83
|
+
|
|
84
|
+
index.fetch(groups.fetch(destination))
|
|
85
|
+
end
|
|
42
86
|
end
|
|
43
|
-
accepts = order.map { |group| dfa.accepts
|
|
87
|
+
accepts = order.map { |group| dfa.accepts.fetch(representatives.fetch(group)) }
|
|
44
88
|
rule_ids = accepts.flatten.map(&:rule_index).uniq.sort
|
|
45
89
|
DFA.new(transitions: transitions, accepts: accepts, ec: dfa.ec, class_count: dfa.class_count,
|
|
46
90
|
start: 0, rule_ids: rule_ids)
|
|
47
91
|
end
|
|
48
92
|
|
|
49
|
-
def
|
|
93
|
+
def group_ids(partitions)
|
|
94
|
+
partitions.each_with_index.with_object({}) do |(partition, group), result|
|
|
95
|
+
partition.each { |state| result[state] = group }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def reachable_groups(dfa, groups, representatives, start_group, dead_group)
|
|
50
100
|
result = []
|
|
51
101
|
queue = [start_group]
|
|
52
102
|
seen = {}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
103
|
+
cursor = 0
|
|
104
|
+
while cursor < queue.length
|
|
105
|
+
group = queue.fetch(cursor)
|
|
106
|
+
cursor += 1
|
|
107
|
+
next if seen[group] || (group == dead_group && group != start_group)
|
|
56
108
|
|
|
57
109
|
seen[group] = true
|
|
58
110
|
result << group
|
|
59
|
-
representative =
|
|
60
|
-
representative.compact.each { |destination| queue << groups
|
|
111
|
+
representative = representatives.fetch(group)
|
|
112
|
+
dfa.transitions.fetch(representative).compact.each { |destination| queue << groups.fetch(destination) }
|
|
61
113
|
end
|
|
62
114
|
result
|
|
63
115
|
end
|
|
64
|
-
|
|
65
|
-
def group_members(dfa, groups, group)
|
|
66
|
-
(0...dfa.states).select { |state| groups[state] == group }
|
|
67
|
-
end
|
|
68
116
|
end
|
|
69
117
|
end
|
|
70
118
|
end
|
data/lib/flexr/automaton/nfa.rb
CHANGED
|
@@ -4,11 +4,6 @@ module Flexr
|
|
|
4
4
|
module Automaton
|
|
5
5
|
NFAState = Struct.new(:epsilon, :transitions, :accepts, keyword_init: true)
|
|
6
6
|
NFATransition = Struct.new(:lo, :hi, :to, keyword_init: true)
|
|
7
|
-
Acceptance = Struct.new(:rule_index, :pattern_index, :bol_only, :end_anchor, keyword_init: true) do
|
|
8
|
-
def inspect
|
|
9
|
-
[rule_index, pattern_index, bol_only, end_anchor].inspect
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
7
|
|
|
13
8
|
class NFA
|
|
14
9
|
attr_reader :states, :start, :byte_classes
|
|
@@ -56,6 +51,8 @@ module Flexr
|
|
|
56
51
|
when Regexp::AST::Empty
|
|
57
52
|
state = @nfa.new_state
|
|
58
53
|
[state, state]
|
|
54
|
+
when Regexp::AST::Fail
|
|
55
|
+
[@nfa.new_state, @nfa.new_state]
|
|
59
56
|
when Regexp::AST::ByteRange
|
|
60
57
|
from = @nfa.new_state
|
|
61
58
|
to = @nfa.new_state
|
|
@@ -75,18 +72,53 @@ module Flexr
|
|
|
75
72
|
end
|
|
76
73
|
[from, to]
|
|
77
74
|
when Regexp::AST::Star
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
@nfa.epsilon(from, to)
|
|
82
|
-
@nfa.epsilon(from, child_start)
|
|
83
|
-
@nfa.epsilon(child_end, child_start)
|
|
84
|
-
@nfa.epsilon(child_end, to)
|
|
85
|
-
[from, to]
|
|
75
|
+
repeat_fragment(node.child, 0, nil)
|
|
76
|
+
when Regexp::AST::Repeat
|
|
77
|
+
repeat_fragment(node.child, node.minimum, node.maximum)
|
|
86
78
|
else
|
|
87
79
|
raise CompileError, "cannot build NFA from #{node.class}"
|
|
88
80
|
end
|
|
89
81
|
end
|
|
82
|
+
|
|
83
|
+
def repeat_fragment(child, minimum, maximum)
|
|
84
|
+
return fragment(Regexp::AST::Empty.new(loc: nil)) if maximum&.zero?
|
|
85
|
+
return star_fragment(child) if minimum.zero? && maximum.nil?
|
|
86
|
+
|
|
87
|
+
from = @nfa.new_state
|
|
88
|
+
cursor = from
|
|
89
|
+
last_start = nil
|
|
90
|
+
minimum.times do
|
|
91
|
+
child_start, child_end = fragment(child)
|
|
92
|
+
@nfa.epsilon(cursor, child_start)
|
|
93
|
+
cursor = child_end
|
|
94
|
+
last_start = child_start
|
|
95
|
+
end
|
|
96
|
+
if maximum.nil?
|
|
97
|
+
@nfa.epsilon(cursor, last_start)
|
|
98
|
+
return [from, cursor]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
(maximum - minimum).times do
|
|
102
|
+
child_start, child_end = fragment(child)
|
|
103
|
+
next_cursor = @nfa.new_state
|
|
104
|
+
@nfa.epsilon(cursor, next_cursor)
|
|
105
|
+
@nfa.epsilon(cursor, child_start)
|
|
106
|
+
@nfa.epsilon(child_end, next_cursor)
|
|
107
|
+
cursor = next_cursor
|
|
108
|
+
end
|
|
109
|
+
[from, cursor]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def star_fragment(child)
|
|
113
|
+
from = @nfa.new_state
|
|
114
|
+
to = @nfa.new_state
|
|
115
|
+
child_start, child_end = fragment(child)
|
|
116
|
+
@nfa.epsilon(from, to)
|
|
117
|
+
@nfa.epsilon(from, child_start)
|
|
118
|
+
@nfa.epsilon(child_end, child_start)
|
|
119
|
+
@nfa.epsilon(child_end, to)
|
|
120
|
+
[from, to]
|
|
121
|
+
end
|
|
90
122
|
end
|
|
91
123
|
end
|
|
92
124
|
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Automaton
|
|
5
|
+
Acceptance = Struct.new(:rule_index, :pattern_index, :bol_only, :end_anchor, keyword_init: true) do
|
|
6
|
+
def inspect
|
|
7
|
+
[rule_index, pattern_index, bol_only, end_anchor].inspect
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
|
|
12
|
+
Machine = Struct.new(:dfa, :state_name, :whole_tokens, keyword_init: true)
|
|
13
|
+
end
|
|
14
|
+
end
|
data/lib/flexr/cli.rb
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "runtime" unless defined?(Flexr::Lexer)
|
|
4
|
+
require_relative "artifact_writer"
|
|
5
|
+
require_relative "generator"
|
|
6
|
+
require_relative "importer"
|
|
7
|
+
require_relative "options"
|
|
8
|
+
require_relative "migration"
|
|
9
|
+
|
|
3
10
|
module Flexr
|
|
4
11
|
module CLI
|
|
5
12
|
COMMANDS = %w[check stats tokens dot explain trace bench import].freeze
|
|
@@ -15,17 +22,19 @@ module Flexr
|
|
|
15
22
|
return version(out) if args.delete("--version")
|
|
16
23
|
|
|
17
24
|
command = COMMANDS.include?(args.first) ? args.shift.to_sym : :generate
|
|
18
|
-
options, output, rule_number, benchmark_args, positionals = parse(args, command)
|
|
25
|
+
options, output, rule_number, benchmark_args, positionals, trace_args = parse(args, command)
|
|
26
|
+
raise ArgumentError, "lrama format is only valid for tokens" if options.format == :lrama && command != :tokens
|
|
19
27
|
spec = positionals.shift
|
|
20
28
|
raise ArgumentError, "a SPEC.rb path is required" unless spec
|
|
21
29
|
raise ArgumentError, "unexpected argument: #{positionals.first}" unless positionals.empty?
|
|
22
30
|
|
|
23
|
-
execute(command, spec, options, output, rule_number, benchmark_args, out, err)
|
|
31
|
+
execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: trace_args)
|
|
24
32
|
rescue ArgumentError => e
|
|
25
33
|
err.puts "error: #{e.message}"
|
|
26
34
|
usage(err, status: EXIT_USAGE)
|
|
27
35
|
rescue Flexr::Error, Errno::ENOENT, Errno::EACCES, SyntaxError => e
|
|
28
|
-
err.puts render_error(e, options: options
|
|
36
|
+
err.puts render_error(e, options: options, path: spec,
|
|
37
|
+
normalize_source: command == :generate || (command == :check && options&.eval_mode))
|
|
29
38
|
EXIT_FAILURE
|
|
30
39
|
rescue StandardError => e
|
|
31
40
|
err.puts "error: #{e.class}: #{e.message}"
|
|
@@ -38,6 +47,7 @@ module Flexr
|
|
|
38
47
|
rule_number = nil
|
|
39
48
|
benchmark_args = []
|
|
40
49
|
positionals = []
|
|
50
|
+
trace_args = {}
|
|
41
51
|
parsing_options = true
|
|
42
52
|
|
|
43
53
|
until args.empty?
|
|
@@ -78,10 +88,32 @@ module Flexr
|
|
|
78
88
|
options.color = required_argument!(args, argument).to_sym
|
|
79
89
|
when "--format"
|
|
80
90
|
options.format = required_argument!(args, argument).to_sym
|
|
91
|
+
when "--target"
|
|
92
|
+
raise ArgumentError, "--target is only valid for the check command" unless command == :check
|
|
93
|
+
options.target = required_argument!(args, argument)
|
|
81
94
|
when "--rule"
|
|
82
95
|
rule_number = Integer(required_argument!(args, argument), 10)
|
|
83
96
|
raise ArgumentError, "--rule must be non-negative" if rule_number.negative?
|
|
84
|
-
when "--
|
|
97
|
+
when "--grammar"
|
|
98
|
+
raise ArgumentError, "--grammar is only valid for check" unless command == :check
|
|
99
|
+
options.grammar = required_argument!(args, argument)
|
|
100
|
+
when "--input", "--generated"
|
|
101
|
+
raise ArgumentError, "#{argument} is only valid for the trace command" unless command == :trace
|
|
102
|
+
if argument == "--generated"
|
|
103
|
+
trace_args[:generated] = true
|
|
104
|
+
else
|
|
105
|
+
trace_args[:input] = args.shift || raise(ArgumentError, "--input requires a value")
|
|
106
|
+
end
|
|
107
|
+
when "--input-file"
|
|
108
|
+
value = required_argument!(args, argument)
|
|
109
|
+
if command == :trace
|
|
110
|
+
trace_args[:input_file] = value
|
|
111
|
+
elsif command == :bench
|
|
112
|
+
benchmark_args.push(argument, value)
|
|
113
|
+
else
|
|
114
|
+
raise ArgumentError, "#{argument} is only valid for the trace or bench command"
|
|
115
|
+
end
|
|
116
|
+
when "--baseline", "--iterations"
|
|
85
117
|
raise ArgumentError, "#{argument} is only valid for the bench command" unless command == :bench
|
|
86
118
|
benchmark_args.push(argument, required_argument!(args, argument))
|
|
87
119
|
else
|
|
@@ -90,14 +122,17 @@ module Flexr
|
|
|
90
122
|
end
|
|
91
123
|
|
|
92
124
|
options.validate!
|
|
93
|
-
|
|
125
|
+
raise ArgumentError, "choose --input or --input-file" if trace_args.key?(:input) && trace_args[:input_file]
|
|
126
|
+
raise ArgumentError, "--generated requires --input or --input-file" if
|
|
127
|
+
trace_args[:generated] && !trace_args.key?(:input) && !trace_args[:input_file]
|
|
128
|
+
[options, output, rule_number, benchmark_args, positionals, trace_args]
|
|
94
129
|
rescue ArgumentError => e
|
|
95
130
|
raise e if e.message.start_with?("unsupported ", "--rule")
|
|
96
131
|
|
|
97
132
|
raise ArgumentError, "invalid option value: #{e.message}"
|
|
98
133
|
end
|
|
99
134
|
|
|
100
|
-
def execute(command, spec, options, output, rule_number, benchmark_args, out, err)
|
|
135
|
+
def execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: {})
|
|
101
136
|
case command
|
|
102
137
|
when :check
|
|
103
138
|
check(spec, options, out)
|
|
@@ -105,14 +140,24 @@ module Flexr
|
|
|
105
140
|
print_stats(spec, options, out)
|
|
106
141
|
when :tokens
|
|
107
142
|
parsed = read_spec(spec)
|
|
108
|
-
|
|
143
|
+
tokens = Array(parsed.config[:declared_tokens])
|
|
144
|
+
if options.format == :lrama
|
|
145
|
+
require_relative "grammar"
|
|
146
|
+
out.puts Grammar.declarations(tokens)
|
|
147
|
+
else
|
|
148
|
+
out.puts tokens.join(" ")
|
|
149
|
+
end
|
|
109
150
|
EXIT_OK
|
|
110
151
|
when :dot
|
|
111
152
|
print_dot(spec, options, out)
|
|
112
153
|
when :explain
|
|
113
154
|
print_explanation(spec, rule_number, out)
|
|
114
155
|
when :trace
|
|
115
|
-
|
|
156
|
+
if trace_args.key?(:input) || trace_args[:input_file]
|
|
157
|
+
print_input_trace(spec, options, trace_args, out)
|
|
158
|
+
else
|
|
159
|
+
print_trace(spec, options, out)
|
|
160
|
+
end
|
|
116
161
|
when :bench
|
|
117
162
|
run_benchmark(spec, options, benchmark_args, out)
|
|
118
163
|
when :import
|
|
@@ -121,13 +166,13 @@ module Flexr
|
|
|
121
166
|
return EXIT_FAILURE unless result.complete?
|
|
122
167
|
|
|
123
168
|
if output
|
|
124
|
-
|
|
169
|
+
ArtifactWriter.write!(output, result.source, source_path: spec)
|
|
125
170
|
else
|
|
126
171
|
out.write(result.source)
|
|
127
172
|
end
|
|
128
173
|
EXIT_OK
|
|
129
174
|
when :generate
|
|
130
|
-
target = output ||
|
|
175
|
+
target = output || ArtifactWriter.default_generated_path(spec)
|
|
131
176
|
Generator.new(spec, output: target, eval_mode: options.eval_mode,
|
|
132
177
|
options: options.generator_options).generate
|
|
133
178
|
EXIT_OK
|
|
@@ -140,18 +185,26 @@ module Flexr
|
|
|
140
185
|
generator.generate
|
|
141
186
|
generator.diagnostics
|
|
142
187
|
else
|
|
143
|
-
|
|
188
|
+
parsed = read_spec(spec)
|
|
189
|
+
compile(parsed, overrides: options.overrides)
|
|
144
190
|
end
|
|
145
|
-
diagnostics = compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)
|
|
191
|
+
diagnostics = (compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)).dup
|
|
192
|
+
if options.grammar
|
|
193
|
+
require_relative "grammar"
|
|
194
|
+
diagnostics.concat(Grammar.diagnostics(read_spec(spec), options.grammar))
|
|
195
|
+
end
|
|
146
196
|
diagnostics.select! do |diagnostic|
|
|
147
|
-
options.warn_level == :all ||
|
|
197
|
+
options.warn_level == :all ||
|
|
198
|
+
(options.warn_level == :default && diagnostic.severity != :info && diagnostic.code != "FLEXR-W016")
|
|
148
199
|
end
|
|
200
|
+
diagnostics.concat(Migration.diagnostics(parsed)) if options.target
|
|
149
201
|
set = DiagnosticSet.new
|
|
150
202
|
diagnostics.each { |diagnostic| set << diagnostic }
|
|
151
203
|
if options.format == :json
|
|
152
204
|
out.puts set.render(format: :json)
|
|
153
205
|
elsif !diagnostics.empty?
|
|
154
|
-
out.puts set.render(format: :human, color: options.color
|
|
206
|
+
out.puts set.render(format: :human, color: options.color, path: spec,
|
|
207
|
+
source: diagnostic_source(spec, normalize: options.eval_mode))
|
|
155
208
|
end
|
|
156
209
|
return EXIT_FAILURE if options.warn_as_error && diagnostics.any?(&:warning?)
|
|
157
210
|
|
|
@@ -251,6 +304,34 @@ module Flexr
|
|
|
251
304
|
EXIT_OK
|
|
252
305
|
end
|
|
253
306
|
|
|
307
|
+
def print_input_trace(spec, options, trace_args, out)
|
|
308
|
+
require_relative "trace"
|
|
309
|
+
parsed = read_spec(spec)
|
|
310
|
+
source = trace_args[:generated] ? generate(spec, options) : File.read(spec)
|
|
311
|
+
scope = Module.new
|
|
312
|
+
scope.module_eval(source, spec, 1)
|
|
313
|
+
namespace = parsed.class_name.start_with?("::") ? Object : scope
|
|
314
|
+
klass = parsed.class_name.split("::").reject(&:empty?).inject(namespace) { |parent, name| parent.const_get(name, false) }
|
|
315
|
+
unless trace_args[:generated]
|
|
316
|
+
options.overrides.each do |name, value|
|
|
317
|
+
klass.public_send(name, value) if %i[backend token_kind accel].include?(name)
|
|
318
|
+
end
|
|
319
|
+
klass.__flexr_config.options.merge!(options.overrides.slice(:max_dfa_states, :warn_level))
|
|
320
|
+
compiled = klass.compile!
|
|
321
|
+
if options.warn_as_error
|
|
322
|
+
diagnostic = Array(compiled.diagnostics).find do |item|
|
|
323
|
+
item.warning? && (options.warn_level == :all || item.code != "FLEXR-W016")
|
|
324
|
+
end
|
|
325
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic) if diagnostic
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
input = trace_args[:input_file] ? File.binread(trace_args[:input_file]) : trace_args.fetch(:input)
|
|
329
|
+
trace = Trace.new(klass.new(input, filename: trace_args[:input_file]))
|
|
330
|
+
trace.run
|
|
331
|
+
out.puts(options.format == :json ? JSON.pretty_generate(steps: trace.steps) : trace.render)
|
|
332
|
+
EXIT_OK
|
|
333
|
+
end
|
|
334
|
+
|
|
254
335
|
def run_benchmark(spec, options, benchmark_args, out)
|
|
255
336
|
require_relative "../../benchmark/run"
|
|
256
337
|
args = ["--spec", spec, *benchmark_args]
|
|
@@ -275,16 +356,16 @@ module Flexr
|
|
|
275
356
|
klass.encoding(config.fetch(:encoding, Encoding::UTF_8))
|
|
276
357
|
Array(config[:declared_tokens]).each { |token| klass.emits(token) }
|
|
277
358
|
config_options = config.fetch(:options, {}).merge(overrides.slice(:experimental, :allow_empty_match))
|
|
278
|
-
config_options.each do |name, value|
|
|
279
|
-
|
|
359
|
+
config_options.slice(*Configuration::BOOLEAN_OPTIONS).each do |name, value|
|
|
360
|
+
klass.option(name) if value
|
|
280
361
|
end
|
|
281
362
|
klass.__flexr_config.options[:max_dfa_states] = overrides[:max_dfa_states] if overrides[:max_dfa_states]
|
|
363
|
+
klass.__flexr_config.options[:warn_level] = overrides[:warn_level] if overrides[:warn_level]
|
|
282
364
|
accel = overrides.fetch(:accel, config_options[:accel])
|
|
283
365
|
klass.accel(accel) if accel
|
|
284
366
|
parsed.states.each do |name, value|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
klass.state(name, inclusive: value[:inclusive]) { nil }
|
|
367
|
+
klass.state(name, inclusive: value[:inclusive]) { nil } unless name.to_sym == :initial
|
|
368
|
+
klass.__flexr_states.fetch(name.to_sym).location = value[:location]
|
|
288
369
|
end
|
|
289
370
|
parsed.rules.each { |rule| klass.__flexr_add_generated_rule(rule.to_h) }
|
|
290
371
|
klass.compile!
|
|
@@ -297,10 +378,30 @@ module Flexr
|
|
|
297
378
|
value
|
|
298
379
|
end
|
|
299
380
|
|
|
300
|
-
def
|
|
381
|
+
def diagnostic_source(path, normalize: false, max_bytes: nil)
|
|
382
|
+
return unless path
|
|
383
|
+
|
|
384
|
+
source = File.binread(path, max_bytes && (max_bytes + 1))
|
|
385
|
+
return if max_bytes && source.bytesize > max_bytes
|
|
386
|
+
|
|
387
|
+
source.force_encoding(Encoding::UTF_8)
|
|
388
|
+
normalize ? source.gsub(/\r\n?/, "\n") : source
|
|
389
|
+
rescue SystemCallError, IOError
|
|
390
|
+
nil
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def render_error(error, options: nil, path: nil, normalize_source: false)
|
|
301
394
|
if error.respond_to?(:diagnostic) && error.diagnostic
|
|
395
|
+
if error.is_a?(LexError) || (error.is_a?(Error) && error.byte_pos)
|
|
396
|
+
path = error.filename
|
|
397
|
+
normalize_source = false
|
|
398
|
+
# shortcut: omit input excerpts above 64 KiB; use bounded line reads if larger inputs need excerpts.
|
|
399
|
+
max_bytes = 64 * 1024
|
|
400
|
+
end
|
|
401
|
+
source = diagnostic_source(path, normalize: normalize_source, max_bytes: max_bytes) if
|
|
402
|
+
error.diagnostic.location && options&.format != :json
|
|
302
403
|
return DiagnosticSet.new.tap { |set| set << error.diagnostic }.render(
|
|
303
|
-
format: options&.format || :human, color: options&.color || :auto
|
|
404
|
+
format: options&.format || :human, color: options&.color || :auto, source: source, path: path
|
|
304
405
|
)
|
|
305
406
|
end
|
|
306
407
|
|
|
@@ -334,7 +435,12 @@ module Flexr
|
|
|
334
435
|
-W, --warn LEVEL all | default | none
|
|
335
436
|
--warn-as-error
|
|
336
437
|
--color WHEN auto | always | never
|
|
337
|
-
--format FMT human | json
|
|
438
|
+
--format FMT human | json | lrama (tokens only)
|
|
439
|
+
--grammar PATH compare emits with grammar %token declarations (check only)
|
|
440
|
+
--input STRING trace input (executes SPEC actions)
|
|
441
|
+
--input-file PATH trace or benchmark input
|
|
442
|
+
--generated trace generated scanner
|
|
443
|
+
--target 2.0 report migration requirements (static check only)
|
|
338
444
|
USAGE
|
|
339
445
|
status
|
|
340
446
|
end
|
data/lib/flexr/codegen/direct.rb
CHANGED
|
@@ -4,49 +4,13 @@ module Flexr
|
|
|
4
4
|
module Codegen
|
|
5
5
|
class Direct < Table
|
|
6
6
|
def generate(state: :initial)
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
dfa = compiled.machines.fetch(state).dfa
|
|
8
|
+
{
|
|
9
|
+
nxt: dfa.transitions.flatten.map { |value| value || -1 }.freeze,
|
|
10
|
+
classes: dfa.class_count
|
|
11
|
+
}.freeze
|
|
9
12
|
end
|
|
10
13
|
|
|
11
|
-
def source(indent: " ")
|
|
12
|
-
lines = []
|
|
13
|
-
lines << "#{indent}def self.__flexr_generated_direct_transition(state_name, state, byte)"
|
|
14
|
-
lines << "#{indent} case state_name"
|
|
15
|
-
compiled.machines.each do |state_name, machine|
|
|
16
|
-
lines << "#{indent} when #{state_name.inspect}"
|
|
17
|
-
lines << "#{indent} class_id = case byte"
|
|
18
|
-
byte_classes(machine.dfa.ec).each do |first, last, class_id|
|
|
19
|
-
selector = first == last ? first.to_s : "#{first}..#{last}"
|
|
20
|
-
lines << "#{indent} when #{selector} then #{class_id}"
|
|
21
|
-
end
|
|
22
|
-
lines << "#{indent} end"
|
|
23
|
-
lines << "#{indent} case state"
|
|
24
|
-
machine.dfa.transitions.each_with_index do |row, state|
|
|
25
|
-
lines << "#{indent} when #{state}"
|
|
26
|
-
lines << "#{indent} case class_id"
|
|
27
|
-
row.each_with_index do |destination, class_id|
|
|
28
|
-
value = destination.nil? ? "nil" : destination
|
|
29
|
-
lines << "#{indent} when #{class_id} then #{value}"
|
|
30
|
-
end
|
|
31
|
-
lines << "#{indent} else nil"
|
|
32
|
-
lines << "#{indent} end"
|
|
33
|
-
end
|
|
34
|
-
lines << "#{indent} else nil"
|
|
35
|
-
lines << "#{indent} end"
|
|
36
|
-
end
|
|
37
|
-
lines << "#{indent} else nil"
|
|
38
|
-
lines << "#{indent} end"
|
|
39
|
-
lines << "#{indent}end"
|
|
40
|
-
"#{lines.join("\n")}\n"
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
private
|
|
44
|
-
|
|
45
|
-
def byte_classes(ec)
|
|
46
|
-
ec.each_with_index.chunk_while { |left, right| right[0] == left[0] }.map do |run|
|
|
47
|
-
[run.first[1], run.last[1], run.first[0]]
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
14
|
end
|
|
51
15
|
end
|
|
52
16
|
end
|
|
@@ -3,14 +3,8 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Codegen
|
|
5
5
|
class Firstmatch < Base
|
|
6
|
-
def initialize(compiled, experimental: false)
|
|
7
|
-
raise CompileError, "firstmatch requires option :experimental" unless experimental
|
|
8
|
-
|
|
9
|
-
super(compiled)
|
|
10
|
-
end
|
|
11
|
-
|
|
12
6
|
def generate
|
|
13
|
-
compiled.rules.
|
|
7
|
+
compiled.rules.flat_map(&:patterns).freeze
|
|
14
8
|
end
|
|
15
9
|
end
|
|
16
10
|
end
|