flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -2,9 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Automaton
|
|
5
|
-
CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
|
|
6
|
-
Machine = Struct.new(:dfa, :state_name, keyword_init: true)
|
|
7
|
-
|
|
8
5
|
class Compiler
|
|
9
6
|
def initialize(spec)
|
|
10
7
|
@spec = spec
|
|
@@ -16,7 +13,13 @@ module Flexr
|
|
|
16
13
|
state_names = effective_states
|
|
17
14
|
machines = state_names.to_h do |state_name|
|
|
18
15
|
rules = rules_for(state_name)
|
|
19
|
-
|
|
16
|
+
dfa = compile_machine(rules)
|
|
17
|
+
plans = Accel.whole_token_plans(dfa, rules) if
|
|
18
|
+
@spec.backend != :firstmatch && @spec.options[:accel] == :strscan
|
|
19
|
+
[state_name, Machine.new(dfa: dfa, state_name: state_name, whole_tokens: plans)]
|
|
20
|
+
rescue CompileError => e
|
|
21
|
+
e.diagnostic.location ||= @spec.states.fetch(state_name).location if e.diagnostic
|
|
22
|
+
raise
|
|
20
23
|
end
|
|
21
24
|
stats = machines.transform_values { |machine| machine.dfa.stats }
|
|
22
25
|
compiled = CompiledSpec.new(machines: machines, rules: @spec.rules, states: state_names, stats: stats)
|
|
@@ -44,15 +47,10 @@ module Flexr
|
|
|
44
47
|
end
|
|
45
48
|
|
|
46
49
|
def compile_machine(rules)
|
|
47
|
-
@
|
|
50
|
+
@active_rules = rules
|
|
48
51
|
normalized = []
|
|
49
52
|
rules.each do |rule|
|
|
50
53
|
rule.pattern_conditions = []
|
|
51
|
-
if reference_rule?(rule)
|
|
52
|
-
validate_reference_patterns(rule)
|
|
53
|
-
next
|
|
54
|
-
end
|
|
55
|
-
|
|
56
54
|
rule.patterns.each_with_index do |pattern, pattern_index|
|
|
57
55
|
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
58
56
|
encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : @spec.encoding
|
|
@@ -66,6 +64,9 @@ module Flexr
|
|
|
66
64
|
normalized_ast = Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
|
|
67
65
|
normalized << [normalized_ast, condition]
|
|
68
66
|
end
|
|
67
|
+
rescue CompileError => e
|
|
68
|
+
e.diagnostic.location ||= rule.location if e.diagnostic
|
|
69
|
+
raise
|
|
69
70
|
end
|
|
70
71
|
return empty_dfa if normalized.empty?
|
|
71
72
|
|
|
@@ -78,32 +79,6 @@ module Flexr
|
|
|
78
79
|
DFA.new(transitions: [[nil]], accepts: [[]], ec: Array.new(256, 0), class_count: 1, start: 0, rule_ids: [])
|
|
79
80
|
end
|
|
80
81
|
|
|
81
|
-
def reference_rule?(rule)
|
|
82
|
-
rule.patterns.any? { |pattern| reference_pattern?(pattern) }
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def reference_pattern?(pattern)
|
|
86
|
-
return false unless pattern.is_a?(::Regexp)
|
|
87
|
-
return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
|
|
88
|
-
|
|
89
|
-
@spec.options[:unicode] == true && @spec.encoding != Encoding::BINARY &&
|
|
90
|
-
pattern.source.match?(/\\[dDwWsS]/)
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def validate_reference_patterns(rule)
|
|
94
|
-
rule.patterns.each_with_index do |pattern, pattern_index|
|
|
95
|
-
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
96
|
-
encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : @spec.encoding
|
|
97
|
-
parser = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
|
|
98
|
-
unicode: @spec.options[:unicode] == true)
|
|
99
|
-
ast = parser.parse
|
|
100
|
-
_body, bol_only, end_anchor = strip_anchors(ast)
|
|
101
|
-
rule.pattern_conditions[pattern_index] = Acceptance.new(
|
|
102
|
-
rule_index: rule.index, pattern_index: pattern_index, bol_only: bol_only, end_anchor: end_anchor
|
|
103
|
-
)
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
|
|
107
82
|
def strip_anchors(ast)
|
|
108
83
|
children = ast.is_a?(Regexp::AST::Seq) ? ast.children.dup : [ast]
|
|
109
84
|
children.shift while children.first.is_a?(Regexp::AST::Empty)
|
|
@@ -131,37 +106,40 @@ module Flexr
|
|
|
131
106
|
|
|
132
107
|
def contains_anchor?(node)
|
|
133
108
|
return true if node.is_a?(Regexp::AST::Anchor)
|
|
109
|
+
return contains_anchor?(node.child) if node.is_a?(Regexp::AST::Repeat) || node.is_a?(Regexp::AST::Star)
|
|
134
110
|
return false unless node.respond_to?(:children)
|
|
135
111
|
|
|
136
112
|
node.children.any? { |child| contains_anchor?(child) }
|
|
137
113
|
end
|
|
138
114
|
|
|
139
115
|
def subset_construction(nfa, ec, class_count)
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
116
|
+
closures = {}
|
|
117
|
+
closure_for = lambda do |state|
|
|
118
|
+
closures[state] ||= epsilon_closure_for(nfa, state)
|
|
119
|
+
end
|
|
120
|
+
transition_closures = indexed_transition_closures(nfa, ec, closure_for)
|
|
121
|
+
start_set = closure_for.call(nfa.start)
|
|
143
122
|
sets = [start_set]
|
|
144
123
|
ids = { start_set => 0 }
|
|
145
124
|
transitions = []
|
|
146
125
|
accepts = []
|
|
147
126
|
queue = [start_set]
|
|
127
|
+
queue_index = 0
|
|
128
|
+
limit = @spec.options.fetch(:max_dfa_states, 100_000)
|
|
129
|
+
limit = 100_000 unless limit.is_a?(Integer) && limit.positive?
|
|
148
130
|
|
|
149
|
-
|
|
150
|
-
set = queue.
|
|
131
|
+
while queue_index < queue.length
|
|
132
|
+
set = queue.fetch(queue_index)
|
|
133
|
+
queue_index += 1
|
|
151
134
|
state_id = ids.fetch(set)
|
|
152
135
|
transitions[state_id] ||= Array.new(class_count)
|
|
153
136
|
accepts[state_id] = accepting_rules(nfa, set)
|
|
154
|
-
|
|
155
|
-
moved = move(nfa, set, representatives[class_id])
|
|
156
|
-
next if moved.zero?
|
|
157
|
-
closure = epsilon_closure(nfa, moved)
|
|
137
|
+
move_closures(set, transition_closures).each do |class_id, closure|
|
|
158
138
|
destination = ids[closure]
|
|
159
139
|
unless destination
|
|
160
140
|
destination = sets.length
|
|
161
|
-
limit = @spec.options.fetch(:max_dfa_states, 100_000)
|
|
162
|
-
limit = 100_000 unless limit.is_a?(Integer) && limit.positive?
|
|
163
141
|
if destination >= limit
|
|
164
|
-
message = "DFA state limit exceeded while compiling
|
|
142
|
+
message = "DFA state limit exceeded while compiling #{@active_rules.map { |rule| Diagnostics.rule_label(rule) }.join(', ')}"
|
|
165
143
|
diagnostic = Diagnostics.error("FLEXR-E006", message,
|
|
166
144
|
help: "raise max_dfa_states or split the listed rules")
|
|
167
145
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
@@ -180,13 +158,12 @@ module Flexr
|
|
|
180
158
|
Minimizer.minimize(dfa)
|
|
181
159
|
end
|
|
182
160
|
|
|
183
|
-
def
|
|
184
|
-
closure =
|
|
185
|
-
stack = []
|
|
186
|
-
nfa.states.each_index { |id| stack << id if set.anybits?(1 << id) }
|
|
161
|
+
def epsilon_closure_for(nfa, state)
|
|
162
|
+
closure = 1 << state
|
|
163
|
+
stack = [state]
|
|
187
164
|
until stack.empty?
|
|
188
|
-
|
|
189
|
-
nfa.states[
|
|
165
|
+
current = stack.pop
|
|
166
|
+
nfa.states[current].epsilon.each do |target|
|
|
190
167
|
next if closure.anybits?(1 << target)
|
|
191
168
|
|
|
192
169
|
closure |= 1 << target
|
|
@@ -196,35 +173,50 @@ module Flexr
|
|
|
196
173
|
closure
|
|
197
174
|
end
|
|
198
175
|
|
|
199
|
-
def
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
176
|
+
def indexed_transition_closures(nfa, ec, closure_for)
|
|
177
|
+
nfa.states.map do |state|
|
|
178
|
+
raw = {}
|
|
179
|
+
state.transitions.each do |transition|
|
|
180
|
+
transition.lo.upto(transition.hi).map { |byte| ec.fetch(byte) }.uniq.each do |class_id|
|
|
181
|
+
raw[class_id] = raw.fetch(class_id, 0) | (1 << transition.to)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
raw.transform_values do |targets|
|
|
185
|
+
each_state_id(targets).reduce(0) { |closure, target| closure | closure_for.call(target) }
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
206
189
|
|
|
207
|
-
|
|
190
|
+
def move_closures(set, transition_closures)
|
|
191
|
+
each_state_id(set).with_object({}) do |state, moves|
|
|
192
|
+
transition_closures.fetch(state).each do |class_id, closure|
|
|
193
|
+
moves[class_id] = moves.fetch(class_id, 0) | closure
|
|
208
194
|
end
|
|
209
195
|
end
|
|
210
|
-
moved
|
|
211
196
|
end
|
|
212
197
|
|
|
213
198
|
def accepting_rules(nfa, set)
|
|
214
|
-
rules = []
|
|
215
|
-
|
|
216
|
-
next if set.nobits?(1 << state)
|
|
217
|
-
|
|
218
|
-
rules.concat(nfa.states[state].accepts)
|
|
199
|
+
rules = each_state_id(set).with_object([]) do |state, result|
|
|
200
|
+
result.concat(nfa.states[state].accepts)
|
|
219
201
|
end
|
|
220
202
|
rules.uniq.sort_by { |acceptance| [acceptance.rule_index, acceptance.pattern_index] }
|
|
221
203
|
end
|
|
222
204
|
|
|
223
|
-
def
|
|
224
|
-
|
|
205
|
+
def each_state_id(set)
|
|
206
|
+
return enum_for(__method__, set) unless block_given?
|
|
207
|
+
|
|
208
|
+
remaining = set
|
|
209
|
+
until remaining.zero?
|
|
210
|
+
bit = remaining & -remaining
|
|
211
|
+
yield bit.bit_length - 1
|
|
212
|
+
remaining ^= bit
|
|
213
|
+
end
|
|
214
|
+
end
|
|
225
215
|
|
|
216
|
+
def validate_rules
|
|
226
217
|
@spec.rules.each do |rule|
|
|
227
218
|
raise CompileError, "rule #{rule.index} has no pattern" if rule.patterns.empty?
|
|
219
|
+
parse_regexp(rule.trailing) if rule.trailing.is_a?(::Regexp)
|
|
228
220
|
next if @spec.options[:allow_empty_match]
|
|
229
221
|
|
|
230
222
|
rule.patterns.each do |pattern|
|
|
@@ -232,9 +224,12 @@ module Flexr
|
|
|
232
224
|
parse_regexp(regexp) if regexp.is_a?(::Regexp)
|
|
233
225
|
next unless regexp.is_a?(::Regexp) && regexp.match?("")
|
|
234
226
|
|
|
235
|
-
diagnostic = Diagnostics.error("FLEXR-E005", "
|
|
227
|
+
diagnostic = Diagnostics.error("FLEXR-E005", "#{Diagnostics.rule_label(rule)} can match an empty string")
|
|
236
228
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
237
229
|
end
|
|
230
|
+
rescue CompileError => e
|
|
231
|
+
e.diagnostic.location ||= rule.location if e.diagnostic
|
|
232
|
+
raise
|
|
238
233
|
end
|
|
239
234
|
end
|
|
240
235
|
|
|
@@ -249,36 +244,46 @@ module Flexr
|
|
|
249
244
|
present = compiled.machines.values.flat_map do |machine|
|
|
250
245
|
machine.dfa.accepts.filter_map { |acceptances| acceptances.min_by(&:rule_index)&.rule_index }
|
|
251
246
|
end
|
|
252
|
-
reference_rules = @spec.rules.select { |rule| reference_rule?(rule) && rule_active_anywhere?(rule) }
|
|
253
|
-
present.concat(reference_rules.map(&:index))
|
|
254
247
|
shadowers = shadowed_rules(compiled)
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
248
|
+
rules_by_index = @spec.rules.to_h { |rule| [rule.index, rule] }
|
|
249
|
+
unreachable = @spec.rules.reject { |rule| present.include?(rule.index) }
|
|
250
|
+
examples = unreachable_examples(compiled, unreachable, rules_by_index)
|
|
251
|
+
diagnostics.concat(unreachable.filter_map do |rule|
|
|
252
|
+
example = examples[rule.index]
|
|
253
|
+
next if example && example.fetch(:winner) == rule.index
|
|
254
|
+
|
|
255
|
+
winner_indices = @spec.backend == :firstmatch || @spec.rules.any?(&:trailing) ? [] : shadowers.fetch(rule.index, [])
|
|
256
|
+
winners = winner_indices.uniq.sort.map { |index| Diagnostics.rule_label(rules_by_index.fetch(index)) }
|
|
257
|
+
suffix = winners.empty? ? "" : " (shadowed by #{winners.join(', ')})"
|
|
258
|
+
Diagnostics.warning("FLEXR-W001", "#{Diagnostics.rule_label(rule)} is unreachable#{suffix}", location: rule.location,
|
|
259
|
+
help: "remove it, reorder the rules, or make its language distinct",
|
|
260
|
+
note: example&.fetch(:note))
|
|
260
261
|
end)
|
|
262
|
+
diagnostics.concat(overlap_diagnostics(compiled, rules_by_index)) if @spec.options[:warn_level] == :all
|
|
261
263
|
|
|
262
|
-
@spec.states.
|
|
264
|
+
@spec.states.each do |state_name, state|
|
|
263
265
|
next if state_name == :initial
|
|
264
266
|
next unless rules_for(state_name).empty?
|
|
265
267
|
|
|
266
268
|
diagnostics << Diagnostics.warning("FLEXR-W002", "state #{state_name.inspect} has no rules",
|
|
269
|
+
location: state.location,
|
|
267
270
|
help: "add a rule to the state or remove the unused state")
|
|
268
271
|
end
|
|
269
272
|
|
|
270
273
|
firstmatch_conflicts(@spec.rules).each do |left, right|
|
|
271
274
|
diagnostics << Diagnostics.warning(
|
|
272
|
-
"FLEXR-W010", "firstmatch
|
|
275
|
+
"FLEXR-W010", "firstmatch #{Diagnostics.rule_label(left)} and #{Diagnostics.rule_label(right)} may change longest-match semantics",
|
|
276
|
+
location: right.location,
|
|
273
277
|
help: "use backend :table unless first-match compatibility is required"
|
|
274
278
|
)
|
|
275
279
|
end
|
|
276
280
|
|
|
277
|
-
max_cells = compiled.stats.
|
|
278
|
-
if max_cells > 1_000_000
|
|
281
|
+
largest_state, max_cells = compiled.stats.map { |name, stat| [name, stat[:states] * stat[:classes]] }.max_by(&:last)
|
|
282
|
+
if max_cells && max_cells > 1_000_000
|
|
279
283
|
diagnostics << Diagnostics.warning(
|
|
280
284
|
"FLEXR-W011",
|
|
281
285
|
"generated transition table is large",
|
|
286
|
+
location: @spec.states.fetch(largest_state).location,
|
|
282
287
|
help: "use backend :direct, table compression, or split the specification"
|
|
283
288
|
)
|
|
284
289
|
end
|
|
@@ -286,21 +291,24 @@ module Flexr
|
|
|
286
291
|
capture_rules.each do |rule|
|
|
287
292
|
diagnostics << Diagnostics.warning(
|
|
288
293
|
"FLEXR-W013",
|
|
289
|
-
"
|
|
294
|
+
"#{Diagnostics.rule_label(rule)} uses a capturing group; flexr treats it as non-capturing",
|
|
295
|
+
location: rule.location,
|
|
290
296
|
help: "rewrite capturing groups as (?:...) and extract text in the action"
|
|
291
297
|
)
|
|
292
298
|
end
|
|
293
299
|
|
|
294
|
-
|
|
300
|
+
undeclared_token_rules.each do |token, rule|
|
|
295
301
|
diagnostics << Diagnostics.warning(
|
|
296
302
|
"FLEXR-W014",
|
|
297
303
|
"token #{token.inspect} is not declared by emits",
|
|
304
|
+
location: rule.location,
|
|
298
305
|
help: "add the token to emits or remove the declaration if it is intentionally private"
|
|
299
306
|
)
|
|
300
307
|
end
|
|
301
308
|
|
|
302
309
|
diagnostics.concat(variable_trailing_rules.map do |rule|
|
|
303
|
-
Diagnostics.warning("FLEXR-W003", "
|
|
310
|
+
Diagnostics.warning("FLEXR-W003", "#{Diagnostics.rule_label(rule)} uses variable-length trailing context",
|
|
311
|
+
location: rule.location,
|
|
304
312
|
help: "make the body or followed_by expression fixed length when possible")
|
|
305
313
|
end)
|
|
306
314
|
|
|
@@ -308,7 +316,8 @@ module Flexr
|
|
|
308
316
|
diagnostics.concat(@spec.rules.select(&:trailing).map do |rule|
|
|
309
317
|
Diagnostics.warning(
|
|
310
318
|
"FLEXR-W012",
|
|
311
|
-
"
|
|
319
|
+
"#{Diagnostics.rule_label(rule)} cannot use region acceleration with trailing context",
|
|
320
|
+
location: rule.location,
|
|
312
321
|
help: "remove trailing context or set accel: :none when the trade-off is intentional"
|
|
313
322
|
)
|
|
314
323
|
end)
|
|
@@ -316,11 +325,138 @@ module Flexr
|
|
|
316
325
|
|
|
317
326
|
if elapsed > 0.5
|
|
318
327
|
diagnostics << Diagnostics.warning("FLEXR-W016", format("DFA construction took %.3fs", elapsed),
|
|
328
|
+
location: @spec.states.fetch(:initial).location,
|
|
319
329
|
help: "use generated mode for production startup")
|
|
320
330
|
end
|
|
321
331
|
diagnostics
|
|
322
332
|
end
|
|
323
333
|
|
|
334
|
+
def overlap_diagnostics(compiled, rules_by_index)
|
|
335
|
+
# shortcut: verify one shortest body and trailing candidate per rule; search alternatives if rejected examples need coverage.
|
|
336
|
+
compiled.machines.flat_map do |state_name, machine|
|
|
337
|
+
active_rules = rules_for(state_name)
|
|
338
|
+
trailing_dfas = diagnostic_trailing_dfas(active_rules)
|
|
339
|
+
next [] unless trailing_dfas
|
|
340
|
+
|
|
341
|
+
has_trailing = !trailing_dfas.empty?
|
|
342
|
+
candidates = Analysis.shadowing_examples(machine.dfa, machine.dfa.rule_ids,
|
|
343
|
+
include_empty: has_trailing, include_unshadowed: has_trailing)
|
|
344
|
+
candidates.filter_map do |index, (bytes, winner)|
|
|
345
|
+
if (trailing = trailing_dfas[index])
|
|
346
|
+
suffix = Analysis.shortest_input(trailing)
|
|
347
|
+
next unless suffix
|
|
348
|
+
|
|
349
|
+
bytes += suffix
|
|
350
|
+
end
|
|
351
|
+
input = bytes.dup.force_encoding(@spec.encoding)
|
|
352
|
+
next if input.empty? || !input.valid_encoding?
|
|
353
|
+
|
|
354
|
+
rule = rules_by_index.fetch(index)
|
|
355
|
+
length = input.bytesize
|
|
356
|
+
if has_trailing
|
|
357
|
+
result = Analysis.trailing_example(machine.dfa, active_rules, rule, input, trailing_dfas,
|
|
358
|
+
encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
|
|
359
|
+
firstmatch: @spec.backend == :firstmatch, same_length: true)
|
|
360
|
+
next unless result && result.first != index
|
|
361
|
+
|
|
362
|
+
winner, length, lookahead = result
|
|
363
|
+
elsif @spec.backend == :firstmatch
|
|
364
|
+
result = Analysis.firstmatch_example(active_rules, rule, input, encoding: @spec.encoding,
|
|
365
|
+
unicode: @spec.options[:unicode] == true, same_length: true)
|
|
366
|
+
next unless result && result.first != index
|
|
367
|
+
|
|
368
|
+
winner, length = result
|
|
369
|
+
end
|
|
370
|
+
selected = Diagnostics.rule_label(rules_by_index.fetch(winner))
|
|
371
|
+
state = state_name == :initial ? "" : " in state #{state_name.inspect}"
|
|
372
|
+
size = "#{length} #{length == 1 ? 'byte' : 'bytes'}"
|
|
373
|
+
reason = if has_trailing
|
|
374
|
+
priority = @spec.backend == :firstmatch ? "first match wins" : "same total length, earlier rule wins"
|
|
375
|
+
"#{priority}; matched #{size}, looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}"
|
|
376
|
+
elsif @spec.backend == :firstmatch
|
|
377
|
+
"first match wins, matched #{size}"
|
|
378
|
+
else
|
|
379
|
+
"#{size}, earlier rule wins"
|
|
380
|
+
end
|
|
381
|
+
match_size = has_trailing ? "total length" : "length"
|
|
382
|
+
Diagnostics.info("FLEXR-I001", "#{Diagnostics.rule_label(rule)} overlaps #{selected} at the same #{match_size}",
|
|
383
|
+
location: rule.location, help: "reorder the rules if the later rule should take priority",
|
|
384
|
+
note: "example: #{input.inspect} selects #{selected}#{state} (#{reason})")
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
def unreachable_examples(compiled, rules, rules_by_index)
|
|
390
|
+
return {} if rules.empty?
|
|
391
|
+
|
|
392
|
+
indices = rules.map(&:index)
|
|
393
|
+
examples = {}
|
|
394
|
+
compiled.machines.each do |state_name, machine|
|
|
395
|
+
active_rules = rules_for(state_name)
|
|
396
|
+
# shortcut: verify only the shortest candidate; search alternatives if rejected examples need coverage.
|
|
397
|
+
candidates = Analysis.shadowing_examples(machine.dfa, indices, include_empty: active_rules.any?(&:trailing))
|
|
398
|
+
next if candidates.empty?
|
|
399
|
+
|
|
400
|
+
trailing_dfas = diagnostic_trailing_dfas(active_rules)
|
|
401
|
+
next unless trailing_dfas
|
|
402
|
+
|
|
403
|
+
candidates.each do |index, (bytes, winner)|
|
|
404
|
+
if (trailing = trailing_dfas[index])
|
|
405
|
+
suffix = Analysis.shortest_input(trailing)
|
|
406
|
+
next unless suffix
|
|
407
|
+
|
|
408
|
+
bytes += suffix
|
|
409
|
+
end
|
|
410
|
+
input = bytes.dup.force_encoding(@spec.encoding)
|
|
411
|
+
next if input.empty? || !input.valid_encoding?
|
|
412
|
+
next if examples[index] && examples[index][1] == index
|
|
413
|
+
|
|
414
|
+
length = lookahead = nil
|
|
415
|
+
if !trailing_dfas.empty?
|
|
416
|
+
result = Analysis.trailing_example(machine.dfa, active_rules, rules_by_index.fetch(index), input, trailing_dfas,
|
|
417
|
+
encoding: @spec.encoding, unicode: @spec.options[:unicode] == true,
|
|
418
|
+
firstmatch: @spec.backend == :firstmatch)
|
|
419
|
+
next unless result
|
|
420
|
+
|
|
421
|
+
winner, length, lookahead = result
|
|
422
|
+
elsif @spec.backend == :firstmatch
|
|
423
|
+
result = Analysis.firstmatch_example(active_rules, rules_by_index.fetch(index), input,
|
|
424
|
+
encoding: @spec.encoding, unicode: @spec.options[:unicode] == true)
|
|
425
|
+
next unless result
|
|
426
|
+
|
|
427
|
+
winner, length = result
|
|
428
|
+
end
|
|
429
|
+
next if winner != index && examples[index] && examples[index].first.bytesize <= input.bytesize
|
|
430
|
+
|
|
431
|
+
examples[index] = [input, winner, state_name, length, lookahead]
|
|
432
|
+
end
|
|
433
|
+
end
|
|
434
|
+
examples.transform_values do |input, winner, state_name, length, lookahead|
|
|
435
|
+
state = state_name == :initial ? "" : " in state #{state_name.inspect}"
|
|
436
|
+
reason = if @spec.backend == :firstmatch
|
|
437
|
+
"first match wins, matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
|
|
438
|
+
elsif lookahead
|
|
439
|
+
"longest total match, then rule order; matched #{length} #{length == 1 ? 'byte' : 'bytes'}"
|
|
440
|
+
else
|
|
441
|
+
"same length, earlier rule wins"
|
|
442
|
+
end
|
|
443
|
+
reason += ", looked ahead #{lookahead} #{lookahead == 1 ? 'byte' : 'bytes'}" if lookahead
|
|
444
|
+
note = "example: #{input.inspect} selects #{Diagnostics.rule_label(rules_by_index.fetch(winner))}#{state} (#{reason})"
|
|
445
|
+
{ note: note, winner: winner }
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
def diagnostic_trailing_dfas(rules)
|
|
450
|
+
rules.select(&:trailing).to_h do |rule|
|
|
451
|
+
dfa = rule.trailing_dfa || Flexr.compile_pattern(rule.trailing, encoding: @spec.encoding,
|
|
452
|
+
options: { unicode: @spec.options[:unicode] == true })
|
|
453
|
+
[rule.index, dfa]
|
|
454
|
+
end
|
|
455
|
+
rescue CompileError
|
|
456
|
+
# shortcut: omit examples for trailing patterns the runtime cannot compile, until that matcher supports them.
|
|
457
|
+
nil
|
|
458
|
+
end
|
|
459
|
+
|
|
324
460
|
def shadowed_rules(compiled)
|
|
325
461
|
shadowers = Hash.new { |hash, key| hash[key] = [] }
|
|
326
462
|
compiled.machines.each_value do |machine|
|
|
@@ -344,10 +480,6 @@ module Flexr
|
|
|
344
480
|
rules.combination(2).to_a
|
|
345
481
|
end
|
|
346
482
|
|
|
347
|
-
def rule_active_anywhere?(rule)
|
|
348
|
-
@spec.states.keys.any? { |state_name| rules_for(state_name).include?(rule) }
|
|
349
|
-
end
|
|
350
|
-
|
|
351
483
|
def capture_rules
|
|
352
484
|
@spec.rules.select do |rule|
|
|
353
485
|
rule.patterns.any? do |pattern|
|
|
@@ -378,21 +510,22 @@ module Flexr
|
|
|
378
510
|
false
|
|
379
511
|
end
|
|
380
512
|
|
|
381
|
-
def
|
|
382
|
-
|
|
513
|
+
def undeclared_token_rules
|
|
514
|
+
declared = Array(@spec.declared_tokens)
|
|
515
|
+
return {} if declared.empty?
|
|
383
516
|
|
|
384
|
-
emitted = @spec.rules.
|
|
517
|
+
emitted = @spec.rules.each_with_object({}) do |rule, rules_by_token|
|
|
385
518
|
action = rule.action
|
|
386
|
-
if action.is_a?(Array) && action.first == :emit
|
|
519
|
+
tokens = if action.is_a?(Array) && action.first == :emit
|
|
387
520
|
[action.last]
|
|
388
521
|
elsif action.is_a?(String)
|
|
389
522
|
action.scan(/\bemit\s*\(?\s*:([A-Za-z_]\w*)/).flatten.map(&:to_sym)
|
|
390
523
|
else
|
|
391
524
|
[]
|
|
392
525
|
end
|
|
526
|
+
tokens.each { |token| rules_by_token[token] ||= rule }
|
|
393
527
|
end
|
|
394
|
-
declared
|
|
395
|
-
emitted.uniq.reject { |token| declared.include?(token) }
|
|
528
|
+
emitted.except(*declared)
|
|
396
529
|
end
|
|
397
530
|
|
|
398
531
|
def variable_trailing_rules
|
data/lib/flexr/automaton/dfa.rb
CHANGED
|
@@ -2,42 +2,81 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Automaton
|
|
5
|
+
class TransitionRows
|
|
6
|
+
include Enumerable
|
|
7
|
+
|
|
8
|
+
attr_reader :length
|
|
9
|
+
|
|
10
|
+
def initialize(length, &loader)
|
|
11
|
+
@length = length
|
|
12
|
+
@loader = loader
|
|
13
|
+
@rows = Array.new(length)
|
|
14
|
+
freeze
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
alias size length
|
|
18
|
+
|
|
19
|
+
def [](state)
|
|
20
|
+
return unless state&.between?(0, length - 1)
|
|
21
|
+
|
|
22
|
+
@rows[state] ||= @loader.call(state).freeze
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def fetch(state)
|
|
26
|
+
return self[state] if state.between?(0, length - 1)
|
|
27
|
+
|
|
28
|
+
raise IndexError, "transition row #{state} is out of bounds"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def each
|
|
32
|
+
return enum_for(__method__) unless block_given?
|
|
33
|
+
|
|
34
|
+
length.times { |state| yield self[state] }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def each_index(&)
|
|
38
|
+
return enum_for(__method__) unless block_given?
|
|
39
|
+
|
|
40
|
+
length.times(&)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def materialized_count
|
|
44
|
+
@rows.count { |row| !row.nil? }
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
5
48
|
class DFA
|
|
6
49
|
attr_reader :transitions, :accepts, :ec, :class_count, :start, :states, :rule_ids, :packed, :direct
|
|
7
50
|
|
|
8
|
-
def initialize(
|
|
9
|
-
|
|
51
|
+
def initialize(accepts:, ec:, class_count:, start:, rule_ids:, transitions: nil, packed: nil, direct: nil,
|
|
52
|
+
state_count: nil)
|
|
53
|
+
@packed = freeze_representation(packed)
|
|
54
|
+
@direct = freeze_representation(direct)
|
|
55
|
+
@states = state_count || transitions&.length || direct_state_count
|
|
56
|
+
raise ArgumentError, "state_count is required without dense transitions" unless @states
|
|
57
|
+
|
|
58
|
+
@transitions = transition_rows(transitions)
|
|
10
59
|
@accepts = accepts.map do |rules|
|
|
11
60
|
rules.map do |acceptance|
|
|
12
|
-
next acceptance if acceptance.is_a?(Acceptance)
|
|
61
|
+
next acceptance.freeze if acceptance.is_a?(Acceptance)
|
|
13
62
|
|
|
14
63
|
Acceptance.new(rule_index: acceptance[0], pattern_index: acceptance[1],
|
|
15
|
-
bol_only: acceptance[2], end_anchor: acceptance[3])
|
|
64
|
+
bol_only: acceptance[2], end_anchor: acceptance[3]).freeze
|
|
16
65
|
end.freeze
|
|
17
66
|
end.freeze
|
|
18
67
|
@ec = ec.freeze
|
|
19
68
|
@class_count = class_count
|
|
20
69
|
@start = start
|
|
21
|
-
@states = transitions.length
|
|
22
70
|
@rule_ids = rule_ids.map { |id| id.respond_to?(:rule_index) ? id.rule_index : id }.uniq.sort.freeze
|
|
23
|
-
|
|
24
|
-
@direct = direct
|
|
71
|
+
freeze
|
|
25
72
|
end
|
|
26
73
|
|
|
27
74
|
def transition(state, byte)
|
|
28
75
|
class_id = @ec[byte]
|
|
76
|
+
return direct_transition(state, class_id) if @direct
|
|
29
77
|
return @transitions[state][class_id] unless @packed
|
|
30
78
|
|
|
31
|
-
|
|
32
|
-
loop do
|
|
33
|
-
index = @packed.fetch(:base).fetch(cursor) + class_id
|
|
34
|
-
return @packed.fetch(:next).fetch(index) if @packed.fetch(:check)[index] == cursor
|
|
35
|
-
|
|
36
|
-
fallback = @packed[:fallback]&.fetch(cursor)
|
|
37
|
-
return @packed.fetch(:default).fetch(cursor) unless fallback
|
|
38
|
-
|
|
39
|
-
cursor = fallback
|
|
40
|
-
end
|
|
79
|
+
packed_transition(state, class_id)
|
|
41
80
|
end
|
|
42
81
|
|
|
43
82
|
# Generated direct lexers use a flattened dispatch representation. The
|
|
@@ -45,8 +84,7 @@ module Flexr
|
|
|
45
84
|
def transition_direct(state, byte)
|
|
46
85
|
if @direct
|
|
47
86
|
class_id = @ec[byte]
|
|
48
|
-
|
|
49
|
-
return value >= 0 ? value : nil
|
|
87
|
+
return direct_transition(state, class_id)
|
|
50
88
|
end
|
|
51
89
|
|
|
52
90
|
@transitions[state][@ec[byte]]
|
|
@@ -65,6 +103,55 @@ module Flexr
|
|
|
65
103
|
def stats
|
|
66
104
|
{ states: states, classes: class_count, accepting_states: accepts.count { |rules| !rules.empty? } }
|
|
67
105
|
end
|
|
106
|
+
|
|
107
|
+
private
|
|
108
|
+
|
|
109
|
+
def transition_rows(transitions)
|
|
110
|
+
return transitions.map(&:freeze).freeze if transitions
|
|
111
|
+
if @packed
|
|
112
|
+
return TransitionRows.new(states) do |state|
|
|
113
|
+
Array.new(class_count) { |class_id| packed_transition(state, class_id) }
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
if @direct
|
|
117
|
+
return TransitionRows.new(states) do |state|
|
|
118
|
+
Array.new(class_count) { |class_id| direct_transition(state, class_id) }
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
raise ArgumentError, "a transition representation is required"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def packed_transition(state, class_id)
|
|
126
|
+
cursor = state
|
|
127
|
+
loop do
|
|
128
|
+
index = @packed.fetch(:base).fetch(cursor) + class_id
|
|
129
|
+
return @packed.fetch(:next).fetch(index) if @packed.fetch(:check)[index] == cursor
|
|
130
|
+
|
|
131
|
+
fallback = @packed[:fallback]&.fetch(cursor)
|
|
132
|
+
return @packed.fetch(:default).fetch(cursor) unless fallback
|
|
133
|
+
|
|
134
|
+
cursor = fallback
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def direct_transition(state, class_id)
|
|
139
|
+
value = @direct.fetch(:nxt).fetch((state * @direct.fetch(:classes)) + class_id)
|
|
140
|
+
value >= 0 ? value : nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def direct_state_count
|
|
144
|
+
return unless @direct
|
|
145
|
+
|
|
146
|
+
@direct.fetch(:nxt).length / @direct.fetch(:classes)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def freeze_representation(representation)
|
|
150
|
+
return unless representation
|
|
151
|
+
|
|
152
|
+
representation.each_value { |value| value.freeze if value.respond_to?(:freeze) }
|
|
153
|
+
representation.freeze
|
|
154
|
+
end
|
|
68
155
|
end
|
|
69
156
|
|
|
70
157
|
class ReferenceDFA
|