flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -19,13 +19,7 @@ module Flexr
|
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
def self_loop_set(dfa, state)
|
|
22
|
-
|
|
23
|
-
next unless destination == state
|
|
24
|
-
|
|
25
|
-
dfa.ec.each_with_index do |value, byte|
|
|
26
|
-
result << byte if value == class_id
|
|
27
|
-
end
|
|
28
|
-
end
|
|
22
|
+
Accel.self_loop_set(dfa, state)
|
|
29
23
|
end
|
|
30
24
|
|
|
31
25
|
def dead_states(dfa)
|
|
@@ -33,6 +27,207 @@ module Flexr
|
|
|
33
27
|
dfa.accepts[state].empty? && dfa.transitions[state].compact.all? { |destination| destination == state }
|
|
34
28
|
end
|
|
35
29
|
end
|
|
30
|
+
|
|
31
|
+
def firstmatch_counterexample(first, second)
|
|
32
|
+
queue = [[first.start, second.start, false, +"".b]]
|
|
33
|
+
visited = { [first.start, second.start, false] => true }
|
|
34
|
+
bytes = joint_byte_representatives(first, second)
|
|
35
|
+
cursor = 0
|
|
36
|
+
|
|
37
|
+
while cursor < queue.length
|
|
38
|
+
first_state, second_state, first_seen, input = queue.fetch(cursor)
|
|
39
|
+
cursor += 1
|
|
40
|
+
bytes.each do |byte|
|
|
41
|
+
next_first = first_state && first.transition(first_state, byte)
|
|
42
|
+
next_second = second_state && second.transition(second_state, byte)
|
|
43
|
+
next unless next_second
|
|
44
|
+
|
|
45
|
+
next_input = input + byte.chr(Encoding::BINARY)
|
|
46
|
+
next_first_seen = first_seen || accepting?(first, next_first)
|
|
47
|
+
return next_input if next_first_seen && accepting?(second, next_second) && !accepting?(first, next_first)
|
|
48
|
+
|
|
49
|
+
key = [next_first, next_second, next_first_seen]
|
|
50
|
+
next if visited[key]
|
|
51
|
+
|
|
52
|
+
visited[key] = true
|
|
53
|
+
queue << [next_first, next_second, next_first_seen, next_input]
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def joint_byte_representatives(first, second)
|
|
60
|
+
representatives = {}
|
|
61
|
+
256.times do |byte|
|
|
62
|
+
key = [first.ec[byte], second.ec[byte]]
|
|
63
|
+
representatives[key] ||= byte
|
|
64
|
+
end
|
|
65
|
+
representatives.values.sort_by { |byte| [byte.between?(32, 126) ? 0 : 1, byte] }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def shadowing_examples(dfa, rule_indices, include_empty: false, include_unshadowed: false)
|
|
69
|
+
remaining = (rule_indices & dfa.rule_ids).to_h { |index| [index, true] }
|
|
70
|
+
return {} if remaining.empty?
|
|
71
|
+
|
|
72
|
+
examples = {}
|
|
73
|
+
queue = [[dfa.start, nil, nil]]
|
|
74
|
+
# Visiting the start again permits a nonempty example when it accepts and loops.
|
|
75
|
+
visited = {}
|
|
76
|
+
bytes = joint_byte_representatives(dfa, dfa)
|
|
77
|
+
cursor = 0
|
|
78
|
+
while cursor < queue.length
|
|
79
|
+
state = queue.fetch(cursor).first
|
|
80
|
+
accepts = dfa.accepts.fetch(state)
|
|
81
|
+
winner = accepts.min_by(&:rule_index)&.rule_index
|
|
82
|
+
if (include_empty || cursor.positive?) && winner
|
|
83
|
+
shadowed = accepts.map(&:rule_index).uniq.select do |index|
|
|
84
|
+
(include_unshadowed || index > winner) && remaining.key?(index)
|
|
85
|
+
end
|
|
86
|
+
unless shadowed.empty?
|
|
87
|
+
input = example_input(queue, cursor)
|
|
88
|
+
shadowed.each do |index|
|
|
89
|
+
examples[index] = [input, winner]
|
|
90
|
+
remaining.delete(index)
|
|
91
|
+
end
|
|
92
|
+
return examples if remaining.empty?
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
bytes.each do |byte|
|
|
97
|
+
target = dfa.transition(state, byte)
|
|
98
|
+
next if target.nil? || visited[target]
|
|
99
|
+
|
|
100
|
+
visited[target] = true
|
|
101
|
+
queue << [target, cursor, byte]
|
|
102
|
+
end
|
|
103
|
+
cursor += 1
|
|
104
|
+
end
|
|
105
|
+
examples
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def example_input(paths, cursor)
|
|
109
|
+
bytes = []
|
|
110
|
+
while (parent = paths.fetch(cursor)[1])
|
|
111
|
+
bytes << paths.fetch(cursor)[2]
|
|
112
|
+
cursor = parent
|
|
113
|
+
end
|
|
114
|
+
bytes.reverse.pack("C*")
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def shortest_input(dfa)
|
|
118
|
+
queue = [[dfa.start, nil, nil]]
|
|
119
|
+
visited = { dfa.start => true }
|
|
120
|
+
bytes = joint_byte_representatives(dfa, dfa)
|
|
121
|
+
cursor = 0
|
|
122
|
+
while cursor < queue.length
|
|
123
|
+
state = queue.fetch(cursor).first
|
|
124
|
+
return example_input(queue, cursor) if accepting?(dfa, state)
|
|
125
|
+
|
|
126
|
+
bytes.each do |byte|
|
|
127
|
+
target = dfa.transition(state, byte)
|
|
128
|
+
next if target.nil? || visited[target]
|
|
129
|
+
|
|
130
|
+
visited[target] = true
|
|
131
|
+
queue << [target, cursor, byte]
|
|
132
|
+
end
|
|
133
|
+
cursor += 1
|
|
134
|
+
end
|
|
135
|
+
nil
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def trailing_example(dfa, rules, target, input, trailing_dfas, encoding:, unicode:, firstmatch:, same_length: false)
|
|
139
|
+
lengths = if firstmatch
|
|
140
|
+
rules.filter_map do |rule|
|
|
141
|
+
next if rule.index > target.index
|
|
142
|
+
|
|
143
|
+
length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
|
|
144
|
+
[rule.index, length] if length
|
|
145
|
+
end
|
|
146
|
+
else
|
|
147
|
+
dfa_match_lengths(dfa, input, encoding: encoding)
|
|
148
|
+
end
|
|
149
|
+
matches = {}
|
|
150
|
+
lengths.each do |index, length|
|
|
151
|
+
trailing = trailing_dfas[index]
|
|
152
|
+
lookahead = trailing ? dfa_match_lengths(trailing, input, offset: length).map(&:last).max : 0
|
|
153
|
+
next unless lookahead
|
|
154
|
+
|
|
155
|
+
total = length + lookahead
|
|
156
|
+
next if matches[index] && matches[index].last > total
|
|
157
|
+
|
|
158
|
+
matches[index] = [length, total]
|
|
159
|
+
end
|
|
160
|
+
return unless matches.key?(target.index)
|
|
161
|
+
|
|
162
|
+
winner, (length, total) = matches.max_by { |index, (_, size)| firstmatch ? -index : [size, -index] }
|
|
163
|
+
return if same_length && total != matches.fetch(target.index).last
|
|
164
|
+
|
|
165
|
+
[winner, length, total - length]
|
|
166
|
+
rescue EncodingError
|
|
167
|
+
nil
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def dfa_match_lengths(dfa, input, offset: 0, encoding: Encoding::BINARY)
|
|
171
|
+
state = dfa.start
|
|
172
|
+
cursor = offset
|
|
173
|
+
matches = []
|
|
174
|
+
loop do
|
|
175
|
+
if encoding != Encoding::UTF_8 || input.byteslice(0, cursor).valid_encoding?
|
|
176
|
+
dfa.accepts.fetch(state).each do |acceptance|
|
|
177
|
+
next if acceptance.end_anchor && cursor < input.bytesize && input.getbyte(cursor) != 0x0a
|
|
178
|
+
|
|
179
|
+
matches << [acceptance.rule_index, cursor - offset]
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
break if cursor == input.bytesize
|
|
183
|
+
|
|
184
|
+
state = dfa.transition(state, input.getbyte(cursor))
|
|
185
|
+
break unless state
|
|
186
|
+
|
|
187
|
+
cursor += 1
|
|
188
|
+
end
|
|
189
|
+
matches
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def firstmatch_example(rules, target, input, encoding:, unicode: false, same_length: false)
|
|
193
|
+
target_length = firstmatch_length(target, input, encoding: encoding, unicode: unicode)
|
|
194
|
+
return unless target_length
|
|
195
|
+
|
|
196
|
+
rules.sort_by(&:index).each do |rule|
|
|
197
|
+
break if rule.index >= target.index
|
|
198
|
+
|
|
199
|
+
length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
|
|
200
|
+
next unless length
|
|
201
|
+
|
|
202
|
+
return nil if same_length && length != target_length
|
|
203
|
+
|
|
204
|
+
return [rule.index, length]
|
|
205
|
+
end
|
|
206
|
+
[target.index, target_length]
|
|
207
|
+
rescue EncodingError
|
|
208
|
+
nil
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def firstmatch_length(rule, input, encoding:, unicode:)
|
|
212
|
+
matches = rule.patterns.each_with_index.filter_map do |pattern, index|
|
|
213
|
+
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
214
|
+
reference = Regexp::Matcher.reference?(regexp, encoding: encoding, unicode: unicode)
|
|
215
|
+
match = Regexp::Matcher.match(regexp, input, encoding: encoding, unicode: unicode, reference: reference)
|
|
216
|
+
next unless match&.begin(0)&.zero?
|
|
217
|
+
|
|
218
|
+
[match[0].bytesize, rule.pattern_conditions.fetch(index)]
|
|
219
|
+
end
|
|
220
|
+
length, condition = matches.max_by(&:first)
|
|
221
|
+
return unless length
|
|
222
|
+
return if encoding == Encoding::UTF_8 && !input.byteslice(0, length).valid_encoding?
|
|
223
|
+
return if condition.end_anchor && length < input.bytesize && input.getbyte(length) != 0x0a
|
|
224
|
+
|
|
225
|
+
length
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def accepting?(dfa, state)
|
|
229
|
+
state && !dfa.accepts.fetch(state).empty?
|
|
230
|
+
end
|
|
36
231
|
end
|
|
37
232
|
end
|
|
38
233
|
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Automaton
|
|
5
|
+
module BackendCostModel
|
|
6
|
+
Metrics = Struct.new(:dense_bytes, :packed_bytes, :lookup_samples, keyword_init: true) do
|
|
7
|
+
def direct_score
|
|
8
|
+
dense_bytes + lookup_samples
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def table_score
|
|
12
|
+
packed_bytes + (lookup_samples * 3)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
def choose(compiled)
|
|
19
|
+
metrics = compiled.machines.values.map { |machine| metrics_for(machine.dfa) }
|
|
20
|
+
direct_score = metrics.sum(&:direct_score)
|
|
21
|
+
table_score = metrics.sum(&:table_score)
|
|
22
|
+
direct_score < table_score ? :direct : :table
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def metrics_for(dfa)
|
|
26
|
+
cells = dfa.states * dfa.class_count
|
|
27
|
+
overrides = dfa.transitions.sum do |row|
|
|
28
|
+
default = row.tally.max_by { |_value, count| count }&.first
|
|
29
|
+
row.count { |value| value != default }
|
|
30
|
+
end
|
|
31
|
+
packed_integers = (overrides * 2) + (dfa.states * 2)
|
|
32
|
+
Metrics.new(
|
|
33
|
+
dense_bytes: cells * 4,
|
|
34
|
+
packed_bytes: packed_integers * 4,
|
|
35
|
+
lookup_samples: dfa.states * [dfa.class_count, 16].min
|
|
36
|
+
).freeze
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|