flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -13,25 +13,41 @@ module Flexr
|
|
|
13
13
|
class Interpreter
|
|
14
14
|
def initialize(lexer)
|
|
15
15
|
@lexer = lexer
|
|
16
|
+
klass = lexer.class
|
|
17
|
+
@compiled = klass.__flexr_compiled
|
|
18
|
+
@rules = klass.__flexr_rules
|
|
19
|
+
@config = klass.__flexr_config
|
|
16
20
|
end
|
|
17
21
|
|
|
18
22
|
def scan
|
|
19
|
-
machine = @
|
|
23
|
+
machine = @compiled.machines.fetch(@lexer.state)
|
|
20
24
|
position = @lexer.byte_pos
|
|
21
25
|
return nil unless @lexer.valid_utf8_at?(position)
|
|
22
|
-
return scan_firstmatch(position) if @
|
|
23
|
-
|
|
26
|
+
return scan_firstmatch(position) if @config.backend == :firstmatch
|
|
27
|
+
if fast_path?(machine)
|
|
28
|
+
if machine.whole_tokens
|
|
29
|
+
match = @lexer.__flexr_whole_token_match(machine.whole_tokens, position)
|
|
30
|
+
return match if match
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
return scan_fast(machine, position)
|
|
34
|
+
end
|
|
24
35
|
|
|
25
36
|
buffer = @lexer.buffer
|
|
26
37
|
state = machine.dfa.start
|
|
27
38
|
best = reference_match(position, buffer)
|
|
28
39
|
cursor = position
|
|
29
|
-
best = consider_acceptances(machine, state, cursor, position, buffer, best) if @
|
|
40
|
+
best = consider_acceptances(machine, state, cursor, position, buffer, best) if @config.options[:allow_empty_match]
|
|
30
41
|
acceleration_regions = acceleration_enabled? ? acceleration_regions_for(machine) : nil
|
|
31
42
|
|
|
32
43
|
while buffer.ensure_available?(cursor + 1)
|
|
33
|
-
if acceleration_regions && (region = acceleration_regions[state])
|
|
34
|
-
|
|
44
|
+
if acceleration_regions && (region = acceleration_regions[state]) &&
|
|
45
|
+
!@disabled_auto_acceleration&.key?(region)
|
|
46
|
+
acceptance = machine.dfa.accepts[state].first
|
|
47
|
+
token_rule = acceptance&.rule_index
|
|
48
|
+
accelerated_end = accelerate(
|
|
49
|
+
region, buffer, cursor, token_rule: token_rule, token_start: position
|
|
50
|
+
)
|
|
35
51
|
if accelerated_end && accelerated_end > cursor
|
|
36
52
|
cursor = accelerated_end
|
|
37
53
|
best = consider_acceptances(machine, state, cursor, position, buffer, best)
|
|
@@ -40,10 +56,13 @@ module Flexr
|
|
|
40
56
|
end
|
|
41
57
|
|
|
42
58
|
byte = buffer.getbyte(cursor)
|
|
59
|
+
@lexer.consume_step!
|
|
43
60
|
state = transition(machine.dfa, state, byte)
|
|
44
61
|
break unless state
|
|
45
62
|
|
|
46
63
|
cursor += 1
|
|
64
|
+
acceptance = machine.dfa.accepts[state].first
|
|
65
|
+
ensure_token_size!(cursor, position, rule: acceptance&.rule_index)
|
|
47
66
|
best = consider_acceptances(machine, state, cursor, position, buffer, best)
|
|
48
67
|
end
|
|
49
68
|
best
|
|
@@ -51,22 +70,57 @@ module Flexr
|
|
|
51
70
|
|
|
52
71
|
def scan_fast(machine, position)
|
|
53
72
|
buffer = @lexer.buffer
|
|
54
|
-
source = buffer.source
|
|
55
73
|
dfa = machine.dfa
|
|
56
74
|
accepts = dfa.accepts
|
|
57
|
-
rules = @
|
|
75
|
+
rules = @rules
|
|
58
76
|
transitions = dfa.transitions
|
|
59
77
|
ec = dfa.ec
|
|
60
78
|
direct = dfa.direct
|
|
79
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
80
|
+
direct_classes = direct&.fetch(:classes)
|
|
81
|
+
source = buffer.stable_source
|
|
82
|
+
guarded_steps = @lexer.scan_steps_guarded?
|
|
83
|
+
text_start = @lexer.more_text_start || position
|
|
84
|
+
max_token_size = @lexer.max_token_size
|
|
85
|
+
token_limit_required = @lexer.token_limit_required?
|
|
61
86
|
state = dfa.start
|
|
62
87
|
cursor = position
|
|
63
88
|
best = nil
|
|
64
89
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
90
|
+
if source && !guarded_steps && !token_limit_required
|
|
91
|
+
while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
|
|
92
|
+
byte = source.getbyte(cursor)
|
|
93
|
+
state = if direct_nxt
|
|
94
|
+
class_id = ec[byte]
|
|
95
|
+
value = direct_nxt[(state * direct_classes) + class_id]
|
|
96
|
+
value >= 0 ? value : nil
|
|
97
|
+
else
|
|
98
|
+
transitions[state][ec[byte]]
|
|
99
|
+
end
|
|
100
|
+
break unless state
|
|
101
|
+
|
|
102
|
+
cursor += 1
|
|
103
|
+
acceptance = accepts[state].first
|
|
104
|
+
next unless acceptance
|
|
105
|
+
|
|
106
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
107
|
+
next if best && cursor == best.total_end_pos && rule.index > best.rule.index
|
|
108
|
+
|
|
109
|
+
best ||= (@match ||= Match.new)
|
|
110
|
+
best.rule = rule
|
|
111
|
+
best.start_pos = position
|
|
112
|
+
best.end_pos = cursor
|
|
113
|
+
best.total_end_pos = cursor
|
|
114
|
+
end
|
|
115
|
+
return best
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
while cursor < buffer.bytesize || buffer.ensure_available?(cursor + 1)
|
|
119
|
+
byte = buffer.getbyte(cursor)
|
|
120
|
+
@lexer.consume_step! if guarded_steps
|
|
121
|
+
state = if direct_nxt
|
|
68
122
|
class_id = ec[byte]
|
|
69
|
-
value =
|
|
123
|
+
value = direct_nxt[(state * direct_classes) + class_id]
|
|
70
124
|
value >= 0 ? value : nil
|
|
71
125
|
else
|
|
72
126
|
transitions[state][ec[byte]]
|
|
@@ -75,6 +129,10 @@ module Flexr
|
|
|
75
129
|
|
|
76
130
|
cursor += 1
|
|
77
131
|
acceptance = accepts[state].first
|
|
132
|
+
if token_limit_required
|
|
133
|
+
size = cursor - text_start
|
|
134
|
+
@lexer.defer_token_size_check!(size, rule: acceptance&.rule_index) if size > max_token_size
|
|
135
|
+
end
|
|
78
136
|
next unless acceptance
|
|
79
137
|
|
|
80
138
|
rule = rules.fetch(acceptance.rule_index)
|
|
@@ -94,7 +152,7 @@ module Flexr
|
|
|
94
152
|
def reference_match(position, buffer)
|
|
95
153
|
return unless reference_rules?
|
|
96
154
|
|
|
97
|
-
candidates = @
|
|
155
|
+
candidates = @rules.filter_map do |rule|
|
|
98
156
|
next unless rule_active?(rule)
|
|
99
157
|
next unless rule.patterns.any? { |pattern| reference_pattern?(pattern) }
|
|
100
158
|
|
|
@@ -102,7 +160,10 @@ module Flexr
|
|
|
102
160
|
condition = rule.pattern_conditions.fetch(pattern_index)
|
|
103
161
|
next if condition.bol_only && !@lexer.beginning_of_line?
|
|
104
162
|
|
|
105
|
-
match = streamed_match(
|
|
163
|
+
match = streamed_match(
|
|
164
|
+
pattern, buffer, position, reference: reference_pattern?(pattern),
|
|
165
|
+
limit: :token, rule: rule
|
|
166
|
+
)
|
|
106
167
|
next unless match&.begin(0)&.zero?
|
|
107
168
|
|
|
108
169
|
end_position = position + match[0].bytesize
|
|
@@ -120,7 +181,7 @@ module Flexr
|
|
|
120
181
|
trailing = trailing_length(rule, buffer, end_position)
|
|
121
182
|
next if rule.trailing && trailing.nil?
|
|
122
183
|
|
|
123
|
-
ensure_token_size!(end_position, position)
|
|
184
|
+
ensure_token_size!(end_position, position, rule: rule)
|
|
124
185
|
Match.new(rule: rule, start_pos: position, end_pos: end_position,
|
|
125
186
|
total_end_pos: end_position + (trailing || 0))
|
|
126
187
|
end
|
|
@@ -128,33 +189,31 @@ module Flexr
|
|
|
128
189
|
end
|
|
129
190
|
|
|
130
191
|
def reference_rules?
|
|
192
|
+
return false unless @config.backend == :firstmatch
|
|
131
193
|
return @reference_rules unless @reference_rules.nil?
|
|
132
194
|
|
|
133
|
-
@reference_rules = @
|
|
195
|
+
@reference_rules = @rules.any? do |rule|
|
|
134
196
|
rule.patterns.any? { |pattern| reference_pattern?(pattern) }
|
|
135
197
|
end
|
|
136
198
|
end
|
|
137
199
|
|
|
138
|
-
def fast_path?
|
|
139
|
-
|
|
200
|
+
def fast_path?(machine)
|
|
201
|
+
@fast_paths ||= {}
|
|
202
|
+
return @fast_paths[machine.dfa] if @fast_paths.key?(machine.dfa)
|
|
140
203
|
|
|
141
|
-
rules = @
|
|
142
|
-
@
|
|
204
|
+
rules = machine.dfa.rule_ids.map { |rule_index| @rules.fetch(rule_index) }
|
|
205
|
+
@fast_paths[machine.dfa] = !@config.options[:allow_empty_match] &&
|
|
143
206
|
!reference_rules? && rules.none?(&:trailing) &&
|
|
144
207
|
rules.none? { |rule| rule.pattern_conditions.any? { |condition| condition&.bol_only || condition&.end_anchor } }
|
|
145
208
|
end
|
|
146
209
|
|
|
147
210
|
def reference_pattern?(pattern)
|
|
148
|
-
|
|
149
|
-
return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
|
|
150
|
-
|
|
151
|
-
@lexer.class.__flexr_config.options[:unicode] == true && @lexer.utf8_input? &&
|
|
152
|
-
pattern.source.match?(/\\[dDwWsS]/)
|
|
211
|
+
Regexp::Matcher.reference?(pattern, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
|
|
153
212
|
end
|
|
154
213
|
|
|
155
214
|
def scan_firstmatch(position)
|
|
156
215
|
buffer = @lexer.buffer
|
|
157
|
-
@
|
|
216
|
+
@rules.sort_by(&:index).each do |rule|
|
|
158
217
|
next unless rule_active?(rule)
|
|
159
218
|
|
|
160
219
|
matches = rule.patterns.each_with_index.filter_map do |pattern, pattern_index|
|
|
@@ -162,10 +221,11 @@ module Flexr
|
|
|
162
221
|
next if condition.bol_only && !@lexer.beginning_of_line?
|
|
163
222
|
|
|
164
223
|
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
165
|
-
streamed_match(
|
|
224
|
+
streamed_match(
|
|
225
|
+
regexp, buffer, position, reference: reference_pattern?(regexp),
|
|
226
|
+
limit: :token, rule: rule
|
|
227
|
+
)
|
|
166
228
|
&.then { |match| [match, condition] }
|
|
167
|
-
rescue ArgumentError, RegexpError
|
|
168
|
-
nil
|
|
169
229
|
end
|
|
170
230
|
match, condition = matches.select { |candidate| candidate[0].begin(0).zero? }
|
|
171
231
|
.max_by { |candidate| candidate[0][0].bytesize }
|
|
@@ -177,7 +237,7 @@ module Flexr
|
|
|
177
237
|
trailing = trailing_length(rule, buffer, end_position)
|
|
178
238
|
next if rule.trailing && trailing.nil?
|
|
179
239
|
|
|
180
|
-
ensure_token_size!(end_position, position)
|
|
240
|
+
ensure_token_size!(end_position, position, rule: rule)
|
|
181
241
|
return reusable_match(rule, position, end_position, end_position + (trailing || 0))
|
|
182
242
|
end
|
|
183
243
|
nil
|
|
@@ -186,28 +246,42 @@ module Flexr
|
|
|
186
246
|
def trailing_length(rule, buffer, position)
|
|
187
247
|
return 0 unless rule.trailing
|
|
188
248
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
249
|
+
dfa = rule.trailing_dfa || trailing_dfa(rule.trailing)
|
|
250
|
+
state = dfa.start
|
|
251
|
+
cursor = position
|
|
252
|
+
longest = 0 if trailing_accept?(dfa, state, buffer, cursor)
|
|
253
|
+
while buffer.ensure_available?(cursor + 1)
|
|
254
|
+
state = dfa.transition(state, buffer.getbyte(cursor))
|
|
255
|
+
break unless state
|
|
256
|
+
|
|
257
|
+
cursor += 1
|
|
258
|
+
@lexer.consume_step!
|
|
259
|
+
size = cursor - position
|
|
260
|
+
@lexer.ensure_lookahead_size!(size, rule: rule)
|
|
261
|
+
longest = size if trailing_accept?(dfa, state, buffer, cursor)
|
|
262
|
+
end
|
|
263
|
+
longest
|
|
264
|
+
end
|
|
192
265
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
266
|
+
def trailing_accept?(dfa, state, buffer, position)
|
|
267
|
+
acceptance = dfa.accepts[state].first
|
|
268
|
+
acceptance && (!acceptance.end_anchor || end_anchor_match?(buffer, position))
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def trailing_dfa(pattern)
|
|
272
|
+
@trailing_dfas ||= {}
|
|
273
|
+
@trailing_dfas[pattern] ||= Flexr.compile_pattern(
|
|
274
|
+
pattern, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
|
|
275
|
+
)
|
|
196
276
|
end
|
|
197
277
|
|
|
198
|
-
def streamed_match(pattern, buffer, position, reference: false)
|
|
278
|
+
def streamed_match(pattern, buffer, position, limit:, rule:, reference: false)
|
|
199
279
|
pattern = regexp_pattern(pattern)
|
|
200
280
|
minimum = minimum_match_bytes(pattern)
|
|
201
281
|
loop do
|
|
202
282
|
subject, tail = stream_subject(buffer, position)
|
|
203
|
-
match =
|
|
204
|
-
|
|
205
|
-
pattern, subject, encoding: @lexer.class.__flexr_config.encoding,
|
|
206
|
-
options: pattern.options, unicode: @lexer.class.__flexr_config.options[:unicode] == true
|
|
207
|
-
)
|
|
208
|
-
else
|
|
209
|
-
pattern.match(subject, 0)
|
|
210
|
-
end
|
|
283
|
+
match = match_stream_pattern(pattern, subject, reference: reference)
|
|
284
|
+
ensure_stream_limit!(limit, position, match[0].bytesize, rule) if match
|
|
211
285
|
return match if match && match[0].bytesize < subject.bytesize
|
|
212
286
|
return match if match && %i[eof invalid].include?(tail)
|
|
213
287
|
if !match && subject.bytesize >= minimum && tail != :incomplete
|
|
@@ -215,12 +289,17 @@ module Flexr
|
|
|
215
289
|
return nil unless first_byte && possible_first_byte?(pattern, first_byte)
|
|
216
290
|
return nil if %i[eof invalid].include?(tail)
|
|
217
291
|
end
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
return
|
|
292
|
+
ensure_stream_limit!(limit, position, subject.bytesize, rule) unless match
|
|
293
|
+
@lexer.consume_step!
|
|
294
|
+
return match unless can_refill_match?(buffer)
|
|
221
295
|
end
|
|
222
296
|
end
|
|
223
297
|
|
|
298
|
+
def match_stream_pattern(pattern, subject, reference:)
|
|
299
|
+
Regexp::Matcher.match(pattern, subject, encoding: @config.encoding,
|
|
300
|
+
unicode: @config.options[:unicode] == true, reference: reference)
|
|
301
|
+
end
|
|
302
|
+
|
|
224
303
|
def stream_subject(buffer, position)
|
|
225
304
|
return [buffer.byteslice(position...buffer.bytesize).to_s.b, buffer.eof_loaded? ? :eof : :end] unless @lexer.utf8_input?
|
|
226
305
|
|
|
@@ -245,7 +324,7 @@ module Flexr
|
|
|
245
324
|
def utf8_status(buffer, position)
|
|
246
325
|
return [:eof, 0] if position >= buffer.bytesize
|
|
247
326
|
|
|
248
|
-
first = buffer.
|
|
327
|
+
first = buffer.getbyte(position)
|
|
249
328
|
return [:complete, 1] if first <= 0x7f
|
|
250
329
|
return [:invalid, 1] unless first.between?(0xc2, 0xf4)
|
|
251
330
|
|
|
@@ -262,13 +341,8 @@ module Flexr
|
|
|
262
341
|
[:complete, length]
|
|
263
342
|
end
|
|
264
343
|
|
|
265
|
-
def can_refill_match?(buffer
|
|
266
|
-
|
|
267
|
-
position + subject_size + 1
|
|
268
|
-
else
|
|
269
|
-
buffer.bytesize + 1
|
|
270
|
-
end
|
|
271
|
-
buffer.ensure_available?(target)
|
|
344
|
+
def can_refill_match?(buffer)
|
|
345
|
+
buffer.ensure_available?(buffer.bytesize + 1)
|
|
272
346
|
end
|
|
273
347
|
|
|
274
348
|
def minimum_match_bytes(pattern)
|
|
@@ -276,7 +350,7 @@ module Flexr
|
|
|
276
350
|
|
|
277
351
|
ast = Regexp::Parser.new(pattern.source, options: pattern.options,
|
|
278
352
|
encoding: pattern.encoding, unicode: true).parse
|
|
279
|
-
minimum_ast_bytes(ast
|
|
353
|
+
minimum_ast_bytes(ast)
|
|
280
354
|
rescue CompileError, RegexpError
|
|
281
355
|
1
|
|
282
356
|
end
|
|
@@ -287,19 +361,20 @@ module Flexr
|
|
|
287
361
|
::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
288
362
|
end
|
|
289
363
|
|
|
290
|
-
def minimum_ast_bytes(node
|
|
364
|
+
def minimum_ast_bytes(node)
|
|
291
365
|
case node
|
|
292
|
-
when Regexp::AST::Empty, Regexp::AST::Anchor then 0
|
|
366
|
+
when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then 0
|
|
293
367
|
when Regexp::AST::ByteRange then 1
|
|
294
368
|
when Regexp::AST::CodepointRange then utf8_length(node.lo)
|
|
295
369
|
when Regexp::AST::CharClass
|
|
296
370
|
ranges = node.ranges.flat_map do |range|
|
|
297
|
-
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase:
|
|
371
|
+
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
|
|
298
372
|
end
|
|
299
373
|
ranges = complement_codepoint_ranges(ranges) if node.negated
|
|
300
374
|
ranges.map { |lo, _hi| utf8_length(lo) }.min || 1
|
|
301
|
-
when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child
|
|
302
|
-
when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child
|
|
375
|
+
when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child) }
|
|
376
|
+
when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child) }.min || 0
|
|
377
|
+
when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child)
|
|
303
378
|
else minimum_unknown_bytes(node)
|
|
304
379
|
end
|
|
305
380
|
end
|
|
@@ -316,24 +391,24 @@ module Flexr
|
|
|
316
391
|
ast = Regexp::Parser.new(pattern.source, options: pattern.options,
|
|
317
392
|
encoding: pattern.encoding, unicode: true).parse
|
|
318
393
|
binary = !@lexer.utf8_input?
|
|
319
|
-
first_byte_ranges(ast,
|
|
394
|
+
first_byte_ranges(ast, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
|
|
320
395
|
rescue CompileError, RegexpError
|
|
321
396
|
true
|
|
322
397
|
end
|
|
323
398
|
|
|
324
|
-
def first_byte_ranges(node,
|
|
399
|
+
def first_byte_ranges(node, binary: false)
|
|
325
400
|
case node
|
|
326
|
-
when Regexp::AST::Empty, Regexp::AST::Anchor then []
|
|
401
|
+
when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then []
|
|
327
402
|
when Regexp::AST::ByteRange then [[node.lo, node.hi]]
|
|
328
403
|
when Regexp::AST::CodepointRange
|
|
329
|
-
ranges =
|
|
404
|
+
ranges = [[node.lo, node.hi]]
|
|
330
405
|
return ranges if binary
|
|
331
406
|
|
|
332
407
|
ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi).map(&:first).map(&:first) }
|
|
333
408
|
.map { |value| [value, value] }
|
|
334
409
|
when Regexp::AST::CharClass
|
|
335
410
|
ranges = node.ranges.flat_map do |range|
|
|
336
|
-
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase:
|
|
411
|
+
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
|
|
337
412
|
end
|
|
338
413
|
ranges = complement_codepoint_ranges(ranges) if node.negated
|
|
339
414
|
return ranges if binary
|
|
@@ -343,14 +418,14 @@ module Flexr
|
|
|
343
418
|
when Regexp::AST::Seq
|
|
344
419
|
ranges = []
|
|
345
420
|
node.children.each do |child|
|
|
346
|
-
ranges.concat(first_byte_ranges(child,
|
|
421
|
+
ranges.concat(first_byte_ranges(child, binary: binary))
|
|
347
422
|
break unless nullable?(child)
|
|
348
423
|
end
|
|
349
424
|
ranges
|
|
350
425
|
when Regexp::AST::Alt
|
|
351
|
-
node.children.flat_map { |child| first_byte_ranges(child,
|
|
352
|
-
when Regexp::AST::Star
|
|
353
|
-
first_byte_ranges(node.child,
|
|
426
|
+
node.children.flat_map { |child| first_byte_ranges(child, binary: binary) }
|
|
427
|
+
when Regexp::AST::Star, Regexp::AST::Repeat
|
|
428
|
+
first_byte_ranges(node.child, binary: binary)
|
|
354
429
|
else
|
|
355
430
|
first_byte_fallback
|
|
356
431
|
end
|
|
@@ -363,6 +438,7 @@ module Flexr
|
|
|
363
438
|
def nullable?(node)
|
|
364
439
|
case node
|
|
365
440
|
when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Star then true
|
|
441
|
+
when Regexp::AST::Repeat then node.minimum.zero? || nullable?(node.child)
|
|
366
442
|
when Regexp::AST::Seq then node.children.all? { |child| nullable?(child) }
|
|
367
443
|
when Regexp::AST::Alt then node.children.any? { |child| nullable?(child) }
|
|
368
444
|
else false
|
|
@@ -400,7 +476,7 @@ module Flexr
|
|
|
400
476
|
|
|
401
477
|
def consider_acceptances(machine, state, cursor, position, buffer, best)
|
|
402
478
|
machine.dfa.accepts[state].each do |acceptance|
|
|
403
|
-
candidate = @
|
|
479
|
+
candidate = @rules.fetch(acceptance.rule_index)
|
|
404
480
|
next if acceptance.bol_only && !@lexer.beginning_of_line?
|
|
405
481
|
next unless @lexer.utf8_boundary?(cursor)
|
|
406
482
|
next unless !acceptance.end_anchor || end_anchor_match?(buffer, cursor)
|
|
@@ -412,7 +488,7 @@ module Flexr
|
|
|
412
488
|
next if best && total_end < best.total_end_pos
|
|
413
489
|
next if best && total_end == best.total_end_pos && acceptance.rule_index > best.rule.index
|
|
414
490
|
|
|
415
|
-
ensure_token_size!(cursor, position)
|
|
491
|
+
ensure_token_size!(cursor, position, rule: candidate)
|
|
416
492
|
best = @match ||= Match.new
|
|
417
493
|
best.rule = candidate
|
|
418
494
|
best.start_pos = position
|
|
@@ -423,7 +499,8 @@ module Flexr
|
|
|
423
499
|
end
|
|
424
500
|
|
|
425
501
|
def acceleration_enabled?
|
|
426
|
-
@
|
|
502
|
+
mode = @config.options.fetch(:accel, :auto)
|
|
503
|
+
mode != :none && !(mode == :auto && @auto_acceleration_off)
|
|
427
504
|
end
|
|
428
505
|
|
|
429
506
|
def reusable_match(rule, start_pos, end_pos, total_end_pos)
|
|
@@ -435,32 +512,63 @@ module Flexr
|
|
|
435
512
|
@match
|
|
436
513
|
end
|
|
437
514
|
|
|
438
|
-
def accelerate(region, buffer, position)
|
|
439
|
-
mode = @
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
515
|
+
def accelerate(region, buffer, position, token_rule:, token_start:)
|
|
516
|
+
mode = @config.options.fetch(:accel, :auto)
|
|
517
|
+
cursor = position
|
|
518
|
+
matched = false
|
|
519
|
+
loop do
|
|
520
|
+
if cursor >= buffer.bytesize
|
|
521
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
522
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
segment = buffer.source
|
|
526
|
+
segment_position = cursor - buffer.base_offset
|
|
527
|
+
length = acceleration_length(region, segment, segment_position, mode)
|
|
528
|
+
record_auto_acceleration_miss(region) if mode == :auto && length.to_i < 8
|
|
529
|
+
return matched ? cursor : nil unless length&.positive?
|
|
530
|
+
|
|
531
|
+
matched = true
|
|
532
|
+
cursor += length
|
|
533
|
+
@lexer.consume_step!(length)
|
|
534
|
+
ensure_token_size!(cursor, token_start, rule: token_rule)
|
|
535
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
448
536
|
end
|
|
449
|
-
|
|
450
|
-
return match_end if match_end && buffer.eof_loaded?
|
|
451
|
-
return unless buffer.ensure_available?(buffer.bytesize + 1)
|
|
537
|
+
end
|
|
452
538
|
|
|
453
|
-
|
|
539
|
+
def acceleration_length(region, segment, position, mode)
|
|
540
|
+
if segment.encoding == Encoding::UTF_8 && region.utf8_regexp
|
|
541
|
+
regexp = region.utf8_regexp
|
|
542
|
+
else
|
|
543
|
+
segment = segment.b unless segment.encoding == Encoding::BINARY
|
|
544
|
+
regexp = region.regexp
|
|
545
|
+
end
|
|
546
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
547
|
+
@acceleration_scanner ||= ::StringScanner.new("".b)
|
|
548
|
+
@acceleration_scanner.string = segment
|
|
549
|
+
@acceleration_scanner.pos = position
|
|
550
|
+
@acceleration_scanner.skip(regexp)
|
|
551
|
+
else
|
|
552
|
+
match = regexp.match(segment.byteslice(position..).to_s, 0)
|
|
553
|
+
match&.begin(0)&.zero? ? match[0].bytesize : nil
|
|
554
|
+
end
|
|
454
555
|
rescue ArgumentError
|
|
455
556
|
nil
|
|
456
557
|
end
|
|
457
558
|
|
|
559
|
+
def record_auto_acceleration_miss(region)
|
|
560
|
+
@auto_acceleration_misses ||= Hash.new(0)
|
|
561
|
+
@auto_acceleration_misses[region] += 1
|
|
562
|
+
@auto_acceleration_miss_total = @auto_acceleration_miss_total.to_i + 1
|
|
563
|
+
@auto_acceleration_off = true if @auto_acceleration_miss_total >= 9
|
|
564
|
+
return if @auto_acceleration_misses[region] < 3
|
|
565
|
+
|
|
566
|
+
@disabled_auto_acceleration ||= {}
|
|
567
|
+
@disabled_auto_acceleration[region] = true
|
|
568
|
+
end
|
|
569
|
+
|
|
458
570
|
def transition(dfa, state, byte)
|
|
459
|
-
if @
|
|
460
|
-
@lexer.class.respond_to?(:__flexr_generated_direct_transition)
|
|
461
|
-
return @lexer.class.__flexr_generated_direct_transition(@lexer.state, state, byte)
|
|
462
|
-
end
|
|
463
|
-
return dfa.transition_direct(state, byte) if @lexer.class.__flexr_config.backend == :direct
|
|
571
|
+
return dfa.transition_direct(state, byte) if @config.backend == :direct
|
|
464
572
|
|
|
465
573
|
dfa.transition(state, byte)
|
|
466
574
|
end
|
|
@@ -474,7 +582,7 @@ module Flexr
|
|
|
474
582
|
|
|
475
583
|
accepting = machine.dfa.accepts[region.state]
|
|
476
584
|
next if accepting.any? do |acceptance|
|
|
477
|
-
rule = @
|
|
585
|
+
rule = @rules.fetch(acceptance.rule_index)
|
|
478
586
|
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
479
587
|
end
|
|
480
588
|
|
|
@@ -486,15 +594,23 @@ module Flexr
|
|
|
486
594
|
buffer.eof?(position) || buffer.getbyte(position) == 0x0a
|
|
487
595
|
end
|
|
488
596
|
|
|
489
|
-
def ensure_token_size!(end_position, position)
|
|
597
|
+
def ensure_token_size!(end_position, position, rule: nil)
|
|
490
598
|
text_start = @lexer.more_text_start || position
|
|
491
|
-
@lexer.defer_token_size_check!(end_position - text_start)
|
|
599
|
+
@lexer.defer_token_size_check!(end_position - text_start, rule: rule)
|
|
600
|
+
end
|
|
601
|
+
|
|
602
|
+
def ensure_stream_limit!(limit, position, size, rule)
|
|
603
|
+
if limit == :lookahead
|
|
604
|
+
@lexer.ensure_lookahead_size!(size, rule: rule)
|
|
605
|
+
else
|
|
606
|
+
ensure_token_size!(position + size, position, rule: rule)
|
|
607
|
+
end
|
|
492
608
|
end
|
|
493
609
|
|
|
494
610
|
def rule_active?(rule)
|
|
495
611
|
return true if rule.states.include?(:initial) && @lexer.state == :initial
|
|
496
612
|
|
|
497
|
-
state = @
|
|
613
|
+
state = @config.states.fetch(@lexer.state)
|
|
498
614
|
return true if state.inclusive && rule.states.include?(:initial)
|
|
499
615
|
|
|
500
616
|
rule.states.include?(@lexer.state)
|
|
@@ -2,24 +2,47 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Runtime
|
|
5
|
-
Location =
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
@column_values =
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
5
|
+
Location = Data.define(:filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end) do
|
|
6
|
+
include Record
|
|
7
|
+
|
|
8
|
+
def initialize(filename: nil, byte_begin: nil, byte_end: nil, line_begin: nil, column_begin: nil,
|
|
9
|
+
line_end: nil, column_end: nil, column_values: nil, eager_columns: false)
|
|
10
|
+
@column_values = [column_begin || column_values&.first, column_end || column_values&.last].freeze
|
|
11
|
+
filename = filename.dup.freeze if filename.is_a?(String)
|
|
12
|
+
column_begin ||= @column_values&.first if eager_columns
|
|
13
|
+
column_end ||= @column_values&.last if eager_columns
|
|
14
|
+
super(filename: filename, byte_begin: byte_begin, byte_end: byte_end, line_begin: line_begin,
|
|
15
|
+
column_begin: column_begin, line_end: line_end, column_end: column_end)
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def column_begin
|
|
18
|
-
|
|
19
|
+
@column_values.first
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
def column_end
|
|
22
|
-
|
|
23
|
+
@column_values.last
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def to_h(&block)
|
|
27
|
+
values = super().merge(column_begin: column_begin, column_end: column_end)
|
|
28
|
+
block ? values.to_h(&block) : values
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def with(**changes)
|
|
32
|
+
columns = @column_values.dup
|
|
33
|
+
columns[0] = changes[:column_begin] if changes.key?(:column_begin)
|
|
34
|
+
columns[1] = changes[:column_end] if changes.key?(:column_end)
|
|
35
|
+
self.class.new(**deconstruct_keys(nil), **changes, column_values: columns)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def ==(other)
|
|
39
|
+
other.instance_of?(self.class) && to_h == other.to_h
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
alias_method :eql?, :==
|
|
43
|
+
|
|
44
|
+
def hash
|
|
45
|
+
[self.class, to_h].hash
|
|
23
46
|
end
|
|
24
47
|
end
|
|
25
48
|
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Runtime
|
|
5
|
+
module Record
|
|
6
|
+
def [](key)
|
|
7
|
+
member = key.is_a?(Integer) ? members.fetch(key) : key.to_sym
|
|
8
|
+
raise NameError, "unknown record member #{key.inspect}" unless members.include?(member)
|
|
9
|
+
|
|
10
|
+
public_send(member)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
data/lib/flexr/runtime/token.rb
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Runtime
|
|
5
|
-
Token =
|
|
5
|
+
Token = Data.define(:type, :value, :location) do
|
|
6
|
+
include Record
|
|
7
|
+
|
|
8
|
+
def initialize(type: nil, value: nil, location: nil)
|
|
9
|
+
super
|
|
10
|
+
end
|
|
11
|
+
end
|
|
6
12
|
end
|
|
7
13
|
end
|