flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
data/lib/flexr/runtime/core.rb
CHANGED
|
@@ -3,20 +3,46 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Runtime
|
|
5
5
|
def initialize(input, filename: nil, error_mode: :raise, max_token_size: 16 * 1024 * 1024,
|
|
6
|
-
|
|
6
|
+
max_lookahead_size: nil, max_buffer_size: 64 * 1024 * 1024,
|
|
7
|
+
max_state_stack: 1024, max_steps: nil, cancellation: nil, retain_input: true,
|
|
8
|
+
chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
|
|
9
|
+
raise ArgumentError, "unsupported error_mode: #{error_mode.inspect}" unless %i[raise token panic].include?(error_mode)
|
|
7
10
|
raise ArgumentError, "max_token_size must be non-negative" if max_token_size.to_i.negative?
|
|
11
|
+
raise ArgumentError, "max_lookahead_size must be non-negative" if max_lookahead_size&.to_i&.negative?
|
|
12
|
+
raise ArgumentError, "max_buffer_size must be non-negative" if max_buffer_size.to_i.negative?
|
|
8
13
|
raise ArgumentError, "max_state_stack must be non-negative" if max_state_stack.to_i.negative?
|
|
14
|
+
raise ArgumentError, "max_steps must be non-negative" if max_steps&.to_i&.negative?
|
|
15
|
+
raise ArgumentError, "cancellation must respond to call" if cancellation && !cancellation.respond_to?(:call)
|
|
9
16
|
|
|
10
17
|
self.class.compile!
|
|
11
|
-
|
|
12
|
-
@
|
|
13
|
-
|
|
18
|
+
config = self.class.__flexr_config
|
|
19
|
+
@buffer = Runtime::Buffer.new(
|
|
20
|
+
input, chunk_size: chunk_size, max_buffer_size: max_buffer_size,
|
|
21
|
+
retain_input: retain_input, filename: filename
|
|
22
|
+
)
|
|
23
|
+
@string_input = input.is_a?(String)
|
|
24
|
+
@stable_input_end = input.bytesize if @string_input && input.frozen?
|
|
25
|
+
@valid_utf8_input = @string_input && input.frozen? && (config.encoding != Encoding::UTF_8 || begin
|
|
26
|
+
bytes = input.encoding == Encoding::UTF_8 ? input : input.dup.force_encoding(Encoding::UTF_8)
|
|
27
|
+
bytes.valid_encoding?
|
|
28
|
+
end)
|
|
29
|
+
@simple_location_input = @string_input && input.frozen? && retain_input &&
|
|
30
|
+
input.ascii_only? && !input.include?("\n")
|
|
31
|
+
@retain_input = retain_input
|
|
14
32
|
@filename = filename
|
|
15
33
|
@error_mode = error_mode
|
|
34
|
+
@token_kind = config.token_kind
|
|
35
|
+
@accel_mode = config.options.fetch(:accel, :auto)
|
|
16
36
|
@max_token_size = max_token_size.to_i
|
|
37
|
+
@token_limit_required = !@string_input || !input.frozen? || input.bytesize > @max_token_size
|
|
38
|
+
@max_lookahead_size = (max_lookahead_size || max_token_size).to_i
|
|
17
39
|
@max_state_stack = max_state_stack.to_i
|
|
40
|
+
@max_steps = max_steps&.to_i
|
|
41
|
+
@cancellation = cancellation.respond_to?(:arity) ? cancellation : cancellation&.method(:call)
|
|
42
|
+
@steps = 0
|
|
18
43
|
@position = 0
|
|
19
44
|
@line = 1
|
|
45
|
+
@column = 1
|
|
20
46
|
@state = :initial
|
|
21
47
|
@state_stack = []
|
|
22
48
|
@pending = nil
|
|
@@ -25,18 +51,22 @@ module Flexr
|
|
|
25
51
|
@match_end = 0
|
|
26
52
|
@text_start = 0
|
|
27
53
|
@text_line = 1
|
|
54
|
+
@text_column = 1
|
|
28
55
|
@bol = true
|
|
29
56
|
@more_start = nil
|
|
30
57
|
@more_line = nil
|
|
58
|
+
@more_column = nil
|
|
31
59
|
@more_requested = false
|
|
32
|
-
@
|
|
60
|
+
@non_progress_signatures = {}
|
|
61
|
+
@active_rule = nil
|
|
33
62
|
@eof_fired_states = {}
|
|
34
63
|
@on_error = nil
|
|
35
64
|
@halted = false
|
|
36
65
|
@interpreter = nil
|
|
66
|
+
@generated_scanner = generated_runtime? && respond_to?(:scan_one, true)
|
|
37
67
|
end
|
|
38
68
|
|
|
39
|
-
attr_reader :filename, :error_mode, :buffer, :max_token_size
|
|
69
|
+
attr_reader :filename, :error_mode, :buffer, :max_token_size, :max_lookahead_size, :steps
|
|
40
70
|
|
|
41
71
|
attr_writer :on_error
|
|
42
72
|
|
|
@@ -49,6 +79,7 @@ module Flexr
|
|
|
49
79
|
|
|
50
80
|
loop do
|
|
51
81
|
return nil if @halted
|
|
82
|
+
consume_step! if scan_steps_guarded?
|
|
52
83
|
|
|
53
84
|
if eof? && @pending.nil?
|
|
54
85
|
eof_action = self.class.__flexr_spec.eof_rules[@state]
|
|
@@ -58,6 +89,7 @@ module Flexr
|
|
|
58
89
|
@match_end = @position
|
|
59
90
|
@text_start = @position
|
|
60
91
|
@text_line = @line
|
|
92
|
+
@text_column = @column
|
|
61
93
|
@matched = nil
|
|
62
94
|
instance_exec(&eof_action)
|
|
63
95
|
token = @pending
|
|
@@ -68,7 +100,8 @@ module Flexr
|
|
|
68
100
|
return nil
|
|
69
101
|
end
|
|
70
102
|
|
|
71
|
-
|
|
103
|
+
@active_rule = nil
|
|
104
|
+
match = if @generated_scanner
|
|
72
105
|
scan_one
|
|
73
106
|
else
|
|
74
107
|
(@interpreter ||= Runtime::Interpreter.new(self)).scan
|
|
@@ -84,22 +117,28 @@ module Flexr
|
|
|
84
117
|
end
|
|
85
118
|
@match_start = match.start_pos
|
|
86
119
|
@match_end = match.end_pos
|
|
120
|
+
scan_state = @state
|
|
87
121
|
if @more_start
|
|
88
122
|
@text_start = @more_start
|
|
89
123
|
@text_line = @more_line
|
|
124
|
+
@text_column = @more_column
|
|
90
125
|
else
|
|
91
126
|
@text_start = @match_start
|
|
92
127
|
@text_line = @line
|
|
128
|
+
@text_column = @column
|
|
93
129
|
end
|
|
94
130
|
@matched = nil
|
|
95
131
|
@more_requested = false
|
|
96
132
|
@position = match.end_pos
|
|
97
133
|
empty_match = match.end_pos == match.start_pos
|
|
134
|
+
@active_rule = match.rule
|
|
98
135
|
execute(match.rule)
|
|
99
136
|
finalize_more
|
|
100
|
-
|
|
101
|
-
|
|
137
|
+
ensure_progress!(match, scan_state, empty_match: empty_match) unless
|
|
138
|
+
@position > match.start_pos && @non_progress_signatures.empty?
|
|
139
|
+
ensure_token_size! if @token_limit_required
|
|
102
140
|
update_position
|
|
141
|
+
discard_consumed_input! unless @retain_input
|
|
103
142
|
token = @pending
|
|
104
143
|
@pending = nil
|
|
105
144
|
return token if token
|
|
@@ -113,7 +152,7 @@ module Flexr
|
|
|
113
152
|
token = next_token
|
|
114
153
|
break unless token
|
|
115
154
|
|
|
116
|
-
if
|
|
155
|
+
if @token_kind == :yield
|
|
117
156
|
yield(*token)
|
|
118
157
|
else
|
|
119
158
|
yield token
|
|
@@ -138,7 +177,7 @@ module Flexr
|
|
|
138
177
|
def text
|
|
139
178
|
return @matched if @matched
|
|
140
179
|
|
|
141
|
-
@matched = @buffer.byteslice(@text_start
|
|
180
|
+
@matched = @buffer.byteslice(@text_start, @match_end - @text_start)
|
|
142
181
|
end
|
|
143
182
|
|
|
144
183
|
def text_bytesize
|
|
@@ -153,8 +192,71 @@ module Flexr
|
|
|
153
192
|
@more_start
|
|
154
193
|
end
|
|
155
194
|
|
|
156
|
-
def defer_token_size_check!(size)
|
|
157
|
-
|
|
195
|
+
def defer_token_size_check!(size, rule: nil)
|
|
196
|
+
return if size <= @max_token_size
|
|
197
|
+
|
|
198
|
+
raise Runtime::TokenTooLargeError.new(
|
|
199
|
+
filename: @filename, byte_pos: @more_start || @position, line: @line,
|
|
200
|
+
rule: diagnostic_rule(rule || @active_rule)
|
|
201
|
+
)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def ensure_lookahead_size!(size, rule: nil)
|
|
205
|
+
return if size <= @max_lookahead_size
|
|
206
|
+
|
|
207
|
+
raise Runtime::LookaheadTooLargeError.new(
|
|
208
|
+
filename: @filename, byte_pos: @position, line: @line,
|
|
209
|
+
rule: diagnostic_rule(rule || @active_rule)
|
|
210
|
+
)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def consume_step!(count = 1)
|
|
214
|
+
return unless scan_steps_guarded?
|
|
215
|
+
|
|
216
|
+
@steps += count
|
|
217
|
+
if @max_steps && @steps > @max_steps
|
|
218
|
+
raise Runtime::StepLimitError.new(
|
|
219
|
+
filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
|
|
220
|
+
)
|
|
221
|
+
end
|
|
222
|
+
return unless cancelled?
|
|
223
|
+
|
|
224
|
+
raise Runtime::CancelledError.new(
|
|
225
|
+
filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
|
|
226
|
+
)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def scan_steps_guarded?
|
|
230
|
+
!@max_steps.nil? || !@cancellation.nil?
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def token_limit_required?
|
|
234
|
+
@token_limit_required
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def __flexr_whole_token_match(plans, position)
|
|
238
|
+
return unless plans && defined?(::StringScanner) && !scan_steps_guarded? && !token_limit_required?
|
|
239
|
+
|
|
240
|
+
source = @buffer.stable_source
|
|
241
|
+
return unless source&.frozen?
|
|
242
|
+
|
|
243
|
+
byte = source.getbyte(position)
|
|
244
|
+
plan = plans[byte] if byte
|
|
245
|
+
return unless plan
|
|
246
|
+
|
|
247
|
+
scanner = @__flexr_whole_scanner ||= ::StringScanner.new(source)
|
|
248
|
+
scanner.pos = position
|
|
249
|
+
length = scanner.skip(plan.regexp)
|
|
250
|
+
return unless length
|
|
251
|
+
|
|
252
|
+
match = @__flexr_whole_match ||= Runtime::Match.new
|
|
253
|
+
match.rule = self.class.__flexr_rules.fetch(plan.rule_index)
|
|
254
|
+
match.start_pos = position
|
|
255
|
+
match.end_pos = position + length
|
|
256
|
+
match.total_end_pos = match.end_pos
|
|
257
|
+
match
|
|
258
|
+
rescue ArgumentError
|
|
259
|
+
nil
|
|
158
260
|
end
|
|
159
261
|
|
|
160
262
|
def utf8_input?
|
|
@@ -162,7 +264,7 @@ module Flexr
|
|
|
162
264
|
end
|
|
163
265
|
|
|
164
266
|
def valid_utf8_at?(position)
|
|
165
|
-
!utf8_input? || @buffer.valid_utf8_at?(position)
|
|
267
|
+
@valid_utf8_input || !utf8_input? || @buffer.valid_utf8_at?(position)
|
|
166
268
|
end
|
|
167
269
|
|
|
168
270
|
def utf8_boundary?(position)
|
|
@@ -184,11 +286,12 @@ module Flexr
|
|
|
184
286
|
end
|
|
185
287
|
|
|
186
288
|
def binary_input
|
|
187
|
-
@buffer.source
|
|
289
|
+
source = @buffer.source
|
|
290
|
+
source.encoding == ::Encoding::BINARY ? source : source.b
|
|
188
291
|
end
|
|
189
292
|
|
|
190
293
|
def emit(type, value = text)
|
|
191
|
-
@pending = case
|
|
294
|
+
@pending = case @token_kind
|
|
192
295
|
when :struct
|
|
193
296
|
Runtime::Token.new(type: type, value: value, location: last_location)
|
|
194
297
|
else
|
|
@@ -201,7 +304,7 @@ module Flexr
|
|
|
201
304
|
end
|
|
202
305
|
|
|
203
306
|
def error!(message)
|
|
204
|
-
error = LexError.new(message, filename: @filename, byte_pos: @match_start, line: @
|
|
307
|
+
error = LexError.new(message, filename: @filename, byte_pos: @match_start, line: @text_line, text: text)
|
|
205
308
|
if @on_error
|
|
206
309
|
action = @on_error.call(error)
|
|
207
310
|
return @pending = nil if action == :skip
|
|
@@ -211,6 +314,11 @@ module Flexr
|
|
|
211
314
|
return @pending = nil
|
|
212
315
|
end
|
|
213
316
|
return emit(:error, text) if action == :token
|
|
317
|
+
|
|
318
|
+
raise Runtime::InvalidRecoveryActionError.new(
|
|
319
|
+
action: action, filename: @filename, byte_pos: @match_start, line: @text_line,
|
|
320
|
+
text: text, rule: @active_rule
|
|
321
|
+
)
|
|
214
322
|
end
|
|
215
323
|
case @error_mode
|
|
216
324
|
when :token
|
|
@@ -229,9 +337,10 @@ module Flexr
|
|
|
229
337
|
def reject
|
|
230
338
|
diagnostic = Diagnostics.error(
|
|
231
339
|
"FLEXR-E013", "reject is not supported by flexr",
|
|
340
|
+
location: [@position, @position],
|
|
232
341
|
help: "use a state transition and less(n) to express the fallback"
|
|
233
342
|
)
|
|
234
|
-
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
343
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
|
|
235
344
|
end
|
|
236
345
|
|
|
237
346
|
def push(name)
|
|
@@ -239,7 +348,8 @@ module Flexr
|
|
|
239
348
|
if @state_stack.length >= @max_state_stack
|
|
240
349
|
raise Runtime::StateStackOverflowError.new(
|
|
241
350
|
"state stack exceeds max_state_stack (#{@max_state_stack})",
|
|
242
|
-
filename: @filename, byte_pos: @position, line: @line, text: text
|
|
351
|
+
filename: @filename, byte_pos: @position, line: @line, text: text,
|
|
352
|
+
rule: @active_rule
|
|
243
353
|
)
|
|
244
354
|
end
|
|
245
355
|
|
|
@@ -266,7 +376,11 @@ module Flexr
|
|
|
266
376
|
matched_bytes = @match_end - @match_start
|
|
267
377
|
raise ArgumentError, "less must not exceed matched bytes" if count.negative? || count > matched_bytes
|
|
268
378
|
|
|
269
|
-
|
|
379
|
+
new_position = @match_start + count
|
|
380
|
+
raise ArgumentError, "less must end at a UTF-8 codepoint boundary" if
|
|
381
|
+
utf8_input? && !@buffer.utf8_boundary?(new_position)
|
|
382
|
+
|
|
383
|
+
@position = new_position
|
|
270
384
|
@match_end = @position
|
|
271
385
|
@matched = nil
|
|
272
386
|
end
|
|
@@ -277,10 +391,14 @@ module Flexr
|
|
|
277
391
|
|
|
278
392
|
def last_location
|
|
279
393
|
line_begin = @text_line || @line
|
|
394
|
+
column_begin = @text_column || @column
|
|
395
|
+
line_end, column_end = location_after(
|
|
396
|
+
@text_start, @match_end, line: line_begin, column: column_begin
|
|
397
|
+
)
|
|
280
398
|
Runtime::Location.new(
|
|
281
399
|
filename: @filename, byte_begin: @text_start, byte_end: @match_end,
|
|
282
|
-
line_begin: line_begin, line_end:
|
|
283
|
-
column_values: [
|
|
400
|
+
line_begin: line_begin, line_end: line_end,
|
|
401
|
+
column_values: [column_begin, column_end],
|
|
284
402
|
eager_columns: self.class.__flexr_config.options[:eager_columns] == true
|
|
285
403
|
)
|
|
286
404
|
end
|
|
@@ -304,9 +422,11 @@ module Flexr
|
|
|
304
422
|
if @more_requested
|
|
305
423
|
@more_start = @text_start
|
|
306
424
|
@more_line = @text_line
|
|
425
|
+
@more_column = @text_column
|
|
307
426
|
else
|
|
308
427
|
@more_start = nil
|
|
309
428
|
@more_line = nil
|
|
429
|
+
@more_column = nil
|
|
310
430
|
end
|
|
311
431
|
@more_requested = false
|
|
312
432
|
end
|
|
@@ -316,34 +436,31 @@ module Flexr
|
|
|
316
436
|
end
|
|
317
437
|
|
|
318
438
|
def ensure_token_size!
|
|
319
|
-
|
|
320
|
-
@candidate_token_size = 0
|
|
321
|
-
return if actual_size <= @max_token_size
|
|
439
|
+
return unless @token_limit_required
|
|
322
440
|
|
|
323
|
-
|
|
441
|
+
actual_size = @match_end - @text_start
|
|
442
|
+
defer_token_size_check!(actual_size, rule: @active_rule)
|
|
324
443
|
end
|
|
325
444
|
|
|
326
445
|
def force_empty_match_progress!
|
|
327
446
|
return if eof?
|
|
328
447
|
|
|
329
|
-
|
|
330
|
-
@position
|
|
331
|
-
@
|
|
332
|
-
@
|
|
448
|
+
length = utf8_input? ? @buffer.utf8_character_length(@position) : 1
|
|
449
|
+
ending = @position + length
|
|
450
|
+
advance_location!(@position, ending)
|
|
451
|
+
@position = ending
|
|
452
|
+
@bol = @buffer.getbyte(@position - 1) == 0x0a
|
|
333
453
|
end
|
|
334
454
|
|
|
335
455
|
def update_position
|
|
336
|
-
if @
|
|
337
|
-
|
|
338
|
-
@
|
|
339
|
-
|
|
340
|
-
elsif @line_tracking_needed && (newline = @buffer.source.index("\n", @match_start)) && newline < @match_end
|
|
341
|
-
@buffer.byteslice(@match_start, length).to_s.count("\n")
|
|
342
|
-
else
|
|
343
|
-
0
|
|
344
|
-
end
|
|
456
|
+
if @simple_location_input
|
|
457
|
+
@column += @match_end - @match_start
|
|
458
|
+
@bol = @match_end.zero?
|
|
459
|
+
return
|
|
345
460
|
end
|
|
346
|
-
|
|
461
|
+
|
|
462
|
+
advance_location!(@match_start, @match_end)
|
|
463
|
+
@bol = @match_end.zero? || @buffer.getbyte(@match_end - 1) == 0x0a
|
|
347
464
|
end
|
|
348
465
|
|
|
349
466
|
def handle_unmatched_byte
|
|
@@ -352,17 +469,22 @@ module Flexr
|
|
|
352
469
|
@match_end = @position + 1
|
|
353
470
|
@text_start = @match_start
|
|
354
471
|
@text_line = @line
|
|
472
|
+
@text_column = @column
|
|
473
|
+
@matched = nil
|
|
355
474
|
@position += 1
|
|
356
|
-
@
|
|
475
|
+
@non_progress_signatures.clear
|
|
476
|
+
advance_location!(@match_start, @match_end)
|
|
357
477
|
@bol = bad.to_s.b == "\n"
|
|
358
478
|
error!("unexpected byte #{bad.inspect}")
|
|
359
479
|
token = @pending
|
|
360
480
|
@pending = nil
|
|
481
|
+
discard_consumed_input! unless @retain_input
|
|
361
482
|
token
|
|
362
483
|
end
|
|
363
484
|
|
|
364
485
|
def eof?
|
|
365
|
-
return @position >= @
|
|
486
|
+
return @position >= @stable_input_end if @stable_input_end
|
|
487
|
+
return @position >= @buffer.bytesize if @string_input
|
|
366
488
|
|
|
367
489
|
@buffer.eof?(@position)
|
|
368
490
|
end
|
|
@@ -370,19 +492,94 @@ module Flexr
|
|
|
370
492
|
def ensure_state!(name)
|
|
371
493
|
return if self.class.__flexr_config.states.key?(name.to_sym)
|
|
372
494
|
|
|
373
|
-
diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}")
|
|
374
|
-
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
495
|
+
diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}", location: [@position, @position])
|
|
496
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
def ensure_progress!(match, scan_state, empty_match:)
|
|
500
|
+
if @position > match.start_pos
|
|
501
|
+
@non_progress_signatures.clear
|
|
502
|
+
return
|
|
503
|
+
end
|
|
504
|
+
return if @halted
|
|
505
|
+
|
|
506
|
+
if empty_match && @state == scan_state && self.class.__flexr_config.options[:allow_empty_match]
|
|
507
|
+
force_empty_match_progress!
|
|
508
|
+
@non_progress_signatures.clear
|
|
509
|
+
return
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
signature = [match.start_pos, scan_state, match.rule.index]
|
|
513
|
+
repeated = @non_progress_signatures.key?(signature)
|
|
514
|
+
@non_progress_signatures[signature] = true
|
|
515
|
+
return if @state != scan_state && !repeated
|
|
516
|
+
|
|
517
|
+
raise Runtime::NonProgressError.new(
|
|
518
|
+
filename: @filename, byte_pos: match.start_pos, line: @line,
|
|
519
|
+
text: text, rule: match.rule
|
|
520
|
+
)
|
|
375
521
|
end
|
|
376
522
|
|
|
377
|
-
def
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
523
|
+
def advance_location!(starting, ending)
|
|
524
|
+
return if ending <= starting
|
|
525
|
+
|
|
526
|
+
if @simple_location_input
|
|
527
|
+
@column += ending - starting
|
|
528
|
+
return
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
value = location_value(starting, ending)
|
|
532
|
+
location_bytes = value.b
|
|
533
|
+
newline_count = location_bytes.count("\n")
|
|
534
|
+
if newline_count.zero?
|
|
535
|
+
@column += display_length(value)
|
|
383
536
|
else
|
|
384
|
-
|
|
537
|
+
tail = value.byteslice(((location_bytes.rindex("\n") || -1) + 1)..).to_s
|
|
538
|
+
@line += newline_count
|
|
539
|
+
@column = display_length(tail) + 1
|
|
385
540
|
end
|
|
386
541
|
end
|
|
542
|
+
|
|
543
|
+
def location_after(starting, ending, line:, column:)
|
|
544
|
+
return [line, column] if ending <= starting
|
|
545
|
+
|
|
546
|
+
value = location_value(starting, ending)
|
|
547
|
+
location_bytes = value.b
|
|
548
|
+
newline_count = location_bytes.count("\n")
|
|
549
|
+
return [line, column + display_length(value)] if newline_count.zero?
|
|
550
|
+
|
|
551
|
+
tail = value.byteslice(((location_bytes.rindex("\n") || -1) + 1)..).to_s
|
|
552
|
+
[line + newline_count, display_length(tail) + 1]
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
def display_length(value)
|
|
556
|
+
return value.bytesize unless utf8_input?
|
|
557
|
+
return value.bytesize if value.ascii_only?
|
|
558
|
+
|
|
559
|
+
utf8 = value.encoding == Encoding::UTF_8 ? value : value.dup.force_encoding(Encoding::UTF_8)
|
|
560
|
+
utf8.valid_encoding? ? utf8.length : value.bytesize
|
|
561
|
+
end
|
|
562
|
+
|
|
563
|
+
def location_value(starting, ending)
|
|
564
|
+
return @matched if @matched && starting == @text_start && ending == @match_end
|
|
565
|
+
|
|
566
|
+
@buffer.byteslice(starting, ending - starting).to_s
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
def discard_consumed_input!
|
|
570
|
+
@buffer.discard_before(@more_start || @position)
|
|
571
|
+
end
|
|
572
|
+
|
|
573
|
+
def cancelled?
|
|
574
|
+
return false unless @cancellation
|
|
575
|
+
|
|
576
|
+
@cancellation.arity.zero? ? @cancellation.call : @cancellation.call(self)
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
def diagnostic_rule(rule)
|
|
580
|
+
return rule unless rule.is_a?(Integer) && rule >= 0
|
|
581
|
+
|
|
582
|
+
self.class.__flexr_rules.fetch(rule, rule)
|
|
583
|
+
end
|
|
387
584
|
end
|
|
388
585
|
end
|
data/lib/flexr/runtime/errors.rb
CHANGED
|
@@ -5,18 +5,80 @@ module Flexr
|
|
|
5
5
|
class TokenTooLargeError < LexError
|
|
6
6
|
CODE = "FLEXR-E012"
|
|
7
7
|
|
|
8
|
-
attr_reader :code
|
|
8
|
+
attr_reader :code, :rule
|
|
9
9
|
|
|
10
|
-
def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil
|
|
10
|
+
def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil,
|
|
11
|
+
rule: nil)
|
|
11
12
|
@code = CODE
|
|
13
|
+
@rule = rule.is_a?(IR::Rule) ? rule.index : rule
|
|
12
14
|
diagnostic = Diagnostics.error(
|
|
13
15
|
CODE,
|
|
14
16
|
message,
|
|
15
|
-
|
|
17
|
+
location: byte_pos && [byte_pos, byte_pos],
|
|
18
|
+
help: "increase max_token_size or split the input token",
|
|
19
|
+
note: rule.nil? ? nil : "#{Diagnostics.rule_label(rule)} at byte #{byte_pos}"
|
|
16
20
|
)
|
|
17
21
|
super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
|
|
18
22
|
end
|
|
19
23
|
end
|
|
20
|
-
|
|
24
|
+
|
|
25
|
+
class ResourceLimitError < LexError
|
|
26
|
+
attr_reader :code, :rule
|
|
27
|
+
|
|
28
|
+
def initialize(code, message, help:, filename: nil, byte_pos: nil, line: nil, text: nil, rule: nil)
|
|
29
|
+
@code = code
|
|
30
|
+
@rule = rule.is_a?(IR::Rule) ? rule.index : rule
|
|
31
|
+
diagnostic = Diagnostics.error(code, message, location: byte_pos && [byte_pos, byte_pos],
|
|
32
|
+
help: help, note: Diagnostics.rule_label(rule))
|
|
33
|
+
super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
class NonProgressError < ResourceLimitError
|
|
38
|
+
def initialize(**context)
|
|
39
|
+
super("FLEXR-E021", "lexer action did not make progress",
|
|
40
|
+
help: "consume input or change to a state that can consume it", **context)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
class BufferTooLargeError < ResourceLimitError
|
|
45
|
+
def initialize(**context)
|
|
46
|
+
super("FLEXR-E022", "streaming buffer exceeds max_buffer_size",
|
|
47
|
+
help: "increase max_buffer_size, shorten tokens, or use retain_input: false", **context)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
class StepLimitError < ResourceLimitError
|
|
52
|
+
def initialize(**context)
|
|
53
|
+
super("FLEXR-E023", "lexer exceeds max_steps",
|
|
54
|
+
help: "increase max_steps or simplify the specification", **context)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
class LookaheadTooLargeError < ResourceLimitError
|
|
59
|
+
def initialize(**context)
|
|
60
|
+
super("FLEXR-E024", "trailing context exceeds max_lookahead_size",
|
|
61
|
+
help: "increase max_lookahead_size or bound the trailing context", **context)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
class StateStackOverflowError < ResourceLimitError
|
|
66
|
+
def initialize(message = "state stack exceeds max_state_stack", **context)
|
|
67
|
+
super("FLEXR-E025", message, help: "increase max_state_stack or remove recursive pushes", **context)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
class CancelledError < ResourceLimitError
|
|
72
|
+
def initialize(**context)
|
|
73
|
+
super("FLEXR-E026", "lexing was cancelled", help: "resume with a new lexer when ready", **context)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
class InvalidRecoveryActionError < ResourceLimitError
|
|
78
|
+
def initialize(action:, **context)
|
|
79
|
+
super("FLEXR-E027", "on_error returned unsupported action #{action.inspect}",
|
|
80
|
+
help: "return :skip, :raise, :halt, or :token", **context)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
21
83
|
end
|
|
22
84
|
end
|