flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -24,7 +24,14 @@ module Flexr
|
|
|
24
24
|
position = @lexer.byte_pos
|
|
25
25
|
return nil unless @lexer.valid_utf8_at?(position)
|
|
26
26
|
return scan_firstmatch(position) if @config.backend == :firstmatch
|
|
27
|
-
|
|
27
|
+
if fast_path?(machine)
|
|
28
|
+
if machine.whole_tokens
|
|
29
|
+
match = @lexer.__flexr_whole_token_match(machine.whole_tokens, position)
|
|
30
|
+
return match if match
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
return scan_fast(machine, position)
|
|
34
|
+
end
|
|
28
35
|
|
|
29
36
|
buffer = @lexer.buffer
|
|
30
37
|
state = machine.dfa.start
|
|
@@ -201,11 +208,7 @@ module Flexr
|
|
|
201
208
|
end
|
|
202
209
|
|
|
203
210
|
def reference_pattern?(pattern)
|
|
204
|
-
|
|
205
|
-
return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
|
|
206
|
-
|
|
207
|
-
@config.options[:unicode] == true && @lexer.utf8_input? &&
|
|
208
|
-
pattern.source.match?(/\\[dDwWsS]/)
|
|
211
|
+
Regexp::Matcher.reference?(pattern, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
|
|
209
212
|
end
|
|
210
213
|
|
|
211
214
|
def scan_firstmatch(position)
|
|
@@ -243,14 +246,33 @@ module Flexr
|
|
|
243
246
|
def trailing_length(rule, buffer, position)
|
|
244
247
|
return 0 unless rule.trailing
|
|
245
248
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
)
|
|
251
|
-
|
|
249
|
+
dfa = rule.trailing_dfa || trailing_dfa(rule.trailing)
|
|
250
|
+
state = dfa.start
|
|
251
|
+
cursor = position
|
|
252
|
+
longest = 0 if trailing_accept?(dfa, state, buffer, cursor)
|
|
253
|
+
while buffer.ensure_available?(cursor + 1)
|
|
254
|
+
state = dfa.transition(state, buffer.getbyte(cursor))
|
|
255
|
+
break unless state
|
|
252
256
|
|
|
253
|
-
|
|
257
|
+
cursor += 1
|
|
258
|
+
@lexer.consume_step!
|
|
259
|
+
size = cursor - position
|
|
260
|
+
@lexer.ensure_lookahead_size!(size, rule: rule)
|
|
261
|
+
longest = size if trailing_accept?(dfa, state, buffer, cursor)
|
|
262
|
+
end
|
|
263
|
+
longest
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def trailing_accept?(dfa, state, buffer, position)
|
|
267
|
+
acceptance = dfa.accepts[state].first
|
|
268
|
+
acceptance && (!acceptance.end_anchor || end_anchor_match?(buffer, position))
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def trailing_dfa(pattern)
|
|
272
|
+
@trailing_dfas ||= {}
|
|
273
|
+
@trailing_dfas[pattern] ||= Flexr.compile_pattern(
|
|
274
|
+
pattern, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
|
|
275
|
+
)
|
|
254
276
|
end
|
|
255
277
|
|
|
256
278
|
def streamed_match(pattern, buffer, position, limit:, rule:, reference: false)
|
|
@@ -269,21 +291,13 @@ module Flexr
|
|
|
269
291
|
end
|
|
270
292
|
ensure_stream_limit!(limit, position, subject.bytesize, rule) unless match
|
|
271
293
|
@lexer.consume_step!
|
|
272
|
-
return match unless can_refill_match?(buffer
|
|
294
|
+
return match unless can_refill_match?(buffer)
|
|
273
295
|
end
|
|
274
296
|
end
|
|
275
297
|
|
|
276
298
|
def match_stream_pattern(pattern, subject, reference:)
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
pattern, subject, encoding: @config.encoding,
|
|
280
|
-
options: pattern.options, unicode: @config.options[:unicode] == true
|
|
281
|
-
)
|
|
282
|
-
else
|
|
283
|
-
pattern.match(subject, 0)
|
|
284
|
-
end
|
|
285
|
-
rescue ArgumentError, RegexpError
|
|
286
|
-
nil
|
|
299
|
+
Regexp::Matcher.match(pattern, subject, encoding: @config.encoding,
|
|
300
|
+
unicode: @config.options[:unicode] == true, reference: reference)
|
|
287
301
|
end
|
|
288
302
|
|
|
289
303
|
def stream_subject(buffer, position)
|
|
@@ -327,13 +341,8 @@ module Flexr
|
|
|
327
341
|
[:complete, length]
|
|
328
342
|
end
|
|
329
343
|
|
|
330
|
-
def can_refill_match?(buffer
|
|
331
|
-
|
|
332
|
-
position + subject_size + 1
|
|
333
|
-
else
|
|
334
|
-
buffer.bytesize + 1
|
|
335
|
-
end
|
|
336
|
-
buffer.ensure_available?(target)
|
|
344
|
+
def can_refill_match?(buffer)
|
|
345
|
+
buffer.ensure_available?(buffer.bytesize + 1)
|
|
337
346
|
end
|
|
338
347
|
|
|
339
348
|
def minimum_match_bytes(pattern)
|
|
@@ -341,7 +350,7 @@ module Flexr
|
|
|
341
350
|
|
|
342
351
|
ast = Regexp::Parser.new(pattern.source, options: pattern.options,
|
|
343
352
|
encoding: pattern.encoding, unicode: true).parse
|
|
344
|
-
minimum_ast_bytes(ast
|
|
353
|
+
minimum_ast_bytes(ast)
|
|
345
354
|
rescue CompileError, RegexpError
|
|
346
355
|
1
|
|
347
356
|
end
|
|
@@ -352,20 +361,20 @@ module Flexr
|
|
|
352
361
|
::Regexp.new(::Regexp.escape(pattern.to_s))
|
|
353
362
|
end
|
|
354
363
|
|
|
355
|
-
def minimum_ast_bytes(node
|
|
364
|
+
def minimum_ast_bytes(node)
|
|
356
365
|
case node
|
|
357
366
|
when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then 0
|
|
358
367
|
when Regexp::AST::ByteRange then 1
|
|
359
368
|
when Regexp::AST::CodepointRange then utf8_length(node.lo)
|
|
360
369
|
when Regexp::AST::CharClass
|
|
361
370
|
ranges = node.ranges.flat_map do |range|
|
|
362
|
-
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase:
|
|
371
|
+
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
|
|
363
372
|
end
|
|
364
373
|
ranges = complement_codepoint_ranges(ranges) if node.negated
|
|
365
374
|
ranges.map { |lo, _hi| utf8_length(lo) }.min || 1
|
|
366
|
-
when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child
|
|
367
|
-
when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child
|
|
368
|
-
when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child
|
|
375
|
+
when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child) }
|
|
376
|
+
when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child) }.min || 0
|
|
377
|
+
when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child)
|
|
369
378
|
else minimum_unknown_bytes(node)
|
|
370
379
|
end
|
|
371
380
|
end
|
|
@@ -382,24 +391,24 @@ module Flexr
|
|
|
382
391
|
ast = Regexp::Parser.new(pattern.source, options: pattern.options,
|
|
383
392
|
encoding: pattern.encoding, unicode: true).parse
|
|
384
393
|
binary = !@lexer.utf8_input?
|
|
385
|
-
first_byte_ranges(ast,
|
|
394
|
+
first_byte_ranges(ast, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
|
|
386
395
|
rescue CompileError, RegexpError
|
|
387
396
|
true
|
|
388
397
|
end
|
|
389
398
|
|
|
390
|
-
def first_byte_ranges(node,
|
|
399
|
+
def first_byte_ranges(node, binary: false)
|
|
391
400
|
case node
|
|
392
401
|
when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then []
|
|
393
402
|
when Regexp::AST::ByteRange then [[node.lo, node.hi]]
|
|
394
403
|
when Regexp::AST::CodepointRange
|
|
395
|
-
ranges =
|
|
404
|
+
ranges = [[node.lo, node.hi]]
|
|
396
405
|
return ranges if binary
|
|
397
406
|
|
|
398
407
|
ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi).map(&:first).map(&:first) }
|
|
399
408
|
.map { |value| [value, value] }
|
|
400
409
|
when Regexp::AST::CharClass
|
|
401
410
|
ranges = node.ranges.flat_map do |range|
|
|
402
|
-
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase:
|
|
411
|
+
range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
|
|
403
412
|
end
|
|
404
413
|
ranges = complement_codepoint_ranges(ranges) if node.negated
|
|
405
414
|
return ranges if binary
|
|
@@ -409,14 +418,14 @@ module Flexr
|
|
|
409
418
|
when Regexp::AST::Seq
|
|
410
419
|
ranges = []
|
|
411
420
|
node.children.each do |child|
|
|
412
|
-
ranges.concat(first_byte_ranges(child,
|
|
421
|
+
ranges.concat(first_byte_ranges(child, binary: binary))
|
|
413
422
|
break unless nullable?(child)
|
|
414
423
|
end
|
|
415
424
|
ranges
|
|
416
425
|
when Regexp::AST::Alt
|
|
417
|
-
node.children.flat_map { |child| first_byte_ranges(child,
|
|
426
|
+
node.children.flat_map { |child| first_byte_ranges(child, binary: binary) }
|
|
418
427
|
when Regexp::AST::Star, Regexp::AST::Repeat
|
|
419
|
-
first_byte_ranges(node.child,
|
|
428
|
+
first_byte_ranges(node.child, binary: binary)
|
|
420
429
|
else
|
|
421
430
|
first_byte_fallback
|
|
422
431
|
end
|
|
@@ -540,8 +549,8 @@ module Flexr
|
|
|
540
549
|
@acceleration_scanner.pos = position
|
|
541
550
|
@acceleration_scanner.skip(regexp)
|
|
542
551
|
else
|
|
543
|
-
match = regexp.match(segment,
|
|
544
|
-
match&.begin(0)
|
|
552
|
+
match = regexp.match(segment.byteslice(position..).to_s, 0)
|
|
553
|
+
match&.begin(0)&.zero? ? match[0].bytesize : nil
|
|
545
554
|
end
|
|
546
555
|
rescue ArgumentError
|
|
547
556
|
nil
|
|
@@ -2,24 +2,47 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Runtime
|
|
5
|
-
Location =
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
@column_values =
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
5
|
+
Location = Data.define(:filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end) do
|
|
6
|
+
include Record
|
|
7
|
+
|
|
8
|
+
def initialize(filename: nil, byte_begin: nil, byte_end: nil, line_begin: nil, column_begin: nil,
|
|
9
|
+
line_end: nil, column_end: nil, column_values: nil, eager_columns: false)
|
|
10
|
+
@column_values = [column_begin || column_values&.first, column_end || column_values&.last].freeze
|
|
11
|
+
filename = filename.dup.freeze if filename.is_a?(String)
|
|
12
|
+
column_begin ||= @column_values&.first if eager_columns
|
|
13
|
+
column_end ||= @column_values&.last if eager_columns
|
|
14
|
+
super(filename: filename, byte_begin: byte_begin, byte_end: byte_end, line_begin: line_begin,
|
|
15
|
+
column_begin: column_begin, line_end: line_end, column_end: column_end)
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def column_begin
|
|
18
|
-
|
|
19
|
+
@column_values.first
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
def column_end
|
|
22
|
-
|
|
23
|
+
@column_values.last
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def to_h(&block)
|
|
27
|
+
values = super().merge(column_begin: column_begin, column_end: column_end)
|
|
28
|
+
block ? values.to_h(&block) : values
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def with(**changes)
|
|
32
|
+
columns = @column_values.dup
|
|
33
|
+
columns[0] = changes[:column_begin] if changes.key?(:column_begin)
|
|
34
|
+
columns[1] = changes[:column_end] if changes.key?(:column_end)
|
|
35
|
+
self.class.new(**deconstruct_keys(nil), **changes, column_values: columns)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def ==(other)
|
|
39
|
+
other.instance_of?(self.class) && to_h == other.to_h
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
alias_method :eql?, :==
|
|
43
|
+
|
|
44
|
+
def hash
|
|
45
|
+
[self.class, to_h].hash
|
|
23
46
|
end
|
|
24
47
|
end
|
|
25
48
|
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Runtime
|
|
5
|
+
module Record
|
|
6
|
+
def [](key)
|
|
7
|
+
member = key.is_a?(Integer) ? members.fetch(key) : key.to_sym
|
|
8
|
+
raise NameError, "unknown record member #{key.inspect}" unless members.include?(member)
|
|
9
|
+
|
|
10
|
+
public_send(member)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
data/lib/flexr/runtime/token.rb
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Runtime
|
|
5
|
-
Token =
|
|
5
|
+
Token = Data.define(:type, :value, :location) do
|
|
6
|
+
include Record
|
|
7
|
+
|
|
8
|
+
def initialize(type: nil, value: nil, location: nil)
|
|
9
|
+
super
|
|
10
|
+
end
|
|
11
|
+
end
|
|
6
12
|
end
|
|
7
13
|
end
|
data/lib/flexr/runtime.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
raise LoadError, "flexr 2.0 requires Ruby 3.4 or newer" if RUBY_VERSION < "3.4"
|
|
4
|
+
|
|
3
5
|
require "json"
|
|
4
6
|
require "monitor"
|
|
5
7
|
require_relative "version"
|
|
@@ -19,6 +21,7 @@ require_relative "unicode/data"
|
|
|
19
21
|
require_relative "unicode/property"
|
|
20
22
|
require_relative "unicode/reference_regexp"
|
|
21
23
|
require_relative "unicode/case_fold"
|
|
24
|
+
require_relative "regexp/matcher"
|
|
22
25
|
require_relative "automaton/types"
|
|
23
26
|
require_relative "automaton/byte_class_set"
|
|
24
27
|
require_relative "automaton/nfa"
|
|
@@ -28,6 +31,7 @@ require_relative "automaton/analysis"
|
|
|
28
31
|
require_relative "automaton/minimizer"
|
|
29
32
|
require_relative "automaton/backend_cost_model"
|
|
30
33
|
require_relative "automaton/accel"
|
|
34
|
+
require_relative "runtime/record"
|
|
31
35
|
require_relative "runtime/location"
|
|
32
36
|
require_relative "runtime/token"
|
|
33
37
|
require_relative "runtime/buffer"
|
|
@@ -38,11 +42,12 @@ require_relative "dsl"
|
|
|
38
42
|
require_relative "lexer"
|
|
39
43
|
require_relative "generated"
|
|
40
44
|
|
|
45
|
+
|
|
41
46
|
module Flexr
|
|
42
47
|
class << self
|
|
43
|
-
def compile_pattern(pattern, options: {})
|
|
48
|
+
def compile_pattern(pattern, options: {}, encoding: nil)
|
|
44
49
|
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
|
|
45
|
-
encoding
|
|
50
|
+
encoding ||= regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
|
|
46
51
|
rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
|
|
47
52
|
state = IR::State.new(name: :initial, inclusive: true, id: 0)
|
|
48
53
|
spec = IR::Spec.new(
|
|
@@ -14,7 +14,7 @@ module Flexr
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def rewrite(source, edits, insertion:, payload:)
|
|
17
|
-
result = source.dup
|
|
17
|
+
result = source.b.dup
|
|
18
18
|
normalized_edits = edits.map do |start_offset, end_offset, replacement|
|
|
19
19
|
start_offset = line_start_offset(source, start_offset) if replacement.empty? && start_offset != insertion
|
|
20
20
|
[start_offset, end_offset, replacement]
|
|
@@ -27,10 +27,10 @@ module Flexr
|
|
|
27
27
|
replacement.bytesize - replaced_size
|
|
28
28
|
end
|
|
29
29
|
normalized_edits.sort_by(&:first).reverse_each do |start_offset, end_offset, replacement|
|
|
30
|
-
result[start_offset...end_offset] = replacement
|
|
30
|
+
result[start_offset...end_offset] = replacement.b
|
|
31
31
|
end
|
|
32
|
-
result.insert(adjusted_insertion, payload)
|
|
33
|
-
result
|
|
32
|
+
result.insert(adjusted_insertion, payload.b)
|
|
33
|
+
result.force_encoding(source.encoding)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
def insert_after_preamble(source, payload)
|
|
@@ -46,26 +46,29 @@ module Flexr
|
|
|
46
46
|
line, next_offset = next_line(source, offset)
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
-
source.dup.insert(offset, payload)
|
|
49
|
+
source.b.dup.insert(offset, payload.b).force_encoding(source.encoding)
|
|
50
50
|
end
|
|
51
51
|
|
|
52
52
|
def indentation(source, offset)
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
binary = source.b
|
|
54
|
+
line_start = binary.rindex("\n", offset - 1)
|
|
55
|
+
binary.byteslice((line_start ? line_start + 1 : 0)...offset).to_s[/\A[ \t]*/].to_s
|
|
55
56
|
end
|
|
56
57
|
|
|
57
58
|
def line_start_offset(source, offset)
|
|
58
|
-
|
|
59
|
+
binary = source.b
|
|
60
|
+
line_start = binary.rindex("\n", offset - 1) unless offset.zero?
|
|
59
61
|
line_start = line_start ? line_start + 1 : 0
|
|
60
|
-
prefix =
|
|
62
|
+
prefix = binary.byteslice(line_start...offset)
|
|
61
63
|
prefix&.match?(/\A[ \t]*\z/) ? line_start : offset
|
|
62
64
|
end
|
|
63
65
|
|
|
64
66
|
def next_line(source, offset)
|
|
65
67
|
return [nil, offset] if offset >= source.bytesize
|
|
66
68
|
|
|
67
|
-
|
|
68
|
-
ending
|
|
69
|
+
binary = source.b
|
|
70
|
+
ending = binary.index("\n", offset)
|
|
71
|
+
ending ? [binary.byteslice(offset..ending), ending + 1] : [binary.byteslice(offset..), source.bytesize]
|
|
69
72
|
end
|
|
70
73
|
|
|
71
74
|
def magic_comment?(line)
|
|
@@ -4,7 +4,7 @@ module Flexr
|
|
|
4
4
|
module Source
|
|
5
5
|
RuleDefinition = Struct.new(
|
|
6
6
|
:index, :patterns, :trailing, :action_source, :action, :states, :bol_only, :end_anchor,
|
|
7
|
-
:pattern_conditions, :bind_action,
|
|
7
|
+
:pattern_conditions, :bind_action, :reads_locals,
|
|
8
8
|
:span, keyword_init: true
|
|
9
9
|
)
|
|
10
10
|
SpecSource = Struct.new(
|
|
@@ -35,18 +35,22 @@ module Flexr
|
|
|
35
35
|
require "prism"
|
|
36
36
|
Prism.parse(@source)
|
|
37
37
|
rescue LoadError => e
|
|
38
|
-
diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation",
|
|
38
|
+
diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation",
|
|
39
|
+
location: [0, 0], help: "install the generator dependency with gem install prism",
|
|
40
|
+
note: e.message)
|
|
39
41
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
40
42
|
end
|
|
41
43
|
unless prism.errors.empty?
|
|
42
44
|
error = prism.errors.first
|
|
43
|
-
diagnostic = Diagnostics.error("FLEXR-E010", error.message
|
|
45
|
+
diagnostic = Diagnostics.error("FLEXR-E010", error.message,
|
|
46
|
+
location: [error.location.start_offset, error.location.end_offset])
|
|
44
47
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
45
48
|
end
|
|
46
49
|
|
|
47
50
|
klass, class_name = find_lexer_class(prism.value)
|
|
48
51
|
raise CompileError, "no class inheriting from Flexr::Lexer found" unless klass
|
|
49
52
|
|
|
53
|
+
@states[:initial][:location] = [klass.location.start_offset, klass.location.end_offset]
|
|
50
54
|
@lexer_scope = class_name.to_s.delete_prefix("::").split("::").reject(&:empty?)
|
|
51
55
|
collect_constants(prism.value)
|
|
52
56
|
collect_body(klass.body, [:initial])
|
|
@@ -64,7 +68,7 @@ module Flexr
|
|
|
64
68
|
def find_lexer_class(node, namespace = [])
|
|
65
69
|
candidates = lexer_class_candidates(node, namespace)
|
|
66
70
|
dsl_candidates = candidates.select { |candidate, _name| lexer_has_dsl?(candidate.body) }
|
|
67
|
-
unsupported_static!("multiple lexer classes require an explicit selection") if dsl_candidates.length > 1
|
|
71
|
+
unsupported_static!("multiple lexer classes require an explicit selection", dsl_candidates[1].first) if dsl_candidates.length > 1
|
|
68
72
|
|
|
69
73
|
dsl_candidates.first || candidates.first
|
|
70
74
|
end
|
|
@@ -174,7 +178,7 @@ module Flexr
|
|
|
174
178
|
reject_unhandled_dsl!(node)
|
|
175
179
|
return
|
|
176
180
|
end
|
|
177
|
-
record_deletion(node) unless %i[rule state all_states].include?(name)
|
|
181
|
+
record_deletion(node) unless %i[rule state all_states on_eof].include?(name)
|
|
178
182
|
case name
|
|
179
183
|
when :rule
|
|
180
184
|
collect_rule(node, states)
|
|
@@ -187,7 +191,13 @@ module Flexr
|
|
|
187
191
|
action = if action_source
|
|
188
192
|
action_source.start_with?("do") ? "proc #{action_source}" : "proc#{action_source}"
|
|
189
193
|
end
|
|
190
|
-
|
|
194
|
+
if action
|
|
195
|
+
state = states.last.to_sym
|
|
196
|
+
@eof_rules[state] = "proc { instance_exec(&self.class.__flexr_generated_eof_actions.fetch(#{state.inspect})) }"
|
|
197
|
+
record_eof_edit(node, state, action)
|
|
198
|
+
else
|
|
199
|
+
record_deletion(node)
|
|
200
|
+
end
|
|
191
201
|
when :emits
|
|
192
202
|
values = positional(node).map { |item| static(item) }.compact.flatten
|
|
193
203
|
@config[:declared_tokens].concat(values.map(&:to_sym))
|
|
@@ -196,7 +206,7 @@ module Flexr
|
|
|
196
206
|
when :token_kind
|
|
197
207
|
@config[:token_kind] = Configuration.token_kind!(static(positional(node).first))
|
|
198
208
|
when :encoding
|
|
199
|
-
@config[:encoding] = static(positional(node).first)
|
|
209
|
+
@config[:encoding] = Configuration.encoding!(static(positional(node).first))
|
|
200
210
|
when :option
|
|
201
211
|
@config[:options] ||= {}
|
|
202
212
|
positional(node).each { |item| @config[:options][Configuration.option!(static(item))] = true }
|
|
@@ -204,6 +214,9 @@ module Flexr
|
|
|
204
214
|
@config[:options] ||= {}
|
|
205
215
|
@config[:options][:accel] = Configuration.accelerator!(static(positional(node).first))
|
|
206
216
|
end
|
|
217
|
+
rescue CompileError => e
|
|
218
|
+
e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic
|
|
219
|
+
raise
|
|
207
220
|
end
|
|
208
221
|
|
|
209
222
|
def collect_state(node, parent_states)
|
|
@@ -221,7 +234,7 @@ module Flexr
|
|
|
221
234
|
existing = @states[name]
|
|
222
235
|
raise ArgumentError, "state #{name.inspect} already declared with inclusive: #{existing[:inclusive]}" if
|
|
223
236
|
existing && existing[:inclusive] != inclusive
|
|
224
|
-
@states[name] ||= { inclusive: inclusive }
|
|
237
|
+
@states[name] ||= { inclusive: inclusive, location: [node.location.start_offset, node.location.end_offset] }
|
|
225
238
|
end
|
|
226
239
|
record_state_wrapper(node, block)
|
|
227
240
|
active_states = parent_states == [:initial] ? names : (parent_states + names).uniq
|
|
@@ -233,7 +246,7 @@ module Flexr
|
|
|
233
246
|
raise ArgumentError, "all_states requires a block" unless block
|
|
234
247
|
|
|
235
248
|
validate_keywords!(node, [])
|
|
236
|
-
unsupported_static!("all_states does not accept arguments") unless positional(node).empty?
|
|
249
|
+
unsupported_static!("all_states does not accept arguments", node) unless positional(node).empty?
|
|
237
250
|
record_state_wrapper(node, block)
|
|
238
251
|
collect_body(block.body, (parent_states + @states.keys).uniq)
|
|
239
252
|
end
|
|
@@ -249,7 +262,7 @@ module Flexr
|
|
|
249
262
|
options = keywords(node)
|
|
250
263
|
validate_keywords!(node, %i[skip emit followed_by])
|
|
251
264
|
action_source = node.block && source_slice(node.block)
|
|
252
|
-
if action_source
|
|
265
|
+
if action_source && reject_action?(node.block)
|
|
253
266
|
diagnostic = Diagnostics.error(
|
|
254
267
|
"FLEXR-E013", "reject is not supported by flexr",
|
|
255
268
|
help: "use a state transition and less(n) to express the fallback"
|
|
@@ -265,6 +278,12 @@ module Flexr
|
|
|
265
278
|
skip: skip, emit: emit, block: block_action, default: "proc { emit(nil, text) }"
|
|
266
279
|
)
|
|
267
280
|
bind_action = !block_action.nil? && action.equal?(block_action)
|
|
281
|
+
reads_locals = false
|
|
282
|
+
if node.block
|
|
283
|
+
each_node_including_self(node.block) do |child|
|
|
284
|
+
reads_locals = true if child.is_a?(Prism::LocalVariableReadNode)
|
|
285
|
+
end
|
|
286
|
+
end
|
|
268
287
|
rule_index = @rules.length
|
|
269
288
|
record_rule_edit(node, rule_index, bind_action ? block_action : nil)
|
|
270
289
|
@rules << RuleDefinition.new(
|
|
@@ -272,13 +291,14 @@ module Flexr
|
|
|
272
291
|
trailing: options[:followed_by] && static(options[:followed_by], scope: @lexer_scope),
|
|
273
292
|
action_source: action_source, action: action, states: states.dup,
|
|
274
293
|
bol_only: false, end_anchor: nil,
|
|
275
|
-
span: [node.location.start_offset, node.location.end_offset], bind_action: bind_action
|
|
294
|
+
span: [node.location.start_offset, node.location.end_offset], bind_action: bind_action, reads_locals: reads_locals
|
|
276
295
|
)
|
|
277
296
|
end
|
|
278
297
|
|
|
279
298
|
def static(node, scope: @lexer_scope)
|
|
280
299
|
StaticEval.new(@source, constants: @constants, scope: scope).call(node)
|
|
281
|
-
rescue StaticResolutionError
|
|
300
|
+
rescue StaticResolutionError => e
|
|
301
|
+
e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic && node
|
|
282
302
|
raise unless @allow_dynamic
|
|
283
303
|
|
|
284
304
|
nil
|
|
@@ -299,7 +319,7 @@ module Flexr
|
|
|
299
319
|
|
|
300
320
|
def validate_keywords!(node, allowed)
|
|
301
321
|
unknown = keywords(node).keys - allowed
|
|
302
|
-
unsupported_static!("unknown keyword #{unknown.first.inspect} for #{node.name}") unless unknown.empty?
|
|
322
|
+
unsupported_static!("unknown keyword #{unknown.first.inspect} for #{node.name}", node) unless unknown.empty?
|
|
303
323
|
end
|
|
304
324
|
|
|
305
325
|
def record_deletion(node)
|
|
@@ -315,6 +335,22 @@ module Flexr
|
|
|
315
335
|
@edits << [*span, replacement]
|
|
316
336
|
end
|
|
317
337
|
|
|
338
|
+
def record_eof_edit(node, state, action)
|
|
339
|
+
span = [node.location.start_offset, node.location.end_offset]
|
|
340
|
+
@spans << span
|
|
341
|
+
@edits << [*span, "__flexr_bind_generated_eof(#{state.inspect}, #{action})"]
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def reject_action?(block)
|
|
345
|
+
found = false
|
|
346
|
+
each_node_including_self(block.body) do |child|
|
|
347
|
+
next unless child.class.name.end_with?("CallNode") && child.name.to_sym == :reject
|
|
348
|
+
|
|
349
|
+
found ||= child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
|
|
350
|
+
end
|
|
351
|
+
found
|
|
352
|
+
end
|
|
353
|
+
|
|
318
354
|
def record_state_wrapper(node, block)
|
|
319
355
|
body = block.body
|
|
320
356
|
return record_deletion(node) unless body
|
|
@@ -334,12 +370,12 @@ module Flexr
|
|
|
334
370
|
next unless child.class.name.end_with?("CallNode")
|
|
335
371
|
if %i[send public_send].include?(child.name.to_sym)
|
|
336
372
|
dynamic_name = dynamically_dispatched_dsl(child)
|
|
337
|
-
unsupported_static!("dynamic dispatch of #{dynamic_name} is not statically supported") if dynamic_name
|
|
373
|
+
unsupported_static!("dynamic dispatch of #{dynamic_name} is not statically supported", child) if dynamic_name
|
|
338
374
|
end
|
|
339
375
|
next unless DSL_NAMES.include?(child.name.to_sym)
|
|
340
376
|
next unless child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
|
|
341
377
|
|
|
342
|
-
unsupported_static!("#{child.name} must be a direct receiverless statement in a lexer or state body")
|
|
378
|
+
unsupported_static!("#{child.name} must be a direct receiverless statement in a lexer or state body", child)
|
|
343
379
|
end
|
|
344
380
|
end
|
|
345
381
|
|
|
@@ -351,9 +387,10 @@ module Flexr
|
|
|
351
387
|
name if DSL_NAMES.include?(name)
|
|
352
388
|
end
|
|
353
389
|
|
|
354
|
-
def unsupported_static!(reason)
|
|
390
|
+
def unsupported_static!(reason, node)
|
|
355
391
|
diagnostic = Diagnostics.error(
|
|
356
392
|
"FLEXR-E017", "DSL structure is not statically supported",
|
|
393
|
+
location: [node.location.start_offset, node.location.end_offset],
|
|
357
394
|
help: "move the DSL call to the lexer or state body, or use --eval", note: reason
|
|
358
395
|
)
|
|
359
396
|
raise StaticResolutionError.new(diagnostic.message, diagnostic: diagnostic)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../flexr"
|
|
4
|
+
|
|
5
|
+
module Flexr
|
|
6
|
+
module Testing
|
|
7
|
+
module Snapshot
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def render(tokens)
|
|
11
|
+
"#{JSON.pretty_generate(encode(tokens))}\n"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def expected(path, actual, update:)
|
|
15
|
+
File.binwrite(path, actual) if update
|
|
16
|
+
File.binread(path).force_encoding(Encoding::UTF_8)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def encode(value)
|
|
20
|
+
case value
|
|
21
|
+
when NilClass, TrueClass, FalseClass, Integer
|
|
22
|
+
value
|
|
23
|
+
when Float
|
|
24
|
+
raise ArgumentError, "unsupported snapshot value: non-finite Float" unless value.finite?
|
|
25
|
+
|
|
26
|
+
value
|
|
27
|
+
when String
|
|
28
|
+
utf8 = value.dup.force_encoding(Encoding::UTF_8)
|
|
29
|
+
utf8.valid_encoding? ? utf8 : { "bytes" => value.bytes }
|
|
30
|
+
when Symbol
|
|
31
|
+
{ "symbol" => value.to_s }
|
|
32
|
+
when Array
|
|
33
|
+
value.map { |item| encode(item) }
|
|
34
|
+
when Hash
|
|
35
|
+
pairs = value.map { |key, item| [encode(key), encode(item)] }
|
|
36
|
+
{ "hash" => pairs.sort_by { |key, _item| JSON.generate(key) } }
|
|
37
|
+
when Runtime::Token
|
|
38
|
+
{ "token" => encode(value.to_h) }
|
|
39
|
+
when Runtime::Location
|
|
40
|
+
{ "location" => encode(value.members.to_h { |name| [name, value.public_send(name)] }) }
|
|
41
|
+
else
|
|
42
|
+
raise ArgumentError, "unsupported snapshot value: #{value.class}; convert it to built-in values first"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|