flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -24,7 +24,14 @@ module Flexr
24
24
  position = @lexer.byte_pos
25
25
  return nil unless @lexer.valid_utf8_at?(position)
26
26
  return scan_firstmatch(position) if @config.backend == :firstmatch
27
- return scan_fast(machine, position) if fast_path?(machine)
27
+ if fast_path?(machine)
28
+ if machine.whole_tokens
29
+ match = @lexer.__flexr_whole_token_match(machine.whole_tokens, position)
30
+ return match if match
31
+ end
32
+
33
+ return scan_fast(machine, position)
34
+ end
28
35
 
29
36
  buffer = @lexer.buffer
30
37
  state = machine.dfa.start
@@ -201,11 +208,7 @@ module Flexr
201
208
  end
202
209
 
203
210
  def reference_pattern?(pattern)
204
- return false unless pattern.is_a?(::Regexp)
205
- return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
206
-
207
- @config.options[:unicode] == true && @lexer.utf8_input? &&
208
- pattern.source.match?(/\\[dDwWsS]/)
211
+ Regexp::Matcher.reference?(pattern, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
209
212
  end
210
213
 
211
214
  def scan_firstmatch(position)
@@ -243,14 +246,33 @@ module Flexr
243
246
  def trailing_length(rule, buffer, position)
244
247
  return 0 unless rule.trailing
245
248
 
246
- regexp = rule.trailing
247
- match = streamed_match(
248
- regexp, buffer, position, reference: reference_pattern?(regexp),
249
- limit: :lookahead, rule: rule
250
- )
251
- return nil unless match&.begin(0)&.zero?
249
+ dfa = rule.trailing_dfa || trailing_dfa(rule.trailing)
250
+ state = dfa.start
251
+ cursor = position
252
+ longest = 0 if trailing_accept?(dfa, state, buffer, cursor)
253
+ while buffer.ensure_available?(cursor + 1)
254
+ state = dfa.transition(state, buffer.getbyte(cursor))
255
+ break unless state
252
256
 
253
- match[0].bytesize.tap { |size| @lexer.ensure_lookahead_size!(size, rule: rule) }
257
+ cursor += 1
258
+ @lexer.consume_step!
259
+ size = cursor - position
260
+ @lexer.ensure_lookahead_size!(size, rule: rule)
261
+ longest = size if trailing_accept?(dfa, state, buffer, cursor)
262
+ end
263
+ longest
264
+ end
265
+
266
+ def trailing_accept?(dfa, state, buffer, position)
267
+ acceptance = dfa.accepts[state].first
268
+ acceptance && (!acceptance.end_anchor || end_anchor_match?(buffer, position))
269
+ end
270
+
271
+ def trailing_dfa(pattern)
272
+ @trailing_dfas ||= {}
273
+ @trailing_dfas[pattern] ||= Flexr.compile_pattern(
274
+ pattern, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
275
+ )
254
276
  end
255
277
 
256
278
  def streamed_match(pattern, buffer, position, limit:, rule:, reference: false)
@@ -269,21 +291,13 @@ module Flexr
269
291
  end
270
292
  ensure_stream_limit!(limit, position, subject.bytesize, rule) unless match
271
293
  @lexer.consume_step!
272
- return match unless can_refill_match?(buffer, position, subject.bytesize, tail)
294
+ return match unless can_refill_match?(buffer)
273
295
  end
274
296
  end
275
297
 
276
298
  def match_stream_pattern(pattern, subject, reference:)
277
- if reference
278
- Unicode::ReferenceRegexp.match(
279
- pattern, subject, encoding: @config.encoding,
280
- options: pattern.options, unicode: @config.options[:unicode] == true
281
- )
282
- else
283
- pattern.match(subject, 0)
284
- end
285
- rescue ArgumentError, RegexpError
286
- nil
299
+ Regexp::Matcher.match(pattern, subject, encoding: @config.encoding,
300
+ unicode: @config.options[:unicode] == true, reference: reference)
287
301
  end
288
302
 
289
303
  def stream_subject(buffer, position)
@@ -327,13 +341,8 @@ module Flexr
327
341
  [:complete, length]
328
342
  end
329
343
 
330
- def can_refill_match?(buffer, position, subject_size, tail)
331
- target = if tail == :incomplete
332
- position + subject_size + 1
333
- else
334
- buffer.bytesize + 1
335
- end
336
- buffer.ensure_available?(target)
344
+ def can_refill_match?(buffer)
345
+ buffer.ensure_available?(buffer.bytesize + 1)
337
346
  end
338
347
 
339
348
  def minimum_match_bytes(pattern)
@@ -341,7 +350,7 @@ module Flexr
341
350
 
342
351
  ast = Regexp::Parser.new(pattern.source, options: pattern.options,
343
352
  encoding: pattern.encoding, unicode: true).parse
344
- minimum_ast_bytes(ast, ignorecase: pattern.options.anybits?(::Regexp::IGNORECASE))
353
+ minimum_ast_bytes(ast)
345
354
  rescue CompileError, RegexpError
346
355
  1
347
356
  end
@@ -352,20 +361,20 @@ module Flexr
352
361
  ::Regexp.new(::Regexp.escape(pattern.to_s))
353
362
  end
354
363
 
355
- def minimum_ast_bytes(node, ignorecase: false)
364
+ def minimum_ast_bytes(node)
356
365
  case node
357
366
  when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then 0
358
367
  when Regexp::AST::ByteRange then 1
359
368
  when Regexp::AST::CodepointRange then utf8_length(node.lo)
360
369
  when Regexp::AST::CharClass
361
370
  ranges = node.ranges.flat_map do |range|
362
- range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: ignorecase) : [range]
371
+ range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
363
372
  end
364
373
  ranges = complement_codepoint_ranges(ranges) if node.negated
365
374
  ranges.map { |lo, _hi| utf8_length(lo) }.min || 1
366
- when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }
367
- when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }.min || 0
368
- when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child, ignorecase: ignorecase)
375
+ when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child) }
376
+ when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child) }.min || 0
377
+ when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child)
369
378
  else minimum_unknown_bytes(node)
370
379
  end
371
380
  end
@@ -382,24 +391,24 @@ module Flexr
382
391
  ast = Regexp::Parser.new(pattern.source, options: pattern.options,
383
392
  encoding: pattern.encoding, unicode: true).parse
384
393
  binary = !@lexer.utf8_input?
385
- first_byte_ranges(ast, pattern.options, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
394
+ first_byte_ranges(ast, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
386
395
  rescue CompileError, RegexpError
387
396
  true
388
397
  end
389
398
 
390
- def first_byte_ranges(node, options, binary: false)
399
+ def first_byte_ranges(node, binary: false)
391
400
  case node
392
401
  when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then []
393
402
  when Regexp::AST::ByteRange then [[node.lo, node.hi]]
394
403
  when Regexp::AST::CodepointRange
395
- ranges = options.anybits?(::Regexp::IGNORECASE) ? Unicode::CaseFold.ranges(node.lo, node.hi) : [[node.lo, node.hi]]
404
+ ranges = [[node.lo, node.hi]]
396
405
  return ranges if binary
397
406
 
398
407
  ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi).map(&:first).map(&:first) }
399
408
  .map { |value| [value, value] }
400
409
  when Regexp::AST::CharClass
401
410
  ranges = node.ranges.flat_map do |range|
402
- range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: options.anybits?(::Regexp::IGNORECASE)) : [range]
411
+ range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
403
412
  end
404
413
  ranges = complement_codepoint_ranges(ranges) if node.negated
405
414
  return ranges if binary
@@ -409,14 +418,14 @@ module Flexr
409
418
  when Regexp::AST::Seq
410
419
  ranges = []
411
420
  node.children.each do |child|
412
- ranges.concat(first_byte_ranges(child, options, binary: binary))
421
+ ranges.concat(first_byte_ranges(child, binary: binary))
413
422
  break unless nullable?(child)
414
423
  end
415
424
  ranges
416
425
  when Regexp::AST::Alt
417
- node.children.flat_map { |child| first_byte_ranges(child, options, binary: binary) }
426
+ node.children.flat_map { |child| first_byte_ranges(child, binary: binary) }
418
427
  when Regexp::AST::Star, Regexp::AST::Repeat
419
- first_byte_ranges(node.child, options, binary: binary)
428
+ first_byte_ranges(node.child, binary: binary)
420
429
  else
421
430
  first_byte_fallback
422
431
  end
@@ -540,8 +549,8 @@ module Flexr
540
549
  @acceleration_scanner.pos = position
541
550
  @acceleration_scanner.skip(regexp)
542
551
  else
543
- match = regexp.match(segment, position)
544
- match&.begin(0) == position ? match.end(0) - position : nil
552
+ match = regexp.match(segment.byteslice(position..).to_s, 0)
553
+ match&.begin(0)&.zero? ? match[0].bytesize : nil
545
554
  end
546
555
  rescue ArgumentError
547
556
  nil
@@ -2,24 +2,47 @@
2
2
 
3
3
  module Flexr
4
4
  module Runtime
5
- Location = Struct.new(
6
- :filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end,
7
- keyword_init: true
8
- ) do
9
- def initialize(**values)
10
- @column_values = values.delete(:column_values)
11
- eager_columns = values.delete(:eager_columns)
12
- super(**values) # rubocop:disable Style/SuperArguments
13
- self.column_begin = @column_values.first if eager_columns && @column_values
14
- self.column_end = @column_values.last if eager_columns && @column_values
5
+ Location = Data.define(:filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end) do
6
+ include Record
7
+
8
+ def initialize(filename: nil, byte_begin: nil, byte_end: nil, line_begin: nil, column_begin: nil,
9
+ line_end: nil, column_end: nil, column_values: nil, eager_columns: false)
10
+ @column_values = [column_begin || column_values&.first, column_end || column_values&.last].freeze
11
+ filename = filename.dup.freeze if filename.is_a?(String)
12
+ column_begin ||= @column_values&.first if eager_columns
13
+ column_end ||= @column_values&.last if eager_columns
14
+ super(filename: filename, byte_begin: byte_begin, byte_end: byte_end, line_begin: line_begin,
15
+ column_begin: column_begin, line_end: line_end, column_end: column_end)
15
16
  end
16
17
 
17
18
  def column_begin
18
- self[:column_begin] ||= @column_values&.first
19
+ @column_values.first
19
20
  end
20
21
 
21
22
  def column_end
22
- self[:column_end] ||= @column_values&.last
23
+ @column_values.last
24
+ end
25
+
26
+ def to_h(&block)
27
+ values = super().merge(column_begin: column_begin, column_end: column_end)
28
+ block ? values.to_h(&block) : values
29
+ end
30
+
31
+ def with(**changes)
32
+ columns = @column_values.dup
33
+ columns[0] = changes[:column_begin] if changes.key?(:column_begin)
34
+ columns[1] = changes[:column_end] if changes.key?(:column_end)
35
+ self.class.new(**deconstruct_keys(nil), **changes, column_values: columns)
36
+ end
37
+
38
+ def ==(other)
39
+ other.instance_of?(self.class) && to_h == other.to_h
40
+ end
41
+
42
+ alias_method :eql?, :==
43
+
44
+ def hash
45
+ [self.class, to_h].hash
23
46
  end
24
47
  end
25
48
  end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Runtime
5
+ module Record
6
+ def [](key)
7
+ member = key.is_a?(Integer) ? members.fetch(key) : key.to_sym
8
+ raise NameError, "unknown record member #{key.inspect}" unless members.include?(member)
9
+
10
+ public_send(member)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -2,6 +2,12 @@
2
2
 
3
3
  module Flexr
4
4
  module Runtime
5
- Token = Struct.new(:type, :value, :location, keyword_init: true)
5
+ Token = Data.define(:type, :value, :location) do
6
+ include Record
7
+
8
+ def initialize(type: nil, value: nil, location: nil)
9
+ super
10
+ end
11
+ end
6
12
  end
7
13
  end
data/lib/flexr/runtime.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ raise LoadError, "flexr 2.0 requires Ruby 3.4 or newer" if RUBY_VERSION < "3.4"
4
+
3
5
  require "json"
4
6
  require "monitor"
5
7
  require_relative "version"
@@ -19,6 +21,7 @@ require_relative "unicode/data"
19
21
  require_relative "unicode/property"
20
22
  require_relative "unicode/reference_regexp"
21
23
  require_relative "unicode/case_fold"
24
+ require_relative "regexp/matcher"
22
25
  require_relative "automaton/types"
23
26
  require_relative "automaton/byte_class_set"
24
27
  require_relative "automaton/nfa"
@@ -28,6 +31,7 @@ require_relative "automaton/analysis"
28
31
  require_relative "automaton/minimizer"
29
32
  require_relative "automaton/backend_cost_model"
30
33
  require_relative "automaton/accel"
34
+ require_relative "runtime/record"
31
35
  require_relative "runtime/location"
32
36
  require_relative "runtime/token"
33
37
  require_relative "runtime/buffer"
@@ -38,11 +42,12 @@ require_relative "dsl"
38
42
  require_relative "lexer"
39
43
  require_relative "generated"
40
44
 
45
+
41
46
  module Flexr
42
47
  class << self
43
- def compile_pattern(pattern, options: {})
48
+ def compile_pattern(pattern, options: {}, encoding: nil)
44
49
  regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
45
- encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
50
+ encoding ||= regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
46
51
  rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
47
52
  state = IR::State.new(name: :initial, inclusive: true, id: 0)
48
53
  spec = IR::Spec.new(
@@ -14,7 +14,7 @@ module Flexr
14
14
  end
15
15
 
16
16
  def rewrite(source, edits, insertion:, payload:)
17
- result = source.dup
17
+ result = source.b.dup
18
18
  normalized_edits = edits.map do |start_offset, end_offset, replacement|
19
19
  start_offset = line_start_offset(source, start_offset) if replacement.empty? && start_offset != insertion
20
20
  [start_offset, end_offset, replacement]
@@ -27,10 +27,10 @@ module Flexr
27
27
  replacement.bytesize - replaced_size
28
28
  end
29
29
  normalized_edits.sort_by(&:first).reverse_each do |start_offset, end_offset, replacement|
30
- result[start_offset...end_offset] = replacement
30
+ result[start_offset...end_offset] = replacement.b
31
31
  end
32
- result.insert(adjusted_insertion, payload)
33
- result
32
+ result.insert(adjusted_insertion, payload.b)
33
+ result.force_encoding(source.encoding)
34
34
  end
35
35
 
36
36
  def insert_after_preamble(source, payload)
@@ -46,26 +46,29 @@ module Flexr
46
46
  line, next_offset = next_line(source, offset)
47
47
  end
48
48
 
49
- source.dup.insert(offset, payload)
49
+ source.b.dup.insert(offset, payload.b).force_encoding(source.encoding)
50
50
  end
51
51
 
52
52
  def indentation(source, offset)
53
- line_start = source.rindex("\n", offset - 1)
54
- source[(line_start ? line_start + 1 : 0)...offset].to_s[/\A[ \t]*/].to_s
53
+ binary = source.b
54
+ line_start = binary.rindex("\n", offset - 1)
55
+ binary.byteslice((line_start ? line_start + 1 : 0)...offset).to_s[/\A[ \t]*/].to_s
55
56
  end
56
57
 
57
58
  def line_start_offset(source, offset)
58
- line_start = source.rindex("\n", offset - 1) unless offset.zero?
59
+ binary = source.b
60
+ line_start = binary.rindex("\n", offset - 1) unless offset.zero?
59
61
  line_start = line_start ? line_start + 1 : 0
60
- prefix = source.byteslice(line_start...offset)
62
+ prefix = binary.byteslice(line_start...offset)
61
63
  prefix&.match?(/\A[ \t]*\z/) ? line_start : offset
62
64
  end
63
65
 
64
66
  def next_line(source, offset)
65
67
  return [nil, offset] if offset >= source.bytesize
66
68
 
67
- ending = source.index("\n", offset)
68
- ending ? [source.byteslice(offset..ending), ending + 1] : [source.byteslice(offset..), source.bytesize]
69
+ binary = source.b
70
+ ending = binary.index("\n", offset)
71
+ ending ? [binary.byteslice(offset..ending), ending + 1] : [binary.byteslice(offset..), source.bytesize]
69
72
  end
70
73
 
71
74
  def magic_comment?(line)
@@ -4,7 +4,7 @@ module Flexr
4
4
  module Source
5
5
  RuleDefinition = Struct.new(
6
6
  :index, :patterns, :trailing, :action_source, :action, :states, :bol_only, :end_anchor,
7
- :pattern_conditions, :bind_action,
7
+ :pattern_conditions, :bind_action, :reads_locals,
8
8
  :span, keyword_init: true
9
9
  )
10
10
  SpecSource = Struct.new(
@@ -35,18 +35,22 @@ module Flexr
35
35
  require "prism"
36
36
  Prism.parse(@source)
37
37
  rescue LoadError => e
38
- diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation", note: e.message)
38
+ diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation",
39
+ location: [0, 0], help: "install the generator dependency with gem install prism",
40
+ note: e.message)
39
41
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
40
42
  end
41
43
  unless prism.errors.empty?
42
44
  error = prism.errors.first
43
- diagnostic = Diagnostics.error("FLEXR-E010", error.message)
45
+ diagnostic = Diagnostics.error("FLEXR-E010", error.message,
46
+ location: [error.location.start_offset, error.location.end_offset])
44
47
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
45
48
  end
46
49
 
47
50
  klass, class_name = find_lexer_class(prism.value)
48
51
  raise CompileError, "no class inheriting from Flexr::Lexer found" unless klass
49
52
 
53
+ @states[:initial][:location] = [klass.location.start_offset, klass.location.end_offset]
50
54
  @lexer_scope = class_name.to_s.delete_prefix("::").split("::").reject(&:empty?)
51
55
  collect_constants(prism.value)
52
56
  collect_body(klass.body, [:initial])
@@ -64,7 +68,7 @@ module Flexr
64
68
  def find_lexer_class(node, namespace = [])
65
69
  candidates = lexer_class_candidates(node, namespace)
66
70
  dsl_candidates = candidates.select { |candidate, _name| lexer_has_dsl?(candidate.body) }
67
- unsupported_static!("multiple lexer classes require an explicit selection") if dsl_candidates.length > 1
71
+ unsupported_static!("multiple lexer classes require an explicit selection", dsl_candidates[1].first) if dsl_candidates.length > 1
68
72
 
69
73
  dsl_candidates.first || candidates.first
70
74
  end
@@ -174,7 +178,7 @@ module Flexr
174
178
  reject_unhandled_dsl!(node)
175
179
  return
176
180
  end
177
- record_deletion(node) unless %i[rule state all_states].include?(name)
181
+ record_deletion(node) unless %i[rule state all_states on_eof].include?(name)
178
182
  case name
179
183
  when :rule
180
184
  collect_rule(node, states)
@@ -187,7 +191,13 @@ module Flexr
187
191
  action = if action_source
188
192
  action_source.start_with?("do") ? "proc #{action_source}" : "proc#{action_source}"
189
193
  end
190
- @eof_rules[states.last.to_sym] = action if action
194
+ if action
195
+ state = states.last.to_sym
196
+ @eof_rules[state] = "proc { instance_exec(&self.class.__flexr_generated_eof_actions.fetch(#{state.inspect})) }"
197
+ record_eof_edit(node, state, action)
198
+ else
199
+ record_deletion(node)
200
+ end
191
201
  when :emits
192
202
  values = positional(node).map { |item| static(item) }.compact.flatten
193
203
  @config[:declared_tokens].concat(values.map(&:to_sym))
@@ -196,7 +206,7 @@ module Flexr
196
206
  when :token_kind
197
207
  @config[:token_kind] = Configuration.token_kind!(static(positional(node).first))
198
208
  when :encoding
199
- @config[:encoding] = static(positional(node).first)
209
+ @config[:encoding] = Configuration.encoding!(static(positional(node).first))
200
210
  when :option
201
211
  @config[:options] ||= {}
202
212
  positional(node).each { |item| @config[:options][Configuration.option!(static(item))] = true }
@@ -204,6 +214,9 @@ module Flexr
204
214
  @config[:options] ||= {}
205
215
  @config[:options][:accel] = Configuration.accelerator!(static(positional(node).first))
206
216
  end
217
+ rescue CompileError => e
218
+ e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic
219
+ raise
207
220
  end
208
221
 
209
222
  def collect_state(node, parent_states)
@@ -221,7 +234,7 @@ module Flexr
221
234
  existing = @states[name]
222
235
  raise ArgumentError, "state #{name.inspect} already declared with inclusive: #{existing[:inclusive]}" if
223
236
  existing && existing[:inclusive] != inclusive
224
- @states[name] ||= { inclusive: inclusive }
237
+ @states[name] ||= { inclusive: inclusive, location: [node.location.start_offset, node.location.end_offset] }
225
238
  end
226
239
  record_state_wrapper(node, block)
227
240
  active_states = parent_states == [:initial] ? names : (parent_states + names).uniq
@@ -233,7 +246,7 @@ module Flexr
233
246
  raise ArgumentError, "all_states requires a block" unless block
234
247
 
235
248
  validate_keywords!(node, [])
236
- unsupported_static!("all_states does not accept arguments") unless positional(node).empty?
249
+ unsupported_static!("all_states does not accept arguments", node) unless positional(node).empty?
237
250
  record_state_wrapper(node, block)
238
251
  collect_body(block.body, (parent_states + @states.keys).uniq)
239
252
  end
@@ -249,7 +262,7 @@ module Flexr
249
262
  options = keywords(node)
250
263
  validate_keywords!(node, %i[skip emit followed_by])
251
264
  action_source = node.block && source_slice(node.block)
252
- if action_source&.match?(/\breject\b/)
265
+ if action_source && reject_action?(node.block)
253
266
  diagnostic = Diagnostics.error(
254
267
  "FLEXR-E013", "reject is not supported by flexr",
255
268
  help: "use a state transition and less(n) to express the fallback"
@@ -265,6 +278,12 @@ module Flexr
265
278
  skip: skip, emit: emit, block: block_action, default: "proc { emit(nil, text) }"
266
279
  )
267
280
  bind_action = !block_action.nil? && action.equal?(block_action)
281
+ reads_locals = false
282
+ if node.block
283
+ each_node_including_self(node.block) do |child|
284
+ reads_locals = true if child.is_a?(Prism::LocalVariableReadNode)
285
+ end
286
+ end
268
287
  rule_index = @rules.length
269
288
  record_rule_edit(node, rule_index, bind_action ? block_action : nil)
270
289
  @rules << RuleDefinition.new(
@@ -272,13 +291,14 @@ module Flexr
272
291
  trailing: options[:followed_by] && static(options[:followed_by], scope: @lexer_scope),
273
292
  action_source: action_source, action: action, states: states.dup,
274
293
  bol_only: false, end_anchor: nil,
275
- span: [node.location.start_offset, node.location.end_offset], bind_action: bind_action
294
+ span: [node.location.start_offset, node.location.end_offset], bind_action: bind_action, reads_locals: reads_locals
276
295
  )
277
296
  end
278
297
 
279
298
  def static(node, scope: @lexer_scope)
280
299
  StaticEval.new(@source, constants: @constants, scope: scope).call(node)
281
- rescue StaticResolutionError
300
+ rescue StaticResolutionError => e
301
+ e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic && node
282
302
  raise unless @allow_dynamic
283
303
 
284
304
  nil
@@ -299,7 +319,7 @@ module Flexr
299
319
 
300
320
  def validate_keywords!(node, allowed)
301
321
  unknown = keywords(node).keys - allowed
302
- unsupported_static!("unknown keyword #{unknown.first.inspect} for #{node.name}") unless unknown.empty?
322
+ unsupported_static!("unknown keyword #{unknown.first.inspect} for #{node.name}", node) unless unknown.empty?
303
323
  end
304
324
 
305
325
  def record_deletion(node)
@@ -315,6 +335,22 @@ module Flexr
315
335
  @edits << [*span, replacement]
316
336
  end
317
337
 
338
+ def record_eof_edit(node, state, action)
339
+ span = [node.location.start_offset, node.location.end_offset]
340
+ @spans << span
341
+ @edits << [*span, "__flexr_bind_generated_eof(#{state.inspect}, #{action})"]
342
+ end
343
+
344
+ def reject_action?(block)
345
+ found = false
346
+ each_node_including_self(block.body) do |child|
347
+ next unless child.class.name.end_with?("CallNode") && child.name.to_sym == :reject
348
+
349
+ found ||= child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
350
+ end
351
+ found
352
+ end
353
+
318
354
  def record_state_wrapper(node, block)
319
355
  body = block.body
320
356
  return record_deletion(node) unless body
@@ -334,12 +370,12 @@ module Flexr
334
370
  next unless child.class.name.end_with?("CallNode")
335
371
  if %i[send public_send].include?(child.name.to_sym)
336
372
  dynamic_name = dynamically_dispatched_dsl(child)
337
- unsupported_static!("dynamic dispatch of #{dynamic_name} is not statically supported") if dynamic_name
373
+ unsupported_static!("dynamic dispatch of #{dynamic_name} is not statically supported", child) if dynamic_name
338
374
  end
339
375
  next unless DSL_NAMES.include?(child.name.to_sym)
340
376
  next unless child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
341
377
 
342
- unsupported_static!("#{child.name} must be a direct receiverless statement in a lexer or state body")
378
+ unsupported_static!("#{child.name} must be a direct receiverless statement in a lexer or state body", child)
343
379
  end
344
380
  end
345
381
 
@@ -351,9 +387,10 @@ module Flexr
351
387
  name if DSL_NAMES.include?(name)
352
388
  end
353
389
 
354
- def unsupported_static!(reason)
390
+ def unsupported_static!(reason, node)
355
391
  diagnostic = Diagnostics.error(
356
392
  "FLEXR-E017", "DSL structure is not statically supported",
393
+ location: [node.location.start_offset, node.location.end_offset],
357
394
  help: "move the DSL call to the lexer or state body, or use --eval", note: reason
358
395
  )
359
396
  raise StaticResolutionError.new(diagnostic.message, diagnostic: diagnostic)
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../flexr"
4
+
5
+ module Flexr
6
+ module Testing
7
+ module Snapshot
8
+ module_function
9
+
10
+ def render(tokens)
11
+ "#{JSON.pretty_generate(encode(tokens))}\n"
12
+ end
13
+
14
+ def expected(path, actual, update:)
15
+ File.binwrite(path, actual) if update
16
+ File.binread(path).force_encoding(Encoding::UTF_8)
17
+ end
18
+
19
+ def encode(value)
20
+ case value
21
+ when NilClass, TrueClass, FalseClass, Integer
22
+ value
23
+ when Float
24
+ raise ArgumentError, "unsupported snapshot value: non-finite Float" unless value.finite?
25
+
26
+ value
27
+ when String
28
+ utf8 = value.dup.force_encoding(Encoding::UTF_8)
29
+ utf8.valid_encoding? ? utf8 : { "bytes" => value.bytes }
30
+ when Symbol
31
+ { "symbol" => value.to_s }
32
+ when Array
33
+ value.map { |item| encode(item) }
34
+ when Hash
35
+ pairs = value.map { |key, item| [encode(key), encode(item)] }
36
+ { "hash" => pairs.sort_by { |key, _item| JSON.generate(key) } }
37
+ when Runtime::Token
38
+ { "token" => encode(value.to_h) }
39
+ when Runtime::Location
40
+ { "location" => encode(value.members.to_h { |name| [name, value.public_send(name)] }) }
41
+ else
42
+ raise ArgumentError, "unsupported snapshot value: #{value.class}; convert it to built-in values first"
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end