flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -13,25 +13,41 @@ module Flexr
13
13
  class Interpreter
14
14
  def initialize(lexer)
15
15
  @lexer = lexer
16
+ klass = lexer.class
17
+ @compiled = klass.__flexr_compiled
18
+ @rules = klass.__flexr_rules
19
+ @config = klass.__flexr_config
16
20
  end
17
21
 
18
22
  def scan
19
- machine = @lexer.class.__flexr_compiled.machines.fetch(@lexer.state)
23
+ machine = @compiled.machines.fetch(@lexer.state)
20
24
  position = @lexer.byte_pos
21
25
  return nil unless @lexer.valid_utf8_at?(position)
22
- return scan_firstmatch(position) if @lexer.class.__flexr_config.backend == :firstmatch
23
- return scan_fast(machine, position) if fast_path?
26
+ return scan_firstmatch(position) if @config.backend == :firstmatch
27
+ if fast_path?(machine)
28
+ if machine.whole_tokens
29
+ match = @lexer.__flexr_whole_token_match(machine.whole_tokens, position)
30
+ return match if match
31
+ end
32
+
33
+ return scan_fast(machine, position)
34
+ end
24
35
 
25
36
  buffer = @lexer.buffer
26
37
  state = machine.dfa.start
27
38
  best = reference_match(position, buffer)
28
39
  cursor = position
29
- best = consider_acceptances(machine, state, cursor, position, buffer, best) if @lexer.class.__flexr_config.options[:allow_empty_match]
40
+ best = consider_acceptances(machine, state, cursor, position, buffer, best) if @config.options[:allow_empty_match]
30
41
  acceleration_regions = acceleration_enabled? ? acceleration_regions_for(machine) : nil
31
42
 
32
43
  while buffer.ensure_available?(cursor + 1)
33
- if acceleration_regions && (region = acceleration_regions[state])
34
- accelerated_end = accelerate(region, buffer, cursor)
44
+ if acceleration_regions && (region = acceleration_regions[state]) &&
45
+ !@disabled_auto_acceleration&.key?(region)
46
+ acceptance = machine.dfa.accepts[state].first
47
+ token_rule = acceptance&.rule_index
48
+ accelerated_end = accelerate(
49
+ region, buffer, cursor, token_rule: token_rule, token_start: position
50
+ )
35
51
  if accelerated_end && accelerated_end > cursor
36
52
  cursor = accelerated_end
37
53
  best = consider_acceptances(machine, state, cursor, position, buffer, best)
@@ -40,10 +56,13 @@ module Flexr
40
56
  end
41
57
 
42
58
  byte = buffer.getbyte(cursor)
59
+ @lexer.consume_step!
43
60
  state = transition(machine.dfa, state, byte)
44
61
  break unless state
45
62
 
46
63
  cursor += 1
64
+ acceptance = machine.dfa.accepts[state].first
65
+ ensure_token_size!(cursor, position, rule: acceptance&.rule_index)
47
66
  best = consider_acceptances(machine, state, cursor, position, buffer, best)
48
67
  end
49
68
  best
@@ -51,22 +70,57 @@ module Flexr
51
70
 
52
71
  def scan_fast(machine, position)
53
72
  buffer = @lexer.buffer
54
- source = buffer.source
55
73
  dfa = machine.dfa
56
74
  accepts = dfa.accepts
57
- rules = @lexer.class.__flexr_rules
75
+ rules = @rules
58
76
  transitions = dfa.transitions
59
77
  ec = dfa.ec
60
78
  direct = dfa.direct
79
+ direct_nxt = direct&.fetch(:nxt)
80
+ direct_classes = direct&.fetch(:classes)
81
+ source = buffer.stable_source
82
+ guarded_steps = @lexer.scan_steps_guarded?
83
+ text_start = @lexer.more_text_start || position
84
+ max_token_size = @lexer.max_token_size
85
+ token_limit_required = @lexer.token_limit_required?
61
86
  state = dfa.start
62
87
  cursor = position
63
88
  best = nil
64
89
 
65
- while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
66
- byte = source.getbyte(cursor)
67
- state = if direct
90
+ if source && !guarded_steps && !token_limit_required
91
+ while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)
92
+ byte = source.getbyte(cursor)
93
+ state = if direct_nxt
94
+ class_id = ec[byte]
95
+ value = direct_nxt[(state * direct_classes) + class_id]
96
+ value >= 0 ? value : nil
97
+ else
98
+ transitions[state][ec[byte]]
99
+ end
100
+ break unless state
101
+
102
+ cursor += 1
103
+ acceptance = accepts[state].first
104
+ next unless acceptance
105
+
106
+ rule = rules.fetch(acceptance.rule_index)
107
+ next if best && cursor == best.total_end_pos && rule.index > best.rule.index
108
+
109
+ best ||= (@match ||= Match.new)
110
+ best.rule = rule
111
+ best.start_pos = position
112
+ best.end_pos = cursor
113
+ best.total_end_pos = cursor
114
+ end
115
+ return best
116
+ end
117
+
118
+ while cursor < buffer.bytesize || buffer.ensure_available?(cursor + 1)
119
+ byte = buffer.getbyte(cursor)
120
+ @lexer.consume_step! if guarded_steps
121
+ state = if direct_nxt
68
122
  class_id = ec[byte]
69
- value = direct[:nxt][(state * direct[:classes]) + class_id]
123
+ value = direct_nxt[(state * direct_classes) + class_id]
70
124
  value >= 0 ? value : nil
71
125
  else
72
126
  transitions[state][ec[byte]]
@@ -75,6 +129,10 @@ module Flexr
75
129
 
76
130
  cursor += 1
77
131
  acceptance = accepts[state].first
132
+ if token_limit_required
133
+ size = cursor - text_start
134
+ @lexer.defer_token_size_check!(size, rule: acceptance&.rule_index) if size > max_token_size
135
+ end
78
136
  next unless acceptance
79
137
 
80
138
  rule = rules.fetch(acceptance.rule_index)
@@ -94,7 +152,7 @@ module Flexr
94
152
  def reference_match(position, buffer)
95
153
  return unless reference_rules?
96
154
 
97
- candidates = @lexer.class.__flexr_rules.filter_map do |rule|
155
+ candidates = @rules.filter_map do |rule|
98
156
  next unless rule_active?(rule)
99
157
  next unless rule.patterns.any? { |pattern| reference_pattern?(pattern) }
100
158
 
@@ -102,7 +160,10 @@ module Flexr
102
160
  condition = rule.pattern_conditions.fetch(pattern_index)
103
161
  next if condition.bol_only && !@lexer.beginning_of_line?
104
162
 
105
- match = streamed_match(pattern, buffer, position, reference: reference_pattern?(pattern))
163
+ match = streamed_match(
164
+ pattern, buffer, position, reference: reference_pattern?(pattern),
165
+ limit: :token, rule: rule
166
+ )
106
167
  next unless match&.begin(0)&.zero?
107
168
 
108
169
  end_position = position + match[0].bytesize
@@ -120,7 +181,7 @@ module Flexr
120
181
  trailing = trailing_length(rule, buffer, end_position)
121
182
  next if rule.trailing && trailing.nil?
122
183
 
123
- ensure_token_size!(end_position, position)
184
+ ensure_token_size!(end_position, position, rule: rule)
124
185
  Match.new(rule: rule, start_pos: position, end_pos: end_position,
125
186
  total_end_pos: end_position + (trailing || 0))
126
187
  end
@@ -128,33 +189,31 @@ module Flexr
128
189
  end
129
190
 
130
191
  def reference_rules?
192
+ return false unless @config.backend == :firstmatch
131
193
  return @reference_rules unless @reference_rules.nil?
132
194
 
133
- @reference_rules = @lexer.class.__flexr_rules.any? do |rule|
195
+ @reference_rules = @rules.any? do |rule|
134
196
  rule.patterns.any? { |pattern| reference_pattern?(pattern) }
135
197
  end
136
198
  end
137
199
 
138
- def fast_path?
139
- return @fast_path unless @fast_path.nil?
200
+ def fast_path?(machine)
201
+ @fast_paths ||= {}
202
+ return @fast_paths[machine.dfa] if @fast_paths.key?(machine.dfa)
140
203
 
141
- rules = @lexer.class.__flexr_rules
142
- @fast_path = !@lexer.class.__flexr_config.options[:allow_empty_match] &&
204
+ rules = machine.dfa.rule_ids.map { |rule_index| @rules.fetch(rule_index) }
205
+ @fast_paths[machine.dfa] = !@config.options[:allow_empty_match] &&
143
206
  !reference_rules? && rules.none?(&:trailing) &&
144
207
  rules.none? { |rule| rule.pattern_conditions.any? { |condition| condition&.bol_only || condition&.end_anchor } }
145
208
  end
146
209
 
147
210
  def reference_pattern?(pattern)
148
- return false unless pattern.is_a?(::Regexp)
149
- return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
150
-
151
- @lexer.class.__flexr_config.options[:unicode] == true && @lexer.utf8_input? &&
152
- pattern.source.match?(/\\[dDwWsS]/)
211
+ Regexp::Matcher.reference?(pattern, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
153
212
  end
154
213
 
155
214
  def scan_firstmatch(position)
156
215
  buffer = @lexer.buffer
157
- @lexer.class.__flexr_rules.sort_by(&:index).each do |rule|
216
+ @rules.sort_by(&:index).each do |rule|
158
217
  next unless rule_active?(rule)
159
218
 
160
219
  matches = rule.patterns.each_with_index.filter_map do |pattern, pattern_index|
@@ -162,10 +221,11 @@ module Flexr
162
221
  next if condition.bol_only && !@lexer.beginning_of_line?
163
222
 
164
223
  regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
165
- streamed_match(regexp, buffer, position, reference: reference_pattern?(regexp))
224
+ streamed_match(
225
+ regexp, buffer, position, reference: reference_pattern?(regexp),
226
+ limit: :token, rule: rule
227
+ )
166
228
  &.then { |match| [match, condition] }
167
- rescue ArgumentError, RegexpError
168
- nil
169
229
  end
170
230
  match, condition = matches.select { |candidate| candidate[0].begin(0).zero? }
171
231
  .max_by { |candidate| candidate[0][0].bytesize }
@@ -177,7 +237,7 @@ module Flexr
177
237
  trailing = trailing_length(rule, buffer, end_position)
178
238
  next if rule.trailing && trailing.nil?
179
239
 
180
- ensure_token_size!(end_position, position)
240
+ ensure_token_size!(end_position, position, rule: rule)
181
241
  return reusable_match(rule, position, end_position, end_position + (trailing || 0))
182
242
  end
183
243
  nil
@@ -186,28 +246,42 @@ module Flexr
186
246
  def trailing_length(rule, buffer, position)
187
247
  return 0 unless rule.trailing
188
248
 
189
- regexp = rule.trailing
190
- match = streamed_match(regexp, buffer, position, reference: reference_pattern?(regexp))
191
- return nil unless match&.begin(0)&.zero?
249
+ dfa = rule.trailing_dfa || trailing_dfa(rule.trailing)
250
+ state = dfa.start
251
+ cursor = position
252
+ longest = 0 if trailing_accept?(dfa, state, buffer, cursor)
253
+ while buffer.ensure_available?(cursor + 1)
254
+ state = dfa.transition(state, buffer.getbyte(cursor))
255
+ break unless state
256
+
257
+ cursor += 1
258
+ @lexer.consume_step!
259
+ size = cursor - position
260
+ @lexer.ensure_lookahead_size!(size, rule: rule)
261
+ longest = size if trailing_accept?(dfa, state, buffer, cursor)
262
+ end
263
+ longest
264
+ end
192
265
 
193
- match[0].bytesize
194
- rescue ArgumentError
195
- nil
266
+ def trailing_accept?(dfa, state, buffer, position)
267
+ acceptance = dfa.accepts[state].first
268
+ acceptance && (!acceptance.end_anchor || end_anchor_match?(buffer, position))
269
+ end
270
+
271
+ def trailing_dfa(pattern)
272
+ @trailing_dfas ||= {}
273
+ @trailing_dfas[pattern] ||= Flexr.compile_pattern(
274
+ pattern, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
275
+ )
196
276
  end
197
277
 
198
- def streamed_match(pattern, buffer, position, reference: false)
278
+ def streamed_match(pattern, buffer, position, limit:, rule:, reference: false)
199
279
  pattern = regexp_pattern(pattern)
200
280
  minimum = minimum_match_bytes(pattern)
201
281
  loop do
202
282
  subject, tail = stream_subject(buffer, position)
203
- match = if reference
204
- Unicode::ReferenceRegexp.match(
205
- pattern, subject, encoding: @lexer.class.__flexr_config.encoding,
206
- options: pattern.options, unicode: @lexer.class.__flexr_config.options[:unicode] == true
207
- )
208
- else
209
- pattern.match(subject, 0)
210
- end
283
+ match = match_stream_pattern(pattern, subject, reference: reference)
284
+ ensure_stream_limit!(limit, position, match[0].bytesize, rule) if match
211
285
  return match if match && match[0].bytesize < subject.bytesize
212
286
  return match if match && %i[eof invalid].include?(tail)
213
287
  if !match && subject.bytesize >= minimum && tail != :incomplete
@@ -215,12 +289,17 @@ module Flexr
215
289
  return nil unless first_byte && possible_first_byte?(pattern, first_byte)
216
290
  return nil if %i[eof invalid].include?(tail)
217
291
  end
218
- return match unless can_refill_match?(buffer, position, subject.bytesize, tail)
219
- rescue ArgumentError, RegexpError
220
- return nil
292
+ ensure_stream_limit!(limit, position, subject.bytesize, rule) unless match
293
+ @lexer.consume_step!
294
+ return match unless can_refill_match?(buffer)
221
295
  end
222
296
  end
223
297
 
298
+ def match_stream_pattern(pattern, subject, reference:)
299
+ Regexp::Matcher.match(pattern, subject, encoding: @config.encoding,
300
+ unicode: @config.options[:unicode] == true, reference: reference)
301
+ end
302
+
224
303
  def stream_subject(buffer, position)
225
304
  return [buffer.byteslice(position...buffer.bytesize).to_s.b, buffer.eof_loaded? ? :eof : :end] unless @lexer.utf8_input?
226
305
 
@@ -245,7 +324,7 @@ module Flexr
245
324
  def utf8_status(buffer, position)
246
325
  return [:eof, 0] if position >= buffer.bytesize
247
326
 
248
- first = buffer.source.getbyte(position)
327
+ first = buffer.getbyte(position)
249
328
  return [:complete, 1] if first <= 0x7f
250
329
  return [:invalid, 1] unless first.between?(0xc2, 0xf4)
251
330
 
@@ -262,13 +341,8 @@ module Flexr
262
341
  [:complete, length]
263
342
  end
264
343
 
265
- def can_refill_match?(buffer, position, subject_size, tail)
266
- target = if tail == :incomplete
267
- position + subject_size + 1
268
- else
269
- buffer.bytesize + 1
270
- end
271
- buffer.ensure_available?(target)
344
+ def can_refill_match?(buffer)
345
+ buffer.ensure_available?(buffer.bytesize + 1)
272
346
  end
273
347
 
274
348
  def minimum_match_bytes(pattern)
@@ -276,7 +350,7 @@ module Flexr
276
350
 
277
351
  ast = Regexp::Parser.new(pattern.source, options: pattern.options,
278
352
  encoding: pattern.encoding, unicode: true).parse
279
- minimum_ast_bytes(ast, ignorecase: pattern.options.anybits?(::Regexp::IGNORECASE))
353
+ minimum_ast_bytes(ast)
280
354
  rescue CompileError, RegexpError
281
355
  1
282
356
  end
@@ -287,19 +361,20 @@ module Flexr
287
361
  ::Regexp.new(::Regexp.escape(pattern.to_s))
288
362
  end
289
363
 
290
- def minimum_ast_bytes(node, ignorecase: false)
364
+ def minimum_ast_bytes(node)
291
365
  case node
292
- when Regexp::AST::Empty, Regexp::AST::Anchor then 0
366
+ when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then 0
293
367
  when Regexp::AST::ByteRange then 1
294
368
  when Regexp::AST::CodepointRange then utf8_length(node.lo)
295
369
  when Regexp::AST::CharClass
296
370
  ranges = node.ranges.flat_map do |range|
297
- range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: ignorecase) : [range]
371
+ range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
298
372
  end
299
373
  ranges = complement_codepoint_ranges(ranges) if node.negated
300
374
  ranges.map { |lo, _hi| utf8_length(lo) }.min || 1
301
- when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }
302
- when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child, ignorecase: ignorecase) }.min || 0
375
+ when Regexp::AST::Seq then node.children.sum { |child| minimum_ast_bytes(child) }
376
+ when Regexp::AST::Alt then node.children.map { |child| minimum_ast_bytes(child) }.min || 0
377
+ when Regexp::AST::Repeat then node.minimum * minimum_ast_bytes(node.child)
303
378
  else minimum_unknown_bytes(node)
304
379
  end
305
380
  end
@@ -316,24 +391,24 @@ module Flexr
316
391
  ast = Regexp::Parser.new(pattern.source, options: pattern.options,
317
392
  encoding: pattern.encoding, unicode: true).parse
318
393
  binary = !@lexer.utf8_input?
319
- first_byte_ranges(ast, pattern.options, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
394
+ first_byte_ranges(ast, binary: binary).any? { |lo, hi| byte.between?(lo, hi) }
320
395
  rescue CompileError, RegexpError
321
396
  true
322
397
  end
323
398
 
324
- def first_byte_ranges(node, options, binary: false)
399
+ def first_byte_ranges(node, binary: false)
325
400
  case node
326
- when Regexp::AST::Empty, Regexp::AST::Anchor then []
401
+ when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Fail then []
327
402
  when Regexp::AST::ByteRange then [[node.lo, node.hi]]
328
403
  when Regexp::AST::CodepointRange
329
- ranges = options.anybits?(::Regexp::IGNORECASE) ? Unicode::CaseFold.ranges(node.lo, node.hi) : [[node.lo, node.hi]]
404
+ ranges = [[node.lo, node.hi]]
330
405
  return ranges if binary
331
406
 
332
407
  ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi).map(&:first).map(&:first) }
333
408
  .map { |value| [value, value] }
334
409
  when Regexp::AST::CharClass
335
410
  ranges = node.ranges.flat_map do |range|
336
- range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: options.anybits?(::Regexp::IGNORECASE)) : [range]
411
+ range.first == Regexp::AST::Property ? property_ranges(range, ignorecase: range[3] == true) : [range]
337
412
  end
338
413
  ranges = complement_codepoint_ranges(ranges) if node.negated
339
414
  return ranges if binary
@@ -343,14 +418,14 @@ module Flexr
343
418
  when Regexp::AST::Seq
344
419
  ranges = []
345
420
  node.children.each do |child|
346
- ranges.concat(first_byte_ranges(child, options, binary: binary))
421
+ ranges.concat(first_byte_ranges(child, binary: binary))
347
422
  break unless nullable?(child)
348
423
  end
349
424
  ranges
350
425
  when Regexp::AST::Alt
351
- node.children.flat_map { |child| first_byte_ranges(child, options, binary: binary) }
352
- when Regexp::AST::Star
353
- first_byte_ranges(node.child, options, binary: binary)
426
+ node.children.flat_map { |child| first_byte_ranges(child, binary: binary) }
427
+ when Regexp::AST::Star, Regexp::AST::Repeat
428
+ first_byte_ranges(node.child, binary: binary)
354
429
  else
355
430
  first_byte_fallback
356
431
  end
@@ -363,6 +438,7 @@ module Flexr
363
438
  def nullable?(node)
364
439
  case node
365
440
  when Regexp::AST::Empty, Regexp::AST::Anchor, Regexp::AST::Star then true
441
+ when Regexp::AST::Repeat then node.minimum.zero? || nullable?(node.child)
366
442
  when Regexp::AST::Seq then node.children.all? { |child| nullable?(child) }
367
443
  when Regexp::AST::Alt then node.children.any? { |child| nullable?(child) }
368
444
  else false
@@ -400,7 +476,7 @@ module Flexr
400
476
 
401
477
  def consider_acceptances(machine, state, cursor, position, buffer, best)
402
478
  machine.dfa.accepts[state].each do |acceptance|
403
- candidate = @lexer.class.__flexr_rules.fetch(acceptance.rule_index)
479
+ candidate = @rules.fetch(acceptance.rule_index)
404
480
  next if acceptance.bol_only && !@lexer.beginning_of_line?
405
481
  next unless @lexer.utf8_boundary?(cursor)
406
482
  next unless !acceptance.end_anchor || end_anchor_match?(buffer, cursor)
@@ -412,7 +488,7 @@ module Flexr
412
488
  next if best && total_end < best.total_end_pos
413
489
  next if best && total_end == best.total_end_pos && acceptance.rule_index > best.rule.index
414
490
 
415
- ensure_token_size!(cursor, position)
491
+ ensure_token_size!(cursor, position, rule: candidate)
416
492
  best = @match ||= Match.new
417
493
  best.rule = candidate
418
494
  best.start_pos = position
@@ -423,7 +499,8 @@ module Flexr
423
499
  end
424
500
 
425
501
  def acceleration_enabled?
426
- @lexer.class.__flexr_config.options.fetch(:accel, :auto) != :none
502
+ mode = @config.options.fetch(:accel, :auto)
503
+ mode != :none && !(mode == :auto && @auto_acceleration_off)
427
504
  end
428
505
 
429
506
  def reusable_match(rule, start_pos, end_pos, total_end_pos)
@@ -435,32 +512,63 @@ module Flexr
435
512
  @match
436
513
  end
437
514
 
438
- def accelerate(region, buffer, position)
439
- mode = @lexer.class.__flexr_config.options.fetch(:accel, :auto)
440
- binary = buffer.source.b
441
- match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)
442
- scanner = StringScanner.new(binary)
443
- scanner.pos = position
444
- length = scanner.skip(region.regexp)
445
- length && scanner.pos
446
- else
447
- region.regexp.match(binary, position)&.then { |match| match.begin(0) == position ? match.end(0) : nil }
515
+ def accelerate(region, buffer, position, token_rule:, token_start:)
516
+ mode = @config.options.fetch(:accel, :auto)
517
+ cursor = position
518
+ matched = false
519
+ loop do
520
+ if cursor >= buffer.bytesize
521
+ return matched ? cursor : nil if buffer.eof_loaded?
522
+ return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
523
+ end
524
+
525
+ segment = buffer.source
526
+ segment_position = cursor - buffer.base_offset
527
+ length = acceleration_length(region, segment, segment_position, mode)
528
+ record_auto_acceleration_miss(region) if mode == :auto && length.to_i < 8
529
+ return matched ? cursor : nil unless length&.positive?
530
+
531
+ matched = true
532
+ cursor += length
533
+ @lexer.consume_step!(length)
534
+ ensure_token_size!(cursor, token_start, rule: token_rule)
535
+ return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
448
536
  end
449
- return match_end if match_end && match_end < buffer.bytesize
450
- return match_end if match_end && buffer.eof_loaded?
451
- return unless buffer.ensure_available?(buffer.bytesize + 1)
537
+ end
452
538
 
453
- accelerate(region, buffer, position)
539
+ def acceleration_length(region, segment, position, mode)
540
+ if segment.encoding == Encoding::UTF_8 && region.utf8_regexp
541
+ regexp = region.utf8_regexp
542
+ else
543
+ segment = segment.b unless segment.encoding == Encoding::BINARY
544
+ regexp = region.regexp
545
+ end
546
+ if %i[strscan auto].include?(mode) && defined?(::StringScanner)
547
+ @acceleration_scanner ||= ::StringScanner.new("".b)
548
+ @acceleration_scanner.string = segment
549
+ @acceleration_scanner.pos = position
550
+ @acceleration_scanner.skip(regexp)
551
+ else
552
+ match = regexp.match(segment.byteslice(position..).to_s, 0)
553
+ match&.begin(0)&.zero? ? match[0].bytesize : nil
554
+ end
454
555
  rescue ArgumentError
455
556
  nil
456
557
  end
457
558
 
559
+ def record_auto_acceleration_miss(region)
560
+ @auto_acceleration_misses ||= Hash.new(0)
561
+ @auto_acceleration_misses[region] += 1
562
+ @auto_acceleration_miss_total = @auto_acceleration_miss_total.to_i + 1
563
+ @auto_acceleration_off = true if @auto_acceleration_miss_total >= 9
564
+ return if @auto_acceleration_misses[region] < 3
565
+
566
+ @disabled_auto_acceleration ||= {}
567
+ @disabled_auto_acceleration[region] = true
568
+ end
569
+
458
570
  def transition(dfa, state, byte)
459
- if @lexer.class.__flexr_config.backend == :direct &&
460
- @lexer.class.respond_to?(:__flexr_generated_direct_transition)
461
- return @lexer.class.__flexr_generated_direct_transition(@lexer.state, state, byte)
462
- end
463
- return dfa.transition_direct(state, byte) if @lexer.class.__flexr_config.backend == :direct
571
+ return dfa.transition_direct(state, byte) if @config.backend == :direct
464
572
 
465
573
  dfa.transition(state, byte)
466
574
  end
@@ -474,7 +582,7 @@ module Flexr
474
582
 
475
583
  accepting = machine.dfa.accepts[region.state]
476
584
  next if accepting.any? do |acceptance|
477
- rule = @lexer.class.__flexr_rules.fetch(acceptance.rule_index)
585
+ rule = @rules.fetch(acceptance.rule_index)
478
586
  acceptance.bol_only || acceptance.end_anchor || rule.trailing
479
587
  end
480
588
 
@@ -486,15 +594,23 @@ module Flexr
486
594
  buffer.eof?(position) || buffer.getbyte(position) == 0x0a
487
595
  end
488
596
 
489
- def ensure_token_size!(end_position, position)
597
+ def ensure_token_size!(end_position, position, rule: nil)
490
598
  text_start = @lexer.more_text_start || position
491
- @lexer.defer_token_size_check!(end_position - text_start)
599
+ @lexer.defer_token_size_check!(end_position - text_start, rule: rule)
600
+ end
601
+
602
+ def ensure_stream_limit!(limit, position, size, rule)
603
+ if limit == :lookahead
604
+ @lexer.ensure_lookahead_size!(size, rule: rule)
605
+ else
606
+ ensure_token_size!(position + size, position, rule: rule)
607
+ end
492
608
  end
493
609
 
494
610
  def rule_active?(rule)
495
611
  return true if rule.states.include?(:initial) && @lexer.state == :initial
496
612
 
497
- state = @lexer.class.__flexr_config.states.fetch(@lexer.state)
613
+ state = @config.states.fetch(@lexer.state)
498
614
  return true if state.inclusive && rule.states.include?(:initial)
499
615
 
500
616
  rule.states.include?(@lexer.state)
@@ -2,24 +2,47 @@
2
2
 
3
3
  module Flexr
4
4
  module Runtime
5
- Location = Struct.new(
6
- :filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end,
7
- keyword_init: true
8
- ) do
9
- def initialize(**values)
10
- @column_values = values.delete(:column_values)
11
- eager_columns = values.delete(:eager_columns)
12
- super(**values) # rubocop:disable Style/SuperArguments
13
- self.column_begin = @column_values.first if eager_columns && @column_values
14
- self.column_end = @column_values.last if eager_columns && @column_values
5
+ Location = Data.define(:filename, :byte_begin, :byte_end, :line_begin, :column_begin, :line_end, :column_end) do
6
+ include Record
7
+
8
+ def initialize(filename: nil, byte_begin: nil, byte_end: nil, line_begin: nil, column_begin: nil,
9
+ line_end: nil, column_end: nil, column_values: nil, eager_columns: false)
10
+ @column_values = [column_begin || column_values&.first, column_end || column_values&.last].freeze
11
+ filename = filename.dup.freeze if filename.is_a?(String)
12
+ column_begin ||= @column_values&.first if eager_columns
13
+ column_end ||= @column_values&.last if eager_columns
14
+ super(filename: filename, byte_begin: byte_begin, byte_end: byte_end, line_begin: line_begin,
15
+ column_begin: column_begin, line_end: line_end, column_end: column_end)
15
16
  end
16
17
 
17
18
  def column_begin
18
- self[:column_begin] ||= @column_values&.first
19
+ @column_values.first
19
20
  end
20
21
 
21
22
  def column_end
22
- self[:column_end] ||= @column_values&.last
23
+ @column_values.last
24
+ end
25
+
26
+ def to_h(&block)
27
+ values = super().merge(column_begin: column_begin, column_end: column_end)
28
+ block ? values.to_h(&block) : values
29
+ end
30
+
31
+ def with(**changes)
32
+ columns = @column_values.dup
33
+ columns[0] = changes[:column_begin] if changes.key?(:column_begin)
34
+ columns[1] = changes[:column_end] if changes.key?(:column_end)
35
+ self.class.new(**deconstruct_keys(nil), **changes, column_values: columns)
36
+ end
37
+
38
+ def ==(other)
39
+ other.instance_of?(self.class) && to_h == other.to_h
40
+ end
41
+
42
+ alias_method :eql?, :==
43
+
44
+ def hash
45
+ [self.class, to_h].hash
23
46
  end
24
47
  end
25
48
  end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Runtime
5
+ module Record
6
+ def [](key)
7
+ member = key.is_a?(Integer) ? members.fetch(key) : key.to_sym
8
+ raise NameError, "unknown record member #{key.inspect}" unless members.include?(member)
9
+
10
+ public_send(member)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -2,6 +2,12 @@
2
2
 
3
3
  module Flexr
4
4
  module Runtime
5
- Token = Struct.new(:type, :value, :location, keyword_init: true)
5
+ Token = Data.define(:type, :value, :location) do
6
+ include Record
7
+
8
+ def initialize(type: nil, value: nil, location: nil)
9
+ super
10
+ end
11
+ end
6
12
  end
7
13
  end