flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -3,20 +3,46 @@
3
3
  module Flexr
4
4
  module Runtime
5
5
  def initialize(input, filename: nil, error_mode: :raise, max_token_size: 16 * 1024 * 1024,
6
- max_state_stack: 1024, chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
6
+ max_lookahead_size: nil, max_buffer_size: 64 * 1024 * 1024,
7
+ max_state_stack: 1024, max_steps: nil, cancellation: nil, retain_input: true,
8
+ chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
9
+ raise ArgumentError, "unsupported error_mode: #{error_mode.inspect}" unless %i[raise token panic].include?(error_mode)
7
10
  raise ArgumentError, "max_token_size must be non-negative" if max_token_size.to_i.negative?
11
+ raise ArgumentError, "max_lookahead_size must be non-negative" if max_lookahead_size&.to_i&.negative?
12
+ raise ArgumentError, "max_buffer_size must be non-negative" if max_buffer_size.to_i.negative?
8
13
  raise ArgumentError, "max_state_stack must be non-negative" if max_state_stack.to_i.negative?
14
+ raise ArgumentError, "max_steps must be non-negative" if max_steps&.to_i&.negative?
15
+ raise ArgumentError, "cancellation must respond to call" if cancellation && !cancellation.respond_to?(:call)
9
16
 
10
17
  self.class.compile!
11
- @buffer = Runtime::Buffer.new(input, chunk_size: chunk_size)
12
- @input = input.is_a?(String) ? input : nil
13
- @line_tracking_needed = !@input || input.include?("\n")
18
+ config = self.class.__flexr_config
19
+ @buffer = Runtime::Buffer.new(
20
+ input, chunk_size: chunk_size, max_buffer_size: max_buffer_size,
21
+ retain_input: retain_input, filename: filename
22
+ )
23
+ @string_input = input.is_a?(String)
24
+ @stable_input_end = input.bytesize if @string_input && input.frozen?
25
+ @valid_utf8_input = @string_input && input.frozen? && (config.encoding != Encoding::UTF_8 || begin
26
+ bytes = input.encoding == Encoding::UTF_8 ? input : input.dup.force_encoding(Encoding::UTF_8)
27
+ bytes.valid_encoding?
28
+ end)
29
+ @simple_location_input = @string_input && input.frozen? && retain_input &&
30
+ input.ascii_only? && !input.include?("\n")
31
+ @retain_input = retain_input
14
32
  @filename = filename
15
33
  @error_mode = error_mode
34
+ @token_kind = config.token_kind
35
+ @accel_mode = config.options.fetch(:accel, :auto)
16
36
  @max_token_size = max_token_size.to_i
37
+ @token_limit_required = !@string_input || !input.frozen? || input.bytesize > @max_token_size
38
+ @max_lookahead_size = (max_lookahead_size || max_token_size).to_i
17
39
  @max_state_stack = max_state_stack.to_i
40
+ @max_steps = max_steps&.to_i
41
+ @cancellation = cancellation.respond_to?(:arity) ? cancellation : cancellation&.method(:call)
42
+ @steps = 0
18
43
  @position = 0
19
44
  @line = 1
45
+ @column = 1
20
46
  @state = :initial
21
47
  @state_stack = []
22
48
  @pending = nil
@@ -25,18 +51,22 @@ module Flexr
25
51
  @match_end = 0
26
52
  @text_start = 0
27
53
  @text_line = 1
54
+ @text_column = 1
28
55
  @bol = true
29
56
  @more_start = nil
30
57
  @more_line = nil
58
+ @more_column = nil
31
59
  @more_requested = false
32
- @candidate_token_size = 0
60
+ @non_progress_signatures = {}
61
+ @active_rule = nil
33
62
  @eof_fired_states = {}
34
63
  @on_error = nil
35
64
  @halted = false
36
65
  @interpreter = nil
66
+ @generated_scanner = generated_runtime? && respond_to?(:scan_one, true)
37
67
  end
38
68
 
39
- attr_reader :filename, :error_mode, :buffer, :max_token_size
69
+ attr_reader :filename, :error_mode, :buffer, :max_token_size, :max_lookahead_size, :steps
40
70
 
41
71
  attr_writer :on_error
42
72
 
@@ -49,6 +79,7 @@ module Flexr
49
79
 
50
80
  loop do
51
81
  return nil if @halted
82
+ consume_step! if scan_steps_guarded?
52
83
 
53
84
  if eof? && @pending.nil?
54
85
  eof_action = self.class.__flexr_spec.eof_rules[@state]
@@ -58,6 +89,7 @@ module Flexr
58
89
  @match_end = @position
59
90
  @text_start = @position
60
91
  @text_line = @line
92
+ @text_column = @column
61
93
  @matched = nil
62
94
  instance_exec(&eof_action)
63
95
  token = @pending
@@ -68,7 +100,8 @@ module Flexr
68
100
  return nil
69
101
  end
70
102
 
71
- match = if generated_runtime? && respond_to?(:scan_one, true)
103
+ @active_rule = nil
104
+ match = if @generated_scanner
72
105
  scan_one
73
106
  else
74
107
  (@interpreter ||= Runtime::Interpreter.new(self)).scan
@@ -84,22 +117,28 @@ module Flexr
84
117
  end
85
118
  @match_start = match.start_pos
86
119
  @match_end = match.end_pos
120
+ scan_state = @state
87
121
  if @more_start
88
122
  @text_start = @more_start
89
123
  @text_line = @more_line
124
+ @text_column = @more_column
90
125
  else
91
126
  @text_start = @match_start
92
127
  @text_line = @line
128
+ @text_column = @column
93
129
  end
94
130
  @matched = nil
95
131
  @more_requested = false
96
132
  @position = match.end_pos
97
133
  empty_match = match.end_pos == match.start_pos
134
+ @active_rule = match.rule
98
135
  execute(match.rule)
99
136
  finalize_more
100
- force_empty_match_progress! if empty_match && self.class.__flexr_config.options[:allow_empty_match]
101
- ensure_token_size!
137
+ ensure_progress!(match, scan_state, empty_match: empty_match) unless
138
+ @position > match.start_pos && @non_progress_signatures.empty?
139
+ ensure_token_size! if @token_limit_required
102
140
  update_position
141
+ discard_consumed_input! unless @retain_input
103
142
  token = @pending
104
143
  @pending = nil
105
144
  return token if token
@@ -113,7 +152,7 @@ module Flexr
113
152
  token = next_token
114
153
  break unless token
115
154
 
116
- if self.class.__flexr_config.token_kind == :yield
155
+ if @token_kind == :yield
117
156
  yield(*token)
118
157
  else
119
158
  yield token
@@ -138,7 +177,7 @@ module Flexr
138
177
  def text
139
178
  return @matched if @matched
140
179
 
141
- @matched = @buffer.byteslice(@text_start...@match_end)
180
+ @matched = @buffer.byteslice(@text_start, @match_end - @text_start)
142
181
  end
143
182
 
144
183
  def text_bytesize
@@ -153,8 +192,71 @@ module Flexr
153
192
  @more_start
154
193
  end
155
194
 
156
- def defer_token_size_check!(size)
157
- @candidate_token_size = size if size > @candidate_token_size
195
+ def defer_token_size_check!(size, rule: nil)
196
+ return if size <= @max_token_size
197
+
198
+ raise Runtime::TokenTooLargeError.new(
199
+ filename: @filename, byte_pos: @more_start || @position, line: @line,
200
+ rule: diagnostic_rule(rule || @active_rule)
201
+ )
202
+ end
203
+
204
+ def ensure_lookahead_size!(size, rule: nil)
205
+ return if size <= @max_lookahead_size
206
+
207
+ raise Runtime::LookaheadTooLargeError.new(
208
+ filename: @filename, byte_pos: @position, line: @line,
209
+ rule: diagnostic_rule(rule || @active_rule)
210
+ )
211
+ end
212
+
213
+ def consume_step!(count = 1)
214
+ return unless scan_steps_guarded?
215
+
216
+ @steps += count
217
+ if @max_steps && @steps > @max_steps
218
+ raise Runtime::StepLimitError.new(
219
+ filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
220
+ )
221
+ end
222
+ return unless cancelled?
223
+
224
+ raise Runtime::CancelledError.new(
225
+ filename: @filename, byte_pos: @position, line: @line, rule: @active_rule
226
+ )
227
+ end
228
+
229
+ def scan_steps_guarded?
230
+ !@max_steps.nil? || !@cancellation.nil?
231
+ end
232
+
233
+ def token_limit_required?
234
+ @token_limit_required
235
+ end
236
+
237
+ def __flexr_whole_token_match(plans, position)
238
+ return unless plans && defined?(::StringScanner) && !scan_steps_guarded? && !token_limit_required?
239
+
240
+ source = @buffer.stable_source
241
+ return unless source&.frozen?
242
+
243
+ byte = source.getbyte(position)
244
+ plan = plans[byte] if byte
245
+ return unless plan
246
+
247
+ scanner = @__flexr_whole_scanner ||= ::StringScanner.new(source)
248
+ scanner.pos = position
249
+ length = scanner.skip(plan.regexp)
250
+ return unless length
251
+
252
+ match = @__flexr_whole_match ||= Runtime::Match.new
253
+ match.rule = self.class.__flexr_rules.fetch(plan.rule_index)
254
+ match.start_pos = position
255
+ match.end_pos = position + length
256
+ match.total_end_pos = match.end_pos
257
+ match
258
+ rescue ArgumentError
259
+ nil
158
260
  end
159
261
 
160
262
  def utf8_input?
@@ -162,7 +264,7 @@ module Flexr
162
264
  end
163
265
 
164
266
  def valid_utf8_at?(position)
165
- !utf8_input? || @buffer.valid_utf8_at?(position)
267
+ @valid_utf8_input || !utf8_input? || @buffer.valid_utf8_at?(position)
166
268
  end
167
269
 
168
270
  def utf8_boundary?(position)
@@ -184,11 +286,12 @@ module Flexr
184
286
  end
185
287
 
186
288
  def binary_input
187
- @buffer.source.b
289
+ source = @buffer.source
290
+ source.encoding == ::Encoding::BINARY ? source : source.b
188
291
  end
189
292
 
190
293
  def emit(type, value = text)
191
- @pending = case self.class.__flexr_config.token_kind
294
+ @pending = case @token_kind
192
295
  when :struct
193
296
  Runtime::Token.new(type: type, value: value, location: last_location)
194
297
  else
@@ -201,7 +304,7 @@ module Flexr
201
304
  end
202
305
 
203
306
  def error!(message)
204
- error = LexError.new(message, filename: @filename, byte_pos: @match_start, line: @line, text: text)
307
+ error = LexError.new(message, filename: @filename, byte_pos: @match_start, line: @text_line, text: text)
205
308
  if @on_error
206
309
  action = @on_error.call(error)
207
310
  return @pending = nil if action == :skip
@@ -211,6 +314,11 @@ module Flexr
211
314
  return @pending = nil
212
315
  end
213
316
  return emit(:error, text) if action == :token
317
+
318
+ raise Runtime::InvalidRecoveryActionError.new(
319
+ action: action, filename: @filename, byte_pos: @match_start, line: @text_line,
320
+ text: text, rule: @active_rule
321
+ )
214
322
  end
215
323
  case @error_mode
216
324
  when :token
@@ -229,9 +337,10 @@ module Flexr
229
337
  def reject
230
338
  diagnostic = Diagnostics.error(
231
339
  "FLEXR-E013", "reject is not supported by flexr",
340
+ location: [@position, @position],
232
341
  help: "use a state transition and less(n) to express the fallback"
233
342
  )
234
- raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
343
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
235
344
  end
236
345
 
237
346
  def push(name)
@@ -239,7 +348,8 @@ module Flexr
239
348
  if @state_stack.length >= @max_state_stack
240
349
  raise Runtime::StateStackOverflowError.new(
241
350
  "state stack exceeds max_state_stack (#{@max_state_stack})",
242
- filename: @filename, byte_pos: @position, line: @line, text: text
351
+ filename: @filename, byte_pos: @position, line: @line, text: text,
352
+ rule: @active_rule
243
353
  )
244
354
  end
245
355
 
@@ -266,7 +376,11 @@ module Flexr
266
376
  matched_bytes = @match_end - @match_start
267
377
  raise ArgumentError, "less must not exceed matched bytes" if count.negative? || count > matched_bytes
268
378
 
269
- @position = @match_start + count
379
+ new_position = @match_start + count
380
+ raise ArgumentError, "less must end at a UTF-8 codepoint boundary" if
381
+ utf8_input? && !@buffer.utf8_boundary?(new_position)
382
+
383
+ @position = new_position
270
384
  @match_end = @position
271
385
  @matched = nil
272
386
  end
@@ -277,10 +391,14 @@ module Flexr
277
391
 
278
392
  def last_location
279
393
  line_begin = @text_line || @line
394
+ column_begin = @text_column || @column
395
+ line_end, column_end = location_after(
396
+ @text_start, @match_end, line: line_begin, column: column_begin
397
+ )
280
398
  Runtime::Location.new(
281
399
  filename: @filename, byte_begin: @text_start, byte_end: @match_end,
282
- line_begin: line_begin, line_end: line_begin + text.to_s.b.count("\n"),
283
- column_values: [column_at(@text_start), column_at(@match_end)],
400
+ line_begin: line_begin, line_end: line_end,
401
+ column_values: [column_begin, column_end],
284
402
  eager_columns: self.class.__flexr_config.options[:eager_columns] == true
285
403
  )
286
404
  end
@@ -304,9 +422,11 @@ module Flexr
304
422
  if @more_requested
305
423
  @more_start = @text_start
306
424
  @more_line = @text_line
425
+ @more_column = @text_column
307
426
  else
308
427
  @more_start = nil
309
428
  @more_line = nil
429
+ @more_column = nil
310
430
  end
311
431
  @more_requested = false
312
432
  end
@@ -316,34 +436,31 @@ module Flexr
316
436
  end
317
437
 
318
438
  def ensure_token_size!
319
- actual_size = @match_end - @text_start
320
- @candidate_token_size = 0
321
- return if actual_size <= @max_token_size
439
+ return unless @token_limit_required
322
440
 
323
- raise Runtime::TokenTooLargeError
441
+ actual_size = @match_end - @text_start
442
+ defer_token_size_check!(actual_size, rule: @active_rule)
324
443
  end
325
444
 
326
445
  def force_empty_match_progress!
327
446
  return if eof?
328
447
 
329
- byte = @buffer.byteslice(@position, 1).to_s.b
330
- @position += 1
331
- @line += 1 if byte == "\n"
332
- @bol = byte == "\n"
448
+ length = utf8_input? ? @buffer.utf8_character_length(@position) : 1
449
+ ending = @position + length
450
+ advance_location!(@position, ending)
451
+ @position = ending
452
+ @bol = @buffer.getbyte(@position - 1) == 0x0a
333
453
  end
334
454
 
335
455
  def update_position
336
- if @match_end > @match_start
337
- length = @match_end - @match_start
338
- @line += if length == 1
339
- @buffer.source.getbyte(@match_start) == 0x0a ? 1 : 0
340
- elsif @line_tracking_needed && (newline = @buffer.source.index("\n", @match_start)) && newline < @match_end
341
- @buffer.byteslice(@match_start, length).to_s.count("\n")
342
- else
343
- 0
344
- end
456
+ if @simple_location_input
457
+ @column += @match_end - @match_start
458
+ @bol = @match_end.zero?
459
+ return
345
460
  end
346
- @bol = @match_end.zero? || @buffer.source.getbyte(@match_end - 1) == 0x0a
461
+
462
+ advance_location!(@match_start, @match_end)
463
+ @bol = @match_end.zero? || @buffer.getbyte(@match_end - 1) == 0x0a
347
464
  end
348
465
 
349
466
  def handle_unmatched_byte
@@ -352,17 +469,22 @@ module Flexr
352
469
  @match_end = @position + 1
353
470
  @text_start = @match_start
354
471
  @text_line = @line
472
+ @text_column = @column
473
+ @matched = nil
355
474
  @position += 1
356
- @line += 1 if bad.to_s.b == "\n"
475
+ @non_progress_signatures.clear
476
+ advance_location!(@match_start, @match_end)
357
477
  @bol = bad.to_s.b == "\n"
358
478
  error!("unexpected byte #{bad.inspect}")
359
479
  token = @pending
360
480
  @pending = nil
481
+ discard_consumed_input! unless @retain_input
361
482
  token
362
483
  end
363
484
 
364
485
  def eof?
365
- return @position >= @buffer.bytesize if @input
486
+ return @position >= @stable_input_end if @stable_input_end
487
+ return @position >= @buffer.bytesize if @string_input
366
488
 
367
489
  @buffer.eof?(@position)
368
490
  end
@@ -370,19 +492,94 @@ module Flexr
370
492
  def ensure_state!(name)
371
493
  return if self.class.__flexr_config.states.key?(name.to_sym)
372
494
 
373
- diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}")
374
- raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
495
+ diagnostic = Diagnostics.error("FLEXR-E003", "undefined state: #{name}", location: [@position, @position])
496
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic, filename: @filename, byte_pos: @position)
497
+ end
498
+
499
+ def ensure_progress!(match, scan_state, empty_match:)
500
+ if @position > match.start_pos
501
+ @non_progress_signatures.clear
502
+ return
503
+ end
504
+ return if @halted
505
+
506
+ if empty_match && @state == scan_state && self.class.__flexr_config.options[:allow_empty_match]
507
+ force_empty_match_progress!
508
+ @non_progress_signatures.clear
509
+ return
510
+ end
511
+
512
+ signature = [match.start_pos, scan_state, match.rule.index]
513
+ repeated = @non_progress_signatures.key?(signature)
514
+ @non_progress_signatures[signature] = true
515
+ return if @state != scan_state && !repeated
516
+
517
+ raise Runtime::NonProgressError.new(
518
+ filename: @filename, byte_pos: match.start_pos, line: @line,
519
+ text: text, rule: match.rule
520
+ )
375
521
  end
376
522
 
377
- def column_at(position)
378
- prefix = @buffer.byteslice(0...position).to_s
379
- last_newline = prefix.rindex("\n")
380
- line_prefix = prefix.byteslice((last_newline ? last_newline + 1 : 0)..).to_s
381
- if utf8_input? && line_prefix.dup.force_encoding(Encoding::UTF_8).valid_encoding?
382
- line_prefix.force_encoding(Encoding::UTF_8).length + 1
523
+ def advance_location!(starting, ending)
524
+ return if ending <= starting
525
+
526
+ if @simple_location_input
527
+ @column += ending - starting
528
+ return
529
+ end
530
+
531
+ value = location_value(starting, ending)
532
+ location_bytes = value.b
533
+ newline_count = location_bytes.count("\n")
534
+ if newline_count.zero?
535
+ @column += display_length(value)
383
536
  else
384
- line_prefix.bytesize + 1
537
+ tail = value.byteslice(((location_bytes.rindex("\n") || -1) + 1)..).to_s
538
+ @line += newline_count
539
+ @column = display_length(tail) + 1
385
540
  end
386
541
  end
542
+
543
+ def location_after(starting, ending, line:, column:)
544
+ return [line, column] if ending <= starting
545
+
546
+ value = location_value(starting, ending)
547
+ location_bytes = value.b
548
+ newline_count = location_bytes.count("\n")
549
+ return [line, column + display_length(value)] if newline_count.zero?
550
+
551
+ tail = value.byteslice(((location_bytes.rindex("\n") || -1) + 1)..).to_s
552
+ [line + newline_count, display_length(tail) + 1]
553
+ end
554
+
555
+ def display_length(value)
556
+ return value.bytesize unless utf8_input?
557
+ return value.bytesize if value.ascii_only?
558
+
559
+ utf8 = value.encoding == Encoding::UTF_8 ? value : value.dup.force_encoding(Encoding::UTF_8)
560
+ utf8.valid_encoding? ? utf8.length : value.bytesize
561
+ end
562
+
563
+ def location_value(starting, ending)
564
+ return @matched if @matched && starting == @text_start && ending == @match_end
565
+
566
+ @buffer.byteslice(starting, ending - starting).to_s
567
+ end
568
+
569
+ def discard_consumed_input!
570
+ @buffer.discard_before(@more_start || @position)
571
+ end
572
+
573
+ def cancelled?
574
+ return false unless @cancellation
575
+
576
+ @cancellation.arity.zero? ? @cancellation.call : @cancellation.call(self)
577
+ end
578
+
579
+ def diagnostic_rule(rule)
580
+ return rule unless rule.is_a?(Integer) && rule >= 0
581
+
582
+ self.class.__flexr_rules.fetch(rule, rule)
583
+ end
387
584
  end
388
585
  end
@@ -5,18 +5,80 @@ module Flexr
5
5
  class TokenTooLargeError < LexError
6
6
  CODE = "FLEXR-E012"
7
7
 
8
- attr_reader :code
8
+ attr_reader :code, :rule
9
9
 
10
- def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil)
10
+ def initialize(message = "token exceeds max_token_size", filename: nil, byte_pos: nil, line: nil, text: nil,
11
+ rule: nil)
11
12
  @code = CODE
13
+ @rule = rule.is_a?(IR::Rule) ? rule.index : rule
12
14
  diagnostic = Diagnostics.error(
13
15
  CODE,
14
16
  message,
15
- help: "increase max_token_size or split the input token"
17
+ location: byte_pos && [byte_pos, byte_pos],
18
+ help: "increase max_token_size or split the input token",
19
+ note: rule.nil? ? nil : "#{Diagnostics.rule_label(rule)} at byte #{byte_pos}"
16
20
  )
17
21
  super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
18
22
  end
19
23
  end
20
- class StateStackOverflowError < LexError; end
24
+
25
+ class ResourceLimitError < LexError
26
+ attr_reader :code, :rule
27
+
28
+ def initialize(code, message, help:, filename: nil, byte_pos: nil, line: nil, text: nil, rule: nil)
29
+ @code = code
30
+ @rule = rule.is_a?(IR::Rule) ? rule.index : rule
31
+ diagnostic = Diagnostics.error(code, message, location: byte_pos && [byte_pos, byte_pos],
32
+ help: help, note: Diagnostics.rule_label(rule))
33
+ super(message, filename: filename, byte_pos: byte_pos, line: line, text: text, diagnostic: diagnostic)
34
+ end
35
+ end
36
+
37
+ class NonProgressError < ResourceLimitError
38
+ def initialize(**context)
39
+ super("FLEXR-E021", "lexer action did not make progress",
40
+ help: "consume input or change to a state that can consume it", **context)
41
+ end
42
+ end
43
+
44
+ class BufferTooLargeError < ResourceLimitError
45
+ def initialize(**context)
46
+ super("FLEXR-E022", "streaming buffer exceeds max_buffer_size",
47
+ help: "increase max_buffer_size, shorten tokens, or use retain_input: false", **context)
48
+ end
49
+ end
50
+
51
+ class StepLimitError < ResourceLimitError
52
+ def initialize(**context)
53
+ super("FLEXR-E023", "lexer exceeds max_steps",
54
+ help: "increase max_steps or simplify the specification", **context)
55
+ end
56
+ end
57
+
58
+ class LookaheadTooLargeError < ResourceLimitError
59
+ def initialize(**context)
60
+ super("FLEXR-E024", "trailing context exceeds max_lookahead_size",
61
+ help: "increase max_lookahead_size or bound the trailing context", **context)
62
+ end
63
+ end
64
+
65
+ class StateStackOverflowError < ResourceLimitError
66
+ def initialize(message = "state stack exceeds max_state_stack", **context)
67
+ super("FLEXR-E025", message, help: "increase max_state_stack or remove recursive pushes", **context)
68
+ end
69
+ end
70
+
71
+ class CancelledError < ResourceLimitError
72
+ def initialize(**context)
73
+ super("FLEXR-E026", "lexing was cancelled", help: "resume with a new lexer when ready", **context)
74
+ end
75
+ end
76
+
77
+ class InvalidRecoveryActionError < ResourceLimitError
78
+ def initialize(action:, **context)
79
+ super("FLEXR-E027", "on_error returned unsupported action #{action.inspect}",
80
+ help: "return :skip, :raise, :halt, or :token", **context)
81
+ end
82
+ end
21
83
  end
22
84
  end