flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,280 @@
1
+ # frozen_string_literal: true
2
+ # Generated by flexr. DO NOT EDIT.
3
+ # source: examples/ruby_subset/lexer.flexr.rb
4
+ # spec-digest: <normalized>
5
+ # artifact-schema: 2
6
+ # runtime-abi: 2
7
+ # unicode: 15.1.0
8
+ # backend: table
9
+ # compiled: true
10
+ # eval: false
11
+ # standalone: false
12
+
13
+ require "flexr"
14
+
15
+ module RubySubset
16
+ class Lexer < Flexr::Lexer
17
+ KEYWORDS = %w[class def end].freeze
18
+ IDENT = /[A-Za-z_][A-Za-z_0-9]*/
19
+
20
+ Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: ["class", "def", "end"], pattern_conditions: [[1, 0, false, false], [1, 1, false, false], [1, 2, false, false]], trailing: nil, action: proc{ emit :KEYWORD, text.to_sym }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/[A-Za-z_][A-Za-z_0-9]*/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: proc{ emit :IDENT, "#{text}:identifier" }, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/"(?:\\.|[^"\\])*"/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: proc do
21
+ value = <<~TOKEN
22
+ #{text[1...-1]}
23
+ TOKEN
24
+ emit :STRING, value.chomp
25
+ end, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:KEYWORD, :IDENT, :STRING], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [], [[2, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [[3, 0, false, false]], [], [], [], [], [[2, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [[1, 1, false, false], [2, 0, false, false]], [[1, 2, false, false], [2, 0, false, false]], [[2, 0, false, false]], [[1, 0, false, false], [2, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 24, 24, 24, 24, 25, 26, 27, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 36, 36, 36, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37], class_count: 38, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3], packed: {base: [0, 9, 23, 22, 51, 72, 93, 0, 102, 1, 3, 4, 130, 151, 172, 193, 214, 235, 256, 277], default: [nil, nil, 2, nil, nil, nil, nil, nil, 2, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 1, 1, nil, 1, nil, 1, nil, 2, nil, 1, 1, 3, 1, nil, 1, 3, nil, 3, 3, 4, 5, 6, 3, 3, 3, 3, 3, 3, 3, 3, 7, 3, 2, 3, 9, 10, 8, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, nil, nil, nil, 9, 10, 11, nil, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 13, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 14, 3, 3, 3, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 9, 10, 11, nil, 3, nil, 3, nil, nil, nil, 3, nil, 15, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 16, 3, 3, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 17, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 18, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 19, 3, 3, nil, 3, nil, nil, nil, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], check: [nil, 0, 0, nil, 0, nil, 0, nil, 0, nil, 1, 1, 0, 1, nil, 1, 0, nil, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 9, 3, 10, 11, 2, 3, nil, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, nil, nil, 2, 2, 2, 2, 2, 2, 4, nil, 4, nil, nil, nil, 4, nil, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, nil, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 8, 6, nil, nil, nil, 6, nil, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 8, 8, 8, 8, 8, 8, 12, nil, 12, nil, nil, nil, 12, nil, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, nil, 13, nil, nil, nil, 13, nil, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, nil, 14, nil, nil, nil, 14, nil, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, nil, 15, nil, nil, nil, 15, nil, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 16, nil, 16, nil, nil, nil, 16, nil, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, nil, 17, nil, nil, nil, 17, nil, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, nil, 18, nil, nil, nil, 18, nil, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, nil, 19, nil, nil, nil, 19, nil, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 38, accepting_states: 14}}, diagnostics: [] } })
26
+ def scan_one
27
+ if @__flexr_generated_scan_state != @state
28
+ machine = self.class.__flexr_compiled.machines.fetch(@state)
29
+ @__flexr_generated_scan_state = @state
30
+ @__flexr_generated_scan_dfa = machine.dfa
31
+ @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
32
+ @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
33
+ end
34
+ dfa = @__flexr_generated_scan_dfa
35
+ return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
36
+ position = byte_pos
37
+ return nil unless valid_utf8_at?(position)
38
+ return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
39
+ cursor = position
40
+ current = dfa.start
41
+ direct = dfa.direct
42
+ direct_nxt = direct&.fetch(:nxt)
43
+ direct_classes = direct&.fetch(:classes)
44
+ ec = dfa.ec
45
+ source = buffer.stable_source
46
+ guarded_steps = scan_steps_guarded?
47
+ token_start = more_text_start || position
48
+ token_limit = max_token_size
49
+ check_token_limit = token_limit_required?
50
+ best = nil
51
+ accel_mode = @accel_mode
52
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
53
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
54
+ if accelerate
55
+ region = __flexr_generated_acceleration_region(dfa, current)
56
+ acceptance = dfa.accepts[current].first
57
+ token_rule = acceptance&.rule_index
58
+ accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
59
+ if accelerated_end && accelerated_end > cursor
60
+ cursor = accelerated_end
61
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
62
+ next
63
+ end
64
+ end
65
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
66
+ consume_step! if guarded_steps
67
+ current = if direct_nxt
68
+ class_id = ec[byte]
69
+ value = direct_nxt[(current * direct_classes) + class_id]
70
+ value >= 0 ? value : nil
71
+ else
72
+ dfa.transition(current, byte)
73
+ end
74
+ break unless current
75
+ cursor += 1
76
+ resource_acceptance = dfa.accepts[current].first
77
+ resource_rule = resource_acceptance&.rule_index
78
+ if check_token_limit
79
+ size = cursor - token_start
80
+ defer_token_size_check!(size, rule: resource_rule) if size > token_limit
81
+ end
82
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
83
+ end
84
+ best
85
+ end
86
+ def __flexr_generated_scan_simple(dfa, position)
87
+ current = dfa.start
88
+ direct = dfa.direct
89
+ direct_nxt = direct&.fetch(:nxt)
90
+ direct_classes = direct&.fetch(:classes)
91
+ ec = dfa.ec
92
+ accepts = dfa.accepts
93
+ rules = self.class.__flexr_rules
94
+ source = buffer.stable_source
95
+ guarded_steps = scan_steps_guarded?
96
+ token_start = more_text_start || position
97
+ token_limit = max_token_size
98
+ check_token_limit = token_limit_required?
99
+ cursor = position
100
+ best = nil
101
+ accel_mode = @accel_mode
102
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
103
+ if source && !guarded_steps && !check_token_limit && !accelerate
104
+ while cursor < source.bytesize
105
+ byte = source.getbyte(cursor)
106
+ current = if direct_nxt
107
+ class_id = ec[byte]
108
+ value = direct_nxt[(current * direct_classes) + class_id]
109
+ value >= 0 ? value : nil
110
+ else
111
+ dfa.transition(current, byte)
112
+ end
113
+ break unless current
114
+ cursor += 1
115
+ acceptance = accepts[current].first
116
+ next unless acceptance
117
+ rule = rules.fetch(acceptance.rule_index)
118
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
119
+ best.rule = rule
120
+ best.start_pos = position
121
+ best.end_pos = cursor
122
+ best.total_end_pos = cursor
123
+ end
124
+ return best
125
+ end
126
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
127
+ if accelerate
128
+ region = __flexr_generated_acceleration_region(dfa, current)
129
+ acceptance = accepts[current].first
130
+ accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
131
+ if accelerated_end && accelerated_end > cursor
132
+ cursor = accelerated_end
133
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
134
+ next
135
+ end
136
+ end
137
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
138
+ consume_step! if guarded_steps
139
+ current = if direct_nxt
140
+ class_id = ec[byte]
141
+ value = direct_nxt[(current * direct_classes) + class_id]
142
+ value >= 0 ? value : nil
143
+ else
144
+ dfa.transition(current, byte)
145
+ end
146
+ break unless current
147
+ cursor += 1
148
+ acceptance = accepts[current].first
149
+ if check_token_limit
150
+ size = cursor - token_start
151
+ defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
152
+ end
153
+ next unless acceptance
154
+ rule = rules.fetch(acceptance.rule_index)
155
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
156
+ best.rule = rule
157
+ best.start_pos = position
158
+ best.end_pos = cursor
159
+ best.total_end_pos = cursor
160
+ end
161
+ best
162
+ end
163
+ def __flexr_generated_acceleration_region(dfa, state)
164
+ @__flexr_generated_accel_regions ||= {}
165
+ regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
166
+ region = regions[state]
167
+ return unless region
168
+ return if @__flexr_generated_disabled_auto_accel&.key?(region)
169
+ return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
170
+ return if dfa.accepts[state].any? do |acceptance|
171
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
172
+ acceptance.bol_only || acceptance.end_anchor || rule.trailing
173
+ end
174
+ region
175
+ end
176
+ def __flexr_generated_accelerate(region, position, token_rule, token_start)
177
+ mode = @accel_mode
178
+ cursor = position
179
+ matched = false
180
+ loop do
181
+ if cursor >= buffer.bytesize
182
+ return matched ? cursor : nil if buffer.eof_loaded?
183
+ return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
184
+ end
185
+ segment = buffer.source
186
+ segment_position = cursor - buffer.base_offset
187
+ if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
188
+ regexp = region.utf8_regexp
189
+ else
190
+ segment = segment.b unless segment.encoding == ::Encoding::BINARY
191
+ regexp = region.regexp
192
+ end
193
+ length = begin
194
+ if %i[strscan auto].include?(mode) && defined?(::StringScanner)
195
+ @__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
196
+ @__flexr_generated_accel_scanner.string = segment
197
+ @__flexr_generated_accel_scanner.pos = segment_position
198
+ @__flexr_generated_accel_scanner.skip(regexp)
199
+ else
200
+ match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
201
+ match && match.begin(0).zero? ? match[0].bytesize : nil
202
+ end
203
+ rescue ArgumentError
204
+ nil
205
+ end
206
+ __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
207
+ return matched ? cursor : nil unless length&.positive?
208
+ matched = true
209
+ cursor += length
210
+ consume_step!(length)
211
+ defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
212
+ return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
213
+ end
214
+ end
215
+ def __flexr_generated_record_auto_accel_miss(region)
216
+ @__flexr_generated_auto_accel_misses ||= Hash.new(0)
217
+ @__flexr_generated_auto_accel_misses[region] += 1
218
+ @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
219
+ @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
220
+ return if @__flexr_generated_auto_accel_misses[region] < 3
221
+ @__flexr_generated_disabled_auto_accel ||= {}
222
+ @__flexr_generated_disabled_auto_accel[region] = true
223
+ end
224
+ def __flexr_generated_fast_path?(dfa)
225
+ @__flexr_generated_fast_paths ||= {}
226
+ return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
227
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
228
+ @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
229
+ self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
230
+ end
231
+ def __flexr_generated_simple_fast_path?(dfa)
232
+ @__flexr_generated_simple_fast_paths ||= {}
233
+ return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
234
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
235
+ @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
236
+ rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
237
+ end
238
+ end
239
+ def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
240
+ dfa.accepts[state].each do |acceptance|
241
+ next if acceptance.bol_only && !beginning_of_line?
242
+ next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
243
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
244
+ next if best && cursor < best.total_end_pos
245
+ next if best && cursor == best.total_end_pos &&
246
+ rule.index > best.rule.index
247
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
248
+ best.rule = rule
249
+ best.start_pos = start_position
250
+ best.end_pos = cursor
251
+ best.total_end_pos = cursor
252
+ end
253
+ best
254
+ end
255
+ def __flexr_generated_execute(rule)
256
+ case rule.index
257
+ when 0
258
+ when 1
259
+ instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
260
+ when 2
261
+ instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
262
+ when 3
263
+ instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))
264
+ else
265
+ instance_exec(&rule.action)
266
+ end
267
+ end
268
+
269
+
270
+
271
+ __flexr_bind_generated_action(1, proc{ emit :KEYWORD, text.to_sym })
272
+ __flexr_bind_generated_action(2, proc{ emit :IDENT, "#{text}:identifier" })
273
+ __flexr_bind_generated_action(3, proc do
274
+ value = <<~TOKEN
275
+ #{text[1...-1]}
276
+ TOKEN
277
+ emit :STRING, value.chomp
278
+ end)
279
+ end
280
+ end