flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
data/Steepfile ADDED
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ target :public_api do
4
+ signature "sig"
5
+ signature "spec/types"
6
+ check "spec/types/public_api.rb"
7
+ end
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "schema": 1,
3
3
  "spec": "examples/json/lexer.flexr.rb",
4
- "source_sha256": "909c8fbc07f27e5be871a6f7e2bb4d0c29996e4362dbb201dd90edd1ae55c7e3",
5
- "source_bytes": 745,
4
+ "source_sha256": "74d57259c5f67f4f6bf64cb870d4165095864fb61c58c9b8bff2d2d88da8b186",
5
+ "source_bytes": 733,
6
6
  "input_sha256": "f15489cc8fa286cd365fb167b3f99d3d216dc90c0de6279fe9f36c8d2215d7a7",
7
7
  "input_bytes": 140000,
8
8
  "tokens": 50000,
@@ -0,0 +1,276 @@
1
+ # frozen_string_literal: true
2
+ # Generated by flexr. DO NOT EDIT.
3
+ # source: examples/calculator/lexer.flexr.rb
4
+ # spec-digest: <normalized>
5
+ # artifact-schema: 2
6
+ # runtime-abi: 2
7
+ # unicode: 15.1.0
8
+ # backend: table
9
+ # compiled: true
10
+ # eval: false
11
+ # standalone: false
12
+
13
+ require "flexr"
14
+
15
+ module CalculatorExample
16
+ class Lexer < Flexr::Lexer
17
+ Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/==/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/=/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/if/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[a-z_][a-z0-9_]*/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/[0-9]+/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\+/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:EQ, :ASSIGN, :IF, :IDENT, :INTEGER, :PLUS], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [[6, 0, false, false]], [[5, 0, false, false]], [[2, 0, false, false]], [[4, 0, false, false]], [[4, 0, false, false]], [[1, 0, false, false]], [[3, 0, false, false], [4, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 15, 15, 15, 15, 16, 17, 17, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20], class_count: 21, state_count: 9, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6], packed: {base: [0, 1, 0, 1, 1, 11, 22, 0, 33], default: [nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 1, 1, 1, 1, 1, 1, 2, nil, 3, 3, 4, 7, 5, nil, 5, 5, 5, 6, 5, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5, 5, nil, nil, nil, 5, nil, 5, 8, 5, 5, 5, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5], check: [nil, 0, 1, 0, 1, 0, 1, 0, nil, 0, 3, 0, 4, 0, nil, 0, 0, 0, 0, 0, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5, 6, nil, nil, nil, 6, nil, 6, 6, 6, 6, 6, 8, nil, nil, nil, 8, nil, 8, 8, 8, 8, 8]}} } }, states: [:initial], stats: {initial: {states: 9, classes: 21, accepting_states: 8}}, diagnostics: [] } })
18
+ def scan_one
19
+ if @__flexr_generated_scan_state != @state
20
+ machine = self.class.__flexr_compiled.machines.fetch(@state)
21
+ @__flexr_generated_scan_state = @state
22
+ @__flexr_generated_scan_dfa = machine.dfa
23
+ @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
24
+ @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
25
+ end
26
+ dfa = @__flexr_generated_scan_dfa
27
+ return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
28
+ position = byte_pos
29
+ return nil unless valid_utf8_at?(position)
30
+ return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
31
+ cursor = position
32
+ current = dfa.start
33
+ direct = dfa.direct
34
+ direct_nxt = direct&.fetch(:nxt)
35
+ direct_classes = direct&.fetch(:classes)
36
+ ec = dfa.ec
37
+ source = buffer.stable_source
38
+ guarded_steps = scan_steps_guarded?
39
+ token_start = more_text_start || position
40
+ token_limit = max_token_size
41
+ check_token_limit = token_limit_required?
42
+ best = nil
43
+ accel_mode = @accel_mode
44
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
45
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
46
+ if accelerate
47
+ region = __flexr_generated_acceleration_region(dfa, current)
48
+ acceptance = dfa.accepts[current].first
49
+ token_rule = acceptance&.rule_index
50
+ accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
51
+ if accelerated_end && accelerated_end > cursor
52
+ cursor = accelerated_end
53
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
54
+ next
55
+ end
56
+ end
57
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
58
+ consume_step! if guarded_steps
59
+ current = if direct_nxt
60
+ class_id = ec[byte]
61
+ value = direct_nxt[(current * direct_classes) + class_id]
62
+ value >= 0 ? value : nil
63
+ else
64
+ dfa.transition(current, byte)
65
+ end
66
+ break unless current
67
+ cursor += 1
68
+ resource_acceptance = dfa.accepts[current].first
69
+ resource_rule = resource_acceptance&.rule_index
70
+ if check_token_limit
71
+ size = cursor - token_start
72
+ defer_token_size_check!(size, rule: resource_rule) if size > token_limit
73
+ end
74
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
75
+ end
76
+ best
77
+ end
78
+ def __flexr_generated_scan_simple(dfa, position)
79
+ current = dfa.start
80
+ direct = dfa.direct
81
+ direct_nxt = direct&.fetch(:nxt)
82
+ direct_classes = direct&.fetch(:classes)
83
+ ec = dfa.ec
84
+ accepts = dfa.accepts
85
+ rules = self.class.__flexr_rules
86
+ source = buffer.stable_source
87
+ guarded_steps = scan_steps_guarded?
88
+ token_start = more_text_start || position
89
+ token_limit = max_token_size
90
+ check_token_limit = token_limit_required?
91
+ cursor = position
92
+ best = nil
93
+ accel_mode = @accel_mode
94
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
95
+ if source && !guarded_steps && !check_token_limit && !accelerate
96
+ while cursor < source.bytesize
97
+ byte = source.getbyte(cursor)
98
+ current = if direct_nxt
99
+ class_id = ec[byte]
100
+ value = direct_nxt[(current * direct_classes) + class_id]
101
+ value >= 0 ? value : nil
102
+ else
103
+ dfa.transition(current, byte)
104
+ end
105
+ break unless current
106
+ cursor += 1
107
+ acceptance = accepts[current].first
108
+ next unless acceptance
109
+ rule = rules.fetch(acceptance.rule_index)
110
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
111
+ best.rule = rule
112
+ best.start_pos = position
113
+ best.end_pos = cursor
114
+ best.total_end_pos = cursor
115
+ end
116
+ return best
117
+ end
118
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
119
+ if accelerate
120
+ region = __flexr_generated_acceleration_region(dfa, current)
121
+ acceptance = accepts[current].first
122
+ accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
123
+ if accelerated_end && accelerated_end > cursor
124
+ cursor = accelerated_end
125
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
126
+ next
127
+ end
128
+ end
129
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
130
+ consume_step! if guarded_steps
131
+ current = if direct_nxt
132
+ class_id = ec[byte]
133
+ value = direct_nxt[(current * direct_classes) + class_id]
134
+ value >= 0 ? value : nil
135
+ else
136
+ dfa.transition(current, byte)
137
+ end
138
+ break unless current
139
+ cursor += 1
140
+ acceptance = accepts[current].first
141
+ if check_token_limit
142
+ size = cursor - token_start
143
+ defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
144
+ end
145
+ next unless acceptance
146
+ rule = rules.fetch(acceptance.rule_index)
147
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
148
+ best.rule = rule
149
+ best.start_pos = position
150
+ best.end_pos = cursor
151
+ best.total_end_pos = cursor
152
+ end
153
+ best
154
+ end
155
+ def __flexr_generated_acceleration_region(dfa, state)
156
+ @__flexr_generated_accel_regions ||= {}
157
+ regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
158
+ region = regions[state]
159
+ return unless region
160
+ return if @__flexr_generated_disabled_auto_accel&.key?(region)
161
+ return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
162
+ return if dfa.accepts[state].any? do |acceptance|
163
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
164
+ acceptance.bol_only || acceptance.end_anchor || rule.trailing
165
+ end
166
+ region
167
+ end
168
+ def __flexr_generated_accelerate(region, position, token_rule, token_start)
169
+ mode = @accel_mode
170
+ cursor = position
171
+ matched = false
172
+ loop do
173
+ if cursor >= buffer.bytesize
174
+ return matched ? cursor : nil if buffer.eof_loaded?
175
+ return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
176
+ end
177
+ segment = buffer.source
178
+ segment_position = cursor - buffer.base_offset
179
+ if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
180
+ regexp = region.utf8_regexp
181
+ else
182
+ segment = segment.b unless segment.encoding == ::Encoding::BINARY
183
+ regexp = region.regexp
184
+ end
185
+ length = begin
186
+ if %i[strscan auto].include?(mode) && defined?(::StringScanner)
187
+ @__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
188
+ @__flexr_generated_accel_scanner.string = segment
189
+ @__flexr_generated_accel_scanner.pos = segment_position
190
+ @__flexr_generated_accel_scanner.skip(regexp)
191
+ else
192
+ match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
193
+ match && match.begin(0).zero? ? match[0].bytesize : nil
194
+ end
195
+ rescue ArgumentError
196
+ nil
197
+ end
198
+ __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
199
+ return matched ? cursor : nil unless length&.positive?
200
+ matched = true
201
+ cursor += length
202
+ consume_step!(length)
203
+ defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
204
+ return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
205
+ end
206
+ end
207
+ def __flexr_generated_record_auto_accel_miss(region)
208
+ @__flexr_generated_auto_accel_misses ||= Hash.new(0)
209
+ @__flexr_generated_auto_accel_misses[region] += 1
210
+ @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
211
+ @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
212
+ return if @__flexr_generated_auto_accel_misses[region] < 3
213
+ @__flexr_generated_disabled_auto_accel ||= {}
214
+ @__flexr_generated_disabled_auto_accel[region] = true
215
+ end
216
+ def __flexr_generated_fast_path?(dfa)
217
+ @__flexr_generated_fast_paths ||= {}
218
+ return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
219
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
220
+ @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
221
+ self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
222
+ end
223
+ def __flexr_generated_simple_fast_path?(dfa)
224
+ @__flexr_generated_simple_fast_paths ||= {}
225
+ return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
226
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
227
+ @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
228
+ rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
229
+ end
230
+ end
231
+ def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
232
+ dfa.accepts[state].each do |acceptance|
233
+ next if acceptance.bol_only && !beginning_of_line?
234
+ next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
235
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
236
+ next if best && cursor < best.total_end_pos
237
+ next if best && cursor == best.total_end_pos &&
238
+ rule.index > best.rule.index
239
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
240
+ best.rule = rule
241
+ best.start_pos = start_position
242
+ best.end_pos = cursor
243
+ best.total_end_pos = cursor
244
+ end
245
+ best
246
+ end
247
+ def __flexr_generated_execute(rule)
248
+ case rule.index
249
+ when 0
250
+ when 1
251
+ emit :EQ
252
+ when 2
253
+ emit :ASSIGN
254
+ when 3
255
+ emit :IF
256
+ when 4
257
+ emit :IDENT
258
+ when 5
259
+ emit :INTEGER, text.to_i
260
+ when 6
261
+ emit :PLUS
262
+ else
263
+ instance_exec(&rule.action)
264
+ end
265
+ end
266
+
267
+
268
+
269
+
270
+
271
+
272
+
273
+
274
+
275
+ end
276
+ end
@@ -0,0 +1,310 @@
1
+ # frozen_string_literal: true
2
+ # Generated by flexr. DO NOT EDIT.
3
+ # source: examples/graphql/lexer.flexr.rb
4
+ # spec-digest: <normalized>
5
+ # artifact-schema: 2
6
+ # runtime-abi: 2
7
+ # unicode: 15.1.0
8
+ # backend: table
9
+ # compiled: true
10
+ # eval: false
11
+ # standalone: false
12
+
13
+ require "flexr"
14
+
15
+ module GraphqlExample
16
+ class Lexer < Flexr::Lexer
17
+ Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n,]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/#[^\n]*/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/[A-Za-z_][A-Za-z_0-9]*/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: [:emit, :NAME], states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: [:emit, :NUMBER], states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/"(?:\\.|[^"\\\r\n])*"/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: [:emit, :STRING], states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\.\.\./], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: [:emit, :SPREAD], states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\{/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: [:emit, :LBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\}/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: [:emit, :RBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\(/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: [:emit, :LPAREN], states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\)/], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: [:emit, :RPAREN], states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/\[/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: [:emit, :LBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 11, patterns: [/\]/], pattern_conditions: [[11, 0, false, false]], trailing: nil, action: [:emit, :RBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 12, patterns: [/:/], pattern_conditions: [[12, 0, false, false]], trailing: nil, action: [:emit, :COLON], states: [:initial], bol_only: false, end_anchor: nil }, { index: 13, patterns: [/=/], pattern_conditions: [[13, 0, false, false]], trailing: nil, action: [:emit, :EQUAL], states: [:initial], bol_only: false, end_anchor: nil }, { index: 14, patterns: [/!/], pattern_conditions: [[14, 0, false, false]], trailing: nil, action: [:emit, :BANG], states: [:initial], bol_only: false, end_anchor: nil }, { index: 15, patterns: [/\$/], pattern_conditions: [[15, 0, false, false]], trailing: nil, action: [:emit, :DOLLAR], states: [:initial], bol_only: false, end_anchor: nil }, { index: 16, patterns: [/@/], pattern_conditions: [[16, 0, false, false]], trailing: nil, action: [:emit, :AT], states: [:initial], bol_only: false, end_anchor: nil }, { index: 17, patterns: [/\|/], pattern_conditions: [[17, 0, false, false]], trailing: nil, action: [:emit, :PIPE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 18, patterns: [/&/], pattern_conditions: [[18, 0, false, false]], trailing: nil, action: [:emit, :AMP], states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:NAME, :STRING, :NUMBER, :SPREAD, :LBRACE, :RBRACE, :LPAREN, :RPAREN, :LBRACKET, :RBRACKET, :COLON, :EQUAL, :BANG, :DOLLAR, :AT, :PIPE, :AMP], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [[14, 0, false, false]], [], [[1, 0, false, false]], [[15, 0, false, false]], [[18, 0, false, false]], [[8, 0, false, false]], [[9, 0, false, false]], [], [], [[3, 0, false, false]], [[3, 0, false, false]], [[12, 0, false, false]], [[13, 0, false, false]], [[16, 0, false, false]], [[2, 0, false, false]], [[10, 0, false, false]], [[11, 0, false, false]], [[6, 0, false, false]], [[17, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [], [], [], [], [], [], [], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [], [[3, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 24, 25, 25, 26, 27, 27, 28, 29, 29, 29, 29, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 35, 36, 37, 38, 38, 38, 38, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 42, 43, 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 49, 49, 49, 49, 49, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50], class_count: 51, state_count: 37, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], packed: {base: [0, 43, 0, 44, 9, 0, 0, 0, 0, 28, 1, 33, 44, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 69, 15, 17, 20, 23, 24, 25, 5, 53, 62, 0, 98, 64, 75], default: [nil, nil, nil, 3, 4, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 3, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 1, 1, nil, 1, nil, 1, 2, 3, 4, 5, nil, 6, nil, 7, 8, nil, nil, 1, 9, 10, 30, 11, 12, 13, 33, 14, nil, 15, 16, 16, 16, 17, nil, 18, nil, 16, nil, 16, 16, 16, 19, 20, 21, 1, 1, nil, 1, nil, 1, 11, 12, 22, 31, nil, nil, 27, 28, 29, nil, 3, 1, 24, 32, 31, 25, 12, 12, 4, 27, 28, nil, 32, nil, 32, 34, 34, 23, nil, 35, nil, 35, nil, 32, 36, 36, 36, 36, nil, nil, nil, 24, 25, 26, nil, 16, 16, 36, 36, nil, nil, nil, 16, 16, 16, nil, nil, nil, nil, 16, nil, 16, 16, 16, nil, nil, 24, 25, 26, nil, 34, 34, nil, nil, nil, nil, nil, nil, 32, nil, nil, nil, nil, nil, nil, nil, nil, 32], check: [nil, 0, 0, nil, 0, nil, 0, 0, 0, 0, 0, 4, 0, nil, 0, 0, nil, nil, 0, 0, 0, 10, 0, 0, 0, 30, 0, nil, 0, 0, 0, 0, 0, nil, 0, nil, 0, nil, 0, 0, 0, 0, 0, 0, 1, 1, 3, 1, 3, 1, 9, 9, 3, 11, 4, 4, 4, 4, 4, 4, 24, 1, 25, 11, 12, 26, 12, 12, 27, 28, 29, 23, 11, nil, 12, 31, 31, 3, nil, 32, nil, 32, nil, 12, 32, 32, 35, 35, nil, 3, 3, 3, 3, 3, 3, 16, 16, 36, 36, nil, nil, nil, 16, 16, 16, nil, nil, nil, nil, 16, nil, 16, 16, 16, 23, 23, 23, 23, 23, 23, 34, 34, nil, nil, nil, nil, nil, nil, 34, nil, nil, nil, nil, nil, nil, nil, nil, 34]}} } }, states: [:initial], stats: {initial: {states: 37, classes: 51, accepting_states: 22}}, diagnostics: [] } })
18
+ def scan_one
19
+ if @__flexr_generated_scan_state != @state
20
+ machine = self.class.__flexr_compiled.machines.fetch(@state)
21
+ @__flexr_generated_scan_state = @state
22
+ @__flexr_generated_scan_dfa = machine.dfa
23
+ @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
24
+ @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
25
+ end
26
+ dfa = @__flexr_generated_scan_dfa
27
+ return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
28
+ position = byte_pos
29
+ return nil unless valid_utf8_at?(position)
30
+ return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
31
+ cursor = position
32
+ current = dfa.start
33
+ direct = dfa.direct
34
+ direct_nxt = direct&.fetch(:nxt)
35
+ direct_classes = direct&.fetch(:classes)
36
+ ec = dfa.ec
37
+ source = buffer.stable_source
38
+ guarded_steps = scan_steps_guarded?
39
+ token_start = more_text_start || position
40
+ token_limit = max_token_size
41
+ check_token_limit = token_limit_required?
42
+ best = nil
43
+ accel_mode = @accel_mode
44
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
45
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
46
+ if accelerate
47
+ region = __flexr_generated_acceleration_region(dfa, current)
48
+ acceptance = dfa.accepts[current].first
49
+ token_rule = acceptance&.rule_index
50
+ accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
51
+ if accelerated_end && accelerated_end > cursor
52
+ cursor = accelerated_end
53
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
54
+ next
55
+ end
56
+ end
57
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
58
+ consume_step! if guarded_steps
59
+ current = if direct_nxt
60
+ class_id = ec[byte]
61
+ value = direct_nxt[(current * direct_classes) + class_id]
62
+ value >= 0 ? value : nil
63
+ else
64
+ dfa.transition(current, byte)
65
+ end
66
+ break unless current
67
+ cursor += 1
68
+ resource_acceptance = dfa.accepts[current].first
69
+ resource_rule = resource_acceptance&.rule_index
70
+ if check_token_limit
71
+ size = cursor - token_start
72
+ defer_token_size_check!(size, rule: resource_rule) if size > token_limit
73
+ end
74
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
75
+ end
76
+ best
77
+ end
78
+ def __flexr_generated_scan_simple(dfa, position)
79
+ current = dfa.start
80
+ direct = dfa.direct
81
+ direct_nxt = direct&.fetch(:nxt)
82
+ direct_classes = direct&.fetch(:classes)
83
+ ec = dfa.ec
84
+ accepts = dfa.accepts
85
+ rules = self.class.__flexr_rules
86
+ source = buffer.stable_source
87
+ guarded_steps = scan_steps_guarded?
88
+ token_start = more_text_start || position
89
+ token_limit = max_token_size
90
+ check_token_limit = token_limit_required?
91
+ cursor = position
92
+ best = nil
93
+ accel_mode = @accel_mode
94
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
95
+ if source && !guarded_steps && !check_token_limit && !accelerate
96
+ while cursor < source.bytesize
97
+ byte = source.getbyte(cursor)
98
+ current = if direct_nxt
99
+ class_id = ec[byte]
100
+ value = direct_nxt[(current * direct_classes) + class_id]
101
+ value >= 0 ? value : nil
102
+ else
103
+ dfa.transition(current, byte)
104
+ end
105
+ break unless current
106
+ cursor += 1
107
+ acceptance = accepts[current].first
108
+ next unless acceptance
109
+ rule = rules.fetch(acceptance.rule_index)
110
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
111
+ best.rule = rule
112
+ best.start_pos = position
113
+ best.end_pos = cursor
114
+ best.total_end_pos = cursor
115
+ end
116
+ return best
117
+ end
118
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
119
+ if accelerate
120
+ region = __flexr_generated_acceleration_region(dfa, current)
121
+ acceptance = accepts[current].first
122
+ accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
123
+ if accelerated_end && accelerated_end > cursor
124
+ cursor = accelerated_end
125
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
126
+ next
127
+ end
128
+ end
129
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
130
+ consume_step! if guarded_steps
131
+ current = if direct_nxt
132
+ class_id = ec[byte]
133
+ value = direct_nxt[(current * direct_classes) + class_id]
134
+ value >= 0 ? value : nil
135
+ else
136
+ dfa.transition(current, byte)
137
+ end
138
+ break unless current
139
+ cursor += 1
140
+ acceptance = accepts[current].first
141
+ if check_token_limit
142
+ size = cursor - token_start
143
+ defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
144
+ end
145
+ next unless acceptance
146
+ rule = rules.fetch(acceptance.rule_index)
147
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
148
+ best.rule = rule
149
+ best.start_pos = position
150
+ best.end_pos = cursor
151
+ best.total_end_pos = cursor
152
+ end
153
+ best
154
+ end
155
+ def __flexr_generated_acceleration_region(dfa, state)
156
+ @__flexr_generated_accel_regions ||= {}
157
+ regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
158
+ region = regions[state]
159
+ return unless region
160
+ return if @__flexr_generated_disabled_auto_accel&.key?(region)
161
+ return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
162
+ return if dfa.accepts[state].any? do |acceptance|
163
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
164
+ acceptance.bol_only || acceptance.end_anchor || rule.trailing
165
+ end
166
+ region
167
+ end
168
+ def __flexr_generated_accelerate(region, position, token_rule, token_start)
169
+ mode = @accel_mode
170
+ cursor = position
171
+ matched = false
172
+ loop do
173
+ if cursor >= buffer.bytesize
174
+ return matched ? cursor : nil if buffer.eof_loaded?
175
+ return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
176
+ end
177
+ segment = buffer.source
178
+ segment_position = cursor - buffer.base_offset
179
+ if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
180
+ regexp = region.utf8_regexp
181
+ else
182
+ segment = segment.b unless segment.encoding == ::Encoding::BINARY
183
+ regexp = region.regexp
184
+ end
185
+ length = begin
186
+ if %i[strscan auto].include?(mode) && defined?(::StringScanner)
187
+ @__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
188
+ @__flexr_generated_accel_scanner.string = segment
189
+ @__flexr_generated_accel_scanner.pos = segment_position
190
+ @__flexr_generated_accel_scanner.skip(regexp)
191
+ else
192
+ match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
193
+ match && match.begin(0).zero? ? match[0].bytesize : nil
194
+ end
195
+ rescue ArgumentError
196
+ nil
197
+ end
198
+ __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
199
+ return matched ? cursor : nil unless length&.positive?
200
+ matched = true
201
+ cursor += length
202
+ consume_step!(length)
203
+ defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
204
+ return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
205
+ end
206
+ end
207
+ def __flexr_generated_record_auto_accel_miss(region)
208
+ @__flexr_generated_auto_accel_misses ||= Hash.new(0)
209
+ @__flexr_generated_auto_accel_misses[region] += 1
210
+ @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
211
+ @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
212
+ return if @__flexr_generated_auto_accel_misses[region] < 3
213
+ @__flexr_generated_disabled_auto_accel ||= {}
214
+ @__flexr_generated_disabled_auto_accel[region] = true
215
+ end
216
+ def __flexr_generated_fast_path?(dfa)
217
+ @__flexr_generated_fast_paths ||= {}
218
+ return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
219
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
220
+ @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
221
+ self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
222
+ end
223
+ def __flexr_generated_simple_fast_path?(dfa)
224
+ @__flexr_generated_simple_fast_paths ||= {}
225
+ return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
226
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
227
+ @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
228
+ rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
229
+ end
230
+ end
231
+ def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
232
+ dfa.accepts[state].each do |acceptance|
233
+ next if acceptance.bol_only && !beginning_of_line?
234
+ next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
235
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
236
+ next if best && cursor < best.total_end_pos
237
+ next if best && cursor == best.total_end_pos &&
238
+ rule.index > best.rule.index
239
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
240
+ best.rule = rule
241
+ best.start_pos = start_position
242
+ best.end_pos = cursor
243
+ best.total_end_pos = cursor
244
+ end
245
+ best
246
+ end
247
+ def __flexr_generated_execute(rule)
248
+ case rule.index
249
+ when 0
250
+ when 1
251
+ when 2
252
+ emit(:NAME, text)
253
+ when 3
254
+ emit(:NUMBER, text)
255
+ when 4
256
+ emit(:STRING, text)
257
+ when 5
258
+ emit(:SPREAD, text)
259
+ when 6
260
+ emit(:LBRACE, text)
261
+ when 7
262
+ emit(:RBRACE, text)
263
+ when 8
264
+ emit(:LPAREN, text)
265
+ when 9
266
+ emit(:RPAREN, text)
267
+ when 10
268
+ emit(:LBRACKET, text)
269
+ when 11
270
+ emit(:RBRACKET, text)
271
+ when 12
272
+ emit(:COLON, text)
273
+ when 13
274
+ emit(:EQUAL, text)
275
+ when 14
276
+ emit(:BANG, text)
277
+ when 15
278
+ emit(:DOLLAR, text)
279
+ when 16
280
+ emit(:AT, text)
281
+ when 17
282
+ emit(:PIPE, text)
283
+ when 18
284
+ emit(:AMP, text)
285
+ else
286
+ instance_exec(&rule.action)
287
+ end
288
+ end
289
+
290
+
291
+
292
+
293
+
294
+
295
+
296
+
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+
307
+
308
+
309
+ end
310
+ end