flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Generated by flexr. DO NOT EDIT.
|
|
3
|
+
# source: examples/calculator/lexer.flexr.rb
|
|
4
|
+
# spec-digest: <normalized>
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
|
+
# unicode: 15.1.0
|
|
8
|
+
# backend: table
|
|
9
|
+
# compiled: true
|
|
10
|
+
# eval: false
|
|
11
|
+
# standalone: false
|
|
12
|
+
|
|
13
|
+
require "flexr"
|
|
14
|
+
|
|
15
|
+
module CalculatorExample
|
|
16
|
+
class Lexer < Flexr::Lexer
|
|
17
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/==/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/=/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/if/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[a-z_][a-z0-9_]*/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/[0-9]+/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\+/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:EQ, :ASSIGN, :IF, :IDENT, :INTEGER, :PLUS], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [[6, 0, false, false]], [[5, 0, false, false]], [[2, 0, false, false]], [[4, 0, false, false]], [[4, 0, false, false]], [[1, 0, false, false]], [[3, 0, false, false], [4, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 14, 15, 15, 15, 15, 15, 16, 17, 17, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20], class_count: 21, state_count: 9, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6], packed: {base: [0, 1, 0, 1, 1, 11, 22, 0, 33], default: [nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 1, 1, 1, 1, 1, 1, 2, nil, 3, 3, 4, 7, 5, nil, 5, 5, 5, 6, 5, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5, 5, nil, nil, nil, 5, nil, 5, 8, 5, 5, 5, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5], check: [nil, 0, 1, 0, 1, 0, 1, 0, nil, 0, 3, 0, 4, 0, nil, 0, 0, 0, 0, 0, 5, nil, nil, nil, 5, nil, 5, 5, 5, 5, 5, 6, nil, nil, nil, 6, nil, 6, 6, 6, 6, 6, 8, nil, nil, nil, 8, nil, 8, 8, 8, 8, 8]}} } }, states: [:initial], stats: {initial: {states: 9, classes: 21, accepting_states: 8}}, diagnostics: [] } })
|
|
18
|
+
def scan_one
|
|
19
|
+
if @__flexr_generated_scan_state != @state
|
|
20
|
+
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
21
|
+
@__flexr_generated_scan_state = @state
|
|
22
|
+
@__flexr_generated_scan_dfa = machine.dfa
|
|
23
|
+
@__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
|
|
24
|
+
@__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
|
|
25
|
+
end
|
|
26
|
+
dfa = @__flexr_generated_scan_dfa
|
|
27
|
+
return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
|
|
28
|
+
position = byte_pos
|
|
29
|
+
return nil unless valid_utf8_at?(position)
|
|
30
|
+
return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
|
|
31
|
+
cursor = position
|
|
32
|
+
current = dfa.start
|
|
33
|
+
direct = dfa.direct
|
|
34
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
35
|
+
direct_classes = direct&.fetch(:classes)
|
|
36
|
+
ec = dfa.ec
|
|
37
|
+
source = buffer.stable_source
|
|
38
|
+
guarded_steps = scan_steps_guarded?
|
|
39
|
+
token_start = more_text_start || position
|
|
40
|
+
token_limit = max_token_size
|
|
41
|
+
check_token_limit = token_limit_required?
|
|
42
|
+
best = nil
|
|
43
|
+
accel_mode = @accel_mode
|
|
44
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
45
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
46
|
+
if accelerate
|
|
47
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
48
|
+
acceptance = dfa.accepts[current].first
|
|
49
|
+
token_rule = acceptance&.rule_index
|
|
50
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
|
|
51
|
+
if accelerated_end && accelerated_end > cursor
|
|
52
|
+
cursor = accelerated_end
|
|
53
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
58
|
+
consume_step! if guarded_steps
|
|
59
|
+
current = if direct_nxt
|
|
60
|
+
class_id = ec[byte]
|
|
61
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
62
|
+
value >= 0 ? value : nil
|
|
63
|
+
else
|
|
64
|
+
dfa.transition(current, byte)
|
|
65
|
+
end
|
|
66
|
+
break unless current
|
|
67
|
+
cursor += 1
|
|
68
|
+
resource_acceptance = dfa.accepts[current].first
|
|
69
|
+
resource_rule = resource_acceptance&.rule_index
|
|
70
|
+
if check_token_limit
|
|
71
|
+
size = cursor - token_start
|
|
72
|
+
defer_token_size_check!(size, rule: resource_rule) if size > token_limit
|
|
73
|
+
end
|
|
74
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
75
|
+
end
|
|
76
|
+
best
|
|
77
|
+
end
|
|
78
|
+
def __flexr_generated_scan_simple(dfa, position)
|
|
79
|
+
current = dfa.start
|
|
80
|
+
direct = dfa.direct
|
|
81
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
82
|
+
direct_classes = direct&.fetch(:classes)
|
|
83
|
+
ec = dfa.ec
|
|
84
|
+
accepts = dfa.accepts
|
|
85
|
+
rules = self.class.__flexr_rules
|
|
86
|
+
source = buffer.stable_source
|
|
87
|
+
guarded_steps = scan_steps_guarded?
|
|
88
|
+
token_start = more_text_start || position
|
|
89
|
+
token_limit = max_token_size
|
|
90
|
+
check_token_limit = token_limit_required?
|
|
91
|
+
cursor = position
|
|
92
|
+
best = nil
|
|
93
|
+
accel_mode = @accel_mode
|
|
94
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
95
|
+
if source && !guarded_steps && !check_token_limit && !accelerate
|
|
96
|
+
while cursor < source.bytesize
|
|
97
|
+
byte = source.getbyte(cursor)
|
|
98
|
+
current = if direct_nxt
|
|
99
|
+
class_id = ec[byte]
|
|
100
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
101
|
+
value >= 0 ? value : nil
|
|
102
|
+
else
|
|
103
|
+
dfa.transition(current, byte)
|
|
104
|
+
end
|
|
105
|
+
break unless current
|
|
106
|
+
cursor += 1
|
|
107
|
+
acceptance = accepts[current].first
|
|
108
|
+
next unless acceptance
|
|
109
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
110
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
111
|
+
best.rule = rule
|
|
112
|
+
best.start_pos = position
|
|
113
|
+
best.end_pos = cursor
|
|
114
|
+
best.total_end_pos = cursor
|
|
115
|
+
end
|
|
116
|
+
return best
|
|
117
|
+
end
|
|
118
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
119
|
+
if accelerate
|
|
120
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
121
|
+
acceptance = accepts[current].first
|
|
122
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
|
|
123
|
+
if accelerated_end && accelerated_end > cursor
|
|
124
|
+
cursor = accelerated_end
|
|
125
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
126
|
+
next
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
130
|
+
consume_step! if guarded_steps
|
|
131
|
+
current = if direct_nxt
|
|
132
|
+
class_id = ec[byte]
|
|
133
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
134
|
+
value >= 0 ? value : nil
|
|
135
|
+
else
|
|
136
|
+
dfa.transition(current, byte)
|
|
137
|
+
end
|
|
138
|
+
break unless current
|
|
139
|
+
cursor += 1
|
|
140
|
+
acceptance = accepts[current].first
|
|
141
|
+
if check_token_limit
|
|
142
|
+
size = cursor - token_start
|
|
143
|
+
defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
|
|
144
|
+
end
|
|
145
|
+
next unless acceptance
|
|
146
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
147
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
148
|
+
best.rule = rule
|
|
149
|
+
best.start_pos = position
|
|
150
|
+
best.end_pos = cursor
|
|
151
|
+
best.total_end_pos = cursor
|
|
152
|
+
end
|
|
153
|
+
best
|
|
154
|
+
end
|
|
155
|
+
def __flexr_generated_acceleration_region(dfa, state)
|
|
156
|
+
@__flexr_generated_accel_regions ||= {}
|
|
157
|
+
regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
|
|
158
|
+
region = regions[state]
|
|
159
|
+
return unless region
|
|
160
|
+
return if @__flexr_generated_disabled_auto_accel&.key?(region)
|
|
161
|
+
return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
|
|
162
|
+
return if dfa.accepts[state].any? do |acceptance|
|
|
163
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
164
|
+
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
165
|
+
end
|
|
166
|
+
region
|
|
167
|
+
end
|
|
168
|
+
def __flexr_generated_accelerate(region, position, token_rule, token_start)
|
|
169
|
+
mode = @accel_mode
|
|
170
|
+
cursor = position
|
|
171
|
+
matched = false
|
|
172
|
+
loop do
|
|
173
|
+
if cursor >= buffer.bytesize
|
|
174
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
175
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
176
|
+
end
|
|
177
|
+
segment = buffer.source
|
|
178
|
+
segment_position = cursor - buffer.base_offset
|
|
179
|
+
if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
|
|
180
|
+
regexp = region.utf8_regexp
|
|
181
|
+
else
|
|
182
|
+
segment = segment.b unless segment.encoding == ::Encoding::BINARY
|
|
183
|
+
regexp = region.regexp
|
|
184
|
+
end
|
|
185
|
+
length = begin
|
|
186
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
187
|
+
@__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
|
|
188
|
+
@__flexr_generated_accel_scanner.string = segment
|
|
189
|
+
@__flexr_generated_accel_scanner.pos = segment_position
|
|
190
|
+
@__flexr_generated_accel_scanner.skip(regexp)
|
|
191
|
+
else
|
|
192
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
193
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
194
|
+
end
|
|
195
|
+
rescue ArgumentError
|
|
196
|
+
nil
|
|
197
|
+
end
|
|
198
|
+
__flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
|
|
199
|
+
return matched ? cursor : nil unless length&.positive?
|
|
200
|
+
matched = true
|
|
201
|
+
cursor += length
|
|
202
|
+
consume_step!(length)
|
|
203
|
+
defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
|
|
204
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
def __flexr_generated_record_auto_accel_miss(region)
|
|
208
|
+
@__flexr_generated_auto_accel_misses ||= Hash.new(0)
|
|
209
|
+
@__flexr_generated_auto_accel_misses[region] += 1
|
|
210
|
+
@__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
|
|
211
|
+
@__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
|
|
212
|
+
return if @__flexr_generated_auto_accel_misses[region] < 3
|
|
213
|
+
@__flexr_generated_disabled_auto_accel ||= {}
|
|
214
|
+
@__flexr_generated_disabled_auto_accel[region] = true
|
|
215
|
+
end
|
|
216
|
+
def __flexr_generated_fast_path?(dfa)
|
|
217
|
+
@__flexr_generated_fast_paths ||= {}
|
|
218
|
+
return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
|
|
219
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
220
|
+
@__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
|
|
221
|
+
self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
|
|
222
|
+
end
|
|
223
|
+
def __flexr_generated_simple_fast_path?(dfa)
|
|
224
|
+
@__flexr_generated_simple_fast_paths ||= {}
|
|
225
|
+
return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
|
|
226
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
227
|
+
@__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
|
|
228
|
+
rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
|
|
232
|
+
dfa.accepts[state].each do |acceptance|
|
|
233
|
+
next if acceptance.bol_only && !beginning_of_line?
|
|
234
|
+
next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
|
|
235
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
236
|
+
next if best && cursor < best.total_end_pos
|
|
237
|
+
next if best && cursor == best.total_end_pos &&
|
|
238
|
+
rule.index > best.rule.index
|
|
239
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
240
|
+
best.rule = rule
|
|
241
|
+
best.start_pos = start_position
|
|
242
|
+
best.end_pos = cursor
|
|
243
|
+
best.total_end_pos = cursor
|
|
244
|
+
end
|
|
245
|
+
best
|
|
246
|
+
end
|
|
247
|
+
def __flexr_generated_execute(rule)
|
|
248
|
+
case rule.index
|
|
249
|
+
when 0
|
|
250
|
+
when 1
|
|
251
|
+
emit :EQ
|
|
252
|
+
when 2
|
|
253
|
+
emit :ASSIGN
|
|
254
|
+
when 3
|
|
255
|
+
emit :IF
|
|
256
|
+
when 4
|
|
257
|
+
emit :IDENT
|
|
258
|
+
when 5
|
|
259
|
+
emit :INTEGER, text.to_i
|
|
260
|
+
when 6
|
|
261
|
+
emit :PLUS
|
|
262
|
+
else
|
|
263
|
+
instance_exec(&rule.action)
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
end
|
|
276
|
+
end
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Generated by flexr. DO NOT EDIT.
|
|
3
|
+
# source: examples/graphql/lexer.flexr.rb
|
|
4
|
+
# spec-digest: <normalized>
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
|
+
# unicode: 15.1.0
|
|
8
|
+
# backend: table
|
|
9
|
+
# compiled: true
|
|
10
|
+
# eval: false
|
|
11
|
+
# standalone: false
|
|
12
|
+
|
|
13
|
+
require "flexr"
|
|
14
|
+
|
|
15
|
+
module GraphqlExample
|
|
16
|
+
class Lexer < Flexr::Lexer
|
|
17
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n,]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/#[^\n]*/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/[A-Za-z_][A-Za-z_0-9]*/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: [:emit, :NAME], states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: [:emit, :NUMBER], states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/"(?:\\.|[^"\\\r\n])*"/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: [:emit, :STRING], states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\.\.\./], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: [:emit, :SPREAD], states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\{/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: [:emit, :LBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\}/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: [:emit, :RBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\(/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: [:emit, :LPAREN], states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\)/], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: [:emit, :RPAREN], states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/\[/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: [:emit, :LBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 11, patterns: [/\]/], pattern_conditions: [[11, 0, false, false]], trailing: nil, action: [:emit, :RBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 12, patterns: [/:/], pattern_conditions: [[12, 0, false, false]], trailing: nil, action: [:emit, :COLON], states: [:initial], bol_only: false, end_anchor: nil }, { index: 13, patterns: [/=/], pattern_conditions: [[13, 0, false, false]], trailing: nil, action: [:emit, :EQUAL], states: [:initial], bol_only: false, end_anchor: nil }, { index: 14, patterns: [/!/], pattern_conditions: [[14, 0, false, false]], trailing: nil, action: [:emit, :BANG], states: [:initial], bol_only: false, end_anchor: nil }, { index: 15, patterns: [/\$/], pattern_conditions: [[15, 0, false, false]], trailing: nil, action: [:emit, :DOLLAR], states: [:initial], bol_only: false, end_anchor: nil }, { index: 16, patterns: [/@/], pattern_conditions: [[16, 0, false, false]], trailing: nil, action: [:emit, :AT], states: [:initial], bol_only: false, end_anchor: nil }, { index: 17, patterns: [/\|/], pattern_conditions: [[17, 0, false, false]], trailing: nil, action: [:emit, :PIPE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 18, patterns: [/&/], pattern_conditions: [[18, 0, false, false]], trailing: nil, action: [:emit, :AMP], states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:NAME, :STRING, :NUMBER, :SPREAD, :LBRACE, :RBRACE, :LPAREN, :RPAREN, :LBRACKET, :RBRACKET, :COLON, :EQUAL, :BANG, :DOLLAR, :AT, :PIPE, :AMP], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [[14, 0, false, false]], [], [[1, 0, false, false]], [[15, 0, false, false]], [[18, 0, false, false]], [[8, 0, false, false]], [[9, 0, false, false]], [], [], [[3, 0, false, false]], [[3, 0, false, false]], [[12, 0, false, false]], [[13, 0, false, false]], [[16, 0, false, false]], [[2, 0, false, false]], [[10, 0, false, false]], [[11, 0, false, false]], [[6, 0, false, false]], [[17, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [], [], [], [], [], [], [], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [], [[3, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 24, 25, 25, 26, 27, 27, 28, 29, 29, 29, 29, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 35, 36, 37, 38, 38, 38, 38, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 42, 43, 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 49, 49, 49, 49, 49, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50], class_count: 51, state_count: 37, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], packed: {base: [0, 43, 0, 44, 9, 0, 0, 0, 0, 28, 1, 33, 44, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 69, 15, 17, 20, 23, 24, 25, 5, 53, 62, 0, 98, 64, 75], default: [nil, nil, nil, 3, 4, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 3, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 1, 1, nil, 1, nil, 1, 2, 3, 4, 5, nil, 6, nil, 7, 8, nil, nil, 1, 9, 10, 30, 11, 12, 13, 33, 14, nil, 15, 16, 16, 16, 17, nil, 18, nil, 16, nil, 16, 16, 16, 19, 20, 21, 1, 1, nil, 1, nil, 1, 11, 12, 22, 31, nil, nil, 27, 28, 29, nil, 3, 1, 24, 32, 31, 25, 12, 12, 4, 27, 28, nil, 32, nil, 32, 34, 34, 23, nil, 35, nil, 35, nil, 32, 36, 36, 36, 36, nil, nil, nil, 24, 25, 26, nil, 16, 16, 36, 36, nil, nil, nil, 16, 16, 16, nil, nil, nil, nil, 16, nil, 16, 16, 16, nil, nil, 24, 25, 26, nil, 34, 34, nil, nil, nil, nil, nil, nil, 32, nil, nil, nil, nil, nil, nil, nil, nil, 32], check: [nil, 0, 0, nil, 0, nil, 0, 0, 0, 0, 0, 4, 0, nil, 0, 0, nil, nil, 0, 0, 0, 10, 0, 0, 0, 30, 0, nil, 0, 0, 0, 0, 0, nil, 0, nil, 0, nil, 0, 0, 0, 0, 0, 0, 1, 1, 3, 1, 3, 1, 9, 9, 3, 11, 4, 4, 4, 4, 4, 4, 24, 1, 25, 11, 12, 26, 12, 12, 27, 28, 29, 23, 11, nil, 12, 31, 31, 3, nil, 32, nil, 32, nil, 12, 32, 32, 35, 35, nil, 3, 3, 3, 3, 3, 3, 16, 16, 36, 36, nil, nil, nil, 16, 16, 16, nil, nil, nil, nil, 16, nil, 16, 16, 16, 23, 23, 23, 23, 23, 23, 34, 34, nil, nil, nil, nil, nil, nil, 34, nil, nil, nil, nil, nil, nil, nil, nil, 34]}} } }, states: [:initial], stats: {initial: {states: 37, classes: 51, accepting_states: 22}}, diagnostics: [] } })
|
|
18
|
+
def scan_one
|
|
19
|
+
if @__flexr_generated_scan_state != @state
|
|
20
|
+
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
21
|
+
@__flexr_generated_scan_state = @state
|
|
22
|
+
@__flexr_generated_scan_dfa = machine.dfa
|
|
23
|
+
@__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
|
|
24
|
+
@__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
|
|
25
|
+
end
|
|
26
|
+
dfa = @__flexr_generated_scan_dfa
|
|
27
|
+
return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
|
|
28
|
+
position = byte_pos
|
|
29
|
+
return nil unless valid_utf8_at?(position)
|
|
30
|
+
return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
|
|
31
|
+
cursor = position
|
|
32
|
+
current = dfa.start
|
|
33
|
+
direct = dfa.direct
|
|
34
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
35
|
+
direct_classes = direct&.fetch(:classes)
|
|
36
|
+
ec = dfa.ec
|
|
37
|
+
source = buffer.stable_source
|
|
38
|
+
guarded_steps = scan_steps_guarded?
|
|
39
|
+
token_start = more_text_start || position
|
|
40
|
+
token_limit = max_token_size
|
|
41
|
+
check_token_limit = token_limit_required?
|
|
42
|
+
best = nil
|
|
43
|
+
accel_mode = @accel_mode
|
|
44
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
45
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
46
|
+
if accelerate
|
|
47
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
48
|
+
acceptance = dfa.accepts[current].first
|
|
49
|
+
token_rule = acceptance&.rule_index
|
|
50
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
|
|
51
|
+
if accelerated_end && accelerated_end > cursor
|
|
52
|
+
cursor = accelerated_end
|
|
53
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
58
|
+
consume_step! if guarded_steps
|
|
59
|
+
current = if direct_nxt
|
|
60
|
+
class_id = ec[byte]
|
|
61
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
62
|
+
value >= 0 ? value : nil
|
|
63
|
+
else
|
|
64
|
+
dfa.transition(current, byte)
|
|
65
|
+
end
|
|
66
|
+
break unless current
|
|
67
|
+
cursor += 1
|
|
68
|
+
resource_acceptance = dfa.accepts[current].first
|
|
69
|
+
resource_rule = resource_acceptance&.rule_index
|
|
70
|
+
if check_token_limit
|
|
71
|
+
size = cursor - token_start
|
|
72
|
+
defer_token_size_check!(size, rule: resource_rule) if size > token_limit
|
|
73
|
+
end
|
|
74
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
75
|
+
end
|
|
76
|
+
best
|
|
77
|
+
end
|
|
78
|
+
def __flexr_generated_scan_simple(dfa, position)
|
|
79
|
+
current = dfa.start
|
|
80
|
+
direct = dfa.direct
|
|
81
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
82
|
+
direct_classes = direct&.fetch(:classes)
|
|
83
|
+
ec = dfa.ec
|
|
84
|
+
accepts = dfa.accepts
|
|
85
|
+
rules = self.class.__flexr_rules
|
|
86
|
+
source = buffer.stable_source
|
|
87
|
+
guarded_steps = scan_steps_guarded?
|
|
88
|
+
token_start = more_text_start || position
|
|
89
|
+
token_limit = max_token_size
|
|
90
|
+
check_token_limit = token_limit_required?
|
|
91
|
+
cursor = position
|
|
92
|
+
best = nil
|
|
93
|
+
accel_mode = @accel_mode
|
|
94
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
95
|
+
if source && !guarded_steps && !check_token_limit && !accelerate
|
|
96
|
+
while cursor < source.bytesize
|
|
97
|
+
byte = source.getbyte(cursor)
|
|
98
|
+
current = if direct_nxt
|
|
99
|
+
class_id = ec[byte]
|
|
100
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
101
|
+
value >= 0 ? value : nil
|
|
102
|
+
else
|
|
103
|
+
dfa.transition(current, byte)
|
|
104
|
+
end
|
|
105
|
+
break unless current
|
|
106
|
+
cursor += 1
|
|
107
|
+
acceptance = accepts[current].first
|
|
108
|
+
next unless acceptance
|
|
109
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
110
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
111
|
+
best.rule = rule
|
|
112
|
+
best.start_pos = position
|
|
113
|
+
best.end_pos = cursor
|
|
114
|
+
best.total_end_pos = cursor
|
|
115
|
+
end
|
|
116
|
+
return best
|
|
117
|
+
end
|
|
118
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
119
|
+
if accelerate
|
|
120
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
121
|
+
acceptance = accepts[current].first
|
|
122
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
|
|
123
|
+
if accelerated_end && accelerated_end > cursor
|
|
124
|
+
cursor = accelerated_end
|
|
125
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
126
|
+
next
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
130
|
+
consume_step! if guarded_steps
|
|
131
|
+
current = if direct_nxt
|
|
132
|
+
class_id = ec[byte]
|
|
133
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
134
|
+
value >= 0 ? value : nil
|
|
135
|
+
else
|
|
136
|
+
dfa.transition(current, byte)
|
|
137
|
+
end
|
|
138
|
+
break unless current
|
|
139
|
+
cursor += 1
|
|
140
|
+
acceptance = accepts[current].first
|
|
141
|
+
if check_token_limit
|
|
142
|
+
size = cursor - token_start
|
|
143
|
+
defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
|
|
144
|
+
end
|
|
145
|
+
next unless acceptance
|
|
146
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
147
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
148
|
+
best.rule = rule
|
|
149
|
+
best.start_pos = position
|
|
150
|
+
best.end_pos = cursor
|
|
151
|
+
best.total_end_pos = cursor
|
|
152
|
+
end
|
|
153
|
+
best
|
|
154
|
+
end
|
|
155
|
+
def __flexr_generated_acceleration_region(dfa, state)
|
|
156
|
+
@__flexr_generated_accel_regions ||= {}
|
|
157
|
+
regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
|
|
158
|
+
region = regions[state]
|
|
159
|
+
return unless region
|
|
160
|
+
return if @__flexr_generated_disabled_auto_accel&.key?(region)
|
|
161
|
+
return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
|
|
162
|
+
return if dfa.accepts[state].any? do |acceptance|
|
|
163
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
164
|
+
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
165
|
+
end
|
|
166
|
+
region
|
|
167
|
+
end
|
|
168
|
+
def __flexr_generated_accelerate(region, position, token_rule, token_start)
|
|
169
|
+
mode = @accel_mode
|
|
170
|
+
cursor = position
|
|
171
|
+
matched = false
|
|
172
|
+
loop do
|
|
173
|
+
if cursor >= buffer.bytesize
|
|
174
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
175
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
176
|
+
end
|
|
177
|
+
segment = buffer.source
|
|
178
|
+
segment_position = cursor - buffer.base_offset
|
|
179
|
+
if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
|
|
180
|
+
regexp = region.utf8_regexp
|
|
181
|
+
else
|
|
182
|
+
segment = segment.b unless segment.encoding == ::Encoding::BINARY
|
|
183
|
+
regexp = region.regexp
|
|
184
|
+
end
|
|
185
|
+
length = begin
|
|
186
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
187
|
+
@__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
|
|
188
|
+
@__flexr_generated_accel_scanner.string = segment
|
|
189
|
+
@__flexr_generated_accel_scanner.pos = segment_position
|
|
190
|
+
@__flexr_generated_accel_scanner.skip(regexp)
|
|
191
|
+
else
|
|
192
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
193
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
194
|
+
end
|
|
195
|
+
rescue ArgumentError
|
|
196
|
+
nil
|
|
197
|
+
end
|
|
198
|
+
__flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
|
|
199
|
+
return matched ? cursor : nil unless length&.positive?
|
|
200
|
+
matched = true
|
|
201
|
+
cursor += length
|
|
202
|
+
consume_step!(length)
|
|
203
|
+
defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
|
|
204
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
def __flexr_generated_record_auto_accel_miss(region)
|
|
208
|
+
@__flexr_generated_auto_accel_misses ||= Hash.new(0)
|
|
209
|
+
@__flexr_generated_auto_accel_misses[region] += 1
|
|
210
|
+
@__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
|
|
211
|
+
@__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
|
|
212
|
+
return if @__flexr_generated_auto_accel_misses[region] < 3
|
|
213
|
+
@__flexr_generated_disabled_auto_accel ||= {}
|
|
214
|
+
@__flexr_generated_disabled_auto_accel[region] = true
|
|
215
|
+
end
|
|
216
|
+
def __flexr_generated_fast_path?(dfa)
|
|
217
|
+
@__flexr_generated_fast_paths ||= {}
|
|
218
|
+
return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
|
|
219
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
220
|
+
@__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
|
|
221
|
+
self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
|
|
222
|
+
end
|
|
223
|
+
def __flexr_generated_simple_fast_path?(dfa)
|
|
224
|
+
@__flexr_generated_simple_fast_paths ||= {}
|
|
225
|
+
return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
|
|
226
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
227
|
+
@__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
|
|
228
|
+
rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
|
|
232
|
+
dfa.accepts[state].each do |acceptance|
|
|
233
|
+
next if acceptance.bol_only && !beginning_of_line?
|
|
234
|
+
next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
|
|
235
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
236
|
+
next if best && cursor < best.total_end_pos
|
|
237
|
+
next if best && cursor == best.total_end_pos &&
|
|
238
|
+
rule.index > best.rule.index
|
|
239
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
240
|
+
best.rule = rule
|
|
241
|
+
best.start_pos = start_position
|
|
242
|
+
best.end_pos = cursor
|
|
243
|
+
best.total_end_pos = cursor
|
|
244
|
+
end
|
|
245
|
+
best
|
|
246
|
+
end
|
|
247
|
+
def __flexr_generated_execute(rule)
|
|
248
|
+
case rule.index
|
|
249
|
+
when 0
|
|
250
|
+
when 1
|
|
251
|
+
when 2
|
|
252
|
+
emit(:NAME, text)
|
|
253
|
+
when 3
|
|
254
|
+
emit(:NUMBER, text)
|
|
255
|
+
when 4
|
|
256
|
+
emit(:STRING, text)
|
|
257
|
+
when 5
|
|
258
|
+
emit(:SPREAD, text)
|
|
259
|
+
when 6
|
|
260
|
+
emit(:LBRACE, text)
|
|
261
|
+
when 7
|
|
262
|
+
emit(:RBRACE, text)
|
|
263
|
+
when 8
|
|
264
|
+
emit(:LPAREN, text)
|
|
265
|
+
when 9
|
|
266
|
+
emit(:RPAREN, text)
|
|
267
|
+
when 10
|
|
268
|
+
emit(:LBRACKET, text)
|
|
269
|
+
when 11
|
|
270
|
+
emit(:RBRACKET, text)
|
|
271
|
+
when 12
|
|
272
|
+
emit(:COLON, text)
|
|
273
|
+
when 13
|
|
274
|
+
emit(:EQUAL, text)
|
|
275
|
+
when 14
|
|
276
|
+
emit(:BANG, text)
|
|
277
|
+
when 15
|
|
278
|
+
emit(:DOLLAR, text)
|
|
279
|
+
when 16
|
|
280
|
+
emit(:AT, text)
|
|
281
|
+
when 17
|
|
282
|
+
emit(:PIPE, text)
|
|
283
|
+
when 18
|
|
284
|
+
emit(:AMP, text)
|
|
285
|
+
else
|
|
286
|
+
instance_exec(&rule.action)
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
end
|
|
310
|
+
end
|