flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Generated by flexr. DO NOT EDIT.
|
|
3
|
+
# source: examples/with_lrama/lexer.flexr.rb
|
|
4
|
+
# spec-digest: <normalized>
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
|
+
# unicode: 15.1.0
|
|
8
|
+
# backend: table
|
|
9
|
+
# compiled: true
|
|
10
|
+
# eval: false
|
|
11
|
+
# standalone: false
|
|
12
|
+
|
|
13
|
+
require "flexr"
|
|
14
|
+
|
|
15
|
+
module WithLrama
|
|
16
|
+
class LramaLexer < Flexr::Lexer
|
|
17
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/[0-9]+/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: [:emit, :INTEGER], states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/-/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: [:emit, :MINUS], states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:INTEGER, :MINUS], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [[2, 0, false, false]], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10], class_count: 11, state_count: 4, start: 0, rule_ids: [0, 1, 2], packed: {base: [0, 1, 0, 1], default: [nil, nil, nil, nil], next: [nil, 1, 1, 1, 1, 1, 1, 2, nil, 3, 3], check: [nil, 0, 1, 0, 1, 0, 1, 0, nil, 0, 3]}} } }, states: [:initial], stats: {initial: {states: 4, classes: 11, accepting_states: 3}}, diagnostics: [] } })
|
|
18
|
+
def scan_one
|
|
19
|
+
if @__flexr_generated_scan_state != @state
|
|
20
|
+
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
21
|
+
@__flexr_generated_scan_state = @state
|
|
22
|
+
@__flexr_generated_scan_dfa = machine.dfa
|
|
23
|
+
@__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
|
|
24
|
+
@__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
|
|
25
|
+
end
|
|
26
|
+
dfa = @__flexr_generated_scan_dfa
|
|
27
|
+
return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
|
|
28
|
+
position = byte_pos
|
|
29
|
+
return nil unless valid_utf8_at?(position)
|
|
30
|
+
return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
|
|
31
|
+
cursor = position
|
|
32
|
+
current = dfa.start
|
|
33
|
+
direct = dfa.direct
|
|
34
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
35
|
+
direct_classes = direct&.fetch(:classes)
|
|
36
|
+
ec = dfa.ec
|
|
37
|
+
source = buffer.stable_source
|
|
38
|
+
guarded_steps = scan_steps_guarded?
|
|
39
|
+
token_start = more_text_start || position
|
|
40
|
+
token_limit = max_token_size
|
|
41
|
+
check_token_limit = token_limit_required?
|
|
42
|
+
best = nil
|
|
43
|
+
accel_mode = @accel_mode
|
|
44
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
45
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
46
|
+
if accelerate
|
|
47
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
48
|
+
acceptance = dfa.accepts[current].first
|
|
49
|
+
token_rule = acceptance&.rule_index
|
|
50
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
|
|
51
|
+
if accelerated_end && accelerated_end > cursor
|
|
52
|
+
cursor = accelerated_end
|
|
53
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
58
|
+
consume_step! if guarded_steps
|
|
59
|
+
current = if direct_nxt
|
|
60
|
+
class_id = ec[byte]
|
|
61
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
62
|
+
value >= 0 ? value : nil
|
|
63
|
+
else
|
|
64
|
+
dfa.transition(current, byte)
|
|
65
|
+
end
|
|
66
|
+
break unless current
|
|
67
|
+
cursor += 1
|
|
68
|
+
resource_acceptance = dfa.accepts[current].first
|
|
69
|
+
resource_rule = resource_acceptance&.rule_index
|
|
70
|
+
if check_token_limit
|
|
71
|
+
size = cursor - token_start
|
|
72
|
+
defer_token_size_check!(size, rule: resource_rule) if size > token_limit
|
|
73
|
+
end
|
|
74
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
75
|
+
end
|
|
76
|
+
best
|
|
77
|
+
end
|
|
78
|
+
def __flexr_generated_scan_simple(dfa, position)
|
|
79
|
+
current = dfa.start
|
|
80
|
+
direct = dfa.direct
|
|
81
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
82
|
+
direct_classes = direct&.fetch(:classes)
|
|
83
|
+
ec = dfa.ec
|
|
84
|
+
accepts = dfa.accepts
|
|
85
|
+
rules = self.class.__flexr_rules
|
|
86
|
+
source = buffer.stable_source
|
|
87
|
+
guarded_steps = scan_steps_guarded?
|
|
88
|
+
token_start = more_text_start || position
|
|
89
|
+
token_limit = max_token_size
|
|
90
|
+
check_token_limit = token_limit_required?
|
|
91
|
+
cursor = position
|
|
92
|
+
best = nil
|
|
93
|
+
accel_mode = @accel_mode
|
|
94
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
95
|
+
if source && !guarded_steps && !check_token_limit && !accelerate
|
|
96
|
+
while cursor < source.bytesize
|
|
97
|
+
byte = source.getbyte(cursor)
|
|
98
|
+
current = if direct_nxt
|
|
99
|
+
class_id = ec[byte]
|
|
100
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
101
|
+
value >= 0 ? value : nil
|
|
102
|
+
else
|
|
103
|
+
dfa.transition(current, byte)
|
|
104
|
+
end
|
|
105
|
+
break unless current
|
|
106
|
+
cursor += 1
|
|
107
|
+
acceptance = accepts[current].first
|
|
108
|
+
next unless acceptance
|
|
109
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
110
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
111
|
+
best.rule = rule
|
|
112
|
+
best.start_pos = position
|
|
113
|
+
best.end_pos = cursor
|
|
114
|
+
best.total_end_pos = cursor
|
|
115
|
+
end
|
|
116
|
+
return best
|
|
117
|
+
end
|
|
118
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
119
|
+
if accelerate
|
|
120
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
121
|
+
acceptance = accepts[current].first
|
|
122
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
|
|
123
|
+
if accelerated_end && accelerated_end > cursor
|
|
124
|
+
cursor = accelerated_end
|
|
125
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
126
|
+
next
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
130
|
+
consume_step! if guarded_steps
|
|
131
|
+
current = if direct_nxt
|
|
132
|
+
class_id = ec[byte]
|
|
133
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
134
|
+
value >= 0 ? value : nil
|
|
135
|
+
else
|
|
136
|
+
dfa.transition(current, byte)
|
|
137
|
+
end
|
|
138
|
+
break unless current
|
|
139
|
+
cursor += 1
|
|
140
|
+
acceptance = accepts[current].first
|
|
141
|
+
if check_token_limit
|
|
142
|
+
size = cursor - token_start
|
|
143
|
+
defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
|
|
144
|
+
end
|
|
145
|
+
next unless acceptance
|
|
146
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
147
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
148
|
+
best.rule = rule
|
|
149
|
+
best.start_pos = position
|
|
150
|
+
best.end_pos = cursor
|
|
151
|
+
best.total_end_pos = cursor
|
|
152
|
+
end
|
|
153
|
+
best
|
|
154
|
+
end
|
|
155
|
+
def __flexr_generated_acceleration_region(dfa, state)
|
|
156
|
+
@__flexr_generated_accel_regions ||= {}
|
|
157
|
+
regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
|
|
158
|
+
region = regions[state]
|
|
159
|
+
return unless region
|
|
160
|
+
return if @__flexr_generated_disabled_auto_accel&.key?(region)
|
|
161
|
+
return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
|
|
162
|
+
return if dfa.accepts[state].any? do |acceptance|
|
|
163
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
164
|
+
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
165
|
+
end
|
|
166
|
+
region
|
|
167
|
+
end
|
|
168
|
+
def __flexr_generated_accelerate(region, position, token_rule, token_start)
|
|
169
|
+
mode = @accel_mode
|
|
170
|
+
cursor = position
|
|
171
|
+
matched = false
|
|
172
|
+
loop do
|
|
173
|
+
if cursor >= buffer.bytesize
|
|
174
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
175
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
176
|
+
end
|
|
177
|
+
segment = buffer.source
|
|
178
|
+
segment_position = cursor - buffer.base_offset
|
|
179
|
+
if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
|
|
180
|
+
regexp = region.utf8_regexp
|
|
181
|
+
else
|
|
182
|
+
segment = segment.b unless segment.encoding == ::Encoding::BINARY
|
|
183
|
+
regexp = region.regexp
|
|
184
|
+
end
|
|
185
|
+
length = begin
|
|
186
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
187
|
+
@__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
|
|
188
|
+
@__flexr_generated_accel_scanner.string = segment
|
|
189
|
+
@__flexr_generated_accel_scanner.pos = segment_position
|
|
190
|
+
@__flexr_generated_accel_scanner.skip(regexp)
|
|
191
|
+
else
|
|
192
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
193
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
194
|
+
end
|
|
195
|
+
rescue ArgumentError
|
|
196
|
+
nil
|
|
197
|
+
end
|
|
198
|
+
__flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
|
|
199
|
+
return matched ? cursor : nil unless length&.positive?
|
|
200
|
+
matched = true
|
|
201
|
+
cursor += length
|
|
202
|
+
consume_step!(length)
|
|
203
|
+
defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
|
|
204
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
def __flexr_generated_record_auto_accel_miss(region)
|
|
208
|
+
@__flexr_generated_auto_accel_misses ||= Hash.new(0)
|
|
209
|
+
@__flexr_generated_auto_accel_misses[region] += 1
|
|
210
|
+
@__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
|
|
211
|
+
@__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
|
|
212
|
+
return if @__flexr_generated_auto_accel_misses[region] < 3
|
|
213
|
+
@__flexr_generated_disabled_auto_accel ||= {}
|
|
214
|
+
@__flexr_generated_disabled_auto_accel[region] = true
|
|
215
|
+
end
|
|
216
|
+
def __flexr_generated_fast_path?(dfa)
|
|
217
|
+
@__flexr_generated_fast_paths ||= {}
|
|
218
|
+
return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
|
|
219
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
220
|
+
@__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
|
|
221
|
+
self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
|
|
222
|
+
end
|
|
223
|
+
def __flexr_generated_simple_fast_path?(dfa)
|
|
224
|
+
@__flexr_generated_simple_fast_paths ||= {}
|
|
225
|
+
return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
|
|
226
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
227
|
+
@__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
|
|
228
|
+
rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
|
|
232
|
+
dfa.accepts[state].each do |acceptance|
|
|
233
|
+
next if acceptance.bol_only && !beginning_of_line?
|
|
234
|
+
next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
|
|
235
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
236
|
+
next if best && cursor < best.total_end_pos
|
|
237
|
+
next if best && cursor == best.total_end_pos &&
|
|
238
|
+
rule.index > best.rule.index
|
|
239
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
240
|
+
best.rule = rule
|
|
241
|
+
best.start_pos = start_position
|
|
242
|
+
best.end_pos = cursor
|
|
243
|
+
best.total_end_pos = cursor
|
|
244
|
+
end
|
|
245
|
+
best
|
|
246
|
+
end
|
|
247
|
+
def __flexr_generated_execute(rule)
|
|
248
|
+
case rule.index
|
|
249
|
+
when 0
|
|
250
|
+
when 1
|
|
251
|
+
emit(:INTEGER, text)
|
|
252
|
+
when 2
|
|
253
|
+
emit(:MINUS, text)
|
|
254
|
+
else
|
|
255
|
+
instance_exec(&rule.action)
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
end
|
|
264
|
+
end
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Generated by flexr. DO NOT EDIT.
|
|
3
|
+
# source: examples/with_racc/lexer.flexr.rb
|
|
4
|
+
# spec-digest: <normalized>
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
|
+
# unicode: 15.1.0
|
|
8
|
+
# backend: table
|
|
9
|
+
# compiled: true
|
|
10
|
+
# eval: false
|
|
11
|
+
# standalone: false
|
|
12
|
+
|
|
13
|
+
require "flexr"
|
|
14
|
+
|
|
15
|
+
module WithRacc
|
|
16
|
+
class RaccLexer < Flexr::Lexer
|
|
17
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/[0-9]+/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: [:emit, :INTEGER], states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\+/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: [:emit, :PLUS], states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:INTEGER, :PLUS], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [[2, 0, false, false]], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10], class_count: 11, state_count: 4, start: 0, rule_ids: [0, 1, 2], packed: {base: [0, 1, 0, 1], default: [nil, nil, nil, nil], next: [nil, 1, 1, 1, 1, 1, 1, 2, nil, 3, 3], check: [nil, 0, 1, 0, 1, 0, 1, 0, nil, 0, 3]}} } }, states: [:initial], stats: {initial: {states: 4, classes: 11, accepting_states: 3}}, diagnostics: [] } })
|
|
18
|
+
def scan_one
|
|
19
|
+
if @__flexr_generated_scan_state != @state
|
|
20
|
+
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
21
|
+
@__flexr_generated_scan_state = @state
|
|
22
|
+
@__flexr_generated_scan_dfa = machine.dfa
|
|
23
|
+
@__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
|
|
24
|
+
@__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
|
|
25
|
+
end
|
|
26
|
+
dfa = @__flexr_generated_scan_dfa
|
|
27
|
+
return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
|
|
28
|
+
position = byte_pos
|
|
29
|
+
return nil unless valid_utf8_at?(position)
|
|
30
|
+
return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
|
|
31
|
+
cursor = position
|
|
32
|
+
current = dfa.start
|
|
33
|
+
direct = dfa.direct
|
|
34
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
35
|
+
direct_classes = direct&.fetch(:classes)
|
|
36
|
+
ec = dfa.ec
|
|
37
|
+
source = buffer.stable_source
|
|
38
|
+
guarded_steps = scan_steps_guarded?
|
|
39
|
+
token_start = more_text_start || position
|
|
40
|
+
token_limit = max_token_size
|
|
41
|
+
check_token_limit = token_limit_required?
|
|
42
|
+
best = nil
|
|
43
|
+
accel_mode = @accel_mode
|
|
44
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
45
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
46
|
+
if accelerate
|
|
47
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
48
|
+
acceptance = dfa.accepts[current].first
|
|
49
|
+
token_rule = acceptance&.rule_index
|
|
50
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
|
|
51
|
+
if accelerated_end && accelerated_end > cursor
|
|
52
|
+
cursor = accelerated_end
|
|
53
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
58
|
+
consume_step! if guarded_steps
|
|
59
|
+
current = if direct_nxt
|
|
60
|
+
class_id = ec[byte]
|
|
61
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
62
|
+
value >= 0 ? value : nil
|
|
63
|
+
else
|
|
64
|
+
dfa.transition(current, byte)
|
|
65
|
+
end
|
|
66
|
+
break unless current
|
|
67
|
+
cursor += 1
|
|
68
|
+
resource_acceptance = dfa.accepts[current].first
|
|
69
|
+
resource_rule = resource_acceptance&.rule_index
|
|
70
|
+
if check_token_limit
|
|
71
|
+
size = cursor - token_start
|
|
72
|
+
defer_token_size_check!(size, rule: resource_rule) if size > token_limit
|
|
73
|
+
end
|
|
74
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
75
|
+
end
|
|
76
|
+
best
|
|
77
|
+
end
|
|
78
|
+
def __flexr_generated_scan_simple(dfa, position)
|
|
79
|
+
current = dfa.start
|
|
80
|
+
direct = dfa.direct
|
|
81
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
82
|
+
direct_classes = direct&.fetch(:classes)
|
|
83
|
+
ec = dfa.ec
|
|
84
|
+
accepts = dfa.accepts
|
|
85
|
+
rules = self.class.__flexr_rules
|
|
86
|
+
source = buffer.stable_source
|
|
87
|
+
guarded_steps = scan_steps_guarded?
|
|
88
|
+
token_start = more_text_start || position
|
|
89
|
+
token_limit = max_token_size
|
|
90
|
+
check_token_limit = token_limit_required?
|
|
91
|
+
cursor = position
|
|
92
|
+
best = nil
|
|
93
|
+
accel_mode = @accel_mode
|
|
94
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
95
|
+
if source && !guarded_steps && !check_token_limit && !accelerate
|
|
96
|
+
while cursor < source.bytesize
|
|
97
|
+
byte = source.getbyte(cursor)
|
|
98
|
+
current = if direct_nxt
|
|
99
|
+
class_id = ec[byte]
|
|
100
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
101
|
+
value >= 0 ? value : nil
|
|
102
|
+
else
|
|
103
|
+
dfa.transition(current, byte)
|
|
104
|
+
end
|
|
105
|
+
break unless current
|
|
106
|
+
cursor += 1
|
|
107
|
+
acceptance = accepts[current].first
|
|
108
|
+
next unless acceptance
|
|
109
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
110
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
111
|
+
best.rule = rule
|
|
112
|
+
best.start_pos = position
|
|
113
|
+
best.end_pos = cursor
|
|
114
|
+
best.total_end_pos = cursor
|
|
115
|
+
end
|
|
116
|
+
return best
|
|
117
|
+
end
|
|
118
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
119
|
+
if accelerate
|
|
120
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
121
|
+
acceptance = accepts[current].first
|
|
122
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
|
|
123
|
+
if accelerated_end && accelerated_end > cursor
|
|
124
|
+
cursor = accelerated_end
|
|
125
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
126
|
+
next
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
130
|
+
consume_step! if guarded_steps
|
|
131
|
+
current = if direct_nxt
|
|
132
|
+
class_id = ec[byte]
|
|
133
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
134
|
+
value >= 0 ? value : nil
|
|
135
|
+
else
|
|
136
|
+
dfa.transition(current, byte)
|
|
137
|
+
end
|
|
138
|
+
break unless current
|
|
139
|
+
cursor += 1
|
|
140
|
+
acceptance = accepts[current].first
|
|
141
|
+
if check_token_limit
|
|
142
|
+
size = cursor - token_start
|
|
143
|
+
defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
|
|
144
|
+
end
|
|
145
|
+
next unless acceptance
|
|
146
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
147
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
148
|
+
best.rule = rule
|
|
149
|
+
best.start_pos = position
|
|
150
|
+
best.end_pos = cursor
|
|
151
|
+
best.total_end_pos = cursor
|
|
152
|
+
end
|
|
153
|
+
best
|
|
154
|
+
end
|
|
155
|
+
def __flexr_generated_acceleration_region(dfa, state)
|
|
156
|
+
@__flexr_generated_accel_regions ||= {}
|
|
157
|
+
regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
|
|
158
|
+
region = regions[state]
|
|
159
|
+
return unless region
|
|
160
|
+
return if @__flexr_generated_disabled_auto_accel&.key?(region)
|
|
161
|
+
return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
|
|
162
|
+
return if dfa.accepts[state].any? do |acceptance|
|
|
163
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
164
|
+
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
165
|
+
end
|
|
166
|
+
region
|
|
167
|
+
end
|
|
168
|
+
def __flexr_generated_accelerate(region, position, token_rule, token_start)
|
|
169
|
+
mode = @accel_mode
|
|
170
|
+
cursor = position
|
|
171
|
+
matched = false
|
|
172
|
+
loop do
|
|
173
|
+
if cursor >= buffer.bytesize
|
|
174
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
175
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
176
|
+
end
|
|
177
|
+
segment = buffer.source
|
|
178
|
+
segment_position = cursor - buffer.base_offset
|
|
179
|
+
if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
|
|
180
|
+
regexp = region.utf8_regexp
|
|
181
|
+
else
|
|
182
|
+
segment = segment.b unless segment.encoding == ::Encoding::BINARY
|
|
183
|
+
regexp = region.regexp
|
|
184
|
+
end
|
|
185
|
+
length = begin
|
|
186
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
187
|
+
@__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
|
|
188
|
+
@__flexr_generated_accel_scanner.string = segment
|
|
189
|
+
@__flexr_generated_accel_scanner.pos = segment_position
|
|
190
|
+
@__flexr_generated_accel_scanner.skip(regexp)
|
|
191
|
+
else
|
|
192
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
193
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
194
|
+
end
|
|
195
|
+
rescue ArgumentError
|
|
196
|
+
nil
|
|
197
|
+
end
|
|
198
|
+
__flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
|
|
199
|
+
return matched ? cursor : nil unless length&.positive?
|
|
200
|
+
matched = true
|
|
201
|
+
cursor += length
|
|
202
|
+
consume_step!(length)
|
|
203
|
+
defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
|
|
204
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
def __flexr_generated_record_auto_accel_miss(region)
|
|
208
|
+
@__flexr_generated_auto_accel_misses ||= Hash.new(0)
|
|
209
|
+
@__flexr_generated_auto_accel_misses[region] += 1
|
|
210
|
+
@__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
|
|
211
|
+
@__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
|
|
212
|
+
return if @__flexr_generated_auto_accel_misses[region] < 3
|
|
213
|
+
@__flexr_generated_disabled_auto_accel ||= {}
|
|
214
|
+
@__flexr_generated_disabled_auto_accel[region] = true
|
|
215
|
+
end
|
|
216
|
+
def __flexr_generated_fast_path?(dfa)
|
|
217
|
+
@__flexr_generated_fast_paths ||= {}
|
|
218
|
+
return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
|
|
219
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
220
|
+
@__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
|
|
221
|
+
self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
|
|
222
|
+
end
|
|
223
|
+
def __flexr_generated_simple_fast_path?(dfa)
|
|
224
|
+
@__flexr_generated_simple_fast_paths ||= {}
|
|
225
|
+
return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
|
|
226
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
227
|
+
@__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
|
|
228
|
+
rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
|
|
232
|
+
dfa.accepts[state].each do |acceptance|
|
|
233
|
+
next if acceptance.bol_only && !beginning_of_line?
|
|
234
|
+
next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
|
|
235
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
236
|
+
next if best && cursor < best.total_end_pos
|
|
237
|
+
next if best && cursor == best.total_end_pos &&
|
|
238
|
+
rule.index > best.rule.index
|
|
239
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
240
|
+
best.rule = rule
|
|
241
|
+
best.start_pos = start_position
|
|
242
|
+
best.end_pos = cursor
|
|
243
|
+
best.total_end_pos = cursor
|
|
244
|
+
end
|
|
245
|
+
best
|
|
246
|
+
end
|
|
247
|
+
def __flexr_generated_execute(rule)
|
|
248
|
+
case rule.index
|
|
249
|
+
when 0
|
|
250
|
+
when 1
|
|
251
|
+
emit(:INTEGER, text)
|
|
252
|
+
when 2
|
|
253
|
+
emit(:PLUS, text)
|
|
254
|
+
else
|
|
255
|
+
instance_exec(&rule.action)
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
end
|
|
264
|
+
end
|