flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
data/lib/flexr/dsl.rb
CHANGED
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module DSL
|
|
5
5
|
DSL_METHODS = %i[rule state all_states on_eof emits backend token_kind encoding option accel].freeze
|
|
6
|
-
AUTO_DIRECT_CELL_THRESHOLD = 100_000
|
|
7
|
-
|
|
8
6
|
def inherited(child)
|
|
9
7
|
super
|
|
10
8
|
child.__flexr_reset!
|
|
@@ -21,23 +19,24 @@ module Flexr
|
|
|
21
19
|
)
|
|
22
20
|
@__flexr_compiled = nil
|
|
23
21
|
@__flexr_generated = false
|
|
24
|
-
@
|
|
22
|
+
@__flexr_generated_actions = []
|
|
23
|
+
@__flexr_generated_eof_actions = {}
|
|
24
|
+
@__flexr_specification_frozen = false
|
|
25
|
+
@__flexr_compile_mutex = Monitor.new
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
def rule(pattern, skip: false, emit: nil, followed_by: nil, &action)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
29
|
+
__flexr_mutate! do
|
|
30
|
+
patterns = normalize_patterns(pattern)
|
|
31
|
+
rule_action = ActionResolver.resolve(
|
|
32
|
+
skip: skip, emit: emit, block: action, default: proc { emit(nil, text) }
|
|
33
|
+
)
|
|
34
|
+
states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
|
|
35
|
+
@__flexr_rules << IR::Rule.new(
|
|
36
|
+
index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
|
|
37
|
+
action: rule_action, states: states, bol_only: false, end_anchor: nil
|
|
38
|
+
)
|
|
35
39
|
end
|
|
36
|
-
states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
|
|
37
|
-
@__flexr_rules << IR::Rule.new(
|
|
38
|
-
index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
|
|
39
|
-
action: rule_action, states: states, bol_only: false, end_anchor: nil
|
|
40
|
-
)
|
|
41
40
|
nil
|
|
42
41
|
end
|
|
43
42
|
|
|
@@ -45,68 +44,85 @@ module Flexr
|
|
|
45
44
|
raise ArgumentError, "state requires a block" unless block
|
|
46
45
|
raise ArgumentError, "state requires a name" if names.empty?
|
|
47
46
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
__flexr_mutate! do
|
|
48
|
+
names.each do |name|
|
|
49
|
+
symbol = name.to_sym
|
|
50
|
+
existing = @__flexr_states[symbol]
|
|
51
|
+
raise ArgumentError, "state #{symbol.inspect} already declared with inclusive: #{existing.inclusive}" if
|
|
52
|
+
existing && existing.inclusive != inclusive
|
|
53
|
+
@__flexr_states[symbol] ||= IR::State.new(
|
|
54
|
+
name: symbol, inclusive: inclusive, id: @__flexr_states.length
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
@__flexr_state_stack.concat(names.map(&:to_sym))
|
|
58
|
+
begin
|
|
59
|
+
class_eval(&block)
|
|
60
|
+
ensure
|
|
61
|
+
names.length.times { @__flexr_state_stack.pop }
|
|
62
|
+
end
|
|
51
63
|
end
|
|
52
|
-
@__flexr_state_stack.concat(names.map(&:to_sym))
|
|
53
|
-
class_eval(&block)
|
|
54
|
-
ensure
|
|
55
|
-
names&.length&.times { @__flexr_state_stack.pop }
|
|
56
64
|
end
|
|
57
65
|
|
|
58
66
|
def all_states(&)
|
|
59
|
-
|
|
67
|
+
raise ArgumentError, "all_states requires a block" unless block_given?
|
|
68
|
+
|
|
69
|
+
__flexr_mutate! do
|
|
70
|
+
names = @__flexr_states.keys
|
|
71
|
+
@__flexr_state_stack.concat(names)
|
|
72
|
+
begin
|
|
73
|
+
class_eval(&)
|
|
74
|
+
ensure
|
|
75
|
+
names.length.times { @__flexr_state_stack.pop }
|
|
76
|
+
end
|
|
77
|
+
end
|
|
60
78
|
end
|
|
61
79
|
|
|
62
80
|
def on_eof(&action)
|
|
63
|
-
|
|
64
|
-
|
|
81
|
+
__flexr_mutate! do
|
|
82
|
+
state_name = @__flexr_state_stack.last || :initial
|
|
83
|
+
@__flexr_eof_rules[state_name] = action
|
|
84
|
+
end
|
|
65
85
|
end
|
|
66
86
|
|
|
67
87
|
def emits(*tokens)
|
|
68
|
-
@__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq!
|
|
88
|
+
__flexr_mutate! { @__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq! }
|
|
69
89
|
end
|
|
70
90
|
|
|
71
91
|
def backend(name)
|
|
72
|
-
@__flexr_config.backend = name
|
|
92
|
+
__flexr_mutate! { @__flexr_config.backend = Configuration.backend!(name) }
|
|
73
93
|
end
|
|
74
94
|
|
|
75
95
|
def token_kind(name)
|
|
76
|
-
|
|
77
|
-
raise ArgumentError, "unsupported token_kind: #{name}" unless %i[array struct yield].include?(value)
|
|
78
|
-
|
|
79
|
-
@__flexr_config.token_kind = value
|
|
96
|
+
__flexr_mutate! { @__flexr_config.token_kind = Configuration.token_kind!(name) }
|
|
80
97
|
end
|
|
81
98
|
|
|
82
99
|
def encoding(value)
|
|
83
|
-
encoding =
|
|
84
|
-
|
|
85
|
-
diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
|
|
86
|
-
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
@__flexr_config.encoding = encoding
|
|
100
|
+
encoding = Configuration.encoding!(value)
|
|
101
|
+
__flexr_mutate! { @__flexr_config.encoding = encoding }
|
|
90
102
|
end
|
|
91
103
|
|
|
92
104
|
def option(*values)
|
|
93
|
-
|
|
105
|
+
__flexr_mutate! do
|
|
106
|
+
values.each { |value| @__flexr_config.options[Configuration.option!(value)] = true }
|
|
107
|
+
end
|
|
94
108
|
end
|
|
95
109
|
|
|
96
110
|
def accel(value)
|
|
97
|
-
@__flexr_config.options[:accel] = value
|
|
111
|
+
__flexr_mutate! { @__flexr_config.options[:accel] = Configuration.accelerator!(value) }
|
|
98
112
|
end
|
|
99
113
|
|
|
100
114
|
def compile!
|
|
115
|
+
return @__flexr_compiled if @__flexr_compiled
|
|
116
|
+
|
|
101
117
|
@__flexr_compile_mutex.synchronize do
|
|
102
118
|
# The ivar is part of the generated/runtime class contract.
|
|
103
|
-
# rubocop:disable Naming/MemoizedInstanceVariableName
|
|
104
119
|
@__flexr_compiled ||= begin
|
|
105
120
|
compiled = Automaton::Compiler.new(__flexr_spec).compile
|
|
106
121
|
@__flexr_config.backend = auto_direct?(compiled) ? :direct : :table if @__flexr_config.backend == :auto
|
|
122
|
+
freeze_specification!
|
|
123
|
+
freeze_compiled!(compiled)
|
|
107
124
|
compiled
|
|
108
125
|
end
|
|
109
|
-
# rubocop:enable Naming/MemoizedInstanceVariableName
|
|
110
126
|
end
|
|
111
127
|
end
|
|
112
128
|
|
|
@@ -131,26 +147,80 @@ module Flexr
|
|
|
131
147
|
)
|
|
132
148
|
end
|
|
133
149
|
|
|
134
|
-
attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled
|
|
150
|
+
attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions,
|
|
151
|
+
:__flexr_generated_eof_actions
|
|
135
152
|
|
|
136
153
|
def __flexr_add_generated_eof(state, action)
|
|
137
|
-
@__flexr_eof_rules[state.to_sym] = action
|
|
154
|
+
__flexr_mutate! { @__flexr_eof_rules[state.to_sym] = action }
|
|
138
155
|
end
|
|
139
156
|
|
|
140
157
|
def __flexr_set_compiled!(compiled)
|
|
141
|
-
@
|
|
158
|
+
@__flexr_compile_mutex.synchronize do
|
|
159
|
+
@__flexr_compiled = freeze_compiled!(compiled)
|
|
160
|
+
freeze_specification!
|
|
161
|
+
end
|
|
142
162
|
end
|
|
143
163
|
|
|
144
164
|
def __flexr_mark_generated!
|
|
145
165
|
@__flexr_generated = true
|
|
146
166
|
end
|
|
147
167
|
|
|
168
|
+
def __flexr_bind_generated_action(index, action)
|
|
169
|
+
raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
|
|
170
|
+
|
|
171
|
+
@__flexr_generated_actions[index] = action
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def __flexr_bind_generated_eof(state, action)
|
|
175
|
+
raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
|
|
176
|
+
|
|
177
|
+
@__flexr_generated_eof_actions[state.to_sym] = action
|
|
178
|
+
end
|
|
179
|
+
|
|
148
180
|
def __flexr_generated?
|
|
149
181
|
@__flexr_generated == true
|
|
150
182
|
end
|
|
151
183
|
|
|
152
184
|
private
|
|
153
185
|
|
|
186
|
+
def __flexr_mutate!
|
|
187
|
+
@__flexr_compile_mutex.synchronize do
|
|
188
|
+
raise FrozenSpecificationError, "lexer specification is immutable after compilation" if
|
|
189
|
+
@__flexr_specification_frozen
|
|
190
|
+
|
|
191
|
+
yield
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def freeze_specification!
|
|
196
|
+
@__flexr_rules.each do |rule|
|
|
197
|
+
rule.patterns.freeze
|
|
198
|
+
rule.states.freeze
|
|
199
|
+
rule.pattern_conditions&.each(&:freeze)
|
|
200
|
+
rule.pattern_conditions&.freeze
|
|
201
|
+
rule.freeze
|
|
202
|
+
end
|
|
203
|
+
@__flexr_states.each_value(&:freeze)
|
|
204
|
+
@__flexr_eof_rules.freeze
|
|
205
|
+
@__flexr_rules.freeze
|
|
206
|
+
@__flexr_states.freeze
|
|
207
|
+
@__flexr_config.options.freeze
|
|
208
|
+
@__flexr_config.declared_tokens.freeze
|
|
209
|
+
@__flexr_config.freeze
|
|
210
|
+
@__flexr_specification_frozen = true
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def freeze_compiled!(compiled)
|
|
214
|
+
compiled.machines.each_value(&:freeze)
|
|
215
|
+
compiled.machines.freeze
|
|
216
|
+
compiled.states.freeze
|
|
217
|
+
compiled.stats.each_value(&:freeze)
|
|
218
|
+
compiled.stats.freeze
|
|
219
|
+
Array(compiled.diagnostics).each(&:freeze)
|
|
220
|
+
compiled.diagnostics.freeze
|
|
221
|
+
compiled.freeze
|
|
222
|
+
end
|
|
223
|
+
|
|
154
224
|
def normalize_patterns(pattern)
|
|
155
225
|
values = pattern.is_a?(Array) ? pattern : [pattern]
|
|
156
226
|
values.each do |value|
|
|
@@ -175,8 +245,7 @@ module Flexr
|
|
|
175
245
|
end
|
|
176
246
|
|
|
177
247
|
def auto_direct?(compiled)
|
|
178
|
-
|
|
179
|
-
cells > AUTO_DIRECT_CELL_THRESHOLD
|
|
248
|
+
Automaton::BackendCostModel.choose(compiled) == :direct
|
|
180
249
|
end
|
|
181
250
|
end
|
|
182
251
|
end
|
data/lib/flexr/errors.rb
CHANGED
|
@@ -2,27 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
class Error < StandardError
|
|
5
|
-
attr_reader :diagnostic
|
|
5
|
+
attr_reader :diagnostic, :filename, :byte_pos
|
|
6
6
|
|
|
7
|
-
def initialize(message = nil, diagnostic: nil)
|
|
7
|
+
def initialize(message = nil, diagnostic: nil, filename: nil, byte_pos: nil)
|
|
8
8
|
@diagnostic = diagnostic
|
|
9
|
+
@filename = filename
|
|
10
|
+
@byte_pos = byte_pos
|
|
9
11
|
super(message || diagnostic&.message)
|
|
10
12
|
end
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
class LexError < Error
|
|
14
|
-
attr_reader :
|
|
16
|
+
attr_reader :line, :text
|
|
15
17
|
|
|
16
18
|
def initialize(message, filename: nil, byte_pos: nil, line: nil, text: nil, diagnostic: nil)
|
|
17
|
-
@filename = filename
|
|
18
|
-
@byte_pos = byte_pos
|
|
19
19
|
@line = line
|
|
20
20
|
@text = text
|
|
21
|
-
super(message, diagnostic: diagnostic)
|
|
21
|
+
super(message, diagnostic: diagnostic, filename: filename, byte_pos: byte_pos)
|
|
22
22
|
end
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
class CompileError < Error; end
|
|
26
26
|
class UnsupportedRegexpError < CompileError; end
|
|
27
27
|
class StaticResolutionError < CompileError; end
|
|
28
|
+
class FrozenSpecificationError < Error; end
|
|
28
29
|
end
|
data/lib/flexr/generated.rb
CHANGED
|
@@ -2,8 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Generated
|
|
5
|
+
ARTIFACT_SCHEMA_VERSION = 2
|
|
6
|
+
RUNTIME_ABI_VERSION = 2
|
|
7
|
+
|
|
5
8
|
module_function
|
|
6
9
|
|
|
10
|
+
def artifact_metadata
|
|
11
|
+
{
|
|
12
|
+
schema_version: ARTIFACT_SCHEMA_VERSION,
|
|
13
|
+
compiler_version: VERSION,
|
|
14
|
+
runtime_abi_version: RUNTIME_ABI_VERSION,
|
|
15
|
+
unicode_version: Unicode::VERSION
|
|
16
|
+
}
|
|
17
|
+
end
|
|
18
|
+
|
|
7
19
|
def install!(klass, payload)
|
|
8
20
|
rules = payload.fetch(:rules)
|
|
9
21
|
config = payload
|
|
@@ -34,22 +46,22 @@ module Flexr
|
|
|
34
46
|
end
|
|
35
47
|
|
|
36
48
|
def install_compiled!(klass, payload)
|
|
49
|
+
begin
|
|
50
|
+
validate_artifact!(payload[:artifact])
|
|
51
|
+
rescue CompileError => e
|
|
52
|
+
caller = caller_locations.find { |frame| frame.base_label != "install_compiled!" }
|
|
53
|
+
path = caller&.absolute_path || caller&.path
|
|
54
|
+
raise unless path && File.file?(path)
|
|
55
|
+
|
|
56
|
+
e.diagnostic.location ||= [0, 0]
|
|
57
|
+
raise CompileError.new(e.message, diagnostic: e.diagnostic, filename: path, byte_pos: 0)
|
|
58
|
+
end
|
|
37
59
|
install!(klass, payload)
|
|
38
60
|
machines = payload.fetch(:compiled).fetch(:machines).transform_values do |machine|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
dfa_data.fetch(:class_count)),
|
|
44
|
-
accepts: dfa_data.fetch(:accepts),
|
|
45
|
-
ec: dfa_data.fetch(:ec),
|
|
46
|
-
class_count: dfa_data.fetch(:class_count),
|
|
47
|
-
start: dfa_data.fetch(:start),
|
|
48
|
-
rule_ids: dfa_data.fetch(:rule_ids),
|
|
49
|
-
packed: packed,
|
|
50
|
-
direct: dfa_data[:direct]
|
|
51
|
-
)
|
|
52
|
-
Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym)
|
|
61
|
+
dfa = load_dfa(machine.fetch(:dfa))
|
|
62
|
+
plans = Automaton::Accel.whole_token_plans(dfa, klass.__flexr_rules) if
|
|
63
|
+
klass.__flexr_config.backend != :firstmatch && klass.__flexr_config.options[:accel] == :strscan
|
|
64
|
+
Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym, whole_tokens: plans)
|
|
53
65
|
end
|
|
54
66
|
compiled = Automaton::CompiledSpec.new(
|
|
55
67
|
machines: machines,
|
|
@@ -65,6 +77,72 @@ module Flexr
|
|
|
65
77
|
klass
|
|
66
78
|
end
|
|
67
79
|
|
|
80
|
+
def validate_artifact!(artifact)
|
|
81
|
+
return incompatible_artifact!("artifact metadata is missing") unless artifact.is_a?(Hash)
|
|
82
|
+
|
|
83
|
+
expected = {
|
|
84
|
+
schema_version: ARTIFACT_SCHEMA_VERSION,
|
|
85
|
+
runtime_abi_version: RUNTIME_ABI_VERSION,
|
|
86
|
+
unicode_version: Unicode::VERSION
|
|
87
|
+
}
|
|
88
|
+
mismatch = expected.find { |name, value| artifact[name] != value }
|
|
89
|
+
compiler_version = artifact[:compiler_version]
|
|
90
|
+
return if mismatch.nil? && compiler_version.is_a?(String) && !compiler_version.empty?
|
|
91
|
+
|
|
92
|
+
detail = if mismatch
|
|
93
|
+
name, value = mismatch
|
|
94
|
+
"#{name}=#{artifact[name].inspect} (expected #{value.inspect})"
|
|
95
|
+
else
|
|
96
|
+
"compiler_version=#{compiler_version.inspect}"
|
|
97
|
+
end
|
|
98
|
+
incompatible_artifact!(detail)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def shareable!(klass)
|
|
102
|
+
unless klass.__flexr_generated? && klass.__flexr_rules.none? { |rule| rule.action.is_a?(Proc) } &&
|
|
103
|
+
klass.__flexr_generated_actions.empty? && klass.__flexr_spec.eof_rules.empty?
|
|
104
|
+
raise ArgumentError, "Ractor sharing requires generated method actions without captured closures or EOF blocks"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
config = klass.__flexr_config
|
|
108
|
+
if config.backend == :firstmatch && klass.__flexr_rules.any? do |rule|
|
|
109
|
+
rule.patterns.any? do |pattern|
|
|
110
|
+
Regexp::Matcher.reference?(pattern, encoding: config.encoding, unicode: config.options[:unicode] == true)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
raise ArgumentError, "Ractor sharing does not support firstmatch reference patterns; use table, direct, or auto"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
original = klass.__flexr_compiled
|
|
117
|
+
rules = original.rules.map do |rule|
|
|
118
|
+
IR::Rule.new(**rule.to_h, trailing_dfa: rule.trailing_dfa && shareable_dfa(rule.trailing_dfa))
|
|
119
|
+
end
|
|
120
|
+
machines = original.machines.transform_values do |machine|
|
|
121
|
+
Automaton::Machine.new(dfa: shareable_dfa(machine.dfa), state_name: machine.state_name,
|
|
122
|
+
whole_tokens: machine.whole_tokens)
|
|
123
|
+
end
|
|
124
|
+
compiled = Automaton::CompiledSpec.new(**original.to_h, machines: machines, rules: rules)
|
|
125
|
+
Ractor.make_shareable([compiled, klass.__flexr_config, klass.__flexr_states, klass.__flexr_spec.eof_rules,
|
|
126
|
+
klass.__flexr_generated_actions, klass.__flexr_generated_eof_actions])
|
|
127
|
+
klass.instance_variable_set(:@__flexr_rules, rules)
|
|
128
|
+
klass.__flexr_set_compiled!(compiled)
|
|
129
|
+
klass
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def shareable_dfa(dfa)
|
|
133
|
+
Automaton::DFA.new(transitions: dfa.transitions.map(&:dup), accepts: dfa.accepts, ec: dfa.ec,
|
|
134
|
+
class_count: dfa.class_count, start: dfa.start, rule_ids: dfa.rule_ids,
|
|
135
|
+
direct: dfa.direct)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def incompatible_artifact!(detail)
|
|
139
|
+
diagnostic = Diagnostics.error(
|
|
140
|
+
"FLEXR-E020", "generated artifact is incompatible with this runtime",
|
|
141
|
+
help: "regenerate the lexer with the installed flexr version", note: detail
|
|
142
|
+
)
|
|
143
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
144
|
+
end
|
|
145
|
+
|
|
68
146
|
def decode_packed(packed)
|
|
69
147
|
return packed unless packed.is_a?(Hash) && packed[:encoding]&.to_sym == :base64
|
|
70
148
|
|
|
@@ -77,6 +155,14 @@ module Flexr
|
|
|
77
155
|
}
|
|
78
156
|
end
|
|
79
157
|
|
|
158
|
+
def load_dfa(data)
|
|
159
|
+
Automaton::DFA.new(
|
|
160
|
+
transitions: data[:transitions], accepts: data.fetch(:accepts), ec: data.fetch(:ec),
|
|
161
|
+
class_count: data.fetch(:class_count), start: data.fetch(:start), rule_ids: data.fetch(:rule_ids),
|
|
162
|
+
packed: decode_packed(data[:packed]), direct: data[:direct], state_count: data.fetch(:state_count)
|
|
163
|
+
)
|
|
164
|
+
end
|
|
165
|
+
|
|
80
166
|
def decode_array(encoded, nil_value: nil)
|
|
81
167
|
values = encoded.unpack1("m0").unpack("l<*")
|
|
82
168
|
return values unless nil_value
|
|
@@ -84,42 +170,30 @@ module Flexr
|
|
|
84
170
|
values.map { |value| value == nil_value ? nil : value }
|
|
85
171
|
end
|
|
86
172
|
|
|
87
|
-
def inflate_packed(packed, state_count, class_count)
|
|
88
|
-
return nil unless packed
|
|
89
|
-
|
|
90
|
-
Array.new(state_count) do |state|
|
|
91
|
-
Array.new(class_count) do |class_id|
|
|
92
|
-
cursor = state
|
|
93
|
-
loop do
|
|
94
|
-
index = packed.fetch(:base).fetch(cursor) + class_id
|
|
95
|
-
break packed.fetch(:next).fetch(index) if packed.fetch(:check)[index] == cursor
|
|
96
|
-
|
|
97
|
-
fallback = packed[:fallback]&.fetch(cursor)
|
|
98
|
-
break packed.fetch(:default).fetch(cursor) unless fallback
|
|
99
|
-
|
|
100
|
-
cursor = fallback
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
173
|
end
|
|
106
174
|
|
|
107
175
|
module DSL
|
|
108
176
|
def __flexr_add_generated_rule(definition)
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
177
|
+
__flexr_mutate! do
|
|
178
|
+
action = definition.fetch(:action)
|
|
179
|
+
@__flexr_rules << IR::Rule.new(
|
|
180
|
+
index: definition.fetch(:index), patterns: Array(definition.fetch(:patterns)),
|
|
181
|
+
trailing: normalize_trailing(definition[:trailing]), action: action,
|
|
182
|
+
states: Array(definition.fetch(:states)).map(&:to_sym),
|
|
183
|
+
bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
|
|
184
|
+
location: definition[:span] || definition[:location],
|
|
185
|
+
trailing_dfa: definition[:trailing_dfa] && Generated.load_dfa(definition[:trailing_dfa]),
|
|
186
|
+
pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
|
|
187
|
+
next unless condition
|
|
188
|
+
|
|
189
|
+
Automaton::Acceptance.new(rule_index: condition[0], pattern_index: condition[1],
|
|
190
|
+
bol_only: condition[2], end_anchor: condition[3])
|
|
191
|
+
end
|
|
192
|
+
)
|
|
193
|
+
end
|
|
194
|
+
rescue CompileError => e
|
|
195
|
+
e.diagnostic.location ||= definition[:span] || definition[:location] if e.diagnostic
|
|
196
|
+
raise
|
|
123
197
|
end
|
|
124
198
|
end
|
|
125
199
|
end
|