flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
data/lib/flexr/dsl.rb CHANGED
@@ -3,8 +3,6 @@
3
3
  module Flexr
4
4
  module DSL
5
5
  DSL_METHODS = %i[rule state all_states on_eof emits backend token_kind encoding option accel].freeze
6
- AUTO_DIRECT_CELL_THRESHOLD = 100_000
7
-
8
6
  def inherited(child)
9
7
  super
10
8
  child.__flexr_reset!
@@ -21,23 +19,24 @@ module Flexr
21
19
  )
22
20
  @__flexr_compiled = nil
23
21
  @__flexr_generated = false
24
- @__flexr_compile_mutex = Mutex.new
22
+ @__flexr_generated_actions = []
23
+ @__flexr_generated_eof_actions = {}
24
+ @__flexr_specification_frozen = false
25
+ @__flexr_compile_mutex = Monitor.new
25
26
  end
26
27
 
27
28
  def rule(pattern, skip: false, emit: nil, followed_by: nil, &action)
28
- patterns = normalize_patterns(pattern)
29
- rule_action = if skip
30
- :skip
31
- elsif emit
32
- [:emit, emit.to_sym]
33
- else
34
- action || proc { emit(nil, text) }
29
+ __flexr_mutate! do
30
+ patterns = normalize_patterns(pattern)
31
+ rule_action = ActionResolver.resolve(
32
+ skip: skip, emit: emit, block: action, default: proc { emit(nil, text) }
33
+ )
34
+ states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
35
+ @__flexr_rules << IR::Rule.new(
36
+ index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
37
+ action: rule_action, states: states, bol_only: false, end_anchor: nil
38
+ )
35
39
  end
36
- states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
37
- @__flexr_rules << IR::Rule.new(
38
- index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
39
- action: rule_action, states: states, bol_only: false, end_anchor: nil
40
- )
41
40
  nil
42
41
  end
43
42
 
@@ -45,68 +44,85 @@ module Flexr
45
44
  raise ArgumentError, "state requires a block" unless block
46
45
  raise ArgumentError, "state requires a name" if names.empty?
47
46
 
48
- names.each do |name|
49
- symbol = name.to_sym
50
- @__flexr_states[symbol] ||= IR::State.new(name: symbol, inclusive: inclusive, id: @__flexr_states.length)
47
+ __flexr_mutate! do
48
+ names.each do |name|
49
+ symbol = name.to_sym
50
+ existing = @__flexr_states[symbol]
51
+ raise ArgumentError, "state #{symbol.inspect} already declared with inclusive: #{existing.inclusive}" if
52
+ existing && existing.inclusive != inclusive
53
+ @__flexr_states[symbol] ||= IR::State.new(
54
+ name: symbol, inclusive: inclusive, id: @__flexr_states.length
55
+ )
56
+ end
57
+ @__flexr_state_stack.concat(names.map(&:to_sym))
58
+ begin
59
+ class_eval(&block)
60
+ ensure
61
+ names.length.times { @__flexr_state_stack.pop }
62
+ end
51
63
  end
52
- @__flexr_state_stack.concat(names.map(&:to_sym))
53
- class_eval(&block)
54
- ensure
55
- names&.length&.times { @__flexr_state_stack.pop }
56
64
  end
57
65
 
58
66
  def all_states(&)
59
- state(*@__flexr_states.keys, &)
67
+ raise ArgumentError, "all_states requires a block" unless block_given?
68
+
69
+ __flexr_mutate! do
70
+ names = @__flexr_states.keys
71
+ @__flexr_state_stack.concat(names)
72
+ begin
73
+ class_eval(&)
74
+ ensure
75
+ names.length.times { @__flexr_state_stack.pop }
76
+ end
77
+ end
60
78
  end
61
79
 
62
80
  def on_eof(&action)
63
- state_name = @__flexr_state_stack.last || :initial
64
- @__flexr_eof_rules[state_name] = action
81
+ __flexr_mutate! do
82
+ state_name = @__flexr_state_stack.last || :initial
83
+ @__flexr_eof_rules[state_name] = action
84
+ end
65
85
  end
66
86
 
67
87
  def emits(*tokens)
68
- @__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq!
88
+ __flexr_mutate! { @__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq! }
69
89
  end
70
90
 
71
91
  def backend(name)
72
- @__flexr_config.backend = name.to_sym
92
+ __flexr_mutate! { @__flexr_config.backend = Configuration.backend!(name) }
73
93
  end
74
94
 
75
95
  def token_kind(name)
76
- value = name.to_sym
77
- raise ArgumentError, "unsupported token_kind: #{name}" unless %i[array struct yield].include?(value)
78
-
79
- @__flexr_config.token_kind = value
96
+ __flexr_mutate! { @__flexr_config.token_kind = Configuration.token_kind!(name) }
80
97
  end
81
98
 
82
99
  def encoding(value)
83
- encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
84
- unless [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
85
- diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
86
- raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
87
- end
88
-
89
- @__flexr_config.encoding = encoding
100
+ encoding = Configuration.encoding!(value)
101
+ __flexr_mutate! { @__flexr_config.encoding = encoding }
90
102
  end
91
103
 
92
104
  def option(*values)
93
- values.each { |value| @__flexr_config.options[value.to_sym] = true }
105
+ __flexr_mutate! do
106
+ values.each { |value| @__flexr_config.options[Configuration.option!(value)] = true }
107
+ end
94
108
  end
95
109
 
96
110
  def accel(value)
97
- @__flexr_config.options[:accel] = value.to_sym
111
+ __flexr_mutate! { @__flexr_config.options[:accel] = Configuration.accelerator!(value) }
98
112
  end
99
113
 
100
114
  def compile!
115
+ return @__flexr_compiled if @__flexr_compiled
116
+
101
117
  @__flexr_compile_mutex.synchronize do
102
118
  # The ivar is part of the generated/runtime class contract.
103
- # rubocop:disable Naming/MemoizedInstanceVariableName
104
119
  @__flexr_compiled ||= begin
105
120
  compiled = Automaton::Compiler.new(__flexr_spec).compile
106
121
  @__flexr_config.backend = auto_direct?(compiled) ? :direct : :table if @__flexr_config.backend == :auto
122
+ freeze_specification!
123
+ freeze_compiled!(compiled)
107
124
  compiled
108
125
  end
109
- # rubocop:enable Naming/MemoizedInstanceVariableName
110
126
  end
111
127
  end
112
128
 
@@ -131,26 +147,80 @@ module Flexr
131
147
  )
132
148
  end
133
149
 
134
- attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled
150
+ attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions,
151
+ :__flexr_generated_eof_actions
135
152
 
136
153
  def __flexr_add_generated_eof(state, action)
137
- @__flexr_eof_rules[state.to_sym] = action
154
+ __flexr_mutate! { @__flexr_eof_rules[state.to_sym] = action }
138
155
  end
139
156
 
140
157
  def __flexr_set_compiled!(compiled)
141
- @__flexr_compiled = compiled
158
+ @__flexr_compile_mutex.synchronize do
159
+ @__flexr_compiled = freeze_compiled!(compiled)
160
+ freeze_specification!
161
+ end
142
162
  end
143
163
 
144
164
  def __flexr_mark_generated!
145
165
  @__flexr_generated = true
146
166
  end
147
167
 
168
+ def __flexr_bind_generated_action(index, action)
169
+ raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
170
+
171
+ @__flexr_generated_actions[index] = action
172
+ end
173
+
174
+ def __flexr_bind_generated_eof(state, action)
175
+ raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
176
+
177
+ @__flexr_generated_eof_actions[state.to_sym] = action
178
+ end
179
+
148
180
  def __flexr_generated?
149
181
  @__flexr_generated == true
150
182
  end
151
183
 
152
184
  private
153
185
 
186
+ def __flexr_mutate!
187
+ @__flexr_compile_mutex.synchronize do
188
+ raise FrozenSpecificationError, "lexer specification is immutable after compilation" if
189
+ @__flexr_specification_frozen
190
+
191
+ yield
192
+ end
193
+ end
194
+
195
+ def freeze_specification!
196
+ @__flexr_rules.each do |rule|
197
+ rule.patterns.freeze
198
+ rule.states.freeze
199
+ rule.pattern_conditions&.each(&:freeze)
200
+ rule.pattern_conditions&.freeze
201
+ rule.freeze
202
+ end
203
+ @__flexr_states.each_value(&:freeze)
204
+ @__flexr_eof_rules.freeze
205
+ @__flexr_rules.freeze
206
+ @__flexr_states.freeze
207
+ @__flexr_config.options.freeze
208
+ @__flexr_config.declared_tokens.freeze
209
+ @__flexr_config.freeze
210
+ @__flexr_specification_frozen = true
211
+ end
212
+
213
+ def freeze_compiled!(compiled)
214
+ compiled.machines.each_value(&:freeze)
215
+ compiled.machines.freeze
216
+ compiled.states.freeze
217
+ compiled.stats.each_value(&:freeze)
218
+ compiled.stats.freeze
219
+ Array(compiled.diagnostics).each(&:freeze)
220
+ compiled.diagnostics.freeze
221
+ compiled.freeze
222
+ end
223
+
154
224
  def normalize_patterns(pattern)
155
225
  values = pattern.is_a?(Array) ? pattern : [pattern]
156
226
  values.each do |value|
@@ -175,8 +245,7 @@ module Flexr
175
245
  end
176
246
 
177
247
  def auto_direct?(compiled)
178
- cells = compiled.stats.values.map { |stats| stats[:states] * stats[:classes] }.max.to_i
179
- cells > AUTO_DIRECT_CELL_THRESHOLD
248
+ Automaton::BackendCostModel.choose(compiled) == :direct
180
249
  end
181
250
  end
182
251
  end
data/lib/flexr/errors.rb CHANGED
@@ -2,27 +2,28 @@
2
2
 
3
3
  module Flexr
4
4
  class Error < StandardError
5
- attr_reader :diagnostic
5
+ attr_reader :diagnostic, :filename, :byte_pos
6
6
 
7
- def initialize(message = nil, diagnostic: nil)
7
+ def initialize(message = nil, diagnostic: nil, filename: nil, byte_pos: nil)
8
8
  @diagnostic = diagnostic
9
+ @filename = filename
10
+ @byte_pos = byte_pos
9
11
  super(message || diagnostic&.message)
10
12
  end
11
13
  end
12
14
 
13
15
  class LexError < Error
14
- attr_reader :filename, :byte_pos, :line, :text
16
+ attr_reader :line, :text
15
17
 
16
18
  def initialize(message, filename: nil, byte_pos: nil, line: nil, text: nil, diagnostic: nil)
17
- @filename = filename
18
- @byte_pos = byte_pos
19
19
  @line = line
20
20
  @text = text
21
- super(message, diagnostic: diagnostic)
21
+ super(message, diagnostic: diagnostic, filename: filename, byte_pos: byte_pos)
22
22
  end
23
23
  end
24
24
 
25
25
  class CompileError < Error; end
26
26
  class UnsupportedRegexpError < CompileError; end
27
27
  class StaticResolutionError < CompileError; end
28
+ class FrozenSpecificationError < Error; end
28
29
  end
@@ -2,8 +2,20 @@
2
2
 
3
3
  module Flexr
4
4
  module Generated
5
+ ARTIFACT_SCHEMA_VERSION = 2
6
+ RUNTIME_ABI_VERSION = 2
7
+
5
8
  module_function
6
9
 
10
+ def artifact_metadata
11
+ {
12
+ schema_version: ARTIFACT_SCHEMA_VERSION,
13
+ compiler_version: VERSION,
14
+ runtime_abi_version: RUNTIME_ABI_VERSION,
15
+ unicode_version: Unicode::VERSION
16
+ }
17
+ end
18
+
7
19
  def install!(klass, payload)
8
20
  rules = payload.fetch(:rules)
9
21
  config = payload
@@ -34,22 +46,22 @@ module Flexr
34
46
  end
35
47
 
36
48
  def install_compiled!(klass, payload)
49
+ begin
50
+ validate_artifact!(payload[:artifact])
51
+ rescue CompileError => e
52
+ caller = caller_locations.find { |frame| frame.base_label != "install_compiled!" }
53
+ path = caller&.absolute_path || caller&.path
54
+ raise unless path && File.file?(path)
55
+
56
+ e.diagnostic.location ||= [0, 0]
57
+ raise CompileError.new(e.message, diagnostic: e.diagnostic, filename: path, byte_pos: 0)
58
+ end
37
59
  install!(klass, payload)
38
60
  machines = payload.fetch(:compiled).fetch(:machines).transform_values do |machine|
39
- dfa_data = machine.fetch(:dfa)
40
- packed = decode_packed(dfa_data[:packed])
41
- dfa = Automaton::DFA.new(
42
- transitions: dfa_data[:transitions] || inflate_packed(packed, dfa_data.fetch(:state_count),
43
- dfa_data.fetch(:class_count)),
44
- accepts: dfa_data.fetch(:accepts),
45
- ec: dfa_data.fetch(:ec),
46
- class_count: dfa_data.fetch(:class_count),
47
- start: dfa_data.fetch(:start),
48
- rule_ids: dfa_data.fetch(:rule_ids),
49
- packed: packed,
50
- direct: dfa_data[:direct]
51
- )
52
- Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym)
61
+ dfa = load_dfa(machine.fetch(:dfa))
62
+ plans = Automaton::Accel.whole_token_plans(dfa, klass.__flexr_rules) if
63
+ klass.__flexr_config.backend != :firstmatch && klass.__flexr_config.options[:accel] == :strscan
64
+ Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym, whole_tokens: plans)
53
65
  end
54
66
  compiled = Automaton::CompiledSpec.new(
55
67
  machines: machines,
@@ -65,6 +77,72 @@ module Flexr
65
77
  klass
66
78
  end
67
79
 
80
+ def validate_artifact!(artifact)
81
+ return incompatible_artifact!("artifact metadata is missing") unless artifact.is_a?(Hash)
82
+
83
+ expected = {
84
+ schema_version: ARTIFACT_SCHEMA_VERSION,
85
+ runtime_abi_version: RUNTIME_ABI_VERSION,
86
+ unicode_version: Unicode::VERSION
87
+ }
88
+ mismatch = expected.find { |name, value| artifact[name] != value }
89
+ compiler_version = artifact[:compiler_version]
90
+ return if mismatch.nil? && compiler_version.is_a?(String) && !compiler_version.empty?
91
+
92
+ detail = if mismatch
93
+ name, value = mismatch
94
+ "#{name}=#{artifact[name].inspect} (expected #{value.inspect})"
95
+ else
96
+ "compiler_version=#{compiler_version.inspect}"
97
+ end
98
+ incompatible_artifact!(detail)
99
+ end
100
+
101
+ def shareable!(klass)
102
+ unless klass.__flexr_generated? && klass.__flexr_rules.none? { |rule| rule.action.is_a?(Proc) } &&
103
+ klass.__flexr_generated_actions.empty? && klass.__flexr_spec.eof_rules.empty?
104
+ raise ArgumentError, "Ractor sharing requires generated method actions without captured closures or EOF blocks"
105
+ end
106
+
107
+ config = klass.__flexr_config
108
+ if config.backend == :firstmatch && klass.__flexr_rules.any? do |rule|
109
+ rule.patterns.any? do |pattern|
110
+ Regexp::Matcher.reference?(pattern, encoding: config.encoding, unicode: config.options[:unicode] == true)
111
+ end
112
+ end
113
+ raise ArgumentError, "Ractor sharing does not support firstmatch reference patterns; use table, direct, or auto"
114
+ end
115
+
116
+ original = klass.__flexr_compiled
117
+ rules = original.rules.map do |rule|
118
+ IR::Rule.new(**rule.to_h, trailing_dfa: rule.trailing_dfa && shareable_dfa(rule.trailing_dfa))
119
+ end
120
+ machines = original.machines.transform_values do |machine|
121
+ Automaton::Machine.new(dfa: shareable_dfa(machine.dfa), state_name: machine.state_name,
122
+ whole_tokens: machine.whole_tokens)
123
+ end
124
+ compiled = Automaton::CompiledSpec.new(**original.to_h, machines: machines, rules: rules)
125
+ Ractor.make_shareable([compiled, klass.__flexr_config, klass.__flexr_states, klass.__flexr_spec.eof_rules,
126
+ klass.__flexr_generated_actions, klass.__flexr_generated_eof_actions])
127
+ klass.instance_variable_set(:@__flexr_rules, rules)
128
+ klass.__flexr_set_compiled!(compiled)
129
+ klass
130
+ end
131
+
132
+ def shareable_dfa(dfa)
133
+ Automaton::DFA.new(transitions: dfa.transitions.map(&:dup), accepts: dfa.accepts, ec: dfa.ec,
134
+ class_count: dfa.class_count, start: dfa.start, rule_ids: dfa.rule_ids,
135
+ direct: dfa.direct)
136
+ end
137
+
138
+ def incompatible_artifact!(detail)
139
+ diagnostic = Diagnostics.error(
140
+ "FLEXR-E020", "generated artifact is incompatible with this runtime",
141
+ help: "regenerate the lexer with the installed flexr version", note: detail
142
+ )
143
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
144
+ end
145
+
68
146
  def decode_packed(packed)
69
147
  return packed unless packed.is_a?(Hash) && packed[:encoding]&.to_sym == :base64
70
148
 
@@ -77,6 +155,14 @@ module Flexr
77
155
  }
78
156
  end
79
157
 
158
+ def load_dfa(data)
159
+ Automaton::DFA.new(
160
+ transitions: data[:transitions], accepts: data.fetch(:accepts), ec: data.fetch(:ec),
161
+ class_count: data.fetch(:class_count), start: data.fetch(:start), rule_ids: data.fetch(:rule_ids),
162
+ packed: decode_packed(data[:packed]), direct: data[:direct], state_count: data.fetch(:state_count)
163
+ )
164
+ end
165
+
80
166
  def decode_array(encoded, nil_value: nil)
81
167
  values = encoded.unpack1("m0").unpack("l<*")
82
168
  return values unless nil_value
@@ -84,42 +170,30 @@ module Flexr
84
170
  values.map { |value| value == nil_value ? nil : value }
85
171
  end
86
172
 
87
- def inflate_packed(packed, state_count, class_count)
88
- return nil unless packed
89
-
90
- Array.new(state_count) do |state|
91
- Array.new(class_count) do |class_id|
92
- cursor = state
93
- loop do
94
- index = packed.fetch(:base).fetch(cursor) + class_id
95
- break packed.fetch(:next).fetch(index) if packed.fetch(:check)[index] == cursor
96
-
97
- fallback = packed[:fallback]&.fetch(cursor)
98
- break packed.fetch(:default).fetch(cursor) unless fallback
99
-
100
- cursor = fallback
101
- end
102
- end
103
- end
104
- end
105
173
  end
106
174
 
107
175
  module DSL
108
176
  def __flexr_add_generated_rule(definition)
109
- action = definition.fetch(:action)
110
- @__flexr_rules << IR::Rule.new(
111
- index: definition.fetch(:index), patterns: Array(definition.fetch(:patterns)),
112
- trailing: normalize_trailing(definition[:trailing]), action: action,
113
- states: Array(definition.fetch(:states)).map(&:to_sym),
114
- bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
115
- location: definition[:span] || definition[:location],
116
- pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
117
- next unless condition
118
-
119
- Automaton::Acceptance.new(rule_index: condition[0], pattern_index: condition[1],
120
- bol_only: condition[2], end_anchor: condition[3])
121
- end
122
- )
177
+ __flexr_mutate! do
178
+ action = definition.fetch(:action)
179
+ @__flexr_rules << IR::Rule.new(
180
+ index: definition.fetch(:index), patterns: Array(definition.fetch(:patterns)),
181
+ trailing: normalize_trailing(definition[:trailing]), action: action,
182
+ states: Array(definition.fetch(:states)).map(&:to_sym),
183
+ bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
184
+ location: definition[:span] || definition[:location],
185
+ trailing_dfa: definition[:trailing_dfa] && Generated.load_dfa(definition[:trailing_dfa]),
186
+ pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
187
+ next unless condition
188
+
189
+ Automaton::Acceptance.new(rule_index: condition[0], pattern_index: condition[1],
190
+ bol_only: condition[2], end_anchor: condition[3])
191
+ end
192
+ )
193
+ end
194
+ rescue CompileError => e
195
+ e.diagnostic.location ||= definition[:span] || definition[:location] if e.diagnostic
196
+ raise
123
197
  end
124
198
  end
125
199
  end