flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -1,5 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "digest"
4
+ require_relative "runtime" unless defined?(Flexr::Lexer)
5
+ require_relative "artifact_writer"
6
+ require_relative "codegen"
7
+ require_relative "source"
8
+
3
9
  module Flexr
4
10
  class Generator
5
11
  attr_reader :diagnostics
@@ -8,15 +14,18 @@ module Flexr
8
14
  INLINE_EMIT_ARGUMENTS = /\A(?::[A-Za-z_]\w*|true|false|nil|-?\d+(?:\.\d+)?|"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|text(?:\.(?:to_f|to_i|bytesize|byteslice\([^()\n]+\)))?|lineno|line)(?:\s*,\s*(?::[A-Za-z_]\w*|true|false|nil|-?\d+(?:\.\d+)?|"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|text(?:\.(?:to_f|to_i|bytesize|byteslice\([^()\n]+\)))?|lineno|line))*\z/
9
15
 
10
16
  RUNTIME_SOURCES = %w[
11
- version.rb errors.rb diagnostics.rb ir.rb
12
- regexp/ast.rb regexp/parser.rb regexp/normalizer.rb regexp/unsupported.rb regexp/char_class.rb
13
- unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb
14
- automaton/byte_class_set.rb automaton/nfa.rb automaton/dfa.rb automaton/compiler.rb
15
- automaton/analysis.rb automaton/minimizer.rb automaton/accel.rb
16
- runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
17
+ version.rb errors.rb diagnostics.rb action_resolver.rb configuration.rb ir.rb
18
+ unicode/version.rb automaton/types.rb automaton/dfa.rb automaton/accel.rb
19
+ runtime/record.rb runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
17
20
  runtime/interpreter.rb runtime/core.rb dsl.rb lexer.rb generated.rb
18
21
  ].freeze
19
22
 
23
+ MATCHER_RUNTIME_SOURCES = %w[
24
+ regexp/ast.rb regexp/parser.rb regexp/unsupported.rb
25
+ unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb
26
+ unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb regexp/matcher.rb
27
+ ].freeze
28
+
20
29
  def initialize(path, output: nil, eval_mode: false, options: {})
21
30
  @path = path
22
31
  @output = output
@@ -26,6 +35,7 @@ module Flexr
26
35
  end
27
36
 
28
37
  def generate
38
+ ArtifactWriter.ensure_distinct!(@path, @output) if @output
29
39
  # Keep generated source stable when a checkout presents the spec with
30
40
  # Windows line endings. All offsets consumed by the parser and
31
41
  # passthrough writer are relative to this normalized source.
@@ -36,7 +46,7 @@ module Flexr
36
46
  parsed = Source::PrismReader.new(source, path: @path).read
37
47
  generate_static(parsed)
38
48
  end
39
- File.binwrite(@output, result) if @output
49
+ ArtifactWriter.write!(@output, result, source_path: @path) if @output
40
50
  result
41
51
  end
42
52
 
@@ -51,24 +61,31 @@ module Flexr
51
61
  install = "Flexr::Generated.install_compiled!(self, #{payload})\n"
52
62
  install = "#{install}#{Codegen::Table.new(compiled).source(indent: indent)}"
53
63
  install = "#{install}#{generated_action_source(parsed, indent)}"
54
- install = "#{install}#{Codegen::Direct.new(compiled).source(indent: indent)}#{indent}" if
55
- effective_backend(parsed) == :direct
56
- result = Source::Passthrough.remove_spans(parsed.source, parsed.dsl_spans, insertion: parsed.first_dsl_offset, payload: install)
57
- result = result.gsub(/^[ \t]+(?=\n)/, "")
58
- result = result.gsub(/^\s*require ["']flexr["']\s*\n/, "") if standalone?(parsed)
59
- digest = Digest::SHA256.hexdigest(payload)
64
+ edits = parsed.dsl_edits.dup
65
+ parsed.rules.each do |rule|
66
+ next unless rule.bind_action && !rule.reads_locals && inline_simple_action(rule.action_source)
67
+
68
+ edits.map! { |edit| edit.first(2) == rule.span ? [*rule.span, ""] : edit }
69
+ end
70
+ edits.concat(parsed.flexr_require_spans.map { |span| [*span, ""] }) if standalone?(parsed)
71
+ result = Source::Passthrough.rewrite(
72
+ parsed.source, edits, insertion: parsed.first_dsl_offset, payload: install
73
+ )
74
+ digest = Digest::SHA256.hexdigest(result)
60
75
  header = [
61
76
  "# Generated by flexr. DO NOT EDIT.",
62
77
  "# source: #{parsed.path}",
63
78
  "# spec-digest: sha256:#{digest}",
79
+ "# artifact-schema: #{Generated::ARTIFACT_SCHEMA_VERSION}",
80
+ "# runtime-abi: #{Generated::RUNTIME_ABI_VERSION}",
64
81
  "# unicode: #{Unicode::VERSION}",
65
82
  "# backend: #{effective_backend(parsed)}",
66
83
  "# compiled: true",
67
84
  "# eval: #{@eval_mode}",
68
85
  "# standalone: #{standalone?(parsed)}"
69
86
  ].join("\n")
70
- prefix = standalone?(parsed) ? "require \"json\"\n#{embedded_runtime}\n" : ""
71
- result.sub(/\A(# frozen_string_literal: true\n)/) { "#{::Regexp.last_match(1)}#{header}\n#{prefix}" }
87
+ prefix = standalone?(parsed) ? "raise LoadError, \"flexr requires Ruby 3.4 or newer\" if RUBY_VERSION < \"3.4\"\nrequire \"json\"\nrequire \"monitor\"\n#{embedded_runtime(parsed)}\n" : ""
88
+ Source::Passthrough.insert_after_preamble(result, "#{header}\n#{prefix}")
72
89
  end
73
90
 
74
91
  def generated_payload(parsed, compiled)
@@ -77,10 +94,11 @@ module Flexr
77
94
  [condition.rule_index, condition.pattern_index, condition.bol_only, condition.end_anchor]
78
95
  end
79
96
  "{ index: #{rule.index}, patterns: #{ruby_literal(rule.patterns)}, pattern_conditions: #{ruby_literal(conditions)}, trailing: #{ruby_literal(rule.trailing)}, " \
80
- "action: #{action_expression(rule.action)}, states: #{rule.states.inspect}, " \
97
+ "#{trailing_dfa_expression(rule, parsed)}" \
98
+ "action: #{generated_rule_action(rule)}, states: #{rule.states.inspect}, " \
81
99
  "bol_only: #{ruby_literal(rule.bol_only)}, end_anchor: #{ruby_literal(rule.end_anchor)} }"
82
100
  end
83
- "{ rules: [#{definitions.join(', ')}], " \
101
+ "{ artifact: #{ruby_literal(artifact_metadata)}, rules: [#{definitions.join(', ')}], " \
84
102
  "backend: #{ruby_literal(effective_backend(parsed))}, " \
85
103
  "token_kind: #{ruby_literal(@options.fetch(:token_kind, parsed.config[:token_kind]))}, " \
86
104
  "encoding: #{encoding_expression(parsed.config[:encoding])}, " \
@@ -93,25 +111,31 @@ module Flexr
93
111
  backend: effective_backend(parsed))} }"
94
112
  end
95
113
 
114
+ def artifact_metadata
115
+ Generated.artifact_metadata
116
+ end
117
+
96
118
  def compile_parsed(parsed)
97
119
  states = parsed.states.each_with_index.to_h do |(name, value), index|
98
- [name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index)]
120
+ [name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index, location: value[:location])]
99
121
  end
100
122
  rules = parsed.rules.map do |rule|
101
123
  IR::Rule.new(index: rule.index, patterns: rule.patterns, trailing: normalize_trailing(rule.trailing),
102
124
  action: rule.action, states: rule.states, bol_only: rule.bol_only,
103
125
  end_anchor: rule.end_anchor, location: rule.span)
126
+ rescue CompileError => e
127
+ e.diagnostic.location ||= rule.span if e.diagnostic
128
+ raise
104
129
  end
105
130
  spec = IR::Spec.new(
106
131
  class_name: parsed.class_name, superclass: "Flexr::Lexer",
107
132
  backend: @options.fetch(:backend, parsed.config[:backend]),
108
133
  token_kind: @options.fetch(:token_kind, parsed.config[:token_kind]),
109
- encoding: parsed.config[:encoding], options: effective_options(parsed),
134
+ encoding: parsed.config[:encoding], options: effective_options(parsed).merge(warn_level: @options[:warn_level]),
110
135
  declared_tokens: parsed.config[:declared_tokens], states: states, rules: rules,
111
136
  eof_rules: parsed.config[:eof_rules] || {}, verbatim: parsed.source
112
137
  )
113
138
  compiled = Automaton::Compiler.new(spec).compile
114
- validate_firstmatch_equivalence!(spec, compiled) if spec.backend == :firstmatch
115
139
  @resolved_backend = resolve_backend(spec.backend, compiled)
116
140
  parsed.rules.each do |rule|
117
141
  compiled_rule = spec.rules.fetch(rule.index)
@@ -128,8 +152,7 @@ module Flexr
128
152
  return requested unless requested == :auto
129
153
  return :table unless compiled
130
154
 
131
- cells = compiled.stats.values.map { |stats| stats[:states] * stats[:classes] }.max.to_i
132
- cells > DSL::AUTO_DIRECT_CELL_THRESHOLD ? :direct : :table
155
+ Automaton::BackendCostModel.choose(compiled)
133
156
  end
134
157
 
135
158
  def normalize_trailing(value)
@@ -144,89 +167,10 @@ module Flexr
144
167
  end
145
168
  end
146
169
 
147
- def validate_firstmatch_equivalence!(spec, compiled)
148
- return if spec.rules.any? do |rule|
149
- rule.trailing || rule.patterns.any? { |pattern| reference_pattern?(pattern, unicode: spec.options[:unicode] == true) }
150
- end
151
-
152
- machine = compiled.machines.fetch(:initial).dfa
153
- random = Random.new(17)
154
- inputs = ["", "a", "aa", "aaa", "ba", "aab"]
155
- inputs.concat(Array.new(9_994) do
156
- Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
157
- end)
158
- inputs.each do |input|
159
- table = table_match(machine, input)
160
- firstmatch = firstmatch_match(spec.rules, input)
161
- next if table == firstmatch
162
-
163
- raise CompileError, "firstmatch differs from table for #{input.inspect}: #{firstmatch.inspect} vs #{table.inspect}"
164
- end
165
- end
166
-
167
- def reference_pattern?(pattern, unicode: false)
168
- return false unless pattern.is_a?(::Regexp)
169
- return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
170
-
171
- unicode && pattern.encoding != Encoding::BINARY && pattern.source.match?(/\\[dDwWsS]/)
172
- end
173
-
174
- def table_match(dfa, input)
175
- state = dfa.start
176
- best = nil
177
- input.each_byte.with_index do |byte, index|
178
- state = dfa.transition(state, byte)
179
- break unless state
180
-
181
- acceptance = dfa.accepts[state].select do |candidate|
182
- !candidate.end_anchor || index + 1 == input.bytesize || input.getbyte(index + 1) == 0x0a
183
- end.min_by(&:rule_index)
184
- next unless acceptance
185
-
186
- candidate = [acceptance.rule_index, index + 1]
187
- best = candidate if best.nil? || candidate[1] > best[1] ||
188
- (candidate[1] == best[1] && candidate[0] < best[0])
189
- end
190
- best
191
- end
192
-
193
- def firstmatch_match(rules, input)
194
- rules.sort_by(&:index).each do |rule|
195
- lengths = rule.patterns.filter_map do |pattern|
196
- regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
197
- match = regexp.match(input, 0)
198
- match&.begin(0)&.zero? ? match[0].bytesize : nil
199
- rescue ArgumentError, RegexpError
200
- nil
201
- end
202
- return [rule.index, lengths.max] unless lengths.empty?
203
- end
204
- nil
205
- end
206
-
207
170
  def compiled_expression(compiled, compression: :none, backend: :table)
208
171
  machines = compiled.machines.map do |name, machine|
209
- dfa = machine.dfa
210
- data = {
211
- accepts: dfa.accepts,
212
- ec: dfa.ec,
213
- class_count: dfa.class_count,
214
- state_count: dfa.states,
215
- start: dfa.start,
216
- rule_ids: dfa.rule_ids
217
- }
218
- pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
219
- if pack_tables
220
- packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
221
- data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
222
- Codegen::TablePacker.encode(packed)
223
- else
224
- packed
225
- end
226
- else
227
- data[:transitions] = dfa.transitions
228
- end
229
- data[:direct] = Codegen::Direct.new(compiled).generate(state: name) if backend == :direct
172
+ direct = Codegen::Direct.new(compiled).generate(state: name) if backend == :direct
173
+ data = dfa_data(machine.dfa, compression: compression, direct: direct)
230
174
  "#{ruby_literal(name)} => { state_name: #{ruby_literal(machine.state_name)}, dfa: #{ruby_literal(data)} }"
231
175
  end
232
176
  # W016 is derived from wall-clock compilation time and must not make
@@ -236,6 +180,38 @@ module Flexr
236
180
  "stats: #{ruby_literal(compiled.stats)}, diagnostics: #{ruby_literal(diagnostics)} }"
237
181
  end
238
182
 
183
+ def trailing_dfa_expression(rule, parsed)
184
+ return "" unless rule.trailing
185
+
186
+ dfa = Flexr.compile_pattern(normalize_trailing(rule.trailing), encoding: parsed.config[:encoding],
187
+ options: { unicode: effective_options(parsed)[:unicode] == true })
188
+ "trailing_dfa: #{ruby_literal(dfa_data(dfa, compression: table_compression(parsed)))}, "
189
+ rescue CompileError => e
190
+ e.diagnostic.location ||= rule.span if e.diagnostic
191
+ raise
192
+ end
193
+
194
+ def dfa_data(dfa, compression:, direct: nil)
195
+ data = {
196
+ accepts: dfa.accepts, ec: dfa.ec, class_count: dfa.class_count,
197
+ state_count: dfa.states, start: dfa.start, rule_ids: dfa.rule_ids
198
+ }
199
+ pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
200
+ if direct
201
+ data[:direct] = direct
202
+ elsif pack_tables
203
+ packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
204
+ data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
205
+ Codegen::TablePacker.encode(packed)
206
+ else
207
+ packed
208
+ end
209
+ else
210
+ data[:transitions] = dfa.transitions
211
+ end
212
+ data
213
+ end
214
+
239
215
  def standalone?(parsed)
240
216
  @options.fetch(:standalone, parsed.config[:options]&.key?(:standalone) || false)
241
217
  end
@@ -253,12 +229,18 @@ module Flexr
253
229
  @options.fetch(:table_compression, parsed.config[:options]&.fetch(:table_compression, :rows) || :rows).to_sym
254
230
  end
255
231
 
256
- def embedded_runtime
257
- RUNTIME_SOURCES.map do |relative|
232
+ def embedded_runtime(parsed)
233
+ sources = RUNTIME_SOURCES.dup
234
+ sources.insert(6, *MATCHER_RUNTIME_SOURCES) if matcher_runtime?(parsed)
235
+ sources.map do |relative|
258
236
  File.binread(File.expand_path(relative, __dir__))
259
237
  end.join("\n")
260
238
  end
261
239
 
240
+ def matcher_runtime?(parsed)
241
+ effective_backend(parsed) == :firstmatch || parsed.rules.any?(&:trailing)
242
+ end
243
+
262
244
  def encoding_expression(encoding)
263
245
  encoding == Encoding::BINARY ? "Encoding::BINARY" : "Encoding::UTF_8"
264
246
  end
@@ -275,11 +257,23 @@ module Flexr
275
257
  "proc { emit(nil, text) }"
276
258
  end
277
259
 
260
+ def generated_rule_action(rule)
261
+ return ruby_literal(:__flexr_generated_method) if rule.action.is_a?(String) &&
262
+ ((!rule.bind_action && !inline_action_body(rule).nil?) || (!rule.reads_locals && inline_simple_action(rule.action_source)))
263
+
264
+ action_expression(rule.action)
265
+ end
266
+
278
267
  def generated_action_source(parsed, indent)
279
268
  lines = ["#{indent}def __flexr_generated_execute(rule)", "#{indent} case rule.index"]
280
269
  parsed.rules.each do |rule|
281
270
  lines << "#{indent} when #{rule.index}"
282
- body = inline_action_body(rule)
271
+ body = inline_simple_action(rule.action_source) if rule.bind_action && !rule.reads_locals
272
+ if rule.bind_action && body.nil?
273
+ lines << "#{indent} instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))"
274
+ next
275
+ end
276
+ body ||= inline_action_body(rule)
283
277
  if body.nil?
284
278
  lines << "#{indent} instance_exec(&rule.action)"
285
279
  else
@@ -344,6 +338,7 @@ module Flexr
344
338
 
345
339
  def inline_simple_action(source)
346
340
  body = source.to_s.strip
341
+ return if body.include?('#{') || body.match?(/text\.byteslice\([^)]*[A-Za-z_]/)
347
342
  body = body[1...-1].strip if body.start_with?("{") && body.end_with?("}")
348
343
  match = INLINE_EMIT.match(body)
349
344
  return unless match
@@ -369,9 +364,7 @@ module Flexr
369
364
  loader = Object.new
370
365
  loader.define_singleton_method(:load_spec) do
371
366
  # --eval is opt-in and intentionally executes trusted specification code.
372
- # rubocop:disable Security/Eval
373
- eval(source, TOPLEVEL_BINDING, @path, 1)
374
- # rubocop:enable Security/Eval
367
+ eval(source, TOPLEVEL_BINDING, @path, 1) # rubocop:disable Security/Eval
375
368
  end
376
369
  loader.instance_variable_set(:@path, @path)
377
370
  loader.load_spec
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Grammar
5
+ module_function
6
+
7
+ def tokens(source)
8
+ names = []
9
+ declaration = false
10
+ source.scan(%r{%\{.*?%\}|/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|<[^>]*>|%%|%[a-zA-Z][\w-]*|[a-zA-Z_]\w*|.}m).each do |token|
11
+ break if token == "%%"
12
+ if token.start_with?("%") && !token.start_with?("%{")
13
+ declaration = token == "%token"
14
+ elsif declaration && token.match?(/\A[A-Za-z_]\w*\z/)
15
+ names << token
16
+ end
17
+ end
18
+ names.uniq
19
+ end
20
+
21
+ def diagnostics(parsed, path)
22
+ expected = tokens(File.read(path))
23
+ emitted = Array(parsed.config[:declared_tokens]).map(&:to_s)
24
+ location = [parsed.first_dsl_offset, parsed.first_dsl_offset]
25
+ (expected - emitted).map do |name|
26
+ Diagnostics.warning("FLEXR-W017", "grammar token #{name} is not declared by emits",
27
+ location: location, note: "grammar: #{path}", help: "declare and emit #{name}, or remove it from the grammar")
28
+ end + (emitted - expected).map do |name|
29
+ Diagnostics.warning("FLEXR-W018", "emitted token #{name} is not declared by the grammar",
30
+ location: location, note: "grammar: #{path}", help: "add a %token declaration or remove #{name} from emits")
31
+ end
32
+ end
33
+
34
+ def declarations(tokens)
35
+ names = tokens.map(&:to_s).uniq
36
+ raise ArgumentError, "Lrama token names must be identifiers" unless names.all? { |name| name.match?(/\A[A-Za-z_]\w*\z/) }
37
+
38
+ names.empty? ? "" : "%token #{names.join(' ')}"
39
+ end
40
+ end
41
+ end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+
3
5
  module Flexr
4
6
  class Importer
5
7
  Result = Struct.new(:source, :warnings, :complete?, keyword_init: true)
@@ -387,46 +389,8 @@ module Flexr
387
389
 
388
390
  def rexical_dfa_counterexample(regexps, automata)
389
391
  first, second = automata
390
- queue = [[first.start, second.start, false, +"".b]]
391
- visited = { [first.start, second.start, false] => true }
392
- bytes = rexical_byte_representatives(first, second)
393
-
394
- until queue.empty?
395
- first_state, second_state, first_seen, input = queue.shift
396
- bytes.each do |byte|
397
- next_first = first_state && first.transition(first_state, byte)
398
- next_second = second.transition(second_state, byte)
399
- next unless next_second
400
-
401
- next_input = input + byte.chr(Encoding::BINARY)
402
- next_first_seen = first_seen || rexical_accepting?(first, next_first)
403
- if next_first_seen && rexical_accepting?(second, next_second) &&
404
- !rexical_accepting?(first, next_first)
405
- lengths = rexical_match_lengths(regexps, next_input)
406
- return lengths if lengths
407
- end
408
-
409
- key = [next_first, next_second, next_first_seen]
410
- next if visited[key]
411
-
412
- visited[key] = true
413
- queue << [next_first, next_second, next_first_seen, next_input]
414
- end
415
- end
416
- nil
417
- end
418
-
419
- def rexical_byte_representatives(first, second)
420
- representatives = {}
421
- 256.times do |byte|
422
- key = [first.ec[byte], second.ec[byte]]
423
- representatives[key] ||= byte
424
- end
425
- representatives.values.sort_by { |byte| [byte.between?(32, 126) ? 0 : 1, byte] }
426
- end
427
-
428
- def rexical_accepting?(dfa, state)
429
- state && !dfa.accepts.fetch(state).empty?
392
+ input = Automaton::Analysis.firstmatch_counterexample(first, second)
393
+ input && rexical_match_lengths(regexps, input)
430
394
  end
431
395
 
432
396
  def rexical_match_lengths(regexps, input)
data/lib/flexr/ir.rb CHANGED
@@ -4,7 +4,7 @@ module Flexr
4
4
  module IR
5
5
  Rule = Struct.new(
6
6
  :index, :patterns, :trailing, :action, :states, :bol_only, :end_anchor, :location,
7
- :pattern_conditions,
7
+ :pattern_conditions, :trailing_dfa,
8
8
  keyword_init: true
9
9
  ) do
10
10
  def skip?
@@ -16,7 +16,7 @@ module Flexr
16
16
  end
17
17
  end
18
18
 
19
- State = Struct.new(:name, :inclusive, :id, keyword_init: true)
19
+ State = Struct.new(:name, :inclusive, :id, :location, keyword_init: true)
20
20
 
21
21
  Spec = Struct.new(
22
22
  :class_name, :superclass, :backend, :token_kind, :encoding, :options,
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Migration
5
+ module_function
6
+
7
+ def diagnostics(parsed)
8
+ require "prism"
9
+ source = parsed.source
10
+ result = [Diagnostics.info("FLEXR-I002", "v2.0 requires Ruby 3.4+; regenerate all generated lexers after upgrading",
11
+ help: "review docs/migration-v2.md and consumer code; static checks cannot find all record mutations or initialization calls")]
12
+ visit = lambda do |node|
13
+ return unless node
14
+
15
+ if node.is_a?(Prism::CallNode) && parsed.dsl_spans.include?([node.location.start_offset, node.location.end_offset]) &&
16
+ node.receiver.nil? && node.name == :token_kind &&
17
+ node.arguments&.arguments&.first.is_a?(Prism::SymbolNode) &&
18
+ node.arguments.arguments.first.unescaped == "struct"
19
+ result << Diagnostics.warning(
20
+ "FLEXR-W020", "structured tokens and locations become immutable Data records in v2.0",
21
+ location: [node.location.start_offset, node.location.end_offset],
22
+ help: "replace field assignments with record.with(...); review consumer code and Struct-specific APIs"
23
+ )
24
+ end
25
+ node.compact_child_nodes.each { |child| visit.call(child) }
26
+ end
27
+ visit.call(Prism.parse(source).value)
28
+ result
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "minitest/assertions"
4
+ require_relative "testing"
5
+
6
+ module Flexr
7
+ module Minitest
8
+ module Assertions
9
+ def assert_tokenize(lexer, input, expected, **)
10
+ assert_equal expected, lexer.new(input, **).tokens
11
+ end
12
+
13
+ def assert_token_snapshot(lexer, input, path, update: false, **)
14
+ actual = Testing::Snapshot.render(lexer.new(input, **).tokens)
15
+ begin
16
+ expected = Testing::Snapshot.expected(path, actual, update: update)
17
+ rescue Errno::ENOENT
18
+ flunk "missing token snapshot #{path}; create it with update: true"
19
+ end
20
+ assert_equal expected, actual, "token snapshot differs: #{path}"
21
+ end
22
+ end
23
+ end
24
+ end
data/lib/flexr/options.rb CHANGED
@@ -1,9 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+
3
5
  module Flexr
4
6
  Options = Struct.new(
5
7
  :backend, :token_kind, :accel, :standalone, :eval_mode, :table_compression,
6
- :table_format, :max_dfa_states, :warn_level, :warn_as_error, :color, :format, :overrides,
8
+ :table_format, :max_dfa_states, :warn_level, :warn_as_error, :color, :format, :overrides, :grammar, :target,
7
9
  keyword_init: true
8
10
  ) do
9
11
  def self.default
@@ -20,14 +22,16 @@ module Flexr
20
22
  end
21
23
 
22
24
  def validate!
23
- validate_value!(backend, %i[table direct firstmatch auto], :backend)
24
- validate_value!(token_kind, %i[array struct yield], :token_kind)
25
- validate_value!(accel, %i[auto strscan regexp none], :accel)
25
+ validate_value!(backend, Configuration::BACKENDS, :backend)
26
+ validate_value!(token_kind, Configuration::TOKEN_KINDS, :token_kind)
27
+ validate_value!(accel, Configuration::ACCELERATORS, :accel)
26
28
  validate_value!(table_compression, %i[none rows full], :table_compression)
27
29
  validate_value!(table_format, %i[literal packed], :table_format)
28
30
  validate_value!(warn_level, %i[all default none], :warn_level)
29
31
  validate_value!(color, %i[auto always never], :color)
30
- validate_value!(format, %i[human json], :format)
32
+ validate_value!(format, %i[human json lrama], :format)
33
+ raise ArgumentError, "unsupported target: #{target.inspect}" unless target.nil? || target == "2.0"
34
+ raise ArgumentError, "--target requires static check; remove --eval" if target && eval_mode
31
35
  raise ArgumentError, "unsupported max_dfa_states: #{max_dfa_states.inspect}" unless max_dfa_states.is_a?(Integer) && max_dfa_states.positive?
32
36
  self
33
37
  end
@@ -1,5 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+ require_relative "artifact_writer"
5
+ require_relative "generator"
6
+
3
7
  module Flexr
4
8
  class RakeTask
5
9
  attr_accessor :spec, :output, :warn_as_error
@@ -17,7 +21,8 @@ module Flexr
17
21
  require "rake"
18
22
  raise ArgumentError, "spec is required" unless @spec
19
23
 
20
- output = @output || @spec.sub(/\.flexr\.rb\z/, ".rb")
24
+ output = @output || ArtifactWriter.default_generated_path(@spec)
25
+ ArtifactWriter.ensure_distinct!(@spec, output)
21
26
  Rake::FileTask.define_task(output => @spec) do
22
27
  Generator.new(@spec, output: output, options: { warn_as_error: @warn_as_error }).generate
23
28
  end
@@ -4,11 +4,13 @@ module Flexr
4
4
  module Regexp
5
5
  module AST
6
6
  Empty = Struct.new(:loc, keyword_init: true)
7
+ Fail = Struct.new(:loc, keyword_init: true)
7
8
  ByteRange = Struct.new(:lo, :hi, :loc, keyword_init: true)
8
9
  CodepointRange = Struct.new(:lo, :hi, :loc, keyword_init: true)
9
10
  Seq = Struct.new(:children, :loc, keyword_init: true)
10
11
  Alt = Struct.new(:children, :loc, keyword_init: true)
11
12
  Star = Struct.new(:child, :loc, keyword_init: true)
13
+ Repeat = Struct.new(:child, :minimum, :maximum, :loc, keyword_init: true)
12
14
  Anchor = Struct.new(:kind, :loc, keyword_init: true)
13
15
  TrailMark = Struct.new(:rule_id, :loc, keyword_init: true)
14
16
  CharClass = Struct.new(:ranges, :negated, :loc, keyword_init: true)
@@ -18,18 +20,19 @@ module Flexr
18
20
  def to_s
19
21
  name = self.class.name.split("::").last
20
22
  case name
21
- when "Empty" then "Empty"
23
+ when "Empty", "Fail" then name
22
24
  when "ByteRange" then "ByteRange(#{lo}..#{hi})"
23
25
  when "CodepointRange" then "CodepointRange(#{lo}..#{hi})"
24
26
  when "Anchor" then "Anchor(#{kind})"
25
27
  when "Star" then "Star(#{child})"
28
+ when "Repeat" then "Repeat(#{child}, #{minimum}, #{maximum || '∞'})"
26
29
  when "Seq", "Alt" then "#{name}(#{children.join(', ')})"
27
30
  else super
28
31
  end
29
32
  end
30
33
  end
31
34
 
32
- [Empty, ByteRange, CodepointRange, Seq, Alt, Star, Anchor, TrailMark, CharClass].each do |node_class|
35
+ [Empty, Fail, ByteRange, CodepointRange, Seq, Alt, Star, Repeat, Anchor, TrailMark, CharClass].each do |node_class|
33
36
  node_class.include(Formatting)
34
37
  end
35
38
 
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Regexp
5
+ module Matcher
6
+ module_function
7
+
8
+ def reference?(pattern, encoding:, unicode: false)
9
+ return false unless pattern.is_a?(::Regexp)
10
+ return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
11
+
12
+ unicode && encoding == Encoding::UTF_8 && pattern.source.match?(/\\[dDwWsS]/)
13
+ end
14
+
15
+ def match(pattern, subject, encoding:, unicode:, reference:)
16
+ if reference
17
+ Unicode::ReferenceRegexp.match(pattern, subject, encoding: encoding, options: pattern.options, unicode: unicode)
18
+ else
19
+ pattern.match(subject, 0)
20
+ end
21
+ rescue ArgumentError, RegexpError
22
+ nil
23
+ end
24
+ end
25
+ end
26
+ end