flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -50,23 +50,46 @@ module Flexr
50
50
  @items.dup
51
51
  end
52
52
 
53
- def render(format: :human, color: :auto)
53
+ def render(format: :human, color: :auto, source: nil, path: nil)
54
54
  return JSON.generate(@items.map(&:to_h)) if format.to_sym == :json
55
55
 
56
- @items.map { |item| render_one(item, color: color_enabled?(color)) }.join("\n")
56
+ @items.map { |item| render_one(item, color: color_enabled?(color), source: source, path: path) }.join("\n")
57
57
  end
58
58
 
59
59
  private
60
60
 
61
- def render_one(item, color:)
61
+ def render_one(item, color:, source:, path:)
62
62
  prefix = "#{item.severity}[#{item.code}]: #{item.message}"
63
63
  prefix = "\e[31m#{prefix}\e[0m" if [true, :always].include?(color)
64
64
  lines = [prefix]
65
+ lines.concat(render_source(item.location, source, path))
65
66
  lines << " help: #{item.help}" if item.help
66
67
  lines << " note: #{item.note}" if item.note
67
68
  lines.join("\n")
68
69
  end
69
70
 
71
+ def render_source(location, source, path)
72
+ return [] unless source.is_a?(String) && location.is_a?(Array) && location.length == 2 &&
73
+ location.all?(Integer)
74
+
75
+ first, last = location
76
+ return [] unless first.between?(0, source.bytesize) && last >= first
77
+
78
+ before = source.byteslice(0, first).b
79
+ line = before.count("\n") + 1
80
+ line_start = (before.rindex("\n") || -1) + 1
81
+ line_end = source.b.index("\n", first) || source.bytesize
82
+ text = source.byteslice(line_start...line_end).delete_suffix("\r")
83
+ prefix = source.byteslice(line_start...first).delete_suffix("\r").scrub
84
+ last = last.clamp(first, [line_start + text.bytesize, first].max)
85
+ width = [source.byteslice(first...last).scrub.length, 1].max
86
+ gutter = " " * line.to_s.length
87
+ # shortcut: carets count characters; use display-cell widths to align wide or combining glyphs.
88
+ indentation = prefix.gsub(/[^\t]/, " ")
89
+ [" --> #{path || '(source)'}:#{line}:#{prefix.length + 1}",
90
+ "#{gutter} |", "#{line} | #{text.scrub}", "#{gutter} | #{indentation}#{'^' * width}"]
91
+ end
92
+
70
93
  def color_enabled?(color)
71
94
  return false if ENV.key?("NO_COLOR") || color == :never || color == false
72
95
  return true if [:always, true].include?(color)
@@ -78,6 +101,15 @@ module Flexr
78
101
  module Diagnostics
79
102
  module_function
80
103
 
104
+ def rule_label(rule)
105
+ return if rule.nil?
106
+ return "rule #{rule}" unless rule.is_a?(IR::Rule)
107
+
108
+ pattern = rule.patterns.one? ? rule.patterns.first : rule.patterns
109
+ label = "rule #{pattern.inspect}"
110
+ rule.trailing ? "#{label}, followed_by: #{rule.trailing.inspect}" : label
111
+ end
112
+
81
113
  def error(code, message, location: nil, help: nil, note: nil)
82
114
  Diagnostic.new(code: code, severity: :error, message: message, location: location, help: help, note: note)
83
115
  end
@@ -86,6 +118,10 @@ module Flexr
86
118
  Diagnostic.new(code: code, severity: :warning, message: message, location: location, help: help, note: note)
87
119
  end
88
120
 
121
+ def info(code, message, location: nil, help: nil, note: nil)
122
+ Diagnostic.new(code: code, severity: :info, message: message, location: location, help: help, note: note)
123
+ end
124
+
89
125
  def raise!(diagnostic)
90
126
  klass = diagnostic.code == "FLEXR-E014" ? UnsupportedRegexpError : CompileError
91
127
  raise klass.new(diagnostic.message, diagnostic: diagnostic)
data/lib/flexr/dsl.rb CHANGED
@@ -20,6 +20,7 @@ module Flexr
20
20
  @__flexr_compiled = nil
21
21
  @__flexr_generated = false
22
22
  @__flexr_generated_actions = []
23
+ @__flexr_generated_eof_actions = {}
23
24
  @__flexr_specification_frozen = false
24
25
  @__flexr_compile_mutex = Monitor.new
25
26
  end
@@ -96,12 +97,7 @@ module Flexr
96
97
  end
97
98
 
98
99
  def encoding(value)
99
- encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
100
- unless [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
101
- diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
102
- raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
103
- end
104
-
100
+ encoding = Configuration.encoding!(value)
105
101
  __flexr_mutate! { @__flexr_config.encoding = encoding }
106
102
  end
107
103
 
@@ -116,9 +112,10 @@ module Flexr
116
112
  end
117
113
 
118
114
  def compile!
115
+ return @__flexr_compiled if @__flexr_compiled
116
+
119
117
  @__flexr_compile_mutex.synchronize do
120
118
  # The ivar is part of the generated/runtime class contract.
121
- # rubocop:disable Naming/MemoizedInstanceVariableName
122
119
  @__flexr_compiled ||= begin
123
120
  compiled = Automaton::Compiler.new(__flexr_spec).compile
124
121
  @__flexr_config.backend = auto_direct?(compiled) ? :direct : :table if @__flexr_config.backend == :auto
@@ -126,7 +123,6 @@ module Flexr
126
123
  freeze_compiled!(compiled)
127
124
  compiled
128
125
  end
129
- # rubocop:enable Naming/MemoizedInstanceVariableName
130
126
  end
131
127
  end
132
128
 
@@ -151,7 +147,8 @@ module Flexr
151
147
  )
152
148
  end
153
149
 
154
- attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions
150
+ attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions,
151
+ :__flexr_generated_eof_actions
155
152
 
156
153
  def __flexr_add_generated_eof(state, action)
157
154
  __flexr_mutate! { @__flexr_eof_rules[state.to_sym] = action }
@@ -174,6 +171,12 @@ module Flexr
174
171
  @__flexr_generated_actions[index] = action
175
172
  end
176
173
 
174
+ def __flexr_bind_generated_eof(state, action)
175
+ raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
176
+
177
+ @__flexr_generated_eof_actions[state.to_sym] = action
178
+ end
179
+
177
180
  def __flexr_generated?
178
181
  @__flexr_generated == true
179
182
  end
data/lib/flexr/errors.rb CHANGED
@@ -2,23 +2,23 @@
2
2
 
3
3
  module Flexr
4
4
  class Error < StandardError
5
- attr_reader :diagnostic
5
+ attr_reader :diagnostic, :filename, :byte_pos
6
6
 
7
- def initialize(message = nil, diagnostic: nil)
7
+ def initialize(message = nil, diagnostic: nil, filename: nil, byte_pos: nil)
8
8
  @diagnostic = diagnostic
9
+ @filename = filename
10
+ @byte_pos = byte_pos
9
11
  super(message || diagnostic&.message)
10
12
  end
11
13
  end
12
14
 
13
15
  class LexError < Error
14
- attr_reader :filename, :byte_pos, :line, :text
16
+ attr_reader :line, :text
15
17
 
16
18
  def initialize(message, filename: nil, byte_pos: nil, line: nil, text: nil, diagnostic: nil)
17
- @filename = filename
18
- @byte_pos = byte_pos
19
19
  @line = line
20
20
  @text = text
21
- super(message, diagnostic: diagnostic)
21
+ super(message, diagnostic: diagnostic, filename: filename, byte_pos: byte_pos)
22
22
  end
23
23
  end
24
24
 
@@ -2,8 +2,8 @@
2
2
 
3
3
  module Flexr
4
4
  module Generated
5
- ARTIFACT_SCHEMA_VERSION = 1
6
- RUNTIME_ABI_VERSION = 1
5
+ ARTIFACT_SCHEMA_VERSION = 2
6
+ RUNTIME_ABI_VERSION = 2
7
7
 
8
8
  module_function
9
9
 
@@ -46,23 +46,22 @@ module Flexr
46
46
  end
47
47
 
48
48
  def install_compiled!(klass, payload)
49
- validate_artifact!(payload[:artifact])
49
+ begin
50
+ validate_artifact!(payload[:artifact])
51
+ rescue CompileError => e
52
+ caller = caller_locations.find { |frame| frame.base_label != "install_compiled!" }
53
+ path = caller&.absolute_path || caller&.path
54
+ raise unless path && File.file?(path)
55
+
56
+ e.diagnostic.location ||= [0, 0]
57
+ raise CompileError.new(e.message, diagnostic: e.diagnostic, filename: path, byte_pos: 0)
58
+ end
50
59
  install!(klass, payload)
51
60
  machines = payload.fetch(:compiled).fetch(:machines).transform_values do |machine|
52
- dfa_data = machine.fetch(:dfa)
53
- packed = decode_packed(dfa_data[:packed])
54
- dfa = Automaton::DFA.new(
55
- transitions: dfa_data[:transitions],
56
- accepts: dfa_data.fetch(:accepts),
57
- ec: dfa_data.fetch(:ec),
58
- class_count: dfa_data.fetch(:class_count),
59
- start: dfa_data.fetch(:start),
60
- rule_ids: dfa_data.fetch(:rule_ids),
61
- packed: packed,
62
- direct: dfa_data[:direct],
63
- state_count: dfa_data.fetch(:state_count)
64
- )
65
- Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym)
61
+ dfa = load_dfa(machine.fetch(:dfa))
62
+ plans = Automaton::Accel.whole_token_plans(dfa, klass.__flexr_rules) if
63
+ klass.__flexr_config.backend != :firstmatch && klass.__flexr_config.options[:accel] == :strscan
64
+ Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym, whole_tokens: plans)
66
65
  end
67
66
  compiled = Automaton::CompiledSpec.new(
68
67
  machines: machines,
@@ -99,6 +98,43 @@ module Flexr
99
98
  incompatible_artifact!(detail)
100
99
  end
101
100
 
101
+ def shareable!(klass)
102
+ unless klass.__flexr_generated? && klass.__flexr_rules.none? { |rule| rule.action.is_a?(Proc) } &&
103
+ klass.__flexr_generated_actions.empty? && klass.__flexr_spec.eof_rules.empty?
104
+ raise ArgumentError, "Ractor sharing requires generated method actions without captured closures or EOF blocks"
105
+ end
106
+
107
+ config = klass.__flexr_config
108
+ if config.backend == :firstmatch && klass.__flexr_rules.any? do |rule|
109
+ rule.patterns.any? do |pattern|
110
+ Regexp::Matcher.reference?(pattern, encoding: config.encoding, unicode: config.options[:unicode] == true)
111
+ end
112
+ end
113
+ raise ArgumentError, "Ractor sharing does not support firstmatch reference patterns; use table, direct, or auto"
114
+ end
115
+
116
+ original = klass.__flexr_compiled
117
+ rules = original.rules.map do |rule|
118
+ IR::Rule.new(**rule.to_h, trailing_dfa: rule.trailing_dfa && shareable_dfa(rule.trailing_dfa))
119
+ end
120
+ machines = original.machines.transform_values do |machine|
121
+ Automaton::Machine.new(dfa: shareable_dfa(machine.dfa), state_name: machine.state_name,
122
+ whole_tokens: machine.whole_tokens)
123
+ end
124
+ compiled = Automaton::CompiledSpec.new(**original.to_h, machines: machines, rules: rules)
125
+ Ractor.make_shareable([compiled, klass.__flexr_config, klass.__flexr_states, klass.__flexr_spec.eof_rules,
126
+ klass.__flexr_generated_actions, klass.__flexr_generated_eof_actions])
127
+ klass.instance_variable_set(:@__flexr_rules, rules)
128
+ klass.__flexr_set_compiled!(compiled)
129
+ klass
130
+ end
131
+
132
+ def shareable_dfa(dfa)
133
+ Automaton::DFA.new(transitions: dfa.transitions.map(&:dup), accepts: dfa.accepts, ec: dfa.ec,
134
+ class_count: dfa.class_count, start: dfa.start, rule_ids: dfa.rule_ids,
135
+ direct: dfa.direct)
136
+ end
137
+
102
138
  def incompatible_artifact!(detail)
103
139
  diagnostic = Diagnostics.error(
104
140
  "FLEXR-E020", "generated artifact is incompatible with this runtime",
@@ -119,6 +155,14 @@ module Flexr
119
155
  }
120
156
  end
121
157
 
158
+ def load_dfa(data)
159
+ Automaton::DFA.new(
160
+ transitions: data[:transitions], accepts: data.fetch(:accepts), ec: data.fetch(:ec),
161
+ class_count: data.fetch(:class_count), start: data.fetch(:start), rule_ids: data.fetch(:rule_ids),
162
+ packed: decode_packed(data[:packed]), direct: data[:direct], state_count: data.fetch(:state_count)
163
+ )
164
+ end
165
+
122
166
  def decode_array(encoded, nil_value: nil)
123
167
  values = encoded.unpack1("m0").unpack("l<*")
124
168
  return values unless nil_value
@@ -138,6 +182,7 @@ module Flexr
138
182
  states: Array(definition.fetch(:states)).map(&:to_sym),
139
183
  bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
140
184
  location: definition[:span] || definition[:location],
185
+ trailing_dfa: definition[:trailing_dfa] && Generated.load_dfa(definition[:trailing_dfa]),
141
186
  pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
142
187
  next unless condition
143
188
 
@@ -146,6 +191,9 @@ module Flexr
146
191
  end
147
192
  )
148
193
  end
194
+ rescue CompileError => e
195
+ e.diagnostic.location ||= definition[:span] || definition[:location] if e.diagnostic
196
+ raise
149
197
  end
150
198
  end
151
199
  end
@@ -15,15 +15,15 @@ module Flexr
15
15
 
16
16
  RUNTIME_SOURCES = %w[
17
17
  version.rb errors.rb diagnostics.rb action_resolver.rb configuration.rb ir.rb
18
- unicode/version.rb automaton/types.rb automaton/dfa.rb automaton/analysis.rb automaton/accel.rb
19
- runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
18
+ unicode/version.rb automaton/types.rb automaton/dfa.rb automaton/accel.rb
19
+ runtime/record.rb runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
20
20
  runtime/interpreter.rb runtime/core.rb dsl.rb lexer.rb generated.rb
21
21
  ].freeze
22
22
 
23
23
  MATCHER_RUNTIME_SOURCES = %w[
24
24
  regexp/ast.rb regexp/parser.rb regexp/unsupported.rb
25
25
  unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb
26
- unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb
26
+ unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb regexp/matcher.rb
27
27
  ].freeze
28
28
 
29
29
  def initialize(path, output: nil, eval_mode: false, options: {})
@@ -62,6 +62,11 @@ module Flexr
62
62
  install = "#{install}#{Codegen::Table.new(compiled).source(indent: indent)}"
63
63
  install = "#{install}#{generated_action_source(parsed, indent)}"
64
64
  edits = parsed.dsl_edits.dup
65
+ parsed.rules.each do |rule|
66
+ next unless rule.bind_action && !rule.reads_locals && inline_simple_action(rule.action_source)
67
+
68
+ edits.map! { |edit| edit.first(2) == rule.span ? [*rule.span, ""] : edit }
69
+ end
65
70
  edits.concat(parsed.flexr_require_spans.map { |span| [*span, ""] }) if standalone?(parsed)
66
71
  result = Source::Passthrough.rewrite(
67
72
  parsed.source, edits, insertion: parsed.first_dsl_offset, payload: install
@@ -79,7 +84,7 @@ module Flexr
79
84
  "# eval: #{@eval_mode}",
80
85
  "# standalone: #{standalone?(parsed)}"
81
86
  ].join("\n")
82
- prefix = standalone?(parsed) ? "require \"json\"\nrequire \"monitor\"\n#{embedded_runtime(parsed)}\n" : ""
87
+ prefix = standalone?(parsed) ? "raise LoadError, \"flexr requires Ruby 3.4 or newer\" if RUBY_VERSION < \"3.4\"\nrequire \"json\"\nrequire \"monitor\"\n#{embedded_runtime(parsed)}\n" : ""
83
88
  Source::Passthrough.insert_after_preamble(result, "#{header}\n#{prefix}")
84
89
  end
85
90
 
@@ -89,7 +94,8 @@ module Flexr
89
94
  [condition.rule_index, condition.pattern_index, condition.bol_only, condition.end_anchor]
90
95
  end
91
96
  "{ index: #{rule.index}, patterns: #{ruby_literal(rule.patterns)}, pattern_conditions: #{ruby_literal(conditions)}, trailing: #{ruby_literal(rule.trailing)}, " \
92
- "action: #{action_expression(rule.action)}, states: #{rule.states.inspect}, " \
97
+ "#{trailing_dfa_expression(rule, parsed)}" \
98
+ "action: #{generated_rule_action(rule)}, states: #{rule.states.inspect}, " \
93
99
  "bol_only: #{ruby_literal(rule.bol_only)}, end_anchor: #{ruby_literal(rule.end_anchor)} }"
94
100
  end
95
101
  "{ artifact: #{ruby_literal(artifact_metadata)}, rules: [#{definitions.join(', ')}], " \
@@ -111,18 +117,21 @@ module Flexr
111
117
 
112
118
  def compile_parsed(parsed)
113
119
  states = parsed.states.each_with_index.to_h do |(name, value), index|
114
- [name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index)]
120
+ [name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index, location: value[:location])]
115
121
  end
116
122
  rules = parsed.rules.map do |rule|
117
123
  IR::Rule.new(index: rule.index, patterns: rule.patterns, trailing: normalize_trailing(rule.trailing),
118
124
  action: rule.action, states: rule.states, bol_only: rule.bol_only,
119
125
  end_anchor: rule.end_anchor, location: rule.span)
126
+ rescue CompileError => e
127
+ e.diagnostic.location ||= rule.span if e.diagnostic
128
+ raise
120
129
  end
121
130
  spec = IR::Spec.new(
122
131
  class_name: parsed.class_name, superclass: "Flexr::Lexer",
123
132
  backend: @options.fetch(:backend, parsed.config[:backend]),
124
133
  token_kind: @options.fetch(:token_kind, parsed.config[:token_kind]),
125
- encoding: parsed.config[:encoding], options: effective_options(parsed),
134
+ encoding: parsed.config[:encoding], options: effective_options(parsed).merge(warn_level: @options[:warn_level]),
126
135
  declared_tokens: parsed.config[:declared_tokens], states: states, rules: rules,
127
136
  eof_rules: parsed.config[:eof_rules] || {}, verbatim: parsed.source
128
137
  )
@@ -160,28 +169,8 @@ module Flexr
160
169
 
161
170
  def compiled_expression(compiled, compression: :none, backend: :table)
162
171
  machines = compiled.machines.map do |name, machine|
163
- dfa = machine.dfa
164
- data = {
165
- accepts: dfa.accepts,
166
- ec: dfa.ec,
167
- class_count: dfa.class_count,
168
- state_count: dfa.states,
169
- start: dfa.start,
170
- rule_ids: dfa.rule_ids
171
- }
172
- pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
173
- if backend == :direct
174
- data[:direct] = Codegen::Direct.new(compiled).generate(state: name)
175
- elsif pack_tables
176
- packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
177
- data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
178
- Codegen::TablePacker.encode(packed)
179
- else
180
- packed
181
- end
182
- else
183
- data[:transitions] = dfa.transitions
184
- end
172
+ direct = Codegen::Direct.new(compiled).generate(state: name) if backend == :direct
173
+ data = dfa_data(machine.dfa, compression: compression, direct: direct)
185
174
  "#{ruby_literal(name)} => { state_name: #{ruby_literal(machine.state_name)}, dfa: #{ruby_literal(data)} }"
186
175
  end
187
176
  # W016 is derived from wall-clock compilation time and must not make
@@ -191,6 +180,38 @@ module Flexr
191
180
  "stats: #{ruby_literal(compiled.stats)}, diagnostics: #{ruby_literal(diagnostics)} }"
192
181
  end
193
182
 
183
+ def trailing_dfa_expression(rule, parsed)
184
+ return "" unless rule.trailing
185
+
186
+ dfa = Flexr.compile_pattern(normalize_trailing(rule.trailing), encoding: parsed.config[:encoding],
187
+ options: { unicode: effective_options(parsed)[:unicode] == true })
188
+ "trailing_dfa: #{ruby_literal(dfa_data(dfa, compression: table_compression(parsed)))}, "
189
+ rescue CompileError => e
190
+ e.diagnostic.location ||= rule.span if e.diagnostic
191
+ raise
192
+ end
193
+
194
+ def dfa_data(dfa, compression:, direct: nil)
195
+ data = {
196
+ accepts: dfa.accepts, ec: dfa.ec, class_count: dfa.class_count,
197
+ state_count: dfa.states, start: dfa.start, rule_ids: dfa.rule_ids
198
+ }
199
+ pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
200
+ if direct
201
+ data[:direct] = direct
202
+ elsif pack_tables
203
+ packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
204
+ data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
205
+ Codegen::TablePacker.encode(packed)
206
+ else
207
+ packed
208
+ end
209
+ else
210
+ data[:transitions] = dfa.transitions
211
+ end
212
+ data
213
+ end
214
+
194
215
  def standalone?(parsed)
195
216
  @options.fetch(:standalone, parsed.config[:options]&.key?(:standalone) || false)
196
217
  end
@@ -236,15 +257,23 @@ module Flexr
236
257
  "proc { emit(nil, text) }"
237
258
  end
238
259
 
260
+ def generated_rule_action(rule)
261
+ return ruby_literal(:__flexr_generated_method) if rule.action.is_a?(String) &&
262
+ ((!rule.bind_action && !inline_action_body(rule).nil?) || (!rule.reads_locals && inline_simple_action(rule.action_source)))
263
+
264
+ action_expression(rule.action)
265
+ end
266
+
239
267
  def generated_action_source(parsed, indent)
240
268
  lines = ["#{indent}def __flexr_generated_execute(rule)", "#{indent} case rule.index"]
241
269
  parsed.rules.each do |rule|
242
270
  lines << "#{indent} when #{rule.index}"
243
- if rule.bind_action
271
+ body = inline_simple_action(rule.action_source) if rule.bind_action && !rule.reads_locals
272
+ if rule.bind_action && body.nil?
244
273
  lines << "#{indent} instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))"
245
274
  next
246
275
  end
247
- body = inline_action_body(rule)
276
+ body ||= inline_action_body(rule)
248
277
  if body.nil?
249
278
  lines << "#{indent} instance_exec(&rule.action)"
250
279
  else
@@ -309,6 +338,7 @@ module Flexr
309
338
 
310
339
  def inline_simple_action(source)
311
340
  body = source.to_s.strip
341
+ return if body.include?('#{') || body.match?(/text\.byteslice\([^)]*[A-Za-z_]/)
312
342
  body = body[1...-1].strip if body.start_with?("{") && body.end_with?("}")
313
343
  match = INLINE_EMIT.match(body)
314
344
  return unless match
@@ -334,9 +364,7 @@ module Flexr
334
364
  loader = Object.new
335
365
  loader.define_singleton_method(:load_spec) do
336
366
  # --eval is opt-in and intentionally executes trusted specification code.
337
- # rubocop:disable Security/Eval
338
- eval(source, TOPLEVEL_BINDING, @path, 1)
339
- # rubocop:enable Security/Eval
367
+ eval(source, TOPLEVEL_BINDING, @path, 1) # rubocop:disable Security/Eval
340
368
  end
341
369
  loader.instance_variable_set(:@path, @path)
342
370
  loader.load_spec
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Grammar
5
+ module_function
6
+
7
+ def tokens(source)
8
+ names = []
9
+ declaration = false
10
+ source.scan(%r{%\{.*?%\}|/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|<[^>]*>|%%|%[a-zA-Z][\w-]*|[a-zA-Z_]\w*|.}m).each do |token|
11
+ break if token == "%%"
12
+ if token.start_with?("%") && !token.start_with?("%{")
13
+ declaration = token == "%token"
14
+ elsif declaration && token.match?(/\A[A-Za-z_]\w*\z/)
15
+ names << token
16
+ end
17
+ end
18
+ names.uniq
19
+ end
20
+
21
+ def diagnostics(parsed, path)
22
+ expected = tokens(File.read(path))
23
+ emitted = Array(parsed.config[:declared_tokens]).map(&:to_s)
24
+ location = [parsed.first_dsl_offset, parsed.first_dsl_offset]
25
+ (expected - emitted).map do |name|
26
+ Diagnostics.warning("FLEXR-W017", "grammar token #{name} is not declared by emits",
27
+ location: location, note: "grammar: #{path}", help: "declare and emit #{name}, or remove it from the grammar")
28
+ end + (emitted - expected).map do |name|
29
+ Diagnostics.warning("FLEXR-W018", "emitted token #{name} is not declared by the grammar",
30
+ location: location, note: "grammar: #{path}", help: "add a %token declaration or remove #{name} from emits")
31
+ end
32
+ end
33
+
34
+ def declarations(tokens)
35
+ names = tokens.map(&:to_s).uniq
36
+ raise ArgumentError, "Lrama token names must be identifiers" unless names.all? { |name| name.match?(/\A[A-Za-z_]\w*\z/) }
37
+
38
+ names.empty? ? "" : "%token #{names.join(' ')}"
39
+ end
40
+ end
41
+ end
data/lib/flexr/ir.rb CHANGED
@@ -4,7 +4,7 @@ module Flexr
4
4
  module IR
5
5
  Rule = Struct.new(
6
6
  :index, :patterns, :trailing, :action, :states, :bol_only, :end_anchor, :location,
7
- :pattern_conditions,
7
+ :pattern_conditions, :trailing_dfa,
8
8
  keyword_init: true
9
9
  ) do
10
10
  def skip?
@@ -16,7 +16,7 @@ module Flexr
16
16
  end
17
17
  end
18
18
 
19
- State = Struct.new(:name, :inclusive, :id, keyword_init: true)
19
+ State = Struct.new(:name, :inclusive, :id, :location, keyword_init: true)
20
20
 
21
21
  Spec = Struct.new(
22
22
  :class_name, :superclass, :backend, :token_kind, :encoding, :options,
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Migration
5
+ module_function
6
+
7
+ def diagnostics(parsed)
8
+ require "prism"
9
+ source = parsed.source
10
+ result = [Diagnostics.info("FLEXR-I002", "v2.0 requires Ruby 3.4+; regenerate all generated lexers after upgrading",
11
+ help: "review docs/migration-v2.md and consumer code; static checks cannot find all record mutations or initialization calls")]
12
+ visit = lambda do |node|
13
+ return unless node
14
+
15
+ if node.is_a?(Prism::CallNode) && parsed.dsl_spans.include?([node.location.start_offset, node.location.end_offset]) &&
16
+ node.receiver.nil? && node.name == :token_kind &&
17
+ node.arguments&.arguments&.first.is_a?(Prism::SymbolNode) &&
18
+ node.arguments.arguments.first.unescaped == "struct"
19
+ result << Diagnostics.warning(
20
+ "FLEXR-W020", "structured tokens and locations become immutable Data records in v2.0",
21
+ location: [node.location.start_offset, node.location.end_offset],
22
+ help: "replace field assignments with record.with(...); review consumer code and Struct-specific APIs"
23
+ )
24
+ end
25
+ node.compact_child_nodes.each { |child| visit.call(child) }
26
+ end
27
+ visit.call(Prism.parse(source).value)
28
+ result
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "minitest/assertions"
4
+ require_relative "testing"
5
+
6
+ module Flexr
7
+ module Minitest
8
+ module Assertions
9
+ def assert_tokenize(lexer, input, expected, **)
10
+ assert_equal expected, lexer.new(input, **).tokens
11
+ end
12
+
13
+ def assert_token_snapshot(lexer, input, path, update: false, **)
14
+ actual = Testing::Snapshot.render(lexer.new(input, **).tokens)
15
+ begin
16
+ expected = Testing::Snapshot.expected(path, actual, update: update)
17
+ rescue Errno::ENOENT
18
+ flunk "missing token snapshot #{path}; create it with update: true"
19
+ end
20
+ assert_equal expected, actual, "token snapshot differs: #{path}"
21
+ end
22
+ end
23
+ end
24
+ end
data/lib/flexr/options.rb CHANGED
@@ -5,7 +5,7 @@ require_relative "runtime" unless defined?(Flexr::Lexer)
5
5
  module Flexr
6
6
  Options = Struct.new(
7
7
  :backend, :token_kind, :accel, :standalone, :eval_mode, :table_compression,
8
- :table_format, :max_dfa_states, :warn_level, :warn_as_error, :color, :format, :overrides,
8
+ :table_format, :max_dfa_states, :warn_level, :warn_as_error, :color, :format, :overrides, :grammar, :target,
9
9
  keyword_init: true
10
10
  ) do
11
11
  def self.default
@@ -29,7 +29,9 @@ module Flexr
29
29
  validate_value!(table_format, %i[literal packed], :table_format)
30
30
  validate_value!(warn_level, %i[all default none], :warn_level)
31
31
  validate_value!(color, %i[auto always never], :color)
32
- validate_value!(format, %i[human json], :format)
32
+ validate_value!(format, %i[human json lrama], :format)
33
+ raise ArgumentError, "unsupported target: #{target.inspect}" unless target.nil? || target == "2.0"
34
+ raise ArgumentError, "--target requires static check; remove --eval" if target && eval_mode
33
35
  raise ArgumentError, "unsupported max_dfa_states: #{max_dfa_states.inspect}" unless max_dfa_states.is_a?(Integer) && max_dfa_states.positive?
34
36
  self
35
37
  end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Regexp
5
+ module Matcher
6
+ module_function
7
+
8
+ def reference?(pattern, encoding:, unicode: false)
9
+ return false unless pattern.is_a?(::Regexp)
10
+ return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
11
+
12
+ unicode && encoding == Encoding::UTF_8 && pattern.source.match?(/\\[dDwWsS]/)
13
+ end
14
+
15
+ def match(pattern, subject, encoding:, unicode:, reference:)
16
+ if reference
17
+ Unicode::ReferenceRegexp.match(pattern, subject, encoding: encoding, options: pattern.options, unicode: unicode)
18
+ else
19
+ pattern.match(subject, 0)
20
+ end
21
+ rescue ArgumentError, RegexpError
22
+ nil
23
+ end
24
+ end
25
+ end
26
+ end