flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
data/lib/flexr/diagnostics.rb
CHANGED
|
@@ -50,23 +50,46 @@ module Flexr
|
|
|
50
50
|
@items.dup
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
def render(format: :human, color: :auto)
|
|
53
|
+
def render(format: :human, color: :auto, source: nil, path: nil)
|
|
54
54
|
return JSON.generate(@items.map(&:to_h)) if format.to_sym == :json
|
|
55
55
|
|
|
56
|
-
@items.map { |item| render_one(item, color: color_enabled?(color)) }.join("\n")
|
|
56
|
+
@items.map { |item| render_one(item, color: color_enabled?(color), source: source, path: path) }.join("\n")
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
private
|
|
60
60
|
|
|
61
|
-
def render_one(item, color:)
|
|
61
|
+
def render_one(item, color:, source:, path:)
|
|
62
62
|
prefix = "#{item.severity}[#{item.code}]: #{item.message}"
|
|
63
63
|
prefix = "\e[31m#{prefix}\e[0m" if [true, :always].include?(color)
|
|
64
64
|
lines = [prefix]
|
|
65
|
+
lines.concat(render_source(item.location, source, path))
|
|
65
66
|
lines << " help: #{item.help}" if item.help
|
|
66
67
|
lines << " note: #{item.note}" if item.note
|
|
67
68
|
lines.join("\n")
|
|
68
69
|
end
|
|
69
70
|
|
|
71
|
+
def render_source(location, source, path)
|
|
72
|
+
return [] unless source.is_a?(String) && location.is_a?(Array) && location.length == 2 &&
|
|
73
|
+
location.all?(Integer)
|
|
74
|
+
|
|
75
|
+
first, last = location
|
|
76
|
+
return [] unless first.between?(0, source.bytesize) && last >= first
|
|
77
|
+
|
|
78
|
+
before = source.byteslice(0, first).b
|
|
79
|
+
line = before.count("\n") + 1
|
|
80
|
+
line_start = (before.rindex("\n") || -1) + 1
|
|
81
|
+
line_end = source.b.index("\n", first) || source.bytesize
|
|
82
|
+
text = source.byteslice(line_start...line_end).delete_suffix("\r")
|
|
83
|
+
prefix = source.byteslice(line_start...first).delete_suffix("\r").scrub
|
|
84
|
+
last = last.clamp(first, [line_start + text.bytesize, first].max)
|
|
85
|
+
width = [source.byteslice(first...last).scrub.length, 1].max
|
|
86
|
+
gutter = " " * line.to_s.length
|
|
87
|
+
# shortcut: carets count characters; use display-cell widths to align wide or combining glyphs.
|
|
88
|
+
indentation = prefix.gsub(/[^\t]/, " ")
|
|
89
|
+
[" --> #{path || '(source)'}:#{line}:#{prefix.length + 1}",
|
|
90
|
+
"#{gutter} |", "#{line} | #{text.scrub}", "#{gutter} | #{indentation}#{'^' * width}"]
|
|
91
|
+
end
|
|
92
|
+
|
|
70
93
|
def color_enabled?(color)
|
|
71
94
|
return false if ENV.key?("NO_COLOR") || color == :never || color == false
|
|
72
95
|
return true if [:always, true].include?(color)
|
|
@@ -78,6 +101,15 @@ module Flexr
|
|
|
78
101
|
module Diagnostics
|
|
79
102
|
module_function
|
|
80
103
|
|
|
104
|
+
def rule_label(rule)
|
|
105
|
+
return if rule.nil?
|
|
106
|
+
return "rule #{rule}" unless rule.is_a?(IR::Rule)
|
|
107
|
+
|
|
108
|
+
pattern = rule.patterns.one? ? rule.patterns.first : rule.patterns
|
|
109
|
+
label = "rule #{pattern.inspect}"
|
|
110
|
+
rule.trailing ? "#{label}, followed_by: #{rule.trailing.inspect}" : label
|
|
111
|
+
end
|
|
112
|
+
|
|
81
113
|
def error(code, message, location: nil, help: nil, note: nil)
|
|
82
114
|
Diagnostic.new(code: code, severity: :error, message: message, location: location, help: help, note: note)
|
|
83
115
|
end
|
|
@@ -86,6 +118,10 @@ module Flexr
|
|
|
86
118
|
Diagnostic.new(code: code, severity: :warning, message: message, location: location, help: help, note: note)
|
|
87
119
|
end
|
|
88
120
|
|
|
121
|
+
def info(code, message, location: nil, help: nil, note: nil)
|
|
122
|
+
Diagnostic.new(code: code, severity: :info, message: message, location: location, help: help, note: note)
|
|
123
|
+
end
|
|
124
|
+
|
|
89
125
|
def raise!(diagnostic)
|
|
90
126
|
klass = diagnostic.code == "FLEXR-E014" ? UnsupportedRegexpError : CompileError
|
|
91
127
|
raise klass.new(diagnostic.message, diagnostic: diagnostic)
|
data/lib/flexr/dsl.rb
CHANGED
|
@@ -20,6 +20,7 @@ module Flexr
|
|
|
20
20
|
@__flexr_compiled = nil
|
|
21
21
|
@__flexr_generated = false
|
|
22
22
|
@__flexr_generated_actions = []
|
|
23
|
+
@__flexr_generated_eof_actions = {}
|
|
23
24
|
@__flexr_specification_frozen = false
|
|
24
25
|
@__flexr_compile_mutex = Monitor.new
|
|
25
26
|
end
|
|
@@ -96,12 +97,7 @@ module Flexr
|
|
|
96
97
|
end
|
|
97
98
|
|
|
98
99
|
def encoding(value)
|
|
99
|
-
encoding =
|
|
100
|
-
unless [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
|
|
101
|
-
diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
|
|
102
|
-
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
103
|
-
end
|
|
104
|
-
|
|
100
|
+
encoding = Configuration.encoding!(value)
|
|
105
101
|
__flexr_mutate! { @__flexr_config.encoding = encoding }
|
|
106
102
|
end
|
|
107
103
|
|
|
@@ -116,9 +112,10 @@ module Flexr
|
|
|
116
112
|
end
|
|
117
113
|
|
|
118
114
|
def compile!
|
|
115
|
+
return @__flexr_compiled if @__flexr_compiled
|
|
116
|
+
|
|
119
117
|
@__flexr_compile_mutex.synchronize do
|
|
120
118
|
# The ivar is part of the generated/runtime class contract.
|
|
121
|
-
# rubocop:disable Naming/MemoizedInstanceVariableName
|
|
122
119
|
@__flexr_compiled ||= begin
|
|
123
120
|
compiled = Automaton::Compiler.new(__flexr_spec).compile
|
|
124
121
|
@__flexr_config.backend = auto_direct?(compiled) ? :direct : :table if @__flexr_config.backend == :auto
|
|
@@ -126,7 +123,6 @@ module Flexr
|
|
|
126
123
|
freeze_compiled!(compiled)
|
|
127
124
|
compiled
|
|
128
125
|
end
|
|
129
|
-
# rubocop:enable Naming/MemoizedInstanceVariableName
|
|
130
126
|
end
|
|
131
127
|
end
|
|
132
128
|
|
|
@@ -151,7 +147,8 @@ module Flexr
|
|
|
151
147
|
)
|
|
152
148
|
end
|
|
153
149
|
|
|
154
|
-
attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions
|
|
150
|
+
attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled, :__flexr_generated_actions,
|
|
151
|
+
:__flexr_generated_eof_actions
|
|
155
152
|
|
|
156
153
|
def __flexr_add_generated_eof(state, action)
|
|
157
154
|
__flexr_mutate! { @__flexr_eof_rules[state.to_sym] = action }
|
|
@@ -174,6 +171,12 @@ module Flexr
|
|
|
174
171
|
@__flexr_generated_actions[index] = action
|
|
175
172
|
end
|
|
176
173
|
|
|
174
|
+
def __flexr_bind_generated_eof(state, action)
|
|
175
|
+
raise FrozenSpecificationError, "generated actions can only be bound on a generated lexer" unless @__flexr_generated
|
|
176
|
+
|
|
177
|
+
@__flexr_generated_eof_actions[state.to_sym] = action
|
|
178
|
+
end
|
|
179
|
+
|
|
177
180
|
def __flexr_generated?
|
|
178
181
|
@__flexr_generated == true
|
|
179
182
|
end
|
data/lib/flexr/errors.rb
CHANGED
|
@@ -2,23 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
class Error < StandardError
|
|
5
|
-
attr_reader :diagnostic
|
|
5
|
+
attr_reader :diagnostic, :filename, :byte_pos
|
|
6
6
|
|
|
7
|
-
def initialize(message = nil, diagnostic: nil)
|
|
7
|
+
def initialize(message = nil, diagnostic: nil, filename: nil, byte_pos: nil)
|
|
8
8
|
@diagnostic = diagnostic
|
|
9
|
+
@filename = filename
|
|
10
|
+
@byte_pos = byte_pos
|
|
9
11
|
super(message || diagnostic&.message)
|
|
10
12
|
end
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
class LexError < Error
|
|
14
|
-
attr_reader :
|
|
16
|
+
attr_reader :line, :text
|
|
15
17
|
|
|
16
18
|
def initialize(message, filename: nil, byte_pos: nil, line: nil, text: nil, diagnostic: nil)
|
|
17
|
-
@filename = filename
|
|
18
|
-
@byte_pos = byte_pos
|
|
19
19
|
@line = line
|
|
20
20
|
@text = text
|
|
21
|
-
super(message, diagnostic: diagnostic)
|
|
21
|
+
super(message, diagnostic: diagnostic, filename: filename, byte_pos: byte_pos)
|
|
22
22
|
end
|
|
23
23
|
end
|
|
24
24
|
|
data/lib/flexr/generated.rb
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Generated
|
|
5
|
-
ARTIFACT_SCHEMA_VERSION =
|
|
6
|
-
RUNTIME_ABI_VERSION =
|
|
5
|
+
ARTIFACT_SCHEMA_VERSION = 2
|
|
6
|
+
RUNTIME_ABI_VERSION = 2
|
|
7
7
|
|
|
8
8
|
module_function
|
|
9
9
|
|
|
@@ -46,23 +46,22 @@ module Flexr
|
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
def install_compiled!(klass, payload)
|
|
49
|
-
|
|
49
|
+
begin
|
|
50
|
+
validate_artifact!(payload[:artifact])
|
|
51
|
+
rescue CompileError => e
|
|
52
|
+
caller = caller_locations.find { |frame| frame.base_label != "install_compiled!" }
|
|
53
|
+
path = caller&.absolute_path || caller&.path
|
|
54
|
+
raise unless path && File.file?(path)
|
|
55
|
+
|
|
56
|
+
e.diagnostic.location ||= [0, 0]
|
|
57
|
+
raise CompileError.new(e.message, diagnostic: e.diagnostic, filename: path, byte_pos: 0)
|
|
58
|
+
end
|
|
50
59
|
install!(klass, payload)
|
|
51
60
|
machines = payload.fetch(:compiled).fetch(:machines).transform_values do |machine|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
accepts: dfa_data.fetch(:accepts),
|
|
57
|
-
ec: dfa_data.fetch(:ec),
|
|
58
|
-
class_count: dfa_data.fetch(:class_count),
|
|
59
|
-
start: dfa_data.fetch(:start),
|
|
60
|
-
rule_ids: dfa_data.fetch(:rule_ids),
|
|
61
|
-
packed: packed,
|
|
62
|
-
direct: dfa_data[:direct],
|
|
63
|
-
state_count: dfa_data.fetch(:state_count)
|
|
64
|
-
)
|
|
65
|
-
Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym)
|
|
61
|
+
dfa = load_dfa(machine.fetch(:dfa))
|
|
62
|
+
plans = Automaton::Accel.whole_token_plans(dfa, klass.__flexr_rules) if
|
|
63
|
+
klass.__flexr_config.backend != :firstmatch && klass.__flexr_config.options[:accel] == :strscan
|
|
64
|
+
Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym, whole_tokens: plans)
|
|
66
65
|
end
|
|
67
66
|
compiled = Automaton::CompiledSpec.new(
|
|
68
67
|
machines: machines,
|
|
@@ -99,6 +98,43 @@ module Flexr
|
|
|
99
98
|
incompatible_artifact!(detail)
|
|
100
99
|
end
|
|
101
100
|
|
|
101
|
+
def shareable!(klass)
|
|
102
|
+
unless klass.__flexr_generated? && klass.__flexr_rules.none? { |rule| rule.action.is_a?(Proc) } &&
|
|
103
|
+
klass.__flexr_generated_actions.empty? && klass.__flexr_spec.eof_rules.empty?
|
|
104
|
+
raise ArgumentError, "Ractor sharing requires generated method actions without captured closures or EOF blocks"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
config = klass.__flexr_config
|
|
108
|
+
if config.backend == :firstmatch && klass.__flexr_rules.any? do |rule|
|
|
109
|
+
rule.patterns.any? do |pattern|
|
|
110
|
+
Regexp::Matcher.reference?(pattern, encoding: config.encoding, unicode: config.options[:unicode] == true)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
raise ArgumentError, "Ractor sharing does not support firstmatch reference patterns; use table, direct, or auto"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
original = klass.__flexr_compiled
|
|
117
|
+
rules = original.rules.map do |rule|
|
|
118
|
+
IR::Rule.new(**rule.to_h, trailing_dfa: rule.trailing_dfa && shareable_dfa(rule.trailing_dfa))
|
|
119
|
+
end
|
|
120
|
+
machines = original.machines.transform_values do |machine|
|
|
121
|
+
Automaton::Machine.new(dfa: shareable_dfa(machine.dfa), state_name: machine.state_name,
|
|
122
|
+
whole_tokens: machine.whole_tokens)
|
|
123
|
+
end
|
|
124
|
+
compiled = Automaton::CompiledSpec.new(**original.to_h, machines: machines, rules: rules)
|
|
125
|
+
Ractor.make_shareable([compiled, klass.__flexr_config, klass.__flexr_states, klass.__flexr_spec.eof_rules,
|
|
126
|
+
klass.__flexr_generated_actions, klass.__flexr_generated_eof_actions])
|
|
127
|
+
klass.instance_variable_set(:@__flexr_rules, rules)
|
|
128
|
+
klass.__flexr_set_compiled!(compiled)
|
|
129
|
+
klass
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def shareable_dfa(dfa)
|
|
133
|
+
Automaton::DFA.new(transitions: dfa.transitions.map(&:dup), accepts: dfa.accepts, ec: dfa.ec,
|
|
134
|
+
class_count: dfa.class_count, start: dfa.start, rule_ids: dfa.rule_ids,
|
|
135
|
+
direct: dfa.direct)
|
|
136
|
+
end
|
|
137
|
+
|
|
102
138
|
def incompatible_artifact!(detail)
|
|
103
139
|
diagnostic = Diagnostics.error(
|
|
104
140
|
"FLEXR-E020", "generated artifact is incompatible with this runtime",
|
|
@@ -119,6 +155,14 @@ module Flexr
|
|
|
119
155
|
}
|
|
120
156
|
end
|
|
121
157
|
|
|
158
|
+
def load_dfa(data)
|
|
159
|
+
Automaton::DFA.new(
|
|
160
|
+
transitions: data[:transitions], accepts: data.fetch(:accepts), ec: data.fetch(:ec),
|
|
161
|
+
class_count: data.fetch(:class_count), start: data.fetch(:start), rule_ids: data.fetch(:rule_ids),
|
|
162
|
+
packed: decode_packed(data[:packed]), direct: data[:direct], state_count: data.fetch(:state_count)
|
|
163
|
+
)
|
|
164
|
+
end
|
|
165
|
+
|
|
122
166
|
def decode_array(encoded, nil_value: nil)
|
|
123
167
|
values = encoded.unpack1("m0").unpack("l<*")
|
|
124
168
|
return values unless nil_value
|
|
@@ -138,6 +182,7 @@ module Flexr
|
|
|
138
182
|
states: Array(definition.fetch(:states)).map(&:to_sym),
|
|
139
183
|
bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
|
|
140
184
|
location: definition[:span] || definition[:location],
|
|
185
|
+
trailing_dfa: definition[:trailing_dfa] && Generated.load_dfa(definition[:trailing_dfa]),
|
|
141
186
|
pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
|
|
142
187
|
next unless condition
|
|
143
188
|
|
|
@@ -146,6 +191,9 @@ module Flexr
|
|
|
146
191
|
end
|
|
147
192
|
)
|
|
148
193
|
end
|
|
194
|
+
rescue CompileError => e
|
|
195
|
+
e.diagnostic.location ||= definition[:span] || definition[:location] if e.diagnostic
|
|
196
|
+
raise
|
|
149
197
|
end
|
|
150
198
|
end
|
|
151
199
|
end
|
data/lib/flexr/generator.rb
CHANGED
|
@@ -15,15 +15,15 @@ module Flexr
|
|
|
15
15
|
|
|
16
16
|
RUNTIME_SOURCES = %w[
|
|
17
17
|
version.rb errors.rb diagnostics.rb action_resolver.rb configuration.rb ir.rb
|
|
18
|
-
unicode/version.rb automaton/types.rb automaton/dfa.rb automaton/
|
|
19
|
-
runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
|
|
18
|
+
unicode/version.rb automaton/types.rb automaton/dfa.rb automaton/accel.rb
|
|
19
|
+
runtime/record.rb runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
|
|
20
20
|
runtime/interpreter.rb runtime/core.rb dsl.rb lexer.rb generated.rb
|
|
21
21
|
].freeze
|
|
22
22
|
|
|
23
23
|
MATCHER_RUNTIME_SOURCES = %w[
|
|
24
24
|
regexp/ast.rb regexp/parser.rb regexp/unsupported.rb
|
|
25
25
|
unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb
|
|
26
|
-
unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb
|
|
26
|
+
unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb regexp/matcher.rb
|
|
27
27
|
].freeze
|
|
28
28
|
|
|
29
29
|
def initialize(path, output: nil, eval_mode: false, options: {})
|
|
@@ -62,6 +62,11 @@ module Flexr
|
|
|
62
62
|
install = "#{install}#{Codegen::Table.new(compiled).source(indent: indent)}"
|
|
63
63
|
install = "#{install}#{generated_action_source(parsed, indent)}"
|
|
64
64
|
edits = parsed.dsl_edits.dup
|
|
65
|
+
parsed.rules.each do |rule|
|
|
66
|
+
next unless rule.bind_action && !rule.reads_locals && inline_simple_action(rule.action_source)
|
|
67
|
+
|
|
68
|
+
edits.map! { |edit| edit.first(2) == rule.span ? [*rule.span, ""] : edit }
|
|
69
|
+
end
|
|
65
70
|
edits.concat(parsed.flexr_require_spans.map { |span| [*span, ""] }) if standalone?(parsed)
|
|
66
71
|
result = Source::Passthrough.rewrite(
|
|
67
72
|
parsed.source, edits, insertion: parsed.first_dsl_offset, payload: install
|
|
@@ -79,7 +84,7 @@ module Flexr
|
|
|
79
84
|
"# eval: #{@eval_mode}",
|
|
80
85
|
"# standalone: #{standalone?(parsed)}"
|
|
81
86
|
].join("\n")
|
|
82
|
-
prefix = standalone?(parsed) ? "
|
|
87
|
+
prefix = standalone?(parsed) ? "raise LoadError, \"flexr requires Ruby 3.4 or newer\" if RUBY_VERSION < \"3.4\"\nrequire \"json\"\nrequire \"monitor\"\n#{embedded_runtime(parsed)}\n" : ""
|
|
83
88
|
Source::Passthrough.insert_after_preamble(result, "#{header}\n#{prefix}")
|
|
84
89
|
end
|
|
85
90
|
|
|
@@ -89,7 +94,8 @@ module Flexr
|
|
|
89
94
|
[condition.rule_index, condition.pattern_index, condition.bol_only, condition.end_anchor]
|
|
90
95
|
end
|
|
91
96
|
"{ index: #{rule.index}, patterns: #{ruby_literal(rule.patterns)}, pattern_conditions: #{ruby_literal(conditions)}, trailing: #{ruby_literal(rule.trailing)}, " \
|
|
92
|
-
"
|
|
97
|
+
"#{trailing_dfa_expression(rule, parsed)}" \
|
|
98
|
+
"action: #{generated_rule_action(rule)}, states: #{rule.states.inspect}, " \
|
|
93
99
|
"bol_only: #{ruby_literal(rule.bol_only)}, end_anchor: #{ruby_literal(rule.end_anchor)} }"
|
|
94
100
|
end
|
|
95
101
|
"{ artifact: #{ruby_literal(artifact_metadata)}, rules: [#{definitions.join(', ')}], " \
|
|
@@ -111,18 +117,21 @@ module Flexr
|
|
|
111
117
|
|
|
112
118
|
def compile_parsed(parsed)
|
|
113
119
|
states = parsed.states.each_with_index.to_h do |(name, value), index|
|
|
114
|
-
[name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index)]
|
|
120
|
+
[name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index, location: value[:location])]
|
|
115
121
|
end
|
|
116
122
|
rules = parsed.rules.map do |rule|
|
|
117
123
|
IR::Rule.new(index: rule.index, patterns: rule.patterns, trailing: normalize_trailing(rule.trailing),
|
|
118
124
|
action: rule.action, states: rule.states, bol_only: rule.bol_only,
|
|
119
125
|
end_anchor: rule.end_anchor, location: rule.span)
|
|
126
|
+
rescue CompileError => e
|
|
127
|
+
e.diagnostic.location ||= rule.span if e.diagnostic
|
|
128
|
+
raise
|
|
120
129
|
end
|
|
121
130
|
spec = IR::Spec.new(
|
|
122
131
|
class_name: parsed.class_name, superclass: "Flexr::Lexer",
|
|
123
132
|
backend: @options.fetch(:backend, parsed.config[:backend]),
|
|
124
133
|
token_kind: @options.fetch(:token_kind, parsed.config[:token_kind]),
|
|
125
|
-
encoding: parsed.config[:encoding], options: effective_options(parsed),
|
|
134
|
+
encoding: parsed.config[:encoding], options: effective_options(parsed).merge(warn_level: @options[:warn_level]),
|
|
126
135
|
declared_tokens: parsed.config[:declared_tokens], states: states, rules: rules,
|
|
127
136
|
eof_rules: parsed.config[:eof_rules] || {}, verbatim: parsed.source
|
|
128
137
|
)
|
|
@@ -160,28 +169,8 @@ module Flexr
|
|
|
160
169
|
|
|
161
170
|
def compiled_expression(compiled, compression: :none, backend: :table)
|
|
162
171
|
machines = compiled.machines.map do |name, machine|
|
|
163
|
-
|
|
164
|
-
data =
|
|
165
|
-
accepts: dfa.accepts,
|
|
166
|
-
ec: dfa.ec,
|
|
167
|
-
class_count: dfa.class_count,
|
|
168
|
-
state_count: dfa.states,
|
|
169
|
-
start: dfa.start,
|
|
170
|
-
rule_ids: dfa.rule_ids
|
|
171
|
-
}
|
|
172
|
-
pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
|
|
173
|
-
if backend == :direct
|
|
174
|
-
data[:direct] = Codegen::Direct.new(compiled).generate(state: name)
|
|
175
|
-
elsif pack_tables
|
|
176
|
-
packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
|
|
177
|
-
data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
|
|
178
|
-
Codegen::TablePacker.encode(packed)
|
|
179
|
-
else
|
|
180
|
-
packed
|
|
181
|
-
end
|
|
182
|
-
else
|
|
183
|
-
data[:transitions] = dfa.transitions
|
|
184
|
-
end
|
|
172
|
+
direct = Codegen::Direct.new(compiled).generate(state: name) if backend == :direct
|
|
173
|
+
data = dfa_data(machine.dfa, compression: compression, direct: direct)
|
|
185
174
|
"#{ruby_literal(name)} => { state_name: #{ruby_literal(machine.state_name)}, dfa: #{ruby_literal(data)} }"
|
|
186
175
|
end
|
|
187
176
|
# W016 is derived from wall-clock compilation time and must not make
|
|
@@ -191,6 +180,38 @@ module Flexr
|
|
|
191
180
|
"stats: #{ruby_literal(compiled.stats)}, diagnostics: #{ruby_literal(diagnostics)} }"
|
|
192
181
|
end
|
|
193
182
|
|
|
183
|
+
def trailing_dfa_expression(rule, parsed)
|
|
184
|
+
return "" unless rule.trailing
|
|
185
|
+
|
|
186
|
+
dfa = Flexr.compile_pattern(normalize_trailing(rule.trailing), encoding: parsed.config[:encoding],
|
|
187
|
+
options: { unicode: effective_options(parsed)[:unicode] == true })
|
|
188
|
+
"trailing_dfa: #{ruby_literal(dfa_data(dfa, compression: table_compression(parsed)))}, "
|
|
189
|
+
rescue CompileError => e
|
|
190
|
+
e.diagnostic.location ||= rule.span if e.diagnostic
|
|
191
|
+
raise
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def dfa_data(dfa, compression:, direct: nil)
|
|
195
|
+
data = {
|
|
196
|
+
accepts: dfa.accepts, ec: dfa.ec, class_count: dfa.class_count,
|
|
197
|
+
state_count: dfa.states, start: dfa.start, rule_ids: dfa.rule_ids
|
|
198
|
+
}
|
|
199
|
+
pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
|
|
200
|
+
if direct
|
|
201
|
+
data[:direct] = direct
|
|
202
|
+
elsif pack_tables
|
|
203
|
+
packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
|
|
204
|
+
data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
|
|
205
|
+
Codegen::TablePacker.encode(packed)
|
|
206
|
+
else
|
|
207
|
+
packed
|
|
208
|
+
end
|
|
209
|
+
else
|
|
210
|
+
data[:transitions] = dfa.transitions
|
|
211
|
+
end
|
|
212
|
+
data
|
|
213
|
+
end
|
|
214
|
+
|
|
194
215
|
def standalone?(parsed)
|
|
195
216
|
@options.fetch(:standalone, parsed.config[:options]&.key?(:standalone) || false)
|
|
196
217
|
end
|
|
@@ -236,15 +257,23 @@ module Flexr
|
|
|
236
257
|
"proc { emit(nil, text) }"
|
|
237
258
|
end
|
|
238
259
|
|
|
260
|
+
def generated_rule_action(rule)
|
|
261
|
+
return ruby_literal(:__flexr_generated_method) if rule.action.is_a?(String) &&
|
|
262
|
+
((!rule.bind_action && !inline_action_body(rule).nil?) || (!rule.reads_locals && inline_simple_action(rule.action_source)))
|
|
263
|
+
|
|
264
|
+
action_expression(rule.action)
|
|
265
|
+
end
|
|
266
|
+
|
|
239
267
|
def generated_action_source(parsed, indent)
|
|
240
268
|
lines = ["#{indent}def __flexr_generated_execute(rule)", "#{indent} case rule.index"]
|
|
241
269
|
parsed.rules.each do |rule|
|
|
242
270
|
lines << "#{indent} when #{rule.index}"
|
|
243
|
-
if rule.bind_action
|
|
271
|
+
body = inline_simple_action(rule.action_source) if rule.bind_action && !rule.reads_locals
|
|
272
|
+
if rule.bind_action && body.nil?
|
|
244
273
|
lines << "#{indent} instance_exec(&self.class.__flexr_generated_actions.fetch(rule.index))"
|
|
245
274
|
next
|
|
246
275
|
end
|
|
247
|
-
body
|
|
276
|
+
body ||= inline_action_body(rule)
|
|
248
277
|
if body.nil?
|
|
249
278
|
lines << "#{indent} instance_exec(&rule.action)"
|
|
250
279
|
else
|
|
@@ -309,6 +338,7 @@ module Flexr
|
|
|
309
338
|
|
|
310
339
|
def inline_simple_action(source)
|
|
311
340
|
body = source.to_s.strip
|
|
341
|
+
return if body.include?('#{') || body.match?(/text\.byteslice\([^)]*[A-Za-z_]/)
|
|
312
342
|
body = body[1...-1].strip if body.start_with?("{") && body.end_with?("}")
|
|
313
343
|
match = INLINE_EMIT.match(body)
|
|
314
344
|
return unless match
|
|
@@ -334,9 +364,7 @@ module Flexr
|
|
|
334
364
|
loader = Object.new
|
|
335
365
|
loader.define_singleton_method(:load_spec) do
|
|
336
366
|
# --eval is opt-in and intentionally executes trusted specification code.
|
|
337
|
-
# rubocop:disable Security/Eval
|
|
338
|
-
eval(source, TOPLEVEL_BINDING, @path, 1)
|
|
339
|
-
# rubocop:enable Security/Eval
|
|
367
|
+
eval(source, TOPLEVEL_BINDING, @path, 1) # rubocop:disable Security/Eval
|
|
340
368
|
end
|
|
341
369
|
loader.instance_variable_set(:@path, @path)
|
|
342
370
|
loader.load_spec
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Grammar
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
def tokens(source)
|
|
8
|
+
names = []
|
|
9
|
+
declaration = false
|
|
10
|
+
source.scan(%r{%\{.*?%\}|/\*.*?\*/|//[^\n]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|<[^>]*>|%%|%[a-zA-Z][\w-]*|[a-zA-Z_]\w*|.}m).each do |token|
|
|
11
|
+
break if token == "%%"
|
|
12
|
+
if token.start_with?("%") && !token.start_with?("%{")
|
|
13
|
+
declaration = token == "%token"
|
|
14
|
+
elsif declaration && token.match?(/\A[A-Za-z_]\w*\z/)
|
|
15
|
+
names << token
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
names.uniq
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def diagnostics(parsed, path)
|
|
22
|
+
expected = tokens(File.read(path))
|
|
23
|
+
emitted = Array(parsed.config[:declared_tokens]).map(&:to_s)
|
|
24
|
+
location = [parsed.first_dsl_offset, parsed.first_dsl_offset]
|
|
25
|
+
(expected - emitted).map do |name|
|
|
26
|
+
Diagnostics.warning("FLEXR-W017", "grammar token #{name} is not declared by emits",
|
|
27
|
+
location: location, note: "grammar: #{path}", help: "declare and emit #{name}, or remove it from the grammar")
|
|
28
|
+
end + (emitted - expected).map do |name|
|
|
29
|
+
Diagnostics.warning("FLEXR-W018", "emitted token #{name} is not declared by the grammar",
|
|
30
|
+
location: location, note: "grammar: #{path}", help: "add a %token declaration or remove #{name} from emits")
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def declarations(tokens)
|
|
35
|
+
names = tokens.map(&:to_s).uniq
|
|
36
|
+
raise ArgumentError, "Lrama token names must be identifiers" unless names.all? { |name| name.match?(/\A[A-Za-z_]\w*\z/) }
|
|
37
|
+
|
|
38
|
+
names.empty? ? "" : "%token #{names.join(' ')}"
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
data/lib/flexr/ir.rb
CHANGED
|
@@ -4,7 +4,7 @@ module Flexr
|
|
|
4
4
|
module IR
|
|
5
5
|
Rule = Struct.new(
|
|
6
6
|
:index, :patterns, :trailing, :action, :states, :bol_only, :end_anchor, :location,
|
|
7
|
-
:pattern_conditions,
|
|
7
|
+
:pattern_conditions, :trailing_dfa,
|
|
8
8
|
keyword_init: true
|
|
9
9
|
) do
|
|
10
10
|
def skip?
|
|
@@ -16,7 +16,7 @@ module Flexr
|
|
|
16
16
|
end
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
State = Struct.new(:name, :inclusive, :id, keyword_init: true)
|
|
19
|
+
State = Struct.new(:name, :inclusive, :id, :location, keyword_init: true)
|
|
20
20
|
|
|
21
21
|
Spec = Struct.new(
|
|
22
22
|
:class_name, :superclass, :backend, :token_kind, :encoding, :options,
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Migration
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
def diagnostics(parsed)
|
|
8
|
+
require "prism"
|
|
9
|
+
source = parsed.source
|
|
10
|
+
result = [Diagnostics.info("FLEXR-I002", "v2.0 requires Ruby 3.4+; regenerate all generated lexers after upgrading",
|
|
11
|
+
help: "review docs/migration-v2.md and consumer code; static checks cannot find all record mutations or initialization calls")]
|
|
12
|
+
visit = lambda do |node|
|
|
13
|
+
return unless node
|
|
14
|
+
|
|
15
|
+
if node.is_a?(Prism::CallNode) && parsed.dsl_spans.include?([node.location.start_offset, node.location.end_offset]) &&
|
|
16
|
+
node.receiver.nil? && node.name == :token_kind &&
|
|
17
|
+
node.arguments&.arguments&.first.is_a?(Prism::SymbolNode) &&
|
|
18
|
+
node.arguments.arguments.first.unescaped == "struct"
|
|
19
|
+
result << Diagnostics.warning(
|
|
20
|
+
"FLEXR-W020", "structured tokens and locations become immutable Data records in v2.0",
|
|
21
|
+
location: [node.location.start_offset, node.location.end_offset],
|
|
22
|
+
help: "replace field assignments with record.with(...); review consumer code and Struct-specific APIs"
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
node.compact_child_nodes.each { |child| visit.call(child) }
|
|
26
|
+
end
|
|
27
|
+
visit.call(Prism.parse(source).value)
|
|
28
|
+
result
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "minitest/assertions"
|
|
4
|
+
require_relative "testing"
|
|
5
|
+
|
|
6
|
+
module Flexr
|
|
7
|
+
module Minitest
|
|
8
|
+
module Assertions
|
|
9
|
+
def assert_tokenize(lexer, input, expected, **)
|
|
10
|
+
assert_equal expected, lexer.new(input, **).tokens
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def assert_token_snapshot(lexer, input, path, update: false, **)
|
|
14
|
+
actual = Testing::Snapshot.render(lexer.new(input, **).tokens)
|
|
15
|
+
begin
|
|
16
|
+
expected = Testing::Snapshot.expected(path, actual, update: update)
|
|
17
|
+
rescue Errno::ENOENT
|
|
18
|
+
flunk "missing token snapshot #{path}; create it with update: true"
|
|
19
|
+
end
|
|
20
|
+
assert_equal expected, actual, "token snapshot differs: #{path}"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
data/lib/flexr/options.rb
CHANGED
|
@@ -5,7 +5,7 @@ require_relative "runtime" unless defined?(Flexr::Lexer)
|
|
|
5
5
|
module Flexr
|
|
6
6
|
Options = Struct.new(
|
|
7
7
|
:backend, :token_kind, :accel, :standalone, :eval_mode, :table_compression,
|
|
8
|
-
:table_format, :max_dfa_states, :warn_level, :warn_as_error, :color, :format, :overrides,
|
|
8
|
+
:table_format, :max_dfa_states, :warn_level, :warn_as_error, :color, :format, :overrides, :grammar, :target,
|
|
9
9
|
keyword_init: true
|
|
10
10
|
) do
|
|
11
11
|
def self.default
|
|
@@ -29,7 +29,9 @@ module Flexr
|
|
|
29
29
|
validate_value!(table_format, %i[literal packed], :table_format)
|
|
30
30
|
validate_value!(warn_level, %i[all default none], :warn_level)
|
|
31
31
|
validate_value!(color, %i[auto always never], :color)
|
|
32
|
-
validate_value!(format, %i[human json], :format)
|
|
32
|
+
validate_value!(format, %i[human json lrama], :format)
|
|
33
|
+
raise ArgumentError, "unsupported target: #{target.inspect}" unless target.nil? || target == "2.0"
|
|
34
|
+
raise ArgumentError, "--target requires static check; remove --eval" if target && eval_mode
|
|
33
35
|
raise ArgumentError, "unsupported max_dfa_states: #{max_dfa_states.inspect}" unless max_dfa_states.is_a?(Integer) && max_dfa_states.positive?
|
|
34
36
|
self
|
|
35
37
|
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Regexp
|
|
5
|
+
module Matcher
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def reference?(pattern, encoding:, unicode: false)
|
|
9
|
+
return false unless pattern.is_a?(::Regexp)
|
|
10
|
+
return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
|
|
11
|
+
|
|
12
|
+
unicode && encoding == Encoding::UTF_8 && pattern.source.match?(/\\[dDwWsS]/)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def match(pattern, subject, encoding:, unicode:, reference:)
|
|
16
|
+
if reference
|
|
17
|
+
Unicode::ReferenceRegexp.match(pattern, subject, encoding: encoding, options: pattern.options, unicode: unicode)
|
|
18
|
+
else
|
|
19
|
+
pattern.match(subject, 0)
|
|
20
|
+
end
|
|
21
|
+
rescue ArgumentError, RegexpError
|
|
22
|
+
nil
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|