flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../flexr"
|
|
4
|
+
|
|
5
|
+
module Flexr
|
|
6
|
+
module Testing
|
|
7
|
+
module Snapshot
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def render(tokens)
|
|
11
|
+
"#{JSON.pretty_generate(encode(tokens))}\n"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def expected(path, actual, update:)
|
|
15
|
+
File.binwrite(path, actual) if update
|
|
16
|
+
File.binread(path).force_encoding(Encoding::UTF_8)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def encode(value)
|
|
20
|
+
case value
|
|
21
|
+
when NilClass, TrueClass, FalseClass, Integer
|
|
22
|
+
value
|
|
23
|
+
when Float
|
|
24
|
+
raise ArgumentError, "unsupported snapshot value: non-finite Float" unless value.finite?
|
|
25
|
+
|
|
26
|
+
value
|
|
27
|
+
when String
|
|
28
|
+
utf8 = value.dup.force_encoding(Encoding::UTF_8)
|
|
29
|
+
utf8.valid_encoding? ? utf8 : { "bytes" => value.bytes }
|
|
30
|
+
when Symbol
|
|
31
|
+
{ "symbol" => value.to_s }
|
|
32
|
+
when Array
|
|
33
|
+
value.map { |item| encode(item) }
|
|
34
|
+
when Hash
|
|
35
|
+
pairs = value.map { |key, item| [encode(key), encode(item)] }
|
|
36
|
+
{ "hash" => pairs.sort_by { |key, _item| JSON.generate(key) } }
|
|
37
|
+
when Runtime::Token
|
|
38
|
+
{ "token" => encode(value.to_h) }
|
|
39
|
+
when Runtime::Location
|
|
40
|
+
{ "location" => encode(value.members.to_h { |name| [name, value.public_send(name)] }) }
|
|
41
|
+
else
|
|
42
|
+
raise ArgumentError, "unsupported snapshot value: #{value.class}; convert it to built-in values first"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
data/lib/flexr/trace.rb
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "runtime" unless defined?(Flexr::Lexer)
|
|
4
|
+
|
|
5
|
+
module Flexr
|
|
6
|
+
class Trace
|
|
7
|
+
module Actions
|
|
8
|
+
private
|
|
9
|
+
|
|
10
|
+
def execute(rule)
|
|
11
|
+
@flexr_trace.record(rule) { super }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
public
|
|
15
|
+
|
|
16
|
+
def instance_exec(*args, &block)
|
|
17
|
+
if block.equal?(self.class.__flexr_spec.eof_rules[state])
|
|
18
|
+
@flexr_trace.record_eof { super }
|
|
19
|
+
else
|
|
20
|
+
super
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def handle_unmatched_byte
|
|
27
|
+
@flexr_trace.record_unmatched { super }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
attr_reader :steps
|
|
32
|
+
|
|
33
|
+
def initialize(lexer)
|
|
34
|
+
@lexer = lexer
|
|
35
|
+
@steps = []
|
|
36
|
+
@config = lexer.class.__flexr_config
|
|
37
|
+
@compiled = lexer.class.__flexr_compiled
|
|
38
|
+
@rules = lexer.class.__flexr_rules
|
|
39
|
+
@lexer.singleton_class.prepend(Actions)
|
|
40
|
+
@lexer.instance_variable_set(:@flexr_trace, self)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def run
|
|
44
|
+
@lexer.tokens
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def record(rule)
|
|
48
|
+
start = @lexer.instance_variable_get(:@match_start)
|
|
49
|
+
ending = @lexer.instance_variable_get(:@match_end)
|
|
50
|
+
state = @lexer.state
|
|
51
|
+
candidates = candidates_at(start)
|
|
52
|
+
selected = candidates.find { |candidate| candidate[:rule] == rule.index }
|
|
53
|
+
reason = if @config.backend == :firstmatch
|
|
54
|
+
"first_match"
|
|
55
|
+
elsif candidates.length == 1
|
|
56
|
+
"only_match"
|
|
57
|
+
elsif selected && candidates.count { |candidate| candidate[:total_length] == selected[:total_length] } > 1
|
|
58
|
+
"rule_order"
|
|
59
|
+
else
|
|
60
|
+
"longest"
|
|
61
|
+
end
|
|
62
|
+
yield
|
|
63
|
+
pending = @lexer.instance_variable_get(:@pending)
|
|
64
|
+
bytes = @lexer.buffer.byteslice(start...ending)
|
|
65
|
+
@steps << { byte_begin: start, byte_end: ending, next_byte: @lexer.byte_pos, state: state, next_state: @lexer.state,
|
|
66
|
+
text: display_text(bytes), bytes: bytes.bytes, candidates: candidates,
|
|
67
|
+
rule: rule.index, patterns: rule.patterns.map(&:inspect), reason: reason,
|
|
68
|
+
action: pending ? "emit" : "skip", token: pending&.inspect }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def record_eof
|
|
72
|
+
position = @lexer.byte_pos
|
|
73
|
+
state = @lexer.state
|
|
74
|
+
yield
|
|
75
|
+
pending = @lexer.instance_variable_get(:@pending)
|
|
76
|
+
@steps << { byte_begin: position, byte_end: position, next_byte: @lexer.byte_pos,
|
|
77
|
+
state: state, next_state: @lexer.state, text: "", bytes: [], candidates: [],
|
|
78
|
+
rule: nil, patterns: [], reason: "eof", action: pending ? "emit" : "skip", token: pending&.inspect }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def record_unmatched
|
|
82
|
+
position = @lexer.byte_pos
|
|
83
|
+
state = @lexer.state
|
|
84
|
+
bytes = @lexer.buffer.byteslice(position, 1)
|
|
85
|
+
token = yield
|
|
86
|
+
@steps << { byte_begin: position, byte_end: position + 1, next_byte: @lexer.byte_pos,
|
|
87
|
+
state: state, next_state: @lexer.state, text: display_text(bytes), bytes: bytes.bytes,
|
|
88
|
+
candidates: [], rule: nil, patterns: [], reason: "unmatched",
|
|
89
|
+
action: token ? "emit" : "skip", token: token&.inspect }
|
|
90
|
+
token
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def render
|
|
94
|
+
steps.map do |step|
|
|
95
|
+
candidates = step[:candidates].map do |candidate|
|
|
96
|
+
"rule #{candidate[:rule]}(#{candidate[:length]}/#{candidate[:total_length]})"
|
|
97
|
+
end.join(" ")
|
|
98
|
+
target = step[:rule] ? "rule #{step[:rule]}" : step[:reason]
|
|
99
|
+
"@#{step[:byte_begin]} #{step[:text].inspect} state=#{step[:state]} " \
|
|
100
|
+
"candidates: #{candidates} -> #{target} #{step[:action]} (#{step[:reason]}) " \
|
|
101
|
+
"state=#{step[:next_state]}#{step[:token] && " #{step[:token]}"}"
|
|
102
|
+
end.join("\n")
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
|
|
107
|
+
def candidates_at(start)
|
|
108
|
+
@config.backend == :firstmatch ? firstmatch_candidates(start) : dfa_candidates(start)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def dfa_candidates(start)
|
|
112
|
+
dfa = @compiled.machines.fetch(@lexer.state).dfa
|
|
113
|
+
best = {}
|
|
114
|
+
walk(dfa, start) do |acceptance, ending|
|
|
115
|
+
next if acceptance.bol_only && !@lexer.beginning_of_line?
|
|
116
|
+
|
|
117
|
+
rule = @rules.fetch(acceptance.rule_index)
|
|
118
|
+
candidate = candidate_for(rule, start, ending)
|
|
119
|
+
next unless candidate
|
|
120
|
+
previous = best[rule.index]
|
|
121
|
+
best[rule.index] = candidate if !previous || candidate[:total_length] >= previous[:total_length]
|
|
122
|
+
end
|
|
123
|
+
best.values.sort_by { |candidate| candidate[:rule] }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def firstmatch_candidates(start)
|
|
127
|
+
state = @config.states.fetch(@lexer.state)
|
|
128
|
+
@lexer.buffer.read_to_end
|
|
129
|
+
ending = @lexer.buffer.bytesize
|
|
130
|
+
if @config.encoding == Encoding::UTF_8
|
|
131
|
+
ending = start
|
|
132
|
+
ending += @lexer.buffer.utf8_character_length(ending) while
|
|
133
|
+
ending < @lexer.buffer.bytesize && @lexer.buffer.valid_utf8_at?(ending)
|
|
134
|
+
end
|
|
135
|
+
subject = @lexer.buffer.byteslice(start...ending).to_s
|
|
136
|
+
subject.force_encoding(@config.encoding)
|
|
137
|
+
@rules.filter_map do |rule|
|
|
138
|
+
next unless rule.states.include?(@lexer.state) || (state.inclusive && rule.states.include?(:initial))
|
|
139
|
+
matches = rule.patterns.each_with_index.filter_map do |pattern, index|
|
|
140
|
+
condition = rule.pattern_conditions.fetch(index)
|
|
141
|
+
next if condition.bol_only && !@lexer.beginning_of_line?
|
|
142
|
+
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern))
|
|
143
|
+
reference = Regexp::Matcher.reference?(regexp, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
|
|
144
|
+
match = Regexp::Matcher.match(regexp, subject, encoding: @config.encoding,
|
|
145
|
+
unicode: @config.options[:unicode] == true, reference: reference)
|
|
146
|
+
[match[0].bytesize, condition] if match&.begin(0)&.zero?
|
|
147
|
+
end
|
|
148
|
+
length, condition = matches.max_by(&:first)
|
|
149
|
+
next unless length
|
|
150
|
+
next if condition.end_anchor && !end_anchor?(start + length)
|
|
151
|
+
|
|
152
|
+
candidate_for(rule, start, start + length)
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def candidate_for(rule, start, ending)
|
|
157
|
+
trailing = if rule.trailing
|
|
158
|
+
dfa = rule.trailing_dfa || (@trailing_dfas ||= {})[rule.index] ||= Flexr.compile_pattern(
|
|
159
|
+
rule.trailing, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
|
|
160
|
+
)
|
|
161
|
+
longest = nil
|
|
162
|
+
walk(dfa, ending) { |_acceptance, cursor| longest = cursor - ending }
|
|
163
|
+
longest
|
|
164
|
+
else
|
|
165
|
+
0
|
|
166
|
+
end
|
|
167
|
+
return if trailing.nil?
|
|
168
|
+
|
|
169
|
+
{ rule: rule.index, patterns: rule.patterns.map(&:inspect), length: ending - start,
|
|
170
|
+
trailing_length: trailing, total_length: ending - start + trailing }
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def walk(dfa, start)
|
|
174
|
+
state = dfa.start
|
|
175
|
+
cursor = start
|
|
176
|
+
loop do
|
|
177
|
+
if @lexer.utf8_boundary?(cursor)
|
|
178
|
+
dfa.accepts[state].each do |acceptance|
|
|
179
|
+
yield acceptance, cursor unless acceptance.end_anchor && !end_anchor?(cursor)
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
break unless @lexer.buffer.ensure_available?(cursor + 1)
|
|
183
|
+
break unless (state = dfa.transition(state, @lexer.buffer.getbyte(cursor)))
|
|
184
|
+
|
|
185
|
+
cursor += 1
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def end_anchor?(position)
|
|
190
|
+
@lexer.buffer.eof?(position) || @lexer.buffer.getbyte(position) == 0x0a
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def display_text(bytes)
|
|
194
|
+
bytes.dup.force_encoding(Encoding::UTF_8).scrub
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|