flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
data/lib/flexr/trace.rb
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "runtime" unless defined?(Flexr::Lexer)
|
|
4
|
+
|
|
5
|
+
module Flexr
|
|
6
|
+
class Trace
|
|
7
|
+
module Actions
|
|
8
|
+
private
|
|
9
|
+
|
|
10
|
+
def execute(rule)
|
|
11
|
+
@flexr_trace.record(rule) { super }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
public
|
|
15
|
+
|
|
16
|
+
def instance_exec(*args, &block)
|
|
17
|
+
if block.equal?(self.class.__flexr_spec.eof_rules[state])
|
|
18
|
+
@flexr_trace.record_eof { super }
|
|
19
|
+
else
|
|
20
|
+
super
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def handle_unmatched_byte
|
|
27
|
+
@flexr_trace.record_unmatched { super }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
attr_reader :steps
|
|
32
|
+
|
|
33
|
+
def initialize(lexer)
|
|
34
|
+
@lexer = lexer
|
|
35
|
+
@steps = []
|
|
36
|
+
@config = lexer.class.__flexr_config
|
|
37
|
+
@compiled = lexer.class.__flexr_compiled
|
|
38
|
+
@rules = lexer.class.__flexr_rules
|
|
39
|
+
@lexer.singleton_class.prepend(Actions)
|
|
40
|
+
@lexer.instance_variable_set(:@flexr_trace, self)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def run
|
|
44
|
+
@lexer.tokens
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def record(rule)
|
|
48
|
+
start = @lexer.instance_variable_get(:@match_start)
|
|
49
|
+
ending = @lexer.instance_variable_get(:@match_end)
|
|
50
|
+
state = @lexer.state
|
|
51
|
+
candidates = candidates_at(start)
|
|
52
|
+
selected = candidates.find { |candidate| candidate[:rule] == rule.index }
|
|
53
|
+
reason = if @config.backend == :firstmatch
|
|
54
|
+
"first_match"
|
|
55
|
+
elsif candidates.length == 1
|
|
56
|
+
"only_match"
|
|
57
|
+
elsif selected && candidates.count { |candidate| candidate[:total_length] == selected[:total_length] } > 1
|
|
58
|
+
"rule_order"
|
|
59
|
+
else
|
|
60
|
+
"longest"
|
|
61
|
+
end
|
|
62
|
+
yield
|
|
63
|
+
pending = @lexer.instance_variable_get(:@pending)
|
|
64
|
+
bytes = @lexer.buffer.byteslice(start...ending)
|
|
65
|
+
@steps << { byte_begin: start, byte_end: ending, next_byte: @lexer.byte_pos, state: state, next_state: @lexer.state,
|
|
66
|
+
text: display_text(bytes), bytes: bytes.bytes, candidates: candidates,
|
|
67
|
+
rule: rule.index, patterns: rule.patterns.map(&:inspect), reason: reason,
|
|
68
|
+
action: pending ? "emit" : "skip", token: pending&.inspect }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def record_eof
|
|
72
|
+
position = @lexer.byte_pos
|
|
73
|
+
state = @lexer.state
|
|
74
|
+
yield
|
|
75
|
+
pending = @lexer.instance_variable_get(:@pending)
|
|
76
|
+
@steps << { byte_begin: position, byte_end: position, next_byte: @lexer.byte_pos,
|
|
77
|
+
state: state, next_state: @lexer.state, text: "", bytes: [], candidates: [],
|
|
78
|
+
rule: nil, patterns: [], reason: "eof", action: pending ? "emit" : "skip", token: pending&.inspect }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def record_unmatched
|
|
82
|
+
position = @lexer.byte_pos
|
|
83
|
+
state = @lexer.state
|
|
84
|
+
bytes = @lexer.buffer.byteslice(position, 1)
|
|
85
|
+
token = yield
|
|
86
|
+
@steps << { byte_begin: position, byte_end: position + 1, next_byte: @lexer.byte_pos,
|
|
87
|
+
state: state, next_state: @lexer.state, text: display_text(bytes), bytes: bytes.bytes,
|
|
88
|
+
candidates: [], rule: nil, patterns: [], reason: "unmatched",
|
|
89
|
+
action: token ? "emit" : "skip", token: token&.inspect }
|
|
90
|
+
token
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def render
|
|
94
|
+
steps.map do |step|
|
|
95
|
+
candidates = step[:candidates].map do |candidate|
|
|
96
|
+
"rule #{candidate[:rule]}(#{candidate[:length]}/#{candidate[:total_length]})"
|
|
97
|
+
end.join(" ")
|
|
98
|
+
target = step[:rule] ? "rule #{step[:rule]}" : step[:reason]
|
|
99
|
+
"@#{step[:byte_begin]} #{step[:text].inspect} state=#{step[:state]} " \
|
|
100
|
+
"candidates: #{candidates} -> #{target} #{step[:action]} (#{step[:reason]}) " \
|
|
101
|
+
"state=#{step[:next_state]}#{step[:token] && " #{step[:token]}"}"
|
|
102
|
+
end.join("\n")
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
|
|
107
|
+
def candidates_at(start)
|
|
108
|
+
@config.backend == :firstmatch ? firstmatch_candidates(start) : dfa_candidates(start)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def dfa_candidates(start)
|
|
112
|
+
dfa = @compiled.machines.fetch(@lexer.state).dfa
|
|
113
|
+
best = {}
|
|
114
|
+
walk(dfa, start) do |acceptance, ending|
|
|
115
|
+
next if acceptance.bol_only && !@lexer.beginning_of_line?
|
|
116
|
+
|
|
117
|
+
rule = @rules.fetch(acceptance.rule_index)
|
|
118
|
+
candidate = candidate_for(rule, start, ending)
|
|
119
|
+
next unless candidate
|
|
120
|
+
previous = best[rule.index]
|
|
121
|
+
best[rule.index] = candidate if !previous || candidate[:total_length] >= previous[:total_length]
|
|
122
|
+
end
|
|
123
|
+
best.values.sort_by { |candidate| candidate[:rule] }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def firstmatch_candidates(start)
|
|
127
|
+
state = @config.states.fetch(@lexer.state)
|
|
128
|
+
@lexer.buffer.read_to_end
|
|
129
|
+
ending = @lexer.buffer.bytesize
|
|
130
|
+
if @config.encoding == Encoding::UTF_8
|
|
131
|
+
ending = start
|
|
132
|
+
ending += @lexer.buffer.utf8_character_length(ending) while
|
|
133
|
+
ending < @lexer.buffer.bytesize && @lexer.buffer.valid_utf8_at?(ending)
|
|
134
|
+
end
|
|
135
|
+
subject = @lexer.buffer.byteslice(start...ending).to_s
|
|
136
|
+
subject.force_encoding(@config.encoding)
|
|
137
|
+
@rules.filter_map do |rule|
|
|
138
|
+
next unless rule.states.include?(@lexer.state) || (state.inclusive && rule.states.include?(:initial))
|
|
139
|
+
matches = rule.patterns.each_with_index.filter_map do |pattern, index|
|
|
140
|
+
condition = rule.pattern_conditions.fetch(index)
|
|
141
|
+
next if condition.bol_only && !@lexer.beginning_of_line?
|
|
142
|
+
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern))
|
|
143
|
+
reference = Regexp::Matcher.reference?(regexp, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
|
|
144
|
+
match = Regexp::Matcher.match(regexp, subject, encoding: @config.encoding,
|
|
145
|
+
unicode: @config.options[:unicode] == true, reference: reference)
|
|
146
|
+
[match[0].bytesize, condition] if match&.begin(0)&.zero?
|
|
147
|
+
end
|
|
148
|
+
length, condition = matches.max_by(&:first)
|
|
149
|
+
next unless length
|
|
150
|
+
next if condition.end_anchor && !end_anchor?(start + length)
|
|
151
|
+
|
|
152
|
+
candidate_for(rule, start, start + length)
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def candidate_for(rule, start, ending)
|
|
157
|
+
trailing = if rule.trailing
|
|
158
|
+
dfa = rule.trailing_dfa || (@trailing_dfas ||= {})[rule.index] ||= Flexr.compile_pattern(
|
|
159
|
+
rule.trailing, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
|
|
160
|
+
)
|
|
161
|
+
longest = nil
|
|
162
|
+
walk(dfa, ending) { |_acceptance, cursor| longest = cursor - ending }
|
|
163
|
+
longest
|
|
164
|
+
else
|
|
165
|
+
0
|
|
166
|
+
end
|
|
167
|
+
return if trailing.nil?
|
|
168
|
+
|
|
169
|
+
{ rule: rule.index, patterns: rule.patterns.map(&:inspect), length: ending - start,
|
|
170
|
+
trailing_length: trailing, total_length: ending - start + trailing }
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def walk(dfa, start)
|
|
174
|
+
state = dfa.start
|
|
175
|
+
cursor = start
|
|
176
|
+
loop do
|
|
177
|
+
if @lexer.utf8_boundary?(cursor)
|
|
178
|
+
dfa.accepts[state].each do |acceptance|
|
|
179
|
+
yield acceptance, cursor unless acceptance.end_anchor && !end_anchor?(cursor)
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
break unless @lexer.buffer.ensure_available?(cursor + 1)
|
|
183
|
+
break unless (state = dfa.transition(state, @lexer.buffer.getbyte(cursor)))
|
|
184
|
+
|
|
185
|
+
cursor += 1
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def end_anchor?(position)
|
|
190
|
+
@lexer.buffer.eof?(position) || @lexer.buffer.getbyte(position) == 0x0a
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def display_text(bytes)
|
|
194
|
+
bytes.dup.force_encoding(Encoding::UTF_8).scrub
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
@@ -13,9 +13,7 @@ module Flexr
|
|
|
13
13
|
"Lower" => "Lowercase", "Lowercase" => "Lowercase",
|
|
14
14
|
"Upper" => "Uppercase", "Uppercase" => "Uppercase"
|
|
15
15
|
}.freeze
|
|
16
|
-
# rubocop:disable Style/MutableConstant
|
|
17
|
-
CACHE = {}
|
|
18
|
-
# rubocop:enable Style/MutableConstant
|
|
16
|
+
CACHE = {} # rubocop:disable Style/MutableConstant
|
|
19
17
|
module_function
|
|
20
18
|
|
|
21
19
|
def ranges(name, negate: false)
|
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Unicode
|
|
5
5
|
module ReferenceRegexp
|
|
6
|
-
# rubocop:disable Style/MutableConstant
|
|
7
|
-
CACHE = {}
|
|
8
|
-
# rubocop:enable Style/MutableConstant
|
|
6
|
+
CACHE = {} # rubocop:disable Style/MutableConstant
|
|
9
7
|
module_function
|
|
10
8
|
|
|
11
9
|
def match(pattern, subject, encoding:, options: 0, unicode: false)
|
|
@@ -22,12 +20,12 @@ module Flexr
|
|
|
22
20
|
return CACHE[key] if CACHE.key?(key)
|
|
23
21
|
|
|
24
22
|
parser = Regexp::Parser.new(pattern.source, options: effective_options, encoding: encoding, unicode: unicode)
|
|
25
|
-
source = source_for(parser.parse
|
|
23
|
+
source = source_for(parser.parse)
|
|
26
24
|
regexp_options = effective_options & ~::Regexp::IGNORECASE
|
|
27
25
|
CACHE[key] = ::Regexp.new(source, regexp_options).freeze
|
|
28
26
|
end
|
|
29
27
|
|
|
30
|
-
def source_for(node
|
|
28
|
+
def source_for(node)
|
|
31
29
|
case node
|
|
32
30
|
when Regexp::AST::Empty, Regexp::AST::Anchor then ""
|
|
33
31
|
when Regexp::AST::Fail then "(?!)"
|
|
@@ -37,7 +35,7 @@ module Flexr
|
|
|
37
35
|
ranges = node.ranges.flat_map do |range|
|
|
38
36
|
if range.first == Regexp::AST::Property
|
|
39
37
|
property_ranges = Unicode::Property.ranges(range[2])
|
|
40
|
-
property_ranges = casefold_ranges(property_ranges) if
|
|
38
|
+
property_ranges = casefold_ranges(property_ranges) if range[3]
|
|
41
39
|
range[1] ? complement(property_ranges) : property_ranges
|
|
42
40
|
else
|
|
43
41
|
[range]
|
|
@@ -45,13 +43,13 @@ module Flexr
|
|
|
45
43
|
end
|
|
46
44
|
ranges = complement(ranges) if node.negated
|
|
47
45
|
codepoint_class(ranges)
|
|
48
|
-
when Regexp::AST::Seq then node.children.map { |child| source_for(child
|
|
46
|
+
when Regexp::AST::Seq then node.children.map { |child| source_for(child) }.join
|
|
49
47
|
when Regexp::AST::Alt
|
|
50
|
-
"(?:#{node.children.map { |child| source_for(child
|
|
51
|
-
when Regexp::AST::Star then "(?:#{source_for(node.child
|
|
48
|
+
"(?:#{node.children.map { |child| source_for(child) }.join('|')})"
|
|
49
|
+
when Regexp::AST::Star then "(?:#{source_for(node.child)})*"
|
|
52
50
|
when Regexp::AST::Repeat
|
|
53
51
|
maximum = node.maximum.nil? ? "" : node.maximum
|
|
54
|
-
"(?:#{source_for(node.child
|
|
52
|
+
"(?:#{source_for(node.child)}){#{node.minimum},#{maximum}}"
|
|
55
53
|
else
|
|
56
54
|
raise CompileError, "unsupported reference AST node: #{node.class}"
|
|
57
55
|
end
|
data/lib/flexr/version.rb
CHANGED
data/lib/flexr.rb
CHANGED
|
@@ -11,4 +11,5 @@ module Flexr
|
|
|
11
11
|
autoload :Options, File.expand_path("flexr/options", __dir__)
|
|
12
12
|
autoload :RakeTask, File.expand_path("flexr/rake_task", __dir__)
|
|
13
13
|
autoload :CLI, File.expand_path("flexr/cli", __dir__)
|
|
14
|
+
autoload :Trace, File.expand_path("flexr/trace", __dir__)
|
|
14
15
|
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ruby_lsp/addon"
|
|
4
|
+
require "flexr/cli"
|
|
5
|
+
|
|
6
|
+
RubyLsp::Addon.depend_on_ruby_lsp!("~> 0.26.11")
|
|
7
|
+
|
|
8
|
+
module RubyLsp
|
|
9
|
+
module Flexr
|
|
10
|
+
class Analysis
|
|
11
|
+
def diagnostics(source)
|
|
12
|
+
return @diagnostics if @source == source
|
|
13
|
+
|
|
14
|
+
diagnostics = begin
|
|
15
|
+
parsed = ::Flexr::Source::PrismReader.new(source).read
|
|
16
|
+
compiled = ::Flexr::CLI.compile(parsed, overrides: { warn_level: :all })
|
|
17
|
+
compiled.diagnostics.reject { |diagnostic| diagnostic.code == "FLEXR-W016" }
|
|
18
|
+
rescue ::Flexr::Error => e
|
|
19
|
+
e.diagnostic ? [e.diagnostic] : []
|
|
20
|
+
rescue ArgumentError => e
|
|
21
|
+
[::Flexr::Diagnostics.error("FLEXR-E017", e.message, location: [0, 0])]
|
|
22
|
+
end
|
|
23
|
+
@source = source.dup.freeze
|
|
24
|
+
@diagnostics = diagnostics
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def message(diagnostic)
|
|
28
|
+
[diagnostic.message, diagnostic.note,
|
|
29
|
+
("help: #{diagnostic.help}" if diagnostic.help)].compact.join("\n\n")
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
class Diagnostics
|
|
34
|
+
def initialize(analysis, encoding)
|
|
35
|
+
@analysis = analysis
|
|
36
|
+
@encoding = encoding
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def run_diagnostic(uri, document)
|
|
40
|
+
return [] unless uri.path&.end_with?(".flexr.rb")
|
|
41
|
+
|
|
42
|
+
source = document.source
|
|
43
|
+
@analysis.diagnostics(source).map do |diagnostic|
|
|
44
|
+
first, last = diagnostic.location || [0, 0]
|
|
45
|
+
severity = { error: Constant::DiagnosticSeverity::ERROR, warning: Constant::DiagnosticSeverity::WARNING,
|
|
46
|
+
info: Constant::DiagnosticSeverity::INFORMATION }.fetch(diagnostic.severity)
|
|
47
|
+
Interface::Diagnostic.new(
|
|
48
|
+
range: Interface::Range.new(start: position(source, first), end: position(source, last)),
|
|
49
|
+
message: @analysis.message(diagnostic), code: diagnostic.code, severity: severity, source: "flexr"
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def position(source, offset)
|
|
57
|
+
prefix = source.byteslice(0, [offset, source.bytesize].min)
|
|
58
|
+
line = prefix.count("\n")
|
|
59
|
+
column = prefix.byteslice(((prefix.b.rindex("\n") || -1) + 1)..).delete_suffix("\r")
|
|
60
|
+
character = case @encoding
|
|
61
|
+
when Encoding::UTF_8 then column.bytesize
|
|
62
|
+
when Encoding::UTF_16LE then column.encode(Encoding::UTF_16LE).bytesize / 2
|
|
63
|
+
else column.length
|
|
64
|
+
end
|
|
65
|
+
Interface::Position.new(line: line, character: character)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
class Hover
|
|
70
|
+
def initialize(builder, analysis, dispatcher)
|
|
71
|
+
@builder = builder
|
|
72
|
+
@analysis = analysis
|
|
73
|
+
dispatcher.register(self, :on_call_node_enter)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def on_call_node_enter(node)
|
|
77
|
+
return unless node.name == :rule && node.receiver.nil?
|
|
78
|
+
|
|
79
|
+
source = node.location.source_lines.join
|
|
80
|
+
diagnostics = @analysis.diagnostics(source).select do |diagnostic|
|
|
81
|
+
diagnostic.location &&
|
|
82
|
+
(node.location.start_offset...node.location.end_offset).cover?(diagnostic.location.first)
|
|
83
|
+
end
|
|
84
|
+
return if diagnostics.empty?
|
|
85
|
+
|
|
86
|
+
text = diagnostics.map { |diagnostic| "#{diagnostic.code}: #{@analysis.message(diagnostic)}" }.join("\n\n")
|
|
87
|
+
@builder.push("```text\n#{text}\n```", category: :documentation)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
class Addon < ::RubyLsp::Addon
|
|
92
|
+
def activate(global_state, _message_queue)
|
|
93
|
+
@analysis = Analysis.new
|
|
94
|
+
global_state.register_formatter("flexr", Diagnostics.new(@analysis, global_state.encoding))
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def deactivate
|
|
98
|
+
@analysis = nil
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def name
|
|
102
|
+
"flexr"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def version
|
|
106
|
+
::Flexr::VERSION
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def create_hover_listener(response_builder, _node_context, dispatcher)
|
|
110
|
+
Hover.new(response_builder, @analysis, dispatcher)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
data/sig/flexr.rbs
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
module Flexr
|
|
2
|
+
VERSION: String
|
|
3
|
+
|
|
4
|
+
type pattern = String | ::Regexp
|
|
5
|
+
type token = [untyped, untyped] | Runtime::Token
|
|
6
|
+
type source_location = [Integer, Integer]
|
|
7
|
+
type cancellation = (^() -> untyped) | (^(Lexer) -> untyped) | _Cancellation | _ContextCancellation
|
|
8
|
+
type recovery_action = :skip | :raise | :token | :halt
|
|
9
|
+
type error_handler = (^(LexError) -> recovery_action) | _ErrorHandler
|
|
10
|
+
|
|
11
|
+
interface _Cancellation
|
|
12
|
+
def call: () -> untyped
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
interface _ContextCancellation
|
|
16
|
+
def call: (Lexer) -> untyped
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
interface _ErrorHandler
|
|
20
|
+
def call: (LexError) -> recovery_action
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
interface _Input
|
|
24
|
+
def read: (Integer) -> String?
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
interface _CompiledPattern
|
|
28
|
+
def accept?: (String) -> bool
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.compile_pattern: (pattern, ?options: Hash[Symbol, untyped], ?encoding: Encoding?) -> _CompiledPattern
|
|
32
|
+
def self.parse_pattern: (pattern, ?options: Hash[Symbol, untyped]) -> untyped
|
|
33
|
+
|
|
34
|
+
module DSL
|
|
35
|
+
def rule: (pattern | Array[pattern], ?skip: bool, ?emit: untyped, ?followed_by: pattern?) ?{ () [self: Lexer] -> untyped } -> nil
|
|
36
|
+
def state: (*(Symbol | String), ?inclusive: bool) { () [self: singleton(Lexer)] -> untyped } -> void
|
|
37
|
+
def all_states: () { () [self: singleton(Lexer)] -> untyped } -> void
|
|
38
|
+
def on_eof: () { () [self: Lexer] -> untyped } -> void
|
|
39
|
+
def emits: (*untyped) -> void
|
|
40
|
+
def backend: (:table | :direct | :auto | :firstmatch | String) -> void
|
|
41
|
+
def token_kind: (:array | :struct | :yield | String) -> void
|
|
42
|
+
def encoding: (Encoding | String) -> void
|
|
43
|
+
def option: (*(Symbol | String)) -> void
|
|
44
|
+
def accel: (:auto | :strscan | :regexp | :none | String) -> void
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
class Lexer
|
|
48
|
+
extend DSL
|
|
49
|
+
|
|
50
|
+
def initialize: (String | _Input, ?filename: String?, ?error_mode: Symbol, ?max_token_size: Integer, ?max_lookahead_size: Integer?, ?max_buffer_size: Integer, ?max_state_stack: Integer, ?max_steps: Integer?, ?cancellation: cancellation?, ?retain_input: bool, ?chunk_size: Integer) -> void
|
|
51
|
+
def next_token: () -> token?
|
|
52
|
+
def tokens: () -> Array[token]
|
|
53
|
+
def each_token: () -> Enumerator[token, self]
|
|
54
|
+
| () { (untyped, ?untyped) -> untyped } -> self
|
|
55
|
+
def racc_next_token: () -> [untyped, untyped]
|
|
56
|
+
|
|
57
|
+
attr_reader filename: String?
|
|
58
|
+
attr_reader error_mode: Symbol
|
|
59
|
+
attr_reader max_token_size: Integer
|
|
60
|
+
attr_reader max_lookahead_size: Integer
|
|
61
|
+
attr_reader steps: Integer
|
|
62
|
+
attr_writer on_error: error_handler?
|
|
63
|
+
def input: () -> String
|
|
64
|
+
def text: () -> String
|
|
65
|
+
def text_bytesize: () -> Integer
|
|
66
|
+
def last_location: () -> Runtime::Location
|
|
67
|
+
def byte_pos: () -> Integer
|
|
68
|
+
def lineno: () -> Integer
|
|
69
|
+
alias line lineno
|
|
70
|
+
def beginning_of_line?: () -> bool
|
|
71
|
+
def state: () -> Symbol
|
|
72
|
+
def binary_input: () -> String
|
|
73
|
+
def emit: (untyped, ?untyped) -> token
|
|
74
|
+
def skip: () -> nil
|
|
75
|
+
def echo: () -> token
|
|
76
|
+
def error!: (String) -> token?
|
|
77
|
+
def push: (Symbol | String) -> Symbol
|
|
78
|
+
alias push_state push
|
|
79
|
+
def pop: () -> Symbol
|
|
80
|
+
alias pop_state pop
|
|
81
|
+
def begin_state: (Symbol | String) -> Symbol
|
|
82
|
+
alias state= begin_state
|
|
83
|
+
def less: (Integer) -> nil
|
|
84
|
+
def more: () -> true
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
module Runtime
|
|
88
|
+
class Token < ::Data
|
|
89
|
+
def self.new: (?type: untyped, ?value: untyped, ?location: Location?) -> instance
|
|
90
|
+
attr_reader type: untyped
|
|
91
|
+
attr_reader value: untyped
|
|
92
|
+
attr_reader location: Location?
|
|
93
|
+
def []: (:location | "location") -> Location?
|
|
94
|
+
| (Integer | Symbol | String) -> untyped
|
|
95
|
+
def with: (?type: untyped, ?value: untyped, ?location: Location?) -> self
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
class Location < ::Data
|
|
99
|
+
def self.new: (?filename: String?, ?byte_begin: Integer?, ?byte_end: Integer?, ?line_begin: Integer?, ?column_begin: Integer?, ?line_end: Integer?, ?column_end: Integer?) -> instance
|
|
100
|
+
attr_reader filename: String?
|
|
101
|
+
attr_reader byte_begin: Integer?
|
|
102
|
+
attr_reader byte_end: Integer?
|
|
103
|
+
attr_reader line_begin: Integer?
|
|
104
|
+
attr_reader line_end: Integer?
|
|
105
|
+
attr_reader column_begin: Integer?
|
|
106
|
+
attr_reader column_end: Integer?
|
|
107
|
+
def []: (:filename | "filename") -> String?
|
|
108
|
+
| (Integer | Symbol | String) -> (Integer | String)?
|
|
109
|
+
def with: (?filename: String?, ?byte_begin: Integer?, ?byte_end: Integer?, ?line_begin: Integer?, ?column_begin: Integer?, ?line_end: Integer?, ?column_end: Integer?) -> self
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
class TokenTooLargeError < LexError
|
|
113
|
+
def initialize: (?String, ?filename: String?, ?byte_pos: Integer?, ?line: Integer?, ?text: String?, ?rule: Integer?) -> void
|
|
114
|
+
attr_reader code: String
|
|
115
|
+
attr_reader rule: Integer?
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
class Error < StandardError
|
|
120
|
+
def initialize: (?String?, ?diagnostic: Diagnostic?, ?filename: String?, ?byte_pos: Integer?) -> void
|
|
121
|
+
attr_reader diagnostic: Diagnostic?
|
|
122
|
+
attr_reader filename: String?
|
|
123
|
+
attr_reader byte_pos: Integer?
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
class LexError < Error
|
|
127
|
+
def initialize: (String, ?filename: String?, ?byte_pos: Integer?, ?line: Integer?, ?text: String?, ?diagnostic: Diagnostic?) -> void
|
|
128
|
+
attr_reader line: Integer?
|
|
129
|
+
attr_reader text: String?
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
class CompileError < Error
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
class UnsupportedRegexpError < CompileError
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
class Diagnostic < ::Struct[untyped]
|
|
139
|
+
def self.new: (?code: String?, ?severity: Symbol?, ?message: String?, ?location: source_location?, ?help: String?, ?note: String?) -> instance
|
|
140
|
+
attr_accessor code: String?
|
|
141
|
+
attr_accessor severity: Symbol?
|
|
142
|
+
attr_accessor message: String?
|
|
143
|
+
attr_accessor location: source_location?
|
|
144
|
+
attr_accessor help: String?
|
|
145
|
+
attr_accessor note: String?
|
|
146
|
+
def error?: () -> bool
|
|
147
|
+
def warning?: () -> bool
|
|
148
|
+
def to_h: () -> Hash[Symbol, untyped]
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
class DiagnosticSet
|
|
152
|
+
include Enumerable[Diagnostic]
|
|
153
|
+
def initialize: () -> void
|
|
154
|
+
def each: () -> Enumerator[Diagnostic, Array[Diagnostic]]
|
|
155
|
+
| () { (Diagnostic) -> untyped } -> Array[Diagnostic]
|
|
156
|
+
def <<: (Diagnostic) -> self
|
|
157
|
+
def any_error?: () -> bool
|
|
158
|
+
def empty?: () -> bool
|
|
159
|
+
def to_a: () -> Array[Diagnostic]
|
|
160
|
+
def render: (?format: Symbol, ?color: Symbol | bool, ?source: String?, ?path: String?) -> String
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
module Diagnostics
|
|
164
|
+
def self.error: (String, String, ?location: source_location?, ?help: String?, ?note: String?) -> Diagnostic
|
|
165
|
+
def self.warning: (String, String, ?location: source_location?, ?help: String?, ?note: String?) -> Diagnostic
|
|
166
|
+
def self.info: (String, String, ?location: source_location?, ?help: String?, ?note: String?) -> Diagnostic
|
|
167
|
+
def self.raise!: (Diagnostic) -> bot
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
class Generator
|
|
171
|
+
def initialize: (String, ?output: String?, ?eval_mode: bool, ?options: Hash[Symbol, untyped]) -> void
|
|
172
|
+
def generate: () -> String
|
|
173
|
+
attr_reader diagnostics: Array[Diagnostic]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
class RakeTask
|
|
177
|
+
def initialize: (?Symbol | String) ?{ (RakeTask) -> untyped } -> void
|
|
178
|
+
attr_accessor spec: String?
|
|
179
|
+
attr_accessor output: String?
|
|
180
|
+
attr_accessor warn_as_error: bool
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
module CLI
|
|
184
|
+
def self.run: (Array[String], ?out: IO | _Output, ?err: IO | _Output) -> Integer
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
interface _Output
|
|
188
|
+
def puts: (*untyped) -> nil
|
|
189
|
+
def write: (String) -> Integer
|
|
190
|
+
end
|
|
191
|
+
end
|
data/site/README.md
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# flexr product site
|
|
2
2
|
|
|
3
|
-
This directory contains the Astro/Starlight product site for flexr.
|
|
3
|
+
This directory contains the Astro/Starlight product site for flexr. The Playground
|
|
4
|
+
runs real Ruby and flexr locally through a pinned ruby.wasm runtime.
|
|
4
5
|
|
|
5
6
|
## Local development
|
|
6
7
|
|
|
@@ -14,9 +15,59 @@ Run the same checks used by the Pages workflow:
|
|
|
14
15
|
|
|
15
16
|
```sh
|
|
16
17
|
pnpm verify
|
|
18
|
+
pnpm test
|
|
17
19
|
pnpm build
|
|
20
|
+
pnpm exec playwright install chromium
|
|
21
|
+
pnpm test:browser
|
|
18
22
|
```
|
|
19
23
|
|
|
20
|
-
|
|
24
|
+
`pnpm dev`, `pnpm build`, and `pnpm test` package the current repository's Ruby
|
|
25
|
+
files and copy the pinned runtime into ignored `public/playground-runtime` assets.
|
|
26
|
+
The WASM binary and source bundle are not checked into Git. The deployment build
|
|
27
|
+
produces them, including the Ruby distribution's license and notice.
|
|
21
28
|
|
|
22
|
-
The
|
|
29
|
+
The first Run lazily creates a worker and downloads the runtime. Every run starts
|
|
30
|
+
a new Ruby VM, loads the specification, compiles with flexr, executes its actions,
|
|
31
|
+
and returns tokens, real diagnostics, DOT, trace steps, and generated Ruby.
|
|
32
|
+
Prism 1.7.0 is available in the pinned Ruby 4.0.0 WASM build, so both runtime and
|
|
33
|
+
generated execution are tested. The static DSL restrictions of `flexr check` and
|
|
34
|
+
the generator apply. Specifications must be self-contained or use the bundled
|
|
35
|
+
Ruby libraries; additional gems and local companion files are unavailable.
|
|
36
|
+
The [ruby.wasm WASI limitations](https://ruby.github.io/ruby.wasm/README_md.html),
|
|
37
|
+
including unsupported thread creation, also apply.
|
|
38
|
+
The original three presets remain editable starting points. TOML, GraphQL, and
|
|
39
|
+
SQL subset presets load the actual repository example sources, packaged at build
|
|
40
|
+
time, so their lexer rules are maintained in one place.
|
|
41
|
+
|
|
42
|
+
The main page stops execution after five seconds; runtime download/startup has a
|
|
43
|
+
30-second limit. Source and input are limited to 20,000 characters each, DFA
|
|
44
|
+
construction to 1024 states, and output to 4 MB. Graphviz loads in a separate
|
|
45
|
+
worker only when Draw DFA is selected and has its own five-second limit.
|
|
46
|
+
|
|
47
|
+
Source/input are not uploaded by the application. Share links store them in the
|
|
48
|
+
URL fragment, and opening a link never executes it. Review shared Ruby before
|
|
49
|
+
running it. Workers isolate the UI and have a virtual filesystem; they are not a
|
|
50
|
+
complete hostile-code or memory sandbox. The page applies a CSP, disables common
|
|
51
|
+
network APIs in the Ruby worker after loading assets, and sanitizes rendered SVG.
|
|
52
|
+
Ruby actions can still modify their VM and interact with worker JavaScript.
|
|
53
|
+
|
|
54
|
+
## Size and startup measurements
|
|
55
|
+
|
|
56
|
+
Measured on 2026-11-21, macOS arm64, Headless Chrome 157, using the local static
|
|
57
|
+
preview server without network throttling:
|
|
58
|
+
|
|
59
|
+
| Asset | Uncompressed bytes | gzip bytes |
|
|
60
|
+
|---|---:|---:|
|
|
61
|
+
| Ruby 4.0.0 and standard library | 32,555,374 | 10,114,484 |
|
|
62
|
+
| flexr Ruby files and bridge | 533,463 | 119,954 |
|
|
63
|
+
| TOML, GraphQL, and SQL presets | 2,750 | 838 |
|
|
64
|
+
|
|
65
|
+
One browser run measured 43 ms for downloading/compiling the runtime and 203 ms
|
|
66
|
+
for starting the VM and loading flexr. These local numbers do not predict a
|
|
67
|
+
remote cold download. The UI reports measured sizes and timings on every run.
|
|
68
|
+
Initial page loading fetches neither Ruby WASM nor Graphviz; the browser test
|
|
69
|
+
checks that no WASM request occurs before Run or when opening a share link.
|
|
70
|
+
|
|
71
|
+
For local Chrome instead of a Playwright download, set
|
|
72
|
+
`PLAYWRIGHT_CHROMIUM_EXECUTABLE` to its executable path. Screenshots from the
|
|
73
|
+
browser tests are written to ignored `test-results/` directories.
|
data/site/astro.config.mjs
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { defineConfig } from 'astro/config';
|
|
1
|
+
import { defineConfig, passthroughImageService } from 'astro/config';
|
|
2
2
|
import starlight from '@astrojs/starlight';
|
|
3
3
|
|
|
4
4
|
export default defineConfig({
|
|
5
5
|
site: 'https://ydah.github.io',
|
|
6
6
|
base: '/flexr',
|
|
7
|
+
image: { service: passthroughImageService() },
|
|
8
|
+
vite: { worker: { format: 'es' } },
|
|
7
9
|
integrations: [
|
|
8
10
|
starlight({
|
|
9
11
|
title: 'flexr',
|