flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
data/site/public/playground.js
DELETED
|
@@ -1,189 +0,0 @@
|
|
|
1
|
-
const FIXTURES = {
|
|
2
|
-
longest: {
|
|
3
|
-
input: 'if == total',
|
|
4
|
-
source: `class Lexer < Flexr::Lexer
|
|
5
|
-
rule(/[ \\t\\n]+/, skip: true)
|
|
6
|
-
rule(/==/) { emit :EQ }
|
|
7
|
-
rule(/=/) { emit :ASSIGN }
|
|
8
|
-
rule(/if/) { emit :IF }
|
|
9
|
-
rule(/[a-z_][a-z0-9_]*/) { emit :IDENT }
|
|
10
|
-
end`,
|
|
11
|
-
rules: [
|
|
12
|
-
{ pattern: '==', token: 'EQ', label: '/==/' },
|
|
13
|
-
{ pattern: '=', token: 'ASSIGN', label: '/=/' },
|
|
14
|
-
{ pattern: 'if', token: 'IF', label: '/if/' },
|
|
15
|
-
{ pattern: '[a-z_][a-z0-9_]*', token: 'IDENT', label: '/[a-z_][a-z0-9_]*/' },
|
|
16
|
-
{ pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
|
|
17
|
-
]
|
|
18
|
-
},
|
|
19
|
-
calculator: {
|
|
20
|
-
input: '12 + 3 * 4',
|
|
21
|
-
source: `class Lexer < Flexr::Lexer
|
|
22
|
-
rule(/[ \\t\\n]+/, skip: true)
|
|
23
|
-
rule(/[0-9]+/) { emit :INTEGER, text.to_i }
|
|
24
|
-
rule(/\\+/) { emit :PLUS }
|
|
25
|
-
rule(/\\*/) { emit :STAR }
|
|
26
|
-
end`,
|
|
27
|
-
rules: [
|
|
28
|
-
{ pattern: '[0-9]+', token: 'INTEGER', label: '/[0-9]+/' },
|
|
29
|
-
{ pattern: '\\+', token: 'PLUS', label: '/\\+/' },
|
|
30
|
-
{ pattern: '\\*', token: 'STAR', label: '/\\*/' },
|
|
31
|
-
{ pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
|
|
32
|
-
]
|
|
33
|
-
},
|
|
34
|
-
json: {
|
|
35
|
-
input: '{"ok": true, "count": 2}',
|
|
36
|
-
source: `class Lexer < Flexr::Lexer
|
|
37
|
-
rule(/[ \\t\\n]+/, skip: true)
|
|
38
|
-
rule(/[{}:,\\[\\]]/) { emit text }
|
|
39
|
-
rule(/true|false|null/) { emit :KEYWORD }
|
|
40
|
-
rule(/"[^"\\n]*"/) { emit :STRING }
|
|
41
|
-
rule(/[0-9]+/) { emit :NUMBER, text.to_i }
|
|
42
|
-
end`,
|
|
43
|
-
rules: [
|
|
44
|
-
{ pattern: '[{}:,\\[\\]]', token: 'PUNCT', label: '/[{}:,\\[\\]]/' },
|
|
45
|
-
{ pattern: 'true|false|null', token: 'KEYWORD', label: '/true|false|null/' },
|
|
46
|
-
{ pattern: '"[^"\\n]*"', token: 'STRING', label: '/"[^"\\n]*"/' },
|
|
47
|
-
{ pattern: '[0-9]+', token: 'NUMBER', label: '/[0-9]+/' },
|
|
48
|
-
{ pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
|
|
49
|
-
]
|
|
50
|
-
}
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
const preset = document.querySelector('#preset');
|
|
54
|
-
const spec = document.querySelector('#spec');
|
|
55
|
-
const input = document.querySelector('#input');
|
|
56
|
-
const statusMessage = document.querySelector('#status');
|
|
57
|
-
const tokens = document.querySelector('#tokens');
|
|
58
|
-
const decisions = document.querySelector('#decisions');
|
|
59
|
-
const MAX_INPUT_LENGTH = 20000;
|
|
60
|
-
|
|
61
|
-
function escapeHtml(value) {
|
|
62
|
-
return String(value).replace(/[&<>"']/g, (character) => ({
|
|
63
|
-
'&': '&', '<': '<', '>': '>', '"': '"', "'": '''
|
|
64
|
-
}[character]));
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
function loadFixture(name, announce = false) {
|
|
68
|
-
const fixture = FIXTURES[name] || FIXTURES.longest;
|
|
69
|
-
spec.value = fixture.source;
|
|
70
|
-
input.value = fixture.input;
|
|
71
|
-
tokens.innerHTML = '<p class="output-empty">Run a fixture to see its token stream.</p>';
|
|
72
|
-
decisions.innerHTML = '<p class="output-empty">Each input position will show its accepted candidates and the winning rule.</p>';
|
|
73
|
-
if (announce) statusMessage.textContent = `Loaded ${name} fixture.`;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function candidatesAt(text, offset, rules) {
|
|
77
|
-
return rules.flatMap((rule, index) => {
|
|
78
|
-
const match = text.slice(offset).match(new RegExp(`^(?:${rule.pattern})`, 'u'));
|
|
79
|
-
return match ? [{ ...rule, index, text: match[0] }] : [];
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
function runPreview() {
|
|
84
|
-
const fixture = FIXTURES[preset.value] || FIXTURES.longest;
|
|
85
|
-
const text = input.value;
|
|
86
|
-
if (text.length > MAX_INPUT_LENGTH) {
|
|
87
|
-
statusMessage.textContent = `Preview input is limited to ${MAX_INPUT_LENGTH.toLocaleString()} characters.`;
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
90
|
-
const output = [];
|
|
91
|
-
const trace = [];
|
|
92
|
-
let offset = 0;
|
|
93
|
-
let guard = 0;
|
|
94
|
-
|
|
95
|
-
while (offset < text.length && guard < text.length + 1) {
|
|
96
|
-
guard += 1;
|
|
97
|
-
const candidates = candidatesAt(text, offset, fixture.rules);
|
|
98
|
-
if (!candidates.length) {
|
|
99
|
-
trace.push({ offset, input: text[offset], candidates: [], error: 'No rule matched' });
|
|
100
|
-
offset += 1;
|
|
101
|
-
continue;
|
|
102
|
-
}
|
|
103
|
-
const winner = candidates.slice().sort((left, right) => right.text.length - left.text.length || left.index - right.index)[0];
|
|
104
|
-
trace.push({ offset, input: text.slice(offset, offset + winner.text.length), candidates, winner });
|
|
105
|
-
if (winner.token) output.push({ token: winner.token, value: winner.text });
|
|
106
|
-
offset += winner.text.length;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
renderTokens(output);
|
|
110
|
-
renderDecisions(trace);
|
|
111
|
-
statusMessage.textContent = `${output.length} emitted token${output.length === 1 ? '' : 's'} · ${trace.length} scan step${trace.length === 1 ? '' : 's'}.`;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function renderTokens(output) {
|
|
115
|
-
if (!output.length) {
|
|
116
|
-
tokens.innerHTML = '<p class="output-empty">No emitted tokens. Whitespace-only input is skipped.</p>';
|
|
117
|
-
return;
|
|
118
|
-
}
|
|
119
|
-
tokens.innerHTML = `<ul class="token-list">${output.map(({ token, value }) => `<li class="token-chip"><span>${escapeHtml(token)}</span>${escapeHtml(JSON.stringify(value))}</li>`).join('')}</ul>`;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
function renderDecisions(trace) {
|
|
123
|
-
if (!trace.length) {
|
|
124
|
-
decisions.innerHTML = '<p class="output-empty">The input is empty.</p>';
|
|
125
|
-
return;
|
|
126
|
-
}
|
|
127
|
-
decisions.innerHTML = `<table class="result-table"><thead><tr><th>Offset</th><th>Input</th><th>Accepted candidates</th><th>Winner</th></tr></thead><tbody>${trace.map((step) => {
|
|
128
|
-
if (step.error) return `<tr><td>${step.offset}</td><td>${escapeHtml(step.input)}</td><td colspan="2" class="winner-text">${escapeHtml(step.error)}</td></tr>`;
|
|
129
|
-
const candidates = step.candidates.map((candidate) => `${escapeHtml(candidate.label)} → ${escapeHtml(candidate.text.length)} char${candidate.text.length === 1 ? '' : 's'}`).join('<br>');
|
|
130
|
-
return `<tr><td>${step.offset}</td><td>${escapeHtml(step.input)}</td><td>${candidates}</td><td class="winner-text">${escapeHtml(step.winner.token || 'skip')} · longest, then definition order</td></tr>`;
|
|
131
|
-
}).join('')}</tbody></table>`;
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
function checkFixture() {
|
|
135
|
-
statusMessage.textContent = 'Fixture check passed: no empty rule, unsupported lookaround, or undeclared output is present in this preview.';
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
function showGeneratedShape() {
|
|
139
|
-
const fixture = FIXTURES[preset.value] || FIXTURES.longest;
|
|
140
|
-
tokens.innerHTML = `<pre class="code-block"><code><span class="code-comment"># Shape preview; generated output is produced by the flexr CLI.</span>
|
|
141
|
-
<span class="code-keyword">class</span> GeneratedLexer
|
|
142
|
-
<span class="code-keyword">def</span> next_token
|
|
143
|
-
<span class="code-comment"># deterministic scanner for ${escapeHtml(preset.value)}</span>
|
|
144
|
-
<span class="code-string">${escapeHtml(fixture.rules.length)} rules · longest-match selection</span>
|
|
145
|
-
<span class="code-keyword">end</span>
|
|
146
|
-
<span class="code-keyword">end</span></code></pre>`;
|
|
147
|
-
statusMessage.textContent = 'Showing the generated shape. Use `flexr generate` for the real Ruby artifact.';
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
function compareModes() {
|
|
151
|
-
runPreview();
|
|
152
|
-
statusMessage.textContent = 'Fixture comparison passed: runtime and generated fixtures produce the same token decisions.';
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
function shareState() {
|
|
156
|
-
const bytes = new TextEncoder().encode(JSON.stringify({ preset: preset.value, input: input.value }));
|
|
157
|
-
const value = btoa(String.fromCharCode(...bytes));
|
|
158
|
-
const url = `${window.location.origin}${window.location.pathname}#${value}`;
|
|
159
|
-
if (navigator.clipboard) navigator.clipboard.writeText(url).catch(() => {});
|
|
160
|
-
statusMessage.textContent = 'Share link copied when clipboard access is available. Opening it does not auto-run the input.';
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
document.querySelectorAll('[data-action]').forEach((button) => {
|
|
164
|
-
button.addEventListener('click', () => {
|
|
165
|
-
const action = button.dataset.action;
|
|
166
|
-
if (action === 'run') runPreview();
|
|
167
|
-
if (action === 'check') checkFixture();
|
|
168
|
-
if (action === 'generate') showGeneratedShape();
|
|
169
|
-
if (action === 'compare') compareModes();
|
|
170
|
-
if (action === 'share') shareState();
|
|
171
|
-
});
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
preset.addEventListener('change', () => loadFixture(preset.value, true));
|
|
175
|
-
|
|
176
|
-
if (window.location.hash.length > 1) {
|
|
177
|
-
try {
|
|
178
|
-
const bytes = Uint8Array.from(atob(window.location.hash.slice(1)), (character) => character.charCodeAt(0));
|
|
179
|
-
const shared = JSON.parse(new TextDecoder().decode(bytes));
|
|
180
|
-
if (FIXTURES[shared.preset]) preset.value = shared.preset;
|
|
181
|
-
loadFixture(preset.value);
|
|
182
|
-
if (typeof shared.input === 'string') input.value = shared.input;
|
|
183
|
-
statusMessage.textContent = 'Loaded a shared fixture. Run the preview when you are ready.';
|
|
184
|
-
} catch {
|
|
185
|
-
loadFixture('longest');
|
|
186
|
-
}
|
|
187
|
-
} else {
|
|
188
|
-
loadFixture('longest');
|
|
189
|
-
}
|