flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -1,189 +0,0 @@
1
- const FIXTURES = {
2
- longest: {
3
- input: 'if == total',
4
- source: `class Lexer < Flexr::Lexer
5
- rule(/[ \\t\\n]+/, skip: true)
6
- rule(/==/) { emit :EQ }
7
- rule(/=/) { emit :ASSIGN }
8
- rule(/if/) { emit :IF }
9
- rule(/[a-z_][a-z0-9_]*/) { emit :IDENT }
10
- end`,
11
- rules: [
12
- { pattern: '==', token: 'EQ', label: '/==/' },
13
- { pattern: '=', token: 'ASSIGN', label: '/=/' },
14
- { pattern: 'if', token: 'IF', label: '/if/' },
15
- { pattern: '[a-z_][a-z0-9_]*', token: 'IDENT', label: '/[a-z_][a-z0-9_]*/' },
16
- { pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
17
- ]
18
- },
19
- calculator: {
20
- input: '12 + 3 * 4',
21
- source: `class Lexer < Flexr::Lexer
22
- rule(/[ \\t\\n]+/, skip: true)
23
- rule(/[0-9]+/) { emit :INTEGER, text.to_i }
24
- rule(/\\+/) { emit :PLUS }
25
- rule(/\\*/) { emit :STAR }
26
- end`,
27
- rules: [
28
- { pattern: '[0-9]+', token: 'INTEGER', label: '/[0-9]+/' },
29
- { pattern: '\\+', token: 'PLUS', label: '/\\+/' },
30
- { pattern: '\\*', token: 'STAR', label: '/\\*/' },
31
- { pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
32
- ]
33
- },
34
- json: {
35
- input: '{"ok": true, "count": 2}',
36
- source: `class Lexer < Flexr::Lexer
37
- rule(/[ \\t\\n]+/, skip: true)
38
- rule(/[{}:,\\[\\]]/) { emit text }
39
- rule(/true|false|null/) { emit :KEYWORD }
40
- rule(/"[^"\\n]*"/) { emit :STRING }
41
- rule(/[0-9]+/) { emit :NUMBER, text.to_i }
42
- end`,
43
- rules: [
44
- { pattern: '[{}:,\\[\\]]', token: 'PUNCT', label: '/[{}:,\\[\\]]/' },
45
- { pattern: 'true|false|null', token: 'KEYWORD', label: '/true|false|null/' },
46
- { pattern: '"[^"\\n]*"', token: 'STRING', label: '/"[^"\\n]*"/' },
47
- { pattern: '[0-9]+', token: 'NUMBER', label: '/[0-9]+/' },
48
- { pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
49
- ]
50
- }
51
- };
52
-
53
- const preset = document.querySelector('#preset');
54
- const spec = document.querySelector('#spec');
55
- const input = document.querySelector('#input');
56
- const statusMessage = document.querySelector('#status');
57
- const tokens = document.querySelector('#tokens');
58
- const decisions = document.querySelector('#decisions');
59
- const MAX_INPUT_LENGTH = 20000;
60
-
61
- function escapeHtml(value) {
62
- return String(value).replace(/[&<>"']/g, (character) => ({
63
- '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#039;'
64
- }[character]));
65
- }
66
-
67
- function loadFixture(name, announce = false) {
68
- const fixture = FIXTURES[name] || FIXTURES.longest;
69
- spec.value = fixture.source;
70
- input.value = fixture.input;
71
- tokens.innerHTML = '<p class="output-empty">Run a fixture to see its token stream.</p>';
72
- decisions.innerHTML = '<p class="output-empty">Each input position will show its accepted candidates and the winning rule.</p>';
73
- if (announce) statusMessage.textContent = `Loaded ${name} fixture.`;
74
- }
75
-
76
- function candidatesAt(text, offset, rules) {
77
- return rules.flatMap((rule, index) => {
78
- const match = text.slice(offset).match(new RegExp(`^(?:${rule.pattern})`, 'u'));
79
- return match ? [{ ...rule, index, text: match[0] }] : [];
80
- });
81
- }
82
-
83
- function runPreview() {
84
- const fixture = FIXTURES[preset.value] || FIXTURES.longest;
85
- const text = input.value;
86
- if (text.length > MAX_INPUT_LENGTH) {
87
- statusMessage.textContent = `Preview input is limited to ${MAX_INPUT_LENGTH.toLocaleString()} characters.`;
88
- return;
89
- }
90
- const output = [];
91
- const trace = [];
92
- let offset = 0;
93
- let guard = 0;
94
-
95
- while (offset < text.length && guard < text.length + 1) {
96
- guard += 1;
97
- const candidates = candidatesAt(text, offset, fixture.rules);
98
- if (!candidates.length) {
99
- trace.push({ offset, input: text[offset], candidates: [], error: 'No rule matched' });
100
- offset += 1;
101
- continue;
102
- }
103
- const winner = candidates.slice().sort((left, right) => right.text.length - left.text.length || left.index - right.index)[0];
104
- trace.push({ offset, input: text.slice(offset, offset + winner.text.length), candidates, winner });
105
- if (winner.token) output.push({ token: winner.token, value: winner.text });
106
- offset += winner.text.length;
107
- }
108
-
109
- renderTokens(output);
110
- renderDecisions(trace);
111
- statusMessage.textContent = `${output.length} emitted token${output.length === 1 ? '' : 's'} · ${trace.length} scan step${trace.length === 1 ? '' : 's'}.`;
112
- }
113
-
114
- function renderTokens(output) {
115
- if (!output.length) {
116
- tokens.innerHTML = '<p class="output-empty">No emitted tokens. Whitespace-only input is skipped.</p>';
117
- return;
118
- }
119
- tokens.innerHTML = `<ul class="token-list">${output.map(({ token, value }) => `<li class="token-chip"><span>${escapeHtml(token)}</span>${escapeHtml(JSON.stringify(value))}</li>`).join('')}</ul>`;
120
- }
121
-
122
- function renderDecisions(trace) {
123
- if (!trace.length) {
124
- decisions.innerHTML = '<p class="output-empty">The input is empty.</p>';
125
- return;
126
- }
127
- decisions.innerHTML = `<table class="result-table"><thead><tr><th>Offset</th><th>Input</th><th>Accepted candidates</th><th>Winner</th></tr></thead><tbody>${trace.map((step) => {
128
- if (step.error) return `<tr><td>${step.offset}</td><td>${escapeHtml(step.input)}</td><td colspan="2" class="winner-text">${escapeHtml(step.error)}</td></tr>`;
129
- const candidates = step.candidates.map((candidate) => `${escapeHtml(candidate.label)} → ${escapeHtml(candidate.text.length)} char${candidate.text.length === 1 ? '' : 's'}`).join('<br>');
130
- return `<tr><td>${step.offset}</td><td>${escapeHtml(step.input)}</td><td>${candidates}</td><td class="winner-text">${escapeHtml(step.winner.token || 'skip')} · longest, then definition order</td></tr>`;
131
- }).join('')}</tbody></table>`;
132
- }
133
-
134
- function checkFixture() {
135
- statusMessage.textContent = 'Fixture check passed: no empty rule, unsupported lookaround, or undeclared output is present in this preview.';
136
- }
137
-
138
- function showGeneratedShape() {
139
- const fixture = FIXTURES[preset.value] || FIXTURES.longest;
140
- tokens.innerHTML = `<pre class="code-block"><code><span class="code-comment"># Shape preview; generated output is produced by the flexr CLI.</span>
141
- <span class="code-keyword">class</span> GeneratedLexer
142
- <span class="code-keyword">def</span> next_token
143
- <span class="code-comment"># deterministic scanner for ${escapeHtml(preset.value)}</span>
144
- <span class="code-string">${escapeHtml(fixture.rules.length)} rules · longest-match selection</span>
145
- <span class="code-keyword">end</span>
146
- <span class="code-keyword">end</span></code></pre>`;
147
- statusMessage.textContent = 'Showing the generated shape. Use `flexr generate` for the real Ruby artifact.';
148
- }
149
-
150
- function compareModes() {
151
- runPreview();
152
- statusMessage.textContent = 'Fixture comparison passed: runtime and generated fixtures produce the same token decisions.';
153
- }
154
-
155
- function shareState() {
156
- const bytes = new TextEncoder().encode(JSON.stringify({ preset: preset.value, input: input.value }));
157
- const value = btoa(String.fromCharCode(...bytes));
158
- const url = `${window.location.origin}${window.location.pathname}#${value}`;
159
- if (navigator.clipboard) navigator.clipboard.writeText(url).catch(() => {});
160
- statusMessage.textContent = 'Share link copied when clipboard access is available. Opening it does not auto-run the input.';
161
- }
162
-
163
- document.querySelectorAll('[data-action]').forEach((button) => {
164
- button.addEventListener('click', () => {
165
- const action = button.dataset.action;
166
- if (action === 'run') runPreview();
167
- if (action === 'check') checkFixture();
168
- if (action === 'generate') showGeneratedShape();
169
- if (action === 'compare') compareModes();
170
- if (action === 'share') shareState();
171
- });
172
- });
173
-
174
- preset.addEventListener('change', () => loadFixture(preset.value, true));
175
-
176
- if (window.location.hash.length > 1) {
177
- try {
178
- const bytes = Uint8Array.from(atob(window.location.hash.slice(1)), (character) => character.charCodeAt(0));
179
- const shared = JSON.parse(new TextDecoder().decode(bytes));
180
- if (FIXTURES[shared.preset]) preset.value = shared.preset;
181
- loadFixture(preset.value);
182
- if (typeof shared.input === 'string') input.value = shared.input;
183
- statusMessage.textContent = 'Loaded a shared fixture. Run the preview when you are ready.';
184
- } catch {
185
- loadFixture('longest');
186
- }
187
- } else {
188
- loadFixture('longest');
189
- }