flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { readFile } from 'node:fs/promises';
|
|
4
|
+
import { createRuntime } from '../src/playground/runtime.mjs';
|
|
5
|
+
import { encodeState, decodeState } from '../src/playground/share.mjs';
|
|
6
|
+
|
|
7
|
+
const binary = await readFile(new URL('../node_modules/@ruby/4.0-wasm-wasi/dist/ruby+stdlib.wasm', import.meta.url));
|
|
8
|
+
const assets = JSON.parse(await readFile(new URL('../public/playground-runtime/flexr.json', import.meta.url), 'utf8'));
|
|
9
|
+
const runtime = await createRuntime(binary, assets);
|
|
10
|
+
|
|
11
|
+
test('executes edited Ruby actions and generates a real scanner with Prism', async () => {
|
|
12
|
+
const source = 'class Lexer < Flexr::Lexer\n emits :INT\n rule(/[0-9]+/) { emit :INT, text.to_i * 2 }\nend';
|
|
13
|
+
const request = { source, input: '12', mode: 'runtime' };
|
|
14
|
+
const result = await runtime.run(request);
|
|
15
|
+
assert.deepEqual(result.tokens, [['INT', 24]]);
|
|
16
|
+
assert.match(result.dot, /digraph flexr/);
|
|
17
|
+
assert.equal(result.steps[0].text, '12');
|
|
18
|
+
assert.match(result.generated, /Generated by flexr/);
|
|
19
|
+
assert.match(result.generated, /def scan_one/);
|
|
20
|
+
const generated = await runtime.run({ ...request, mode: 'generated' });
|
|
21
|
+
assert.deepEqual(generated.tokens, result.tokens);
|
|
22
|
+
assert.deepEqual(generated.steps, result.steps);
|
|
23
|
+
assert.equal(result.ruby_version, '4.0.0');
|
|
24
|
+
assert.equal(result.prism_version, '1.7.0');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('renders real diagnostic source locations and rejects malformed Ruby', async () => {
|
|
28
|
+
const result = await runtime.run({ source: 'class Lexer < Flexr::Lexer\n rule(/a(?=b)/, emit: :A)\nend', input: 'ab', mode: 'runtime' });
|
|
29
|
+
assert.match(result.error, /FLEXR-E014/);
|
|
30
|
+
assert.match(result.error, /playground.flexr.rb:2:/);
|
|
31
|
+
assert.match(result.error, /\^/);
|
|
32
|
+
const syntax = await runtime.run({ source: 'class Lexer <', input: '', mode: 'runtime' });
|
|
33
|
+
assert.match(syntax.error, /FLEXR-E010/);
|
|
34
|
+
const unmatched = await runtime.run({ source: 'class Lexer < Flexr::Lexer\n rule(/a/, emit: :A)\nend', input: 'b', mode: 'runtime' });
|
|
35
|
+
assert.match(unmatched.error, /unexpected byte/);
|
|
36
|
+
assert.match(unmatched.error, /input.txt:1:/);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('each run starts a fresh Ruby VM', async () => {
|
|
40
|
+
const source = 'class ::AbsoluteLexer < Flexr::Lexer\n rule(/a/, emit: :A)\nend';
|
|
41
|
+
assert.deepEqual((await runtime.run({ source, input: 'a', mode: 'runtime' })).tokens, [['A', 'a']]);
|
|
42
|
+
assert.deepEqual((await runtime.run({ source, input: 'a', mode: 'runtime' })).tokens, [['A', 'a']]);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test('console truncation preserves valid UTF-8 and emitted tokens', async () => {
|
|
46
|
+
const source = 'class Lexer < Flexr::Lexer\n rule(/a/) do\n puts 0x3042.chr(Encoding::UTF_8) * 2000\n emit :A\n end\nend';
|
|
47
|
+
for (const mode of ['runtime', 'generated']) {
|
|
48
|
+
const result = await runtime.run({ source, input: 'a', mode });
|
|
49
|
+
assert.equal(result.error, undefined, result.error);
|
|
50
|
+
assert.deepEqual(result.tokens, [['A', 'a']]);
|
|
51
|
+
assert.ok(result.console.length < 2000);
|
|
52
|
+
assert.match(result.console, /^あ+\uFFFD$/);
|
|
53
|
+
assert.doesNotThrow(() => new TextDecoder('utf-8', { fatal: true }).decode(new TextEncoder().encode(result.console)));
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('literal Unicode actions and surrounding constants survive generation', async () => {
|
|
58
|
+
const source = 'class Lexer < Flexr::Lexer\n LABEL = "前"\n rule(/a/) { emit :TEXT, LABEL + "あ" }\n TRAILER = "後"\nend';
|
|
59
|
+
for (const mode of ['runtime', 'generated']) {
|
|
60
|
+
const result = await runtime.run({ source, input: 'a', mode });
|
|
61
|
+
assert.equal(result.error, undefined, result.error);
|
|
62
|
+
assert.deepEqual(result.tokens, [['TEXT', '前あ']]);
|
|
63
|
+
assert.match(result.generated, /TRAILER = "後"/);
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('share links retain Unicode source and input without running code', () => {
|
|
68
|
+
const state = { source: 'class Lexer; # café\nend', input: '日本語', mode: 'runtime' };
|
|
69
|
+
assert.deepEqual(decodeState(encodeState(state)), state);
|
|
70
|
+
assert.throws(() => decodeState('x'.repeat(300_000)), /too large/);
|
|
71
|
+
assert.throws(() => decodeState(btoa(JSON.stringify({ source: 5, input: '' }))), /Invalid/);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('language presets use the repository examples and run in both modes', async () => {
|
|
75
|
+
const presets = JSON.parse(await readFile(new URL('../public/playground-runtime/presets.json', import.meta.url), 'utf8'));
|
|
76
|
+
for (const name of ['toml', 'graphql', 'sql']) {
|
|
77
|
+
const preset = presets[name];
|
|
78
|
+
assert.equal(preset.source, await readFile(new URL(`../../examples/${name}/lexer.flexr.rb`, import.meta.url), 'utf8'));
|
|
79
|
+
const result = await runtime.run({ ...preset, mode: 'runtime' });
|
|
80
|
+
assert.equal(result.error, undefined, result.error);
|
|
81
|
+
assert.ok(result.tokens.length > 0);
|
|
82
|
+
const generated = await runtime.run({ ...preset, mode: 'generated' });
|
|
83
|
+
assert.deepEqual(generated.tokens, result.tokens);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ruby_lsp/internal"
|
|
4
|
+
require "ruby_lsp/test_helper"
|
|
5
|
+
require "ruby_lsp/flexr/addon"
|
|
6
|
+
require "open3"
|
|
7
|
+
|
|
8
|
+
RSpec.describe "Ruby LSP flexr add-on" do
|
|
9
|
+
include RubyLsp::TestHelper
|
|
10
|
+
|
|
11
|
+
def diagnostics(server, uri, id: 1)
|
|
12
|
+
server.process_message(id: id, method: "textDocument/diagnostic", params: { textDocument: { uri: uri } })
|
|
13
|
+
messages = []
|
|
14
|
+
result = server.pop_response
|
|
15
|
+
until result.is_a?(RubyLsp::Result)
|
|
16
|
+
messages << result.inspect
|
|
17
|
+
result = server.pop_response
|
|
18
|
+
end
|
|
19
|
+
expect(result.response).not_to be_nil, messages.join("\n")
|
|
20
|
+
result.response.items
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def open_server(source, path: "unsaved.flexr.rb", encodings: nil, &block)
|
|
24
|
+
uri = URI::Generic.from_path(path: File.join(Dir.pwd, path))
|
|
25
|
+
with_server(source.dup, uri, load_addons: false) do |server, opened_uri|
|
|
26
|
+
server.global_state.apply_options(initializationOptions: { linters: ["flexr"] },
|
|
27
|
+
capabilities: { general: { positionEncodings: encodings } })
|
|
28
|
+
addon = RubyLsp::Flexr::Addon.new
|
|
29
|
+
addon.activate(server.global_state, Queue.new)
|
|
30
|
+
RubyLsp::Addon.addons << addon
|
|
31
|
+
begin
|
|
32
|
+
block.call(server, opened_uri)
|
|
33
|
+
ensure
|
|
34
|
+
RubyLsp::Addon.addons.delete(addon)
|
|
35
|
+
addon.deactivate
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
{ "utf-8" => 18, "utf-32" => 15 }.each do |encoding, column|
|
|
41
|
+
it "uses negotiated #{encoding} positions" do
|
|
42
|
+
source = "class EditorLexer < Flexr::Lexer\n label = '😀'; rule(/a(?=b)/, skip: true)\nend\n"
|
|
43
|
+
open_server(source, encodings: [encoding]) do |server, uri|
|
|
44
|
+
diagnostic = diagnostics(server, uri).find { |item| item.to_hash[:code] == "FLEXR-E014" }
|
|
45
|
+
expect(diagnostic.range.start.to_hash).to eq(line: 1, character: column)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
it "reports warnings from unsaved source with UTF-16 positions and CRLF" do
|
|
51
|
+
source = "class EditorLexer < Flexr::Lexer\r\n emits :WORD, :IF\r\n rule(/[a-z]+/, emit: :WORD)\r\n label = '😀'; rule(/if/, emit: :IF)\r\nend\r\n"
|
|
52
|
+
open_server(source) do |server, uri|
|
|
53
|
+
diagnostic = diagnostics(server, uri).find { |item| item.to_hash[:code] == "FLEXR-W001" }
|
|
54
|
+
expect(diagnostic.source).to eq("flexr")
|
|
55
|
+
expect(diagnostic.severity).to eq(RubyLsp::Constant::DiagnosticSeverity::WARNING)
|
|
56
|
+
expect(diagnostic.range.start.to_hash).to eq(line: 3, character: 16)
|
|
57
|
+
expect(diagnostic.message).to include("unreachable", "example:", '"if"')
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
it "refreshes errors after unsaved edits and clears them when fixed" do
|
|
62
|
+
source = "class EditorLexer < Flexr::Lexer\n rule(/a(?=b)/, skip: true)\nend\n"
|
|
63
|
+
open_server(source) do |server, uri|
|
|
64
|
+
expect(diagnostics(server, uri).map(&:code)).to include("FLEXR-E014")
|
|
65
|
+
server.process_message(method: "textDocument/didChange", params: {
|
|
66
|
+
textDocument: { uri: uri, version: 2 },
|
|
67
|
+
contentChanges: [{ range: { start: { line: 0, character: 0 }, end: { line: 3, character: 0 } },
|
|
68
|
+
text: source.sub("a(?=b)", "ab") }]
|
|
69
|
+
})
|
|
70
|
+
expect(server.instance_variable_get(:@store).get(uri).source).to include("rule(/ab/")
|
|
71
|
+
expect(diagnostics(server, uri, id: 2).map(&:code)).not_to include("FLEXR-E014")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
it "reports invalid configuration repeatedly and recovers after a fixing edit" do
|
|
76
|
+
source = "class EditorLexer < Flexr::Lexer\n backend :bogus\n rule(/a/, skip: true)\nend\n"
|
|
77
|
+
open_server(source) do |server, uri|
|
|
78
|
+
[1, 2].each do |id|
|
|
79
|
+
error = diagnostics(server, uri, id: id).find { |item| item.to_hash[:code] == "FLEXR-E017" }
|
|
80
|
+
expect(error.message).to include("unsupported backend", "bogus")
|
|
81
|
+
expect(error.range.start.to_hash).to eq(line: 0, character: 0)
|
|
82
|
+
end
|
|
83
|
+
server.process_message(method: "textDocument/didChange", params: {
|
|
84
|
+
textDocument: { uri: uri, version: 2 },
|
|
85
|
+
contentChanges: [{ range: { start: { line: 0, character: 0 }, end: { line: 4, character: 0 } }, text: source }]
|
|
86
|
+
})
|
|
87
|
+
expect(diagnostics(server, uri, id: 3).map(&:code)).to include("FLEXR-E017")
|
|
88
|
+
|
|
89
|
+
server.process_message(method: "textDocument/didChange", params: {
|
|
90
|
+
textDocument: { uri: uri, version: 3 },
|
|
91
|
+
contentChanges: [{ range: { start: { line: 0, character: 0 }, end: { line: 4, character: 0 } },
|
|
92
|
+
text: source.sub(":bogus", ":table") }]
|
|
93
|
+
})
|
|
94
|
+
expect(diagnostics(server, uri, id: 4).map(&:code)).not_to include("FLEXR-E017")
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
it "shows counterexamples and overlap information on rule hover without a diagnostics request" do
|
|
99
|
+
source = "class EditorLexer < Flexr::Lexer\n emits :WORD, :IF\n rule(/[a-z]+/, emit: :WORD)\n rule(/if/, emit: :IF)\nend\n"
|
|
100
|
+
open_server(source) do |server, uri|
|
|
101
|
+
server.process_message(id: 1, method: "textDocument/hover", params: {
|
|
102
|
+
textDocument: { uri: uri }, position: { line: 3, character: 3 }
|
|
103
|
+
})
|
|
104
|
+
response = pop_result(server).response
|
|
105
|
+
expect(response.contents.value).to include("FLEXR-W001", "FLEXR-I001", '"if"')
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
it "does not diagnose ordinary Ruby files or execute specification code" do
|
|
110
|
+
source = "raise 'must not execute'\nclass EditorLexer < Flexr::Lexer\n rule(/a(?=b)/, skip: true)\nend\n"
|
|
111
|
+
open_server(source, path: "ordinary.rb") do |server, uri|
|
|
112
|
+
expect(diagnostics(server, uri).map(&:source)).not_to include("flexr")
|
|
113
|
+
end
|
|
114
|
+
open_server(source) do |server, uri|
|
|
115
|
+
expect(diagnostics(server, uri).map(&:code)).to include("FLEXR-E014")
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
it "is discovered and activated by Ruby LSP from the gem load path" do
|
|
120
|
+
Dir.mktmpdir do |directory|
|
|
121
|
+
script = File.join(directory, "discover.rb")
|
|
122
|
+
File.write(script, <<~RUBY)
|
|
123
|
+
require "ruby_lsp/internal"
|
|
124
|
+
server = RubyLsp::Server.new(test_mode: true)
|
|
125
|
+
server.global_state.apply_options(initializationOptions: { linters: ["flexr"] })
|
|
126
|
+
server.load_addons
|
|
127
|
+
puts JSON.generate(RubyLsp::Addon.addons.map { |addon| [addon.name, addon.error?] })
|
|
128
|
+
server.run_shutdown
|
|
129
|
+
RUBY
|
|
130
|
+
output, errors, status = Open3.capture3(RbConfig.ruby, "-Ilib", script, chdir: File.expand_path("..", __dir__))
|
|
131
|
+
expect(status).to be_success, errors
|
|
132
|
+
expect(JSON.parse(output)).to include(["flexr", false])
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
data/tools/docs_verify.rb
CHANGED
|
@@ -13,6 +13,7 @@ module FlexrDocumentation
|
|
|
13
13
|
MARKER_END = "<!-- flexr-help:end -->"
|
|
14
14
|
MARKDOWN_FILES = [
|
|
15
15
|
File.join(ROOT, "README.md"),
|
|
16
|
+
File.join(ROOT, "CHANGELOG.md"),
|
|
16
17
|
File.join(ROOT, "CONTRIBUTING.md"),
|
|
17
18
|
*Dir[File.join(ROOT, "docs/**/*.md")],
|
|
18
19
|
*Dir[File.join(ROOT, "examples/**/README.md")]
|
|
@@ -101,8 +102,8 @@ module FlexrDocumentation
|
|
|
101
102
|
end
|
|
102
103
|
end
|
|
103
104
|
|
|
104
|
-
def command(*
|
|
105
|
-
Open3.capture3(RbConfig.ruby, "-Ilib", "exe/flexr",
|
|
105
|
+
def command(*)
|
|
106
|
+
Open3.capture3(RbConfig.ruby, "-Ilib", "exe/flexr", *, chdir: ROOT)
|
|
106
107
|
end
|
|
107
108
|
|
|
108
109
|
def ruby_json(script, path, input)
|
data/tools/golden.rb
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FlexrVerification
|
|
4
|
+
module Golden
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
def normalize(source)
|
|
8
|
+
lines = source.gsub(/\r\n?/, "\n").lines
|
|
9
|
+
header = lines.index("# Generated by flexr. DO NOT EDIT.\n")
|
|
10
|
+
raise ArgumentError, "missing generated digest header" unless
|
|
11
|
+
header && lines[header + 2]&.match?(/\A# spec-digest: sha256:[a-f0-9]{64}\n\z/)
|
|
12
|
+
|
|
13
|
+
lines[header + 2] = "# spec-digest: <normalized>\n"
|
|
14
|
+
source = lines.join
|
|
15
|
+
version = compiler_version_node(source)
|
|
16
|
+
normalized = source.b.dup
|
|
17
|
+
normalized[version.location.start_offset...version.location.end_offset] = '"<compiler-version>"'
|
|
18
|
+
normalized.force_encoding(source.encoding)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def compiler_version_node(source)
|
|
22
|
+
require "prism"
|
|
23
|
+
parsed = Prism.parse(source)
|
|
24
|
+
raise ArgumentError, "invalid generated Ruby: #{parsed.errors.first.message}" unless parsed.errors.empty?
|
|
25
|
+
|
|
26
|
+
pending = [parsed.value]
|
|
27
|
+
installations = []
|
|
28
|
+
until pending.empty?
|
|
29
|
+
node = pending.pop
|
|
30
|
+
if node.is_a?(Prism::CallNode) && node.name == :install_compiled! &&
|
|
31
|
+
node.receiver&.slice&.delete_prefix("::") == "Flexr::Generated"
|
|
32
|
+
installations << node
|
|
33
|
+
end
|
|
34
|
+
pending.concat(node.child_nodes.compact)
|
|
35
|
+
end
|
|
36
|
+
raise ArgumentError, "expected one generated installation" unless installations.length == 1
|
|
37
|
+
|
|
38
|
+
payload = installations.first.arguments&.arguments&.[](1)
|
|
39
|
+
version = hash_value(hash_value(payload, "artifact"), "compiler_version")
|
|
40
|
+
raise ArgumentError, "expected a literal compiler version" unless version.is_a?(Prism::StringNode)
|
|
41
|
+
|
|
42
|
+
version
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def hash_value(node, name)
|
|
46
|
+
return unless node.is_a?(Prism::HashNode)
|
|
47
|
+
|
|
48
|
+
fields = node.elements.select do |entry|
|
|
49
|
+
entry.is_a?(Prism::AssocNode) && entry.key.is_a?(Prism::SymbolNode) && entry.key.unescaped == name
|
|
50
|
+
end
|
|
51
|
+
fields.first.value if fields.length == 1
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "open3"
|
|
7
|
+
require "shellwords"
|
|
8
|
+
require "tempfile"
|
|
9
|
+
|
|
10
|
+
module FlexrVerification
|
|
11
|
+
module FailureCase
|
|
12
|
+
VERSION = 1
|
|
13
|
+
DIRECTORY = "tmp/verification-failures"
|
|
14
|
+
|
|
15
|
+
module_function
|
|
16
|
+
|
|
17
|
+
def abort_with_record(message, root:, task:, input:, expected:, actual:, **details)
|
|
18
|
+
warn message
|
|
19
|
+
directory = File.expand_path(ENV.fetch("FLEXR_FAILURE_DIR", DIRECTORY), root)
|
|
20
|
+
record = {
|
|
21
|
+
version: VERSION,
|
|
22
|
+
task: task,
|
|
23
|
+
revision: revision(root),
|
|
24
|
+
ruby: RUBY_DESCRIPTION,
|
|
25
|
+
unicode: Flexr::Unicode::VERSION,
|
|
26
|
+
gems: Gem.loaded_specs.slice("bundler", "flexr", "json", "prism", "rake").transform_values { |gem| gem.version.to_s },
|
|
27
|
+
**details,
|
|
28
|
+
input: encode_bytes(input),
|
|
29
|
+
expected: snapshot(expected),
|
|
30
|
+
actual: snapshot(actual)
|
|
31
|
+
}
|
|
32
|
+
FileUtils.mkdir_p(directory)
|
|
33
|
+
path = Tempfile.create(["#{task.tr(':', '-')}-", ".tmp"], directory) do |file|
|
|
34
|
+
file.write("#{JSON.pretty_generate(record)}\n")
|
|
35
|
+
file.close
|
|
36
|
+
target = "#{file.path.delete_suffix('.tmp')}.json"
|
|
37
|
+
File.rename(file.path, target)
|
|
38
|
+
target
|
|
39
|
+
end
|
|
40
|
+
warn "Saved verification failure to #{path}"
|
|
41
|
+
warn "Replay with FLEXR_FAILURE_CASE=#{Shellwords.escape(path)} bundle exec rake verification:replay"
|
|
42
|
+
abort
|
|
43
|
+
rescue SystemCallError, IOError, JSON::GeneratorError => e
|
|
44
|
+
abort "Could not save verification failure: #{e.message}"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def read(path)
|
|
48
|
+
record = JSON.parse(File.binread(path))
|
|
49
|
+
raise ArgumentError, "unsupported verification failure format" unless record.is_a?(Hash) && record["version"] == VERSION
|
|
50
|
+
|
|
51
|
+
number = record.fetch("case")
|
|
52
|
+
raise ArgumentError, "invalid case number" unless number.is_a?(Integer) && number.positive?
|
|
53
|
+
|
|
54
|
+
record
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def section(record, name)
|
|
58
|
+
value = record.fetch(name)
|
|
59
|
+
raise ArgumentError, "invalid #{name} in verification failure" unless value.is_a?(Hash)
|
|
60
|
+
|
|
61
|
+
value
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def encode_bytes(string)
|
|
65
|
+
{ encoding: string.encoding.name, hex: string.unpack1("H*") }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def decode_bytes(record)
|
|
69
|
+
hex = record.fetch("hex")
|
|
70
|
+
encoding = record.fetch("encoding")
|
|
71
|
+
valid_hex = hex.is_a?(String) && hex.match?(/\A[0-9a-f]*\z/) && hex.length.even?
|
|
72
|
+
raise ArgumentError, "invalid encoded bytes" unless valid_hex && encoding.is_a?(String)
|
|
73
|
+
|
|
74
|
+
[hex].pack("H*").force_encoding(Encoding.find(encoding))
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def snapshot(value)
|
|
78
|
+
case value
|
|
79
|
+
when Hash then value.transform_values { |item| snapshot(item) }
|
|
80
|
+
when Array then value.map { |item| snapshot(item) }
|
|
81
|
+
when String then value.ascii_only? || (value.encoding == Encoding::UTF_8 && value.valid_encoding?) ? value : encode_bytes(value)
|
|
82
|
+
when Symbol then { symbol: snapshot(value.to_s) }
|
|
83
|
+
when Float then value.finite? ? value : { float: value.to_s }
|
|
84
|
+
when Numeric, true, false, nil then value
|
|
85
|
+
else { class: value.class.name, inspect: snapshot(value.inspect) }
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def revision(root)
|
|
90
|
+
commit = git(root, "rev-parse", "HEAD")&.strip
|
|
91
|
+
return unless commit
|
|
92
|
+
|
|
93
|
+
diff = git(root, "diff", "--binary", "--no-ext-diff", "--no-textconv", "HEAD", "--")
|
|
94
|
+
untracked = git(root, "ls-files", "--others", "--exclude-standard", "-z")
|
|
95
|
+
return { commit: commit } unless diff && untracked
|
|
96
|
+
|
|
97
|
+
digest = Digest::SHA256.new.update(diff)
|
|
98
|
+
untracked.split("\0").each do |path|
|
|
99
|
+
absolute = File.join(root, path)
|
|
100
|
+
content = if File.symlink?(absolute)
|
|
101
|
+
Digest::SHA256.hexdigest(File.readlink(absolute))
|
|
102
|
+
elsif File.file?(absolute)
|
|
103
|
+
Digest::SHA256.file(absolute).hexdigest
|
|
104
|
+
end
|
|
105
|
+
digest.update("\0#{path}\0#{content}")
|
|
106
|
+
end
|
|
107
|
+
{ commit: commit, dirty: !diff.empty? || !untracked.empty?, worktree_sha256: digest.hexdigest }
|
|
108
|
+
rescue SystemCallError
|
|
109
|
+
{ commit: commit } if commit
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def git(root, *)
|
|
113
|
+
stdout, status = Open3.capture2("git", *, chdir: root, err: File::NULL, binmode: true)
|
|
114
|
+
stdout if status.success?
|
|
115
|
+
rescue Errno::ENOENT
|
|
116
|
+
nil
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: flexr
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yudai Takada
|
|
@@ -18,23 +18,32 @@ extensions: []
|
|
|
18
18
|
extra_rdoc_files: []
|
|
19
19
|
files:
|
|
20
20
|
- ".rubocop.yml"
|
|
21
|
+
- CHANGELOG.md
|
|
21
22
|
- CONTRIBUTING.md
|
|
22
23
|
- LICENSE.txt
|
|
23
24
|
- README.md
|
|
24
25
|
- Rakefile
|
|
26
|
+
- Steepfile
|
|
25
27
|
- benchmark/baselines/json.json
|
|
26
28
|
- benchmark/baselines/json_handwritten.rb
|
|
27
29
|
- benchmark/baselines/json_rexical.rex
|
|
28
30
|
- benchmark/corpora/README.md
|
|
29
31
|
- benchmark/corpora/generate_json.rb
|
|
30
|
-
- benchmark/golden/calculator_lexer.
|
|
31
|
-
- benchmark/golden/
|
|
32
|
-
- benchmark/golden/
|
|
33
|
-
- benchmark/golden/
|
|
34
|
-
- benchmark/golden/
|
|
35
|
-
- benchmark/golden/
|
|
36
|
-
- benchmark/golden/
|
|
32
|
+
- benchmark/golden/calculator_lexer.rb
|
|
33
|
+
- benchmark/golden/graphql_lexer.rb
|
|
34
|
+
- benchmark/golden/json_lexer.rb
|
|
35
|
+
- benchmark/golden/regexp_tokenizer.rb
|
|
36
|
+
- benchmark/golden/ruby_subset_lexer.rb
|
|
37
|
+
- benchmark/golden/sql_lexer.rb
|
|
38
|
+
- benchmark/golden/toml_lexer.rb
|
|
39
|
+
- benchmark/golden/toy_lang_lexer.rb
|
|
40
|
+
- benchmark/golden/with_lrama_lexer.rb
|
|
41
|
+
- benchmark/golden/with_racc_lexer.rb
|
|
42
|
+
- benchmark/lrama_lexer.rb
|
|
43
|
+
- benchmark/matrix.rb
|
|
44
|
+
- benchmark/profile.rb
|
|
37
45
|
- benchmark/run.rb
|
|
46
|
+
- benchmark/state_dispatch.rb
|
|
38
47
|
- docs/README.md
|
|
39
48
|
- docs/RELEASING.md
|
|
40
49
|
- docs/adr/0001-byte-level-dfa.md
|
|
@@ -64,6 +73,7 @@ files:
|
|
|
64
73
|
- docs/how-to/use-states.md
|
|
65
74
|
- docs/how-to/use-trailing-context.md
|
|
66
75
|
- docs/internals/README.md
|
|
76
|
+
- docs/migration-v2.md
|
|
67
77
|
- docs/perf-log.md
|
|
68
78
|
- docs/reference/README.md
|
|
69
79
|
- docs/reference/actions.md
|
|
@@ -75,16 +85,23 @@ files:
|
|
|
75
85
|
- docs/reference/generated-artifacts.md
|
|
76
86
|
- docs/reference/public-api.md
|
|
77
87
|
- docs/reference/regexp.md
|
|
88
|
+
- docs/reference/ruby-lsp.md
|
|
78
89
|
- docs/reference/runtime.md
|
|
90
|
+
- docs/reference/testing.md
|
|
79
91
|
- docs/reference/tokens-and-locations.md
|
|
80
|
-
- docs/releases/v1.1.0.md
|
|
81
92
|
- docs/tutorial/build-a-calculator-lexer.md
|
|
82
93
|
- examples/calculator/README.md
|
|
83
94
|
- examples/calculator/lexer.flexr.rb
|
|
95
|
+
- examples/graphql/README.md
|
|
96
|
+
- examples/graphql/lexer.flexr.rb
|
|
84
97
|
- examples/json/README.md
|
|
85
98
|
- examples/json/lexer.flexr.rb
|
|
86
99
|
- examples/ruby_subset/README.md
|
|
87
100
|
- examples/ruby_subset/lexer.flexr.rb
|
|
101
|
+
- examples/sql/README.md
|
|
102
|
+
- examples/sql/lexer.flexr.rb
|
|
103
|
+
- examples/toml/README.md
|
|
104
|
+
- examples/toml/lexer.flexr.rb
|
|
88
105
|
- examples/toy_lang/README.md
|
|
89
106
|
- examples/toy_lang/lexer.flexr.rb
|
|
90
107
|
- examples/with_lrama/README.md
|
|
@@ -92,6 +109,8 @@ files:
|
|
|
92
109
|
- examples/with_racc/README.md
|
|
93
110
|
- examples/with_racc/lexer.flexr.rb
|
|
94
111
|
- exe/flexr
|
|
112
|
+
- gemfiles/ruby_lsp.gemfile
|
|
113
|
+
- gemfiles/types.gemfile
|
|
95
114
|
- lib/flexr.rb
|
|
96
115
|
- lib/flexr/action_resolver.rb
|
|
97
116
|
- lib/flexr/artifact_writer.rb
|
|
@@ -117,29 +136,37 @@ files:
|
|
|
117
136
|
- lib/flexr/errors.rb
|
|
118
137
|
- lib/flexr/generated.rb
|
|
119
138
|
- lib/flexr/generator.rb
|
|
139
|
+
- lib/flexr/grammar.rb
|
|
120
140
|
- lib/flexr/importer.rb
|
|
121
141
|
- lib/flexr/ir.rb
|
|
122
142
|
- lib/flexr/lexer.rb
|
|
143
|
+
- lib/flexr/migration.rb
|
|
144
|
+
- lib/flexr/minitest.rb
|
|
123
145
|
- lib/flexr/options.rb
|
|
124
146
|
- lib/flexr/rake_task.rb
|
|
125
147
|
- lib/flexr/regexp/ast.rb
|
|
126
148
|
- lib/flexr/regexp/char_class.rb
|
|
149
|
+
- lib/flexr/regexp/matcher.rb
|
|
127
150
|
- lib/flexr/regexp/normalizer.rb
|
|
128
151
|
- lib/flexr/regexp/parser.rb
|
|
129
152
|
- lib/flexr/regexp/tokenizer.flexr.rb
|
|
130
153
|
- lib/flexr/regexp/tokenizer.rb
|
|
131
154
|
- lib/flexr/regexp/unsupported.rb
|
|
155
|
+
- lib/flexr/rspec.rb
|
|
132
156
|
- lib/flexr/runtime.rb
|
|
133
157
|
- lib/flexr/runtime/buffer.rb
|
|
134
158
|
- lib/flexr/runtime/core.rb
|
|
135
159
|
- lib/flexr/runtime/errors.rb
|
|
136
160
|
- lib/flexr/runtime/interpreter.rb
|
|
137
161
|
- lib/flexr/runtime/location.rb
|
|
162
|
+
- lib/flexr/runtime/record.rb
|
|
138
163
|
- lib/flexr/runtime/token.rb
|
|
139
164
|
- lib/flexr/source.rb
|
|
140
165
|
- lib/flexr/source/passthrough.rb
|
|
141
166
|
- lib/flexr/source/prism_reader.rb
|
|
142
167
|
- lib/flexr/source/static_eval.rb
|
|
168
|
+
- lib/flexr/testing.rb
|
|
169
|
+
- lib/flexr/trace.rb
|
|
143
170
|
- lib/flexr/unicode/case_fold.rb
|
|
144
171
|
- lib/flexr/unicode/data.rb
|
|
145
172
|
- lib/flexr/unicode/data/LICENSE-UNICODE.txt
|
|
@@ -151,12 +178,16 @@ files:
|
|
|
151
178
|
- lib/flexr/unicode/utf8_splitter.rb
|
|
152
179
|
- lib/flexr/unicode/version.rb
|
|
153
180
|
- lib/flexr/version.rb
|
|
181
|
+
- lib/ruby_lsp/flexr/addon.rb
|
|
182
|
+
- sig/flexr.rbs
|
|
154
183
|
- site/README.md
|
|
155
184
|
- site/astro.config.mjs
|
|
156
185
|
- site/package.json
|
|
186
|
+
- site/playwright.config.mjs
|
|
157
187
|
- site/pnpm-lock.yaml
|
|
158
188
|
- site/pnpm-workspace.yaml
|
|
159
|
-
- site/
|
|
189
|
+
- site/ruby/playground.rb
|
|
190
|
+
- site/scripts/prepare-playground.mjs
|
|
160
191
|
- site/scripts/verify-site.mjs
|
|
161
192
|
- site/src/content.config.ts
|
|
162
193
|
- site/src/content/docs/benchmarks.md
|
|
@@ -181,18 +212,30 @@ files:
|
|
|
181
212
|
- site/src/layouts/SiteLayout.astro
|
|
182
213
|
- site/src/pages/index.astro
|
|
183
214
|
- site/src/pages/playground.astro
|
|
215
|
+
- site/src/playground/client.js
|
|
216
|
+
- site/src/playground/graph-worker.mjs
|
|
217
|
+
- site/src/playground/presets.mjs
|
|
218
|
+
- site/src/playground/runtime.mjs
|
|
219
|
+
- site/src/playground/share.mjs
|
|
220
|
+
- site/src/playground/worker.mjs
|
|
184
221
|
- site/src/styles/custom.css
|
|
222
|
+
- site/src/styles/landing.css
|
|
223
|
+
- site/tests/browser/playground.spec.mjs
|
|
224
|
+
- site/tests/playground-runtime.test.mjs
|
|
185
225
|
- site/tsconfig.json
|
|
226
|
+
- test/ruby_lsp_addon_spec.rb
|
|
186
227
|
- tools/coverage.rb
|
|
187
228
|
- tools/docs_verify.rb
|
|
188
229
|
- tools/gen_unicode_tables.rb
|
|
230
|
+
- tools/golden.rb
|
|
189
231
|
- tools/regexp_tokenizer_reference.rb
|
|
232
|
+
- tools/verification_failure.rb
|
|
190
233
|
homepage: https://github.com/ydah/flexr
|
|
191
234
|
licenses:
|
|
192
235
|
- MIT
|
|
193
236
|
metadata:
|
|
194
237
|
rubygems_mfa_required: 'true'
|
|
195
|
-
generator_ruby_version: ">= 3.
|
|
238
|
+
generator_ruby_version: ">= 3.4 (Prism)"
|
|
196
239
|
homepage_uri: https://github.com/ydah/flexr
|
|
197
240
|
source_code_uri: https://github.com/ydah/flexr/tree/main
|
|
198
241
|
bug_tracker_uri: https://github.com/ydah/flexr/issues
|
|
@@ -203,14 +246,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
203
246
|
requirements:
|
|
204
247
|
- - ">="
|
|
205
248
|
- !ruby/object:Gem::Version
|
|
206
|
-
version: 3.
|
|
249
|
+
version: 3.4.0
|
|
207
250
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
208
251
|
requirements:
|
|
209
252
|
- - ">="
|
|
210
253
|
- !ruby/object:Gem::Version
|
|
211
254
|
version: '0'
|
|
212
255
|
requirements: []
|
|
213
|
-
rubygems_version: 4.0.
|
|
256
|
+
rubygems_version: 4.0.20
|
|
214
257
|
specification_version: 4
|
|
215
258
|
summary: Ruby-native lexer generator with runtime and generated modes
|
|
216
259
|
test_files: []
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
d01c0b69f4f2fedd572e90000f4fc670524b01089a6942b94b1fc1c1cd27f95e
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
f6d8418031b657a328d3ad62fcd780a43bbf4731e9d0e390120dd219894a515e
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
1f53aa34bf24a92fd03096648b3290bbfda76222078c0c58966e259fd3a13aca
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
5bb597bfacaf3217d2852185a9c4e90833a3cb6da02e114817d00d00dd3a3aec
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0285b7b1ceff88a7eddcd1f7d3ac87b0a4bce1e4162f5e64205549aacd576dce
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
6abde13a982ee9f529cc2182dad6cefd1f8608af295981fdfe8671e6b46c7f99
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
cb3836416ff7db7877efa68dc4f921755af9ebfb1e03aaa440892e12b6b7ac35
|
data/docs/releases/v1.1.0.md
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
# flexr v1.1.0
|
|
2
|
-
|
|
3
|
-
Release date: 2026-08-13
|
|
4
|
-
|
|
5
|
-
v1.1.0 strengthens the runtime, generated, and standalone lexer contract while
|
|
6
|
-
keeping the stable public API focused on deterministic, byte-oriented lexing.
|
|
7
|
-
|
|
8
|
-
## Highlights
|
|
9
|
-
|
|
10
|
-
- Runtime, generated, and standalone modes now share action precedence,
|
|
11
|
-
nested-state behavior, static-evaluation rules, immutable compiled
|
|
12
|
-
specifications, and structured error outcomes.
|
|
13
|
-
- Runtime resource usage is bounded with token, lookahead, buffer, step, state
|
|
14
|
-
stack, cancellation, and non-progress guards. Streaming input supports
|
|
15
|
-
sliding-buffer retention and incremental location tracking.
|
|
16
|
-
- Unicode properties and POSIX classes use the vendored Unicode 15.1.0 data
|
|
17
|
-
through the DFA path, including property-aware longest-match behavior and
|
|
18
|
-
Unicode-safe empty-match progress.
|
|
19
|
-
- Regexp parsing and normalization now fail closed for unsupported syntax and
|
|
20
|
-
cover empty-language AST nodes, symbolic repetition, class intersections,
|
|
21
|
-
trailing hyphens, shorthand classes, and exact first-match counterexamples.
|
|
22
|
-
- Compiler and generated-runtime paths use indexed construction, Hopcroft
|
|
23
|
-
minimization, lazy transition representations, backend cost selection, and
|
|
24
|
-
state-specific scanner fast paths.
|
|
25
|
-
- Generated artifact writes are collision-safe, atomic, no-op aware, and carry
|
|
26
|
-
schema, runtime ABI, compiler, and Unicode metadata.
|
|
27
|
-
|
|
28
|
-
## Compatibility notes
|
|
29
|
-
|
|
30
|
-
- Stable backends remain `table`, `direct`, and `auto`.
|
|
31
|
-
- `firstmatch` remains experimental and intentionally does not provide the
|
|
32
|
-
longest-match guarantee.
|
|
33
|
-
- Generated files should be regenerated with flexr v1.1.0 when upgrading from
|
|
34
|
-
v1.0.x. Older generated artifacts remain loadable when their artifact
|
|
35
|
-
metadata is compatible.
|
|
36
|
-
- Unicode behavior is pinned to the vendored Unicode 15.1.0 snapshot rather
|
|
37
|
-
than the host Ruby version.
|
|
38
|
-
|
|
39
|
-
## Verification
|
|
40
|
-
|
|
41
|
-
The release candidate passed the repository's RSpec, RuboCop, documentation,
|
|
42
|
-
generation, parity, differential, fuzz, Unicode, coverage, benchmark, and gem
|
|
43
|
-
build gates on the release environment. See [the release procedure](../RELEASING.md)
|
|
44
|
-
for the commands to repeat before tagging and publishing.
|