flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,85 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { readFile } from 'node:fs/promises';
4
+ import { createRuntime } from '../src/playground/runtime.mjs';
5
+ import { encodeState, decodeState } from '../src/playground/share.mjs';
6
+
7
+ const binary = await readFile(new URL('../node_modules/@ruby/4.0-wasm-wasi/dist/ruby+stdlib.wasm', import.meta.url));
8
+ const assets = JSON.parse(await readFile(new URL('../public/playground-runtime/flexr.json', import.meta.url), 'utf8'));
9
+ const runtime = await createRuntime(binary, assets);
10
+
11
+ test('executes edited Ruby actions and generates a real scanner with Prism', async () => {
12
+ const source = 'class Lexer < Flexr::Lexer\n emits :INT\n rule(/[0-9]+/) { emit :INT, text.to_i * 2 }\nend';
13
+ const request = { source, input: '12', mode: 'runtime' };
14
+ const result = await runtime.run(request);
15
+ assert.deepEqual(result.tokens, [['INT', 24]]);
16
+ assert.match(result.dot, /digraph flexr/);
17
+ assert.equal(result.steps[0].text, '12');
18
+ assert.match(result.generated, /Generated by flexr/);
19
+ assert.match(result.generated, /def scan_one/);
20
+ const generated = await runtime.run({ ...request, mode: 'generated' });
21
+ assert.deepEqual(generated.tokens, result.tokens);
22
+ assert.deepEqual(generated.steps, result.steps);
23
+ assert.equal(result.ruby_version, '4.0.0');
24
+ assert.equal(result.prism_version, '1.7.0');
25
+ });
26
+
27
+ test('renders real diagnostic source locations and rejects malformed Ruby', async () => {
28
+ const result = await runtime.run({ source: 'class Lexer < Flexr::Lexer\n rule(/a(?=b)/, emit: :A)\nend', input: 'ab', mode: 'runtime' });
29
+ assert.match(result.error, /FLEXR-E014/);
30
+ assert.match(result.error, /playground.flexr.rb:2:/);
31
+ assert.match(result.error, /\^/);
32
+ const syntax = await runtime.run({ source: 'class Lexer <', input: '', mode: 'runtime' });
33
+ assert.match(syntax.error, /FLEXR-E010/);
34
+ const unmatched = await runtime.run({ source: 'class Lexer < Flexr::Lexer\n rule(/a/, emit: :A)\nend', input: 'b', mode: 'runtime' });
35
+ assert.match(unmatched.error, /unexpected byte/);
36
+ assert.match(unmatched.error, /input.txt:1:/);
37
+ });
38
+
39
+ test('each run starts a fresh Ruby VM', async () => {
40
+ const source = 'class ::AbsoluteLexer < Flexr::Lexer\n rule(/a/, emit: :A)\nend';
41
+ assert.deepEqual((await runtime.run({ source, input: 'a', mode: 'runtime' })).tokens, [['A', 'a']]);
42
+ assert.deepEqual((await runtime.run({ source, input: 'a', mode: 'runtime' })).tokens, [['A', 'a']]);
43
+ });
44
+
45
+ test('console truncation preserves valid UTF-8 and emitted tokens', async () => {
46
+ const source = 'class Lexer < Flexr::Lexer\n rule(/a/) do\n puts 0x3042.chr(Encoding::UTF_8) * 2000\n emit :A\n end\nend';
47
+ for (const mode of ['runtime', 'generated']) {
48
+ const result = await runtime.run({ source, input: 'a', mode });
49
+ assert.equal(result.error, undefined, result.error);
50
+ assert.deepEqual(result.tokens, [['A', 'a']]);
51
+ assert.ok(result.console.length < 2000);
52
+ assert.match(result.console, /^あ+\uFFFD$/);
53
+ assert.doesNotThrow(() => new TextDecoder('utf-8', { fatal: true }).decode(new TextEncoder().encode(result.console)));
54
+ }
55
+ });
56
+
57
+ test('literal Unicode actions and surrounding constants survive generation', async () => {
58
+ const source = 'class Lexer < Flexr::Lexer\n LABEL = "前"\n rule(/a/) { emit :TEXT, LABEL + "あ" }\n TRAILER = "後"\nend';
59
+ for (const mode of ['runtime', 'generated']) {
60
+ const result = await runtime.run({ source, input: 'a', mode });
61
+ assert.equal(result.error, undefined, result.error);
62
+ assert.deepEqual(result.tokens, [['TEXT', '前あ']]);
63
+ assert.match(result.generated, /TRAILER = "後"/);
64
+ }
65
+ });
66
+
67
+ test('share links retain Unicode source and input without running code', () => {
68
+ const state = { source: 'class Lexer; # café\nend', input: '日本語', mode: 'runtime' };
69
+ assert.deepEqual(decodeState(encodeState(state)), state);
70
+ assert.throws(() => decodeState('x'.repeat(300_000)), /too large/);
71
+ assert.throws(() => decodeState(btoa(JSON.stringify({ source: 5, input: '' }))), /Invalid/);
72
+ });
73
+
74
+ test('language presets use the repository examples and run in both modes', async () => {
75
+ const presets = JSON.parse(await readFile(new URL('../public/playground-runtime/presets.json', import.meta.url), 'utf8'));
76
+ for (const name of ['toml', 'graphql', 'sql']) {
77
+ const preset = presets[name];
78
+ assert.equal(preset.source, await readFile(new URL(`../../examples/${name}/lexer.flexr.rb`, import.meta.url), 'utf8'));
79
+ const result = await runtime.run({ ...preset, mode: 'runtime' });
80
+ assert.equal(result.error, undefined, result.error);
81
+ assert.ok(result.tokens.length > 0);
82
+ const generated = await runtime.run({ ...preset, mode: 'generated' });
83
+ assert.deepEqual(generated.tokens, result.tokens);
84
+ }
85
+ });
@@ -0,0 +1,135 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ruby_lsp/internal"
4
+ require "ruby_lsp/test_helper"
5
+ require "ruby_lsp/flexr/addon"
6
+ require "open3"
7
+
8
+ RSpec.describe "Ruby LSP flexr add-on" do
9
+ include RubyLsp::TestHelper
10
+
11
+ def diagnostics(server, uri, id: 1)
12
+ server.process_message(id: id, method: "textDocument/diagnostic", params: { textDocument: { uri: uri } })
13
+ messages = []
14
+ result = server.pop_response
15
+ until result.is_a?(RubyLsp::Result)
16
+ messages << result.inspect
17
+ result = server.pop_response
18
+ end
19
+ expect(result.response).not_to be_nil, messages.join("\n")
20
+ result.response.items
21
+ end
22
+
23
+ def open_server(source, path: "unsaved.flexr.rb", encodings: nil, &block)
24
+ uri = URI::Generic.from_path(path: File.join(Dir.pwd, path))
25
+ with_server(source.dup, uri, load_addons: false) do |server, opened_uri|
26
+ server.global_state.apply_options(initializationOptions: { linters: ["flexr"] },
27
+ capabilities: { general: { positionEncodings: encodings } })
28
+ addon = RubyLsp::Flexr::Addon.new
29
+ addon.activate(server.global_state, Queue.new)
30
+ RubyLsp::Addon.addons << addon
31
+ begin
32
+ block.call(server, opened_uri)
33
+ ensure
34
+ RubyLsp::Addon.addons.delete(addon)
35
+ addon.deactivate
36
+ end
37
+ end
38
+ end
39
+
40
+ { "utf-8" => 18, "utf-32" => 15 }.each do |encoding, column|
41
+ it "uses negotiated #{encoding} positions" do
42
+ source = "class EditorLexer < Flexr::Lexer\n label = '😀'; rule(/a(?=b)/, skip: true)\nend\n"
43
+ open_server(source, encodings: [encoding]) do |server, uri|
44
+ diagnostic = diagnostics(server, uri).find { |item| item.to_hash[:code] == "FLEXR-E014" }
45
+ expect(diagnostic.range.start.to_hash).to eq(line: 1, character: column)
46
+ end
47
+ end
48
+ end
49
+
50
+ it "reports warnings from unsaved source with UTF-16 positions and CRLF" do
51
+ source = "class EditorLexer < Flexr::Lexer\r\n emits :WORD, :IF\r\n rule(/[a-z]+/, emit: :WORD)\r\n label = '😀'; rule(/if/, emit: :IF)\r\nend\r\n"
52
+ open_server(source) do |server, uri|
53
+ diagnostic = diagnostics(server, uri).find { |item| item.to_hash[:code] == "FLEXR-W001" }
54
+ expect(diagnostic.source).to eq("flexr")
55
+ expect(diagnostic.severity).to eq(RubyLsp::Constant::DiagnosticSeverity::WARNING)
56
+ expect(diagnostic.range.start.to_hash).to eq(line: 3, character: 16)
57
+ expect(diagnostic.message).to include("unreachable", "example:", '"if"')
58
+ end
59
+ end
60
+
61
+ it "refreshes errors after unsaved edits and clears them when fixed" do
62
+ source = "class EditorLexer < Flexr::Lexer\n rule(/a(?=b)/, skip: true)\nend\n"
63
+ open_server(source) do |server, uri|
64
+ expect(diagnostics(server, uri).map(&:code)).to include("FLEXR-E014")
65
+ server.process_message(method: "textDocument/didChange", params: {
66
+ textDocument: { uri: uri, version: 2 },
67
+ contentChanges: [{ range: { start: { line: 0, character: 0 }, end: { line: 3, character: 0 } },
68
+ text: source.sub("a(?=b)", "ab") }]
69
+ })
70
+ expect(server.instance_variable_get(:@store).get(uri).source).to include("rule(/ab/")
71
+ expect(diagnostics(server, uri, id: 2).map(&:code)).not_to include("FLEXR-E014")
72
+ end
73
+ end
74
+
75
+ it "reports invalid configuration repeatedly and recovers after a fixing edit" do
76
+ source = "class EditorLexer < Flexr::Lexer\n backend :bogus\n rule(/a/, skip: true)\nend\n"
77
+ open_server(source) do |server, uri|
78
+ [1, 2].each do |id|
79
+ error = diagnostics(server, uri, id: id).find { |item| item.to_hash[:code] == "FLEXR-E017" }
80
+ expect(error.message).to include("unsupported backend", "bogus")
81
+ expect(error.range.start.to_hash).to eq(line: 0, character: 0)
82
+ end
83
+ server.process_message(method: "textDocument/didChange", params: {
84
+ textDocument: { uri: uri, version: 2 },
85
+ contentChanges: [{ range: { start: { line: 0, character: 0 }, end: { line: 4, character: 0 } }, text: source }]
86
+ })
87
+ expect(diagnostics(server, uri, id: 3).map(&:code)).to include("FLEXR-E017")
88
+
89
+ server.process_message(method: "textDocument/didChange", params: {
90
+ textDocument: { uri: uri, version: 3 },
91
+ contentChanges: [{ range: { start: { line: 0, character: 0 }, end: { line: 4, character: 0 } },
92
+ text: source.sub(":bogus", ":table") }]
93
+ })
94
+ expect(diagnostics(server, uri, id: 4).map(&:code)).not_to include("FLEXR-E017")
95
+ end
96
+ end
97
+
98
+ it "shows counterexamples and overlap information on rule hover without a diagnostics request" do
99
+ source = "class EditorLexer < Flexr::Lexer\n emits :WORD, :IF\n rule(/[a-z]+/, emit: :WORD)\n rule(/if/, emit: :IF)\nend\n"
100
+ open_server(source) do |server, uri|
101
+ server.process_message(id: 1, method: "textDocument/hover", params: {
102
+ textDocument: { uri: uri }, position: { line: 3, character: 3 }
103
+ })
104
+ response = pop_result(server).response
105
+ expect(response.contents.value).to include("FLEXR-W001", "FLEXR-I001", '"if"')
106
+ end
107
+ end
108
+
109
+ it "does not diagnose ordinary Ruby files or execute specification code" do
110
+ source = "raise 'must not execute'\nclass EditorLexer < Flexr::Lexer\n rule(/a(?=b)/, skip: true)\nend\n"
111
+ open_server(source, path: "ordinary.rb") do |server, uri|
112
+ expect(diagnostics(server, uri).map(&:source)).not_to include("flexr")
113
+ end
114
+ open_server(source) do |server, uri|
115
+ expect(diagnostics(server, uri).map(&:code)).to include("FLEXR-E014")
116
+ end
117
+ end
118
+
119
+ it "is discovered and activated by Ruby LSP from the gem load path" do
120
+ Dir.mktmpdir do |directory|
121
+ script = File.join(directory, "discover.rb")
122
+ File.write(script, <<~RUBY)
123
+ require "ruby_lsp/internal"
124
+ server = RubyLsp::Server.new(test_mode: true)
125
+ server.global_state.apply_options(initializationOptions: { linters: ["flexr"] })
126
+ server.load_addons
127
+ puts JSON.generate(RubyLsp::Addon.addons.map { |addon| [addon.name, addon.error?] })
128
+ server.run_shutdown
129
+ RUBY
130
+ output, errors, status = Open3.capture3(RbConfig.ruby, "-Ilib", script, chdir: File.expand_path("..", __dir__))
131
+ expect(status).to be_success, errors
132
+ expect(JSON.parse(output)).to include(["flexr", false])
133
+ end
134
+ end
135
+ end
data/tools/coverage.rb CHANGED
@@ -7,7 +7,12 @@ Coverage.start(lines: true)
7
7
 
8
8
  status = RSpec::Core::Runner.run(["spec", "--format", "progress", "--no-color"], File::NULL, $stdout)
9
9
  result = Coverage.result
10
- files = Dir[File.expand_path("../lib/flexr/**/*.rb", __dir__)]
10
+ files = Dir[File.expand_path("../lib/flexr/**/*.rb", __dir__)].reject do |file|
11
+ # Generated artifacts duplicate the scanner template's executable lines.
12
+ # Their behavior is exercised by the suite, while the template is counted
13
+ # once as the maintainable source of those lines.
14
+ File.foreach(file).first(3).any? { |line| line.include?("Generated by flexr") }
15
+ end
11
16
  totals = files.map do |file|
12
17
  lines = result.fetch(file, {}).fetch(:lines, [])
13
18
  executable = lines.count { |count| !count.nil? }
data/tools/docs_verify.rb CHANGED
@@ -13,6 +13,7 @@ module FlexrDocumentation
13
13
  MARKER_END = "<!-- flexr-help:end -->"
14
14
  MARKDOWN_FILES = [
15
15
  File.join(ROOT, "README.md"),
16
+ File.join(ROOT, "CHANGELOG.md"),
16
17
  File.join(ROOT, "CONTRIBUTING.md"),
17
18
  *Dir[File.join(ROOT, "docs/**/*.md")],
18
19
  *Dir[File.join(ROOT, "examples/**/README.md")]
@@ -101,8 +102,8 @@ module FlexrDocumentation
101
102
  end
102
103
  end
103
104
 
104
- def command(*arguments)
105
- Open3.capture3(RbConfig.ruby, "-Ilib", "exe/flexr", *arguments, chdir: ROOT)
105
+ def command(*)
106
+ Open3.capture3(RbConfig.ruby, "-Ilib", "exe/flexr", *, chdir: ROOT)
106
107
  end
107
108
 
108
109
  def ruby_json(script, path, input)
data/tools/golden.rb ADDED
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FlexrVerification
4
+ module Golden
5
+ module_function
6
+
7
+ def normalize(source)
8
+ lines = source.gsub(/\r\n?/, "\n").lines
9
+ header = lines.index("# Generated by flexr. DO NOT EDIT.\n")
10
+ raise ArgumentError, "missing generated digest header" unless
11
+ header && lines[header + 2]&.match?(/\A# spec-digest: sha256:[a-f0-9]{64}\n\z/)
12
+
13
+ lines[header + 2] = "# spec-digest: <normalized>\n"
14
+ source = lines.join
15
+ version = compiler_version_node(source)
16
+ normalized = source.b.dup
17
+ normalized[version.location.start_offset...version.location.end_offset] = '"<compiler-version>"'
18
+ normalized.force_encoding(source.encoding)
19
+ end
20
+
21
+ def compiler_version_node(source)
22
+ require "prism"
23
+ parsed = Prism.parse(source)
24
+ raise ArgumentError, "invalid generated Ruby: #{parsed.errors.first.message}" unless parsed.errors.empty?
25
+
26
+ pending = [parsed.value]
27
+ installations = []
28
+ until pending.empty?
29
+ node = pending.pop
30
+ if node.is_a?(Prism::CallNode) && node.name == :install_compiled! &&
31
+ node.receiver&.slice&.delete_prefix("::") == "Flexr::Generated"
32
+ installations << node
33
+ end
34
+ pending.concat(node.child_nodes.compact)
35
+ end
36
+ raise ArgumentError, "expected one generated installation" unless installations.length == 1
37
+
38
+ payload = installations.first.arguments&.arguments&.[](1)
39
+ version = hash_value(hash_value(payload, "artifact"), "compiler_version")
40
+ raise ArgumentError, "expected a literal compiler version" unless version.is_a?(Prism::StringNode)
41
+
42
+ version
43
+ end
44
+
45
+ def hash_value(node, name)
46
+ return unless node.is_a?(Prism::HashNode)
47
+
48
+ fields = node.elements.select do |entry|
49
+ entry.is_a?(Prism::AssocNode) && entry.key.is_a?(Prism::SymbolNode) && entry.key.unescaped == name
50
+ end
51
+ fields.first.value if fields.length == 1
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "fileutils"
5
+ require "json"
6
+ require "open3"
7
+ require "shellwords"
8
+ require "tempfile"
9
+
10
+ module FlexrVerification
11
+ module FailureCase
12
+ VERSION = 1
13
+ DIRECTORY = "tmp/verification-failures"
14
+
15
+ module_function
16
+
17
+ def abort_with_record(message, root:, task:, input:, expected:, actual:, **details)
18
+ warn message
19
+ directory = File.expand_path(ENV.fetch("FLEXR_FAILURE_DIR", DIRECTORY), root)
20
+ record = {
21
+ version: VERSION,
22
+ task: task,
23
+ revision: revision(root),
24
+ ruby: RUBY_DESCRIPTION,
25
+ unicode: Flexr::Unicode::VERSION,
26
+ gems: Gem.loaded_specs.slice("bundler", "flexr", "json", "prism", "rake").transform_values { |gem| gem.version.to_s },
27
+ **details,
28
+ input: encode_bytes(input),
29
+ expected: snapshot(expected),
30
+ actual: snapshot(actual)
31
+ }
32
+ FileUtils.mkdir_p(directory)
33
+ path = Tempfile.create(["#{task.tr(':', '-')}-", ".tmp"], directory) do |file|
34
+ file.write("#{JSON.pretty_generate(record)}\n")
35
+ file.close
36
+ target = "#{file.path.delete_suffix('.tmp')}.json"
37
+ File.rename(file.path, target)
38
+ target
39
+ end
40
+ warn "Saved verification failure to #{path}"
41
+ warn "Replay with FLEXR_FAILURE_CASE=#{Shellwords.escape(path)} bundle exec rake verification:replay"
42
+ abort
43
+ rescue SystemCallError, IOError, JSON::GeneratorError => e
44
+ abort "Could not save verification failure: #{e.message}"
45
+ end
46
+
47
+ def read(path)
48
+ record = JSON.parse(File.binread(path))
49
+ raise ArgumentError, "unsupported verification failure format" unless record.is_a?(Hash) && record["version"] == VERSION
50
+
51
+ number = record.fetch("case")
52
+ raise ArgumentError, "invalid case number" unless number.is_a?(Integer) && number.positive?
53
+
54
+ record
55
+ end
56
+
57
+ def section(record, name)
58
+ value = record.fetch(name)
59
+ raise ArgumentError, "invalid #{name} in verification failure" unless value.is_a?(Hash)
60
+
61
+ value
62
+ end
63
+
64
+ def encode_bytes(string)
65
+ { encoding: string.encoding.name, hex: string.unpack1("H*") }
66
+ end
67
+
68
+ def decode_bytes(record)
69
+ hex = record.fetch("hex")
70
+ encoding = record.fetch("encoding")
71
+ valid_hex = hex.is_a?(String) && hex.match?(/\A[0-9a-f]*\z/) && hex.length.even?
72
+ raise ArgumentError, "invalid encoded bytes" unless valid_hex && encoding.is_a?(String)
73
+
74
+ [hex].pack("H*").force_encoding(Encoding.find(encoding))
75
+ end
76
+
77
+ def snapshot(value)
78
+ case value
79
+ when Hash then value.transform_values { |item| snapshot(item) }
80
+ when Array then value.map { |item| snapshot(item) }
81
+ when String then value.ascii_only? || (value.encoding == Encoding::UTF_8 && value.valid_encoding?) ? value : encode_bytes(value)
82
+ when Symbol then { symbol: snapshot(value.to_s) }
83
+ when Float then value.finite? ? value : { float: value.to_s }
84
+ when Numeric, true, false, nil then value
85
+ else { class: value.class.name, inspect: snapshot(value.inspect) }
86
+ end
87
+ end
88
+
89
+ def revision(root)
90
+ commit = git(root, "rev-parse", "HEAD")&.strip
91
+ return unless commit
92
+
93
+ diff = git(root, "diff", "--binary", "--no-ext-diff", "--no-textconv", "HEAD", "--")
94
+ untracked = git(root, "ls-files", "--others", "--exclude-standard", "-z")
95
+ return { commit: commit } unless diff && untracked
96
+
97
+ digest = Digest::SHA256.new.update(diff)
98
+ untracked.split("\0").each do |path|
99
+ absolute = File.join(root, path)
100
+ content = if File.symlink?(absolute)
101
+ Digest::SHA256.hexdigest(File.readlink(absolute))
102
+ elsif File.file?(absolute)
103
+ Digest::SHA256.file(absolute).hexdigest
104
+ end
105
+ digest.update("\0#{path}\0#{content}")
106
+ end
107
+ { commit: commit, dirty: !diff.empty? || !untracked.empty?, worktree_sha256: digest.hexdigest }
108
+ rescue SystemCallError
109
+ { commit: commit } if commit
110
+ end
111
+
112
+ def git(root, *)
113
+ stdout, status = Open3.capture2("git", *, chdir: root, err: File::NULL, binmode: true)
114
+ stdout if status.success?
115
+ rescue Errno::ENOENT
116
+ nil
117
+ end
118
+ end
119
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flexr
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yudai Takada
@@ -18,23 +18,32 @@ extensions: []
18
18
  extra_rdoc_files: []
19
19
  files:
20
20
  - ".rubocop.yml"
21
+ - CHANGELOG.md
21
22
  - CONTRIBUTING.md
22
23
  - LICENSE.txt
23
24
  - README.md
24
25
  - Rakefile
26
+ - Steepfile
25
27
  - benchmark/baselines/json.json
26
28
  - benchmark/baselines/json_handwritten.rb
27
29
  - benchmark/baselines/json_rexical.rex
28
30
  - benchmark/corpora/README.md
29
31
  - benchmark/corpora/generate_json.rb
30
- - benchmark/golden/calculator_lexer.sha256
31
- - benchmark/golden/json_lexer.sha256
32
- - benchmark/golden/regexp_tokenizer.sha256
33
- - benchmark/golden/ruby_subset_lexer.sha256
34
- - benchmark/golden/toy_lang_lexer.sha256
35
- - benchmark/golden/with_lrama_lexer.sha256
36
- - benchmark/golden/with_racc_lexer.sha256
32
+ - benchmark/golden/calculator_lexer.rb
33
+ - benchmark/golden/graphql_lexer.rb
34
+ - benchmark/golden/json_lexer.rb
35
+ - benchmark/golden/regexp_tokenizer.rb
36
+ - benchmark/golden/ruby_subset_lexer.rb
37
+ - benchmark/golden/sql_lexer.rb
38
+ - benchmark/golden/toml_lexer.rb
39
+ - benchmark/golden/toy_lang_lexer.rb
40
+ - benchmark/golden/with_lrama_lexer.rb
41
+ - benchmark/golden/with_racc_lexer.rb
42
+ - benchmark/lrama_lexer.rb
43
+ - benchmark/matrix.rb
44
+ - benchmark/profile.rb
37
45
  - benchmark/run.rb
46
+ - benchmark/state_dispatch.rb
38
47
  - docs/README.md
39
48
  - docs/RELEASING.md
40
49
  - docs/adr/0001-byte-level-dfa.md
@@ -64,6 +73,7 @@ files:
64
73
  - docs/how-to/use-states.md
65
74
  - docs/how-to/use-trailing-context.md
66
75
  - docs/internals/README.md
76
+ - docs/migration-v2.md
67
77
  - docs/perf-log.md
68
78
  - docs/reference/README.md
69
79
  - docs/reference/actions.md
@@ -75,15 +85,23 @@ files:
75
85
  - docs/reference/generated-artifacts.md
76
86
  - docs/reference/public-api.md
77
87
  - docs/reference/regexp.md
88
+ - docs/reference/ruby-lsp.md
78
89
  - docs/reference/runtime.md
90
+ - docs/reference/testing.md
79
91
  - docs/reference/tokens-and-locations.md
80
92
  - docs/tutorial/build-a-calculator-lexer.md
81
93
  - examples/calculator/README.md
82
94
  - examples/calculator/lexer.flexr.rb
95
+ - examples/graphql/README.md
96
+ - examples/graphql/lexer.flexr.rb
83
97
  - examples/json/README.md
84
98
  - examples/json/lexer.flexr.rb
85
99
  - examples/ruby_subset/README.md
86
100
  - examples/ruby_subset/lexer.flexr.rb
101
+ - examples/sql/README.md
102
+ - examples/sql/lexer.flexr.rb
103
+ - examples/toml/README.md
104
+ - examples/toml/lexer.flexr.rb
87
105
  - examples/toy_lang/README.md
88
106
  - examples/toy_lang/lexer.flexr.rb
89
107
  - examples/with_lrama/README.md
@@ -91,47 +109,66 @@ files:
91
109
  - examples/with_racc/README.md
92
110
  - examples/with_racc/lexer.flexr.rb
93
111
  - exe/flexr
112
+ - gemfiles/ruby_lsp.gemfile
113
+ - gemfiles/types.gemfile
94
114
  - lib/flexr.rb
115
+ - lib/flexr/action_resolver.rb
116
+ - lib/flexr/artifact_writer.rb
95
117
  - lib/flexr/automaton/accel.rb
96
118
  - lib/flexr/automaton/analysis.rb
119
+ - lib/flexr/automaton/backend_cost_model.rb
97
120
  - lib/flexr/automaton/byte_class_set.rb
98
121
  - lib/flexr/automaton/compiler.rb
99
122
  - lib/flexr/automaton/dfa.rb
100
123
  - lib/flexr/automaton/minimizer.rb
101
124
  - lib/flexr/automaton/nfa.rb
125
+ - lib/flexr/automaton/types.rb
102
126
  - lib/flexr/cli.rb
127
+ - lib/flexr/codegen.rb
103
128
  - lib/flexr/codegen/base.rb
104
129
  - lib/flexr/codegen/direct.rb
105
130
  - lib/flexr/codegen/firstmatch.rb
106
131
  - lib/flexr/codegen/table.rb
107
132
  - lib/flexr/codegen/table_packer.rb
133
+ - lib/flexr/configuration.rb
108
134
  - lib/flexr/diagnostics.rb
109
135
  - lib/flexr/dsl.rb
110
136
  - lib/flexr/errors.rb
111
137
  - lib/flexr/generated.rb
112
138
  - lib/flexr/generator.rb
139
+ - lib/flexr/grammar.rb
113
140
  - lib/flexr/importer.rb
114
141
  - lib/flexr/ir.rb
115
142
  - lib/flexr/lexer.rb
143
+ - lib/flexr/migration.rb
144
+ - lib/flexr/minitest.rb
116
145
  - lib/flexr/options.rb
117
146
  - lib/flexr/rake_task.rb
118
147
  - lib/flexr/regexp/ast.rb
119
148
  - lib/flexr/regexp/char_class.rb
149
+ - lib/flexr/regexp/matcher.rb
120
150
  - lib/flexr/regexp/normalizer.rb
121
151
  - lib/flexr/regexp/parser.rb
122
152
  - lib/flexr/regexp/tokenizer.flexr.rb
123
153
  - lib/flexr/regexp/tokenizer.rb
124
154
  - lib/flexr/regexp/unsupported.rb
155
+ - lib/flexr/rspec.rb
156
+ - lib/flexr/runtime.rb
125
157
  - lib/flexr/runtime/buffer.rb
126
158
  - lib/flexr/runtime/core.rb
127
159
  - lib/flexr/runtime/errors.rb
128
160
  - lib/flexr/runtime/interpreter.rb
129
161
  - lib/flexr/runtime/location.rb
162
+ - lib/flexr/runtime/record.rb
130
163
  - lib/flexr/runtime/token.rb
164
+ - lib/flexr/source.rb
131
165
  - lib/flexr/source/passthrough.rb
132
166
  - lib/flexr/source/prism_reader.rb
133
167
  - lib/flexr/source/static_eval.rb
168
+ - lib/flexr/testing.rb
169
+ - lib/flexr/trace.rb
134
170
  - lib/flexr/unicode/case_fold.rb
171
+ - lib/flexr/unicode/data.rb
135
172
  - lib/flexr/unicode/data/LICENSE-UNICODE.txt
136
173
  - lib/flexr/unicode/data/UNICODE_VERSION
137
174
  - lib/flexr/unicode/data/case_folding.rb
@@ -139,13 +176,18 @@ files:
139
176
  - lib/flexr/unicode/property.rb
140
177
  - lib/flexr/unicode/reference_regexp.rb
141
178
  - lib/flexr/unicode/utf8_splitter.rb
179
+ - lib/flexr/unicode/version.rb
142
180
  - lib/flexr/version.rb
181
+ - lib/ruby_lsp/flexr/addon.rb
182
+ - sig/flexr.rbs
143
183
  - site/README.md
144
184
  - site/astro.config.mjs
145
185
  - site/package.json
186
+ - site/playwright.config.mjs
146
187
  - site/pnpm-lock.yaml
147
188
  - site/pnpm-workspace.yaml
148
- - site/public/playground.js
189
+ - site/ruby/playground.rb
190
+ - site/scripts/prepare-playground.mjs
149
191
  - site/scripts/verify-site.mjs
150
192
  - site/src/content.config.ts
151
193
  - site/src/content/docs/benchmarks.md
@@ -170,18 +212,30 @@ files:
170
212
  - site/src/layouts/SiteLayout.astro
171
213
  - site/src/pages/index.astro
172
214
  - site/src/pages/playground.astro
215
+ - site/src/playground/client.js
216
+ - site/src/playground/graph-worker.mjs
217
+ - site/src/playground/presets.mjs
218
+ - site/src/playground/runtime.mjs
219
+ - site/src/playground/share.mjs
220
+ - site/src/playground/worker.mjs
173
221
  - site/src/styles/custom.css
222
+ - site/src/styles/landing.css
223
+ - site/tests/browser/playground.spec.mjs
224
+ - site/tests/playground-runtime.test.mjs
174
225
  - site/tsconfig.json
226
+ - test/ruby_lsp_addon_spec.rb
175
227
  - tools/coverage.rb
176
228
  - tools/docs_verify.rb
177
229
  - tools/gen_unicode_tables.rb
230
+ - tools/golden.rb
178
231
  - tools/regexp_tokenizer_reference.rb
232
+ - tools/verification_failure.rb
179
233
  homepage: https://github.com/ydah/flexr
180
234
  licenses:
181
235
  - MIT
182
236
  metadata:
183
237
  rubygems_mfa_required: 'true'
184
- generator_ruby_version: ">= 3.3 (Prism)"
238
+ generator_ruby_version: ">= 3.4 (Prism)"
185
239
  homepage_uri: https://github.com/ydah/flexr
186
240
  source_code_uri: https://github.com/ydah/flexr/tree/main
187
241
  bug_tracker_uri: https://github.com/ydah/flexr/issues
@@ -192,14 +246,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
192
246
  requirements:
193
247
  - - ">="
194
248
  - !ruby/object:Gem::Version
195
- version: 3.1.0
249
+ version: 3.4.0
196
250
  required_rubygems_version: !ruby/object:Gem::Requirement
197
251
  requirements:
198
252
  - - ">="
199
253
  - !ruby/object:Gem::Version
200
254
  version: '0'
201
255
  requirements: []
202
- rubygems_version: 4.0.6
256
+ rubygems_version: 4.0.20
203
257
  specification_version: 4
204
258
  summary: Ruby-native lexer generator with runtime and generated modes
205
259
  test_files: []
@@ -1 +0,0 @@
1
- 6359976d9cdc72772eff8c72cf421d965e913febad3dbc0ca0f2057a0faea3b4
@@ -1 +0,0 @@
1
- 913cf85ba7e1a3128eea0a4b06077486fead2d56357b669685ae6e26bdd55715
@@ -1 +0,0 @@
1
- 7f20a1024eea1d674aee73a2baf7a32c52fa178f2d158d24e49f9c9b77a63960
@@ -1 +0,0 @@
1
- 0d99b92c658d56cbef7d9c33ac3e5770d83e379b94cf412f8b028f59d44910b8
@@ -1 +0,0 @@
1
- 16943c2d8a86999148dd6bcd2ed644671032d1e6b4acbe891d180ff0810cd97a
@@ -1 +0,0 @@
1
- 7627b0fafbfdbdea850d02e67c3e9c24eab8512cdc74571a986323d397848071
@@ -1 +0,0 @@
1
- 037fa90c634117522c1e882d08c437c7b79653f59ca94e6cf6c115cd7897610f