flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "stringio"
4
+
5
+ module FlexrPlayground
6
+ module_function
7
+
8
+ def run(request)
9
+ path = "/playground.flexr.rb"
10
+ source = request.fetch("source")
11
+ File.write(path, source)
12
+ stdout = $stdout
13
+ stderr = $stderr
14
+ $stdout = StringIO.new
15
+ $stderr = StringIO.new
16
+ options = Flexr::Options.default
17
+ options.set(:max_dfa_states, 1024)
18
+ options.set(:warn_level, :all)
19
+ options.color = :never
20
+ diagnostics = StringIO.new
21
+ Flexr::CLI.check(path, options, diagnostics)
22
+ generated = Flexr::Generator.new(path, options: options.generator_options).generate
23
+ parsed = Flexr::CLI.read_spec(path)
24
+ scope = Module.new
25
+ scope.module_eval(request["mode"] == "generated" ? generated : source, path, 1)
26
+ namespace = parsed.class_name.start_with?("::") ? Object : scope
27
+ klass = parsed.class_name.split("::").reject(&:empty?).inject(namespace) { |parent, name| parent.const_get(name, false) }
28
+ unless request["mode"] == "generated"
29
+ klass.__flexr_config.options[:max_dfa_states] = 1024
30
+ klass.__flexr_config.options[:warn_level] = :all
31
+ end
32
+ input_path = "/input.txt"
33
+ File.write(input_path, request.fetch("input"))
34
+ lexer = klass.new(request.fetch("input"), filename: input_path, max_steps: 1_000_000, max_token_size: 65_536,
35
+ max_lookahead_size: 65_536, max_buffer_size: 1_048_576)
36
+ trace = Flexr::Trace.new(lexer)
37
+ tokens = trace.run
38
+ dot = StringIO.new
39
+ Flexr::CLI.print_dot(path, options, dot)
40
+ JSON.generate(tokens: tokens, steps: trace.steps, diagnostics: diagnostics.string,
41
+ generated: generated, dot: dot.string, ruby_version: RUBY_VERSION,
42
+ prism_version: Prism::VERSION,
43
+ console: ($stdout.string + $stderr.string).byteslice(0, 4096).force_encoding(Encoding::UTF_8).scrub)
44
+ rescue StandardError, SyntaxError, LoadError, SystemExit => e
45
+ if e.is_a?(Flexr::LexError) && !e.diagnostic && e.byte_pos
46
+ diagnostic = Flexr::Diagnostics.error("FLEXR-E000", e.message,
47
+ location: [e.byte_pos, e.byte_pos + e.text.to_s.bytesize])
48
+ e = Flexr::LexError.new(e.message, diagnostic: diagnostic,
49
+ filename: e.filename, byte_pos: e.byte_pos, line: e.line, text: e.text)
50
+ end
51
+ JSON.generate(error: Flexr::CLI.render_error(e, options: options, path: path),
52
+ ruby_version: RUBY_VERSION, prism_version: defined?(Prism::VERSION) && Prism::VERSION)
53
+ ensure
54
+ $stdout = stdout
55
+ $stderr = stderr
56
+ end
57
+ end
@@ -0,0 +1,35 @@
1
+ import { cp, mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
2
+ import { fileURLToPath } from 'node:url';
3
+ import path from 'node:path';
4
+
5
+ const site = fileURLToPath(new URL('..', import.meta.url));
6
+ const root = path.dirname(site);
7
+ const output = path.join(site, 'public/playground-runtime');
8
+ const files = {};
9
+ async function collect(directory) {
10
+ for (const entry of await readdir(path.join(root, directory), { withFileTypes: true })) {
11
+ const relative = path.posix.join(directory, entry.name);
12
+ if (entry.isDirectory()) await collect(relative);
13
+ else if (entry.name.endsWith('.rb')) files[`/app/${relative}`] = await readFile(path.join(root, relative), 'utf8');
14
+ }
15
+ }
16
+ await collect('lib');
17
+ files['/app/playground.rb'] = await readFile(path.join(site, 'ruby/playground.rb'), 'utf8');
18
+ await mkdir(output, { recursive: true });
19
+ const payload = JSON.stringify({ files });
20
+ await writeFile(path.join(output, 'flexr.json'), payload);
21
+ const inputs = {
22
+ toml: 'title = "flexr"',
23
+ graphql: 'query { user(id: 12) { name } }',
24
+ sql: 'select name from users where id >= 12;',
25
+ };
26
+ const presets = {};
27
+ for (const [name, input] of Object.entries(inputs)) {
28
+ presets[name] = { source: await readFile(path.join(root, `examples/${name}/lexer.flexr.rb`), 'utf8'), input };
29
+ }
30
+ await writeFile(path.join(output, 'presets.json'), JSON.stringify(presets));
31
+ const wasm = path.join(site, 'node_modules/@ruby/4.0-wasm-wasi/dist');
32
+ await cp(path.join(wasm, 'ruby+stdlib.wasm'), path.join(output, 'ruby.wasm'));
33
+ await cp(path.join(wasm, 'LICENSE'), path.join(output, 'LICENSE'));
34
+ await cp(path.join(wasm, 'NOTICE'), path.join(output, 'NOTICE'));
35
+ console.log(`Playground: packaged ${Object.keys(files).length} Ruby files (${Buffer.byteLength(payload)} bytes).`);
@@ -10,7 +10,9 @@ const required = [
10
10
  'src/content/docs/learn/generation.md',
11
11
  'src/content/docs/reference/dsl.md',
12
12
  'src/content/docs/reference/diagnostics.md',
13
- 'public/playground.js'
13
+ 'src/playground/client.js',
14
+ 'src/playground/worker.mjs',
15
+ 'ruby/playground.rb'
14
16
  ];
15
17
 
16
18
  const missing = required.filter((file) => !existsSync(resolve(root, file)));
@@ -30,8 +32,8 @@ const requiredPhrases = [
30
32
  'longest match',
31
33
  'Runtime mode',
32
34
  'Generated mode',
33
- 'fixture-backed',
34
- 'does not execute arbitrary Ruby'
35
+ 'separate worker',
36
+ 'source and input stay in your browser'
35
37
  ];
36
38
  const missingPhrases = requiredPhrases.filter((phrase) => !source.toLowerCase().includes(phrase.toLowerCase()));
37
39
  if (missingPhrases.length) {
@@ -3,7 +3,7 @@ title: Matching semantics
3
3
  description: Understand how flexr chooses a rule when several rules accept the same position.
4
4
  ---
5
5
 
6
- At each input position, flexr selects the rule that consumes the most characters. If multiple rules consume the same number, the rule defined first wins.
6
+ With the default longest-match backends, flexr selects the rule that consumes the most bytes at each input position. If multiple rules consume the same number, the rule defined first wins.
7
7
 
8
8
  ```ruby
9
9
  rule(/==/) { emit :EQ }
@@ -12,4 +12,6 @@ rule(/=/) { emit :ASSIGN }
12
12
 
13
13
  Input `==` produces `EQ`, regardless of the order above. Equal-length alternatives are resolved by definition order, so putting keyword rules before a broad identifier rule is the conventional way to reserve keywords.
14
14
 
15
- The [playground](/flexr/playground/) shows candidates and the winner for each scan step. The `firstmatch` backend is experimental because it changes this selection model.
15
+ The stable `firstmatch` backend instead selects the first rule with a match at the current position. Ruby regexp alternation order applies within that rule, so `/a|ab/` matches `a` on input `ab`. Select this behavior explicitly with `backend :firstmatch`.
16
+
17
+ The [playground](/flexr/playground/) shows candidates and the winner for each scan step.
@@ -12,4 +12,12 @@ flexr specifications are Ruby programs. Rule actions, `--eval`, and generated ac
12
12
  - Use `flexr check` for diagnostics, not as a sandbox.
13
13
  - Keep build-time inputs and generator versions pinned in CI.
14
14
 
15
- The hosted playground currently uses fixed fixtures and does not execute arbitrary Ruby. A future WASM worker must keep that boundary explicit, enforce resource limits, and never imply that parsing source is equivalent to sandboxing actions.
15
+ The Playground executes real Ruby in a separate worker using a virtual filesystem.
16
+ The application does not upload source or input; share links store them in a URL
17
+ fragment and do not run automatically. Review shared source before selecting Run.
18
+
19
+ Runs have a five-second timeout and can be stopped immediately. Input size,
20
+ automaton construction, and output are bounded. The page restricts network access
21
+ and sanitizes output, but a worker is not a complete hostile-code or memory
22
+ sandbox. Ruby actions can modify their VM and access worker JavaScript. Parsing a
23
+ specification does not make its actions safe.
@@ -10,7 +10,7 @@ bundle exec flexr check lexer.flexr.rb
10
10
  bundle exec flexr generate lexer.flexr.rb -o lexer.rb
11
11
  ```
12
12
 
13
- The generator can use `table`, `direct`, or `auto` backends. `firstmatch` is experimental and requires an explicit option because it does not preserve the usual longest-match contract.
13
+ The generator supports the longest-match `table`, `direct`, and `auto` backends. Select `backend :firstmatch` explicitly to use the first matching rule in definition order. This backend is stable and requires no experimental option.
14
14
 
15
15
  ## Runtime dependencies
16
16
 
@@ -10,10 +10,10 @@ description: The stable lexer specification methods and their role.
10
10
  | `emits(*kinds)` | Declare token kinds for diagnostics and parser integration |
11
11
  | `state(name)` | Define a named start condition |
12
12
  | `all_states` | Apply a rule to every state |
13
- | `backend(name)` | Choose `table`, `direct`, or `auto`; `firstmatch` is experimental |
13
+ | `backend(name)` | Choose longest-match `table`, `direct`, or `auto`, or rule-order `firstmatch` |
14
14
  | `token_kind(name)` | Choose the token return shape |
15
15
  | `encoding(name)` | Select the input encoding contract |
16
16
  | `option(name)` | Enable an explicit compiler option |
17
17
  | `accel(name)` | Configure acceleration where compatible |
18
18
 
19
- Rules are evaluated using [longest-match semantics](/flexr/concepts/matching-semantics/). The repository's [DSL reference](https://github.com/ydah/flexr/blob/main/docs/reference/dsl.md) includes option defaults and examples.
19
+ The default backends use [longest-match semantics](/flexr/concepts/matching-semantics/); `firstmatch` selects the first matching rule. The repository's [DSL reference](https://github.com/ydah/flexr/blob/main/docs/reference/dsl.md) includes option defaults and examples.
@@ -5,11 +5,13 @@ description: Separate stable user APIs from experimental and internal implementa
5
5
 
6
6
  ## Public and stable
7
7
 
8
- `Flexr::Lexer`, the lexer DSL, runtime token/location structures, the documented CLI, and `Flexr::RakeTask` are the supported entry points for normal use.
8
+ `Flexr::Lexer`, the lexer DSL, immutable runtime token/location Data records, the documented CLI, and `Flexr::RakeTask` are the supported entry points for normal use.
9
+
10
+ The `firstmatch` backend is stable and uses rule-order semantics in runtime and generated modes. Select it explicitly with `backend :firstmatch`; no experimental option is required.
9
11
 
10
12
  ## Experimental
11
13
 
12
- The `firstmatch` backend and any option explicitly marked experimental may change behavior or API shape between releases. Opt in deliberately and test the resulting token stream.
14
+ Options explicitly marked experimental may change behavior or API shape between releases.
13
15
 
14
16
  ## Internal
15
17
 
@@ -13,4 +13,7 @@ description: Return shapes, semantic values, and source positions.
13
13
 
14
14
  Locations use byte offsets for the input position and track line/column as the lexer consumes text. Keep the location with the token when a parser needs useful syntax errors.
15
15
 
16
+ In v2, structured tokens and locations are immutable `Data` records. Read fields
17
+ with attribute readers or `[]`, and use `with` to create a replacement.
18
+
16
19
  For exact defaults and the `Location` data structure, use the repository's [tokens and locations reference](https://github.com/ydah/flexr/blob/main/docs/reference/tokens-and-locations.md).
@@ -4,9 +4,10 @@ import '../styles/custom.css';
4
4
  export interface Props {
5
5
  title: string;
6
6
  description: string;
7
+ playground?: boolean;
7
8
  }
8
9
 
9
- const { title, description } = Astro.props;
10
+ const { title, description, playground = false } = Astro.props;
10
11
  const siteTitle = `${title} · flexr`;
11
12
  const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
12
13
  ---
@@ -16,6 +17,14 @@ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL :
16
17
  <meta charset="UTF-8" />
17
18
  <meta name="viewport" content="width=device-width" />
18
19
  <meta name="description" content={description} />
20
+ <meta name="theme-color" content="#161518" />
21
+ {playground && <meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' blob: 'unsafe-eval' 'wasm-unsafe-eval'; worker-src 'self' blob:; connect-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; img-src 'self' data:; object-src 'none'; base-uri 'self'" />}
22
+ <meta property="og:title" content={siteTitle} />
23
+ <meta property="og:description" content={description} />
24
+ <meta property="og:type" content="website" />
25
+ <link rel="preconnect" href="https://fonts.googleapis.com" />
26
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
27
+ <link href="https://fonts.googleapis.com/css2?family=Newsreader:ital,opsz,wght@0,6..72,400..700;1,6..72,400..600&amp;family=Public+Sans:wght@400;500;600;700&amp;display=swap" rel="stylesheet" />
19
28
  <title>{siteTitle}</title>
20
29
  </head>
21
30
  <body>
@@ -25,11 +34,10 @@ const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL :
25
34
  <a class="site-nav-brand" href={base} aria-label="flexr home">flexr<span>/</span></a>
26
35
  <nav class="site-nav-links" aria-label="Primary navigation">
27
36
  <a href={`${base}playground/`}>Playground</a>
28
- <a href={`${base}learn/getting-started/`}>Learn</a>
29
37
  <a href={`${base}reference/dsl/`}>Reference</a>
30
38
  <a href="https://github.com/ydah/flexr">GitHub</a>
31
39
  </nav>
32
- <a class="site-nav-cta" href={`${base}learn/getting-started/`}>Get started <span aria-hidden="true">↗</span></a>
40
+ <a class="site-nav-cta" href={`${base}learn/getting-started/`}>Get started</a>
33
41
  </div>
34
42
  </header>
35
43
  <div id="main-content">
@@ -1,174 +1,88 @@
1
1
  ---
2
2
  import SiteLayout from '../layouts/SiteLayout.astro';
3
+ import '../styles/landing.css';
3
4
 
4
5
  const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
5
6
  const repo = 'https://github.com/ydah/flexr';
6
7
  ---
7
8
  <SiteLayout
8
- title="Ruby-native lexer tooling"
9
- description="flexr lets parser authors write lexer specifications in ordinary Ruby, inspect matching decisions, and generate deterministic Ruby for deployment."
9
+ title="A lexer generator that speaks Ruby"
10
+ description="Define lexer rules in Ruby, understand every match, and generate deterministic Ruby with flexr."
10
11
  >
11
- <div class="home-shell">
12
- <header class="home-hero">
13
- <div>
14
- <div class="eyebrow">Ruby lexer tooling for parser authors</div>
15
- <h1>Write lexers in Ruby. See why each token wins.</h1>
16
- <p class="hero-copy">
17
- flexr lets you write a lexer specification as ordinary Ruby, run it while developing,
18
- inspect its matching behavior, and generate deterministic Ruby for deployment.
12
+ <main class="landing">
13
+ <section class="hero" aria-labelledby="hero-title">
14
+ <div class="hero-heading">
15
+ <p>Lexer generation, in Ruby</p>
16
+ <h1 id="hero-title">Write the rules.<br /><em>Know what wins.</em></h1>
17
+ </div>
18
+ <div class="hero-intro">
19
+ <p>
20
+ flexr keeps lexer specifications in the language they serve. Define readable rules,
21
+ inspect every decision, then generate deterministic Ruby for deployment.
19
22
  </p>
20
23
  <div class="hero-actions">
21
- <a class="button button-primary" href={`${base}playground/`}>Open the playground</a>
22
- <a class="button button-secondary" href={`${base}learn/getting-started/`}>Get started</a>
24
+ <a class="action action-primary" href={`${base}learn/getting-started/`}>Build your first lexer</a>
25
+ <a class="action action-link" href={`${base}playground/`}>Open the playground</a>
23
26
  </div>
24
27
  </div>
28
+ </section>
25
29
 
26
- <div class="demo-window" aria-label="Longest-match example">
27
- <div class="window-bar">
28
- <span class="window-dot"></span>
29
- <span class="window-dot"></span>
30
- <span class="window-dot"></span>
31
- <span>lexer.flexr.rb</span>
32
- </div>
33
- <pre class="code-block"><code><span class="code-keyword">class</span> Lexer &lt; Flexr::Lexer
34
- rule(<span class="code-string">/==/</span>) &#123; emit <span class="code-string">:EQ</span> &#125;
35
- rule(<span class="code-string">/=/</span>) &#123; emit <span class="code-string">:ASSIGN</span> &#125;
36
- rule(<span class="code-string">/if/</span>) &#123; emit <span class="code-string">:IF</span> &#125;
37
- rule(<span class="code-string">/[a-z_][a-z0-9_]*/</span>) &#123; emit <span class="code-string">:IDENT</span> &#125;
38
- <span class="code-keyword">end</span></code></pre>
39
- <div class="decision-panel">
40
- <div class="decision-label"><span>input: <strong>if == total</strong></span><span>step 02</span></div>
41
- <div class="decision-row winner"><strong>/==/ → EQ</strong><span>longest match · 2 chars</span></div>
42
- <div class="decision-row"><strong>/=/ → ASSIGN</strong><span>shorter match · 1 char</span></div>
43
- </div>
30
+ <section class="specimen" aria-labelledby="specimen-title">
31
+ <header>
32
+ <p>One input, four decisions</p>
33
+ <h2 id="specimen-title">A match you can read.</h2>
34
+ </header>
35
+ <div class="source-line">
36
+ <span>source</span>
37
+ <code>if total == 42</code>
44
38
  </div>
45
- </header>
39
+ <ol class="token-line" aria-label="Generated token stream">
40
+ <li><span>if</span><strong>IF</strong></li>
41
+ <li><span>total</span><strong>IDENT</strong></li>
42
+ <li class="token-accent"><span>==</span><strong>EQ</strong></li>
43
+ <li><span>42</span><strong>INTEGER</strong></li>
44
+ </ol>
45
+ <p class="decision"><code>/==/</code> wins over <code>/=/</code> because the longest match wins.</p>
46
+ </section>
46
47
 
47
- <main>
48
- <section class="home-section" aria-labelledby="why-heading">
49
- <div class="section-heading">
50
- <div>
51
- <div class="eyebrow">Why flexr</div>
52
- <h2>One specification, two useful ways to run it.</h2>
53
- </div>
54
- <p>
55
- Keep the feedback loop in Ruby. When the lexer is ready to ship, compile the same
56
- rules into a predictable artifact and keep the behavior easy to reason about.
57
- </p>
58
- </div>
59
- <div class="card-grid">
60
- <article class="site-card">
61
- <div class="card-index">01 / RUBY-NATIVE</div>
62
- <h3>Ordinary Ruby specifications</h3>
63
- <p>Use classes, constants, interpolation, and familiar Ruby actions without learning a separate lexer language.</p>
64
- </article>
65
- <article class="site-card">
66
- <div class="card-index">02 / DECISIONS</div>
67
- <h3>Matching you can explain</h3>
68
- <p>Longest match wins. If matches have the same length, the rule defined first wins. The contract stays visible.</p>
69
- </article>
70
- <article class="site-card">
71
- <div class="card-index">03 / PARITY</div>
72
- <h3>Runtime and generated parity</h3>
73
- <p>Develop with the runtime interpreter, then compare its token stream with generated Ruby before deployment.</p>
74
- </article>
75
- <article class="site-card">
76
- <div class="card-index">04 / DIAGNOSTICS</div>
77
- <h3>Actionable checks</h3>
78
- <p>Find unsupported regexp constructs, unreachable rules, undeclared tokens, and other problems before they reach a parser.</p>
79
- </article>
80
- </div>
81
- </section>
82
-
83
- <section class="home-section" aria-labelledby="workflow-heading">
84
- <div class="section-heading">
85
- <div>
86
- <div class="eyebrow">A practical workflow</div>
87
- <h2>Write. Inspect. Generate. Integrate.</h2>
88
- </div>
89
- </div>
90
- <div class="workflow-grid">
91
- <article class="workflow-step">
92
- <div class="card-index">01</div>
93
- <h3>Write</h3>
94
- <p>Define rules and actions in a Ruby class that looks like the code around it.</p>
95
- </article>
96
- <article class="workflow-step">
97
- <div class="card-index">02</div>
98
- <h3>Inspect</h3>
99
- <p>Check the specification and make a matching decision visible when rules compete.</p>
100
- </article>
101
- <article class="workflow-step">
102
- <div class="card-index">03</div>
103
- <h3>Generate</h3>
104
- <p>Produce deterministic Ruby for environments where startup and deployment shape matter.</p>
105
- </article>
106
- <article class="workflow-step">
107
- <div class="card-index">04</div>
108
- <h3>Integrate</h3>
109
- <p>Return tokens to Racc, Lrama, or your own parser protocol with explicit locations.</p>
110
- </article>
111
- </div>
112
- </section>
113
-
114
- <section class="home-section" aria-labelledby="modes-heading">
115
- <div class="section-heading">
116
- <div>
117
- <div class="eyebrow">Choose your mode</div>
118
- <h2>Fast feedback first. Reproducible output when it matters.</h2>
119
- </div>
120
- <p>Both modes consume the same lexer design. The choice is about the deployment boundary, not a second language.</p>
121
- </div>
122
- <div class="compare-panel" id="modes-heading">
123
- <div>
124
- <h3>Runtime mode</h3>
125
- <p>Load the specification directly with the <code>flexr</code> runtime. It is a strong fit for iteration, tests, and applications that value a compact source of truth.</p>
126
- <div class="home-links"><a class="button button-secondary" href={`${base}learn/runtime-mode/`}>Runtime guide</a></div>
127
- </div>
128
- <div>
129
- <h3>Generated mode</h3>
130
- <p>Compile the specification to Ruby and commit or build the generated artifact. Use standalone output when the deployed process should not load the gem at runtime.</p>
131
- <div class="home-links"><a class="button button-secondary" href={`${base}learn/generation/`}>Generation guide</a></div>
132
- </div>
133
- </div>
134
- </section>
48
+ <section class="principles" aria-labelledby="principles-title">
49
+ <header>
50
+ <p>Why flexr</p>
51
+ <h2 id="principles-title">One specification, from first test to release.</h2>
52
+ </header>
53
+ <div class="principle-list">
54
+ <article>
55
+ <h3>Ruby-native</h3>
56
+ <p>Use familiar classes, constants, and actions. There is no separate lexer language to learn.</p>
57
+ </article>
58
+ <article>
59
+ <h3>Explainable</h3>
60
+ <p>Longest match wins. Source order resolves ties. Unsupported patterns surface early.</p>
61
+ </article>
62
+ <article>
63
+ <h3>Ready both ways</h3>
64
+ <p><a href={`${base}learn/runtime-mode/`}>Runtime mode</a> keeps iteration quick. <a href={`${base}learn/generation/`}>Generated mode</a> keeps deployment reproducible.</p>
65
+ </article>
66
+ </div>
67
+ </section>
135
68
 
136
- <section class="home-section" aria-labelledby="trust-heading">
137
- <div class="section-heading">
138
- <div>
139
- <div class="eyebrow">Make the boundary explicit</div>
140
- <h2>Useful power, clear constraints.</h2>
141
- </div>
142
- <p>flexr is designed to be inspectable. Know what runs at build time, what remains Ruby code, and what the regexp compiler accepts.</p>
143
- </div>
144
- <div class="trust-grid">
145
- <article class="trust-panel">
146
- <div class="card-index">SECURITY</div>
147
- <h3>Actions are Ruby code</h3>
148
- <p>Only run specifications and generated artifacts you trust. Static checks do not make arbitrary actions safe.</p>
149
- </article>
150
- <article class="trust-panel">
151
- <div class="card-index">COMPATIBILITY</div>
152
- <h3>Regexp support is documented</h3>
153
- <p>Check the compatibility matrix before porting lookarounds, backreferences, or open-ended repetition.</p>
154
- </article>
155
- <article class="trust-panel">
156
- <div class="card-index">EXPERIMENTAL</div>
157
- <h3>Experimental features say so</h3>
158
- <p><code>firstmatch</code> is experimental because it changes the usual longest-match contract and needs an explicit option.</p>
159
- </article>
160
- <article class="trust-panel">
161
- <div class="card-index">OPEN SOURCE</div>
162
- <h3>Read the implementation</h3>
163
- <p>Inspect examples, diagnostics, generated output, and the benchmark log in the repository.</p>
164
- <div class="home-links"><a class="button button-secondary" href={repo}>View on GitHub</a></div>
165
- </article>
166
- </div>
167
- </section>
168
- </main>
69
+ <section class="start" aria-labelledby="start-title">
70
+ <div>
71
+ <p>Ready when you are</p>
72
+ <h2 id="start-title">Start with one Ruby file.</h2>
73
+ </div>
74
+ <code>gem install flexr</code>
75
+ <a class="action action-primary" href={`${base}learn/getting-started/`}>Read the quick start</a>
76
+ </section>
169
77
 
170
- <footer class="site-footer">
171
- <span class="site-mark">flexr</span> · Ruby-native lexer tooling · <a href={`${base}examples/`}>Examples</a> · <a href={`${base}reference/cli/`}>CLI reference</a> · <a href={repo}>Source</a>
78
+ <footer class="landing-footer">
79
+ <a class="footer-brand" href={base}>flexr/</a>
80
+ <nav aria-label="Footer navigation">
81
+ <a href={`${base}reference/dsl/`}>Reference</a>
82
+ <a href={`${base}examples/`}>Examples</a>
83
+ <a href={repo}>GitHub</a>
84
+ </nav>
85
+ <p>MIT licensed.</p>
172
86
  </footer>
173
- </div>
87
+ </main>
174
88
  </SiteLayout>
@@ -3,62 +3,64 @@ import SiteLayout from '../layouts/SiteLayout.astro';
3
3
 
4
4
  const base = import.meta.env.BASE_URL.endsWith('/') ? import.meta.env.BASE_URL : `${import.meta.env.BASE_URL}/`;
5
5
  ---
6
- <SiteLayout
7
- title="Playground"
8
- description="Explore flexr's longest-match decision model with an interactive, fixture-backed lexer preview."
9
- >
6
+ <SiteLayout title="Playground" description="Run real flexr Ruby specifications locally, inspect diagnostics and trace decisions, and generate a scanner." playground>
10
7
  <main class="playground-shell">
11
- <a class="back-link" href={base}>← Back to flexr</a>
8
+ <a class="back-link" href={base}>Back to flexr</a>
12
9
  <header class="playground-header">
13
- <div class="eyebrow">Interactive preview · flexr 1.0</div>
14
- <h1>See the decision, not just the token.</h1>
15
- <p>
16
- Choose a fixture, edit its input, and inspect which rule wins at each position. This
17
- preview is static and fixture-backed: it does not execute arbitrary Ruby in your browser.
18
- Use the repository examples and CLI for full runtime and generation behavior.
19
- </p>
10
+ <h1>Follow every match.</h1>
11
+ <p>Edit Ruby, run the real lexer, and inspect its decisions. Source and input stay in your browser.</p>
20
12
  <div class="playground-notice" role="note">
21
- Safe boundary: this page sends no source or input to a server. Arbitrary actions and
22
- <code>--eval</code> are intentionally unavailable here.
13
+ Run executes Ruby and its actions in a separate worker. Review shared source before running it.
14
+ Runs stop after five seconds; Stop cancels the worker immediately. Ruby loads only on your first run.
23
15
  </div>
24
16
  </header>
25
-
26
17
  <section class="workbench" aria-label="flexr playground workbench">
27
18
  <div class="workbench-panel">
28
- <div class="panel-heading"><span>Specification fixture</span><small>read-only model</small></div>
29
- <select id="preset" aria-label="Choose a playground fixture">
30
- <option value="longest">Longest match</option>
31
- <option value="calculator">Calculator</option>
32
- <option value="json">JSON tokens</option>
19
+ <div class="panel-heading"><label for="spec">Ruby specification</label><small>editable</small></div>
20
+ <select id="preset" aria-label="Choose a playground preset">
21
+ <option value="longest">Longest match</option><option value="calculator">Calculator</option><option value="json">JSON tokens</option>
22
+ <option value="toml">TOML subset</option><option value="graphql">GraphQL subset</option><option value="sql">SQL subset</option>
33
23
  </select>
34
- <textarea id="spec" aria-label="Lexer specification" readonly></textarea>
35
- <div class="workbench-actions">
36
- <button class="button button-secondary" type="button" data-action="check">Check</button>
37
- <button class="button button-secondary" type="button" data-action="generate">Show generated shape</button>
38
- </div>
24
+ <textarea id="spec" aria-label="Lexer specification" spellcheck="false" maxlength="20000"></textarea>
39
25
  </div>
40
-
41
26
  <div class="workbench-panel">
42
- <div class="panel-heading"><span>Input</span><small>edit, then run</small></div>
43
- <textarea id="input" class="input-area" aria-label="Input to tokenize"></textarea>
27
+ <div class="panel-heading"><label for="input">Input</label><small>up to 20,000 characters</small></div>
28
+ <textarea id="input" class="input-area" aria-label="Input to tokenize" spellcheck="false" maxlength="20000"></textarea>
29
+ <select id="mode" aria-label="Execution mode"><option value="runtime">Runtime mode</option><option value="generated">Generated mode</option></select>
44
30
  <div class="workbench-actions">
45
- <button class="button button-primary" type="button" data-action="run">Run preview</button>
46
- <button class="button button-secondary" type="button" data-action="compare">Compare modes</button>
31
+ <button class="button button-primary" type="button" data-action="run">Run</button>
32
+ <button class="button button-secondary" type="button" data-action="stop" disabled>Stop</button>
47
33
  <button class="button button-secondary" type="button" data-action="share">Copy share link</button>
48
34
  </div>
49
- <p id="status" class="workbench-status" role="status" aria-live="polite"></p>
35
+ <input id="share-link" class="share-link" aria-label="Share link" readonly hidden />
36
+ <p id="status" class="workbench-status" role="status" aria-live="polite">Choose a preset or write a specification, then select Run.</p>
37
+ <p id="metrics" class="workbench-status"></p>
50
38
  </div>
51
-
52
39
  <div class="workbench-panel full">
53
- <div class="panel-heading"><span>Tokens</span><small>output</small></div>
54
- <div id="tokens" class="workbench-output"><p class="output-empty">Run a fixture to see its token stream.</p></div>
40
+ <div class="panel-heading"><span>Tokens</span></div>
41
+ <pre id="tokens" class="workbench-output">Run a specification to see its token stream.</pre>
42
+ </div>
43
+ <div class="workbench-panel full">
44
+ <div class="panel-heading"><span>Diagnostics</span></div>
45
+ <pre id="diagnostics" class="workbench-output">Compiler warnings and source locations appear here.</pre>
46
+ </div>
47
+ <div class="workbench-panel full">
48
+ <div class="panel-heading"><span>Match decisions</span><span id="step-position">No scan steps.</span></div>
49
+ <div class="workbench-actions">
50
+ <button class="button button-secondary" type="button" data-action="previous">Previous step</button>
51
+ <input id="step" type="range" min="0" max="0" value="0" aria-label="Trace step" disabled />
52
+ <button class="button button-secondary" type="button" data-action="next">Next step</button>
53
+ </div>
54
+ <pre id="decisions" class="workbench-output">Run a specification to inspect its decisions.</pre>
55
55
  </div>
56
-
57
56
  <div class="workbench-panel full">
58
- <div class="panel-heading"><span>Match decisions</span><small>why the winner won</small></div>
59
- <div id="decisions" class="workbench-output"><p class="output-empty">Each input position will show its accepted candidates and the winning rule.</p></div>
57
+ <div class="panel-heading"><span>DFA</span><button type="button" data-action="draw">Draw DFA</button></div>
58
+ <div id="graph" class="workbench-output graph-output">Run a specification to inspect its DFA.</div>
59
+ <details><summary>DOT output</summary><pre id="dot" class="workbench-output"></pre></details>
60
60
  </div>
61
+ <details class="workbench-panel full"><summary>Generated Ruby</summary><pre id="generated" class="workbench-output">The real generated scanner appears after a run.</pre></details>
62
+ <details class="workbench-panel full"><summary>Console output</summary><pre id="console" class="workbench-output"></pre></details>
61
63
  </section>
62
64
  </main>
63
- <script is:inline src={`${base}playground.js`} defer></script>
65
+ <script>import '../playground/client.js';</script>
64
66
  </SiteLayout>