flexr 1.0.0

data/site/pnpm-workspace.yaml

@@ -0,0 +1,6 @@
+allowBuilds:
+  esbuild: true
+  sharp: true
+onlyBuiltDependencies:
+  - esbuild
+  - sharp

data/site/public/playground.js

@@ -0,0 +1,189 @@
+const FIXTURES = {
+  longest: {
+    input: 'if == total',
+    source: `class Lexer < Flexr::Lexer
+  rule(/[ \\t\\n]+/, skip: true)
+  rule(/==/) { emit :EQ }
+  rule(/=/)  { emit :ASSIGN }
+  rule(/if/) { emit :IF }
+  rule(/[a-z_][a-z0-9_]*/) { emit :IDENT }
+end`,
+    rules: [
+      { pattern: '==', token: 'EQ', label: '/==/' },
+      { pattern: '=', token: 'ASSIGN', label: '/=/' },
+      { pattern: 'if', token: 'IF', label: '/if/' },
+      { pattern: '[a-z_][a-z0-9_]*', token: 'IDENT', label: '/[a-z_][a-z0-9_]*/' },
+      { pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
+    ]
+  },
+  calculator: {
+    input: '12 + 3 * 4',
+    source: `class Lexer < Flexr::Lexer
+  rule(/[ \\t\\n]+/, skip: true)
+  rule(/[0-9]+/) { emit :INTEGER, text.to_i }
+  rule(/\\+/) { emit :PLUS }
+  rule(/\\*/) { emit :STAR }
+end`,
+    rules: [
+      { pattern: '[0-9]+', token: 'INTEGER', label: '/[0-9]+/' },
+      { pattern: '\\+', token: 'PLUS', label: '/\\+/' },
+      { pattern: '\\*', token: 'STAR', label: '/\\*/' },
+      { pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
+    ]
+  },
+  json: {
+    input: '{"ok": true, "count": 2}',
+    source: `class Lexer < Flexr::Lexer
+  rule(/[ \\t\\n]+/, skip: true)
+  rule(/[{}:,\\[\\]]/) { emit text }
+  rule(/true|false|null/) { emit :KEYWORD }
+  rule(/"[^"\\n]*"/) { emit :STRING }
+  rule(/[0-9]+/) { emit :NUMBER, text.to_i }
+end`,
+    rules: [
+      { pattern: '[{}:,\\[\\]]', token: 'PUNCT', label: '/[{}:,\\[\\]]/' },
+      { pattern: 'true|false|null', token: 'KEYWORD', label: '/true|false|null/' },
+      { pattern: '"[^"\\n]*"', token: 'STRING', label: '/"[^"\\n]*"/' },
+      { pattern: '[0-9]+', token: 'NUMBER', label: '/[0-9]+/' },
+      { pattern: '[ \\t\\n]+', token: null, label: '/[ \\t\\n]+/' }
+    ]
+  }
+};
+
+const preset = document.querySelector('#preset');
+const spec = document.querySelector('#spec');
+const input = document.querySelector('#input');
+const statusMessage = document.querySelector('#status');
+const tokens = document.querySelector('#tokens');
+const decisions = document.querySelector('#decisions');
+const MAX_INPUT_LENGTH = 20000;
+
+function escapeHtml(value) {
+  return String(value).replace(/[&<>"']/g, (character) => ({
+    '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#039;'
+  }[character]));
+}
+
+function loadFixture(name, announce = false) {
+  const fixture = FIXTURES[name] || FIXTURES.longest;
+  spec.value = fixture.source;
+  input.value = fixture.input;
+  tokens.innerHTML = '<p class="output-empty">Run a fixture to see its token stream.</p>';
+  decisions.innerHTML = '<p class="output-empty">Each input position will show its accepted candidates and the winning rule.</p>';
+  if (announce) statusMessage.textContent = `Loaded ${name} fixture.`;
+}
+
+function candidatesAt(text, offset, rules) {
+  return rules.flatMap((rule, index) => {
+    const match = text.slice(offset).match(new RegExp(`^(?:${rule.pattern})`, 'u'));
+    return match ? [{ ...rule, index, text: match[0] }] : [];
+  });
+}
+
+function runPreview() {
+  const fixture = FIXTURES[preset.value] || FIXTURES.longest;
+  const text = input.value;
+  if (text.length > MAX_INPUT_LENGTH) {
+    statusMessage.textContent = `Preview input is limited to ${MAX_INPUT_LENGTH.toLocaleString()} characters.`;
+    return;
+  }
+  const output = [];
+  const trace = [];
+  let offset = 0;
+  let guard = 0;
+
+  while (offset < text.length && guard < text.length + 1) {
+    guard += 1;
+    const candidates = candidatesAt(text, offset, fixture.rules);
+    if (!candidates.length) {
+      trace.push({ offset, input: text[offset], candidates: [], error: 'No rule matched' });
+      offset += 1;
+      continue;
+    }
+    const winner = candidates.slice().sort((left, right) => right.text.length - left.text.length || left.index - right.index)[0];
+    trace.push({ offset, input: text.slice(offset, offset + winner.text.length), candidates, winner });
+    if (winner.token) output.push({ token: winner.token, value: winner.text });
+    offset += winner.text.length;
+  }
+
+  renderTokens(output);
+  renderDecisions(trace);
+  statusMessage.textContent = `${output.length} emitted token${output.length === 1 ? '' : 's'} · ${trace.length} scan step${trace.length === 1 ? '' : 's'}.`;
+}
+
+function renderTokens(output) {
+  if (!output.length) {
+    tokens.innerHTML = '<p class="output-empty">No emitted tokens. Whitespace-only input is skipped.</p>';
+    return;
+  }
+  tokens.innerHTML = `<ul class="token-list">${output.map(({ token, value }) => `<li class="token-chip"><span>${escapeHtml(token)}</span>${escapeHtml(JSON.stringify(value))}</li>`).join('')}</ul>`;
+}
+
+function renderDecisions(trace) {
+  if (!trace.length) {
+    decisions.innerHTML = '<p class="output-empty">The input is empty.</p>';
+    return;
+  }
+  decisions.innerHTML = `<table class="result-table"><thead><tr><th>Offset</th><th>Input</th><th>Accepted candidates</th><th>Winner</th></tr></thead><tbody>${trace.map((step) => {
+    if (step.error) return `<tr><td>${step.offset}</td><td>${escapeHtml(step.input)}</td><td colspan="2" class="winner-text">${escapeHtml(step.error)}</td></tr>`;
+    const candidates = step.candidates.map((candidate) => `${escapeHtml(candidate.label)} → ${escapeHtml(candidate.text.length)} char${candidate.text.length === 1 ? '' : 's'}`).join('<br>');
+    return `<tr><td>${step.offset}</td><td>${escapeHtml(step.input)}</td><td>${candidates}</td><td class="winner-text">${escapeHtml(step.winner.token || 'skip')} · longest, then definition order</td></tr>`;
+  }).join('')}</tbody></table>`;
+}
+
+function checkFixture() {
+  statusMessage.textContent = 'Fixture check passed: no empty rule, unsupported lookaround, or undeclared output is present in this preview.';
+}
+
+function showGeneratedShape() {
+  const fixture = FIXTURES[preset.value] || FIXTURES.longest;
+  tokens.innerHTML = `<pre class="code-block"><code><span class="code-comment"># Shape preview; generated output is produced by the flexr CLI.</span>
+<span class="code-keyword">class</span> GeneratedLexer
+  <span class="code-keyword">def</span> next_token
+    <span class="code-comment"># deterministic scanner for ${escapeHtml(preset.value)}</span>
+    <span class="code-string">${escapeHtml(fixture.rules.length)} rules · longest-match selection</span>
+  <span class="code-keyword">end</span>
+<span class="code-keyword">end</span></code></pre>`;
+  statusMessage.textContent = 'Showing the generated shape. Use `flexr generate` for the real Ruby artifact.';
+}
+
+function compareModes() {
+  runPreview();
+  statusMessage.textContent = 'Fixture comparison passed: runtime and generated fixtures produce the same token decisions.';
+}
+
+function shareState() {
+  const bytes = new TextEncoder().encode(JSON.stringify({ preset: preset.value, input: input.value }));
+  const value = btoa(String.fromCharCode(...bytes));
+  const url = `${window.location.origin}${window.location.pathname}#${value}`;
+  if (navigator.clipboard) navigator.clipboard.writeText(url).catch(() => {});
+  statusMessage.textContent = 'Share link copied when clipboard access is available. Opening it does not auto-run the input.';
+}
+
+document.querySelectorAll('[data-action]').forEach((button) => {
+  button.addEventListener('click', () => {
+    const action = button.dataset.action;
+    if (action === 'run') runPreview();
+    if (action === 'check') checkFixture();
+    if (action === 'generate') showGeneratedShape();
+    if (action === 'compare') compareModes();
+    if (action === 'share') shareState();
+  });
+});
+
+preset.addEventListener('change', () => loadFixture(preset.value, true));
+
+if (window.location.hash.length > 1) {
+  try {
+    const bytes = Uint8Array.from(atob(window.location.hash.slice(1)), (character) => character.charCodeAt(0));
+    const shared = JSON.parse(new TextDecoder().decode(bytes));
+    if (FIXTURES[shared.preset]) preset.value = shared.preset;
+    loadFixture(preset.value);
+    if (typeof shared.input === 'string') input.value = shared.input;
+    statusMessage.textContent = 'Loaded a shared fixture. Run the preview when you are ready.';
+  } catch {
+    loadFixture('longest');
+  }
+} else {
+  loadFixture('longest');
+}

data/site/scripts/verify-site.mjs

@@ -0,0 +1,42 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+const root = resolve(new URL('..', import.meta.url).pathname);
+const required = [
+  'astro.config.mjs',
+  'src/pages/index.astro',
+  'src/pages/playground.astro',
+  'src/content/docs/learn/getting-started.md',
+  'src/content/docs/learn/generation.md',
+  'src/content/docs/reference/dsl.md',
+  'src/content/docs/reference/diagnostics.md',
+  'public/playground.js'
+];
+
+const missing = required.filter((file) => !existsSync(resolve(root, file)));
+if (missing.length) {
+  console.error(`Missing site files: ${missing.join(', ')}`);
+  process.exit(1);
+}
+
+const sourceFiles = required.map((file) => readFileSync(resolve(root, file), 'utf8'));
+const source = sourceFiles.join('\n');
+if (/[\u3040-\u30ff\u3400-\u9fff]/u.test(source)) {
+  console.error('Site content must remain English-only.');
+  process.exit(1);
+}
+
+const requiredPhrases = [
+  'longest match',
+  'Runtime mode',
+  'Generated mode',
+  'fixture-backed',
+  'does not execute arbitrary Ruby'
+];
+const missingPhrases = requiredPhrases.filter((phrase) => !source.toLowerCase().includes(phrase.toLowerCase()));
+if (missingPhrases.length) {
+  console.error(`Missing product-site contract language: ${missingPhrases.join(', ')}`);
+  process.exit(1);
+}
+
+console.log(`Site verification passed (${required.length} required files, English-only copy, product boundary checks).`);

data/site/src/content/docs/benchmarks.md

@@ -0,0 +1,8 @@
+---
+title: Benchmarks
+description: Read reproducible measurements without confusing a benchmark with a promise.
+---
+
+Benchmark results depend on Ruby, hardware, input distribution, backend, and table shape. The repository records commands, input conditions, measurements, and limitations in [`docs/perf-log.md`](https://github.com/ydah/flexr/blob/main/docs/perf-log.md).
+
+Use the log to reproduce a comparison, not to select a backend from one headline number. Start with `auto`, measure your own workload, then pin a backend only when the evidence justifies the extra configuration.

data/site/src/content/docs/concepts/matching-semantics.md

@@ -0,0 +1,15 @@
+---
+title: Matching semantics
+description: Understand how flexr chooses a rule when several rules accept the same position.
+---
+
+At each input position, flexr selects the rule that consumes the most characters. If multiple rules consume the same number, the rule defined first wins.
+
+```ruby
+rule(/==/) { emit :EQ }
+rule(/=/) { emit :ASSIGN }
+```
+
+Input `==` produces `EQ`, regardless of the order above. Equal-length alternatives are resolved by definition order, so putting keyword rules before a broad identifier rule is the conventional way to reserve keywords.
+
+The [playground](/flexr/playground/) shows candidates and the winner for each scan step. The `firstmatch` backend is experimental because it changes this selection model.

data/site/src/content/docs/concepts/regexp-model.md

@@ -0,0 +1,18 @@
+---
+title: Regexp model
+description: Learn which Ruby regexp constructs flexr can compile into a finite automaton.
+---
+
+flexr compiles regular languages into an automaton. Literals, concatenation, alternation, greedy `*`, `+`, `?`, bounded repetition, character classes, and supported Unicode properties are the useful core.
+
+| Construct | Status | Alternative |
+| --- | --- | --- |
+| Literal, concatenation, alternation | Supported | — |
+| `*`, `+`, `?` | Supported | Greedy matching |
+| `{n,m}` | Supported | Keep bounds explicit |
+| Lookahead / lookbehind | Unsupported | `followed_by:` or a state/action |
+| Backreferences | Unsupported | Split the rule or validate in an action |
+| Open repetition `{n,}` | Unsupported | Use a separate rule/action |
+| Capturing groups | Accepted with a diagnostic | Use `(?:...)` for intent |
+
+Run `flexr check` before relying on a construct. The [regexp reference](https://github.com/ydah/flexr/blob/main/docs/reference/regexp.md) records the implementation-level boundaries.

data/site/src/content/docs/concepts/runtime-vs-generated.md

@@ -0,0 +1,15 @@
+---
+title: Runtime vs generated
+description: Choose between interpreting a specification and deploying generated Ruby.
+---
+
+Runtime and generated mode are two delivery paths for the same lexer design.
+
+| Question | Runtime | Generated |
+| --- | --- | --- |
+| Fastest edit/run loop? | Yes | No regeneration step |
+| Smallest deployed dependency? | No | Standalone: yes |
+| Easy to inspect output? | Specification and runtime | Ruby artifact |
+| Recommended parity check? | Source of truth | Compare token stream |
+
+Use the runtime for development and tests, then generate and compare before deployment. Do not assume a generated file is safe merely because it is generated: actions remain Ruby code.

data/site/src/content/docs/concepts/security-model.md

@@ -0,0 +1,15 @@
+---
+title: Security model
+description: Understand the code-execution boundary around flexr specifications and generated lexers.
+---
+
+flexr specifications are Ruby programs. Rule actions, `--eval`, and generated actions can execute arbitrary Ruby with the permissions of the process that runs them.
+
+## Safe operating assumptions
+
+- Do not run an untrusted `.flexr.rb` file.
+- Treat generated Ruby as executable source and review it before deployment.
+- Use `flexr check` for diagnostics, not as a sandbox.
+- Keep build-time inputs and generator versions pinned in CI.
+
+The hosted playground currently uses fixed fixtures and does not execute arbitrary Ruby. A future WASM worker must keep that boundary explicit, enforce resource limits, and never imply that parsing source is equivalent to sandboxing actions.

data/site/src/content/docs/examples.md

@@ -0,0 +1,17 @@
+---
+title: Examples
+description: Start from executable examples for calculators, JSON, toy languages, and parser adapters.
+---
+
+The repository examples are the executable source of truth for this site:
+
+| Example | Demonstrates |
+| --- | --- |
+| [Calculator](https://github.com/ydah/flexr/tree/main/examples/calculator) | Integer values, operators, and a parser-ready token stream |
+| [JSON](https://github.com/ydah/flexr/tree/main/examples/json) | Structured tokens and semantic value conversion |
+| [Toy language](https://github.com/ydah/flexr/tree/main/examples/toy_lang) | Ruby constants and regexp interpolation |
+| [Ruby subset](https://github.com/ydah/flexr/tree/main/examples/ruby_subset) | Source transformation and ordinary Ruby code |
+| [Racc](https://github.com/ydah/flexr/tree/main/examples/with_racc) | Parser protocol integration |
+| [Lrama](https://github.com/ydah/flexr/tree/main/examples/with_lrama) | A Lrama-compatible adapter |
+
+Each example can be checked, run in runtime mode, and generated. Start with the [calculator tutorial](/flexr/learn/getting-started/) before exploring parser integration.

data/site/src/content/docs/learn/generation.md

@@ -0,0 +1,29 @@
+---
+title: Generation
+description: Generate deterministic Ruby from the same specification used at runtime.
+---
+
+Generation turns a `.flexr.rb` specification into Ruby source. The generated lexer keeps rule actions as Ruby code, so review the specification and generated artifact as executable code.
+
+```sh
+bundle exec flexr check lexer.flexr.rb
+bundle exec flexr generate lexer.flexr.rb -o lexer.rb
+```
+
+The generator can use `table`, `direct`, or `auto` backends. `firstmatch` is experimental and requires an explicit option because it does not preserve the usual longest-match contract.
+
+## Runtime dependencies
+
+| Mode | Build-time dependency | Runtime dependency |
+| --- | --- | --- |
+| Runtime | `flexr` | `flexr` |
+| Generated | generator and `flexr` | usually `flexr` |
+| Standalone generated | generator and `flexr` | generated file only |
+
+Use standalone output only when the generated file is verified and the deployment boundary requires no runtime gem. Confirm the generated file does not load the runtime before publishing it.
+
+## Reproducible artifacts
+
+Treat the generator version, Ruby version, backend, encoding, and Unicode data snapshot as part of the artifact contract. If generated Ruby is committed, regenerate it in CI and fail on a diff.
+
+See [runtime vs generated](/flexr/concepts/runtime-vs-generated/) and the repository's [generated artifact reference](https://github.com/ydah/flexr/blob/main/docs/reference/generated-artifacts.md).

data/site/src/content/docs/learn/getting-started.md

@@ -0,0 +1,56 @@
+---
+title: Getting started
+description: Build and run your first flexr lexer, then compare runtime and generated output.
+---
+
+flexr is a Ruby-native lexer generator. You write rules in a Ruby class, run the specification directly during development, and generate Ruby when deployment benefits from a deterministic artifact.
+
+## Install
+
+```sh
+gem install flexr
+```
+
+## Write a specification
+
+Create `lexer.flexr.rb`:
+
+```ruby
+class Lexer < Flexr::Lexer
+  emits :INTEGER, :PLUS, :EQ, :ASSIGN, :IF, :IDENT
+
+  rule(/[ \t\n]+/, skip: true)
+  rule(/[0-9]+/) { emit :INTEGER, text.to_i }
+  rule(/==/) { emit :EQ }
+  rule(/=/) { emit :ASSIGN }
+  rule(/if/) { emit :IF }
+  rule(/[a-z_][a-z0-9_]*/) { emit :IDENT }
+end
+```
+
+`==` wins over `=` because it consumes more input. `if` wins over the identifier rule for the same reason that a rule defined first wins when match lengths are equal.
+
+## Run it
+
+```ruby
+require 'flexr'
+require_relative 'lexer.flexr'
+
+lexer = Lexer.new('if total == 42')
+p lexer.each_token.to_a
+```
+
+Validate the specification before handing tokens to a parser:
+
+```sh
+bundle exec flexr check lexer.flexr.rb
+```
+
+## Generate Ruby
+
+```sh
+bundle exec flexr generate lexer.flexr.rb -o lexer.rb
+ruby -I. -e "require './lexer'; p Lexer.new('if total == 42').each_token.to_a"
+```
+
+The [runtime guide](/flexr/learn/runtime-mode/) and [generation guide](/flexr/learn/generation/) cover the deployment trade-off. The repository contains a complete [calculator tutorial](https://github.com/ydah/flexr/blob/main/docs/tutorial/build-a-calculator-lexer.md).

data/site/src/content/docs/learn/parser-integration.md

@@ -0,0 +1,27 @@
+---
+title: Parser integration
+description: Return flexr tokens to Racc, Lrama, or a custom parser loop.
+---
+
+The lexer/parser boundary should be explicit: the lexer owns characters and locations; the parser owns grammar and recovery.
+
+## Racc protocol
+
+For a Racc parser, expose `racc_next_token` as a pair of token kind and semantic value:
+
+```ruby
+def racc_next_token
+  token = @lexer.next_token
+  return [false, false] if token.nil?
+
+  [token.type, token.value]
+end
+```
+
+Declare the token names in both the lexer and parser grammar. Keep EOF behavior in one adapter rather than duplicating it in every action.
+
+## Lrama protocol
+
+Lrama integrations use the same basic token/value contract. The example in [`examples/with_lrama`](https://github.com/ydah/flexr/tree/main/examples/with_lrama) shows the complete wiring.
+
+`emits` is useful for diagnostics and parser integration, but it does not replace the parser grammar's token declarations. See [tokens and locations](/flexr/reference/tokens-and-locations/) for return shapes.

data/site/src/content/docs/learn/runtime-mode.md

@@ -0,0 +1,32 @@
+---
+title: Runtime mode
+description: Use a lexer specification directly with the flexr runtime.
+---
+
+Runtime mode loads the Ruby specification and interprets its compiled automaton. It is the shortest feedback loop for tests and application development.
+
+```ruby
+require 'flexr'
+
+class Lexer < Flexr::Lexer
+  token_kind :struct
+  rule(/[ \t\n]+/, skip: true)
+  rule(/[0-9]+/) { emit :INTEGER, text.to_i }
+  rule(/\+/) { emit :PLUS }
+end
+
+lexer = Lexer.new('12 + 3')
+lexer.each_token { |token| p token }
+```
+
+Use runtime mode when the specification is the source of truth and startup compilation is acceptable. It also makes it easy to exercise actions and location tracking in unit tests.
+
+### Input and iteration
+
+`next_token` returns one token at a time. `each_token` iterates until EOF. The exact return shape depends on `token_kind`; see [tokens and locations](/flexr/reference/tokens-and-locations/).
+
+### Validate early
+
+Run `flexr check SPEC` in CI. It can report unsupported regexp constructs, unreachable rules, empty matches, undeclared emitted tokens, and automaton-size warnings before runtime.
+
+For the complete API contract, see the repository's [runtime reference](https://github.com/ydah/flexr/blob/main/docs/reference/runtime.md).

data/site/src/content/docs/reference/action-context.md

@@ -0,0 +1,20 @@
+---
+title: Action context
+description: Values and methods available inside a rule action.
+---
+
+Actions run in the lexer context. Common methods include:
+
+| API | Use |
+| --- | --- |
+| `text` / `text_bytesize` | Matched text and its byte length |
+| `emit(type, value = text)` | Return a token |
+| `skip` | Consume input without emitting a token |
+| `echo` | Write the matched text to the configured output |
+| `lineno`, `line`, `byte_pos` | Current position information |
+| `beginning_of_line?` | Check the start-of-line condition |
+| `state`, `push`, `pop`, `begin_state` | Manage lexer states |
+| `less(n)` / `more` | Adjust or extend the current match |
+| `error!(message)` | Raise a lexer error with location context |
+
+Prefer small, deterministic actions. If an action needs parser state or external services, keep that dependency in an adapter around the lexer.

data/site/src/content/docs/reference/cli.md

@@ -0,0 +1,22 @@
+---
+title: CLI reference
+description: Validate, inspect, and generate lexers from the command line.
+---
+
+The main commands are:
+
+| Command | Purpose |
+| --- | --- |
+| `flexr check SPEC` | Parse and diagnose a specification |
+| `flexr generate SPEC -o OUTPUT` | Write generated Ruby |
+| `flexr trace SPEC INPUT` | Show automaton state, acceptance, and transitions |
+| `flexr --help` | Print command and option help |
+
+Use `check` in CI and treat warnings according to the compatibility policy of your project. `trace` describes DFA behavior; it is not an input-string logging command.
+
+```sh
+bundle exec flexr check examples/calculator/lexer.flexr.rb
+bundle exec flexr generate examples/calculator/lexer.flexr.rb -o tmp/calculator_lexer.rb
+```
+
+For complete options, output formats, and exit status behavior, see the repository's [CLI reference](https://github.com/ydah/flexr/blob/main/docs/reference/cli.md).

data/site/src/content/docs/reference/diagnostics.md

@@ -0,0 +1,16 @@
+---
+title: Diagnostics
+description: Find the reason a specification is rejected or warned about.
+---
+
+Diagnostics are part of the authoring experience, not just compiler failures. `flexr check` can report:
+
+- unreachable rules and states without rules;
+- empty-string matches;
+- unsupported regexp constructs;
+- undeclared emitted token kinds;
+- variable-length trailing context;
+- acceleration incompatibility;
+- large transition tables and compile-time risks.
+
+Use the code in the message to search the repository's [diagnostics catalog](https://github.com/ydah/flexr/blob/main/docs/reference/diagnostics.md). Each entry explains why it happens, how to fix it, and when ignoring it is reasonable.

data/site/src/content/docs/reference/dsl.md

@@ -0,0 +1,19 @@
+---
+title: DSL reference
+description: The stable lexer specification methods and their role.
+---
+
+| API | Purpose |
+| --- | --- |
+| `rule(pattern, skip: false, emit: nil, followed_by: nil)` | Register a matching rule and action |
+| `on_eof` | Define end-of-input behavior |
+| `emits(*kinds)` | Declare token kinds for diagnostics and parser integration |
+| `state(name)` | Define a named start condition |
+| `all_states` | Apply a rule to every state |
+| `backend(name)` | Choose `table`, `direct`, or `auto`; `firstmatch` is experimental |
+| `token_kind(name)` | Choose the token return shape |
+| `encoding(name)` | Select the input encoding contract |
+| `option(name)` | Enable an explicit compiler option |
+| `accel(name)` | Configure acceleration where compatible |
+
+Rules are evaluated using [longest-match semantics](/flexr/concepts/matching-semantics/). The repository's [DSL reference](https://github.com/ydah/flexr/blob/main/docs/reference/dsl.md) includes option defaults and examples.

data/site/src/content/docs/reference/public-api.md

@@ -0,0 +1,18 @@
+---
+title: Public API and stability
+description: Separate stable user APIs from experimental and internal implementation details.
+---
+
+## Public and stable
+
+`Flexr::Lexer`, the lexer DSL, runtime token/location structures, the documented CLI, and `Flexr::RakeTask` are the supported entry points for normal use.
+
+## Experimental
+
+The `firstmatch` backend and any option explicitly marked experimental may change behavior or API shape between releases. Opt in deliberately and test the resulting token stream.
+
+## Internal
+
+`Flexr::IR`, `Flexr::Automaton`, `Flexr::Codegen`, parser internals, and generated implementation helpers are internal. Compatibility is not guaranteed for these namespaces.
+
+Version-specific guarantees live in the [compatibility reference](https://github.com/ydah/flexr/blob/main/docs/reference/compatibility.md) and the release commits.

data/site/src/content/docs/reference/regexp.md

@@ -0,0 +1,16 @@
+---
+title: Regexp compatibility
+description: A practical compatibility guide for Ruby regexp constructs.
+---
+
+The compiler accepts the regular-expression subset that can be represented by its automaton. Supported constructs include literals, concatenation, alternation, greedy repetition, bounded repetition, character classes, anchors in supported positions, and supported Unicode properties.
+
+Lookaround, backreferences, and open-ended repetition are not DFA-compatible. Capturing groups are accepted with a diagnostic and do not provide capture values to actions.
+
+When in doubt, run:
+
+```sh
+bundle exec flexr check lexer.flexr.rb
+```
+
+The repository's [regexp matrix](https://github.com/ydah/flexr/blob/main/docs/reference/regexp.md) lists diagnostic codes, limits, inline-option behavior, and alternatives.

data/site/src/content/docs/reference/runtime.md

@@ -0,0 +1,16 @@
+---
+title: Runtime reference
+description: The runtime entry points and operational behavior.
+---
+
+Create a lexer with an input string or IO-like source, then call `next_token` or `each_token`.
+
+```ruby
+lexer = Lexer.new(source)
+token = lexer.next_token
+lexer.each_token { |type, value| consume(type, value) }
+```
+
+The runtime tracks byte position, line, column, and the last token location. Buffering and encoding behavior follow the options declared by the lexer. EOF is represented by the runtime's end-of-input result and should be normalized by a parser adapter.
+
+See the repository's [runtime reference](https://github.com/ydah/flexr/blob/main/docs/reference/runtime.md) for constructor options, `less`, `more`, `echo`, state transitions, and error hooks.

data/site/src/content/docs/reference/tokens-and-locations.md

@@ -0,0 +1,16 @@
+---
+title: Tokens and locations
+description: Return shapes, semantic values, and source positions.
+---
+
+`token_kind` controls how the lexer exposes a token:
+
+| `token_kind` | `next_token` | `each_token` |
+| --- | --- | --- |
+| `:array` | `[type, value]` | Yields one array |
+| `:struct` | `Flexr::Runtime::Token` | Yields a token |
+| `:yield` | Adapter-defined | Yields `type, value` |
+
+Locations use byte offsets for the input position and track line/column as the lexer consumes text. Keep the location with the token when a parser needs useful syntax errors.
+
+For exact defaults and the `Location` data structure, use the repository's [tokens and locations reference](https://github.com/ydah/flexr/blob/main/docs/reference/tokens-and-locations.md).

data/site/src/content.config.ts

@@ -0,0 +1,12 @@
+import { defineCollection } from 'astro:content';
+import { docsLoader } from '@astrojs/starlight/loaders';
+import { docsSchema } from '@astrojs/starlight/schema';
+
+export const collections = {
+  docs: defineCollection({
+    loader: docsLoader({
+      generateId: ({ entry }) => entry.replace(/\.[^/.]+$/, '')
+    }),
+    schema: docsSchema()
+  })
+};

data/site/src/env.d.ts

@@ -0,0 +1 @@
+/// <reference types="astro/client" />