@uniflowed/vite 0.0.0-alpha.1 → 0.0.0-alpha.5

package/driver.js

@@ -1,3 +1,5 @@
+// @noflow
+//
 // Plain JavaScript: the host runs this file directly.
 //
 // The driver `uf dev`, `uf build` and `uf preview` spawn.
@@ -23,6 +25,7 @@ import { pathToFileURL } from "node:url";
 
 import { emit, errorEvent, eventLogger } from "./internal/events.js";
 import { loadUfConfig, projectConfig } from "./internal/config.js";
+import { withProjectConfig } from "./merge.js";
 import { VIRTUAL, scanRoutes } from "./internal/routes.js";
 
 function argument(name) {
@@ -41,10 +44,15 @@ const root = path.resolve(argument("--root") ?? process.cwd());
 process.env.UF_PROJECT_ROOT = root;
 
 // The config imports `@uniflowed/config`, which is Flow. Node needs the loader
-// hooks for that; Bun is started with `--preload ./bun-preload.js` instead
-// and has no `register`.
+// hooks for that; Bun is started with `--preload` on the same package's
+// preload instead, and has no `register`.
+//
+// The hooks live in `@uniflowed/host` rather than here: they are how Flow runs
+// on a Capability JS Host, and nothing in them is Vite's. `uf test` reaches for
+// the same package, which is what stopped a test run from depending on a
+// bundler it never loads.
 if (typeof Bun === "undefined" && typeof Deno === "undefined") {
-  register("./internal/node-hooks.js", import.meta.url, { data: { root } });
+  register("@uniflowed/host/internal/node-hooks.js", import.meta.url, { data: { root } });
 }
 
 process.stdin.on("end", () => process.exit(0));
@@ -78,16 +86,21 @@ async function viteConfig(config, mode) {
   const userPlugins = Array.isArray(config.plugins) ? config.plugins : [];
   const host = argument("--host") ?? dev.host ?? "127.0.0.1";
   const port = Number(argument("--port") ?? dev.port ?? 5173);
-  const allowedHosts = Array.isArray(dev.allowedHosts) && dev.allowedHosts.length > 0 ? dev.allowedHosts : undefined;
-
-  return {
+  const allowedHosts =
+    Array.isArray(dev.allowedHosts) && dev.allowedHosts.length > 0 ? dev.allowedHosts : undefined;
+
+  // What uf generates from the semantics it owns: where the project is, which
+  // plugins make Flow compile, and the few settings uf enforces rather than
+  // merely passes on — `allowedHosts` gates binding a routable address, and
+  // `manifest` is how the prerender finds its assets.
+  const generated = {
     root,
     configFile: false,
     envFile: false,
     mode,
     clearScreen: false,
     customLogger: eventLogger(argument("--log-level") ?? "info"),
-    plugins: [uniflowed({ root, config }), ...userPlugins],
+    plugins: [uniflowed({ root, config })],
     server: {
       host,
       port,
@@ -106,6 +119,14 @@ async function viteConfig(config, mode) {
       emptyOutDir: true,
     },
   };
+
+  // Then the project's own Vite configuration, merged over it. uf does not
+  // read this and does not need to: an option added to Vite tomorrow works in
+  // a uf project tomorrow, rather than after a uf release that names it.
+  return withProjectConfig(generated, {
+    ...(config.vite ?? {}),
+    plugins: [...(config.vite?.plugins ?? []), ...userPlugins],
+  });
 }
 
 /**
@@ -138,13 +159,27 @@ async function dev() {
   const assets = { scripts: [`/@id/${VIRTUAL.client}`], styles: [], preloads: [] };
 
   server.middlewares.use(async (request, response, next) => {
-    if (request.method !== "GET" && request.method !== "HEAD") {
-      next();
-      return;
-    }
     const url = request.originalUrl ?? request.url ?? "/";
     try {
       const entry = await server.ssrLoadModule(VIRTUAL.server);
+
+      // Route handlers first, and for every method: a handler is the only
+      // thing that answers a POST, and it may also answer a GET for a path
+      // that has no page.
+      const handled = await entry.dispatch(await toRequest(request, server.config));
+      if (handled != null) {
+        await send(response, handled);
+        return;
+      }
+
+      // Only a navigation reaches the renderer. A page cannot answer a POST,
+      // and letting one try would turn a missing handler into a rendered page
+      // with a 200 rather than a 404.
+      if (request.method !== "GET" && request.method !== "HEAD") {
+        next();
+        return;
+      }
+
       const result = await entry.render(url, assets);
       const html = await server.transformIndexHtml(url, result.html);
       response.statusCode = result.status ?? 200;
@@ -162,7 +197,9 @@ async function dev() {
   emit("listening", {
     local: urls.local,
     network: urls.network,
-    routes: scanRoutes(path.resolve(root, config.app?.router?.root ?? "app")).routes.map((route) => route.path),
+    routes: scanRoutes(path.resolve(root, config.app?.router?.root ?? "app")).routes.map(
+      (route) => route.path,
+    ),
   });
 
   const shutdown = async () => {
@@ -173,6 +210,62 @@ async function dev() {
   process.on("SIGTERM", shutdown);
 }
 
+/**
+ * A Node request as a `Request`.
+ *
+ * The handler contract is the platform's, so the adapter belongs here rather
+ * than in every handler. The body is read as a stream where the host supports
+ * it, because a handler that accepts an upload should not need the whole thing
+ * buffered before it starts.
+ */
+async function toRequest(incoming, config) {
+  const host = incoming.headers.host ?? "localhost";
+  const protocol = config?.server?.https == null ? "http" : "https";
+  const url = new URL(incoming.originalUrl ?? incoming.url ?? "/", `${protocol}://${host}`);
+
+  const headers = new Headers();
+  for (const [name, value] of Object.entries(incoming.headers)) {
+    if (value == null) continue;
+    for (const entry of Array.isArray(value) ? value : [value]) {
+      headers.append(name, entry);
+    }
+  }
+
+  const method = (incoming.method ?? "GET").toUpperCase();
+  const init = { method, headers };
+  if (method !== "GET" && method !== "HEAD") {
+    // `duplex` is required by the specification whenever a body is a stream,
+    // and Node throws without it.
+    init.body = incoming;
+    init.duplex = "half";
+  }
+  return new Request(url, init);
+}
+
+/** Write a `Response` to a Node response. */
+async function send(outgoing, result) {
+  outgoing.statusCode = result.status;
+  if (result.statusText !== "") {
+    outgoing.statusMessage = result.statusText;
+  }
+  for (const [name, value] of result.headers) {
+    outgoing.setHeader(name, value);
+  }
+  if (result.body == null) {
+    outgoing.end();
+    return;
+  }
+  // Streamed rather than buffered, so a handler returning a large or
+  // open-ended body is not read into memory first.
+  const reader = result.body.getReader();
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    outgoing.write(value);
+  }
+  outgoing.end();
+}
+
 async function preview() {
   const { preview: startPreview } = await import("vite");
   const config = await loadConfig();
@@ -236,13 +329,23 @@ async function build() {
     const file = htmlPathFor(outDir, url);
     mkdirSync(path.dirname(file), { recursive: true });
     writeFileSync(file, result.html);
-    emit("page", { url, file: path.relative(root, file), status: result.status, bytes: Buffer.byteLength(result.html) });
+    emit("page", {
+      url,
+      file: path.relative(root, file),
+      status: result.status,
+      bytes: Buffer.byteLength(result.html),
+    });
   }
   if (server.notFound != null) {
     const result = await server.render("/__uf_not_found__", assets);
     const file = path.join(outDir, "404.html");
     writeFileSync(file, result.html);
-    emit("page", { url: "/404", file: path.relative(root, file), status: 404, bytes: Buffer.byteLength(result.html) });
+    emit("page", {
+      url: "/404",
+      file: path.relative(root, file),
+      status: 404,
+      bytes: Buffer.byteLength(result.html),
+    });
   }
 
   emit("done", { outDir: path.relative(root, outDir), pages: pages.length });
@@ -346,9 +449,12 @@ function fillParams(routePath, params) {
     .map((segment) => {
       if (segment.endsWith("*")) {
         const value = params[segment.slice(1, -1)];
-        return Array.isArray(value) ? value.map(encodeURIComponent).join("/") : encodeURIComponent(String(value ?? ""));
+        return Array.isArray(value)
+          ? value.map(encodeURIComponent).join("/")
+          : encodeURIComponent(String(value ?? ""));
       }
-      if (segment.startsWith(":")) return encodeURIComponent(String(params[segment.slice(1)] ?? ""));
+      if (segment.startsWith(":"))
+        return encodeURIComponent(String(params[segment.slice(1)] ?? ""));
       return segment;
     })
     .join("/");
@@ -356,5 +462,7 @@ function fillParams(routePath, params) {
 
 function htmlPathFor(outDir, url) {
   const pathname = url.split("?")[0].replace(/^\/+/, "");
-  return pathname === "" ? path.join(outDir, "index.html") : path.join(outDir, pathname, "index.html");
+  return pathname === ""
+    ? path.join(outDir, "index.html")
+    : path.join(outDir, pathname, "index.html");
 }

package/index.js

@@ -1,3 +1,5 @@
+// @noflow
+//
 // Plain JavaScript: Vite imports this module directly, before any transform.
 //
 // `@uniflowed/vite` — uf, as Vite plugins.
@@ -15,7 +17,8 @@
 //                 development it also renders every HTML request on the
 //                 server, so `uf dev` serves the same markup `uf build` writes.
 // * `uf:mdx`    — `@mdx-js/rollup`, configured for React with GitHub-flavoured
-//                 markdown, front matter and heading ids, so `.mdx` works with
+//                 markdown, front matter, heading ids and build-time syntax
+//                 highlighting, so `.mdx` works with
 //                 no configuration.
 //
 // `uniflowed(options)` returns the array; a project that wants to add a plugin
@@ -26,6 +29,8 @@ import path from "node:path";
 
 import mdx from "@mdx-js/rollup";
 import rehypeSlug from "rehype-slug";
+
+import { highlightPlugin } from "./internal/highlight.js";
 import remarkFrontmatter from "remark-frontmatter";
 import remarkGfm from "remark-gfm";
 import remarkMdxFrontmatter from "remark-mdx-frontmatter";
@@ -44,12 +49,22 @@ import {
   scanRoutes,
   serverModuleSource,
 } from "./internal/routes.js";
-import { TransformService, isFlowModule } from "./transform.js";
+import { TransformService, isFlowModule } from "@uniflowed/host/transform";
 
 /** A resolved virtual id: Vite's convention is a leading NUL byte. */
 const resolved = (id) => `\0${id}`;
 const VIRTUAL_IDS = new Set(Object.values(VIRTUAL));
 
+/**
+ * Prefix of the virtual module that carries one source module's StyleX rules.
+ *
+ * Not NUL-prefixed, unlike the virtual modules above: Vite's CSS pipeline keys
+ * off the `.css` extension of a *resolvable* id, and a NUL-prefixed id is
+ * excluded from it. The prefix is distinctive enough that nothing else can
+ * collide with it.
+ */
+const STYLE_PREFIX = "uf-style:";
+
 /** The URL a NUL-prefixed module is served at in development. */
 export function devUrlFor(id) {
   return `/@id/__x00__${id}`;
@@ -89,6 +104,15 @@ function flowPlugin({ routerRoot, appEntry, command }) {
   let server = null;
   /** @type {TransformService | null} */
   let service = null;
+  /**
+   * Each module's compiled stylesheet, keyed by the virtual id serving it.
+   *
+   * A map rather than one accumulated sheet: Vite asks for a module's CSS when
+   * it loads that module, re-asks when the module changes, and drops it when
+   * the module goes away. One shared sheet would have to be invalidated by
+   * hand, which is the part that goes wrong.
+   */
+  const styles = new Map();
 
   const ensureService = () => {
     service ??= new TransformService({ command, root });
@@ -144,6 +168,10 @@ function flowPlugin({ routerRoot, appEntry, command }) {
     resolveId(id) {
       if (id === RUNTIME_PUBLIC_PATH) return RUNTIME_RESOLVED_ID;
       if (VIRTUAL_IDS.has(id)) return resolved(id);
+      // A module's own stylesheet, which `transform` below asked for by
+      // importing this id. Returning it unchanged marks it resolved without
+      // Vite going to the filesystem for a file that does not exist.
+      if (id.startsWith(STYLE_PREFIX)) return id;
       return null;
     },
 
@@ -152,6 +180,7 @@ function flowPlugin({ routerRoot, appEntry, command }) {
       if (id === resolved(VIRTUAL.routes)) return routesModuleSource(scanRoutes(appRoot));
       if (id === resolved(VIRTUAL.client)) return clientModuleSource(entryPath);
       if (id === resolved(VIRTUAL.server)) return serverModuleSource(entryPath);
+      if (id.startsWith(STYLE_PREFIX)) return styles.get(id) ?? "";
       return null;
     },
 
@@ -169,9 +198,24 @@ function flowPlugin({ routerRoot, appEntry, command }) {
         this.warn?.(`${diagnostic.function ?? "a function"}: ${diagnostic.message}`);
       }
       const map = out.map == null ? null : JSON.parse(out.map);
-      if (!refresh) return { code: out.code, map };
+      // StyleX. `uf transform` compiled the module's `stylex.create` calls into
+      // class names and handed back the rules they declared; the rules become a
+      // module of their own that this one imports.
+      //
+      // Handing the CSS to Vite as a module, rather than collecting it here and
+      // writing a stylesheet at the end, is what keeps uf out of the CSS
+      // business: Vite already injects a stylesheet in dev, extracts it in a
+      // build, code-splits it per chunk, and replaces it over HMR. A module
+      // whose styles are gone stops importing it, and Vite notices.
+      let output = out.code;
+      if (out.css != null && out.css !== "") {
+        const styleId = `${STYLE_PREFIX}${cleanId(id)}.css`;
+        styles.set(styleId, out.css);
+        output = `import ${JSON.stringify(styleId)};\n${output}`;
+      }
+      if (!refresh) return { code: output, map };
       const relative = path.relative(root, cleanId(id)).split(path.sep).join("/");
-      return addRefreshWrapper(out.code, map, relative);
+      return addRefreshWrapper(output, map, relative);
     },
 
     buildEnd() {
@@ -248,12 +292,23 @@ function flowPlugin({ routerRoot, appEntry, command }) {
 function mdxPlugin(markdown) {
   const mdxConfig = markdown.mdx ?? {};
   if (mdxConfig.enabled === false) return { name: "uf:mdx" };
+
+  // Highlighting is on unless a project turns it off, and it happens here
+  // rather than in the browser: the colours are in the HTML, so a code sample
+  // is readable before any JavaScript loads and no highlighter is shipped.
+  const highlight = highlightPlugin(mdxConfig.highlight);
+  const rehypePlugins = highlight == null ? [rehypeSlug] : [rehypeSlug, highlight];
+
   return {
     enforce: "pre",
     ...mdx({
       jsxImportSource: "react",
-      remarkPlugins: [remarkGfm, remarkFrontmatter, [remarkMdxFrontmatter, { name: "frontmatter" }]],
-      rehypePlugins: [rehypeSlug],
+      remarkPlugins: [
+        remarkGfm,
+        remarkFrontmatter,
+        [remarkMdxFrontmatter, { name: "frontmatter" }],
+      ],
+      rehypePlugins,
     }),
     name: "uf:mdx",
   };

package/internal/config.js

@@ -1,3 +1,5 @@
+// @noflow
+//
 // Plain JavaScript: executed by the host that runs Vite, before any transform.
 //
 // Loading `uf.config.js`.
@@ -15,11 +17,12 @@
 // data and cannot use the functions.
 
 import { createHash } from "node:crypto";
-import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 
-import { transformFlow } from "../transform.js";
+import { transformFlow } from "@uniflowed/host/transform";
+import { writeAtomically } from "@uniflowed/host/write-atomically";
 
 /** The one config file name uf reads. */
 export const CONFIG_FILES = ["uf.config.js"];
@@ -63,14 +66,18 @@ export async function loadUfConfig(root) {
 
   const source = readFileSync(file, "utf8");
   if (source.length > MAX_CONFIG_BYTES) {
-    throw new Error(`uf: ${file} is ${source.length} bytes, over the ${MAX_CONFIG_BYTES} byte ceiling`);
+    throw new Error(
+      `uf: ${file} is ${source.length} bytes, over the ${MAX_CONFIG_BYTES} byte ceiling`,
+    );
   }
 
   const compiled = await compileConfig(source, file, root);
   const module = await import(pathToFileURL(compiled).href);
   const config = module.default;
   if (config == null || typeof config !== "object") {
-    throw new Error(`uf: ${path.relative(root, file)} must \`export default defineConfig({ ... })\``);
+    throw new Error(
+      `uf: ${path.relative(root, file)} must \`export default defineConfig({ ... })\``,
+    );
   }
   return { config, file };
 }
@@ -88,8 +95,13 @@ async function compileConfig(source, file, root) {
 
   const out = await transformFlow(source, file, { root, sourceMap: false });
   const code = rewriteRelativeImports(out?.code ?? source, path.dirname(file));
-  mkdirSync(directory, { recursive: true });
-  writeFileSync(target, `// Compiled from ${file}. Do not edit; edit the source.\n${code}`);
+  // Atomically, because two `uf` commands in one project write this same path
+  // at the same time — the hash is of the source, so they agree on the name —
+  // and `writeFileSync` truncates before it writes. A reader that caught it
+  // mid-write imported a module with no exports and reported it as
+  // `uf.config.js must export default defineConfig({ ... })`, which is a
+  // sentence about a file that is perfectly correct. See ubugeeei-prod/uf#240.
+  writeAtomically(target, `// Compiled from ${file}. Do not edit; edit the source.\n${code}`);
   return target;
 }
 
@@ -126,7 +138,10 @@ export function projectConfig(config) {
     JSON.stringify(config, (key, value) => {
       if (typeof value === "function") return undefined;
       if (key === "plugins" && Array.isArray(value)) {
-        return value.flat(Infinity).map(pluginName).filter((name) => name != null);
+        return value
+          .flat(Infinity)
+          .map(pluginName)
+          .filter((name) => name != null);
       }
       return value;
     }),

package/internal/events.js

@@ -1,3 +1,5 @@
+// @noflow
+//
 // Plain JavaScript: executed by the host that runs Vite, before any transform.
 //
 // The driver's control channel.

package/internal/flow-grammar-shim.js

@@ -0,0 +1,241 @@
+// @noflow
+//
+// The Flow syntax a JavaScript grammar cannot parse, shown to it as
+// JavaScript it can, and taken back afterwards.
+//
+// # Why re-tagging the word is not enough
+//
+// `internal/flow-keywords.js` colours Flow's words after the grammar has run.
+// That works for a word the grammar tokenised and mis-labelled. It cannot work
+// for syntax that stops the grammar, because there are then no tokens to
+// re-label — the whole construct arrives as one unstyled run. Two pieces of
+// Flow do that, and both are common enough that this repository's own
+// documentation hit them on its first page about Flow.
+//
+// **A `component` or `hook` declaration.** A JavaScript grammar has no
+// production for an identifier where a declaration keyword belongs, so it
+// gives up on the rest of the line:
+//
+//     export component Avatar(src: string, size: number = 32) {
+//     └ keyword ┘ └ name ┘ └───────── one grey token ─────────┘
+//
+// The parameter names, their types and the default value were all the same
+// undifferentiated grey. `export hook useNow(…) {` was worse: the grammar's
+// state did not recover, and all eight lines of that sample came out as one
+// grey token each — a code block with no highlighting at all, on the page
+// whose subject is Flow's syntax.
+//
+// **An exact object type, `{| … |}`.** This one is quieter and travels
+// further. The grammar reads the `{|` as a brace and a bitwise or, and every
+// line *after* it in the same block is then mis-scoped: `export` came out in
+// the colour of a function call, `return` likewise, and a JSX tag lost its
+// element colour. One type annotation discoloured the rest of the sample.
+//
+// # What this does instead
+//
+// It hands the grammar `function` where the source says `component` or `hook`,
+// and a plain brace where the source says `{|`, takes the tokens that
+// produces, and rebuilds them over the original text. The grammar then walks
+// the parameter list, the return type and the body the way it does for any
+// function, and `flow-keywords.js` recolours the restored words.
+//
+// # Why this cannot corrupt the sample
+//
+// Because {@link restoreLine} never copies from the text the grammar saw. It
+// maps each token's boundaries back into the original line and slices *that*,
+// so the concatenation of a restored line is the original line by
+// construction, whatever the grammar decided to do with a stand-in. A
+// mis-shimmed line can come out with the wrong colours. It cannot come out
+// saying `function`.
+//
+// # What it does not cover
+//
+// The declaration rewrite is anchored to the start of a line, after an
+// optional `export` or `export default`, which is where a declaration begins
+// and where the grammar breaks. A declaration written anywhere else is left to
+// the ordinary path.
+//
+// Neither rewrite asks whether the line is inside a string or a comment, so a
+// line of quoted sample code that opens with `component Name(` is rewritten
+// too. The text still survives exactly; only its colours are a function's
+// rather than a string's, and `flow-keywords.js` still refuses to call the
+// word a keyword. Buying the remaining fidelity would mean lexing the block
+// twice, once here and once there, to fix a case that is a code sample inside
+// a code sample.
+
+/**
+ * A `component` or `hook` declaration head at the start of a line.
+ *
+ * The name and the opening bracket are matched but not captured: requiring
+ * them is what distinguishes a declaration from `const component = 1`, and
+ * consuming them would mean putting them back.
+ */
+const DECLARATION_HEAD =
+  /^([ \t]*(?:export[ \t]+(?:default[ \t]+)?)?)(component|hook)(?=[ \t]+[A-Za-z_$][\w$]*[ \t]*[(<])/;
+
+/**
+ * The word a declaration keyword is shown as.
+ *
+ * `function` and not `function*`: a generator tokenises identically here, and
+ * the length no longer has to match now that restoration maps positions rather
+ * than assuming they line up.
+ */
+const DECLARATION_STAND_IN = "function";
+
+/**
+ * The braces of an exact object type, and the ordinary braces they are shown
+ * as.
+ *
+ * Same length in both directions, so the rest of the line does not move; the
+ * mapping would cope either way, but a rewrite that cannot shift anything is
+ * one less thing to reason about. `{||}` — the empty exact object — is two
+ * adjacent rewrites rather than an overlapping one, which is why these are
+ * matched as a pair of two-character sequences rather than as one bracket.
+ */
+const EXACT_OBJECT = /\{\||\|\}/g;
+
+const EXACT_OBJECT_STAND_INS = { "{|": "{ ", "|}": " }" };
+
+/**
+ * `source` rewritten for the grammar, with the undo that belongs to it.
+ *
+ * The undo is returned rather than exported separately because it closes over
+ * the original lines, and pairing the wrong undo with a rewrite is the one
+ * mistake that would matter. `restore` is the identity when nothing was
+ * rewritten, so the caller has no case to distinguish.
+ *
+ * @param {string} source
+ * @returns {{code: string, restore: (lines: Array<Array<object>>) => Array<Array<object>>}}
+ */
+export function shimFlowGrammar(source) {
+  const original = source.split("\n");
+  const edits = new Map();
+
+  const shimmed = original.map((line, index) => {
+    const lineEdits = editsFor(line);
+    if (lineEdits.length === 0) {
+      return line;
+    }
+    edits.set(index, lineEdits);
+    return rewrite(line, lineEdits);
+  });
+
+  if (edits.size === 0) {
+    return { code: source, restore: (lines) => lines };
+  }
+  return {
+    code: shimmed.join("\n"),
+    restore: (lines) => restore(lines, edits, original),
+  };
+}
+
+/**
+ * Every rewrite one line needs, in the order they occur.
+ *
+ * Ordered and non-overlapping, because {@link mapping} walks them once and
+ * accumulates the shift each one makes. The declaration head is found first
+ * and starts at the line's indentation, so it can never overlap an exact
+ * object brace, which needs a `{` or a `|`.
+ */
+function editsFor(line) {
+  const found = [];
+  const head = DECLARATION_HEAD.exec(line);
+  if (head != null) {
+    found.push({ column: head[1].length, text: head[2], standIn: DECLARATION_STAND_IN });
+  }
+  for (const brace of line.matchAll(EXACT_OBJECT)) {
+    found.push({
+      column: brace.index ?? 0,
+      text: brace[0],
+      standIn: EXACT_OBJECT_STAND_INS[brace[0]],
+    });
+  }
+  return found;
+}
+
+/** `line` with every rewrite applied, left to right. */
+function rewrite(line, edits) {
+  let out = "";
+  let at = 0;
+  for (const edit of edits) {
+    out += line.slice(at, edit.column) + edit.standIn;
+    at = edit.column + edit.text.length;
+  }
+  return out + line.slice(at);
+}
+
+/**
+ * The tokenised block, rebuilt over the original source.
+ *
+ * Offsets are recomputed rather than adjusted: a stand-in that is not the
+ * length of the text it stands for moves everything after it on the line, and
+ * every line after that one, so there is no correct delta to add. Walking the
+ * restored lines gives the right answer directly.
+ */
+function restore(lines, edits, original) {
+  let offset = 0;
+  return lines.map((tokens, index) => {
+    const text = original[index] ?? tokens.map((token) => token.content).join("");
+    const lineEdits = edits.get(index);
+    const restored =
+      lineEdits == null ? reoffset(tokens, offset) : restoreLine(tokens, text, lineEdits, offset);
+    offset += text.length + 1;
+    return restored;
+  });
+}
+
+/** A line the grammar saw unchanged, with its offsets moved to the original. */
+function reoffset(tokens, offset) {
+  let at = 0;
+  return tokens.map((token) => {
+    const placed = { ...token, offset: offset + at };
+    at += token.content.length;
+    return placed;
+  });
+}
+
+/**
+ * One rewritten line's tokens, re-cut over the original text.
+ *
+ * Every token boundary is a position in the line the grammar saw; {@link
+ * mapping} turns it into a position in the line the reader gets. A boundary
+ * that falls *inside* a stand-in maps to the end of the text it stands for, so
+ * that text always lands in exactly one piece and no character of the line is
+ * dropped — the pieces still tile the line end to end even when the grammar
+ * split a stand-in or swallowed it into a longer token.
+ */
+function restoreLine(tokens, text, edits, offset) {
+  const map = mapping(edits);
+  const out = [];
+  let seen = 0;
+  let at = 0;
+  for (const token of tokens) {
+    const from = map(seen);
+    seen += token.content.length;
+    const to = map(seen);
+    if (to <= from) {
+      continue;
+    }
+    out.push({ ...token, content: text.slice(from, to), offset: offset + at });
+    at += to - from;
+  }
+  return out;
+}
+
+/** A position in the rewritten line, as a position in the original one. */
+function mapping(edits) {
+  return (at) => {
+    let shift = 0;
+    for (const edit of edits) {
+      const from = edit.column + shift;
+      if (at <= from) {
+        return at - shift;
+      }
+      if (at < from + edit.standIn.length) {
+        return edit.column + edit.text.length;
+      }
+      shift += edit.standIn.length - edit.text.length;
+    }
+    return at - shift;
+  };
+}