@daugasauron/dolly-build 0.1.5 → 0.1.6

package/scripts/dolly-build.mjs

@@ -1,11 +1,13 @@
 #!/usr/bin/env node
-// An npm user's image: builds a Dollyfile FROM an installed image package, with
-// the packages it INSTALLs from their packages, in a browser with that browser's
-// Dolly runtime, into a site a static host serves as it lies (docs/embedding.md,
-// "Your own image").
-//   npx dolly-build init NAME [PACKAGE…]   writes Dollyfile-NAME FROM pi-minimal and public/ (the page)
+// An npm user's image: builds a Dollyfile that INSTALLs installed packages, or starts
+// FROM an installed image, in a browser with that browser's Dolly runtime, into a
+// site a static host serves as it lies (docs/npm.md).
+//   npx dolly-build init NAME [PACKAGE…]   writes Dollyfile-NAME, Pi's project, public/ (the page) and a CI workflow
+//   npx dolly-build add NAME [PACKAGE…]    adds INSTALL rows to Dollyfile-NAME, and writes every row's installed pin
 //   npx dolly-build Dollyfile-NAME OUT [--browser chrome|chromium|firefox]
-// OUT is the files of public/, workspace.json listing public/workspace/, the runtime
+// init and add install with the project's package manager what they name and lack.
+// OUT is the files of public/, workspace.json and skills.json listing public/workspace/
+// and public/skills/, the runtime
 // (dolly/), the host modules the image declares (dolly-host-NAME/),
 // coi-serviceworker.js for hosts without isolation headers, and image/, the image's
 // directory as an image package lays it out (catalog, metadata, packs, recipes,
@@ -14,9 +16,11 @@
 // (pnpm links there what it keeps elsewhere); the runtime runs only in the browser.
 import { cp, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises";
 import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { createServer } from "node:http";
 import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
+import { Readable } from "node:stream";
 import { pathToFileURL } from "node:url";
 import { parseArgs } from "node:util";
 import { chromium, firefox } from "playwright-core";
@@ -27,37 +31,99 @@ import { ensureSnapshotPacks } from "./share-pages-snapshots.mjs";
 import { verifySnapshotIdentity } from "./snapshot-identity.mjs";
 import { decodeSystemSnapshot } from "./system-snapshot-format.mjs";
 import { readWasmInterface } from "./wasm-interface.mjs";
-import { loadRecipeGraph } from "../src/dollyfile-graph.mjs";
-import { imageFileName, inspectDollyfile } from "../src/dollyfile-view.mjs";
-import { imageInputs } from "../src/image-inputs.mjs";
-import { validName } from "../src/image-names.mjs";
-import { siteReference, sitePath } from "../src/static-asset.mjs";
+import { loadRecipeGraph } from "../site/dollyfile-graph.mjs";
+import { imageFileName, inspectDollyfile, recipeFileName } from "../site/dollyfile-view.mjs";
+import { imageInputs } from "../page/image-inputs.mjs";
+import { validName } from "../page/image-names.mjs";
+import { siteReference, sitePath } from "../page/static-asset.mjs";
+
+// A build reaches the network through Node (relay below), and through the proxy the
+// environment names, as a CI runner's often does, which Node's fetch takes only with
+// NODE_USE_ENV_PROXY set: dolly-build runs itself again with it.
+if (["HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"].some(name => process.env[name]) && !process.env.NODE_USE_ENV_PROXY) {
+  const { status } = spawnSync(process.execPath, process.argv.slice(1), { stdio: "inherit",
+    env: { ...process.env, NODE_USE_ENV_PROXY: "1" } });
+  process.exit(status ?? 1);
+}
 
 function fail(message) {
   console.error(`dolly-build: ${message}`);
   process.exit(1);
 }
-const scope = resolve("node_modules/@daugasauron");
-if (!existsSync(join(scope, "dolly"))) fail("run it where node_modules/@daugasauron/dolly is installed");
 const load = path => import(pathToFileURL(path).href);
-const { positionals, values: { browser: browserName } } = parseArgs({ allowPositionals: true,
-  options: { browser: { type: "string", default: "chrome" } } });
+const usage = `usage: dolly-build init NAME [PACKAGE...]   a project: Pi with the packages named
+       dolly-build add NAME [PACKAGE...]    Dollyfile-NAME with these packages too, every row at its installed pin
+       dolly-build Dollyfile-NAME OUT [--browser chrome|chromium|firefox]   the site, built in a browser`;
+const { positionals, values: { browser: browserName } } = (() => {
+  try { return parseArgs({ allowPositionals: true, options: { browser: { type: "string" } } }); } catch { fail(usage); }
+})();
 // Firefox's system add-on updater resolves its host even behind a proxy.
 const browsers = {
   chrome: options => chromium.launch({ channel: "chrome", headless: true, ...options }),
   chromium: options => chromium.launch({ headless: true, ...options }),
   firefox: options => firefox.launch({ headless: true, firefoxUserPrefs: { "extensions.systemAddon.update.enabled": false }, ...options }),
 };
-const usage = "usage: dolly-build init NAME [PACKAGE...] | Dollyfile-NAME OUT [--browser chrome|chromium|firefox]";
-const init = positionals[0] === "init";
-if (!browsers[browserName] || (init ? positionals.length < 2 : positionals.length !== 2)) fail(usage);
+// Without --browser: the first of them this machine has.
+async function launch(options) {
+  for (const name of browserName ? [browserName] : Object.keys(browsers)) {
+    try { return await browsers[name](options); } catch (error) {
+      if (!/is not found at|Executable doesn't exist/.test(error.message)) throw error;
+    }
+  }
+  fail(`${browserName ?? "a browser"} is not installed: install Chrome, or Playwright's Chromium with ` +
+    "npx playwright-core install chromium-headless-shell");
+}
+const project = ["init", "add"].includes(positionals[0]);
+if ((browserName && !browsers[browserName]) || (project ? positionals.length < 2 : positionals.length !== 2)) fail(usage);
+const scope = resolve("node_modules/@daugasauron"), beside = resolve(import.meta.dirname, "../..");
+const hostPackage = name => `@daugasauron/dolly-host-${name === "download" ? "save" : name}`;
+// Pi's packages, which init's recipe installs: those dolly-build depends on, pi-minimal's
+// (scripts/package-npm.mjs), in its order.
+const own = JSON.parse(await readFile(join(import.meta.dirname, "../package.json"), "utf8"));
+const piPackages = Object.keys(own.dependencies ?? {})
+  .filter(dependency => dependency.startsWith("@daugasauron/dolly-package-")).map(dependency => dependency.replace("@daugasauron/dolly-package-", ""));
+
+// init and add install what they name and the project lacks, at dolly-build's version, with
+// the package manager that runs them: the runtime, each PACKAGE, and the device host modules
+// they declare (init: Pi's too, and the keyboard's), which packages.json lists for every
+// published package and image (scripts/package-npm.mjs). add also installs at that version
+// each Dolly package the project depends on and has at another, as after dolly-build's
+// update, and the host modules of the packages its recipe installs and of the image it
+// starts FROM, which a new version may add to.
+if (project) {
+  const { packages: published, images } = JSON.parse(await readFile(join(import.meta.dirname, "../packages.json"), "utf8"));
+  const named = positionals.slice(2), unknown = named.filter(name => !published[name]);
+  if (!validName(positionals[1])) fail(`${positionals[1]}: a name is lowercase letters, digits and dashes, at most 32 bytes`);
+  if (unknown.length) fail(`${unknown.join(", ")}: no such package. The packages: ${Object.keys(published).sort().join(", ")}`);
+  const recipePackages = positionals[0] === "init" ? [] : (await readFile(`Dollyfile-${positionals[1]}`, "utf8").catch(() => ""))
+    .split("\n").map(line => /^(?:FROM|INSTALL)\s+(\S+)/.exec(line)?.[1]).filter(Boolean).map(location => imageFileName(recipeFileName(location)));
+  const hosts = [...positionals[0] === "init" ? ["input", ...piPackages.flatMap(name => published[name])] : [],
+    ...[...named, ...recipePackages].flatMap(name => published[name] ?? images[name] ?? [])];
+  // The project's own Dolly packages that are installed at another version.
+  const manifest = async directory => JSON.parse(await readFile(join(directory, "package.json"), "utf8").catch(() => "{}"));
+  const stale = [];
+  for (const name of positionals[0] === "add" ? Object.keys((await manifest(".")).dependencies ?? {}).filter(name => name.startsWith("@daugasauron/dolly")) : []) {
+    if ((await manifest(join(scope, basename(name)))).version !== own.version) stale.push(name);
+  }
+  const missing = [...new Set([...stale,
+    ...["@daugasauron/dolly", ...hosts.map(hostPackage)].filter(name => !existsSync(join(scope, basename(name)))),
+    ...named.map(name => `@daugasauron/dolly-package-${name}`).filter(name => ![scope, beside].some(parent => existsSync(join(parent, basename(name)))))])];
+  if (missing.length) {
+    const manager = /^(pnpm|yarn)\//.exec(process.env.npm_config_user_agent ?? "")?.[1] ?? "npm";
+    const command = [manager, manager === "npm" ? "install" : "add", manager === "yarn" ? "--exact" : "--save-exact",
+      ...missing.map(name => `${name}@${own.version}`)];
+    console.log(`dolly-build: ${command.join(" ")}`);
+    if (spawnSync(command[0], command.slice(1), { stdio: "inherit" }).status !== 0) fail("that install failed: run it, then this command again");
+  }
+}
+if (!existsSync(join(scope, "dolly"))) fail("run it where node_modules/@daugasauron/dolly is installed");
 
 // The installed image and package directories (dolly-image-NAME, dolly-package-NAME):
 // each holds its own image's packs, and the catalog and metadata of that image's
 // closure. npm installs the packages a package installs beside it, pnpm beside its
-// real path.
+// real path, and those dolly-build depends on beside dolly-build's.
 const directories = [];
-for (const pending = [scope]; pending.length;) {
+for (const pending = [scope, beside]; pending.length;) {
   const parent = pending.pop();
   for (const name of (await readdir(parent)).filter(name => /^dolly-(image|package)-/.test(name))) {
     const directory = await realpath(join(parent, name));
@@ -83,52 +149,79 @@ for (const name of await readdir(join(scope, "dolly/host"))) {
 for (const name of (await readdir(scope)).filter(name => name.startsWith("dolly-host-"))) {
   modules.set(JSON.parse(await readFile(join(scope, name, "module.json"), "utf8")).name, `${name}/module.json`);
 }
-const hostPackage = name => `@daugasauron/dolly-host-${name === "download" ? "save" : name}`;
 
-try {
-  if (init) await scaffold(positionals[1], positionals.slice(2));
-  else await build(...positionals.map(path => resolve(path)));
-} catch (error) {
-  fail(error.message);
-}
-
-// A project: the recipe (FROM pi-minimal at its pin, INSTALL of each PACKAGE,
-// the template's rows, the base's ENTRY) and public/, the page's files: the page,
-// its theme and the workspace.
+// A project: the recipe (INSTALL of Pi's packages and of each PACKAGE at the installed
+// pins, then the template's rows and ENTRY), public/, the page's files (the page, its
+// theme, the workspace and a skill), and the workflow that builds the site in CI.
+const installRow = definition => `INSTALL ${siteReference(definition.dollyfile)} ${definition.sha256}`;
 async function scaffold(name, packages) {
-  if (!validName(name)) fail(`${name}: a name is lowercase letters, digits and dashes, at most 32 bytes`);
-  const base = definitionOf(installed("pi-minimal", "image"));
-  const installs = packages.map(image => definitionOf(installed(image, "package")));
-  // snapshot@0 is the page's Save: a save is kept in the browser, and the page opens it again.
-  const hosts = [...new Set([base, ...installs].flatMap(definition => definition.hostRequirements).concat("snapshot@0"))].sort();
-  const row = (operation, definition) => `${operation} ${siteReference(definition.dollyfile)} ${definition.sha256}\n`;
+  const installs = [...new Set([...piPackages, ...packages])].map(image => definitionOf(installed(image, "package")));
+  // input@0 is the terminal's keyboard; snapshot@0 the page's Save: a save is kept in the
+  // browser, and the page opens it again.
+  const hosts = [...new Set(installs.flatMap(definition => definition.hostRequirements).concat("input@0", "snapshot@0"))].sort();
   const template = async file => (await readFile(join(import.meta.dirname, "dolly-build-init", file), "utf8")).replaceAll("__NAME__", name);
+  const pages = (await readdir(join(import.meta.dirname, "dolly-build-init/public"), { recursive: true, withFileTypes: true }))
+    .filter(entry => entry.isFile()).map(entry => relative(join(import.meta.dirname, "dolly-build-init"), join(entry.parentPath, entry.name)).split(sep).join("/")).sort();
   const files = {
     [`Dollyfile-${name}`]: `DOLLY 7\nAPPLICATION ${name}\n${hosts.map(host => `REQUIRES HOST ${host}\n`).join("")}\n` +
-      row("FROM", base) + installs.map(definition => row("INSTALL", definition)).join("") + "\n" + await template("Dollyfile") +
-      `\nENTRY ${base.entry.join(" ")}\n`,
-    "public/index.html": await template("public/index.html"),
-    "public/theme.json": await template("public/theme.json"),
-    "public/workspace/README.md": await template("public/workspace/README.md"),
+      installs.map(definition => `${installRow(definition)}\n`).join("") + "\n" + await template("Dollyfile"),
+    ...Object.fromEntries(await Promise.all(pages.map(async file => [file, await template(file)]))),
+    [`.github/workflows/${name}.yml`]: await template("workflow.yml"),
   };
   const present = Object.keys(files).filter(file => existsSync(file));
   if (present.length) fail(`${present.join(", ")}: already here`);
-  await mkdir("public/workspace", { recursive: true });
-  for (const [file, text] of Object.entries(files)) await writeFile(file, text);
+  for (const [file, text] of Object.entries(files)) {
+    await mkdir(dirname(file), { recursive: true });
+    await writeFile(file, text);
+  }
   // The site and node_modules are built, not kept: a .gitignore, unless the project has one.
   if (!existsSync(".gitignore")) await writeFile(".gitignore", `node_modules/\n${name}/\n`);
-  const missing = hosts.map(host => host.split("@")[0]).filter(name => !modules.has(name));
   console.log(`dolly-build: wrote ${Object.keys(files).join(", ")}\n` +
-    (missing.length ? `  npm install ${missing.map(hostPackage).join(" ")}   the recipe declares ${missing.join(", ")}\n` : "") +
     `  npx dolly-build Dollyfile-${name} ${name}   builds the image and the site, ${name}/ (again after a change)\n` +
     `  python3 -m http.server -d ${name} -b 127.0.0.1 8080   serves it, as any static host does`);
 }
 
+// Dollyfile-NAME with every FROM and INSTALL row at the pin of the installed package that
+// holds its recipe, as after an update of the packages, and a row for each PACKAGE it
+// lacks, with the host modules those, the packages it installs and the image it starts FROM
+// declare at their new pins: FROM carries a base's files, not its host modules.
+async function add(name, packages) {
+  const file = `Dollyfile-${name}`, lines = (await readFile(file, "utf8").catch(() =>
+    fail(`${file}: no such recipe here (dolly-build init ${name} writes one)`))).split("\n");
+  const held = new Set(), rows = [], repeated = [];
+  let repinned = 0;
+  for (const [index, line] of lines.entries()) {
+    const [, operation, location, pin] = /^(FROM|INSTALL)\s+(\S+)\s+([0-9a-f]{64})\s*$/.exec(line) ?? [];
+    if (!operation) continue;
+    rows.push(index);
+    const directory = directories.find(candidate => candidate.image === imageFileName(recipeFileName(location)));
+    if (!directory) continue;
+    held.add(directory.image);
+    const definition = definitionOf(directory);
+    repeated.push(definition);
+    if (definition.sha256 !== pin) repinned++;
+    lines[index] = `${operation} ${siteReference(definition.dollyfile)} ${definition.sha256}`;
+  }
+  if (!rows.length) fail(`${file}: no FROM or INSTALL row to add after`);
+  const added = [...new Set(packages)].filter(image => !held.has(image)).map(image => definitionOf(installed(image, "package")));
+  lines.splice(rows.at(-1) + 1, 0, ...added.map(installRow));
+  const declared = new Set(lines.map(line => /^REQUIRES HOST (\S+)/.exec(line)?.[1]));
+  const hosts = [...new Set([...repeated, ...added].flatMap(definition => definition.hostRequirements))].filter(host => !declared.has(host));
+  lines.splice(lines.findLastIndex(line => line.startsWith("REQUIRES HOST ")) + 1, 0, ...hosts.map(host => `REQUIRES HOST ${host}`));
+  await writeFile(file, lines.join("\n"));
+  console.log(`dolly-build: ${file}: ${added.length ? `added ${added.map(definition => definition.image).join(", ")}` : "no package added"}` +
+    `${hosts.length ? ` (${hosts.join(", ")})` : ""}; rows at a new pin: ${repinned}\n` +
+    `  npx dolly-build ${file} ${name}   builds it`);
+}
+
 async function build(recipePath, outArgument) {
   const filename = basename(recipePath), source = await readFile(recipePath, "utf8");
   const parsed = inspectDollyfile(source, filename), image = parsed.image;
   if (imageFileName(filename) !== image) fail(`${filename}: name it Dollyfile-${image}`);
-  if (!parsed.from) fail(`${filename}: FROM an installed image`);
+  // The build runs the recipe engine of its base, or of the dollyfile package it installs.
+  const engine = parsed.artifacts.some(({ operation, location }) => operation === "install" &&
+    imageFileName(recipeFileName(location)) === "dollyfile");
+  if (!parsed.from && !engine) fail(`${filename}: FROM an installed image, or INSTALL the dollyfile package`);
   if (!existsSync("public/index.html")) fail("public/index.html: the page of the site (dolly-build init writes one)");
   // A rebuild replaces the site it wrote before, and nothing else.
   const previous = join(outArgument, "image");
@@ -137,7 +230,7 @@ async function build(recipePath, outArgument) {
   }
   // Every row that names an image names one of the installed directories at its pin.
   const holders = parsed.artifacts.map(reference => holder(reference) ?? fail(`${filename}:${reference.line}: no installed ` +
-    `package holds ${reference.location} at that pin (dolly-build init writes the rows of the installed ones)`));
+    `package holds ${reference.location} at that pin (dolly-build add ${image} writes the installed pins)`));
   // One catalog of their closures: the same image at the same pin in each.
   const records = new Map(), staticSources = new Map(), first = new Map();
   for (const directory of holders) {
@@ -164,11 +257,10 @@ async function build(recipePath, outArgument) {
     // modules the image declares, as their packages hold them, and the worker
     // that isolates the page where the host sends no headers.
     await cp("public", staging, { recursive: true });
-    const workspace = join(staging, "workspace");
-    if (existsSync(workspace)) {
-      const files = (await readdir(workspace, { recursive: true, withFileTypes: true })).filter(entry => entry.isFile())
-        .map(entry => relative(workspace, join(entry.parentPath, entry.name)).split(sep).join("/")).sort();
-      await writeFile(join(staging, "workspace.json"), `${JSON.stringify(files)}\n`);
+    for (const tree of ["workspace", "skills"].filter(tree => existsSync(join(staging, tree)))) {
+      const files = (await readdir(join(staging, tree), { recursive: true, withFileTypes: true })).filter(entry => entry.isFile())
+        .map(entry => relative(join(staging, tree), join(entry.parentPath, entry.name)).split(sep).join("/")).sort();
+      await writeFile(join(staging, `${tree}.json`), `${JSON.stringify(files)}\n`);
     }
     for (const name of new Set(["dolly", ...declared.map(module => modules.get(module).split("/")[0])])) {
       await cp(join(scope, name), join(staging, name), { recursive: true, dereference: true });
@@ -194,6 +286,36 @@ async function built(directory, image, source, identities) {
     metadata?.buildId === identities.imageBuildId && metadata?.runtimeBuildId === identities.buildId;
 }
 
+// A request of the build's, which the page's broker admitted, fetched by Node: a host
+// without CORS headers, a redirect between hosts and the environment's proxy (above)
+// work as on any machine. /relay/SCHEME/HOST/PATH?QUERY names it; a build's steps reach
+// any host, as a session's programs do on Dolly's site, and each host is named once.
+const reached = new Set();
+const unsent = new Set(["host", "connection", "origin", "referer", "cookie", "accept-encoding", "content-length"]);
+const unreturned = new Set(["connection", "content-encoding", "content-length", "transfer-encoding", "set-cookie"]);
+async function relay(request, response, url) {
+  const [scheme, ...rest] = url.pathname.slice("/relay/".length).split("/");
+  if (!["http", "https"].includes(scheme) || !rest.length) return response.writeHead(400).end();
+  const target = new URL(`${scheme}://${rest.join("/")}${url.search}`);
+  if (!reached.has(target.origin)) {
+    reached.add(target.origin);
+    process.stderr.write(`dolly-build: the build reaches ${target.origin}\n`);
+  }
+  const headers = Object.entries(request.headers).filter(([name]) => !unsent.has(name) && !name.startsWith("sec-"));
+  const chunks = [];
+  for await (const chunk of request) chunks.push(chunk);
+  try {
+    const answer = await fetch(target, { method: request.method, headers, redirect: "follow",
+      body: ["GET", "HEAD"].includes(request.method) ? undefined : Buffer.concat(chunks) });
+    // x-dolly-relay-url: where the redirects followed ended, for the page (page-image-build.mjs).
+    response.writeHead(answer.status, { ...Object.fromEntries([...answer.headers].filter(([name]) => !unreturned.has(name))),
+      ...isolationHeaders, "x-dolly-relay-url": answer.url });
+    if (answer.body) Readable.fromWeb(answer.body).pipe(response); else response.end();
+  } catch (error) {
+    response.writeHead(502, { "content-type": "text/plain" }).end(`${target.origin}: ${error.cause?.message ?? error.message}\n`);
+  }
+}
+
 // The image directory, laid out as an image package: the closures' metadata and
 // recipes, this recipe beside them, the image's metadata and packs, its package
 // index and its index.mjs. Every installed directory's packs are served for the
@@ -216,7 +338,9 @@ async function buildImage(directory, { filename, source, image, parsed, holders,
   const roots = [["/", scope], ["/image/", directory],
     ...directories.map(installed => ["/image/dist/packs/", join(installed.directory, "dist/packs")])];
   const server = createServer(async (request, response) => {
-    const path = decodeURIComponent(new URL(request.url, "http://127.0.0.1").pathname);
+    const url = new URL(request.url, "http://127.0.0.1");
+    if (url.pathname.startsWith("/relay/")) return relay(request, response, url);
+    const path = decodeURIComponent(url.pathname);
     if (path === "/") return response.writeHead(200, { ...isolationHeaders, "content-type": "text/html" }).end(page);
     for (const [prefix, root] of roots.toReversed().filter(([prefix]) => path.startsWith(prefix))) {
       const file = resolve(root, path.slice(prefix.length));
@@ -228,25 +352,25 @@ async function buildImage(directory, { filename, source, image, parsed, holders,
     response.writeHead(404).end();
   });
   // The browser's own requests (its vendor's updates, clock, autofill) go to a proxy that
-  // refuses them: it reaches this server and the recipes' sources alone.
+  // refuses them: the page reaches this server alone, the build's network through relay().
   const refusal = createServer((request, response) => response.writeHead(403).end()).on("connect", (request, socket) => socket.destroy());
   await Promise.all([server, refusal].map(listener => new Promise(listening => listener.listen(0, "127.0.0.1", listening))));
   const origin = `http://127.0.0.1:${server.address().port}`;
-  // The recipes' own upstream URLs are the only network the build may use.
-  const rules = graph.records.flatMap(record => record.sources.filter(entry => sitePath(entry.location) === null))
-    .map(entry => ({ origin: new URL(entry.location).origin, path: new URL(entry.location).pathname, methods: ["GET"] }));
-  const browser = await browsers[browserName]({ proxy: { server: `http://127.0.0.1:${refusal.address().port}`,
-    bypass: [...new Set(["127.0.0.1", ...rules.map(rule => new URL(rule.origin).hostname)])].join(",") } });
+  // What the build asks of this server through the relay is not a host it reaches.
+  reached.add(origin);
+  // The environment's proxy is the relay's alone: Firefox takes HTTP_PROXY from its
+  // environment over the proxy it is given and sent its page's own address there.
+  const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => !/^(?:https?|all|no)_proxy$/i.test(name)));
+  const browser = await launch({ env, proxy: { server: `http://127.0.0.1:${refusal.address().port}`, bypass: "127.0.0.1" } });
   let inputs;
   try {
     const tab = await browser.newPage();
     tab.setDefaultTimeout(0);
     await tab.exposeFunction("dollyBuildLog", text => void process.stderr.write(text));
-    await tab.addInitScript(rules => { globalThis.DOLLY_HTTP_POLICY = { maxRequests: 256, rules }; }, rules);
     await tab.goto(`${origin}/`);
     ({ inputs } = await tab.evaluate(`(${buildImageInPage})(${JSON.stringify(`${origin}/dolly/`)}, ` +
       `${JSON.stringify(`${origin}/image/`)}, ${JSON.stringify(image)}, undefined, ` +
-      `${JSON.stringify([...modules.values()].map(path => `${origin}/${path}`))})`));
+      `${JSON.stringify([...modules.values()].map(path => `${origin}/${path}`))}, ${JSON.stringify(`${origin}/relay/`)})`));
     await savePageSnapshot(tab, join(directory, `dist/dolly-${image}-system.snapshot`));
   } finally {
     await browser.close();
@@ -276,3 +400,11 @@ async function buildImage(directory, { filename, source, image, parsed, holders,
   await writeFile(join(directory, "amy-index.txt"), amyIndex(catalog.filter(record => held.includes(record.image)), descriptions));
   await writeFile(join(directory, "index.mjs"), indexSource(image, declared.map(name => `../${modules.get(name)}`)));
 }
+
+// Last, once every constant above is initialized: the relay reads them during a build.
+try {
+  if (project) await (positionals[0] === "init" ? scaffold : add)(positionals[1], positionals.slice(2));
+  else await build(...positionals.map(path => resolve(path)));
+} catch (error) {
+  fail(error.message);
+}

package/scripts/dollyfile-graph.mjs

@@ -1,9 +1,9 @@
 import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
-import { loadRecipeGraph } from "../src/dollyfile-graph.mjs";
-import { recipeFileName } from "../src/dollyfile-view.mjs";
+import { loadRecipeGraph } from "../site/dollyfile-graph.mjs";
+import { recipeFileName } from "../site/dollyfile-view.mjs";
 import { recipeFiles } from "./recipe-files.mjs";
-import { siteReference } from "../src/static-asset.mjs";
+import { siteReference } from "../page/static-asset.mjs";
 
 // Loads catalog recipe graphs from a project directory, sharing parsed recipes.
 export function createDollyfileGraphLoader(projectDir) {

package/scripts/image-directory.mjs

@@ -1,16 +1,15 @@
-import { createHash } from "node:crypto";
+import { sha256 } from "./sha256.mjs";
 import { recipeRecords } from "./dollyfile-graph.mjs";
-import { alphabetical } from "../src/alphabetical.mjs";
-import { siteReference } from "../src/static-asset.mjs";
+import { alphabetical } from "../site/alphabetical.mjs";
+import { siteReference } from "../page/static-asset.mjs";
+import { recipesDigest } from "../page/image-inputs.mjs";
 
-// The generated files of an image directory (src/images.mjs): its catalog,
+// The generated files of an image directory (page/images.mjs): its catalog,
 // dist/dolly-images.json, each image's metadata, dist/dolly-IMAGE-system-snapshot.json,
 // its package index amy-index.txt, and the index.mjs of a directory boot() loads.
 // The catalog's build, the npm packages and dolly-build (an npm user's image)
 // write them here.
 
-const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
-
 // An image's catalog record: its recipe, at FILENAME in the directory, and the
 // recipe graph it roots.
 export function catalogRecord({ image, filename, source, parsed }, graph) {
@@ -21,7 +20,7 @@ export function catalogRecord({ image, filename, source, parsed }, graph) {
     dollyfile: filename,
     byteLength: Buffer.byteLength(source),
     sha256: sha256(source),
-    recipes: recipeRecords(graph),
+    recipesSha256: recipesDigest(recipeRecords(graph)),
     artifacts: graph.artifacts,
     hostRequirements: graph.root.hostRequirements,
   };

package/scripts/page-image-build.mjs

@@ -1,22 +1,36 @@
 import { open, rm } from "node:fs/promises";
-import { MAX_SNAPSHOT_BYTES } from "../src/snapshot-records.mjs";
+import { MAX_SNAPSHOT_BYTES } from "../page/snapshot-records.mjs";
 
 // Runs in a page (pass its source to page.evaluate): builds IMAGE of the image
 // directory at IMAGES, or "custom" from CUSTOM, after its missing dependencies,
 // with the builder and HTTP broker of the runtime at RUNTIME and the host
 // modules whose module.json URLs are MODULES; on Dolly's site both directories
-// are the site and the modules every one it builds (host/manifests.mjs). Logs
+// are the site and the modules every one it builds (host/manifests.mjs). The
+// broker admits what POLICY admits (docs/http.md), without one any URL. With
+// RELAY, a URL prefix of the page's own origin, the broker fetches each remote
+// request it admits from RELAY + scheme/host/path?query instead. Logs
 // through the exposed dollyBuildLog(text), keeps the snapshot's Blob in
 // globalThis.dollySnapshot and returns { inputs, dependenciesBuilt }.
-export async function buildImageInPage(runtime, images, image, custom, modules) {
+export async function buildImageInPage(runtime, images, image, custom, modules, relay, policy) {
+  // What the runtime needs of the browser, said before anything is built.
+  if (!crossOriginIsolated) throw new Error("the build page is not cross-origin isolated");
+  if (!WebAssembly.validate(new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 5, 3, 1, 4, 1]))) {
+    throw new Error("this browser has no WebAssembly memory64, which Dolly needs (Chrome 138, Firefox 146 or later)");
+  }
   const log = text => void globalThis.dollyBuildLog(text);
-  const [registry, policy, transport, builder, graph] = await Promise.all([
-    "src/images.mjs", "host/http/policy.mjs", "host/http/local-services.mjs",
-    "src/image-builder.mjs", "src/image-build.mjs",
+  const [registry, http, transport, builder, graph] = await Promise.all([
+    "page/images.mjs", "host/http/policy.mjs", "host/http/local-services.mjs",
+    "site/image-builder.mjs", "site/image-build.mjs",
   ].map(path => import(new URL(path, runtime).href)));
   await registry.loadImages(images);
-  const network = { ...transport.localServicesTransport(new policy.DollyHttpPolicy(globalThis.DOLLY_HTTP_POLICY,
-    registry.bootstrapSources(), new URL(images))), site: images };
+  // A relayed answer is from the URL the relay reached, after its redirects: a program
+  // resolves what it links against that, as without a relay, never against the relay's.
+  const remoteFetch = relay && (async (url, init) => {
+    const response = await fetch(`${relay}${url.protocol.slice(0, -1)}/${url.host}${url.pathname}${url.search}`, init);
+    return Object.defineProperty(response, "url", { value: response.headers.get("x-dolly-relay-url") ?? url.href });
+  });
+  const network = { ...transport.localServicesTransport(new http.DollyHttpPolicy(policy,
+    registry.bootstrapSources(), new URL(images)), {}, remoteFetch), site: images };
   modules ??= (await import(new URL("host/manifests.mjs", runtime).href)).hostModuleUrls;
   const build = (name, artifacts) => builder.buildImage(name, artifacts, { http: { network } }, log,
     { modules, customSource: name === "custom" ? custom : undefined });
@@ -39,8 +53,13 @@ export async function savePageSnapshot(page, output) {
   const file = await open(output, "wx");
   try {
     for (let offset = 0; offset < length; offset += piece) {
-      const bytes = Buffer.from(await page.evaluate(async ([offset, piece]) =>
-        new Uint8Array(await globalThis.dollySnapshot.slice(offset, offset + piece).arrayBuffer()).toBase64(), [offset, piece]), "base64");
+      // A data URL's base64, which every browser the runtime runs in makes.
+      const bytes = Buffer.from(await page.evaluate(([offset, piece]) => new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(reader.result.slice(reader.result.indexOf(",") + 1));
+        reader.onerror = () => reject(reader.error);
+        reader.readAsDataURL(globalThis.dollySnapshot.slice(offset, offset + piece));
+      }), [offset, piece]), "base64");
       await file.write(bytes, 0, bytes.length, offset);
     }
   } catch (error) {

package/scripts/recipe-files.mjs

@@ -1,8 +1,8 @@
 import { readdir } from "node:fs/promises";
 import { resolve } from "node:path";
-import { siteReference } from "../src/static-asset.mjs";
-import { imageFileName } from "../src/dollyfile-view.mjs";
-import { validName } from "../src/image-names.mjs";
+import { siteReference } from "../page/static-asset.mjs";
+import { imageFileName } from "../site/dollyfile-view.mjs";
+import { validName } from "../page/image-names.mjs";
 
 // Recipes are published at their checkout path: core images at the top
 // level, each demo's in demos/DEMO/. Returns site reference ->

package/scripts/release-layout.mjs

@@ -1,7 +1,7 @@
 // A published version's directory and public path: /vX.Y.Z/.
 export const versionName = /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
 
-// The version a packaged site carries in its src/version.mjs, read as data.
+// The version a packaged site carries in its page/version.mjs, read as data.
 export function releaseVersion(source) {
   const name = `v${/^export const DOLLY_VERSION = "([^"]*)";$/m.exec(source)?.[1]}`;
   if (!versionName.test(name)) throw new Error("the release carries no version");

package/scripts/serve.mjs

@@ -4,11 +4,12 @@ import { readFile, readlink } from "node:fs/promises";
 import { createServer } from "node:http";
 import { basename, extname, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
-import { sha256 } from "./snapshot-identity.mjs";
+import { sha256 } from "./sha256.mjs";
 import { releaseVersion, robotsText } from "./release-layout.mjs";
 
 export const mimeTypes = new Map([
   [".html", "text/html; charset=utf-8"],
+  [".css", "text/css; charset=utf-8"],
   [".md", "text/markdown; charset=utf-8"],
   [".txt", "text/plain; charset=utf-8"],
   [".wat", "text/plain; charset=utf-8"],
@@ -22,6 +23,8 @@ export const mimeTypes = new Map([
   [".js", "text/javascript; charset=utf-8"],
   [".mjs", "text/javascript; charset=utf-8"],
   [".wasm", "application/wasm"],
+  [".woff", "font/woff"],
+  [".svg", "image/svg+xml"],
   [".woff2", "font/woff2"],
   [".png", "image/png"],
   [".mp4", "video/mp4"],
@@ -33,12 +36,19 @@ export const isolationHeaders = {
   "cache-control": "no-store",
 };
 const releaseDigest = /^[0-9a-f]{64}$/;
+// The site's Content-Security-Policy, as Pages and this server send it: the
+// strict policy of docs/embedding.md, with the inline styles of its menu and
+// lists, and every destination its default HTTP policy admits.
+export const siteContentSecurityPolicy = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval' blob:; " +
+  "worker-src 'self' blob:; img-src 'self' blob:; style-src 'self' 'unsafe-inline'; connect-src *; " +
+  "object-src 'none'; base-uri 'none'";
 
 // Serves RELEASES/current as it would be deployed: / leads to its version, the
 // release's files are under /vX.Y.Z/, robots.txt speaks for that version, and
-// every other path is 404. Nothing is cacheable: a candidate keeps its version
-// while it changes. dist/ and the source checkout are build inputs, never the
-// running app.
+// every other path is 404. A candidate keeps its version while it changes, so
+// a file is cached only as its digest: the browser asks again each time and
+// keeps what it has while the digest is the same (ETag). dist/ and the source
+// checkout are build inputs, never the running app.
 export function createReleaseServer(releases) {
   let current;
   async function release() {
@@ -55,8 +65,8 @@ export function createReleaseServer(releases) {
           files.has(match[2])) throw new Error("invalid release file manifest");
       files.set(match[2], match[1]);
     }
-    const version = await readFile(resolve(releases, digest, "src/version.mjs"));
-    if (sha256(version) !== files.get("src/version.mjs")) throw new Error("published file changed");
+    const version = await readFile(resolve(releases, digest, "page/version.mjs"));
+    if (sha256(version) !== files.get("page/version.mjs")) throw new Error("published file changed");
     return current = { digest, files, base: `/${releaseVersion(version.toString())}/` };
   }
   return createServer(async (request, response) => {
@@ -84,13 +94,33 @@ export function createReleaseServer(releases) {
         response.writeHead(308, { ...isolationHeaders, location: `${path}/` }).end();
         return;
       }
+      const etag = `"${files.get(file)}"`;
+      if (request.headers["if-none-match"] === etag) {
+        response.writeHead(304, { ...isolationHeaders, "cache-control": "no-cache", etag }).end();
+        return;
+      }
       const body = await readFile(resolve(releases, digest, file));
       if (sha256(body) !== files.get(file)) throw new Error("published file changed");
-      response.writeHead(200, {
-        ...isolationHeaders,
+      const headers = {
+        ...isolationHeaders, "cache-control": "no-cache", etag, "content-security-policy": siteContentSecurityPolicy,
         "content-type": /^Dollyfile(?:-|$)/.test(basename(file)) ? "text/plain; charset=utf-8" :
-          mimeTypes.get(extname(file)) ?? "application/octet-stream",
-      });
+          mimeTypes.get(extname(file)) ?? "application/octet-stream", "accept-ranges": "bytes",
+      };
+      // A video plays from ranges, as the hosts of a deployment serve them.
+      const range = /^bytes=(\d*)-(\d*)$/.exec(request.headers.range ?? "");
+      if (range && (range[1] || range[2])) {
+        const start = range[1] ? Number(range[1]) : Math.max(0, body.length - Number(range[2]));
+        const end = range[1] && range[2] ? Math.min(Number(range[2]), body.length - 1) : body.length - 1;
+        if (start > end) {
+          response.writeHead(416, { ...isolationHeaders, "content-range": `bytes */${body.length}` }).end();
+          return;
+        }
+        response.writeHead(206, { ...headers, "content-range": `bytes ${start}-${end}/${body.length}`,
+          "content-length": end + 1 - start });
+        response.end(request.method === "HEAD" ? undefined : body.subarray(start, end + 1));
+        return;
+      }
+      response.writeHead(200, { ...headers, "content-length": body.length });
       response.end(request.method === "HEAD" ? undefined : body);
     } catch {
       response.writeHead(404, isolationHeaders).end("not found");

package/scripts/sha256.mjs

@@ -0,0 +1,4 @@
+import { createHash } from "node:crypto";
+
+// The lowercase hex SHA-256 of bytes or text, as recipes and records pin them.
+export const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");