@daugasauron/dolly-build 0.0.0-stage → 0.1.5

package/scripts/image-directory.mjs

@@ -0,0 +1,85 @@
+import { createHash } from "node:crypto";
+import { recipeRecords } from "./dollyfile-graph.mjs";
+import { alphabetical } from "../src/alphabetical.mjs";
+import { siteReference } from "../src/static-asset.mjs";
+
+// The generated files of an image directory (src/images.mjs): its catalog,
+// dist/dolly-images.json, each image's metadata, dist/dolly-IMAGE-system-snapshot.json,
+// its package index amy-index.txt, and the index.mjs of a directory boot() loads.
+// The catalog's build, the npm packages and dolly-build (an npm user's image)
+// write them here.
+
+const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
+
+// An image's catalog record: its recipe, at FILENAME in the directory, and the
+// recipe graph it roots.
+export function catalogRecord({ image, filename, source, parsed }, graph) {
+  return {
+    image,
+    role: parsed.role,
+    entry: parsed.entry,
+    dollyfile: filename,
+    byteLength: Buffer.byteLength(source),
+    sha256: sha256(source),
+    recipes: recipeRecords(graph),
+    artifacts: graph.artifacts,
+    hostRequirements: graph.root.hostRequirements,
+  };
+}
+
+// The catalog: its images' records and the files their recipes read by site path.
+export const catalogJson = (images, staticSources = []) => `${JSON.stringify({ images, staticSources }, null, 2)}\n`;
+
+// An image's metadata before it is packed: the snapshot SNAPSHOT (its bytes,
+// decoded as PARSED, with the ENTRY it was checked for), built from GRAPH on
+// INPUTS by the runtime whose identities are BUILD.
+export function snapshotMetadata(image, graph, inputs, snapshot, parsed, entry, build) {
+  return {
+    image,
+    buildId: build.imageBuildId,
+    runtimeBuildId: build.buildId,
+    formatVersion: 2,
+    identityVersion: 2,
+    inputs,
+    recipes: recipeRecords(graph),
+    hostRequirements: graph.root.hostRequirements,
+    entry,
+    manifest: parsed.manifest,
+    byteLength: snapshot.length,
+    sha256: sha256(snapshot),
+  };
+}
+
+export const metadataJson = metadata => `${JSON.stringify(metadata, null, 2)}\n`;
+
+// An image directory's index.mjs, for boot(): the directory, IMAGE, and the
+// manifests of the host modules it declares at PATHS, relative to the directory.
+export const indexSource = (image, paths) => `// The ${image} image for boot(): its directory, and the manifests of the host modules it
+// declares, from the packages that hold them.
+export const images = new URL("./", import.meta.url).href;
+export const image = ${JSON.stringify(image)};
+export const modules = ${JSON.stringify(paths)}
+  .map(path => new URL(path, import.meta.url).href);
+`;
+
+// The package index people and amy read: "NAME RECIPE SHA256 DESCRIPTION" per
+// package, the INSTALL row's operands and the description the menu shows, of
+// PACKAGES ({ image, dollyfile, sha256 } records) described by DESCRIPTIONS.
+// amy prints a description in a terminal: a README's Markdown link becomes its text.
+export const plainText = description => description.replaceAll("`", "").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1");
+export const amyIndex = (packages, descriptions) => [...packages]
+  .sort((left, right) => alphabetical(left.image, right.image))
+  .map(({ image, dollyfile, sha256 }) => `${image} ${siteReference(dollyfile)} ${sha256} ${plainText(descriptions.get(image))}\n`)
+  .join("");
+
+// The packages an image of CATALOG holds, by name: those its build installs
+// (its record's install rows, which name theirs too) and, but in a package, its base's.
+export function heldPackages(catalog, image) {
+  const record = catalog.find(candidate => candidate.image === image);
+  return [...new Set(record.artifacts.flatMap(({ operation, image: name }) => operation === "install" ? [name]
+    : operation === "from" && record.role !== "package" ? heldPackages(catalog, name) : []))];
+}
+
+// The descriptions of an amy-index.txt's packages, by name.
+export const indexDescriptions = text => new Map(text.split("\n").filter(Boolean)
+  .map(line => /^(\S+) \S+ \S+ (.*)$/.exec(line).slice(1)));

package/scripts/page-image-build.mjs

@@ -0,0 +1,53 @@
+import { open, rm } from "node:fs/promises";
+import { MAX_SNAPSHOT_BYTES } from "../src/snapshot-records.mjs";
+
+// Runs in a page (pass its source to page.evaluate): builds IMAGE of the image
+// directory at IMAGES, or "custom" from CUSTOM, after its missing dependencies,
+// with the builder and HTTP broker of the runtime at RUNTIME and the host
+// modules whose module.json URLs are MODULES; on Dolly's site both directories
+// are the site and the modules every one it builds (host/manifests.mjs). Logs
+// through the exposed dollyBuildLog(text), keeps the snapshot's Blob in
+// globalThis.dollySnapshot and returns { inputs, dependenciesBuilt }.
+export async function buildImageInPage(runtime, images, image, custom, modules) {
+  const log = text => void globalThis.dollyBuildLog(text);
+  const [registry, policy, transport, builder, graph] = await Promise.all([
+    "src/images.mjs", "host/http/policy.mjs", "host/http/local-services.mjs",
+    "src/image-builder.mjs", "src/image-build.mjs",
+  ].map(path => import(new URL(path, runtime).href)));
+  await registry.loadImages(images);
+  const network = { ...transport.localServicesTransport(new policy.DollyHttpPolicy(globalThis.DOLLY_HTTP_POLICY,
+    registry.bootstrapSources(), new URL(images))), site: images };
+  modules ??= (await import(new URL("host/manifests.mjs", runtime).href)).hostModuleUrls;
+  const build = (name, artifacts) => builder.buildImage(name, artifacts, { http: { network } }, log,
+    { modules, customSource: name === "custom" ? custom : undefined });
+  let dependenciesBuilt = 0;
+  const dependencies = await graph.prepareImageArtifacts(image, custom, (name, artifacts) => {
+    dependenciesBuilt++;
+    return build(name, artifacts);
+  }, text => log(`${text}\n`));
+  const { bytes, inputs } = await build(image, dependencies);
+  globalThis.dollySnapshot = bytes;
+  return { inputs, dependenciesBuilt };
+}
+
+// Copies the snapshot buildImageInPage left in PAGE to the new file OUTPUT, in
+// pieces. Not by the page's fetch: Playwright keeps the body of every request a
+// page sends, so an upload held the whole snapshot a second time.
+export async function savePageSnapshot(page, output) {
+  const length = await page.evaluate(() => globalThis.dollySnapshot.size), piece = 16 * 1024 * 1024;
+  if (length <= 0 || length > MAX_SNAPSHOT_BYTES) throw new Error("invalid snapshot size");
+  const file = await open(output, "wx");
+  try {
+    for (let offset = 0; offset < length; offset += piece) {
+      const bytes = Buffer.from(await page.evaluate(async ([offset, piece]) =>
+        new Uint8Array(await globalThis.dollySnapshot.slice(offset, offset + piece).arrayBuffer()).toBase64(), [offset, piece]), "base64");
+      await file.write(bytes, 0, bytes.length, offset);
+    }
+  } catch (error) {
+    await rm(output, { force: true });
+    throw error;
+  } finally {
+    await file.close();
+  }
+  return length;
+}

package/scripts/recipe-files.mjs

@@ -0,0 +1,31 @@
+import { readdir } from "node:fs/promises";
+import { resolve } from "node:path";
+import { siteReference } from "../src/static-asset.mjs";
+import { imageFileName } from "../src/dollyfile-view.mjs";
+import { validName } from "../src/image-names.mjs";
+
+// Recipes are published at their checkout path: core images at the top
+// level, each demo's in demos/DEMO/. Returns site reference ->
+// project-relative file. Image names are unique.
+export async function recipeFiles(projectDir) {
+  const files = new Map(), names = new Map();
+  async function scan(directory) {
+    for (const entry of await entries(resolve(projectDir, directory))) {
+      if (!entry.isFile() || !validName(imageFileName(entry.name))) continue;
+      const path = directory === "." ? entry.name : `${directory}/${entry.name}`;
+      if (names.has(entry.name)) throw new Error(`${path}: ${entry.name} is already ${names.get(entry.name)}`);
+      names.set(entry.name, path);
+      files.set(siteReference(path), path);
+    }
+  }
+  await scan(".");
+  for (const demo of await entries(resolve(projectDir, "demos"))) {
+    if (demo.isDirectory()) await scan(`demos/${demo.name}`);
+  }
+  return files;
+}
+
+async function entries(directory) {
+  try { return await readdir(directory, { withFileTypes: true }); }
+  catch (error) { if (error.code === "ENOENT") return []; throw error; }
+}

package/scripts/release-layout.mjs

@@ -0,0 +1,29 @@
+// A published version's directory and public path: /vX.Y.Z/.
+export const versionName = /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
+
+// The version a packaged site carries in its src/version.mjs, read as data.
+export function releaseVersion(source) {
+  const name = `v${/^export const DOLLY_VERSION = "([^"]*)";$/m.exec(source)?.[1]}`;
+  if (!versionName.test(name)) throw new Error("the release carries no version");
+  return name;
+}
+
+export function compareVersions(left, right) {
+  const [a, b] = [left, right].map(name => versionName.exec(name).slice(1).map(Number));
+  return a[0] - b[0] || a[1] - b[1] || a[2] - b[2];
+}
+
+// The robots.txt at a host's root, for the versions it serves: the newest
+// version's llms.txt as comments, without its Markdown marks and with its
+// links as that version's paths, then one rule per version for its bulk
+// (dist/: packs, sources, the runtime). A rule is an exact prefix, which
+// every crawler reads and which matches no page.
+export function robotsText(about, names) {
+  const base = `https://site.invalid/${[...names].sort(compareVersions).at(-1)}/`;
+  const comments = about.trimEnd().split("\n").map(line => `# ${line.replace(/^(#+|>) ?/, "")
+    .replace(/\[([^\]]+)\]\((\S+)\)/g, (_, label, target) => {
+      const url = new URL(target, base);
+      return `${label} ${url.origin === "https://site.invalid" ? url.pathname + url.hash : url.href}`;
+    })}`.trimEnd());
+  return [...comments, "", "User-agent: *", ...names.map(name => `Disallow: /${name}/dist/`), ""].join("\n");
+}

package/scripts/serve.mjs

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+
+import { readFile, readlink } from "node:fs/promises";
+import { createServer } from "node:http";
+import { basename, extname, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { sha256 } from "./snapshot-identity.mjs";
+import { releaseVersion, robotsText } from "./release-layout.mjs";
+
+export const mimeTypes = new Map([
+  [".html", "text/html; charset=utf-8"],
+  [".md", "text/markdown; charset=utf-8"],
+  [".txt", "text/plain; charset=utf-8"],
+  [".wat", "text/plain; charset=utf-8"],
+  [".h", "text/plain; charset=utf-8"],
+  [".c", "text/plain; charset=utf-8"],
+  [".cpp", "text/plain; charset=utf-8"],
+  [".patch", "text/plain; charset=utf-8"],
+  [".py", "text/plain; charset=utf-8"],
+  [".json", "application/json; charset=utf-8"],
+  [".sh", "text/plain; charset=utf-8"],
+  [".js", "text/javascript; charset=utf-8"],
+  [".mjs", "text/javascript; charset=utf-8"],
+  [".wasm", "application/wasm"],
+  [".woff2", "font/woff2"],
+  [".png", "image/png"],
+  [".mp4", "video/mp4"],
+]);
+export const isolationHeaders = {
+  "cross-origin-opener-policy": "same-origin",
+  "cross-origin-embedder-policy": "require-corp",
+  "cross-origin-resource-policy": "same-origin",
+  "cache-control": "no-store",
+};
+const releaseDigest = /^[0-9a-f]{64}$/;
+
+// Serves RELEASES/current as it would be deployed: / leads to its version, the
+// release's files are under /vX.Y.Z/, robots.txt speaks for that version, and
+// every other path is 404. Nothing is cacheable: a candidate keeps its version
+// while it changes. dist/ and the source checkout are build inputs, never the
+// running app.
+export function createReleaseServer(releases) {
+  let current;
+  async function release() {
+    const digest = await readlink(resolve(releases, "current"));
+    if (current?.digest === digest) return current;
+    if (!releaseDigest.test(digest)) throw new Error("invalid release ID");
+    const manifest = await readFile(resolve(releases, digest, "release/files.sha256"), "utf8");
+    if (sha256(manifest) !== digest) throw new Error("release manifest changed");
+    const files = new Map();
+    for (const row of manifest.trimEnd().split("\n")) {
+      const match = /^([0-9a-f]{64})  (.+)$/.exec(row);
+      if (!match || /[\\\0]/.test(match[2]) ||
+          match[2].split("/").some(part => !part || part === "." || part === "..") ||
+          files.has(match[2])) throw new Error("invalid release file manifest");
+      files.set(match[2], match[1]);
+    }
+    const version = await readFile(resolve(releases, digest, "src/version.mjs"));
+    if (sha256(version) !== files.get("src/version.mjs")) throw new Error("published file changed");
+    return current = { digest, files, base: `/${releaseVersion(version.toString())}/` };
+  }
+  return createServer(async (request, response) => {
+    try {
+      if (!["GET", "HEAD"].includes(request.method)) throw new Error("unsupported method");
+      const { digest, files, base } = await release();
+      const path = decodeURIComponent(new URL(request.url, "http://127.0.0.1").pathname);
+      if (path === "/" || path === "/llms.txt") {
+        response.writeHead(302, { ...isolationHeaders, location: base + path.slice(1) }).end();
+        return;
+      }
+      if (path === "/robots.txt") {
+        const about = await readFile(resolve(releases, digest, "llms.txt"));
+        if (sha256(about) !== files.get("llms.txt")) throw new Error("published file changed");
+        response.writeHead(200, { ...isolationHeaders, "content-type": mimeTypes.get(".txt") });
+        response.end(robotsText(about.toString(), [base.slice(1, -1)]));
+        return;
+      }
+      if (!path.startsWith(base)) throw new Error("unversioned path");
+      // A directory's address ends in a slash, as its page's relative links expect.
+      const relative = path.slice(base.length), page = `${relative}${relative && !relative.endsWith("/") ? "/" : ""}index.html`;
+      const file = files.has(relative) ? relative : page;
+      if (!files.has(file)) throw new Error("not published");
+      if (file === page && !path.endsWith("/")) {
+        response.writeHead(308, { ...isolationHeaders, location: `${path}/` }).end();
+        return;
+      }
+      const body = await readFile(resolve(releases, digest, file));
+      if (sha256(body) !== files.get(file)) throw new Error("published file changed");
+      response.writeHead(200, {
+        ...isolationHeaders,
+        "content-type": /^Dollyfile(?:-|$)/.test(basename(file)) ? "text/plain; charset=utf-8" :
+          mimeTypes.get(extname(file)) ?? "application/octet-stream",
+      });
+      response.end(request.method === "HEAD" ? undefined : body);
+    } catch {
+      response.writeHead(404, isolationHeaders).end("not found");
+    }
+  });
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const releases = process.argv[2] ? resolve(process.argv[2]) : resolve(import.meta.dirname, "../build/releases");
+  try {
+    if (!releaseDigest.test(await readlink(resolve(releases, "current")))) throw new Error("invalid current release");
+  } catch {
+    throw new Error("No published Dolly app. Run npm run publish after building; failed builds leave it untouched.");
+  }
+  const server = createReleaseServer(releases);
+  server.listen(Number(process.env.DOLLY_PORT ?? 8080), "127.0.0.1", () => {
+    console.log(`dolly: http://127.0.0.1:${server.address().port}/`);
+  });
+}

package/scripts/share-pages-snapshots.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+import { createHash, randomUUID } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { createGzip } from "node:zlib";
+import { decodeSnapshotRecords, validateSnapshotPacks, MAX_SNAPSHOT_BYTES } from "../src/snapshot-records.mjs";
+import { metadataJson } from "./image-directory.mjs";
+
+// A record is either decoded (`data`) or indexed in a snapshot file on disk
+// (`byteLength`, `source`, `offset`); the catalog is never held in memory.
+const CHUNK_BYTES = 4 * 1024 * 1024;
+const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+const digest = bytes => createHash("sha256").update(bytes).digest("hex");
+const recordLength = record => record.byteLength ?? record.data.length;
+const recordKey = (kind, path) => createHash("sha256").update(`${kind}\0${path}\0`);
+
+export function splitSnapshotRecords(records) {
+  const entries = [...records].map(([path, record]) => ({ path, record, encodedPath: Buffer.from(path) }))
+    .sort((left, right) => Buffer.compare(left.encodedPath, right.encodedPath));
+  const parts = [];
+  let part = new Map(), size = 16;
+  const finish = () => {
+    if (part.size) parts.push(part);
+    part = new Map(); size = 16;
+  };
+  for (const { path, record, encodedPath } of entries) {
+    const length = 16 + encodedPath.length + recordLength(record);
+    if (size + length > 4 * 1024 * 1024) finish();
+    part.set(path, record);
+    size += length;
+    // Large files stand alone. Path landmarks let later packs stabilize after
+    // small edits without tying their boundaries to the catalog's image names.
+    if (size >= 4 * 1024 * 1024 || (size >= 2 * 1024 * 1024 &&
+        (createHash("sha256").update(encodedPath).digest()[0] & 31) === 0)) finish();
+  }
+  finish();
+  return parts;
+}
+
+function snapshotHeader(count) {
+  const header = Buffer.alloc(16, "DOLLYSNP", "latin1");
+  header.writeUInt32LE(2, 8);
+  header.writeUInt32LE(count, 12);
+  return header;
+}
+
+function recordHeader(kind, encodedPath, byteLength) {
+  const header = Buffer.alloc(16 + encodedPath.length);
+  header.writeUInt32LE(kind, 0);
+  header.writeUInt32LE(encodedPath.length, 4);
+  header.writeBigUInt64LE(BigInt(byteLength), 8);
+  header.set(encodedPath, 16);
+  return header;
+}
+
+async function readExact(handle, buffer, position, length) {
+  let done = 0;
+  while (done < length) {
+    const { bytesRead } = await handle.read(buffer, done, length - done, position + done);
+    if (!bytesRead) throw new Error("truncated snapshot record");
+    done += bytesRead;
+  }
+  return buffer.subarray(0, length);
+}
+
+async function* indexedData(record) {
+  const hash = recordKey(record.kind, record.path);
+  for (let done = 0; done < record.byteLength;) {
+    const length = Math.min(CHUNK_BYTES, record.byteLength - done);
+    const chunk = await readExact(record.source.handle, Buffer.allocUnsafe(length), record.offset + done, length);
+    hash.update(chunk);
+    done += length;
+    yield chunk;
+  }
+  if (hash.digest("hex") !== record.key) throw new Error(`${record.source.name}: packed record mismatch: ${record.path}`);
+}
+
+async function* decodedData(record) { yield record.data; }
+
+// Encodes like encodeSnapshotRecords, as chunks of at least CHUNK_BYTES.
+async function* snapshotChunks(records, readData) {
+  const entries = [...records].map(([path, record]) => ({ encodedPath: Buffer.from(path), record }))
+    .sort((left, right) => Buffer.compare(left.encodedPath, right.encodedPath));
+  let pending = [snapshotHeader(entries.length)], size = pending[0].length;
+  for (const { encodedPath, record } of entries) {
+    pending.push(recordHeader(record.kind, encodedPath, recordLength(record)));
+    size += pending.at(-1).length;
+    for await (const chunk of readData(record)) {
+      pending.push(chunk);
+      size += chunk.length;
+      if (size >= CHUNK_BYTES) { yield Buffer.concat(pending); pending = []; size = 0; }
+    }
+  }
+  if (size) yield Buffer.concat(pending);
+}
+
+async function writeSnapshotPack(directory, records, readData) {
+  const hash = createHash("sha256"), temporary = resolve(directory, "packs", `${randomUUID()}.tmp`);
+  let byteLength = 0;
+  try {
+    await pipeline(async function* () {
+      for await (const chunk of snapshotChunks(records, readData)) {
+        hash.update(chunk);
+        byteLength += chunk.length;
+        yield chunk;
+      }
+    }, createGzip({ level: 6 }), createWriteStream(temporary));
+    const sha256 = hash.digest("hex"), path = resolve(directory, "packs", `${sha256}.snapshot.gz`);
+    await rename(temporary, path);
+    return { sha256, byteLength, encodedByteLength: (await stat(path)).size };
+  } finally { await rm(temporary, { force: true }); }
+}
+
+export async function ensureSnapshotPacks(directory, metadata) {
+  if (metadata.encoding === "packs") {
+    const present = await Promise.all(validateSnapshotPacks(metadata).map(async pack => {
+      try { return (await stat(resolve(directory, "packs", `${pack.sha256}.snapshot.gz`))).size === pack.encodedByteLength; }
+      catch (error) { if (error.code === "ENOENT") return false; throw error; }
+    }));
+    if (present.every(Boolean)) return metadata;
+  }
+  const bytes = await readFile(resolve(directory, `dolly-${metadata.image}-system.snapshot`));
+  if (bytes.length !== metadata.byteLength || digest(bytes) !== metadata.sha256) throw new Error("snapshot mismatch");
+  await mkdir(resolve(directory, "packs"), { recursive: true });
+  const packs = [];
+  for (const records of splitSnapshotRecords(decodeSnapshotRecords(bytes))) {
+    packs.push(await writeSnapshotPack(directory, records, decodedData));
+  }
+  const result = { ...metadata, encoding: "packs", packs };
+  validateSnapshotPacks(result);
+  return result;
+}
+
+// Reads the snapshot once, sequentially, keeping one chunk and the record
+// index: path, kind, size, file offset and content key. The structure is
+// validated in full by the acceptance step, which merges the packs again.
+async function indexSnapshot(source) {
+  const { handle, metadata } = source;
+  if ((await handle.stat()).size !== metadata.byteLength) throw new Error(`${source.name}: snapshot mismatch`);
+  const fileHash = createHash("sha256"), buffer = Buffer.allocUnsafe(CHUNK_BYTES), records = [];
+  let position = 0;
+  const read = async (length, hash) => {
+    const bytes = await readExact(handle, buffer, position, length);
+    fileHash.update(bytes);
+    hash?.update(bytes);
+    position += length;
+    return bytes;
+  };
+  const header = await read(16);
+  const count = header.readUInt32LE(12);
+  if (header.toString("latin1", 0, 8) !== "DOLLYSNP" || header.readUInt32LE(8) !== 2 || count === 0 || count > 100_000) {
+    throw new Error(`${source.name}: invalid Dolly snapshot header`);
+  }
+  for (let index = 0; index < count; index += 1) {
+    const head = await read(16);
+    const kind = head.readUInt32LE(0), pathLength = head.readUInt32LE(4), byteLength = Number(head.readBigUInt64LE(8));
+    if (kind < 1 || kind > 3 || pathLength < 2 || pathLength >= 4096 || byteLength > MAX_SNAPSHOT_BYTES ||
+        position > metadata.byteLength - pathLength - byteLength) throw new Error(`${source.name}: invalid snapshot record size`);
+    const path = decoder.decode(await read(pathLength)), hash = recordKey(kind, path), offset = position;
+    for (let done = 0; done < byteLength; done += Math.min(CHUNK_BYTES, byteLength - done)) await read(Math.min(CHUNK_BYTES, byteLength - done), hash);
+    records.push({ key: hash.digest("hex"), path, kind, byteLength, source, offset });
+  }
+  if (position !== metadata.byteLength) throw new Error(`${source.name}: trailing snapshot bytes`);
+  if (fileHash.digest("hex") !== metadata.sha256) throw new Error(`${source.name}: snapshot mismatch`);
+  return records;
+}
+
+export async function shareSnapshots(directory, snapshots) {
+  const images = [], identical = new Map();
+  try {
+    for (const name of (await readdir(directory)).sort()) {
+      if (!/^dolly-.+-system-snapshot\.json$/.test(name)) continue;
+      const metadata = JSON.parse(await readFile(resolve(directory, name), "utf8"));
+      const image = { metadata, name, packs: [], records: 0, packed: 0,
+        handle: await open(resolve(snapshots, `dolly-${metadata.image}-system.snapshot`)) };
+      images.push(image);
+      for (const record of await indexSnapshot(image)) {
+        image.records += 1;
+        let shared = identical.get(record.key);
+        if (!shared) { shared = { ...record, images: [] }; identical.set(record.key, shared); }
+        shared.images.push(image);
+      }
+    }
+    // Group identical records by the images that need them. This keeps request
+    // counts small while each file's bytes appear in exactly one published pack.
+    const groups = new Map();
+    for (const shared of identical.values()) {
+      const key = shared.images.map(image => image.metadata.image).join(",");
+      if (!groups.has(key)) groups.set(key, { images: shared.images, records: new Map() });
+      groups.get(key).records.set(shared.path, shared);
+    }
+    await mkdir(resolve(directory, "packs"), { recursive: true });
+    const packs = new Set();
+    let packedBytes = 0;
+    for (const group of groups.values()) {
+      for (const records of splitSnapshotRecords(group.records)) {
+        const pack = await writeSnapshotPack(directory, records, indexedData);
+        packs.add(pack.sha256);
+        packedBytes += pack.encodedByteLength;
+        for (const image of group.images) {
+          image.packs.push(pack);
+          image.packed += records.size;
+        }
+      }
+    }
+    for (const image of images) {
+      if (image.packed !== image.records) throw new Error(`${image.name}: packed reconstruction mismatch`);
+      image.packs.sort((left, right) => left.sha256.localeCompare(right.sha256));
+      const metadata = { ...image.metadata, encoding: "packs", packs: image.packs };
+      validateSnapshotPacks(metadata);
+      await writeFile(resolve(directory, image.name), metadataJson(metadata));
+    }
+    console.log(`dolly: ${images.length} images share ${packs.size} packs (${packedBytes} compressed bytes)`);
+    return { images: images.length, packs: packs.size, packedBytes };
+  } finally { await Promise.all(images.map(image => image.handle.close())); }
+}
+if (process.argv[1] === import.meta.filename) await shareSnapshots(resolve(process.argv[2]), resolve(process.argv[3]));

package/scripts/snapshot-identity.mjs

@@ -0,0 +1,36 @@
+import { createHash } from "node:crypto";
+import { recipeRecords } from "./dollyfile-graph.mjs";
+import { decodeSnapshotEnvironment, validateSnapshotEntry } from "./system-snapshot-format.mjs";
+
+const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+export const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
+
+export function verifySnapshotIdentity(definition, graph, parsed, processContract, processAbiDigest) {
+  const image = definition.image;
+  const recipes = recipeRecords(graph);
+  const selected = decoder.decode(parsed.files.get("/etc/dolly/image") ?? new Uint8Array());
+  if (selected !== image) throw new Error(`snapshot selected image ${selected}, expected ${image}`);
+  const expectedLock = "DOLLY-RECIPES 1\n" +
+    recipes.map((recipe) => `${recipe.locator} ${recipe.sha256}\n`).join("");
+  const actualLock = decoder.decode(parsed.files.get("/etc/dolly/recipes.lock") ?? new Uint8Array());
+  if (actualLock !== expectedLock) throw new Error("snapshot recipe lock does not match source");
+  const sourceByPath = new Map(graph.records.map((record) => [record.location, record.source]));
+  for (const recipe of recipes) {
+    const embedded = parsed.files.get(recipe.retainedPath);
+    if (!embedded || sha256(embedded) !== recipe.sha256 ||
+        decoder.decode(embedded) !== sourceByPath.get(recipe.sourcePath)) {
+      throw new Error(`snapshot did not retain the exact ${recipe.sourcePath}`);
+    }
+  }
+  const selectedBytes = parsed.files.get("/etc/dolly/Dollyfile");
+  if (!selectedBytes || decoder.decode(selectedBytes) !== definition.source) {
+    throw new Error("snapshot canonical Dollyfile does not match the selected recipe");
+  }
+  // Runtime ordering determines ENV, not the advisory source inspection graph.
+  decodeSnapshotEnvironment(parsed.files.get("/etc/dolly/environment"));
+  const entry = validateSnapshotEntry(parsed, processContract, processAbiDigest);
+  if (JSON.stringify(entry) !== JSON.stringify(definition.parsed.entry)) {
+    throw new Error("snapshot ENTRY does not match the selected recipe");
+  }
+  return entry;
+}