@daugasauron/dolly 0.1.4 → 0.1.5

package/src/image-builder.mjs

@@ -0,0 +1,63 @@
+import { buildHost, hostRegistry } from "../host/modules.mjs";
+import { hostRequirement } from "../host/requirements.mjs";
+import { imageBase, imageDefinition } from "./images.mjs";
+import { inspectDollyfile } from "./dollyfile-view.mjs";
+
+// Disposable Wasm userspace on the runtime the image declares, with the build
+// host's modules, configured as the caller's host says (host.builder): the
+// caller's browser policy and no display, file picker, local service or ENTRY.
+// MODULES are the URLs of the module.json of each host module the page has.
+// Used for dependencies and Studio builds. ARTIFACTS are the build's inputs,
+// each one's bytes a Blob the Worker reads; it resolves to the built artifact,
+// verified and in the image cache.
+export async function buildImage(image, artifacts, configuration, report, { modules, customSource, signal } = {}) {
+  signal?.throwIfAborted();
+  const declared = image === "custom" ? inspectDollyfile(customSource).hostRequirements
+    : imageDefinition(image).hostRequirements;
+  let worker;
+  // The build registers the build host and every module the image and its
+  // inputs declare: it enables only the runtime and the build host, but a step
+  // may run a program linked against a declared module's client (transfer's
+  // download --help), which the Worker admits by that module's ABI.
+  const needed = new Set([...buildHost, ...declared, ...artifacts.flatMap(artifact => artifact.hostRequirements)]
+    .map(value => hostRequirement(value).name));
+  const manifests = await Promise.all(modules.map(async url => ({ url, ...(await import(url, { with: { type: "json" } })).default })));
+  const registry = await hostRegistry(manifests.filter(({ name }) => needed.has(name)).map(({ url }) => url));
+  const host = await registry.createHost("browser", registry.buildHostFor(declared),
+    { send: message => worker.postMessage(message), configuration });
+  let abort, result;
+  const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
+  try {
+    signal?.throwIfAborted();
+    worker = new Worker(new URL("./build-worker.mjs", import.meta.url), {
+      type: "module", name: `dolly-build-${image}`,
+    });
+    result = await new Promise((resolve, reject) => {
+      abort = () => reject(signal.reason);
+      signal?.addEventListener("abort", abort, { once: true });
+      worker.addEventListener("error", event => reject(new Error(event.message || "Image build worker failed")), { once: true });
+      worker.addEventListener("message", ({ data: message }) => {
+        try {
+          void host.handle(message).catch(reject);
+          if (message.type === "bootstrap") report(message.text);
+          else if (message.type === "bootstrap-bytes") report(decoder.decode(message.bytes, { stream: true }));
+          else if (message.type === "image-built") resolve(message.artifact);
+          else if (message.type === "error") reject(new Error(message.message));
+        } catch (error) { reject(error); }
+      });
+      // An input's Blob is cloned by reference: every Worker reads the one copy.
+      worker.postMessage({ type: "configure", mode: "rebuild", image, buildOnly: true,
+        artifacts: artifacts.map(({ buildId, recipeSha256, sha256, bytes }) => ({ buildId, recipeSha256, sha256, bytes })),
+        images: imageBase.href, modules: registry.urls, hostModules: host.enabled, hostConfiguration: host.configuration,
+        ...(customSource === undefined ? {} : { customSource }) });
+    });
+  } finally {
+    signal?.removeEventListener("abort", abort);
+    host.dispose();
+    worker?.terminate();
+    const tail = decoder.decode();
+    if (tail && !signal?.aborted) report(tail);
+  }
+  signal?.throwIfAborted();
+  return result;
+}

package/src/images.mjs

@@ -1,10 +1,21 @@
 // The images a page runs and where their files are. An image directory holds
-// dist/dolly-images.mjs (its catalog), each image's metadata and packs under
+// dist/dolly-images.json (its catalog), each image's metadata and packs under
 // dist/, and the recipes and sources at their site paths. The site is one such
 // directory; an embedding names its own (boot's `images`). The page and the
 // Worker each load it once, before anything reads it.
 export let DOLLY_IMAGES = [], DOLLY_STATIC_SOURCES = [], imageBase;
 export async function loadImages(base) {
   imageBase = new URL(base);
-  ({ DOLLY_IMAGES, DOLLY_STATIC_SOURCES } = await import(new URL("dist/dolly-images.mjs", imageBase).href));
+  ({ images: DOLLY_IMAGES, staticSources: DOLLY_STATIC_SOURCES } =
+    (await import(new URL("dist/dolly-images.json", imageBase).href, { with: { type: "json" } })).default);
 }
+
+// The catalog's record of an image, by name; undefined for one it lacks.
+export const imageDefinition = name => DOLLY_IMAGES.find(definition => definition.image === name);
+
+// The files of the directory a build reads by site path, with their sizes:
+// every recipe and every source (http@0's trusted sources).
+export const bootstrapSources = () => [
+  ...DOLLY_IMAGES.map(definition => ({ path: `/${definition.dollyfile}`, byteLength: definition.byteLength })),
+  ...DOLLY_STATIC_SOURCES,
+];

package/src/pack-store.mjs

@@ -0,0 +1,141 @@
+// The packs this browser has read, kept in origin-private storage so that an
+// image starts again, and a session saved on it opens, without downloading
+// them. A pack is a file named by its SHA-256, written as its bytes stream to
+// the reader and kept only once the reader has checked them against that
+// digest; a kept pack is checked the same way at every use, and removed when
+// it fails. Every image read leaves a record of its packs; once no image is
+// being read on the origin, what the records of this runtime's images do not
+// name is removed. Only the page and its Workers write here, from an image's
+// trusted metadata; nothing of the guest reaches it.
+import { DOLLY_IMAGE_BUILD_ID } from "../dist/dolly-image-build-id.mjs";
+import { imageBase, imageDefinition } from "./images.mjs";
+import { sha256 } from "./static-asset.mjs";
+
+const STORE = "dolly-packs", LOCK = "dolly-packs";
+const encoder = new TextEncoder();
+
+async function storeDirectories() {
+  const root = await (await navigator.storage.getDirectory()).getDirectoryHandle(STORE, { create: true });
+  const [packs, records] = await Promise.all(["packs", "images"].map(name => root.getDirectoryHandle(name, { create: true })));
+  return { packs, records };
+}
+
+// Holds the store's lock in MODE until the function it resolves to is
+// called, which resolves once the lock is released.
+const hold = mode => new Promise((held, refused) => {
+  const released = navigator.locks.request(LOCK, { mode }, () => new Promise(release => held(() => {
+    release();
+    return released;
+  })));
+  released.catch(refused);
+});
+
+async function writeFile(directory, name, data) {
+  const writable = await (await directory.getFileHandle(name, { create: true })).createWritable();
+  await writable.write(data);
+  await writable.close();
+}
+
+// Where the browser refuses origin-private storage or Web Locks (and in
+// Node), every pack is downloaded.
+const noStore = {
+  kept: async () => null,
+  keeping: async (pack, body) => ({ body, keep() {}, discard() {} }),
+  close() {},
+};
+
+// The store as the reader of IMAGE's PACKS uses it: kept(pack) is a pack's
+// part read from here, or null; keeping(pack, body) the part of a download,
+// written here as it is read. A part's keep() is called once its bytes match
+// the pack's digest, discard() otherwise. close(complete) ends the reading;
+// while it lasts, nothing is removed.
+export async function openPackStore(image, packs) {
+  let store, release;
+  try {
+    store = await storeDirectories();
+    release = await hold("shared");
+    const record = { directory: imageBase.href, image, buildId: DOLLY_IMAGE_BUILD_ID, packs: packs.map(pack => pack.sha256) };
+    await writeFile(store.records, await sha256(encoder.encode(`${record.directory}\0${image}`)), JSON.stringify(record));
+  } catch {
+    release?.();
+    return noStore;
+  }
+  return {
+    async kept(pack) {
+      try {
+        const file = await (await store.packs.getFileHandle(pack.sha256)).getFile();
+        if (file.size !== pack.encodedByteLength) return null;
+        return { body: file.stream(), keep() {}, discard: () => store.packs.removeEntry(pack.sha256).catch(() => {}) };
+      } catch { return null; }
+    },
+    async keeping(pack, body) {
+      // The file holds what was written only once it is closed. A write that
+      // fails, such as one over the quota, leaves the pack unkept.
+      let writable = await store.packs.getFileHandle(pack.sha256, { create: true })
+        .then(handle => handle.createWritable()).catch(() => null);
+      const discard = async () => {
+        const unkept = writable;
+        writable = null;
+        await unkept?.abort().catch(() => {});
+      };
+      return {
+        body: body.pipeThrough(new TransformStream({
+          async transform(chunk, controller) {
+            await writable?.write(chunk).catch(discard);
+            controller.enqueue(chunk);
+          },
+        })),
+        keep: () => writable?.close().catch(discard),
+        discard,
+      };
+    },
+    close(complete) {
+      void release().then(() => complete &&
+        navigator.locks.request(LOCK, { ifAvailable: true }, lock => lock && prune(store))).catch(() => {});
+    },
+  };
+}
+
+// Removes the records of images no page of this runtime starts (another
+// runtime's, and those this directory's catalog no longer has), then every
+// pack the remaining records do not name. An image rebuilt since it was last
+// read keeps its former packs until it is read again.
+async function prune({ packs, records }) {
+  const used = new Set(), stale = [], unused = [];
+  for await (const [name, handle] of records) {
+    try {
+      const record = JSON.parse(await (await handle.getFile()).text());
+      if (record.buildId === DOLLY_IMAGE_BUILD_ID && (record.directory !== imageBase.href || imageDefinition(record.image))) {
+        for (const pack of record.packs) used.add(pack);
+        continue;
+      }
+    } catch {}
+    stale.push(name);
+  }
+  for (const name of stale) await records.removeEntry(name).catch(() => {});
+  for await (const name of packs.keys()) if (!used.has(name)) unused.push(name);
+  for (const name of unused) await packs.removeEntry(name).catch(() => {});
+}
+
+// What the store holds: its packs and their bytes.
+export async function packStoreSize() {
+  let count = 0, bytes = 0;
+  try {
+    const store = await (await navigator.storage.getDirectory()).getDirectoryHandle(STORE);
+    for await (const handle of (await store.getDirectoryHandle("packs")).values()) {
+      count += 1;
+      bytes += (await handle.getFile()).size;
+    }
+  } catch (error) {
+    if (error.name !== "NotFoundError") throw error;
+  }
+  return { count, bytes };
+}
+
+// Removes the store once no image is being read on the origin.
+export async function clearPackStore() {
+  await navigator.locks.request(LOCK, async () =>
+    (await navigator.storage.getDirectory()).removeEntry(STORE, { recursive: true }).catch(error => {
+      if (error.name !== "NotFoundError") throw error;
+    }));
+}

package/src/packaged-snapshot.mjs

@@ -1,13 +1,16 @@
 // The images of an image directory as the page and the Worker load them: each
 // one's metadata, checked against the catalog and the recipes the directory
-// serves, and its packs, streamed or whole.
+// serves, and its packs: streamed into a booting kernel, or merged in path
+// order into the image's canonical snapshot.
 import { DOLLY_IMAGE_BUILD_ID } from "../dist/dolly-image-build-id.mjs";
 import { DOLLY_IMAGES, imageBase } from "./images.mjs";
 import { imageInputs, imageInputsMatch } from "./image-inputs.mjs";
 import { hostRequirements } from "../host/requirements.mjs";
 import { siteReference, sitePath, decodeStaticAsset, sha256 } from "./static-asset.mjs";
-import { mergeSnapshotRecords, validateSnapshotPacks, MAX_SNAPSHOT_BYTES as snapshotSizeLimit } from "./snapshot-records.mjs";
+import { scanSnapshotRecords, validateSnapshotPacks, MAX_SNAPSHOT_BYTES as snapshotSizeLimit } from "./snapshot-records.mjs";
 import { unretainedPath } from "./image-names.mjs";
+import { openPackStore } from "./pack-store.mjs";
+import { Sha256 } from "./sha256.mjs";
 const definitionOf = image => DOLLY_IMAGES.find(definition => definition.image === image);
 const encoder = new TextEncoder();
 const expectedRecipes = image => definitionOf(image).recipes;
@@ -20,50 +23,74 @@ function validSnapshotPath(path) {
     !unretainedPath(path);
 }
 
-async function verifyVisibleRecipes(recipes) {
-  for (const recipe of recipes) {
+// The recipes the directory serves for IMAGE (its page shows them) are the ones
+// that built the image a boot runs. They are fetched together.
+export async function verifyVisibleRecipes(image) {
+  await Promise.all(expectedRecipes(image).map(async recipe => {
     const response = await fetch(new URL(sitePath(recipe.sourcePath).slice(1), imageBase), {
-      cache: "no-store", credentials: "same-origin", redirect: "error",
+      credentials: "same-origin", redirect: "error",
     });
     if (!response.ok) throw new Error(`${recipe.sourcePath} returned HTTP ${response.status}`);
     const bytes = await response.arrayBuffer();
     if (bytes.byteLength !== recipe.byteLength || await sha256(bytes) !== recipe.sha256) {
       throw new Error(`${recipe.sourcePath} does not match the packaged snapshot recipe`);
     }
+  }));
+}
+
+// The image a FROM, COPY or INSTALL row names: its recipe at that path and pin.
+const referenced = ({ location, sha256 }) => DOLLY_IMAGES.find(candidate =>
+  siteReference(candidate.dollyfile) === location && candidate.sha256 === sha256);
+
+// The images IMAGE is built from, each once and before those built from it,
+// IMAGE last; a cycle is refused.
+function inputClosure(image, order = [], path = []) {
+  if (path.includes(image)) throw new Error("packaged image cycle");
+  if (order.includes(image)) return order;
+  for (const reference of definitionOf(image).artifacts) {
+    const dependency = referenced(reference);
+    if (!dependency) throw new Error("packaged image input is missing");
+    inputClosure(dependency.image, order, [...path, image]);
   }
+  order.push(image);
+  return order;
 }
 
-export async function loadPackagedSnapshotMetadata(image, checked = new Map(), active = new Set()) {
-  if (checked.has(image)) return checked.get(image);
-  if (active.has(image)) throw new Error("packaged image cycle");
-  active.add(image);
-  const metadataUrl = new URL(
-    `dist/dolly-${image}-system-snapshot.mjs`, imageBase,
-  );
-  let metadata;
+async function importMetadata(image) {
   try {
-    ({ DOLLY_SYSTEM_SNAPSHOT: metadata } = await import(metadataUrl.href));
+    return (await import(new URL(`dist/dolly-${image}-system-snapshot.json`, imageBase).href, { with: { type: "json" } })).default;
   } catch {
     throw new Error(`The packaged ${image} snapshot is missing. Run npm run image -- ${image} first.`);
   }
+}
+
+// An image's metadata, checked against the catalog and the metadata of every
+// image it is built from, which are fetched together.
+export async function loadPackagedSnapshotMetadata(image) {
+  const closure = inputClosure(image);
+  const loaded = await Promise.all(closure.map(importMetadata));
+  const checked = new Map();
+  closure.forEach((name, index) => checked.set(name, checkedMetadata(name, loaded[index], checked)));
+  return checked.get(image);
+}
+
+// METADATA, checked as IMAGE's against the catalog and CHECKED, the metadata
+// of the images it is built from.
+function checkedMetadata(image, metadata, checked) {
   const recipes = expectedRecipes(image);
   if (metadata === null || typeof metadata !== "object" ||
       metadata.image !== image || metadata.buildId !== DOLLY_IMAGE_BUILD_ID ||
       metadata.formatVersion !== 2 || metadata.identityVersion !== 2 ||
       JSON.stringify(metadata.recipes) !== JSON.stringify(recipes) ||
-      JSON.stringify(hostRequirements(metadata.hostRequirements)) !== JSON.stringify(definitionOf(image).hostRequirements ?? []) ||
+      JSON.stringify(hostRequirements(metadata.hostRequirements)) !== JSON.stringify(definitionOf(image).hostRequirements) ||
       !Number.isSafeInteger(metadata.byteLength) || metadata.byteLength <= 0 ||
-      metadata.byteLength > snapshotSizeLimit ||
-      (![undefined, "gzip", "packs"].includes(metadata.encoding)) ||
-      (metadata.encoding === "gzip" &&
-       (!Number.isSafeInteger(metadata.encodedByteLength) ||
-        metadata.encodedByteLength <= 0 || metadata.encodedByteLength > snapshotSizeLimit)) ||
+      metadata.byteLength > snapshotSizeLimit || metadata.encoding !== "packs" ||
       !/^[0-9a-f]{64}$/.test(metadata.sha256) ||
       !Array.isArray(metadata.manifest) || metadata.manifest.length === 0 ||
       metadata.manifest.length > 100_000) {
     throw new Error("The packaged system snapshot metadata does not match this Dolly build");
   }
-  if (metadata.encoding === "packs") validateSnapshotPacks(metadata);
+  validateSnapshotPacks(metadata);
   const inputs = imageInputs(metadata.inputs);
   const references = [...new Set(definitionOf(image).artifacts.map(reference => reference.sha256))].sort();
   if (JSON.stringify(inputs.map(input => input.recipeSha256)) !== JSON.stringify(references)) {
@@ -90,86 +117,231 @@ export async function loadPackagedSnapshotMetadata(image, checked = new Map(), a
       throw new Error(`The packaged system snapshot is missing ${required}`);
     }
   }
-  const expectedInputs = [];
-  for (const reference of definitionOf(image).artifacts) {
-    const dependency = DOLLY_IMAGES.find(candidate =>
-      siteReference(candidate.dollyfile) === reference.location && candidate.sha256 === reference.sha256);
-    if (!dependency) throw new Error("packaged image input is missing");
-    const parent = await loadPackagedSnapshotMetadata(dependency.image, checked, active);
-    expectedInputs.push({ recipeSha256: reference.sha256, sha256: parent.sha256 });
-  }
+  const expectedInputs = definitionOf(image).artifacts.map(reference =>
+    ({ recipeSha256: reference.sha256, sha256: checked.get(referenced(reference).image).sha256 }));
   if (!imageInputsMatch(inputs, expectedInputs)) throw new Error("The packaged system snapshot has stale image inputs");
-  if (active.size === 1) await verifyVisibleRecipes(recipes);
-  active.delete(image);
-  checked.set(image, metadata);
   return metadata;
 }
 
-async function* packagedSnapshotParts(image, metadata, signal) {
-  const packed = metadata.encoding === "packs";
-  const parts = packed ? validateSnapshotPacks(metadata) : [metadata];
-  for (const part of parts) {
-    signal?.throwIfAborted();
-    const compressed = packed || metadata.encoding === "gzip";
-    const url = packed
-      ? new URL(`dist/packs/${part.sha256}.snapshot.gz`, imageBase)
-      : new URL(`dist/dolly-${image}-system.snapshot${compressed ? ".gz" : ""}`, imageBase);
-    const init = { cache: packed ? "force-cache" : "no-store", credentials: "same-origin", redirect: "error", signal };
-    const expected = compressed ? part.encodedByteLength : part.byteLength;
-    const response = await decodeStaticAsset(await fetch(url, init), url, init, expected);
-    if (!response.ok || !response.body) throw new Error(`snapshot returned HTTP ${response.status}`);
-    const declared = response.headers.get("content-length");
-    if (declared !== null && Number(declared) !== expected) throw new Error("snapshot HTTP content length mismatch");
-    const body = compressed ? response.body.pipeThrough(new DecompressionStream("gzip")) : response.body;
-    yield { ...part, body };
+// An image is the records of its packs, each a gzip-compressed snapshot named
+// by its digest, which images that hold the same files share. Each pack's
+// compressed body, kept in this browser (pack-store.mjs) or downloaded and
+// checked against its declared length. The consumer checks each part against
+// its digest and then calls its verified(), which keeps a downloaded pack; a
+// part it does not verify is not kept, or no longer.
+async function* packagedSnapshotParts(metadata, signal) {
+  const packs = validateSnapshotPacks(metadata), store = await openPackStore(metadata.image, packs);
+  let complete = false;
+  try {
+    for (const pack of packs) {
+      signal?.throwIfAborted();
+      const part = await store.kept(pack) ?? await store.keeping(pack, await downloadedPack(pack, signal));
+      let verified = false;
+      try {
+        yield { ...pack, body: part.body, verified: () => { verified = true; return part.keep(); } };
+      } finally {
+        if (!verified) await part.discard();
+      }
+    }
+    complete = true;
+  } finally {
+    store.close(complete);
+  }
+}
+
+async function downloadedPack(pack, signal) {
+  const url = new URL(`dist/packs/${pack.sha256}.snapshot.gz`, imageBase);
+  const init = { cache: "force-cache", credentials: "same-origin", redirect: "error", signal };
+  const response = await decodeStaticAsset(await fetch(url, init), url, init, pack.encodedByteLength);
+  if (!response.ok || !response.body) throw new Error(`snapshot returned HTTP ${response.status}`);
+  const declared = response.headers.get("content-length");
+  if (declared !== null && Number(declared) !== pack.encodedByteLength) throw new Error("snapshot HTTP content length mismatch");
+  return response.body;
+}
+
+const gunzip = body => body.pipeThrough(new DecompressionStream("gzip"));
+
+// The chunks of `body`; a consumer that stops early cancels it.
+async function* chunksOf(body, signal) {
+  const reader = body.getReader();
+  let finished = false;
+  try {
+    for (;;) {
+      signal?.throwIfAborted();
+      const { done, value } = await reader.read();
+      if (done) break;
+      yield value;
+    }
+    finished = true;
+  } finally {
+    if (!finished) await reader.cancel().catch(() => {});
+    reader.releaseLock();
   }
 }
 
+async function readBytes(body, length, signal) {
+  const bytes = new Uint8Array(length);
+  let offset = 0;
+  for await (const chunk of chunksOf(body, signal)) {
+    if (chunk.length > length - offset) throw new Error("snapshot exceeds declared size");
+    bytes.set(chunk, offset);
+    offset += chunk.length;
+  }
+  if (offset !== length) throw new Error("snapshot integrity mismatch");
+  return bytes;
+}
+
+// The stream of `bytes` in pieces of at most 16 KiB, so that decompressing it
+// holds only a little of the output at a time.
+function streamOf(bytes) {
+  let offset = 0;
+  return new ReadableStream({
+    pull(controller) {
+      const end = Math.min(offset + 16 * 1024, bytes.length);
+      controller.enqueue(bytes.subarray(offset, end));
+      offset = end;
+      if (offset === bytes.length) controller.close();
+    },
+  }, { highWaterMark: 0 });
+}
+
 // The sink owns incremental hashing. Every part is checked before another is
 // requested; the caller verifies the complete canonical image before booting.
-export async function streamPackagedSystemSnapshot(image, metadata, write, endPart, signal) {
-  for await (const part of packagedSnapshotParts(image, metadata, signal)) {
-    const reader = part.body.getReader();
+export async function streamPackagedSystemSnapshot(metadata, write, endPart, signal) {
+  for await (const part of packagedSnapshotParts(metadata, signal)) {
     let size = 0;
-    try {
-      for (;;) {
-        signal?.throwIfAborted();
-        const { done, value } = await reader.read();
-        if (done) break;
-        if (value.length > part.byteLength - size) throw new Error("snapshot exceeds declared size");
-        await write(value);
-        size += value.length;
+    for await (const chunk of chunksOf(gunzip(part.body), signal)) {
+      if (chunk.length > part.byteLength - size) throw new Error("snapshot exceeds declared size");
+      await write(chunk);
+      size += chunk.length;
+    }
+    if (size !== part.byteLength || await endPart() !== part.sha256) throw new Error("snapshot integrity mismatch");
+    await part.verified();
+  }
+}
+
+function snapshotHeader(count) {
+  const bytes = new Uint8Array(16), view = new DataView(bytes.buffer);
+  bytes.set(encoder.encode("DOLLYSNP"));
+  view.setUint32(8, 2, true);
+  view.setUint32(12, count, true);
+  return bytes;
+}
+
+function recordHeader(kind, path, size) {
+  const bytes = new Uint8Array(16 + path.length), view = new DataView(bytes.buffer);
+  view.setUint32(0, kind, true);
+  view.setUint32(4, path.length, true);
+  view.setBigUint64(8, BigInt(size), true);
+  bytes.set(path, 16);
+  return bytes;
+}
+
+function compareBytes(left, right) {
+  for (let index = 0; index < Math.min(left.length, right.length); index += 1) {
+    if (left[index] !== right[index]) return left[index] - right[index];
+  }
+  return left.length - right.length;
+}
+
+// One pack's bytes, decompressed again as its records are reached in path order.
+class PackReader {
+  constructor(compressed) {
+    this.chunks = chunksOf(gunzip(streamOf(compressed)));
+    this.chunk = new Uint8Array(0);
+    this.position = 0; // the pack offset of this.chunk
+  }
+
+  // The pack's bytes [offset, offset + length), skipping those before it.
+  async *read(offset, length) {
+    const end = offset + length;
+    while (this.position < end) {
+      if (this.chunk.length === 0) {
+        const { done, value } = await this.chunks.next();
+        if (done) throw new Error("truncated snapshot pack");
+        this.chunk = value;
       }
-      if (size !== part.byteLength || await endPart() !== part.sha256) throw new Error("snapshot integrity mismatch");
-    } catch (error) {
-      await reader.cancel().catch(() => {});
-      throw error;
-    } finally { reader.releaseLock(); }
-  }
-}
-
-export async function loadPackagedSystemSnapshot(image, metadata, signal) {
-  const parts = [];
-  for await (const part of packagedSnapshotParts(image, metadata, signal)) {
-    const reader = part.body.getReader(), bytes = new Uint8Array(part.byteLength);
-    let offset = 0;
-    try {
-      for (;;) {
-        signal?.throwIfAborted();
-        const { done, value } = await reader.read();
-        if (done) break;
-        if (value.length > bytes.length - offset) throw new Error("snapshot exceeds declared size");
-        bytes.set(value, offset);
-        offset += value.length;
+      const start = Math.max(offset - this.position, 0), taken = Math.min(this.chunk.length, end - this.position);
+      if (taken > start) yield this.chunk.subarray(start, taken);
+      this.chunk = this.chunk.subarray(taken);
+      this.position += taken;
+    }
+  }
+
+  close() { return this.chunks.return(); }
+}
+
+// Checks every pack against its hash as it decompresses, keeping only its
+// compressed bytes and where its records lie; returns the image's records in
+// path order, each pack's read from its bytes decompressed once more.
+async function packedRecords(metadata, signal) {
+  const records = [];
+  for await (const part of packagedSnapshotParts(metadata, signal)) {
+    const pack = { compressed: await readBytes(part.body, part.encodedByteLength, signal), unread: 0 };
+    const hash = new Sha256();
+    const scanned = await scanSnapshotRecords((async function* () {
+      for await (const chunk of chunksOf(gunzip(streamOf(pack.compressed)), signal)) {
+        hash.update(chunk);
+        yield chunk;
+      }
+    })(), () => false);
+    if (scanned.byteLength !== part.byteLength || hash.hex() !== part.sha256) throw new Error("snapshot integrity mismatch");
+    await part.verified();
+    for (const [path, { kind, offset, size }] of scanned.records) {
+      records.push({ pack, kind, path: encoder.encode(path), offset, size });
+      pack.unread += 1;
+    }
+  }
+  records.sort((left, right) => compareBytes(left.path, right.path));
+  return (async function* () {
+    yield snapshotHeader(records.length);
+    for (const { pack, kind, path, offset, size } of records) {
+      yield recordHeader(kind, path, size);
+      pack.reader ??= new PackReader(pack.compressed);
+      yield* pack.reader.read(offset, size);
+      if (--pack.unread === 0) {
+        await pack.reader.close();
+        pack.reader = pack.compressed = null;
+      }
+    }
+  })();
+}
+
+// Joins pieces into chunks of 1 MiB, the last one shorter.
+async function* joined(pieces) {
+  const size = 1024 * 1024;
+  let chunk = new Uint8Array(size), used = 0;
+  for await (let piece of pieces) {
+    while (piece.length !== 0) {
+      const taken = Math.min(piece.length, size - used);
+      chunk.set(piece.subarray(0, taken), used);
+      used += taken;
+      piece = piece.subarray(taken);
+      if (used === size) {
+        yield chunk;
+        chunk = new Uint8Array(size);
+        used = 0;
       }
-      if (offset !== bytes.length || await sha256(bytes) !== part.sha256) throw new Error("snapshot integrity mismatch");
-      parts.push(bytes);
-    } catch (error) {
-      await reader.cancel().catch(() => {});
-      throw error;
-    } finally { reader.releaseLock(); }
-  }
-  const bytes = parts.length === 1 ? parts[0] : mergeSnapshotRecords(parts);
-  if (bytes.length !== metadata.byteLength || await sha256(bytes) !== metadata.sha256) throw new Error("packed snapshot integrity mismatch");
-  return bytes.buffer;
+    }
+  }
+  if (used !== 0) yield chunk.subarray(0, used);
+}
+
+// The image's canonical snapshot in chunks, merged from its packs as they are
+// read: the page holds the packs' compressed bytes, not the image. Every pack
+// matches its hash before the first chunk; the chunks end in an error, not
+// their end, unless together they match the image's hash.
+export async function packagedSystemSnapshot(metadata, signal) {
+  const pieces = await packedRecords(metadata, signal);
+  return (async function* () {
+    const hash = new Sha256();
+    let size = 0;
+    for await (const chunk of joined(pieces)) {
+      if (chunk.length > metadata.byteLength - size) throw new Error("snapshot exceeds declared size");
+      size += chunk.length;
+      hash.update(chunk);
+      yield chunk;
+    }
+    if (size !== metadata.byteLength || hash.hex() !== metadata.sha256) throw new Error("packed snapshot integrity mismatch");
+  })();
 }