@daugasauron/dolly 0.1.5 → 0.1.6

package/{src → site}/session-store.mjs

@@ -1,11 +1,13 @@
 import { alphabetical } from "./alphabetical.mjs";
-import { validName } from "./image-names.mjs";
-import { imageInputs } from "./image-inputs.mjs";
-import { MAX_SNAPSHOT_BYTES } from "./snapshot-records.mjs";
-import { DOLLY_VERSION } from "./version.mjs";
-
-export const DOLLY_SESSION_FORMAT_VERSION = 3;
-export const DOLLY_SESSION_MAX_BYTES = 512 * 1024 * 1024;
+import { validName } from "../page/image-names.mjs";
+import { imageInputs } from "../page/image-inputs.mjs";
+import { MAX_SNAPSHOT_BYTES } from "../page/snapshot-records.mjs";
+import { DOLLY_VERSION } from "../page/version.mjs";
+
+// A save is the whole filesystem but /dev, /proc and /run (host/snapshot/kernel.c),
+// at most 8 GiB before and after compression.
+export const DOLLY_SESSION_FORMAT_VERSION = 4;
+export const DOLLY_SESSION_MAX_BYTES = 8 * 1024 * 1024 * 1024;
 export const DOLLY_SESSION_METADATA_MAX_BYTES = 1024 * 1024;
 
 // Saves belong to a version: every version's pages open their own database.
@@ -43,16 +45,17 @@ export function customSessionIdentity(custom) {
   return `custom:${artifact.recipeSha256}:${artifact.sha256}`;
 }
 
-export function sessionCompatible(record, definitions, buildId, imageBuildId) {
+// A save holds its whole filesystem, so it opens wherever its programs run: on
+// a runtime with the image build ID of the one that made it (buildId; a rebuilt
+// kernel keeps it while the images stay valid), whatever became of its image's
+// recipe. The image need only still be named (its host modules are the
+// catalog's), a custom one by the recipe it carries.
+export function sessionCompatible(record, definitions, buildId) {
   if (record.formatVersion !== DOLLY_SESSION_FORMAT_VERSION || record.buildId !== buildId) return false;
   if (record.image === "custom") {
-    try {
-      return record.imageIdentity === customSessionIdentity(record.customImage) &&
-        record.customImage.artifact.buildId === imageBuildId;
-    } catch { return false; }
+    try { return record.imageIdentity === customSessionIdentity(record.customImage); } catch { return false; }
   }
-  return record.customImage === undefined && definitions.some(({ image }) => image === record.image) &&
-    record.imageIdentity === sessionImageIdentity(definitions, record.image);
+  return record.customImage === undefined && definitions.some(({ image }) => image === record.image);
 }
 
 export function sessionLoadUrl(name, applicationBase) {
@@ -92,16 +95,36 @@ async function collectStream(stream, maximum) {
   }
 }
 
+// The compressed save, a Blob of its parts: no buffer holds it whole.
+async function collectBlob(stream, maximum) {
+  const reader = stream.getReader(), parts = [];
+  let length = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      length += value.byteLength;
+      if (length > maximum) throw new Error("Dolly session exceeds its size limit");
+      parts.push(value);
+    }
+    return new Blob(parts);
+  } catch (error) {
+    await reader.cancel(error).catch(() => {});
+    throw error;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
 export async function encodeSessionStream(produce) {
   const stream = new CompressionStream("gzip"), writer = stream.writable.getWriter();
-  const encoded = collectStream(stream.readable, DOLLY_SESSION_MAX_BYTES);
+  const encoded = collectBlob(stream.readable, DOLLY_SESSION_MAX_BYTES);
   // The reader can fail while the producer is awaiting its next mailbox chunk.
   encoded.catch(() => {});
   try {
     await produce(bytes => writer.write(bytes));
     await writer.close();
-    // Blob storage needs fixed buffers; decoded buffers transfer directly to Wasm.
-    return { encoding: "gzip", bytes: (await encoded).transferToFixedLength() };
+    return { encoding: "gzip", bytes: await encoded };
   } catch (error) {
     await writer.abort(error).catch(() => {});
     await encoded.catch(() => {});
@@ -109,22 +132,10 @@ export async function encodeSessionStream(produce) {
   }
 }
 
-export async function decodeSessionSnapshot(record) {
-  if (record === null || typeof record !== "object" ||
-      !(record.bytes instanceof ArrayBuffer) || record.bytes.byteLength === 0 ||
-      record.bytes.byteLength > DOLLY_SESSION_MAX_BYTES || record.encoding !== "gzip") {
-    throw new Error("Stored Dolly session is invalid");
-  }
-  let offset = 0;
-  const stream = new ReadableStream({
-    pull(controller) {
-      if (offset === record.bytes.byteLength) { controller.close(); return; }
-      const length = Math.min(65536, record.bytes.byteLength - offset);
-      controller.enqueue(new Uint8Array(record.bytes, offset, length));
-      offset += length;
-    },
-  }).pipeThrough(new DecompressionStream("gzip"));
-  return collectStream(stream, DOLLY_SESSION_MAX_BYTES);
+// A whole save in one buffer, for a test that reads its records: at most 2 GiB.
+export async function decodeSessionSnapshot(bytes) {
+  if (!(bytes instanceof Blob) || bytes.size === 0) throw new Error("Stored Dolly session is invalid");
+  return collectStream(bytes.stream().pipeThrough(new DecompressionStream("gzip")), 2 * 1024 * 1024 * 1024);
 }
 
 function openDatabase() {
@@ -164,7 +175,6 @@ export async function loadStoredSession(name) {
   if (!(record.bytes instanceof Blob) || record.bytes.size === 0 || record.bytes.size > DOLLY_SESSION_MAX_BYTES) {
     throw new Error("Stored Dolly session is invalid");
   }
-  record.bytes = await record.bytes.arrayBuffer();
   return record;
 }
 
@@ -176,8 +186,8 @@ export function validateSessionRecord(record) {
       typeof record.image !== "string" || !validName(record.image) ||
       typeof record.imageIdentity !== "string" || record.imageIdentity.length > 256 ||
       !Number.isSafeInteger(record.updatedAt) ||
-      !(record.bytes instanceof ArrayBuffer) || record.bytes.byteLength === 0 ||
-      record.bytes.byteLength > DOLLY_SESSION_MAX_BYTES || record.encoding !== "gzip") {
+      !(record.bytes instanceof Blob) || record.bytes.size === 0 ||
+      record.bytes.size > DOLLY_SESSION_MAX_BYTES || record.encoding !== "gzip") {
     throw new TypeError("invalid Dolly session record");
   }
   if (record.image === "custom" ? record.imageIdentity !== customSessionIdentity(record.customImage)
@@ -186,8 +196,7 @@ export function validateSessionRecord(record) {
 
 export async function saveStoredSession(record, { overwrite = true } = {}) {
   validateSessionRecord(record);
-  const stored = { ...record, bytes: new Blob([record.bytes]) };
-  await transaction("readwrite", (store) => overwrite ? store.put(stored) : store.add(stored));
+  await transaction("readwrite", (store) => overwrite ? store.put(record) : store.add(record));
 }
 
 export async function deleteStoredSession(name) {

package/src/runtime-worker.mjs

@@ -1,57 +0,0 @@
-// The runtime Worker boot() starts: it restores an image of the image directory
-// from its packs into a new kernel and runs its ENTRY (worker-image.mjs).
-import { imageDefinition } from "./images.mjs";
-import { loadPackagedSnapshotMetadata, streamPackagedSystemSnapshot, verifyVisibleRecipes } from "./packaged-snapshot.mjs";
-import { siteReference, hex } from "./static-asset.mjs";
-import { checkedMemoryRange, runImage } from "./worker-image.mjs";
-
-await runImage(async ({ image, files: given = {} }, host) => {
-  const definition = imageDefinition(image);
-  if (!definition) throw new Error("invalid Dolly image selection");
-  const [metadata] = await Promise.all([loadPackagedSnapshotMetadata(image), verifyVisibleRecipes(image)]);
-  host.require(definition.hostRequirements);
-  host.admit(definition.hostRequirements);
-  return {
-    name: image,
-    prepare({ files }) {
-      files.write("/etc/dolly/recipe.locator", siteReference(definition.dollyfile));
-      // The kernel restores a snapshot against the list of its paths.
-      files.write("/etc/dolly/image.manifest", `${metadata.manifest.join("\n")}\n`);
-    },
-    async restore({ kernel, memory, files, stage }) {
-      stage("loading precompiled userspace snapshot...");
-      const capacity = 1024 * 1024;
-      const address = kernel.dolly_snapshot_restore_address(BigInt(capacity));
-      const staging = () => {
-        const range = checkedMemoryRange(memory, address, capacity);
-        return new Uint8Array(memory.buffer, range.address, range.size);
-      };
-      staging().set(Uint8Array.from(metadata.sha256.match(/../g), byte => parseInt(byte, 16)));
-      if (kernel.dolly_bootstrap_snapshot_begin(BigInt(metadata.byteLength)) !== 0)
-        throw new Error("Dolly snapshot stream initialization failed");
-      let loaded = 0, reported = 0;
-      await streamPackagedSystemSnapshot(metadata, bytes => {
-        for (let offset = 0; offset < bytes.length; offset += capacity) {
-          const chunk = bytes.subarray(offset, offset + capacity);
-          staging().set(chunk);
-          if (kernel.dolly_snapshot_stream_write(BigInt(chunk.length), 0) !== 0)
-            throw new Error("Dolly rejected a snapshot stream record");
-        }
-        loaded += bytes.length;
-        if (performance.now() - reported >= 1000) {
-          const mib = 1024 * 1024;
-          stage(`loading userspace: ${Math.min(Math.floor(loaded / mib), Math.ceil(metadata.byteLength / mib))} / ${Math.ceil(metadata.byteLength / mib)} MiB`);
-          reported = performance.now();
-        }
-      }, () => {
-        if (kernel.dolly_snapshot_stream_write(0n, 1) !== 0)
-          throw new Error("Dolly rejected an incomplete snapshot part");
-        return hex(staging().subarray(0, 32));
-      });
-      const status = kernel.dolly_bootstrap_snapshot_end();
-      // The embedding's files (boot's `files`) lie over the image's.
-      if (status === 0) for (const [path, contents] of Object.entries(given)) files.write(path, contents);
-      return { status };
-    },
-  };
-});