@daugasauron/dolly 0.1.5-rc.0 → 0.1.6

package/host/packages/service.mjs

@@ -1,6 +1,5 @@
-import { DOLLY_IMAGES, imageBase } from "../../src/images.mjs";
-import { loadPackagedSnapshotMetadata } from "../../src/packaged-snapshot.mjs";
-import { MAX_SNAPSHOT_BYTES } from "../../src/snapshot-records.mjs";
+import { imageBase } from "../../page/images.mjs";
+import { MAX_SNAPSHOT_BYTES } from "../../page/snapshot-records.mjs";
 
 const PACKAGES_ORIGIN = "https://packages.dolly.invalid";
 const PACKAGE_LIMITS = Object.freeze({
@@ -9,35 +8,58 @@ const PACKAGE_LIMITS = Object.freeze({
   // Snapshots served per page: enough for every package, not for a loop.
   maxSnapshots: 64,
 });
-const packagePath = /^\/v1\/packages\/([0-9a-f]{64})$/;
+const directoryPath = "/v1/directory";
+const packagePath = /^\/v1\/packages\/([0-9a-f]{64})\/from\/([^/]+)$/;
 
 function reply(body, status, type = "text/plain; charset=utf-8") {
   return new Response(body, { status, headers: { "content-type": type, "cache-control": "no-store" } });
 }
 
-// GET /v1/packages/SHA256 is the exact snapshot of the package whose recipe
-// has that pin, verified from the published packs and merged as the guest
-// reads it. One snapshot is served at a time, until the guest has read it or
-// the broker's timeout cancels it.
+// The image directory a path segment names, or null: an http or https URL of a
+// directory, written as the URL parser writes it, with nothing but a path.
+function directoryOf(segment) {
+  let text;
+  try { text = decodeURIComponent(segment); } catch { return null; }
+  const url = URL.parse(text);
+  return url?.href === text && /^https?:$/.test(url.protocol) && url.pathname.endsWith("/") &&
+    !url.username && !url.password && !url.search && !url.hash ? url : null;
+}
+
+// GET /v1/directory is the image directory the page loads its images from,
+// amy's server unless the session names another. GET /v1/packages/SHA256/from/
+// DIRECTORY is the exact snapshot of the package whose recipe has that pin in
+// that directory's catalog, verified from its packs and merged as the guest
+// reads it. Another directory than the page's is read only where the page's
+// HTTP policy admits a GET of it (`admit`), and only packages built for this
+// runtime pass. One snapshot is served at a time, until the guest has read it
+// or the broker's timeout cancels it.
 export class PackageService {
-  constructor() {
+  constructor(admit) {
+    this.admit = admit;
     this.active = false;
     this.served = 0;
   }
   get origin() { return PACKAGES_ORIGIN; }
   authorize(url, method, bytes) {
-    return method === "GET" && bytes === 0 && packagePath.test(url.pathname) ? PACKAGE_LIMITS : undefined;
+    return method === "GET" && bytes === 0 && (url.pathname === directoryPath || packagePath.test(url.pathname))
+      ? PACKAGE_LIMITS : undefined;
   }
   async fetch(url, init) {
-    const sha256 = packagePath.exec(url.pathname)[1];
-    const definition = DOLLY_IMAGES.find(candidate => candidate.role === "package" && candidate.sha256 === sha256);
-    if (!definition) return reply("This release publishes no package with that recipe pin\n", 404);
-    if (this.active) return reply("Another package snapshot is being served\n", 409);
-    if (this.served >= PACKAGE_LIMITS.maxSnapshots) return reply("Package snapshot quota exceeded\n", 429);
+    if (url.pathname === directoryPath) return reply(imageBase.href);
+    const [, sha256, segment] = packagePath.exec(url.pathname);
+    const directory = directoryOf(segment);
+    if (!directory) return reply("not the URL of an image directory\n", 400);
+    if (directory.href !== imageBase.href) {
+      try { this.admit(directory); } catch {
+        return reply(`this page's HTTP policy does not admit ${directory.href}\n`, 403);
+      }
+    }
+    if (this.active) return reply("another package snapshot is being served\n", 409);
+    if (this.served >= PACKAGE_LIMITS.maxSnapshots) return reply("this page's package snapshot quota is spent\n", 429);
     this.active = true;
-    this.served += 1;
     // The Worker (worker.mjs) answers each message in turn: the first with the
-    // packs checked, every later one with the snapshot's next chunk.
+    // package found and its packs checked, every later one with the snapshot's
+    // next chunk.
     const worker = new Worker(new URL("./worker.mjs", import.meta.url), { type: "module", name: "dolly-packages" });
     let released = false, refuse;
     const release = () => {
@@ -56,12 +78,16 @@ export class PackageService {
       else worker.postMessage(message);
     });
     try {
-      const metadata = await loadPackagedSnapshotMetadata(definition.image);
-      await ask({ base: imageBase.href, metadata });
+      if ((await ask({ directory: directory.href, sha256 })).missing) {
+        release();
+        return reply(`${directory.href} publishes no package with that recipe pin\n`, 404);
+      }
     } catch (error) {
       release();
-      return reply(`${definition.image}: ${error.message}\n`, 502);
+      return reply(`${directory.href}: ${error.message}\n`, 502);
     }
+    // A package found counts against the quota; a miss or an offline directory does not.
+    this.served += 1;
     const body = new ReadableStream({
       async pull(controller) {
         try {

package/host/packages/transport.mjs

@@ -0,0 +1,123 @@
+import { DOLLY_ERRNO as errno } from "../../page/process-constants.mjs";
+import * as A from "./abi.mjs";
+
+const capacity = A.DOLLY_PACKAGES_CHUNK_CAPACITY;
+const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+const encoder = new TextEncoder();
+// Snapshots served per page: enough for every package, not for a loop.
+const maxSnapshots = 64;
+// The index is one chunk.
+const maxIndexBytes = capacity;
+
+// The Worker's end of packages@0's mailbox (dolly-packages-0.wat): it answers
+// each request the kernel reports, one chunk at a time, from the page's own
+// image directory, `imageBase`; another directory is refused until the page's
+// HTTP policy judges it here.
+export class PackagesTransport {
+  constructor(buffer, address, imageBase) {
+    this.imageBase = imageBase;
+    if (!(buffer instanceof SharedArrayBuffer) || !Number.isSafeInteger(address) || address <= 0 || address % 4 ||
+        address > buffer.byteLength - A.DOLLY_PACKAGES_CHUNK_OFFSET - capacity) throw new TypeError("invalid packages mailbox");
+    this.words = new Int32Array(buffer, address, 16);
+    this.directory = new Uint8Array(buffer, address + A.DOLLY_PACKAGES_DIRECTORY_OFFSET, A.DOLLY_PACKAGES_DIRECTORY_CAPACITY);
+    this.pin = new Uint8Array(buffer, address + A.DOLLY_PACKAGES_PIN_OFFSET, A.DOLLY_PACKAGES_PIN_BYTES);
+    this.bytes = new Uint8Array(buffer, address + A.DOLLY_PACKAGES_CHUNK_OFFSET, capacity);
+    this.active = null;
+    this.served = 0;
+    this.reported = Atomics.load(this.words, A.DOLLY_PACKAGES_WORD_REQUEST);
+    this.retiredSeen = Atomics.load(this.words, A.DOLLY_PACKAGES_WORD_CANCELLED);
+    Atomics.store(this.words, A.DOLLY_PACKAGES_WORD_ENABLED, 1);
+  }
+
+  // At each service tick: a new request is answered, a retired one ends.
+  service() {
+    const request = Atomics.load(this.words, A.DOLLY_PACKAGES_WORD_REQUEST);
+    const cancelled = Atomics.load(this.words, A.DOLLY_PACKAGES_WORD_CANCELLED);
+    if (request !== this.reported) {
+      this.reported = request;
+      void this.answer();
+    }
+    if (cancelled !== this.retiredSeen) {
+      this.retiredSeen = cancelled;
+      if (this.active && this.retired(this.active.sequence)) this.active.retire();
+    }
+  }
+
+  retired(sequence) {
+    return Atomics.load(this.words, A.DOLLY_PACKAGES_WORD_REQUEST) !== sequence ||
+      Atomics.load(this.words, A.DOLLY_PACKAGES_WORD_CANCELLED) === sequence;
+  }
+
+  async answer() {
+    const words = this.words, sequence = Atomics.load(words, A.DOLLY_PACKAGES_WORD_REQUEST);
+    if (this.active || sequence === Atomics.load(words, A.DOLLY_PACKAGES_WORD_COMPLETED)) return;
+    let retire, worker;
+    const retirement = new Promise(resolve => { retire = resolve; });
+    this.active = { sequence, retire };
+    let size = 0;
+    const publish = async (bytes, eof, error = 0) => {
+      // The kernel notifies the consumed word after writing each chunk.
+      for (let consumed; (consumed = Atomics.load(words, A.DOLLY_PACKAGES_WORD_CONSUMED)) !== Atomics.load(words, A.DOLLY_PACKAGES_WORD_CHUNK);) {
+        if (this.retired(sequence)) return;
+        await Promise.race([Atomics.waitAsync(words, A.DOLLY_PACKAGES_WORD_CONSUMED, consumed).value, retirement]);
+      }
+      if (this.retired(sequence)) return;
+      this.bytes.set(bytes);
+      Atomics.store(words, A.DOLLY_PACKAGES_WORD_LENGTH, bytes.length);
+      Atomics.store(words, A.DOLLY_PACKAGES_WORD_ERROR, error);
+      Atomics.store(words, A.DOLLY_PACKAGES_WORD_EOF, eof ? 1 : 0);
+      Atomics.store(words, A.DOLLY_PACKAGES_WORD_SIZE, size);
+      Atomics.add(words, A.DOLLY_PACKAGES_WORD_CHUNK, 1);
+    };
+    const whole = bytes => { size = bytes.length; return publish(bytes, true); };
+    const refuse = code => publish(new Uint8Array(), true, code);
+    try {
+      const kind = Atomics.load(words, A.DOLLY_PACKAGES_WORD_KIND);
+      const directory = decoder.decode(this.directory.slice(0, Atomics.load(words, A.DOLLY_PACKAGES_WORD_DIRECTORY_LENGTH)));
+      if (kind === A.DOLLY_PACKAGES_KIND_DIRECTORY) return await whole(encoder.encode(this.imageBase.href));
+      if (directory !== this.imageBase.href) return await refuse(errno.EACCES);
+      if (kind === A.DOLLY_PACKAGES_KIND_INDEX) {
+        const response = await fetch(new URL("amy-index.txt", this.imageBase), { credentials: "same-origin", redirect: "error" });
+        const length = Number(response.headers.get("content-length") ?? 0);
+        if (!response.ok || length > maxIndexBytes) return await refuse(response.ok ? errno.EFBIG : errno.ENOENT);
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        return await (bytes.length > maxIndexBytes ? refuse(errno.EFBIG) : whole(bytes));
+      }
+      if (this.served >= maxSnapshots) return await refuse(errno.EDQUOT);
+      // The packages Worker (worker.mjs) finds the package and merges its
+      // checked packs into its snapshot, a chunk per message.
+      worker = new Worker(new URL("./worker.mjs", import.meta.url), { type: "module", name: "dolly-packages" });
+      const ask = message => new Promise((resolve, reject) => {
+        worker.onmessage = ({ data }) => data.error === undefined ? resolve(data) : reject(new Error(data.error));
+        worker.onerror = event => { event.preventDefault(); reject(new Error(event.message || "package Worker failed")); };
+        worker.postMessage(message);
+      });
+      const found = await ask({ directory, sha256: decoder.decode(this.pin.slice()) });
+      if (found.missing) return await refuse(errno.ENOENT);
+      if (!Number.isSafeInteger(found.size) || found.size <= 0 || found.size > A.DOLLY_PACKAGES_MAX_SIZE) return await refuse(errno.EFBIG);
+      // A package found counts against the quota; a miss does not.
+      this.served += 1;
+      size = found.size;
+      let sent = 0;
+      for (;;) {
+        const { done, value } = await ask({});
+        if (done) break;
+        for (let offset = 0; offset < value.length; offset += capacity) {
+          const piece = value.subarray(offset, Math.min(value.length, offset + capacity));
+          sent += piece.length;
+          await publish(piece, sent === size);
+          if (this.retired(sequence)) return;
+        }
+      }
+      if (sent !== size) await refuse(errno.EIO);
+    } catch {
+      await refuse(errno.EIO);
+    } finally {
+      worker?.terminate();
+      Atomics.store(words, A.DOLLY_PACKAGES_WORD_COMPLETED, sequence);
+      this.active = null;
+    }
+  }
+
+  close() { Atomics.store(this.words, A.DOLLY_PACKAGES_WORD_ENABLED, 0); this.active?.retire(); }
+}

package/host/packages/worker.mjs

@@ -1,16 +1,20 @@
-// The packages@0 Worker merges one package's packs into its snapshot, a chunk
-// per request, off the page's thread: the page must answer the guest's reads
-// at once, since a guest that finds no record waits 10 ms.
-import { loadImages } from "../../src/images.mjs";
-import { packagedSystemSnapshot } from "../../src/packaged-snapshot.mjs";
+// The packages@0 Worker reads one package of an image directory and merges its
+// packs into its snapshot, a chunk per request, off the page's thread: the page
+// must answer the guest's reads at once, since a guest that finds no record
+// waits 10 ms.
+import { DOLLY_IMAGES, loadImages } from "../../page/images.mjs";
+import { loadPackagedSnapshotMetadata, packagedSystemSnapshot } from "../../page/packaged-snapshot.mjs";
 
 let chunks;
 onmessage = async ({ data }) => {
   try {
-    if (data.metadata) {
-      await loadImages(data.base);
-      chunks = await packagedSystemSnapshot(data.metadata);
-      postMessage({ done: false });
+    if (data.directory) {
+      await loadImages(data.directory);
+      const definition = DOLLY_IMAGES.find(candidate => candidate.role === "package" && candidate.sha256 === data.sha256);
+      if (!definition) return postMessage({ missing: true });
+      const metadata = await loadPackagedSnapshotMetadata(definition.image);
+      chunks = await packagedSystemSnapshot(metadata);
+      postMessage({ done: false, size: metadata.byteLength });
     } else {
       const { done, value } = await chunks.next();
       postMessage({ done, value }, done ? [] : [value.buffer]);

package/host/requirements.mjs

@@ -1,6 +1,6 @@
 import { DOLLY_HOST_RECORD_BYTES as recordBytes, DOLLY_HOST_NAME_BYTES as nameBytes,
   DOLLY_HOST_DIGEST_BYTES as digestBytes, DOLLY_HOST_MAX_RECORDS as maxRecords } from "./abi.mjs";
-import { hex } from "../src/static-asset.mjs";
+import { hex } from "../page/static-asset.mjs";
 
 const pattern = /^([a-z][a-z0-9-]{0,30})@(0|[1-9][0-9]{0,4})$/;
 

package/host/runtime/abi.mjs

@@ -1,4 +1,10 @@
 // Generated from abi/dolly-supervisor-0.wat, abi/dolly-image-0.wat, abi/dolly-process-0.wat, abi/dolly-process-gate-0.wat, include/dolly/process.h.
+export const DOLLY_PROCESS_EVENT_LAUNCH = 1;
+export const DOLLY_PROCESS_EVENT_INTERRUPT = 2;
+export const DOLLY_PROCESS_EVENT_TERMINATE = 3;
+export const DOLLY_PROCESS_EVENT_WAKE = 4;
+export const DOLLY_PROCESS_EVENT_KIND_SHIFT = 48;
+export const DOLLY_PROCESS_EVENT_VALUE_SHIFT = 32;
 export const DOLLY_TERMINAL_WORD_RESULT_SEQUENCE = 0;
 export const DOLLY_TERMINAL_WORD_RESULT_STATUS = 1;
 export const DOLLY_TERMINAL_WORD_FOREGROUND_PID = 2;
@@ -6,4 +12,30 @@ export const DOLLY_TERMINAL_WORD_FOREGROUND_INTERRUPTIBLE = 3;
 export const DOLLY_TERMINAL_WORD_INTERRUPT_SEQUENCE = 4;
 export const DOLLY_TERMINAL_WORD_INTERRUPT_TARGET_PID = 5;
 export const DOLLY_FS_BLOCK_SIZE = 1048576;
+export const DOLLY_DEVICE_HEADER_BYTES = 32;
+export const DOLLY_DEVICE_REPLY_HEADER_BYTES = 64;
+export const DOLLY_DEVICE_REPLY_WORD_STATE = 0;
+export const DOLLY_DEVICE_REPLY_WORD_SCOPE = 1;
+export const DOLLY_DEVICE_REPLY_WORD_SEQUENCE = 2;
+export const DOLLY_DEVICE_REPLY_WORD_ERRNO = 3;
+export const DOLLY_DEVICE_REPLY_WORD_LENGTH = 4;
+export const DOLLY_DEVICE_REPLY_PUBLISHED = 1;
+export const DOLLY_SNAPSHOT_VERSION = 2;
+export const DOLLY_SNAPSHOT_HEADER_BYTES = 16;
+export const DOLLY_SNAPSHOT_RECORD_HEADER_BYTES = 16;
+export const DOLLY_SNAPSHOT_RECORD_DIRECTORY = 1;
+export const DOLLY_SNAPSHOT_RECORD_FILE = 2;
+export const DOLLY_SNAPSHOT_RECORD_SYMLINK = 3;
+export const DOLLY_SNAPSHOT_RECORD_DELETED = 4;
+export const DOLLY_SNAPSHOT_MAX_PATHS = 100000;
+export const DOLLY_SNAPSHOT_MAX_MEBIBYTES = 2048;
+export const DOLLY_SNAPSHOT_PATH_MIN_BYTES = 2;
+export const DOLLY_SNAPSHOT_PATH_MAX_BYTES = 4095;
+export const DOLLY_SNAPSHOT_SYMLINK_MIN_BYTES = 1;
+export const DOLLY_SNAPSHOT_SYMLINK_MAX_BYTES = 4095;
+export const DOLLY_ENTRY_VERSION = 1;
+export const DOLLY_ENTRY_HEADER_BYTES = 16;
+export const DOLLY_ENTRY_MAX_ARGUMENTS = 256;
+export const DOLLY_ENTRY_ARGUMENT_MAX_BYTES = 4096;
+export const DOLLY_ENTRY_MAX_BYTES = 65536;
 export const DOLLY_ERRNO = Object.freeze({ E2BIG: 1, EACCES: 2, EADDRINUSE: 3, EADDRNOTAVAIL: 4, EAFNOSUPPORT: 5, EAGAIN: 6, EALREADY: 7, EBADF: 8, EBADMSG: 9, EBUSY: 10, ECANCELED: 11, ECHILD: 12, ECONNABORTED: 13, ECONNREFUSED: 14, ECONNRESET: 15, EDEADLK: 16, EDESTADDRREQ: 17, EDOM: 18, EDQUOT: 19, EEXIST: 20, EFAULT: 21, EFBIG: 22, EHOSTUNREACH: 23, EIDRM: 24, EILSEQ: 25, EINPROGRESS: 26, EINTR: 27, EINVAL: 28, EIO: 29, EISCONN: 30, EISDIR: 31, ELOOP: 32, EMFILE: 33, EMLINK: 34, EMSGSIZE: 35, EMULTIHOP: 36, ENAMETOOLONG: 37, ENETDOWN: 38, ENETRESET: 39, ENETUNREACH: 40, ENFILE: 41, ENOBUFS: 42, ENODEV: 43, ENOENT: 44, ENOEXEC: 45, ENOLCK: 46, ENOLINK: 47, ENOMEM: 48, ENOMSG: 49, ENOPROTOOPT: 50, ENOSPC: 51, ENOSYS: 52, ENOTCONN: 53, ENOTDIR: 54, ENOTEMPTY: 55, ENOTRECOVERABLE: 56, ENOTSOCK: 57, ENOTTY: 59, ENXIO: 60, EOVERFLOW: 61, EOWNERDEAD: 62, EPERM: 63, EPIPE: 64, EPROTO: 65, EPROTONOSUPPORT: 66, EPROTOTYPE: 67, ERANGE: 68, EROFS: 69, ESPIPE: 70, ESRCH: 71, ESTALE: 72, ETIMEDOUT: 73, ETXTBSY: 74, EXDEV: 75, ENODATA: 116, ENOTSUP: 138 });

package/host/runtime/lease-bridge.mjs

@@ -1,21 +1,26 @@
-import { DOLLY_ERRNO as E } from "./abi.mjs";
+import { DOLLY_DEVICE_REPLY_HEADER_BYTES as header, DOLLY_DEVICE_REPLY_PUBLISHED, DOLLY_DEVICE_REPLY_WORD_ERRNO,
+  DOLLY_DEVICE_REPLY_WORD_LENGTH, DOLLY_DEVICE_REPLY_WORD_SCOPE, DOLLY_DEVICE_REPLY_WORD_SEQUENCE, DOLLY_DEVICE_REPLY_WORD_STATE,
+  DOLLY_ERRNO as E } from "./abi.mjs";
 
-// The Worker side of a device leased per process (src/device-lease.c). Each
+// The Worker side of a device leased per process (kernel/device-lease.c). Each
 // packet a process submits (dolly_device_header, runtime.h, then the
 // operation's body) is checked whole here and goes to the page as
 // `TYPE-request` with its operation, scope, sequence and body; the page's
-// `TYPE-complete` is published in the lease's reply slot, a 64-byte header and
+// `TYPE-complete` is published in the lease's reply slot, a header and
 // `replyBytes`. `version`, `open` and `close` are the device's.
 export function createLeaseBridge({ type, version, slots: count, packetBytes, replyBytes, open, close }, memory, mailbox, send, complete) {
-  const slots = new Array(count), stride = 64 + replyBytes;
+  const slots = new Array(count), stride = header + replyBytes;
   if (!Number.isSafeInteger(mailbox) || mailbox <= 0 || mailbox % 4 ||
       mailbox > memory.buffer.byteLength - count * stride) throw new Error(`Invalid ${type} mailbox`);
   // The reply to the call a slot's lease waits for; its state is published last.
   function publish(slot, entry, error, bytes) {
-    const words = new Int32Array(memory.buffer, mailbox + slot * stride, 16);
-    new Uint8Array(memory.buffer, mailbox + slot * stride + 64, bytes.length).set(bytes);
-    Atomics.store(words, 1, entry.scope); Atomics.store(words, 2, entry.sequence);
-    Atomics.store(words, 3, error); Atomics.store(words, 4, bytes.length); Atomics.store(words, 0, 1);
+    const words = new Int32Array(memory.buffer, mailbox + slot * stride, header / 4);
+    new Uint8Array(memory.buffer, mailbox + slot * stride + header, bytes.length).set(bytes);
+    Atomics.store(words, DOLLY_DEVICE_REPLY_WORD_SCOPE, entry.scope);
+    Atomics.store(words, DOLLY_DEVICE_REPLY_WORD_SEQUENCE, entry.sequence);
+    Atomics.store(words, DOLLY_DEVICE_REPLY_WORD_ERRNO, error);
+    Atomics.store(words, DOLLY_DEVICE_REPLY_WORD_LENGTH, bytes.length);
+    Atomics.store(words, DOLLY_DEVICE_REPLY_WORD_STATE, DOLLY_DEVICE_REPLY_PUBLISHED);
     entry.pending = false;
   }
   function acknowledge(message) {

package/host/runtime/module.json

@@ -10,7 +10,7 @@
   "host": "runtime.mjs",
   "contracts": ["../../abi/dolly-supervisor-0.wat", "../../abi/dolly-image-0.wat"],
   "process": ["../../abi/dolly-process-0.wat", "../../abi/dolly-process-gate-0.wat"],
-  "headers": ["../../include/dolly/runtime.h", "../../include/dolly/process.h", "../../include/dolly/host.h", "../../include/dolly/host-abi.h", "../../include/dolly/toolchain.h"],
-  "kernel": ["../../src/dolly.c", "../../src/process-kernel.c", "../../src/system-snapshot.c", "../../src/device-lease.c", "../../src/file-blocks.cpp", "../../src/libc-host.c"],
-  "client": ["../../src/process/runtime-adapter.c", "../../src/process/socket-refusals.c", "../../src/process/libc-stubs.c"]
+  "headers": ["../../include/dolly/runtime.h", "../../include/dolly/process.h", "../../include/dolly/host.h", "../../include/dolly/host-abi.h", "../../include/dolly/image-abi.h", "../../include/dolly/toolchain.h"],
+  "kernel": ["../../kernel/dolly.c", "../../kernel/process-kernel.c", "../../kernel/pty.c", "../../kernel/system-snapshot.c", "../../kernel/device-lease.c", "../../kernel/file-blocks.cpp", "../../kernel/libc-host.c"],
+  "client": ["../../process/runtime-adapter.c", "../../process/socket-refusals.c", "../../process/libc-stubs.c"]
 }

package/host/runtime/runtime.mjs

@@ -1,6 +1,6 @@
-import { DollyProcessSupervisor } from "../../src/process-supervisor.mjs";
+import { DollyProcessSupervisor } from "../../page/process-supervisor.mjs";
 import { createLeaseBridge } from "./lease-bridge.mjs";
-import { instantiateKernelPlugin } from "../../src/kernel-plugin.mjs";
+import { instantiateKernelPlugin } from "../../page/kernel-plugin.mjs";
 import { DOLLY_ERRNO as E } from "./abi.mjs";
 import { DOLLY_TERMINAL_WORD_RESULT_SEQUENCE, DOLLY_TERMINAL_WORD_RESULT_STATUS, DOLLY_TERMINAL_WORD_FOREGROUND_PID,
   DOLLY_TERMINAL_WORD_FOREGROUND_INTERRUPTIBLE, DOLLY_TERMINAL_WORD_INTERRUPT_SEQUENCE,
@@ -219,7 +219,7 @@ export function worker({ send, runtimeBase, abi, service }) {
     },
     // A resident library linked into the kernel (--dolly-kernel-plugin), such as the display's.
     kernelPlugin(bytes, kernel) { return instantiateKernelPlugin(bytes, kernel, memory); },
-    // The Worker side of a device a module leases per process (src/device-lease.c).
+    // The Worker side of a device a module leases per process (kernel/device-lease.c).
     leaseBridge(lease, mailbox, send) {
       return createLeaseBridge(lease, memory, mailbox, send, () => supervisor?.serviceDeferred());
     },

package/host/snapshot/abi.mjs

@@ -1,5 +1,5 @@
 // Generated from host/snapshot/dolly-snapshot-0.wat, include/dolly/process.h.
-export const DOLLY_SESSION_HEADER_SIZE = 64;
+export const DOLLY_SESSION_MAILBOX_HEADER_SIZE = 64;
 export const DOLLY_SESSION_TRANSFER_CAPACITY = 1048576;
 export const DOLLY_SESSION_WORD_REQUEST_SEQUENCE = 0;
 export const DOLLY_SESSION_WORD_COMPLETED_SEQUENCE = 1;
@@ -11,4 +11,5 @@ export const DOLLY_SESSION_WORD_CHUNK_EOF = 6;
 export const DOLLY_SESSION_WORD_TOTAL_SIZE_LOW = 7;
 export const DOLLY_SESSION_WORD_TOTAL_SIZE_HIGH = 8;
 export const DOLLY_SESSION_WORD_CANCELLED_SEQUENCE = 9;
+export const DOLLY_SESSION_WORD_CHANGES = 10;
 export const DOLLY_ERRNO = Object.freeze({ E2BIG: 1, EACCES: 2, EADDRINUSE: 3, EADDRNOTAVAIL: 4, EAFNOSUPPORT: 5, EAGAIN: 6, EALREADY: 7, EBADF: 8, EBADMSG: 9, EBUSY: 10, ECANCELED: 11, ECHILD: 12, ECONNABORTED: 13, ECONNREFUSED: 14, ECONNRESET: 15, EDEADLK: 16, EDESTADDRREQ: 17, EDOM: 18, EDQUOT: 19, EEXIST: 20, EFAULT: 21, EFBIG: 22, EHOSTUNREACH: 23, EIDRM: 24, EILSEQ: 25, EINPROGRESS: 26, EINTR: 27, EINVAL: 28, EIO: 29, EISCONN: 30, EISDIR: 31, ELOOP: 32, EMFILE: 33, EMLINK: 34, EMSGSIZE: 35, EMULTIHOP: 36, ENAMETOOLONG: 37, ENETDOWN: 38, ENETRESET: 39, ENETUNREACH: 40, ENFILE: 41, ENOBUFS: 42, ENODEV: 43, ENOENT: 44, ENOEXEC: 45, ENOLCK: 46, ENOLINK: 47, ENOMEM: 48, ENOMSG: 49, ENOPROTOOPT: 50, ENOSPC: 51, ENOSYS: 52, ENOTCONN: 53, ENOTDIR: 54, ENOTEMPTY: 55, ENOTRECOVERABLE: 56, ENOTSOCK: 57, ENOTTY: 59, ENXIO: 60, EOVERFLOW: 61, EOWNERDEAD: 62, EPERM: 63, EPIPE: 64, EPROTO: 65, EPROTONOSUPPORT: 66, EPROTOTYPE: 67, ERANGE: 68, EROFS: 69, ESPIPE: 70, ESRCH: 71, ESTALE: 72, ETIMEDOUT: 73, ETXTBSY: 74, EXDEV: 75, ENODATA: 116, ENOTSUP: 138 });

package/host/snapshot/module.json

@@ -2,7 +2,7 @@
   "name": "snapshot",
   "version": 0,
   "gives": "save the session in this browser and restore it",
-  "fixed": "its baseline is the image as loaded",
+  "fixed": "a save is the whole filesystem but /dev, /proc and /run",
   "dependencies": ["runtime@0"],
   "phase": "image",
   "imports": [],

package/host/snapshot/snapshot.mjs

@@ -1,14 +1,16 @@
 import { SessionTransport } from "./transport.mjs";
+import { DOLLY_SESSION_TRANSFER_CAPACITY } from "./abi.mjs";
 import { mountSessionSave } from "./ui.mjs";
-import { DOLLY_BUILD_ID } from "../../dist/dolly-build-id.mjs";
-import { DOLLY_IMAGE_BUILD_ID } from "../../dist/dolly-image-build-id.mjs";
-import { DOLLY_IMAGES } from "../../src/images.mjs";
-import { DOLLY_SESSION_FORMAT_VERSION, decodeSessionSnapshot, loadStoredSession, sessionCompatible,
-  sessionLoadUrl, validSessionName, DOLLY_SESSION_MAX_BYTES } from "../../src/session-store.mjs";
+import { DOLLY_IMAGE_BUILD_ID } from "../../page/generated.mjs";
+import { DOLLY_IMAGES } from "../../page/images.mjs";
+import { sha256 } from "../../page/static-asset.mjs";
+import { DOLLY_SESSION_FORMAT_VERSION, loadStoredSession, sessionCompatible,
+  sessionLoadUrl, validSessionName, DOLLY_SESSION_MAX_BYTES } from "../../site/session-store.mjs";
 
 // A session route (/session/?name=NAME), or boot()'s session option on a page
-// of one's own, boots the saved session's image and restores its delta; with
-// recover=1 the site's route unpacks the files into a fresh system image instead.
+// of one's own, boots the saved session's image from the save, which is its whole
+// filesystem: no image is loaded. With recover=1 the site's route unpacks the
+// files into a fresh system image instead.
 export async function boot(route) {
   if (!route.loadSession) return undefined;
   if (route.mode !== "snapshot") throw new Error("invalid Dolly route configuration");
@@ -25,14 +27,17 @@ export async function boot(route) {
     if (record.formatVersion !== DOLLY_SESSION_FORMAT_VERSION) throw new Error("This save uses an unsupported recovery format");
     if (!DOLLY_IMAGES.some(definition => definition.image === "system")) throw new Error("File recovery needs the system image in this distribution");
     image = "system";
-  } else if (!sessionCompatible(record, DOLLY_IMAGES, DOLLY_BUILD_ID, DOLLY_IMAGE_BUILD_ID)) {
-    throw new Error(`This save belongs to an older runtime or image recipe. It has not been deleted or overwritten.${listed}`);
+  } else if (!sessionCompatible(record, DOLLY_IMAGES, DOLLY_IMAGE_BUILD_ID)) {
+    throw new Error(`This save was made on another build of this version. It has not been deleted or overwritten.${listed}`);
+  } else if (image === "custom" &&
+      await sha256(new TextEncoder().encode(record.customImage.source)) !== record.customImage.artifact.recipeSha256) {
+    // A custom save's Dollyfile names the boot's host modules. The record carries both the file
+    // and its digest, so this proves the save is whole, not where it came from: an imported save
+    // boots with the modules its file names, as a Dollyfile pasted into /custom/ would.
+    throw new Error(`This save's Dollyfile is not the one its image was built from.${listed}`);
   }
-  const bytes = await decodeSessionSnapshot(record);
-  record.bytes.transfer(0);
-  record.bytes = undefined;
   return { image, label: `${recovering ? "RECOVER FILES FROM" : "RESTORE SESSION"} ${name}`,
-    custom: image === "custom" ? record.customImage : undefined, configuration: { bytes, name, recovering } };
+    custom: image === "custom" ? record.customImage : undefined, configuration: { save: record.bytes, name, recovering, updatedAt: record.updatedAt } };
 }
 
 // The page's capture: the Worker reads the session's chunks out of the mailbox
@@ -57,70 +62,88 @@ function sessionCapture({ send, ask }) {
 }
 
 export function browser(page) {
-  const { bytes, name, recovering } = page.configuration, capture = sessionCapture(page);
-  const session = mountSessionSave(page, () => capture, name === undefined ? null : { name, recovering });
+  const { save, name, recovering, updatedAt } = page.configuration, capture = sessionCapture(page);
+  const session = mountSessionSave(page, () => capture, name === undefined ? null : { name, recovering, updatedAt });
   return {
-    configuration: bytes === undefined ? {} : { bytes, ...(recovering ? { recover: name } : {}) },
-    transfers: bytes ? [bytes] : [],
+    configuration: save === undefined ? {} : { save, ...(recovering ? { recover: name } : {}) },
     page: { get sessionName() { return session.name; }, saveSession: session.save },
     claimsKey: session.claimsKey,
     entryStarted: session.entryStarted,
     // An ended image leaves the session this tab saved or restored to open again.
     ended: () => session.name && page.site ? [{ text: `Open saved session ${session.name}`,
       href: sessionLoadUrl(session.name, page.site).href }] : [],
-    messages: { "snapshot-chunk": capture.chunk },
+    messages: { "snapshot-chunk": capture.chunk, "snapshot-changed": ({ changed }) => session.changed(changed) },
     dispose() { session.abort(); },
   };
 }
 
-export function worker({ get, ask, configuration: { bytes, recover } }) {
-  let kernel, transport, cancelled;
-  if (bytes !== undefined && (!(bytes instanceof ArrayBuffer) || bytes.byteLength < 16 ||
-      bytes.byteLength > DOLLY_SESSION_MAX_BYTES)) throw new Error("invalid Dolly session snapshot");
-  if (recover !== undefined && (!validSessionName(recover) || bytes === undefined)) {
-    throw new Error("invalid Dolly file recovery request");
+export function worker({ get, ask, send, configuration: { save, recover } }) {
+  let kernel, transport, cancelled, changed = false;
+  // One step of a save, and word to the page when processes have changed what
+  // the last save holds, or no longer have (the kernel publishes its count here).
+  const service = () => {
+    kernel?.dolly_session_service();
+    if (transport && transport.changed !== changed) send({ type: "snapshot-changed", changed: changed = transport.changed });
+  };
+  if (save !== undefined && (!(save instanceof Blob) || save.size === 0 || save.size > DOLLY_SESSION_MAX_BYTES)) {
+    throw new Error("invalid Dolly session snapshot");
   }
-  function restore(kernel) {
-    const memory = get("runtime").memory;
-    const address = Number(kernel.dolly_session_restore_address(BigInt(bytes.byteLength)));
-    if (!Number.isSafeInteger(address) || address <= 0 || address > memory.buffer.byteLength - bytes.byteLength) {
-      throw new Error("invalid session restore memory range");
-    }
-    new Uint8Array(memory.buffer, address, bytes.byteLength).set(new Uint8Array(bytes));
-    if (kernel.dolly_session_restore(BigInt(bytes.byteLength)) !== 0) {
-      throw new Error("Dolly session filesystem restore failed; the saved copy is unchanged");
-    }
+  if (recover !== undefined && (!validSessionName(recover) || save === undefined)) {
+    throw new Error("invalid Dolly file recovery request");
   }
-  async function recoverFiles({ supervisor, stage, files }) {
-    const path = "/tmp/dolly-session-recovery.delta", destination = `/workspace/recovered-${recover}`;
-    stage(`recovering saved files into ${destination}...`);
-    files.write(path, new Uint8Array(bytes));
-    try {
-      const program = "/usr/bin/session-recover";
-      if ((await supervisor.spawn([program, path, destination])).status !== 0) {
-        throw new Error("File recovery failed; the original saved session is unchanged");
+  // Hands the kernel the save a transfer range at a time, as it is decompressed.
+  async function stream({ kernel, memory, stage }, doing, failed) {
+    const address = Number(kernel.dolly_session_transfer_address());
+    const reader = save.stream().pipeThrough(new DecompressionStream("gzip")).getReader();
+    let taken = 0, reported = performance.now();
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      for (let offset = 0; offset < value.length; offset += DOLLY_SESSION_TRANSFER_CAPACITY) {
+        const piece = value.subarray(offset, offset + DOLLY_SESSION_TRANSFER_CAPACITY);
+        new Uint8Array(memory.buffer, address, piece.length).set(piece);
+        if (kernel.dolly_session_restore_write(BigInt(piece.length), 0) !== 0) throw failed();
       }
-    } finally { files.remove(path); }
+      taken += value.length;
+      if (performance.now() - reported >= 1000) {
+        stage(`${doing}: ${Math.floor(taken / 1024 / 1024)} MiB`);
+        reported = performance.now();
+      }
+    }
+    if (kernel.dolly_session_restore_write(0n, 1) !== 0) throw failed();
+    save = undefined;
   }
+  const restores = () => save !== undefined && recover === undefined;
   return {
+    restoresFilesystem: restores,
+    // The save is the whole filesystem: the kernel empties the boot's and takes it.
+    async restoreFilesystem(context) {
+      if (!restores()) return false;
+      context.stage("restoring the saved session...");
+      if (context.kernel.dolly_session_restore_begin() !== 0) throw new Error("Dolly could not empty the filesystem for the saved session");
+      await stream(context, "restoring the saved session",
+        () => new Error("Dolly session filesystem restore failed; the saved copy is unchanged"));
+      context.stage("saved session restored");
+      return true;
+    },
+    // File recovery: the kernel takes the save's files and directories under
+    // /home and /workspace into a new directory of the system image.
     async imageRestored(context) {
-      const { kernel, image, stage } = context;
-      if (kernel.dolly_session_base_capture() !== 0) throw new Error("Dolly could not index the session baseline");
-      if (bytes === undefined) return;
-      if (recover !== undefined) {
-        if (image !== "system") throw new Error("invalid Dolly file recovery request");
-        await recoverFiles(context);
-      } else {
-        stage("restoring named session filesystem...");
-        restore(kernel);
-        stage("named session filesystem restored");
-      }
-      bytes.transfer(0);
-      bytes = undefined;
+      if (recover === undefined) return;
+      if (context.image !== "system") throw new Error("invalid Dolly file recovery request");
+      const destination = `/workspace/recovered-${recover}`, { kernel, memory, stage } = context;
+      stage(`recovering saved files into ${destination}...`);
+      const failed = () => new Error("File recovery failed; the original saved session is unchanged");
+      const names = new TextEncoder().encode(`${destination}\0/home\0/workspace\0`);
+      new Uint8Array(memory.buffer, Number(kernel.dolly_session_transfer_address()), names.length).set(names);
+      if (kernel.dolly_session_recover_begin(BigInt(names.length)) !== 0) throw failed();
+      await stream(context, "recovering saved files", failed);
     },
-    service() { kernel?.dolly_session_service(); },
+    service,
     messages: {
       "snapshot-capture"() {
+        // The count before the capture clears it, so the page has it ahead of the reply.
+        service();
         const controller = cancelled = new AbortController();
         return transport.capture({ signal: controller.signal,
           onChunk: chunk => ask({ type: "snapshot-chunk", bytes: chunk }, [chunk.buffer]) });