@daugasauron/dolly 0.1.4 → 0.1.5

package/src/session-store.mjs

@@ -1,9 +1,10 @@
 import { alphabetical } from "./alphabetical.mjs";
 import { validName } from "./image-names.mjs";
 import { imageInputs } from "./image-inputs.mjs";
+import { MAX_SNAPSHOT_BYTES } from "./snapshot-records.mjs";
 import { DOLLY_VERSION } from "./version.mjs";
 
-export const DOLLY_SESSION_FORMAT_VERSION = 2;
+export const DOLLY_SESSION_FORMAT_VERSION = 3;
 export const DOLLY_SESSION_MAX_BYTES = 512 * 1024 * 1024;
 export const DOLLY_SESSION_METADATA_MAX_BYTES = 1024 * 1024;
 
@@ -31,7 +32,7 @@ export function customSessionIdentity(custom) {
       typeof artifact.recipeSha256 !== "string" || typeof artifact.sha256 !== "string" ||
       !/^[0-9a-f]{64}$/.test(artifact.recipeSha256) || !/^[0-9a-f]{64}$/.test(artifact.sha256) ||
       !Number.isSafeInteger(artifact.byteLength) || artifact.byteLength <= 0 ||
-      artifact.byteLength > DOLLY_SESSION_MAX_BYTES ||
+      artifact.byteLength > MAX_SNAPSHOT_BYTES ||
       (policies !== undefined && (!Array.isArray(policies) || policies.length === 0 || policies.length > 16 ||
         new TextEncoder().encode(JSON.stringify(policies)).byteLength > 65536))) {
     throw new TypeError("invalid custom session base");
@@ -111,11 +112,9 @@ export async function encodeSessionStream(produce) {
 export async function decodeSessionSnapshot(record) {
   if (record === null || typeof record !== "object" ||
       !(record.bytes instanceof ArrayBuffer) || record.bytes.byteLength === 0 ||
-      record.bytes.byteLength > DOLLY_SESSION_MAX_BYTES ||
-      !["gzip", "identity"].includes(record.encoding)) {
+      record.bytes.byteLength > DOLLY_SESSION_MAX_BYTES || record.encoding !== "gzip") {
     throw new Error("Stored Dolly session is invalid");
   }
-  if (record.encoding === "identity") return record.bytes.slice(0);
   let offset = 0;
   const stream = new ReadableStream({
     pull(controller) {
@@ -178,8 +177,7 @@ export function validateSessionRecord(record) {
       typeof record.imageIdentity !== "string" || record.imageIdentity.length > 256 ||
       !Number.isSafeInteger(record.updatedAt) ||
       !(record.bytes instanceof ArrayBuffer) || record.bytes.byteLength === 0 ||
-      record.bytes.byteLength > DOLLY_SESSION_MAX_BYTES ||
-      !["gzip", "identity"].includes(record.encoding)) {
+      record.bytes.byteLength > DOLLY_SESSION_MAX_BYTES || record.encoding !== "gzip") {
     throw new TypeError("invalid Dolly session record");
   }
   if (record.image === "custom" ? record.imageIdentity !== customSessionIdentity(record.customImage)

package/src/sha256.mjs

@@ -0,0 +1,65 @@
+// SHA-256 over bytes given in pieces, for a digest of more bytes than the page
+// should hold at once; WebCrypto digests only a whole buffer.
+const K = Int32Array.of(
+  0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+  0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+  0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+  0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+  0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+  0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+  0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+  0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2);
+
+export class Sha256 {
+  state = Int32Array.of(0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19);
+  block = new Uint8Array(64);
+  used = 0;
+  length = 0;
+  words = new Int32Array(64);
+
+  compress(view, offset) {
+    const w = this.words, s = this.state;
+    for (let i = 0; i < 16; i++) w[i] = view.getInt32(offset + 4 * i);
+    for (let i = 16; i < 64; i++) {
+      const x = w[i - 15], y = w[i - 2];
+      w[i] = (((x >>> 7 | x << 25) ^ (x >>> 18 | x << 14) ^ x >>> 3) + w[i - 7] +
+        ((y >>> 17 | y << 15) ^ (y >>> 19 | y << 13) ^ y >>> 10) + w[i - 16]) | 0;
+    }
+    let a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7];
+    for (let i = 0; i < 64; i++) {
+      const t1 = (h + ((e >>> 6 | e << 26) ^ (e >>> 11 | e << 21) ^ (e >>> 25 | e << 7)) + (e & f ^ ~e & g) + K[i] + w[i]) | 0;
+      const t2 = (((a >>> 2 | a << 30) ^ (a >>> 13 | a << 19) ^ (a >>> 22 | a << 10)) + (a & b ^ a & c ^ b & c)) | 0;
+      h = g; g = f; f = e; e = (d + t1) | 0; d = c; c = b; b = a; a = (t1 + t2) | 0;
+    }
+    s[0] += a; s[1] += b; s[2] += c; s[3] += d; s[4] += e; s[5] += f; s[6] += g; s[7] += h;
+  }
+
+  update(bytes) {
+    this.length += bytes.length;
+    let offset = 0;
+    if (this.used !== 0) {
+      offset = Math.min(64 - this.used, bytes.length);
+      this.block.set(bytes.subarray(0, offset), this.used);
+      this.used += offset;
+      if (this.used < 64) return;
+      this.compress(new DataView(this.block.buffer), 0);
+      this.used = 0;
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    for (; offset + 64 <= bytes.length; offset += 64) this.compress(view, offset);
+    this.block.set(bytes.subarray(offset));
+    this.used = bytes.length - offset;
+  }
+
+  // The lowercase hex digest; the hash takes no more bytes after it.
+  hex() {
+    const bits = this.length * 8;
+    this.update(Uint8Array.of(0x80));
+    this.update(new Uint8Array((120 - this.used) % 64));
+    const view = new DataView(this.block.buffer);
+    view.setUint32(56, Math.floor(bits / 2 ** 32));
+    view.setUint32(60, bits >>> 0);
+    this.compress(view, 0);
+    return [...this.state].map(word => (word >>> 0).toString(16).padStart(8, "0")).join("");
+  }
+}

package/src/snapshot-records.mjs

@@ -1,6 +1,9 @@
 import { unretainedPath } from "./image-names.mjs";
 const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 export const MAX_SNAPSHOT_BYTES = 2 * 1024 * 1024 * 1024;
+// Packs average 2 MiB or more (scripts/share-pages-snapshots.mjs): an image of
+// MAX_SNAPSHOT_BYTES needs about 1,024, which this bound leaves room for.
+const MAX_SNAPSHOT_PACKS = MAX_SNAPSHOT_BYTES / (1024 * 1024);
 
 function compareBytes(left, right) {
   for (let index = 0; index < Math.min(left.length, right.length); index += 1) {
@@ -9,47 +12,156 @@ function compareBytes(left, right) {
   return left.length - right.length;
 }
 
-export function decodeSnapshotRecords(input) {
-  const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
+// The 16-byte snapshot header: its record count.
+function decodeHeader(bytes) {
   const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-  if (bytes.length < 16 || bytes.length > MAX_SNAPSHOT_BYTES ||
-      decoder.decode(bytes.subarray(0, 8)) !== "DOLLYSNP" || view.getUint32(8, true) !== 2) {
+  if (bytes.length < 16 || decoder.decode(bytes.subarray(0, 8)) !== "DOLLYSNP" || view.getUint32(8, true) !== 2) {
     throw new Error("invalid Dolly snapshot header");
   }
   const count = view.getUint32(12, true);
   if (count === 0 || count > 100_000) throw new Error("invalid Dolly snapshot record count");
+  return count;
+}
+
+// A record's 16-byte header: its kind, path length and data size.
+function decodeRecordHeader(bytes) {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const kind = view.getUint32(0, true), pathLength = view.getUint32(4, true), size = view.getBigUint64(8, true);
+  if (kind < 1 || kind > 3 || pathLength < 2 || pathLength >= 4096 ||
+      size > BigInt(MAX_SNAPSHOT_BYTES) || (kind === 1 && size !== 0n) ||
+      (kind === 3 && (size === 0n || size >= 4096n))) throw new Error("invalid snapshot record size");
+  return { kind, pathLength, size: Number(size) };
+}
+
+// The path of PATHBYTES, valid and after PREVIOUS in byte order.
+function decodePath(pathBytes, previous) {
+  const path = decoder.decode(pathBytes);
+  if (!path.startsWith("/") || /[\0\\\r\n]/.test(path) ||
+      path.slice(1).split("/").some(part => !part || part === "." || part === "..") ||
+      unretainedPath(path) ||
+      previous && compareBytes(previous, pathBytes) >= 0) throw new Error(`invalid snapshot path ${path}`);
+  return path;
+}
+
+function checkParents(records) {
+  for (const path of records.keys()) {
+    for (let slash = path.indexOf("/", 1); slash !== -1; slash = path.indexOf("/", slash + 1)) {
+      const parent = records.get(path.slice(0, slash));
+      if (parent && parent.kind !== 1) throw new Error(`snapshot parent is not a directory: ${path}`);
+    }
+  }
+}
+
+export function decodeSnapshotRecords(input) {
+  const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
+  if (bytes.length > MAX_SNAPSHOT_BYTES) throw new Error("invalid Dolly snapshot header");
+  const count = decodeHeader(bytes.subarray(0, 16));
   const records = new Map();
   let offset = 16, previous;
   for (let index = 0; index < count; index += 1) {
     if (offset > bytes.length - 16) throw new Error("truncated snapshot record");
-    const kind = view.getUint32(offset, true), pathLength = view.getUint32(offset + 4, true);
-    const size = view.getBigUint64(offset + 8, true);
+    const { kind, pathLength, size } = decodeRecordHeader(bytes.subarray(offset, offset + 16));
     offset += 16;
-    if (kind < 1 || kind > 3 || pathLength < 2 || pathLength >= 4096 ||
-        size > BigInt(MAX_SNAPSHOT_BYTES) || (kind === 1 && size !== 0n) ||
-        (kind === 3 && (size === 0n || size >= 4096n))) throw new Error("invalid snapshot record size");
-    if (offset > bytes.length - pathLength - Number(size)) throw new Error("truncated snapshot record");
+    if (offset > bytes.length - pathLength - size) throw new Error("truncated snapshot record");
     const pathBytes = bytes.subarray(offset, offset + pathLength);
-    const path = decoder.decode(pathBytes);
-    if (!path.startsWith("/") || /[\0\\\r\n]/.test(path) ||
-        path.slice(1).split("/").some(part => !part || part === "." || part === "..") ||
-        unretainedPath(path) ||
-        previous && compareBytes(previous, pathBytes) >= 0) throw new Error(`invalid snapshot path ${path}`);
+    const path = decodePath(pathBytes, previous);
     previous = pathBytes;
     offset += pathLength;
-    const data = bytes.subarray(offset, offset + Number(size));
+    const data = bytes.subarray(offset, offset + size);
     if (kind === 3 && data.includes(0)) throw new Error("invalid snapshot symlink");
     records.set(path, { kind, data });
-    offset += Number(size);
+    offset += size;
   }
   if (offset !== bytes.length) throw new Error("trailing snapshot bytes");
-  for (const path of records.keys()) {
-    for (let slash = path.indexOf("/", 1); slash !== -1; slash = path.indexOf("/", slash + 1)) {
-      const parent = records.get(path.slice(0, slash));
-      if (parent && parent.kind !== 1) throw new Error(`snapshot parent is not a directory: ${path}`);
+  checkParents(records);
+  return records;
+}
+
+// Bytes that arrive in pieces (an iterable of Uint8Array), read in order.
+class PieceReader {
+  constructor(pieces) {
+    this.pieces = pieces[Symbol.asyncIterator]?.() ?? pieces[Symbol.iterator]();
+    this.chunk = new Uint8Array(0);
+    this.position = 0;
+  }
+
+  async #fill() {
+    const { done, value } = await this.pieces.next();
+    if (done) return false;
+    if (!(value instanceof Uint8Array)) throw new TypeError("invalid snapshot piece");
+    this.chunk = value;
+    return true;
+  }
+
+  #advance(length) {
+    this.position += length;
+    if (this.position > MAX_SNAPSHOT_BYTES) throw new Error("snapshot exceeds its size limit");
+  }
+
+  // The next LENGTH bytes, a view into the current piece when they lie in one.
+  async take(length) {
+    if (length === 0) return new Uint8Array(0);
+    if (this.chunk.length === 0 && !await this.#fill()) throw new Error("truncated snapshot record");
+    if (this.chunk.length >= length) {
+      const bytes = this.chunk.subarray(0, length);
+      this.chunk = this.chunk.subarray(length);
+      this.#advance(length);
+      return bytes;
     }
+    const bytes = new Uint8Array(length);
+    for (let filled = 0; filled < length;) {
+      if (this.chunk.length === 0 && !await this.#fill()) throw new Error("truncated snapshot record");
+      const taken = Math.min(this.chunk.length, length - filled);
+      bytes.set(this.chunk.subarray(0, taken), filled);
+      this.chunk = this.chunk.subarray(taken);
+      filled += taken;
+    }
+    this.#advance(length);
+    return bytes;
   }
-  return records;
+
+  async skip(length) {
+    while (length !== 0) {
+      if (this.chunk.length === 0 && !await this.#fill()) throw new Error("truncated snapshot record");
+      const taken = Math.min(this.chunk.length, length);
+      this.chunk = this.chunk.subarray(taken);
+      this.#advance(taken);
+      length -= taken;
+    }
+  }
+
+  async ended() {
+    while (this.chunk.length === 0) if (!await this.#fill()) return true;
+    return false;
+  }
+}
+
+// The records of a snapshot that arrives in pieces, checked as
+// decodeSnapshotRecords checks them without holding the snapshot: every path
+// with its kind, where its data lies (offset, size), and the data of the paths
+// KEEP selects (symlink targets always). Resolves to the records and the
+// snapshot's size.
+export async function scanSnapshotRecords(pieces, keep) {
+  const reader = new PieceReader(pieces);
+  const count = decodeHeader(await reader.take(16));
+  const records = new Map();
+  let previous;
+  for (let index = 0; index < count; index += 1) {
+    const { kind, pathLength, size } = decodeRecordHeader(await reader.take(16));
+    const pathBytes = await reader.take(pathLength);
+    const path = decodePath(pathBytes, previous);
+    previous = pathBytes;
+    const offset = reader.position;
+    let data;
+    if (kind === 3 || (kind === 2 && keep(path))) {
+      data = await reader.take(size);
+      if (kind === 3 && data.includes(0)) throw new Error("invalid snapshot symlink");
+    } else await reader.skip(size);
+    records.set(path, { kind, data, offset, size });
+  }
+  if (!await reader.ended()) throw new Error("trailing snapshot bytes");
+  checkParents(records);
+  return { records, byteLength: reader.position };
 }
 
 export function encodeSnapshotRecords(records) {
@@ -91,7 +203,7 @@ export function mergeSnapshotRecords(parts) {
 }
 
 export function validateSnapshotPacks(metadata) {
-  if (!Array.isArray(metadata.packs) || metadata.packs.length === 0 || metadata.packs.length > 256) throw new Error("invalid snapshot packs");
+  if (!Array.isArray(metadata.packs) || metadata.packs.length === 0 || metadata.packs.length > MAX_SNAPSHOT_PACKS) throw new Error("invalid snapshot packs");
   const hashes = new Set();
   let size = 16;
   for (const pack of metadata.packs) {

package/src/static-asset.mjs

@@ -55,12 +55,39 @@ export async function boundedAssetBody(response, limit, signal) {
   return bytes;
 }
 
+// A file a host stores in parts is a JSON manifest at its path that begins
+// with these bytes, and no file stored whole does (the Pages exporter checks),
+// so the page knows a manifest by its content: a host needs no header for it.
+// Pages before v0.1.5 read the X-Dolly-Parts header instead.
+export const PARTS_MANIFEST_START = '{"byteLength":';
+
+// Whether a whole response's body begins as a manifest, read from a copy of it.
+async function isPartsManifest(response) {
+  if (response.status !== 200 || !response.body) return false;
+  const start = new TextEncoder().encode(PARTS_MANIFEST_START), reader = response.clone().body.getReader();
+  const head = new Uint8Array(start.length);
+  let size = 0;
+  try {
+    while (size < start.length) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      const taken = value.subarray(0, start.length - size);
+      head.set(taken, size);
+      size += taken.length;
+    }
+  } finally {
+    // Not awaited: cancelling a copy settles only once the response is read too.
+    reader.cancel().catch(() => {});
+  }
+  return size === start.length && head.every((byte, index) => byte === start[index]);
+}
+
 // Streams verified parts, holding at most one 20 MiB part per request. Each
 // part matches the manifest hash before any of its bytes are delivered.
 export async function decodeStaticAsset(response, target, init, maximumBytes, fetchRequest = globalThis.fetch.bind(globalThis)) {
-  if (response.headers.get("x-dolly-parts") !== "1") return response;
+  if (!await isPartsManifest(response)) return response;
   const url = new URL(target);
-  if (response.status !== 200 || (init.method ?? "GET") !== "GET" || url.search || url.hash ||
+  if ((init.method ?? "GET") !== "GET" || url.search || url.hash ||
       url.username || url.password || !/^https?:$/.test(url.protocol)) throw new Error("invalid multipart asset request");
   const manifest = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(await boundedAssetBody(response, 65536, init.signal)));
   if (!Number.isSafeInteger(manifest.byteLength) || manifest.byteLength <= 0 ||

package/src/terminal-text.mjs

@@ -1,3 +1,6 @@
+import { DOLLY_POINTER_ACTION_DRAG as drag, DOLLY_POINTER_ACTION_PRESS as press,
+  DOLLY_POINTER_ACTION_RELEASE as release } from "../host/input/abi.mjs";
+
 // How the page's handle (docs/embedding.md) reads the terminal, over the page APIs of
 // display@0, input@0 and the runtime: the screen's text is the terminal's own
 // selection of every cell, made with pointer records.
@@ -36,10 +39,10 @@ export function terminalText(page) {
       const y = geometry.paddingY + Math.floor(geometry.cellHeight / 2);
       const endX = x + (dimensions.cols - 1) * geometry.cellWidth, endY = y + (dimensions.rows - 1) * geometry.cellHeight;
       let selected;
-      await inputTransport.pushPointer(x, y, 1, shifted);
+      await inputTransport.pushPointer(x, y, press, shifted);
       await waitFor(async () => await transport.copySelection() === null, "terminal selection reset");
-      await inputTransport.pushPointer(endX, endY, 2, shifted);
-      await inputTransport.pushPointer(endX, endY, 0, shifted);
+      await inputTransport.pushPointer(endX, endY, drag, shifted);
+      await inputTransport.pushPointer(endX, endY, release, shifted);
       if (await waitFor(async () => (selected = await transport.copySelection()) !== null, null, attempts)) {
         return selected.split("\n").slice(-dimensions.rows).join("\n");
       }
@@ -62,8 +65,8 @@ export function terminalText(page) {
     const geometry = transport.geometry();
     const x = geometry.paddingX + Math.floor(geometry.cellWidth / 2);
     const y = geometry.paddingY + Math.floor(geometry.cellHeight / 2);
-    await inputTransport.pushPointer(x, y, 1, shifted);
-    await inputTransport.pushPointer(x, y, 0, shifted);
+    await inputTransport.pushPointer(x, y, press, shifted);
+    await inputTransport.pushPointer(x, y, release, shifted);
     await waitFor(() => inputTransport.inputIdle(), "terminal selection cleanup");
     return pid;
   }

package/src/version.mjs

@@ -1,3 +1,3 @@
 // The version being developed: the release tag vX.Y.Z, the site path /vX.Y.Z/
 // and the prefix of every recipe reference. package.json holds the same value.
-export const DOLLY_VERSION = "0.1.4";
+export const DOLLY_VERSION = "0.1.5";

package/src/worker-image.mjs

@@ -3,7 +3,6 @@
 // image into that kernel, and runs its ENTRY. runtime-worker.mjs restores an
 // image of the image directory; the site's build-worker.mjs builds one or
 // restores a custom one.
-import { DOLLY_BUILD_ID } from "../dist/dolly-build-id.mjs";
 import { loadImages } from "./images.mjs";
 import { decodeImageEntry } from "./image-entry.mjs";
 import { MAX_SNAPSHOT_BYTES } from "./snapshot-records.mjs";
@@ -28,7 +27,7 @@ const configured = new Promise((resolve, reject) => {
 const { hostRegistry } = await import("../host/modules.mjs");
 
 // The runtime's own files: the kernel and the contracts beside this module.
-export const runtimeBase = new URL("../", import.meta.url);
+const runtimeBase = new URL("../", import.meta.url);
 export const kernelArtifact = path => {
   if (path !== "dolly.wasm" && path !== "dolly.data") throw new Error("unknown fixed kernel artifact");
   return new URL(`dist/${path}`, runtimeBase).href;
@@ -45,7 +44,7 @@ export function checkedMemoryRange(memory, addressValue, sizeValue) {
   return { address, size };
 }
 
-const stage = text => self.postMessage({ type: "bootstrap", text: `${text}\n`, error: false });
+const stage = text => self.postMessage({ type: "bootstrap", text: `${text}\n` });
 
 function acknowledged(type, failure) {
   return new Promise((resolve, reject) => {
@@ -111,8 +110,7 @@ export async function runImage(image) {
     }
     stage("starting image entry...");
     const entryReady = acknowledged("entry-ready-ack", "browser did not acknowledge image startup");
-    self.postMessage({ type: "ready", image: runtimeImage, routeImage: steps.name, buildId: DOLLY_BUILD_ID,
-      hostModules: host.enabled });
+    self.postMessage({ type: "ready", image: runtimeImage });
     await entryReady;
 
     // The image ends with its ENTRY process, however that ended: a program that