@daugasauron/dolly-build 0.0.0-stage → 0.1.5

package/src/process-abi.mjs

@@ -0,0 +1,69 @@
+import { formatWasmType, sameWasmType } from "./wasm-interface.mjs";
+import { hex } from "./static-asset.mjs";
+
+export function unique(entries, key) {
+  const result = new Map();
+  for (const entry of entries) {
+    const name = key(entry);
+    if (result.has(name)) throw new TypeError(`duplicate process interface entry ${name}`);
+    result.set(name, entry);
+  }
+  return result;
+}
+
+// Both the build tools and browser admission call this exact validator on
+// parsed bytes. The contract is assembled from abi/dolly-process-0.wat.
+export function validateProcessInterface(contract, process, digest) {
+  const fail = message => { throw new TypeError(`${process.label}: ${message}`); };
+  const section = name => {
+    const entries = process.customSectionData.filter(x => x.name === name);
+    if (entries.length !== 1) fail(`expected exactly one ${name} section`);
+    return entries[0].data;
+  };
+  if (process.customSections.includes("dylink.0")) {
+    fail("a process executable must not be a side module");
+  }
+  const stamp = hex(section("dolly.process"));
+  if (stamp !== digest) fail("wrong dolly.process stamp");
+  const key = x => `${x.module}.${x.name}`;
+  const allowed = unique(contract.imports, key);
+  const actual = unique(process.imports, key);
+  if (allowed.size !== 2 || !allowed.has("env.memory") || !allowed.has("dolly_process_0.call")) {
+    throw new TypeError("process contract must contain only memory and call");
+  }
+  for (const name of actual.keys()) if (!allowed.has(name)) fail(`import ${name} is outside dolly-process-0`);
+  if (actual.size !== allowed.size) fail("expected exactly the two dolly-process-0 imports");
+  let memory;
+  for (const [name, expected] of allowed) {
+    const entry = actual.get(name);
+    if (!entry) fail(`missing required import ${name}`);
+    const type = entry.type;
+    if (name === "env.memory") {
+      if (type.kind !== "memory" || expected.type.kind !== "memory" ||
+          type.address64 !== expected.type.address64 || type.shared !== expected.type.shared ||
+          type.minimum < expected.type.minimum || type.maximum === null ||
+          expected.type.maximum === null || type.maximum > expected.type.maximum ||
+          type.minimum > type.maximum) {
+        fail(`process memory is outside ${formatWasmType(expected.type)}`);
+      }
+      const bytes = section("dolly.process.memory");
+      if (bytes.length !== 16) fail("expected a 16-byte dolly.process.memory section");
+      const record = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+      if (record.getBigUint64(0, true) !== type.minimum ||
+          record.getBigUint64(8, true) !== type.maximum) {
+        fail("dolly.process.memory does not match its memory import");
+      }
+      memory = { initial: type.minimum, maximum: type.maximum };
+    } else if (!sameWasmType(type, expected.type)) {
+      fail(`import ${name} must be ${formatWasmType(expected.type)}`);
+    }
+  }
+  const exports = unique(process.exports, x => x.name);
+  for (const expected of contract.exports) {
+    const entry = exports.get(expected.name);
+    if (!entry || !sameWasmType(entry.type, expected.type)) {
+      fail(`missing or incompatible process export ${expected.name}`);
+    }
+  }
+  return memory;
+}

package/src/snapshot-records.mjs

@@ -0,0 +1,221 @@
+import { unretainedPath } from "./image-names.mjs";
+const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+export const MAX_SNAPSHOT_BYTES = 2 * 1024 * 1024 * 1024;
+// Packs average 2 MiB or more (scripts/share-pages-snapshots.mjs): an image of
+// MAX_SNAPSHOT_BYTES needs about 1,024, which this bound leaves room for.
+const MAX_SNAPSHOT_PACKS = MAX_SNAPSHOT_BYTES / (1024 * 1024);
+
+function compareBytes(left, right) {
+  for (let index = 0; index < Math.min(left.length, right.length); index += 1) {
+    if (left[index] !== right[index]) return left[index] - right[index];
+  }
+  return left.length - right.length;
+}
+
+// The 16-byte snapshot header: its record count.
+function decodeHeader(bytes) {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  if (bytes.length < 16 || decoder.decode(bytes.subarray(0, 8)) !== "DOLLYSNP" || view.getUint32(8, true) !== 2) {
+    throw new Error("invalid Dolly snapshot header");
+  }
+  const count = view.getUint32(12, true);
+  if (count === 0 || count > 100_000) throw new Error("invalid Dolly snapshot record count");
+  return count;
+}
+
+// A record's 16-byte header: its kind, path length and data size.
+function decodeRecordHeader(bytes) {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const kind = view.getUint32(0, true), pathLength = view.getUint32(4, true), size = view.getBigUint64(8, true);
+  if (kind < 1 || kind > 3 || pathLength < 2 || pathLength >= 4096 ||
+      size > BigInt(MAX_SNAPSHOT_BYTES) || (kind === 1 && size !== 0n) ||
+      (kind === 3 && (size === 0n || size >= 4096n))) throw new Error("invalid snapshot record size");
+  return { kind, pathLength, size: Number(size) };
+}
+
+// The path of PATHBYTES, valid and after PREVIOUS in byte order.
+function decodePath(pathBytes, previous) {
+  const path = decoder.decode(pathBytes);
+  if (!path.startsWith("/") || /[\0\\\r\n]/.test(path) ||
+      path.slice(1).split("/").some(part => !part || part === "." || part === "..") ||
+      unretainedPath(path) ||
+      previous && compareBytes(previous, pathBytes) >= 0) throw new Error(`invalid snapshot path ${path}`);
+  return path;
+}
+
+function checkParents(records) {
+  for (const path of records.keys()) {
+    for (let slash = path.indexOf("/", 1); slash !== -1; slash = path.indexOf("/", slash + 1)) {
+      const parent = records.get(path.slice(0, slash));
+      if (parent && parent.kind !== 1) throw new Error(`snapshot parent is not a directory: ${path}`);
+    }
+  }
+}
+
+export function decodeSnapshotRecords(input) {
+  const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
+  if (bytes.length > MAX_SNAPSHOT_BYTES) throw new Error("invalid Dolly snapshot header");
+  const count = decodeHeader(bytes.subarray(0, 16));
+  const records = new Map();
+  let offset = 16, previous;
+  for (let index = 0; index < count; index += 1) {
+    if (offset > bytes.length - 16) throw new Error("truncated snapshot record");
+    const { kind, pathLength, size } = decodeRecordHeader(bytes.subarray(offset, offset + 16));
+    offset += 16;
+    if (offset > bytes.length - pathLength - size) throw new Error("truncated snapshot record");
+    const pathBytes = bytes.subarray(offset, offset + pathLength);
+    const path = decodePath(pathBytes, previous);
+    previous = pathBytes;
+    offset += pathLength;
+    const data = bytes.subarray(offset, offset + size);
+    if (kind === 3 && data.includes(0)) throw new Error("invalid snapshot symlink");
+    records.set(path, { kind, data });
+    offset += size;
+  }
+  if (offset !== bytes.length) throw new Error("trailing snapshot bytes");
+  checkParents(records);
+  return records;
+}
+
+// Bytes that arrive in pieces (an iterable of Uint8Array), read in order.
+class PieceReader {
+  constructor(pieces) {
+    this.pieces = pieces[Symbol.asyncIterator]?.() ?? pieces[Symbol.iterator]();
+    this.chunk = new Uint8Array(0);
+    this.position = 0;
+  }
+
+  async #fill() {
+    const { done, value } = await this.pieces.next();
+    if (done) return false;
+    if (!(value instanceof Uint8Array)) throw new TypeError("invalid snapshot piece");
+    this.chunk = value;
+    return true;
+  }
+
+  #advance(length) {
+    this.position += length;
+    if (this.position > MAX_SNAPSHOT_BYTES) throw new Error("snapshot exceeds its size limit");
+  }
+
+  // The next LENGTH bytes, a view into the current piece when they lie in one.
+  async take(length) {
+    if (length === 0) return new Uint8Array(0);
+    if (this.chunk.length === 0 && !await this.#fill()) throw new Error("truncated snapshot record");
+    if (this.chunk.length >= length) {
+      const bytes = this.chunk.subarray(0, length);
+      this.chunk = this.chunk.subarray(length);
+      this.#advance(length);
+      return bytes;
+    }
+    const bytes = new Uint8Array(length);
+    for (let filled = 0; filled < length;) {
+      if (this.chunk.length === 0 && !await this.#fill()) throw new Error("truncated snapshot record");
+      const taken = Math.min(this.chunk.length, length - filled);
+      bytes.set(this.chunk.subarray(0, taken), filled);
+      this.chunk = this.chunk.subarray(taken);
+      filled += taken;
+    }
+    this.#advance(length);
+    return bytes;
+  }
+
+  async skip(length) {
+    while (length !== 0) {
+      if (this.chunk.length === 0 && !await this.#fill()) throw new Error("truncated snapshot record");
+      const taken = Math.min(this.chunk.length, length);
+      this.chunk = this.chunk.subarray(taken);
+      this.#advance(taken);
+      length -= taken;
+    }
+  }
+
+  async ended() {
+    while (this.chunk.length === 0) if (!await this.#fill()) return true;
+    return false;
+  }
+}
+
+// The records of a snapshot that arrives in pieces, checked as
+// decodeSnapshotRecords checks them without holding the snapshot: every path
+// with its kind, where its data lies (offset, size), and the data of the paths
+// KEEP selects (symlink targets always). Resolves to the records and the
+// snapshot's size.
+export async function scanSnapshotRecords(pieces, keep) {
+  const reader = new PieceReader(pieces);
+  const count = decodeHeader(await reader.take(16));
+  const records = new Map();
+  let previous;
+  for (let index = 0; index < count; index += 1) {
+    const { kind, pathLength, size } = decodeRecordHeader(await reader.take(16));
+    const pathBytes = await reader.take(pathLength);
+    const path = decodePath(pathBytes, previous);
+    previous = pathBytes;
+    const offset = reader.position;
+    let data;
+    if (kind === 3 || (kind === 2 && keep(path))) {
+      data = await reader.take(size);
+      if (kind === 3 && data.includes(0)) throw new Error("invalid snapshot symlink");
+    } else await reader.skip(size);
+    records.set(path, { kind, data, offset, size });
+  }
+  if (!await reader.ended()) throw new Error("trailing snapshot bytes");
+  checkParents(records);
+  return { records, byteLength: reader.position };
+}
+
+export function encodeSnapshotRecords(records) {
+  const encoder = new TextEncoder();
+  const entries = [...records].map(([path, record]) => ({ path: encoder.encode(path), ...record }))
+    .sort((left, right) => compareBytes(left.path, right.path));
+  const size = entries.reduce((sum, record) => sum + 16 + record.path.length + record.data.length, 16);
+  if (size > MAX_SNAPSHOT_BYTES || entries.length === 0 || entries.length > 100_000) throw new Error("snapshot exceeds record limits");
+  const bytes = new Uint8Array(size), view = new DataView(bytes.buffer);
+  bytes.set(encoder.encode("DOLLYSNP"));
+  view.setUint32(8, 2, true);
+  view.setUint32(12, entries.length, true);
+  let offset = 16;
+  for (const record of entries) {
+    view.setUint32(offset, record.kind, true);
+    view.setUint32(offset + 4, record.path.length, true);
+    view.setBigUint64(offset + 8, BigInt(record.data.length), true);
+    bytes.set(record.path, offset + 16);
+    bytes.set(record.data, offset + 16 + record.path.length);
+    offset += 16 + record.path.length + record.data.length;
+  }
+  return bytes;
+}
+
+export function mergeSnapshotRecords(parts) {
+  const records = new Map();
+  let size = 16;
+  for (const part of parts) {
+    for (const [path, record] of decodeSnapshotRecords(part)) {
+      if (records.has(path)) throw new Error(`duplicate packed snapshot path: ${path}`);
+      size += 16 + new TextEncoder().encode(path).length + record.data.length;
+      if (size > MAX_SNAPSHOT_BYTES || records.size >= 100_000) throw new Error("packed snapshot exceeds limits");
+      records.set(path, record);
+    }
+  }
+  const bytes = encodeSnapshotRecords(records);
+  decodeSnapshotRecords(bytes);
+  return bytes;
+}
+
+export function validateSnapshotPacks(metadata) {
+  if (!Array.isArray(metadata.packs) || metadata.packs.length === 0 || metadata.packs.length > MAX_SNAPSHOT_PACKS) throw new Error("invalid snapshot packs");
+  const hashes = new Set();
+  let size = 16;
+  for (const pack of metadata.packs) {
+    if (!/^[0-9a-f]{64}$/.test(pack.sha256) || hashes.has(pack.sha256) ||
+        !Number.isSafeInteger(pack.byteLength) || pack.byteLength < 16 || pack.byteLength > MAX_SNAPSHOT_BYTES ||
+        !Number.isSafeInteger(pack.encodedByteLength) || pack.encodedByteLength <= 0 || pack.encodedByteLength > MAX_SNAPSHOT_BYTES) {
+      throw new Error("invalid snapshot pack descriptor");
+    }
+    hashes.add(pack.sha256);
+    size += pack.byteLength - 16;
+    if (size > MAX_SNAPSHOT_BYTES) throw new Error("packed snapshot exceeds size limit");
+  }
+  if (size !== metadata.byteLength) throw new Error("snapshot pack sizes do not match image");
+  return metadata.packs;
+}

package/src/static-asset.mjs

@@ -0,0 +1,124 @@
+import { MAX_SNAPSHOT_BYTES } from "./snapshot-records.mjs";
+import { DOLLY_VERSION } from "./version.mjs";
+
+// Static delivery for an already authorized bootstrap input or snapshot pack.
+// Hosts cap file sizes, so assets travel as verified parts; nothing static is
+// larger than the largest snapshot.
+export const STATIC_PART_BYTES = 20 * 1024 * 1024;
+export const staticMaxParts = () => Math.ceil(MAX_SNAPSHOT_BYTES / STATIC_PART_BYTES);
+
+export const hex = bytes =>
+  Array.from(bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes), byte => byte.toString(16).padStart(2, "0")).join("");
+
+export async function sha256(bytes) {
+  return hex(await crypto.subtle.digest("SHA-256", bytes));
+}
+
+// Recipes name a file this site publishes as a path that starts with the
+// version, /v0.1.0/Dollyfile-system. Every page serves its own version's
+// files wherever it is mounted, so where their bytes come from never changes
+// recipe text, pins or image identity.
+const sitePrefix = `/v${DOLLY_VERSION}`;
+export const siteReference = path => `${sitePrefix}/${path}`;
+
+// The public site, for the pages of another site that link to it. A recipe
+// that names it by URL instead of a site path fails the lint.
+export const PUBLIC_ORIGIN = "https://slopshare.dev";
+
+// The path ("/Dollyfile-system") a site reference names among this page's
+// files, or null for an absolute URL. Another version's files are not here.
+export function sitePath(reference) {
+  if (!reference.startsWith("/")) return null;
+  if (!reference.startsWith(`${sitePrefix}/`)) {
+    throw new Error(`${reference}: this is Dolly ${DOLLY_VERSION}, which reads only ${sitePrefix}/ paths`);
+  }
+  return reference.slice(sitePrefix.length);
+}
+
+export async function boundedAssetBody(response, limit, signal) {
+  if (!response.ok || !response.body) throw new Error(`asset returned HTTP ${response.status}`);
+  const reader = response.body.getReader(), chunks = [];
+  let size = 0;
+  try {
+    for (;;) {
+      signal?.throwIfAborted();
+      const { done, value } = await reader.read();
+      if (done) break;
+      size += value.length;
+      if (size > limit) { await reader.cancel(); throw new Error("asset exceeds declared size"); }
+      chunks.push(value);
+    }
+  } finally { reader.releaseLock(); }
+  const bytes = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; }
+  return bytes;
+}
+
+// A file a host stores in parts is a JSON manifest at its path that begins
+// with these bytes, and no file stored whole does (the Pages exporter checks),
+// so the page knows a manifest by its content: a host needs no header for it.
+// Pages before v0.1.5 read the X-Dolly-Parts header instead.
+export const PARTS_MANIFEST_START = '{"byteLength":';
+
+// Whether a whole response's body begins as a manifest, read from a copy of it.
+async function isPartsManifest(response) {
+  if (response.status !== 200 || !response.body) return false;
+  const start = new TextEncoder().encode(PARTS_MANIFEST_START), reader = response.clone().body.getReader();
+  const head = new Uint8Array(start.length);
+  let size = 0;
+  try {
+    while (size < start.length) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      const taken = value.subarray(0, start.length - size);
+      head.set(taken, size);
+      size += taken.length;
+    }
+  } finally {
+    // Not awaited: cancelling a copy settles only once the response is read too.
+    reader.cancel().catch(() => {});
+  }
+  return size === start.length && head.every((byte, index) => byte === start[index]);
+}
+
+// Streams verified parts, holding at most one 20 MiB part per request. Each
+// part matches the manifest hash before any of its bytes are delivered.
+export async function decodeStaticAsset(response, target, init, maximumBytes, fetchRequest = globalThis.fetch.bind(globalThis)) {
+  if (!await isPartsManifest(response)) return response;
+  const url = new URL(target);
+  if ((init.method ?? "GET") !== "GET" || url.search || url.hash ||
+      url.username || url.password || !/^https?:$/.test(url.protocol)) throw new Error("invalid multipart asset request");
+  const manifest = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(await boundedAssetBody(response, 65536, init.signal)));
+  if (!Number.isSafeInteger(manifest.byteLength) || manifest.byteLength <= 0 ||
+      manifest.byteLength > maximumBytes || manifest.byteLength > MAX_SNAPSHOT_BYTES ||
+      !Array.isArray(manifest.parts) || manifest.parts.length < 2 || manifest.parts.length > staticMaxParts() ||
+      manifest.parts.some(part => !Number.isSafeInteger(part.byteLength) || part.byteLength <= 0 ||
+        part.byteLength > STATIC_PART_BYTES || !/^[0-9a-f]{64}$/.test(part.sha256)) ||
+      manifest.parts.reduce((size, part) => size + part.byteLength, 0) !== manifest.byteLength) {
+    throw new Error("invalid multipart asset manifest");
+  }
+  let index = 0;
+  const body = new ReadableStream({
+    async pull(controller) {
+      if (index === manifest.parts.length) { controller.close(); return; }
+      const part = manifest.parts[index];
+      // The manifest supplies no destinations. Parts are fixed siblings, with
+      // no guest headers, credentials, query strings or redirect authority.
+      const partURL = new URL(url);
+      partURL.pathname += `.part-${index++}`;
+      const result = await fetchRequest(partURL, { method: "GET", credentials: "omit", redirect: "error",
+        referrerPolicy: "no-referrer", cache: "force-cache", signal: init.signal });
+      const chunk = await boundedAssetBody(result, part.byteLength, init.signal);
+      if (chunk.length !== part.byteLength || await sha256(chunk) !== part.sha256) throw new Error("asset part integrity mismatch");
+      controller.enqueue(chunk);
+    },
+  }, { highWaterMark: 0 });
+  const headers = new Headers(response.headers);
+  headers.delete("x-dolly-parts");
+  headers.delete("content-encoding");
+  headers.set("content-length", String(manifest.byteLength));
+  const decoded = new Response(body, { status: response.status, statusText: response.statusText, headers });
+  Object.defineProperty(decoded, "url", { value: response.url });
+  return decoded;
+}

package/src/version.mjs

@@ -0,0 +1,3 @@
+// The version being developed: the release tag vX.Y.Z, the site path /vX.Y.Z/
+// and the prefix of every recipe reference. package.json holds the same value.
+export const DOLLY_VERSION = "0.1.5";