@daugasauron/dolly 0.0.0-stage → 0.1.4-rc.0

package/src/wasm-interface.mjs

@@ -0,0 +1,440 @@
+const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+const valueTypes = new Map([
+  [0x7f, "i32"],
+  [0x7e, "i64"],
+  [0x7d, "f32"],
+  [0x7c, "f64"],
+  [0x7b, "v128"],
+  [0x70, "funcref"],
+  [0x6f, "externref"],
+  [0x69, "exnref"],
+  [0x74, "nullexnref"],
+]);
+
+export class Reader {
+  constructor(bytes, label = "WebAssembly binary") {
+    this.bytes = bytes;
+    this.offset = 0;
+    this.label = label;
+  }
+
+  get done() {
+    return this.offset === this.bytes.length;
+  }
+
+  fail(message) {
+    throw new Error(`${this.label} at byte ${this.offset}: ${message}`);
+  }
+
+  u8() {
+    if (this.offset >= this.bytes.length) this.fail("unexpected end of file");
+    return this.bytes[this.offset++];
+  }
+
+  take(length) {
+    if (!Number.isSafeInteger(length) || length < 0 || this.offset + length > this.bytes.length) {
+      this.fail(`invalid byte range of length ${length}`);
+    }
+    const result = this.bytes.subarray(this.offset, this.offset + length);
+    this.offset += length;
+    return result;
+  }
+
+  subreader(length, label) {
+    return new Reader(this.take(length), label);
+  }
+
+  unsigned(bits = 32) {
+    let result = 0n;
+    let shift = 0n;
+    const limit = Math.ceil(bits / 7);
+    for (let index = 0; index < limit; index += 1) {
+      const byte = this.u8();
+      result |= BigInt(byte & 0x7f) << shift;
+      if ((byte & 0x80) === 0) {
+        if (result >= (1n << BigInt(bits))) this.fail(`unsigned LEB exceeds ${bits} bits`);
+        return result;
+      }
+      shift += 7n;
+    }
+    this.fail(`unterminated unsigned ${bits}-bit LEB`);
+  }
+
+  signed(bits = 32) {
+    let result = 0n;
+    let shift = 0n;
+    let byte = 0;
+    const limit = Math.ceil(bits / 7);
+    for (let index = 0; index < limit; index += 1) {
+      byte = this.u8();
+      result |= BigInt(byte & 0x7f) << shift;
+      shift += 7n;
+      if ((byte & 0x80) === 0) {
+        if ((byte & 0x40) !== 0 && shift < BigInt(bits)) result |= (-1n) << shift;
+        return result;
+      }
+    }
+    this.fail(`unterminated signed ${bits}-bit LEB`);
+  }
+
+  u32() {
+    return Number(this.unsigned(32));
+  }
+
+  string() {
+    const length = this.u32();
+    try {
+      return decoder.decode(this.take(length));
+    } catch {
+      this.fail("invalid UTF-8 string");
+    }
+  }
+}
+
+function readValueType(reader) {
+  const byte = reader.u8();
+  const type = valueTypes.get(byte);
+  if (!type) reader.fail(`unsupported value type 0x${byte.toString(16)}`);
+  return type;
+}
+
+function readVector(reader, readElement, result = []) {
+  const count = reader.u32();
+  if (count > reader.bytes.length - reader.offset) reader.fail("vector count exceeds section size");
+  for (let index = 0; index < count; index++) result.push(readElement(reader));
+  return result;
+}
+
+function readLimits(reader) {
+  const flags = reader.u32();
+  if ((flags & ~0x7) !== 0) reader.fail(`unsupported limits flags 0x${flags.toString(16)}`);
+  const hasMaximum = (flags & 0x1) !== 0;
+  const shared = (flags & 0x2) !== 0;
+  const address64 = (flags & 0x4) !== 0;
+  const readBound = () => reader.unsigned(address64 ? 64 : 32);
+  const minimum = readBound();
+  const maximum = hasMaximum ? readBound() : null;
+  if (shared && maximum === null) reader.fail("shared limits require a maximum");
+  return { minimum, maximum, shared, address64 };
+}
+
+function readTableType(reader) {
+  return { kind: "table", element: readValueType(reader), ...readLimits(reader) };
+}
+
+function readMemoryType(reader) {
+  return { kind: "memory", ...readLimits(reader) };
+}
+
+function readGlobalType(reader) {
+  return { kind: "global", value: readValueType(reader), mutable: reader.u8() !== 0 };
+}
+
+function readTagType(reader, types) {
+  const attribute = reader.u32();
+  return { kind: "tag", attribute, type: functionType(types, reader.u32(), reader) };
+}
+
+function skipConstantExpression(reader) {
+  for (;;) {
+    const opcode = reader.u8();
+    switch (opcode) {
+      case 0x0b:
+        return;
+      case 0x41:
+        reader.signed(32);
+        break;
+      case 0x42:
+        reader.signed(64);
+        break;
+      case 0x43:
+        reader.take(4);
+        break;
+      case 0x44:
+        reader.take(8);
+        break;
+      case 0x23:
+      case 0xd2:
+        reader.u32();
+        break;
+      case 0xd0:
+        readValueType(reader);
+        break;
+      case 0x6a: // i32.add
+      case 0x6b: // i32.sub
+      case 0x6c: // i32.mul
+      case 0x7c: // i64.add
+      case 0x7d: // i64.sub
+      case 0x7e: // i64.mul
+        break;
+      default:
+        reader.fail(`unsupported constant-expression opcode 0x${opcode.toString(16)}`);
+    }
+  }
+}
+
+// An element of an expression segment: the function a ref.func names, or
+// null for ref.null.
+function readElementExpression(reader) {
+  const opcode = reader.u8();
+  let index = null;
+  if (opcode === 0xd2) index = reader.u32();
+  else if (opcode === 0xd0) readValueType(reader);
+  else reader.fail(`unsupported element expression opcode 0x${opcode.toString(16)}`);
+  if (reader.u8() !== 0x0b) reader.fail("element expression has more than one instruction");
+  return index;
+}
+
+// The functions an element segment puts into a table: an active segment's,
+// none for a passive or declarative one (flag bit 0).
+function readElementSegment(reader) {
+  const flags = reader.u32();
+  if (flags > 7) reader.fail(`unsupported element segment flags ${flags}`);
+  const active = (flags & 0x1) === 0;
+  if (flags === 2 || flags === 6) reader.u32();
+  if (active) skipConstantExpression(reader);
+  if (flags & 0x4) {
+    if (flags !== 4) readValueType(reader);
+    const indices = readVector(reader, readElementExpression);
+    return active ? indices.filter(index => index !== null) : [];
+  }
+  if (flags !== 0 && reader.u8() !== 0x00) reader.fail("unsupported element kind");
+  const indices = readVector(reader, item => item.u32());
+  return active ? indices : [];
+}
+
+// Binaryen's --fpcast-emu leaves one signature in the table, which every
+// indirect call uses: thunks of i64 parameters, sixteen or more
+// (max-func-params), and an i64 result. Their parameter count, or 0.
+function castEmulationParameters(tableFunctions) {
+  const [type] = tableFunctions;
+  const thunk = type && type.params.length >= 16 && type.params.every(param => param === "i64") &&
+    type.results.length === 1 && type.results[0] === "i64";
+  return thunk && tableFunctions.every(other => other === type) ? type.params.length : 0;
+}
+
+function functionType(types, index, reader) {
+  const type = types[index];
+  if (!type) reader.fail(`function references missing type ${index}`);
+  return type;
+}
+
+export function parseWasmInterface(input, label = "WebAssembly binary") {
+  const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
+  const reader = new Reader(bytes, label);
+  const expectedHeader = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00];
+  for (const byte of expectedHeader) {
+    if (reader.u8() !== byte) reader.fail("invalid WebAssembly header");
+  }
+
+  const types = [];
+  const imports = [];
+  const definedFunctionTypes = [];
+  const importedTables = [];
+  const definedTables = [];
+  const importedMemories = [];
+  const definedMemories = [];
+  const importedGlobals = [];
+  const definedGlobals = [];
+  const importedTags = [];
+  const definedTags = [];
+  const rawExports = [];
+  const tableFunctionIndices = [];
+  const customSections = [];
+  const customSectionData = [];
+
+  while (!reader.done) {
+    const id = reader.u8();
+    const size = reader.u32();
+    const section = reader.subreader(size, `${label} section ${id}`);
+
+    if (id === 0) {
+      const name = section.string();
+      customSections.push(name);
+      customSectionData.push({ name, data: section.take(section.bytes.length - section.offset) });
+    } else if (id === 1) {
+      readVector(section, (item) => {
+        if (item.u8() !== 0x60) item.fail("only function types are supported");
+        return {
+          kind: "func",
+          params: readVector(item, readValueType),
+          results: readVector(item, readValueType),
+        };
+      }, types);
+    } else if (id === 2) {
+      readVector(section, (item) => {
+        const module = item.string();
+        const name = item.string();
+        const externalKind = item.u8();
+        let type;
+        if (externalKind === 0) {
+          const typeIndex = item.u32();
+          type = functionType(types, typeIndex, item);
+        } else if (externalKind === 1) {
+          type = readTableType(item);
+          importedTables.push(type);
+        } else if (externalKind === 2) {
+          type = readMemoryType(item);
+          importedMemories.push(type);
+        } else if (externalKind === 3) {
+          type = readGlobalType(item);
+          importedGlobals.push(type);
+        } else if (externalKind === 4) {
+          type = readTagType(item, types);
+          importedTags.push(type);
+        } else {
+          item.fail(`unknown import kind ${externalKind}`);
+        }
+        return { module, name, type };
+      }, imports);
+    } else if (id === 3) {
+      readVector(section, (item) => item.u32(), definedFunctionTypes);
+    } else if (id === 4) {
+      readVector(section, readTableType, definedTables);
+    } else if (id === 5) {
+      readVector(section, readMemoryType, definedMemories);
+    } else if (id === 6) {
+      readVector(section, (item) => {
+        const type = readGlobalType(item);
+        skipConstantExpression(item);
+        return type;
+      }, definedGlobals);
+    } else if (id === 7) {
+      readVector(section, (item) => ({
+        name: item.string(),
+        externalKind: item.u8(),
+        index: item.u32(),
+      }), rawExports);
+    } else if (id === 9) {
+      // One by one: a large program's table (Mojo's, Codex's) exceeds the
+      // arguments a spread may pass.
+      for (const indices of readVector(section, readElementSegment)) for (const index of indices) tableFunctionIndices.push(index);
+    } else if (id === 13) {
+      readVector(section, item => readTagType(item, types), definedTags);
+    } else {
+      section.take(section.bytes.length - section.offset);
+    }
+
+    if (!section.done) section.fail("section has trailing bytes");
+  }
+
+  const importedFunctions = imports.filter((entry) => entry.type.kind === "func").map((entry) => entry.type);
+  const functions = [
+    ...importedFunctions,
+    ...definedFunctionTypes.map((index) => functionType(types, index, reader)),
+  ];
+  const tables = [...importedTables, ...definedTables];
+  const memories = [...importedMemories, ...definedMemories];
+  const globals = [...importedGlobals, ...definedGlobals];
+  const collections = [functions, tables, memories, globals, [...importedTags, ...definedTags]];
+  const kindNames = ["func", "table", "memory", "global", "tag"];
+  const exports = rawExports.map(({ name, externalKind, index }) => {
+    const collection = collections[externalKind];
+    const type = collection?.[index];
+    if (!type) reader.fail(`export ${name} references unknown ${kindNames[externalKind] ?? externalKind} ${index}`);
+    return { name, type };
+  });
+
+  const tableFunctions = tableFunctionIndices.map(index =>
+    functions[index] ?? reader.fail(`element segment references unknown function ${index}`));
+
+  return {
+    label,
+    imports,
+    exports,
+    castEmulationParameters: castEmulationParameters(tableFunctions),
+    customSections,
+    customSectionData,
+  };
+}
+
+function encodeUnsigned(value) {
+  let remaining = BigInt(value);
+  const bytes = [];
+  do {
+    let byte = Number(remaining & 0x7fn);
+    remaining >>= 7n;
+    if (remaining !== 0n) byte |= 0x80;
+    bytes.push(byte);
+  } while (remaining !== 0n);
+  return Uint8Array.from(bytes);
+}
+
+function joinBytes(parts) {
+  const length = parts.reduce((total, part) => total + part.length, 0);
+  const result = new Uint8Array(length);
+  let offset = 0;
+  for (const part of parts) {
+    result.set(part, offset);
+    offset += part.length;
+  }
+  return result;
+}
+
+export function appendCustomSection(input, name, data) {
+  const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
+  const encodedName = new TextEncoder().encode(name);
+  const payload = joinBytes([encodeUnsigned(encodedName.length), encodedName, data]);
+  const section = joinBytes([Uint8Array.of(0), encodeUnsigned(payload.length), payload]);
+  return joinBytes([bytes, section]);
+}
+
+function formatLimits(type) {
+  const width = type.address64 ? "64" : "32";
+  const maximum = type.maximum === null ? "*" : type.maximum;
+  return `${width}(min=${type.minimum},max=${maximum}${type.shared ? ",shared" : ""})`;
+}
+
+export function formatWasmType(type) {
+  if (type.kind === "func") {
+    return `func(${type.params.join(",")})->(${type.results.join(",")})`;
+  }
+  if (type.kind === "global") {
+    return `global(${type.mutable ? "mut " : ""}${type.value})`;
+  }
+  if (type.kind === "memory") return `memory${formatLimits(type)}`;
+  if (type.kind === "table") return `table${formatLimits(type)}:${type.element}`;
+  if (type.kind === "tag") return `tag(${formatWasmType(type.type)})`;
+  throw new Error(`unknown WebAssembly type kind ${type.kind}`);
+}
+
+function arraysEqual(left, right) {
+  return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+
+export function sameWasmType(left, right) {
+  if (left.kind !== right.kind) return false;
+  if (left.kind === "func") {
+    return arraysEqual(left.params, right.params) && arraysEqual(left.results, right.results);
+  }
+  if (left.kind === "global") return left.value === right.value && left.mutable === right.mutable;
+  if (left.kind === "tag") return left.attribute === right.attribute && sameWasmType(left.type, right.type);
+  if (left.kind === "memory") {
+    return left.address64 === right.address64
+      && left.shared === right.shared
+      && left.minimum === right.minimum
+      && left.maximum === right.maximum;
+  }
+  if (left.kind === "table") {
+    return left.element === right.element
+      && left.address64 === right.address64
+      && left.minimum === right.minimum
+      && left.maximum === right.maximum;
+  }
+  return false;
+}
+
+export function providerSatisfiesImport(provider, required, { dynamicTable = false } = {}) {
+  if (provider.kind !== required.kind) return false;
+  if (["func", "global", "tag"].includes(provider.kind)) return sameWasmType(provider, required);
+  if (provider.kind === "memory" || provider.kind === "table") {
+    if (provider.address64 !== required.address64 || provider.shared !== required.shared) return false;
+    if (provider.kind === "table" && provider.element !== required.element) return false;
+    if (!dynamicTable && provider.minimum < required.minimum) return false;
+    if (required.maximum !== null) {
+      if (provider.maximum === null || provider.maximum > required.maximum) return false;
+    }
+    return true;
+  }
+  return false;
+}

package/src/worker-image.mjs

@@ -0,0 +1,137 @@
+// What a runtime Worker does around the image it runs: it takes the page's
+// configure message, starts the host modules and a kernel, has `image` put the
+// image into that kernel, and runs its ENTRY. runtime-worker.mjs restores an
+// image of the image directory; the site's build-worker.mjs builds one or
+// restores a custom one.
+import { DOLLY_BUILD_ID } from "../dist/dolly-build-id.mjs";
+import { loadImages } from "./images.mjs";
+import { decodeImageEntry } from "./image-entry.mjs";
+import { MAX_SNAPSHOT_BYTES } from "./snapshot-records.mjs";
+import { terminalFailureReason } from "./process-supervisor.mjs";
+import { bootFiles } from "../host/runtime/boot-files.mjs";
+
+const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
+
+// Listen before the host registry loads its manifests: the page's configure
+// message may arrive while that import is still pending.
+const configured = new Promise((resolve, reject) => {
+  const timeout = setTimeout(
+    () => reject(new Error("Dolly boot configuration was not provided")), 10_000,
+  );
+  self.addEventListener("message", function configure(event) {
+    if (event.data?.type !== "configure") return;
+    self.removeEventListener("message", configure);
+    clearTimeout(timeout);
+    resolve(event.data);
+  });
+});
+const { hostRegistry } = await import("../host/modules.mjs");
+
+// The runtime's own files: the kernel and the contracts beside this module.
+export const runtimeBase = new URL("../", import.meta.url);
+export const kernelArtifact = path => {
+  if (path !== "dolly.wasm" && path !== "dolly.data") throw new Error("unknown fixed kernel artifact");
+  return new URL(`dist/${path}`, runtimeBase).href;
+};
+
+export function checkedMemoryRange(memory, addressValue, sizeValue) {
+  const address = Number(addressValue);
+  const size = Number(sizeValue);
+  if (!Number.isSafeInteger(address) || !Number.isSafeInteger(size) ||
+      address <= 0 || size <= 0 || size > MAX_SNAPSHOT_BYTES ||
+      address > memory.buffer.byteLength - size) {
+    throw new Error("Wasm published an invalid system snapshot memory range");
+  }
+  return { address, size };
+}
+
+const stage = text => self.postMessage({ type: "bootstrap", text: `${text}\n`, error: false });
+
+function acknowledged(type, failure) {
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => reject(new Error(failure)), 10_000);
+    self.addEventListener("message", function acknowledge(event) {
+      if (event.data?.type !== type) return;
+      self.removeEventListener("message", acknowledge);
+      clearTimeout(timeout);
+      resolve();
+    });
+  });
+}
+
+// `image(config, host)` checks what the page asked for and returns its steps:
+// prepare({ kernel, memory, files, stage }) before the kernel's modules start,
+// restore({ kernel, memory, files, stage }) to { status, supervisor? },
+// and an optional finish({ kernel, stage }) once the supervisor runs; `name` is
+// the image the route asked for ("custom" names none).
+export async function runImage(image) {
+  let host;
+  self.addEventListener("message", event => { void host?.handle(event.data).catch(error => {
+    host.dispose();
+    self.postMessage({ type: "error", message: error.message });
+  }); });
+  try {
+    const config = await configured;
+    if (!Array.isArray(config.modules) || config.modules.length > 64 ||
+        !config.modules.every(url => typeof url === "string")) throw new Error("invalid host module list");
+    if (typeof config.images !== "string") throw new Error("invalid image directory");
+    await loadImages(config.images);
+    host = await (await hostRegistry(config.modules)).createHost("worker", config.hostModules, {
+      send: (message, transfers = []) => self.postMessage(message, transfers),
+      configuration: config.hostConfiguration,
+      resources: { runtimeBase },
+    });
+    const steps = await image(config, host);
+    stage("creating wasm64 userspace kernel...");
+    const memory = host.kernel.memory;
+    await host.kernel.growing;
+    // Fixed deployment input, never a filename or URL supplied by Wasm.
+    const kernelModule = await WebAssembly.compileStreaming(fetch(kernelArtifact("dolly.wasm")));
+    const kernel = new WebAssembly.Instance(kernelModule, host.imports(kernelModule)).exports;
+    kernel._initialize();
+    const files = bootFiles(kernel, memory);
+    await steps.prepare({ kernel, memory, files, stage });
+    stage("Dolly runtime loaded");
+    await host.start("kernel", { kernel, memory });
+    const restored = await steps.restore({ kernel, memory, files, stage });
+    if (restored.status !== 0) throw new Error(`Dolly bootstrap failed with status ${restored.status}`);
+    if (config.buildOnly) {
+      self.close();
+      return;
+    }
+    const supervisor = restored.supervisor ?? await host.kernel.supervisor(kernel);
+    await steps.finish?.({ kernel, stage });
+
+    await host.imageRestored({ kernel, supervisor, stage, files, image: steps.name });
+    if (kernel.dolly_bootstrap_environment() !== 0) throw new Error("Dolly image environment is invalid");
+    await host.start("image", { kernel, memory });
+    const runtimeImage = decoder.decode(files.read("/etc/dolly/image", 256));
+    if (steps.name !== "custom" && runtimeImage !== steps.name) {
+      throw new Error(`Dollyfile selected image ${runtimeImage}, expected ${steps.name}`);
+    }
+    stage("starting image entry...");
+    const entryReady = acknowledged("entry-ready-ack", "browser did not acknowledge image startup");
+    self.postMessage({ type: "ready", image: runtimeImage, routeImage: steps.name, buildId: DOLLY_BUILD_ID,
+      hostModules: host.enabled });
+    await entryReady;
+
+    // The image ends with its ENTRY process, however that ended: a program that
+    // failed or could not start is the image's ending, not the runtime's failure.
+    const entry = async () => supervisor.spawn(decodeImageEntry(files.read("/etc/dolly/entry", 64 * 1024)), { foreground: true });
+    const ending = await entry().catch(error => {
+      console.error(error);
+      return { status: 126, signal: 0, failure: error.reason ?? terminalFailureReason(error) };
+    });
+    // The modules' Worker sides let go first: display@0 gives the page its last frame.
+    host.dispose();
+    self.postMessage({ type: "exited", ...ending });
+  } catch (error) {
+    self.postMessage({
+      type: "error",
+      message: error instanceof Error ? error.message : String(error),
+      stack: error instanceof Error ? error.stack ?? "" : "",
+    });
+  } finally {
+    host?.dispose();
+  }
+}