@daugasauron/dolly 0.1.5-rc.0 → 0.1.6

package/{src → page}/packaged-snapshot.mjs

@@ -2,18 +2,18 @@
 // one's metadata, checked against the catalog and the recipes the directory
 // serves, and its packs: streamed into a booting kernel, or merged in path
 // order into the image's canonical snapshot.
-import { DOLLY_IMAGE_BUILD_ID } from "../dist/dolly-image-build-id.mjs";
+import { DOLLY_IMAGE_BUILD_ID } from "./generated.mjs";
 import { DOLLY_IMAGES, imageBase } from "./images.mjs";
-import { imageInputs, imageInputsMatch } from "./image-inputs.mjs";
+import { imageInputs, imageInputsMatch, recipesDigest } from "./image-inputs.mjs";
 import { hostRequirements } from "../host/requirements.mjs";
-import { siteReference, sitePath, decodeStaticAsset, sha256 } from "./static-asset.mjs";
-import { scanSnapshotRecords, validateSnapshotPacks, MAX_SNAPSHOT_BYTES as snapshotSizeLimit } from "./snapshot-records.mjs";
+import { siteReference, decodeStaticAsset, hex } from "./static-asset.mjs";
+import { scanSnapshotRecords, validateSnapshotPacks, snapshotHeader, recordHeader, MAX_SNAPSHOT_BYTES as snapshotSizeLimit } from "./snapshot-records.mjs";
+import { DOLLY_SNAPSHOT_MAX_PATHS } from "../host/runtime/abi.mjs";
 import { unretainedPath } from "./image-names.mjs";
 import { openPackStore } from "./pack-store.mjs";
 import { Sha256 } from "./sha256.mjs";
 const definitionOf = image => DOLLY_IMAGES.find(definition => definition.image === image);
 const encoder = new TextEncoder();
-const expectedRecipes = image => definitionOf(image).recipes;
 
 function validSnapshotPath(path) {
   return typeof path === "string" && path.startsWith("/") && path.length > 1 &&
@@ -23,21 +23,6 @@ function validSnapshotPath(path) {
     !unretainedPath(path);
 }
 
-// The recipes the directory serves for IMAGE (its page shows them) are the ones
-// that built the image a boot runs. They are fetched together.
-export async function verifyVisibleRecipes(image) {
-  await Promise.all(expectedRecipes(image).map(async recipe => {
-    const response = await fetch(new URL(sitePath(recipe.sourcePath).slice(1), imageBase), {
-      credentials: "same-origin", redirect: "error",
-    });
-    if (!response.ok) throw new Error(`${recipe.sourcePath} returned HTTP ${response.status}`);
-    const bytes = await response.arrayBuffer();
-    if (bytes.byteLength !== recipe.byteLength || await sha256(bytes) !== recipe.sha256) {
-      throw new Error(`${recipe.sourcePath} does not match the packaged snapshot recipe`);
-    }
-  }));
-}
-
 // The image a FROM, COPY or INSTALL row names: its recipe at that path and pin.
 const referenced = ({ location, sha256 }) => DOLLY_IMAGES.find(candidate =>
   siteReference(candidate.dollyfile) === location && candidate.sha256 === sha256);
@@ -64,6 +49,14 @@ async function importMetadata(image) {
   }
 }
 
+// The metadata an image starts from: its own, checked against its catalog
+// record (its recipes, the pins of its inputs, its modules). That the snapshots
+// of its inputs are the ones it was built on is the build's and the release's
+// to check (loadPackagedSnapshotMetadata, scripts/site-release.mjs).
+export async function loadImageMetadata(image) {
+  return checkedMetadata(image, await importMetadata(image), null);
+}
+
 // An image's metadata, checked against the catalog and the metadata of every
 // image it is built from, which are fetched together.
 export async function loadPackagedSnapshotMetadata(image) {
@@ -75,19 +68,18 @@ export async function loadPackagedSnapshotMetadata(image) {
 }
 
 // METADATA, checked as IMAGE's against the catalog and CHECKED, the metadata
-// of the images it is built from.
+// of the images it is built from, unless CHECKED is null.
 function checkedMetadata(image, metadata, checked) {
-  const recipes = expectedRecipes(image);
   if (metadata === null || typeof metadata !== "object" ||
       metadata.image !== image || metadata.buildId !== DOLLY_IMAGE_BUILD_ID ||
       metadata.formatVersion !== 2 || metadata.identityVersion !== 2 ||
-      JSON.stringify(metadata.recipes) !== JSON.stringify(recipes) ||
+      !Array.isArray(metadata.recipes) || recipesDigest(metadata.recipes) !== definitionOf(image).recipesSha256 ||
       JSON.stringify(hostRequirements(metadata.hostRequirements)) !== JSON.stringify(definitionOf(image).hostRequirements) ||
       !Number.isSafeInteger(metadata.byteLength) || metadata.byteLength <= 0 ||
       metadata.byteLength > snapshotSizeLimit || metadata.encoding !== "packs" ||
       !/^[0-9a-f]{64}$/.test(metadata.sha256) ||
       !Array.isArray(metadata.manifest) || metadata.manifest.length === 0 ||
-      metadata.manifest.length > 100_000) {
+      metadata.manifest.length > DOLLY_SNAPSHOT_MAX_PATHS) {
     throw new Error("The packaged system snapshot metadata does not match this Dolly build");
   }
   validateSnapshotPacks(metadata);
@@ -117,6 +109,7 @@ function checkedMetadata(image, metadata, checked) {
       throw new Error(`The packaged system snapshot is missing ${required}`);
     }
   }
+  if (checked === null) return metadata;
   const expectedInputs = definitionOf(image).artifacts.map(reference =>
     ({ recipeSha256: reference.sha256, sha256: checked.get(referenced(reference).image).sha256 }));
   if (!imageInputsMatch(inputs, expectedInputs)) throw new Error("The packaged system snapshot has stale image inputs");
@@ -129,13 +122,32 @@ function checkedMetadata(image, metadata, checked) {
 // checked against its declared length. The consumer checks each part against
 // its digest and then calls its verified(), which keeps a downloaded pack; a
 // part it does not verify is not kept, or no longer.
+// Packs are asked for ahead of the one being read, at most PREFETCH_PACKS of
+// them and PREFETCH_BYTES of compressed bytes (one at least), and handed over
+// in order: on a network their round trips overlap instead of adding up.
+const PREFETCH_PACKS = 8, PREFETCH_BYTES = 64 * 1024 * 1024;
 async function* packagedSnapshotParts(metadata, signal) {
   const packs = validateSnapshotPacks(metadata), store = await openPackStore(metadata.image, packs);
-  let complete = false;
+  const opened = async pack => {
+    const kept = await store.kept(pack);
+    return kept ? { part: kept, downloaded: false }
+      : { part: await store.keeping(pack, await downloadedPack(pack, signal)), downloaded: true };
+  };
+  const ahead = [];
+  let next = 0, aheadBytes = 0, complete = false;
   try {
-    for (const pack of packs) {
+    while (next < packs.length || ahead.length) {
       signal?.throwIfAborted();
-      const part = await store.kept(pack) ?? await store.keeping(pack, await downloadedPack(pack, signal));
+      while (next < packs.length && (!ahead.length ||
+             ahead.length < PREFETCH_PACKS && aheadBytes + packs[next].encodedByteLength <= PREFETCH_BYTES)) {
+        const pack = packs[next++], opening = opened(pack);
+        opening.catch(() => {});
+        ahead.push({ pack, opening });
+        aheadBytes += pack.encodedByteLength;
+      }
+      const { pack, opening } = ahead.shift();
+      aheadBytes -= pack.encodedByteLength;
+      const { part } = await opening;
       let verified = false;
       try {
         yield { ...pack, body: part.body, verified: () => { verified = true; return part.keep(); } };
@@ -145,6 +157,14 @@ async function* packagedSnapshotParts(metadata, signal) {
     }
     complete = true;
   } finally {
+    // Packs asked for and never read: their downloads stop and are not kept;
+    // a kept one stays kept.
+    for (const { opening } of ahead) {
+      void opening.then(({ part, downloaded }) => {
+        void part.body.cancel().catch(() => {});
+        if (downloaded) void part.discard();
+      }, () => {});
+    }
     store.close(complete);
   }
 }
@@ -205,37 +225,27 @@ function streamOf(bytes) {
   }, { highWaterMark: 0 });
 }
 
-// The sink owns incremental hashing. Every part is checked before another is
-// requested; the caller verifies the complete canonical image before booting.
+// Each part's bytes go to `write` as they arrive and are checked against the
+// pack's digest (WebCrypto's SHA-256, which is native) before `endPart` and
+// before another part is read; a mismatch ends the restore.
 export async function streamPackagedSystemSnapshot(metadata, write, endPart, signal) {
   for await (const part of packagedSnapshotParts(metadata, signal)) {
+    const bytes = new Uint8Array(part.byteLength);
     let size = 0;
     for await (const chunk of chunksOf(gunzip(part.body), signal)) {
       if (chunk.length > part.byteLength - size) throw new Error("snapshot exceeds declared size");
+      bytes.set(chunk, size);
       await write(chunk);
       size += chunk.length;
     }
-    if (size !== part.byteLength || await endPart() !== part.sha256) throw new Error("snapshot integrity mismatch");
+    if (size !== part.byteLength || hex(await crypto.subtle.digest("SHA-256", bytes)) !== part.sha256) {
+      throw new Error("snapshot integrity mismatch");
+    }
+    await endPart();
     await part.verified();
   }
 }
 
-function snapshotHeader(count) {
-  const bytes = new Uint8Array(16), view = new DataView(bytes.buffer);
-  bytes.set(encoder.encode("DOLLYSNP"));
-  view.setUint32(8, 2, true);
-  view.setUint32(12, count, true);
-  return bytes;
-}
-
-function recordHeader(kind, path, size) {
-  const bytes = new Uint8Array(16 + path.length), view = new DataView(bytes.buffer);
-  view.setUint32(0, kind, true);
-  view.setUint32(4, path.length, true);
-  view.setBigUint64(8, BigInt(size), true);
-  bytes.set(path, 16);
-  return bytes;
-}
 
 function compareBytes(left, right) {
   for (let index = 0; index < Math.min(left.length, right.length); index += 1) {

package/{src → page}/process-constants.mjs

@@ -117,6 +117,7 @@ export const DOLLY_PROCESS_TERMINAL_MODE_SET = 4;
 export const DOLLY_PROCESS_TERMINAL_SIZE = 5;
 export const DOLLY_PROCESS_TERMINAL_PUBLISH_RESULT = 6;
 export const DOLLY_PROCESS_TERMINAL_FOREGROUND = 7;
+export const DOLLY_PROCESS_TERMINAL_SIZE_SET = 8;
 export const DOLLY_PROCESS_LOCK_SHARED = 0;
 export const DOLLY_PROCESS_LOCK_EXCLUSIVE = 1;
 export const DOLLY_PROCESS_LOCK_UNLOCK = 2;
@@ -204,41 +205,97 @@ export const DOLLY_ERRNO = Object.freeze({
 });
 export const DOLLY_PROCESS_SIZEOF = Object.freeze({
   dolly_process_vector_sizes: 16,
+  dolly_process_exit_request: 8,
   dolly_process_fd_io_request: 16,
   dolly_process_fd_pread_request: 24,
   dolly_process_fd_truncate_request: 16,
   dolly_process_fd_request: 8,
   dolly_process_timestamp: 16,
   dolly_process_fd_times_request: 40,
+  dolly_process_io_result: 8,
   dolly_process_fd_dup_request: 16,
+  dolly_process_fd_dup_response: 8,
   dolly_process_fd_flags: 8,
   dolly_process_fd_lock_request: 32,
   dolly_process_fd_lock_response: 24,
+  dolly_process_pipe_request: 8,
+  dolly_process_pipe_response: 8,
   dolly_process_poll_request: 16,
   dolly_process_poll_query: 8,
   dolly_process_poll_response: 16,
   dolly_process_poll_result: 8,
-  dolly_process_directory_entry: 24,
   dolly_process_directory_request: 24,
-  dolly_process_stat_response: 88,
-  dolly_process_filesystem_stat_response: 96,
-  dolly_process_spawn_request: 72,
-  dolly_process_fd_mapping: 8,
+  dolly_process_directory_entry: 24,
+  dolly_process_fd_seek_request: 16,
+  dolly_process_fd_seek_response: 8,
+  dolly_process_clock_request: 16,
+  dolly_process_clock_response: 8,
+  dolly_process_clock_sleep_request: 16,
   dolly_process_exec_state: 16,
+  dolly_process_fd_mapping: 8,
+  dolly_process_spawn_request: 72,
+  dolly_process_spawn_response: 8,
   dolly_process_exec_request: 48,
-  dolly_process_pipe_request: 8,
-  dolly_process_two_path_request: 16,
-  dolly_process_path_times_request: 48,
-  dolly_process_terminal_request: 24,
-  dolly_process_terminal_response: 16,
-  dolly_process_clock_sleep_request: 16,
-  dolly_process_exit_request: 8,
   dolly_process_wait_request: 8,
   dolly_process_wait_response: 16,
+  dolly_process_info_response: 8,
   dolly_process_signal_request: 8,
-  dolly_process_alarm: 16,
   dolly_process_signal_dispositions: 8,
-  dolly_process_fd_seek_request: 16,
-  dolly_process_clock_request: 16,
+  dolly_process_alarm: 16,
+  dolly_process_terminal_request: 24,
+  dolly_process_terminal_response: 16,
   dolly_process_path_request: 16,
+  dolly_process_path_times_request: 48,
+  dolly_process_path_open_response: 8,
+  dolly_process_stat_response: 88,
+  dolly_process_filesystem_stat_response: 96,
+  dolly_process_two_path_request: 16,
+});
+export const DOLLY_PROCESS_OFFSETOF = Object.freeze({
+  dolly_process_vector_sizes: Object.freeze({"count":0, "reserved":4, "bytes":8}),
+  dolly_process_exit_request: Object.freeze({"status":0, "signal_number":4}),
+  dolly_process_fd_io_request: Object.freeze({"descriptor":0, "reserved":4, "size":8}),
+  dolly_process_fd_pread_request: Object.freeze({"descriptor":0, "reserved":4, "offset":8, "size":16}),
+  dolly_process_fd_truncate_request: Object.freeze({"descriptor":0, "reserved":4, "size":8}),
+  dolly_process_fd_request: Object.freeze({"descriptor":0, "reserved":4}),
+  dolly_process_timestamp: Object.freeze({"seconds":0, "nanoseconds":8, "flags":12}),
+  dolly_process_fd_times_request: Object.freeze({"descriptor":0, "reserved":4, "access":8, "modification":24}),
+  dolly_process_io_result: Object.freeze({"size":0}),
+  dolly_process_fd_dup_request: Object.freeze({"source_descriptor":0, "target_descriptor":4, "flags":8, "reserved":12}),
+  dolly_process_fd_dup_response: Object.freeze({"descriptor":0, "reserved":4}),
+  dolly_process_fd_flags: Object.freeze({"descriptor":0, "flags":4}),
+  dolly_process_fd_lock_request: Object.freeze({"descriptor":0, "flags":4, "type":8, "whence":12, "start":16, "length":24}),
+  dolly_process_fd_lock_response: Object.freeze({"type":0, "pid":4, "start":8, "length":16}),
+  dolly_process_pipe_request: Object.freeze({"flags":0, "reserved":4}),
+  dolly_process_pipe_response: Object.freeze({"read_descriptor":0, "write_descriptor":4}),
+  dolly_process_poll_request: Object.freeze({"deadline_nanoseconds":0, "count":8, "reserved":12}),
+  dolly_process_poll_query: Object.freeze({"descriptor":0, "events":4, "reserved":6}),
+  dolly_process_poll_response: Object.freeze({"ready":0, "count":4, "reserved":8}),
+  dolly_process_poll_result: Object.freeze({"events":0, "reserved":2, "reserved2":4}),
+  dolly_process_directory_request: Object.freeze({"descriptor":0, "maximum_entries":4, "cookie":8, "reserved":16}),
+  dolly_process_directory_entry: Object.freeze({"inode":0, "next_cookie":8, "file_type":16, "name_size":20}),
+  dolly_process_fd_seek_request: Object.freeze({"descriptor":0, "whence":4, "offset":8}),
+  dolly_process_fd_seek_response: Object.freeze({"offset":0}),
+  dolly_process_clock_request: Object.freeze({"clock_id":0, "reserved":4, "precision_nanoseconds":8}),
+  dolly_process_clock_response: Object.freeze({"nanoseconds":0}),
+  dolly_process_clock_sleep_request: Object.freeze({"clock_id":0, "flags":4, "deadline_nanoseconds":8}),
+  dolly_process_exec_state: Object.freeze({"blocked_signals":0, "ignored_signals":4, "pending_signals":8, "reserved":12}),
+  dolly_process_fd_mapping: Object.freeze({"source_descriptor":0, "target_descriptor":4}),
+  dolly_process_spawn_request: Object.freeze({"flags":0, "argument_count":4, "environment_count":8, "cwd_size":12, "mapping_count":16, "descriptor_inheritance":20, "reserved":24, "path_size":28, "argument_bytes":32, "environment_bytes":40, "deadline_nanoseconds":48, "signals":56}),
+  dolly_process_spawn_response: Object.freeze({"pid":0, "reserved":4}),
+  dolly_process_exec_request: Object.freeze({"signals":0, "argument_count":16, "environment_count":20, "path_size":24, "reserved":28, "argument_bytes":32, "environment_bytes":40}),
+  dolly_process_wait_request: Object.freeze({"pid":0, "flags":4}),
+  dolly_process_wait_response: Object.freeze({"pid":0, "status":4, "signal_number":8, "reserved":12}),
+  dolly_process_info_response: Object.freeze({"pid":0, "parent_pid":4}),
+  dolly_process_signal_request: Object.freeze({"pid":0, "signal_number":4}),
+  dolly_process_signal_dispositions: Object.freeze({"ignored_signals":0, "handled_signals":4}),
+  dolly_process_alarm: Object.freeze({"value_nanoseconds":0, "interval_nanoseconds":8}),
+  dolly_process_terminal_request: Object.freeze({"operation":0, "descriptor":4, "flags":8, "reserved":12, "deadline_nanoseconds":16}),
+  dolly_process_terminal_response: Object.freeze({"value":0, "columns":8, "rows":12}),
+  dolly_process_path_request: Object.freeze({"directory_descriptor":0, "flags":4, "reserved":8, "path_size":12}),
+  dolly_process_path_times_request: Object.freeze({"directory_descriptor":0, "flags":4, "reserved":8, "path_size":12, "access":16, "modification":32}),
+  dolly_process_path_open_response: Object.freeze({"descriptor":0, "reserved":4}),
+  dolly_process_stat_response: Object.freeze({"device":0, "inode":8, "size":16, "access_nanoseconds":24, "modification_nanoseconds":32, "change_nanoseconds":40, "blocks":48, "mode":56, "link_count":60, "user":64, "group":68, "block_size":72, "file_type":76, "reserved":80}),
+  dolly_process_filesystem_stat_response: Object.freeze({"type":0, "block_size":8, "blocks":16, "blocks_free":24, "blocks_available":32, "files":40, "files_free":48, "maximum_name_length":56, "fragment_size":64, "flags":72, "filesystem_id":80, "reserved":88}),
+  dolly_process_two_path_request: Object.freeze({"old_directory_descriptor":0, "new_directory_descriptor":4, "old_path_size":8, "new_path_size":12}),
 });