@daugasauron/dolly-build 0.1.5-rc.0 → 0.1.6

package/scripts/share-pages-snapshots.mjs

@@ -1,18 +1,20 @@
 #!/usr/bin/env node
 import { createHash, randomUUID } from "node:crypto";
+import { sha256 as digest } from "./sha256.mjs";
 import { createWriteStream } from "node:fs";
 import { mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { pipeline } from "node:stream/promises";
 import { createGzip } from "node:zlib";
-import { decodeSnapshotRecords, validateSnapshotPacks, MAX_SNAPSHOT_BYTES } from "../src/snapshot-records.mjs";
+import { decodeSnapshotRecords, decodeSnapshotHeader, decodeRecordHeader, snapshotHeader, recordHeader,
+  validateSnapshotPacks } from "../page/snapshot-records.mjs";
+import { DOLLY_SNAPSHOT_HEADER_BYTES, DOLLY_SNAPSHOT_RECORD_HEADER_BYTES } from "../host/runtime/abi.mjs";
 import { metadataJson } from "./image-directory.mjs";
 
 // A record is either decoded (`data`) or indexed in a snapshot file on disk
 // (`byteLength`, `source`, `offset`); the catalog is never held in memory.
 const CHUNK_BYTES = 4 * 1024 * 1024;
 const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-const digest = bytes => createHash("sha256").update(bytes).digest("hex");
 const recordLength = record => record.byteLength ?? record.data.length;
 const recordKey = (kind, path) => createHash("sha256").update(`${kind}\0${path}\0`);
 
@@ -20,13 +22,13 @@ export function splitSnapshotRecords(records) {
   const entries = [...records].map(([path, record]) => ({ path, record, encodedPath: Buffer.from(path) }))
     .sort((left, right) => Buffer.compare(left.encodedPath, right.encodedPath));
   const parts = [];
-  let part = new Map(), size = 16;
+  let part = new Map(), size = DOLLY_SNAPSHOT_HEADER_BYTES;
   const finish = () => {
     if (part.size) parts.push(part);
-    part = new Map(); size = 16;
+    part = new Map(); size = DOLLY_SNAPSHOT_HEADER_BYTES;
   };
   for (const { path, record, encodedPath } of entries) {
-    const length = 16 + encodedPath.length + recordLength(record);
+    const length = DOLLY_SNAPSHOT_RECORD_HEADER_BYTES + encodedPath.length + recordLength(record);
     if (size + length > 4 * 1024 * 1024) finish();
     part.set(path, record);
     size += length;
@@ -39,22 +41,6 @@ export function splitSnapshotRecords(records) {
   return parts;
 }
 
-function snapshotHeader(count) {
-  const header = Buffer.alloc(16, "DOLLYSNP", "latin1");
-  header.writeUInt32LE(2, 8);
-  header.writeUInt32LE(count, 12);
-  return header;
-}
-
-function recordHeader(kind, encodedPath, byteLength) {
-  const header = Buffer.alloc(16 + encodedPath.length);
-  header.writeUInt32LE(kind, 0);
-  header.writeUInt32LE(encodedPath.length, 4);
-  header.writeBigUInt64LE(BigInt(byteLength), 8);
-  header.set(encodedPath, 16);
-  return header;
-}
-
 async function readExact(handle, buffer, position, length) {
   let done = 0;
   while (done < length) {
@@ -148,16 +134,10 @@ async function indexSnapshot(source) {
     position += length;
     return bytes;
   };
-  const header = await read(16);
-  const count = header.readUInt32LE(12);
-  if (header.toString("latin1", 0, 8) !== "DOLLYSNP" || header.readUInt32LE(8) !== 2 || count === 0 || count > 100_000) {
-    throw new Error(`${source.name}: invalid Dolly snapshot header`);
-  }
+  const count = decodeSnapshotHeader(await read(DOLLY_SNAPSHOT_HEADER_BYTES));
   for (let index = 0; index < count; index += 1) {
-    const head = await read(16);
-    const kind = head.readUInt32LE(0), pathLength = head.readUInt32LE(4), byteLength = Number(head.readBigUInt64LE(8));
-    if (kind < 1 || kind > 3 || pathLength < 2 || pathLength >= 4096 || byteLength > MAX_SNAPSHOT_BYTES ||
-        position > metadata.byteLength - pathLength - byteLength) throw new Error(`${source.name}: invalid snapshot record size`);
+    const { kind, pathLength, size: byteLength } = decodeRecordHeader(await read(DOLLY_SNAPSHOT_RECORD_HEADER_BYTES));
+    if (position > metadata.byteLength - pathLength - byteLength) throw new Error(`${source.name}: invalid snapshot record size`);
     const path = decoder.decode(await read(pathLength)), hash = recordKey(kind, path), offset = position;
     for (let done = 0; done < byteLength; done += Math.min(CHUNK_BYTES, byteLength - done)) await read(Math.min(CHUNK_BYTES, byteLength - done), hash);
     records.push({ key: hash.digest("hex"), path, kind, byteLength, source, offset });

package/scripts/snapshot-identity.mjs

@@ -1,9 +1,8 @@
-import { createHash } from "node:crypto";
 import { recipeRecords } from "./dollyfile-graph.mjs";
 import { decodeSnapshotEnvironment, validateSnapshotEntry } from "./system-snapshot-format.mjs";
+import { sha256 } from "./sha256.mjs";
 
 const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-export const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
 
 export function verifySnapshotIdentity(definition, graph, parsed, processContract, processAbiDigest) {
   const image = definition.image;

package/scripts/system-snapshot-format.mjs

@@ -1,8 +1,10 @@
 import { Buffer } from "node:buffer";
-import { parseWasmInterface } from "../src/wasm-interface.mjs";
-import { validateProcessInterface } from "../src/process-abi.mjs";
-import { decodeImageEntry } from "../src/image-entry.mjs";
-import { decodeSnapshotRecords } from "../src/snapshot-records.mjs";
+import { parseWasmInterface } from "../page/wasm-interface.mjs";
+import { validateProcessInterface } from "../page/process-abi.mjs";
+import { decodeImageEntry } from "../page/image-entry.mjs";
+import { decodeSnapshotRecords } from "../page/snapshot-records.mjs";
+import { DOLLY_SNAPSHOT_RECORD_DIRECTORY as DIRECTORY, DOLLY_SNAPSHOT_RECORD_FILE as FILE,
+  DOLLY_SNAPSHOT_RECORD_SYMLINK as SYMLINK } from "../host/runtime/abi.mjs";
 
 const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
 
@@ -12,8 +14,8 @@ export function decodeSystemSnapshot(input) {
   const files = new Map();
   const directories = new Set(["/"]);
   for (const [path, { kind, data }] of entries) {
-    if (kind === 2) files.set(path, data);
-    if (kind === 1) directories.add(path);
+    if (kind === FILE) files.set(path, data);
+    if (kind === DIRECTORY) directories.add(path);
     for (let slash = path.indexOf("/", 1); slash !== -1; slash = path.indexOf("/", slash + 1)) {
       directories.add(path.slice(0, slash));
     }
@@ -33,7 +35,7 @@ export function resolveSnapshotFile(snapshot, path) {
     if (part === "..") { resolved.pop(); continue; }
     const current = "/" + [...resolved, part].join("/");
     const record = snapshot.entries.get(current);
-    if (record?.kind === 3) {
+    if (record?.kind === SYMLINK) {
       if (++links > 40) throw new Error(`snapshot symlink loop: ${path}`);
       const target = decoder.decode(record.data);
       if (target.startsWith("/")) resolved.length = 0;
@@ -43,13 +45,13 @@ export function resolveSnapshotFile(snapshot, path) {
     if (pending.length && !snapshot.directories.has(current)) {
       throw new Error(`snapshot parent is missing or not a directory: ${current}`);
     }
-    if (!pending.length && record?.kind !== 2) {
+    if (!pending.length && record?.kind !== FILE) {
       throw new Error(`snapshot file is not retained: ${current}`);
     }
     resolved.push(part);
   }
   const record = snapshot.entries.get("/" + resolved.join("/"));
-  if (record?.kind !== 2) throw new Error(`snapshot file is not retained: ${path}`);
+  if (record?.kind !== FILE) throw new Error(`snapshot file is not retained: ${path}`);
   return record.data;
 }
 

package/scripts/wasm-interface.mjs

@@ -1,6 +1,6 @@
 import { readFile } from "node:fs/promises";
-import { parseWasmInterface } from "../src/wasm-interface.mjs";
-export * from "../src/wasm-interface.mjs";
+import { parseWasmInterface } from "../page/wasm-interface.mjs";
+export * from "../page/wasm-interface.mjs";
 
 export async function readWasmInterface(path) {
   return parseWasmInterface(await readFile(path), path);

package/{src → site}/dollyfile-graph.mjs

@@ -1,5 +1,5 @@
 import { inspectDollyfile, recipeFileName, imageFileName } from "./dollyfile-view.mjs";
-import { sha256 } from "./static-asset.mjs";
+import { sha256 } from "../page/static-asset.mjs";
 
 const key = object => `${object.type}:${object.name}`;
 const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });

package/{src → site}/dollyfile-view.mjs

@@ -1,5 +1,5 @@
 import { hostRequirement, hostRequirements } from "../host/requirements.mjs";
-import { unretainedPath, validName } from "./image-names.mjs";
+import { unretainedPath, validName } from "../page/image-names.mjs";
 
 export const MAX_DOLLYFILE_BYTES = 128 * 1024;