@dcasegr/skein 0.0.0-stage → 0.1.2

package/dist/host/boot.js

@@ -0,0 +1,643 @@
+// The bootstrap loader (issue #4): one loader, two sources, no genesis by code
+// alone. An instance boots from a **system tree** (docs/BOOTSTRAP.md):
+//
+//   bin/<name>.wasm   a handler program's module (WASI), or
+//   bin/<name>.cid    the CID (raw, bafkrei…) of a module the source or the kernel holds
+//   bin/<name>.json   optional: the program record's {inputs, services, description}
+//   etc/config.json   optional: {defaults, peers, names, collect, feeds, roles} (keys hex, $self, $infer or a provider's $<name>; feeds: feeds.ts)
+//   etc/dispatch.json [{transport?, address, prefix?, filters?, program?, fn?, …}] (#77, #143: the routes; genesis.ts DispatchSpec)
+//   etc/apps.json     optional (#141, #152): {install: [<directory> | {url, hash}], routes?: [<root's own route>]}
+//   … anything else: the instance's files (SOUL.md, skills/ …)
+//   (etc/subscriptions.json, etc/routes.json, etc/reads.json — the forms before #77 and #115 — are refused, #143)
+//
+// The loader **pre-fills the store** with the tree's objects (git-raw blobs
+// and trees, each hash-checked by the kernel's `putblock`; modules as raw
+// blocks), puts a program record per bin/ module, and writes the genesis from
+// the tree's config and routes, naming the tree — the kernel sets
+// `main` to it when it processes the genesis. That is the state an authorised
+// `objects` + `head` pair would have produced, written directly, and a
+// function of the log (a replay copies the tree and gets the same state).
+//
+// The sources only differ in where the objects come from (`Objects`): a
+// directory scanned into memory (A), a store file (`--boot <cid> --from`),
+// or a verified chain packet (B, packet.ts). An instance with no tree takes
+// the stock system in code (genesis.ts codeSystem) through the same writer; a
+// packet with a checkpoint (#30's state record) is restored instead: its
+// blocks put, the state pointer moved, nothing rebuilt. The router's
+// hydration of an empty store and `skein-host add --boot/--packet` both call
+// `boot` here.
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, posix, resolve } from "node:path";
+import { gunzipSync } from "node:zlib";
+import { fileURLToPath } from "node:url";
+import { CID } from "multiformats/cid";
+import { encode, parse as parseCid } from "../runtime/cid.js";
+import { programRecord, RAW, rawCid, wasmKind } from "../runtime/programs.js";
+import { GIT_RAW, hashBlob, hashTree, parseTree, sha1Cid } from "../runtime/tree.js";
+import { scan } from "../dev/scan.js";
+import { now as clockNow } from "./clock.js";
+import * as dagCbor from "@ipld/dag-cbor";
+import { appRecordIn, mergeConfig, planInstall, planRootRoute, readStoredApp } from "./plan.js";
+import { CLAIM_ROW, codeSystem, HANDSHAKE_ROW, rowsOf, resolveSystem, STOCK_HTTP, writeSystemGenesis } from "./genesis.js";
+import { appIgnore } from "./install.js";
+export const DAG_CBOR = 0x71;
+// The module and program-record helpers live in ../runtime/programs.ts (no node APIs: the plan uses them in a browser too, #92).
+export { programRecord, RAW, rawCid, wasmKind };
+export const BIN = "bin";
+export const CONFIG = "etc/config.json";
+export const DISPATCH = "etc/dispatch.json";
+/** The forms before #77 and #115, refused (#143: they spoke of senders). */
+export const GONE = ["etc/subscriptions.json", "etc/routes.json", "etc/reads.json"];
+/** #141, #152: the apps an image installs at birth, and root's own routes (ImageApps). */
+export const APPS = "etc/apps.json";
+/** Blocks in memory: what a scanned directory or a verified packet becomes. */
+export class MemBlocks {
+    map = new Map();
+    async get(cid) { return this.map.get(cid.toString())?.bytes; }
+    async bytes(cid) { const b = this.map.get(cid.toString()); if (!b)
+        throw new Error(`not found: ${cid}`); return b.bytes; }
+    async has(cid) { return this.map.has(cid.toString()); }
+    async putBlock(cid, bytes) { this.map.set(cid.toString(), { cid, bytes }); }
+}
+/** A git object's body if it is a `<type>` object. */
+export function gitBody(object, type) {
+    const nul = object.indexOf(0);
+    if (nul < 0)
+        return undefined;
+    const head = Buffer.from(object.subarray(0, nul)).toString("latin1");
+    return head === `${type} ${object.length - nul - 1}` ? object.subarray(nul + 1) : undefined;
+}
+// ---------------------------------------------------------------- sources
+/** Source A: a directory on disk, hashed into git objects in memory (scan.ts). */
+export async function dirSource(dir, o = {}) {
+    const objects = new MemBlocks();
+    const root = await scan(objects, dir, o);
+    return { root, objects };
+}
+/** An entry's key: its path, or its tarball's tree id. */
+export const appKey = (a) => typeof a === "string" ? posix.normalize(a).replace(/\/$/, "") : a.hash;
+const isImageApp = (a) => typeof a === "string" ? a.length > 0 : !!a && typeof a === "object" && typeof a.url === "string" && typeof a.hash === "string" && /^[0-9a-f]{40}$/.test(a.hash) && Object.keys(a).length === 2;
+const need = async (objects, cid, what) => {
+    const b = await objects.get(cid);
+    if (!b)
+        throw new Error(`incomplete: ${what} (${cid}) is not in the source`);
+    return b;
+};
+/** Every git object reachable from `root` (trees, blobs, gitlinked records), each checked against its id. */
+export async function treeObjects(objects, root) {
+    const out = [];
+    const seen = new Set();
+    const visit = async (cid, what, tree) => {
+        if (seen.has(cid.toString()))
+            return;
+        seen.add(cid.toString());
+        if (cid.code !== GIT_RAW)
+            throw new Error(`${what}: ${cid} is not a git object`);
+        const bytes = await need(objects, cid, what);
+        if (!Buffer.from(createHash("sha1").update(bytes).digest()).equals(Buffer.from(cid.multihash.digest)))
+            throw new Error(`hash mismatch: ${what} (${cid})`);
+        if (!tree) {
+            out.push({ cid, bytes });
+            return;
+        }
+        const entries = parseTree(bytes, cid);
+        out.push({ cid, bytes, entries });
+        for (const e of entries)
+            await visit(e.cid, `${what}/${e.name}`, e.mode === "40000");
+    };
+    await visit(root, "", true);
+    return out;
+}
+/** Read a system tree out of `objects` (docs/BOOTSTRAP.md): its objects, programs, modules, config and subscriptions. */
+export async function readSystemTree(objects, root) {
+    const all = await treeObjects(objects, root);
+    const byCid = new Map(all.map((x) => [x.cid.toString(), x]));
+    const entriesOf = (cid) => cid ? byCid.get(cid.toString())?.entries : undefined;
+    const at = (path) => {
+        let dir = root, found;
+        for (const seg of path.split("/")) {
+            found = entriesOf(dir)?.find((e) => e.name === seg);
+            if (!found)
+                return undefined;
+            dir = found.mode === "40000" ? found.cid : undefined;
+        }
+        return found;
+    };
+    const text = (e, path) => {
+        const b = gitBody(byCid.get(e.cid.toString()).bytes, "blob");
+        if (!b)
+            throw new Error(`${path}: not a file`);
+        return Buffer.from(b).toString("utf8");
+    };
+    const json = (path) => {
+        const e = at(path);
+        if (!e)
+            return undefined;
+        try {
+            return JSON.parse(text(e, path));
+        }
+        catch (x) {
+            throw new Error(`${path}: ${x.message}`);
+        }
+    };
+    const bin = at(BIN);
+    const programs = [];
+    const modules = [];
+    const binEntries = bin?.mode === "40000" ? entriesOf(bin.cid) ?? [] : [];
+    const names = [...new Set(binEntries.map((e) => /^(.+)\.(wasm|cid)$/.exec(e.name)?.[1]).filter((n) => !!n))].sort();
+    for (const name of names) {
+        const wasm = binEntries.find((e) => e.name === `${name}.wasm`), cidFile = binEntries.find((e) => e.name === `${name}.cid`);
+        if (wasm && cidFile)
+            throw new Error(`bin/${name}: both a .wasm and a .cid`);
+        let module;
+        if (wasm) {
+            const bytes = gitBody(byCid.get(wasm.cid.toString()).bytes, "blob");
+            // A preview1 core module (version 1) or a WASI 0.2 component (#34: version 0x0d, layer 1): stored as a raw
+            // block either way; the kernel's runner tells them apart by this preamble when it runs one.
+            if (wasmKind(bytes) === undefined)
+                throw new Error(`bin/${name}.wasm: not a wasm module or component`);
+            module = rawCid(bytes);
+            modules.push({ cid: module, bytes: new Uint8Array(bytes), name });
+        }
+        else {
+            const t = text(cidFile, `bin/${name}.cid`).trim();
+            try {
+                module = parseCid(t);
+            }
+            catch {
+                throw new Error(`bin/${name}.cid: not a CID: ${JSON.stringify(t)}`);
+            }
+            if (module.code !== RAW)
+                throw new Error(`bin/${name}.cid: ${module} is not a raw module CID`);
+            modules.push({ cid: module, bytes: await objects.get(module), name });
+        }
+        const meta = json(`${BIN}/${name}.json`) ?? {};
+        programs.push({ name, module, record: programRecord(name, module, meta) });
+    }
+    for (const g of GONE)
+        if (at(g))
+            throw new Error(`${g}: the form before #77/#115 is gone (#143: it spoke of senders) — name routes in ${DISPATCH}`);
+    const dispatch = json(DISPATCH);
+    if (dispatch === undefined)
+        throw new Error(`${DISPATCH}: missing (a system tree names its routes)`);
+    if (!Array.isArray(dispatch))
+        throw new Error(`${DISPATCH}: not a list`);
+    const config = json(CONFIG) ?? {};
+    if (config.defaults && Object.values(config.defaults).some((v) => typeof v !== "string"))
+        throw new Error(`${CONFIG}: every default is a string`);
+    const apps = json(APPS);
+    checkImageApps(apps);
+    const appTrees = {};
+    for (const a of apps?.install ?? []) {
+        if (typeof a !== "string")
+            continue;
+        const e = at(appKey(a));
+        if (e?.mode === "40000")
+            appTrees[appKey(a)] = e.cid;
+    }
+    return { ...(apps ? { apps, appTrees } : {}), root, objects: all.map(({ cid, bytes }) => ({ cid, bytes })), programs, modules, config, dispatch };
+}
+/** etc/apps.json's shape (#141, #152), refused with what it wants. */
+export function checkImageApps(apps) {
+    if (apps === undefined)
+        return;
+    if (!apps || !Array.isArray(apps.install) || !apps.install.every(isImageApp) || (apps.routes !== undefined && !Array.isArray(apps.routes)))
+        throw new Error(`${APPS}: want {install: [<directory, relative to the image> | {url: <tarball>, hash: <its tree's git id, 40 hex>}], routes?: [<root's own route>]}`);
+    if (apps.reads !== undefined)
+        throw new Error(`${APPS}: reads: gone (#143) — root's own read is a route in \`routes\` (filters only)`);
+    const keys = apps.install.map(appKey);
+    const twice = keys.filter((k, i) => keys.indexOf(k) !== i);
+    if (twice.length)
+        throw new Error(`${APPS}: ${twice.join(", ")} listed twice`);
+}
+/**
+ * Boot an empty store through its kernel: pre-fill and write the genesis (a
+ * tree, or the stock system), or restore a checkpoint. The kernel processes
+ * the genesis when it starts (the dispatch rows; `main` → the tree).
+ */
+export async function boot(k, src, c, time = clockNow()) {
+    if (await k.store.log.tip())
+        throw new Error("the store already has a log: boot only into an empty store");
+    if (src.kind === "checkpoint") {
+        for (const b of src.blocks)
+            await k.putBlock(b.cid, b.bytes);
+        await k.restore(src.state);
+        return { state: src.state, objects: src.blocks.length, programs: [] };
+    }
+    // A tree takes from the kernel only the front door, unless its bin/ has one (#70: the middleware every provider's answer comes in through, as `local`); the other pinned records would sit in the store unreferenced (a replay would not reproduce them). No shell (#83): the shell is an app.
+    if (src.kind === "code") {
+        const kernelPrograms = await k.call("programs", c.mailbox ? ["frontdoor", "messagebox"] : undefined);
+        const entry = await writeSystemGenesis(k, c, codeSystem(c, kernelPrograms), time);
+        return { entry, objects: 0, programs: Object.keys(kernelPrograms) };
+    }
+    const t = await readSystemTree(src.objects, src.root);
+    const kernelPrograms = t.programs.some((p) => p.name === "frontdoor") ? {} : await k.call("programs", ["frontdoor"]);
+    let n = 0;
+    for (const o of t.objects)
+        if (!(await k.hasBlock(o.cid))) {
+            await k.putBlock(o.cid, o.bytes);
+            n++;
+        }
+    for (const m of t.modules) {
+        if (m.bytes) {
+            if (!(await k.hasBlock(m.cid))) {
+                await k.putBlock(m.cid, m.bytes);
+                n++;
+            }
+        }
+        else if (!(await k.hasBlock(m.cid)))
+            throw new Error(`incomplete: bin/${m.name}.cid names ${m.cid}, which neither the source nor the kernel holds`);
+    }
+    // The front door is the kernel's when the tree brings none; every handler comes from bin/
+    const programs = { ...(kernelPrograms.frontdoor ? { frontdoor: kernelPrograms.frontdoor } : {}) };
+    for (const p of t.programs)
+        programs[p.name] = await k.store.put(p.record);
+    // A tree that names no http route takes the stock ones (the handshake, the messagebox's, the explorer), as a tree
+    // with no etc/routes.json did before #143; each is dropped when the tree lacks its program.
+    // The handshake route is the transport's own: a tree that names http routes of its own gets it too.
+    const specs = t.dispatch.some((r) => r.transport === "http")
+        ? [...t.dispatch, ...(t.dispatch.some((r) => r.transport === "http" && r.address === HANDSHAKE_ROW.address) ? [] : [HANDSHAKE_ROW])]
+        : [...t.dispatch, ...STOCK_HTTP];
+    let s = resolveSystem(c, programs, t.config, t.root, specs);
+    const isClaim = (r) => r.program === "kernel" && r.fn === "claim";
+    // An image (#89, #143): no root, so it must be claimable — its routes carry the claim route.
+    if (!c.root?.length && !s.dispatch.some(isClaim))
+        throw new Error(`an image (a genesis with no root) needs a claim route in ${DISPATCH}: ${JSON.stringify(CLAIM_ROW)}`);
+    // An image booted with root named (#142: the host skein): born as a claim would leave it — no claim route.
+    if (c.root?.length && s.dispatch.some(isClaim))
+        s = { ...s, dispatch: s.dispatch.filter((r) => !isClaim(r)) };
+    let apps;
+    if (t.apps) {
+        const born = await installAtBirth(k, src.objects, t, s, c, src.apps);
+        s = { ...s, dispatch: [...s.dispatch, ...born.rows], heads: born.heads };
+        n += born.put;
+        apps = born.apps;
+    }
+    const entry = await writeSystemGenesis(k, c, s, time);
+    return { entry, tree: t.root, objects: n, programs: Object.keys(programs), ...(apps ? { apps } : {}) };
+}
+/**
+ * #141, #152: the apps an image installs at birth (etc/apps.json), each a directory install — the
+ * app's tree (in the image; a directory beside it, `resolved` by the source; a tarball fetched and
+ * checked, appTarball), its objects put — planned as an install by messages would be (plan.ts:
+ * readStoredApp over the app's tree, planInstall) against the instance as its genesis will stand —
+ * the same records, routes and heads — and root's own routes (planRootRoute). An app's config given
+ * at creation (`appConfig`) is merged over its manifest's, as an install's `--config` is. Their
+ * objects and records are put into the store; the routes and heads go into the genesis.
+ */
+export async function installAtBirth(k, imageObjects, t, s, c, resolved = {}) {
+    const fetched = new MemBlocks();
+    const objects = anyOf(imageObjects, fetched);
+    const local = new MemBlocks();
+    const bytes = async (cid) => {
+        const b = local.map.get(cid.toString())?.bytes ?? await objects.get(cid);
+        if (!b)
+            throw new Error(`not found: ${cid}`);
+        return b;
+    };
+    const store = {
+        bytes,
+        has: async (cid) => local.map.has(cid.toString()) || await k.hasBlock(cid),
+        get: async (cid) => dagCbor.decode(await bytes(cid)),
+        putBlock: async (cid, b) => { local.map.set(cid.toString(), { cid, bytes: b }); },
+    };
+    const view = {
+        store, identity: c.identity, programs: s.programs,
+        addressBook: (c.addressBook ?? []).map((e) => ({ key: Buffer.from(e.key).toString("hex"), transport: e.transport, address: e.address, ...(e.handle ? { handle: e.handle } : {}), ...(e.domain ? { domain: e.domain } : {}) })),
+        heads: [], dispatch: [...s.dispatch],
+    };
+    const rows = [];
+    const heads = new Map();
+    let put = 0;
+    const put1 = async (r) => { if (!(await k.hasBlock(r.cid))) {
+        await k.putBlock(r.cid, r.bytes);
+        put++;
+    } };
+    const apps = [];
+    for (const entry of t.apps.install) {
+        const key = appKey(entry);
+        let tree = t.appTrees?.[key] ?? resolved[key];
+        if (!tree && typeof entry === "string")
+            throw new Error(`${APPS}: ${entry} is not a directory of the image (a directory beside it is read when the image is booted from its directory)`);
+        if (!tree)
+            tree = await appTarball(entry, imageObjects, fetched);
+        if (!t.appTrees?.[key])
+            for (const o of await treeObjects(objects, tree))
+                await put1(o);
+        const app = await readStoredApp(store, tree);
+        // #142: an app's config at birth (the host skein's onboard: its domain, origin, name and note from host.env), merged as `--config` is.
+        const config = c.appConfig?.[app.checked.manifest.name];
+        if (config)
+            app.checked.manifest.config = mergeConfig(app.checked.manifest.config, config);
+        const p = await planInstall(app, view, { modules: objects });
+        if (p.start)
+            throw new Error(`${APPS}: ${p.app}: its start message is root's, sent by an install by message; a genesis carries no message (install ${p.app} after the claim)`);
+        for (const r of p.records) {
+            local.map.set(r.cid.toString(), r);
+            await put1(r);
+        }
+        for (const h of p.heads) {
+            heads.set(h.name, h.tree);
+            view.heads = [...view.heads.filter((x) => x.name !== h.name), { name: h.name, root: h.tree }];
+        }
+        for (const r of p.rows) {
+            rows.push(r.row);
+            view.dispatch.push(r.row);
+        }
+        apps.push({ name: p.app, version: p.version, notes: p.notes });
+    }
+    // #147: a root route's handler may be `<app>.<role>` — an app installed above, its program record (as `skein routes add`).
+    const named = { ...s.programs };
+    for (const r of t.apps.routes ?? []) {
+        const role = typeof r.program === "string" ? /^([a-z0-9][a-z0-9-]*)\.([A-Za-z0-9_-]+)$/.exec(r.program) : null;
+        if (!role || named[r.program])
+            continue;
+        const app = await appRecordIn(view, role[1]);
+        if (!app)
+            throw new Error(`${APPS}: route ${r.address}: handler ${r.program}: no app ${role[1]} installed at birth`);
+        const cid = app.record.programs[role[2]];
+        if (!cid)
+            throw new Error(`${APPS}: route ${r.address}: handler ${r.program}: app ${role[1]} has no program ${role[2]} (${Object.keys(app.record.programs).join(", ")})`);
+        named[r.program] = cid;
+    }
+    for (const r of rowsOf(t.apps.routes ?? [], named)) {
+        const p = await planRootRoute(view, "add", r);
+        for (const x of p.rows) {
+            rows.push(x.row);
+            view.dispatch.push(x.row);
+        }
+    }
+    return { rows, heads: Object.fromEntries(heads), put, apps };
+}
+/** The program records' CIDs a tree's bin/ yields (as `boot` puts them), without a kernel. */
+export function programCid(record) { return encode(record).cid; }
+/** The stock system as files (`skein-host system <dir>`): bin/*.cid + bin/*.json from the kernel's records, etc/. */
+export async function stockSystemFiles(k) {
+    const programs = await k.call("programs");
+    const files = {};
+    for (const [name, cid] of Object.entries(programs)) {
+        const r = await k.store.get(cid);
+        files[`${BIN}/${name}.cid`] = `${r.code.wasm.toString()}\n`;
+        files[`${BIN}/${name}.json`] = `${JSON.stringify({ inputs: r.inputs, services: r.services, description: r.description }, null, 2)}\n`;
+    }
+    const { DEFAULTS } = await import("../runtime/log.js");
+    const { STOCK_DISPATCH, STOCK_ROLES, STOCK_SCOPES } = await import("./genesis.js");
+    files[CONFIG] = `${JSON.stringify({ defaults: DEFAULTS, collect: ["completions"], scopes: STOCK_SCOPES, roles: STOCK_ROLES }, null, 2)}\n`;
+    // The stock routes (#77, #143): the boxes, the events and the HTTP routes (the admin routes are every genesis's).
+    files[DISPATCH] = `${JSON.stringify(STOCK_DISPATCH, null, 2)}\n`;
+    return files;
+}
+/**
+ * Boot a store file that no router holds yet (`skein-host add --boot/--packet`,
+ * tests): start its kernel, `boot`, stop it. The router then hydrates it as
+ * any other store; the kernel processes the genesis at that first start.
+ */
+export async function bootStore(o, src, c, time) {
+    const { Kernel, KernelHost } = await import("./kernel.js");
+    const { mkdirSync } = await import("node:fs");
+    const { dirname } = await import("node:path");
+    const { parseRocksSpec } = await import("../runtime/index-store.js");
+    // #158: a RocksDB set boots in the host's one kernel process (this node process's, shared with its router, or one of its own).
+    const rocks = parseRocksSpec(o.db);
+    // The settled split: a SQLite set boots on the wasm kernel (this node process's machine for it).
+    const { parseSqliteSpec } = await import("../runtime/index-store.js");
+    const sq = parseSqliteSpec(o.db);
+    if (sq) {
+        const { WasmHost } = await import("./wasm-kernel.js");
+        const net = o.env?.SKEIN_NETWORK ?? process.env.SKEIN_NETWORK;
+        const host = WasmHost.acquire(sq.db, { network: net ?? "main", log: o.log });
+        try {
+            const k = await host.open({ db: o.db, handle: o.handle, domain: o.domain, log: o.log });
+            try {
+                return await boot(k, src, c, time);
+            }
+            finally {
+                await k.stop();
+            }
+        }
+        finally {
+            await host.release();
+        }
+    }
+    if (rocks) {
+        const host = KernelHost.acquire(rocks.dir, { command: o.command, env: o.env, log: o.log });
+        try {
+            const k = await host.open({ db: o.db, handle: o.handle, domain: o.domain, log: o.log });
+            try {
+                return await boot(k, src, c, time);
+            }
+            finally {
+                await k.stop();
+            }
+        }
+        finally {
+            await host.release();
+        }
+    }
+    if (!o.db.startsWith("lmdb:"))
+        mkdirSync(dirname(o.db), { recursive: true }); // an LMDB set (#96): the kernel makes its environment
+    const k = new Kernel({ db: o.db, handle: o.handle, domain: o.domain, command: o.command, env: o.env, log: o.log });
+    try {
+        return await boot(k, src, c, time);
+    }
+    finally {
+        await k.stop(5000);
+    }
+}
+// ---------------------------------------------------------------- more sources
+/** Blocks out of a store (a store file opened read-only, or anything with `bytes`). */
+export function storeObjects(s) {
+    // A block the store does not have (or cannot read) is absent: the loader refuses what names it, with the CID.
+    return { get: async (cid) => { try {
+            return await s.bytes(cid);
+        }
+        catch {
+            return undefined;
+        } } };
+}
+/** The first source that has the block. */
+export function anyOf(...sources) {
+    return { get: async (cid) => { for (const s of sources) {
+            const b = await s.get(cid);
+            if (b)
+                return b;
+        } return undefined; } };
+}
+/** The modules in a wasm directory (the repo's wasm/: the pinned builds), by their raw CIDs; hashed on first use. */
+export function wasmDirObjects(dir) {
+    let byCid;
+    return {
+        get: async (cid) => {
+            if (cid.code !== RAW)
+                return undefined;
+            const { readdirSync, readFileSync } = await import("node:fs");
+            const { join } = await import("node:path");
+            if (!byCid) {
+                byCid = new Map();
+                for (const f of readdirSync(dir))
+                    if (f.endsWith(".wasm") || f.endsWith(".zip"))
+                        byCid.set(rawCid(readFileSync(join(dir, f))).toString(), join(dir, f));
+            }
+            const p = byCid.get(cid.toString());
+            return p ? new Uint8Array(readFileSync(p)) : undefined;
+        },
+    };
+}
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../..");
+/** The repo's pinned modules (wasm/): what a system tree's `bin/*.cid` names. */
+export const WASM_DIR = join(ROOT, "wasm");
+/** The default image (#89): a user skein's — no root in it, a claim route. */
+export const DEFAULT_IMAGE = join(ROOT, "images/default");
+/** The host image's own part (#142): merged over the default image (mergeImages), the host skein's image. */
+export const HOST_IMAGE = join(ROOT, "images/host");
+/**
+ * Two image trees merged (#142: images are starting trees that merge): every path of `over` added to
+ * `base`. A directory in both is merged; a file in both must be the same blob — except `etc/apps.json`,
+ * whose `install` and `routes` lists are joined (an app in both, by appKey, is refused; a root route of
+ * `over` at the same transport, address and prefix as one of `base`'s replaces it, #147). Any other path in both is a
+ * collision, refused with its path. The installer's own checks (boxes, heads, paths) run when the merged
+ * image's apps are installed at birth (installAtBirth). The new trees and the merged apps.json go into
+ * `out`; every other object stays where it was.
+ */
+export async function mergeImages(objects, base, over, out) {
+    const tree = async (cid) => parseTree(await need(objects, cid, "an image tree"), cid);
+    const blob = async (cid, path) => {
+        const b = gitBody(await need(objects, cid, path), "blob");
+        if (!b)
+            throw new Error(`${path}: not a file`);
+        return Buffer.from(b).toString("utf8");
+    };
+    const merge = async (a, b, at) => {
+        const entries = new Map((await tree(a)).map((e) => [e.name, e]));
+        for (const e of await tree(b)) {
+            const path = at ? `${at}/${e.name}` : e.name;
+            const had = entries.get(e.name);
+            if (!had || had.cid.equals(e.cid)) {
+                entries.set(e.name, e);
+                continue;
+            }
+            if (had.mode === "40000" && e.mode === "40000") {
+                entries.set(e.name, { ...e, cid: await merge(had.cid, e.cid, path) });
+                continue;
+            }
+            if (path === APPS && had.mode !== "40000" && e.mode !== "40000") {
+                const x = JSON.parse(await blob(had.cid, path)), y = JSON.parse(await blob(e.cid, path));
+                const theirs = new Set((y.install ?? []).map(appKey));
+                const both = (x.install ?? []).map(appKey).filter((p) => theirs.has(p));
+                if (both.length)
+                    throw new Error(`${APPS}: ${both.join(", ")} installed by both images`);
+                // #147: a root route of `over` at a route of `base`'s (same transport, address, prefix) replaces it.
+                const key = (r) => `${r.transport ?? "mailbox"} ${r.address}${r.prefix ? " prefix" : ""}`;
+                const mine = new Set((y.routes ?? []).map(key));
+                const routes = [...(x.routes ?? []).filter((r) => !mine.has(key(r))), ...(y.routes ?? [])];
+                const h = hashBlob(new TextEncoder().encode(`${JSON.stringify({ install: [...(x.install ?? []), ...(y.install ?? [])], ...(routes.length ? { routes } : {}) }, null, 2)}\n`));
+                await out.putBlock(h.cid, h.object);
+                entries.set(e.name, { ...e, cid: h.cid });
+                continue;
+            }
+            throw new Error(`images: ${path} is in both images (a merge adds paths; it does not replace one)`);
+        }
+        const t = hashTree([...entries.values()]);
+        await out.putBlock(t.cid, t.object);
+        return t.cid;
+    };
+    return await merge(base, over, "");
+}
+/**
+ * #152: an image directory with no `bin/` is an overlay — it carries what it adds (an app repository's
+ * `image/`: `etc/apps.json` alone) and is merged over the default image before boot, as images/host is;
+ * one with `bin/` is a complete image of its own.
+ */
+export function isOverlayImage(dir) {
+    const bin = join(dir, BIN);
+    return !(existsSync(bin) && statSync(bin).isDirectory());
+}
+/**
+ * #142, #152: an overlay image directory (`dir`: images/host, an app repository's `image/`) merged over
+ * `base` (mergeImages: an app in both refused, a root route at the same place replaced, any other path in
+ * both refused unless it is the same blob). The apps each names beside it are kept, by appKey.
+ */
+export async function overlayImage(base, dir) {
+    const own = await imageDir(dir);
+    const merged = new MemBlocks();
+    const objects = anyOf(merged, base.objects, own.objects);
+    const root = await mergeImages(objects, base.root, own.root, merged);
+    return { root, objects, apps: { ...base.apps, ...own.apps } };
+}
+/**
+ * An image as a boot source (#89, #90): a directory (default: the default
+ * image), its objects with the repo's pinned modules beside them, and the
+ * apps it names by a directory outside it (imageDir).
+ */
+export async function imageSource(dir = DEFAULT_IMAGE) {
+    const d = await imageDir(dir);
+    return { kind: "tree", root: d.root, objects: anyOf(d.objects, wasmDirObjects(WASM_DIR)), apps: d.apps };
+}
+/**
+ * #152: an image directory scanned, and each app its etc/apps.json names by a directory outside it
+ * (`..`, `../<dir>`, an absolute path) scanned from there as `skein install <dir>` reads it
+ * (install.ts appIgnore): the trees by appKey (BootSource `apps`), their objects with the image's.
+ */
+export async function imageDir(dir) {
+    const d = await dirSource(dir);
+    const apps = {};
+    const file = join(dir, APPS);
+    if (!existsSync(file))
+        return { ...d, apps };
+    let list;
+    try {
+        list = JSON.parse(readFileSync(file, "utf8"));
+    }
+    catch (x) {
+        throw new Error(`${APPS}: ${x.message}`);
+    }
+    checkImageApps(list);
+    for (const a of list.install) {
+        if (typeof a !== "string")
+            continue;
+        const key = appKey(a);
+        if (posix.isAbsolute(key) || key === ".." || key.startsWith("../"))
+            apps[key] = await scan(d.objects, resolve(dir, a), { ignore: appIgnore });
+    }
+    return { ...d, apps };
+}
+/**
+ * #152: an app by release tarball, `hash` its tree's git id: the tree the image's objects hold
+ * already, else fetched from `url` (gzip or plain tar), unpacked (a tarball holding one directory,
+ * as a tag's archive does, is that directory), scanned into `into` as `skein install <dir>` reads
+ * a directory (install.ts appIgnore), and refused unless its tree is `hash`.
+ */
+export async function appTarball(a, have, into, get = fetch) {
+    const want = sha1Cid(Buffer.from(a.hash, "hex"));
+    if (await have.get(want))
+        return want;
+    const r = await get(a.url);
+    if (!r.ok)
+        throw new Error(`${APPS}: ${a.url}: ${r.status} ${r.statusText}`);
+    let bytes = new Uint8Array(await r.arrayBuffer());
+    if (bytes[0] === 0x1f && bytes[1] === 0x8b)
+        bytes = gunzipSync(bytes);
+    const dir = mkdtempSync(join(tmpdir(), "skein-app-"));
+    try {
+        const x = spawnSync("tar", ["-x", "-f", "-", "-C", dir], { input: bytes });
+        if (x.status !== 0)
+            throw new Error(`${APPS}: ${a.url}: not a tarball (${x.stderr.toString().trim() || `tar exit ${x.status}`})`);
+        const top = readdirSync(dir, { withFileTypes: true });
+        const root = await scan(into, top.length === 1 && top[0].isDirectory() ? join(dir, top[0].name) : dir, { ignore: appIgnore });
+        if (!root.equals(want))
+            throw new Error(`${APPS}: ${a.url}: its tree is ${Buffer.from(root.multihash.digest).toString("hex")}, not ${a.hash}`);
+        return root;
+    }
+    finally {
+        rmSync(dir, { recursive: true, force: true });
+    }
+}
+/** Source B: a packet file, verified (packet.ts) — a system tree to boot, or a checkpoint to restore. */
+export async function packetSource(bytes, o = {}) {
+    const { readPacket } = await import("./packet.js");
+    const v = await readPacket(bytes, o);
+    return v.kind === "tree" ? { kind: "tree", root: v.scope, objects: v.blocks } : { kind: "checkpoint", state: v.scope, blocks: v.list };
+}