@dcasegr/skein 0.1.2 → 0.2.8

package/dist/host/router.d.ts

@@ -4,6 +4,7 @@ import { CID } from "multiformats/cid";
 import type { Stamp } from "../runtime/syscalls.ts";
 import { Broadcaster, type ArcConfig } from "./arc.ts";
 import { Feeds, type FeedSpec } from "./feeds.ts";
+import { TeranodeTracker } from "./teranode.ts";
 import { Cron } from "./cron.ts";
 import { type DispatchRow } from "../runtime/dispatch.ts";
 import { type BootSource, type Booted } from "./boot.ts";
@@ -11,7 +12,7 @@ import { ImageChain } from "./image-chain.ts";
 import { OverlaySync } from "./sync.ts";
 import { type AddressSeed, type DispatchSpec, type Genesis2Config, type Libp2pSpec } from "./genesis.ts";
 import { HANDLE, type HostDb, type InstanceRow } from "./instances.ts";
-import { type KernelSurface } from "./kernel.ts";
+import { type HostChainTip, type KernelSurface } from "./kernel.ts";
 import { type Live } from "./liveness.ts";
 import { EventStreams } from "./events.ts";
 import { P2PHost, type Beacon, type InboundAnswer, type InboundCall, type P2PHostConfig, type Subscription } from "./p2p.ts";
@@ -53,6 +54,21 @@ export interface RouterOptions {
     };
     /** The host's headers feed (#102, SKEIN_HEADERS_URL): an SSE stream of block headers every enabled instance whose dispatch table takes events in box `chain` is subscribed to. */
     headersFeed?: string;
+    /**
+     * #156: the host's headers tracker over the Teranode P2P network (teranode.ts): its bootstrap
+     * peers (the network's, SKEIN_TERANODE_BOOTSTRAP), its first-sync hub (the network's default,
+     * SKEIN_HEADERS_HUB), its key file ($SKEIN_HOME/teranode.key). Beside the headers feed or alone:
+     * both feed the same intake. Absent (SKEIN_TERANODE=off, or a network with neither): no tracker.
+     */
+    teranode?: {
+        bootstrap: string[];
+        hub?: string;
+        peersFile?: string;
+        page?: number;
+        bulkPage?: number;
+        maxCrawl?: number;
+        discoveryMs?: number;
+    };
     /** The host's Arcade (#58, #65, arc.ts): where the broadcast events go, the status subscription, the `status` provider. Absent: no broadcaster (a broadcast event is dropped). */
     arc?: ArcConfig;
     /** Tests: how the broadcaster reaches Arcade. */
@@ -111,6 +127,11 @@ export interface RouterOptions {
     libp2p?: P2PHostConfig;
     /** An instance's libp2p peer key (signer.ts peerKey). Absent: no libp2p host; the kernels' `libp2p` is refused. */
     peerKeyFor?(handle: string): PrivateKey;
+    /**
+     * #156: the host node's peer key (P2PHost.hostNode: the Teranode tracker rides it; on a one-key
+     * host every instance's peer key, so they share it). Absent: a fresh key per start.
+     */
+    hostPeerKey?: PrivateKey;
     /**
      * A provider's key (#70, signer.ts providerKey: the HTTP proxy `fetch`, the
      * `waker`, `cron`, `libp2p`, `status`), which each new genesis's address
@@ -150,6 +171,17 @@ export interface RouterOptions {
      * stock wiring takes, from `handle`, handed over after its step's commit. Absent: logged as ignored.
      */
     appEvent?(handle: string, record: Record<string, unknown>): void;
+    /**
+     * An application host's reads (shruggr/open-exchange 0.11.0, the public read API): a GET for an
+     * instance (its hostname, `/@<handle>/`, or the root instance's hostname), asked AFTER the host's own
+     * reads there (the instance card, `/<app>/.live/<topic>`) and BEFORE the request is forwarded to the
+     * instance's program. `route` is the path at the instance (`/amm/pools`); `undefined`: not the host's,
+     * forwarded as before. An answer with `stream` is held open (RouterResponse). What the host needs
+     * for its reads is the Router's public surface: `dispatch` (in-process reads of the instances),
+     * `p2p.live.live(handle, topic)` (the beats kept), `events.subscribe(handle, query, send, close)` (an
+     * instance's emitted events), `o.db.list("enabled")` (the instances). Absent: nothing is asked.
+     */
+    appRead?(handle: string, route: string, url: URL, req: RouterRequest): Promise<RouterResponse | undefined>;
 }
 interface Loaded {
     row: InstanceRow;
@@ -180,6 +212,13 @@ export interface RouterResponse {
     status: number;
     headers: Record<string, string>;
     body: Uint8Array;
+    /**
+     * A response held open (an event stream an `appRead` answers): on a socket (Router.handler) the
+     * status and headers are written, then `stream` is called with `send` (writes a chunk) and `end`
+     * (closes the response); the function it returns is called once when the client goes. `body` is
+     * not written. In process (Router.dispatch's caller) it is not called: the caller gets `body`.
+     */
+    stream?(send: (chunk: string) => void, end: () => void): () => void;
 }
 export { HANDLE };
 /** Handles no instance may take on this host (H3), besides the router origin's own first label (Router.reserved): `id` (the router's own in production) and `host` (the host skein's). */
@@ -262,6 +301,13 @@ export declare class Router {
     private control?;
     /** The default image (#89), with its chain part (#132): what createInstance boots a new skein from. */
     readonly image: ImageChain;
+    /** #156: the headers tracker over the Teranode mesh (teranode.ts), when the host runs one. */
+    tracker?: TeranodeTracker;
+    /** #156: the libp2p the host node runs on: the instances' (`p2p`), else one for the tracker alone. */
+    hostP2p?: P2PHost;
+    private readonly p2pConfig;
+    /** #156: the header intake, one run at a time (the image chain, then the host chains). */
+    private intake;
     /** #130: each billed instance as last read (its host row's terms, the kernel's billing state), and what this host metered since its last tick. */
     private views;
     private periods;
@@ -392,6 +438,33 @@ export declare class Router {
      * libp2p arrivals, which do not queue here, need no queue.
      */
     serial<T>(handle: string, f: () => Promise<T>): Promise<T>;
+    /** #156: the key the host's headers arrive under (feeds.ts): the feed's URL, else the tracker's own. */
+    headersKey(): string | undefined;
+    /** #156: the Teranode tracker started: its node on the mesh, its first sync begun (`tracker.ready`). */
+    private startTracker;
+    /**
+     * #156: a run of headers from either source (the feed's, `at` undefined: image-chain.ts places
+     * each; the tracker's, at height `at`): into the image's chain part, then every host chain synced
+     * from it. One run at a time.
+     */
+    takeHeaders(raws: Uint8Array[], at?: number): Promise<void>;
+    /** #156: the host chains this host keeps once per database (its SQLite one, its RocksDB one, and those its rows name). */
+    private hostHolders;
+    /** #156: each host chain this host keeps (by database) and its tip (undefined: empty). */
+    hostChainTips(): Promise<Array<{
+        name: string;
+        tip: HostChainTip | undefined;
+    }>>;
+    /** #156: every host chain brought to the image chain's tip (from `from` when a run replaced from there). */
+    private syncHolders;
+    /**
+     * #156: a host chain brought to the image chain: an EMPTY one imports the image's chain part
+     * (`chain-import`, as an instance born with it would bring it: one frame) where its kernel can
+     * read it (the wasm kernel's SQLite database), else takes the image's headers from genesis in
+     * `headers` runs; one behind takes the headers it lacks; one the image does not hold at its
+     * height (a fork) from a little below it, or from `from`.
+     */
+    private syncHolder;
     /**
      * The network the host chain is on: SKEIN_NETWORK (`network`), else the genesis defaults'
      * `walletNetwork` (a test host's), else the kernel's default (main).

package/dist/host/router.js

@@ -117,7 +117,8 @@ import { CID } from "multiformats/cid";
 import { rootIdentity } from "../runtime/identity.js";
 import { DEFAULTS, short, stampMs } from "../runtime/log.js";
 import { ARC_ROUTE, Broadcaster } from "./arc.js";
-import { DEFAULT_BOX, Feeds, feedsOf, txCid } from "./feeds.js";
+import { BACKFILL_RUN, DEFAULT_BOX, Feeds, feedsOf, txCid } from "./feeds.js";
+import { hashOf, TeranodeTracker } from "./teranode.js";
 import { Cron, dbSchedules } from "./cron.js";
 import { currentDispatch, takesEvent, takesMail } from "../runtime/dispatch.js";
 import { isSet, parseRocksSpec, parseSqliteSpec, storeSpec } from "../runtime/index-store.js";
@@ -136,9 +137,9 @@ import { keyBytes, keyHex } from "./genesis.js";
 import { HANDLE, pathOnly, TOKEN_HANDLE, topicsOf, validHandle } from "./instances.js";
 import { signClaim } from "../client/raw.js";
 import { DoorAnswered, Kernel, KernelHost } from "./kernel.js";
-import { foldLiveness, livenessEvent, livenessOf } from "./liveness.js";
+import { foldLiveness, LIVENESS_MAX_MS, livenessEvent, livenessOf } from "./liveness.js";
 import { EventStreams, eventsQuery, sseMessage } from "./events.js";
-import { beaconEvent, beaconsOf, DEFAULT_LISTEN, emittedEvents, foldBeacons, foldSubscriptions, libp2pConfig, P2PHost, subscribedTopics, subscriptionEvent, subscriptionsOf } from "./p2p.js";
+import { beaconEvent, beaconsOf, LOOPBACK_LISTEN, emittedEvents, foldBeacons, foldSubscriptions, libp2pConfig, P2PHost, subscribedTopics, subscriptionEvent, subscriptionsOf } from "./p2p.js";
 import { peerIdFromMultihash, peerIdFromString } from "@libp2p/peer-id";
 import { MESSAGE_KEY_ID, MESSAGE_PROTOCOL, Providers, tooLarge } from "./providers.js";
 import { closeControl, listenControl } from "./control.js";
@@ -314,6 +315,13 @@ export class Router {
     control;
     /** The default image (#89), with its chain part (#132): what createInstance boots a new skein from. */
     image;
+    /** #156: the headers tracker over the Teranode mesh (teranode.ts), when the host runs one. */
+    tracker;
+    /** #156: the libp2p the host node runs on: the instances' (`p2p`), else one for the tracker alone. */
+    hostP2p;
+    p2pConfig;
+    /** #156: the header intake, one run at a time (the image chain, then the host chains). */
+    intake = Promise.resolve();
     /** #130: each billed instance as last read (its host row's terms, the kernel's billing state), and what this host metered since its last tick. */
     views = new Map();
     periods = new Map();
@@ -387,9 +395,12 @@ export class Router {
                 holds: async (h, txid) => await (await this.hydrate(h)).kernel.hasBlock(txCid(txid)),
             });
         }
+        // #156: the host's libp2p (one node per key; the host node among them) — for the instances, and for the tracker on its own.
+        const hostConfig = { ...(o.libp2p ?? { listen: LOOPBACK_LISTEN, bootstrap: [], dht: "off", relays: [], mdns: false }), network: this.chainNetwork() ?? "main" };
+        this.p2pConfig = hostConfig;
         if (o.peerKeyFor) {
             this.p2p = new P2PHost({
-                host: o.libp2p ?? { listen: DEFAULT_LISTEN, bootstrap: [], dht: "off", relays: [], mdns: false },
+                host: hostConfig, ...(o.hostPeerKey ? { hostKey: o.hostPeerKey } : {}),
                 keyOf: o.peerKeyFor, discoveryMs: o.libp2pDiscoveryMs,
                 inbound: (h, c) => this.p2pInbound(h, c),
                 log: (s, l) => this.say(s, l),
@@ -397,8 +408,10 @@ export class Router {
                 sign: (h, data) => this.signAs(h, data),
                 identity: async (h) => { const id = this.o.db.get(h)?.identity; return id ? keyBytes(id) : undefined; },
                 beat: (h, topic, b) => { this.sync?.beat(h, topic, b); void this.rememberPeer(h, topic, b.from); },
+                beats: o.db,
             });
             const p2pHost = this.p2p;
+            this.hostP2p = p2pHost;
             this.sync = new OverlaySync({
                 overlayOf: (h, liveTopic) => this.loaded.get(h)?.beacons.find((b) => b.topic === liveTopic)?.app,
                 local: (h, app, fn, query) => this.overlayRead(h, app, fn, query),
@@ -523,11 +536,28 @@ export class Router {
     }
     /** Hydrate every enabled row once (recovery at hydrate time: what a waiting thread awaits is handed to the providers again), then let them idle out. */
     async start() {
+        // #132, #156: the host's headers — the feed's and the Teranode tracker's — into one intake: the image's chain part, then the host chains.
+        const key = this.headersKey();
+        if (key)
+            this.feeds.listen(key, (raws, at) => this.takeHeaders(raws, at));
         // #132: the image's chain part, filled from the feed's history (from genesis once; then from near its tip), then grown by the feed.
         if (this.o.headersFeed) {
-            void this.image.start();
-            this.feeds.listen(this.o.headersFeed, (raws) => { void this.image.headers(raws); });
-        }
+            const started = this.image.start();
+            void started.then(() => this.syncHolders()).catch((e) => this.say("router", `host chain: not filled from the image chain: ${e.message}`));
+            // With a feed, its history makes the first sync; the tracker follows the mesh once that is done.
+            if (this.o.teranode)
+                void started.then(() => this.startTracker()).catch((e) => this.say("router", `headers: the Teranode tracker not started: ${e.message}`));
+        }
+        else if (this.o.teranode) {
+            // #156: the tracker is the source: its first sync (a hub's, when the network has one) before any instance steps.
+            await this.startTracker().catch((e) => this.say("router", `headers: the Teranode tracker not started: ${e.message}`));
+            if (this.tracker?.o.hub)
+                await this.tracker.ready;
+        }
+        // #156: a host chain that is empty (or behind) filled from the image chain, before any instance is hydrated.
+        await this.syncHolders().catch((e) => this.say("router", `host chain: not filled from the image chain: ${e.message}`));
+        // #138: the stored liveness beats older than the longest window (a day) go before any instance reloads them.
+        this.o.db.pruneBeats(stampMs(this.now()) - LIVENESS_MAX_MS);
         for (const row of this.o.db.list("enabled")) {
             try {
                 await this.hydrate(row.handle);
@@ -590,7 +620,8 @@ export class Router {
         this.control = undefined;
         this.providers.stop();
         this.events.stop();
-        await Promise.all([this.cron.stop(), this.feeds.stop(), this.arc?.stop(), this.p2p?.stop()]);
+        await Promise.all([this.cron.stop(), this.feeds.stop(), this.arc?.stop(), this.tracker?.stop()]);
+        await Promise.all([this.p2p?.stop(), this.hostP2p !== this.p2p ? this.hostP2p?.stop() : undefined]); // #156: the host node with them
         // A kernel still hydrating: its load finishes (or fails) first, then it is stopped with the rest.
         await Promise.all([...this.loading.values()].map((p) => p.catch(() => { }))); // a failed load was reported to whoever asked for it
         await Promise.all([...this.loaded.values()].map((l) => l.kernel.stop()));
@@ -815,6 +846,128 @@ export class Router {
         return next;
     }
     // ---------------------------------------------------------------- the host chain (#156)
+    /** #156: the key the host's headers arrive under (feeds.ts): the feed's URL, else the tracker's own. */
+    headersKey() {
+        return this.o.headersFeed ?? (this.o.teranode ? `teranode:${this.chainNetwork() ?? "main"}` : undefined);
+    }
+    /** #156: the Teranode tracker started: its node on the mesh, its first sync begun (`tracker.ready`). */
+    async startTracker() {
+        const t = this.o.teranode;
+        if (!t || this.tracker || this.stopped)
+            return;
+        const key = this.headersKey();
+        // #156: no instance libp2p (no peer keys: a test's host): the host's libp2p for the tracker alone.
+        this.hostP2p ??= new P2PHost({
+            host: this.p2pConfig, ...(this.o.hostPeerKey ? { hostKey: this.o.hostPeerKey } : {}), discoveryMs: this.o.libp2pDiscoveryMs,
+            keyOf: () => { throw new Error("no instance libp2p on this host"); },
+            inbound: async () => ({ verdict: "ignore" }),
+            log: (s, l) => this.say(s, l),
+        });
+        this.tracker = new TeranodeTracker({
+            network: this.chainNetwork() ?? "main", ...t, p2p: this.hostP2p,
+            chain: {
+                tip: () => this.image.tip(),
+                find: (hash) => this.image.find(hash),
+                // A long run (a first sync, a deep catch-up): into the image and the host chains, then each running instance its chain event; a short one as the feed's headers go.
+                take: async (at, raws) => {
+                    if (raws.length <= 4 * BACKFILL_RUN)
+                        return await this.feeds.inject(key, raws, at);
+                    await this.takeHeaders(raws, at);
+                    for (const h of [...this.loaded.keys()])
+                        void this.chainBirth(h).catch((e) => this.say(h, `host chain: the chain event not sent: ${e.message}`));
+                },
+            },
+            log: (l) => this.say("router", l),
+        });
+        this.image.ready = this.tracker.ready;
+        await this.tracker.start();
+        this.image.ready = this.tracker.ready;
+    }
+    /**
+     * #156: a run of headers from either source (the feed's, `at` undefined: image-chain.ts places
+     * each; the tracker's, at height `at`): into the image's chain part, then every host chain synced
+     * from it. One run at a time.
+     */
+    takeHeaders(raws, at) {
+        const p = this.intake.then(async () => {
+            if (at === undefined)
+                await this.image.headers(raws);
+            else
+                await this.image.put(at, raws);
+            await this.syncHolders(at);
+        });
+        this.intake = p.catch(() => { });
+        return p;
+    }
+    /** #156: the host chains this host keeps once per database (its SQLite one, its RocksDB one, and those its rows name). */
+    hostHolders() {
+        const sq = new Set(), rocks = new Set();
+        if (this.o.sqliteStore)
+            sq.add(this.o.sqliteStore);
+        if (this.o.rocksStore)
+            rocks.add(this.o.rocksStore);
+        for (const row of this.o.db.list("enabled")) {
+            const spec = storeSpec(row);
+            const r = parseRocksSpec(spec), q = parseSqliteSpec(spec);
+            if (r)
+                rocks.add(r.dir);
+            if (q)
+                sq.add(q.db);
+        }
+        return [
+            ...[...sq].map((db) => { const w = this.wasmHost(db); return { name: db, tip: () => w.chainTip(), headers: (raws) => w.chainHeaders(raws), importChain: w.importChain.bind(w) }; }),
+            ...[...rocks].map((dir) => { const k = this.kernelHost(dir); return { name: dir, tip: () => k.chainTip(), headers: (raws) => k.chainHeaders(raws) }; }),
+        ];
+    }
+    /** #156: each host chain this host keeps (by database) and its tip (undefined: empty). */
+    async hostChainTips() {
+        return await Promise.all(this.hostHolders().map(async (h) => ({ name: h.name, tip: await h.tip() })));
+    }
+    /** #156: every host chain brought to the image chain's tip (from `from` when a run replaced from there). */
+    async syncHolders(from) {
+        for (const h of this.hostHolders())
+            await this.syncHolder(h.name, h, from).catch((e) => this.say("router", `host chain ${h.name}: not synced from the image chain: ${e.message}`));
+    }
+    /**
+     * #156: a host chain brought to the image chain: an EMPTY one imports the image's chain part
+     * (`chain-import`, as an instance born with it would bring it: one frame) where its kernel can
+     * read it (the wasm kernel's SQLite database), else takes the image's headers from genesis in
+     * `headers` runs; one behind takes the headers it lacks; one the image does not hold at its
+     * height (a fork) from a little below it, or from `from`.
+     */
+    async syncHolder(name, h, from) {
+        const top = this.image.height();
+        if (top < 0)
+            return;
+        const tip = await h.tip();
+        const t0 = Date.now();
+        if (!tip && h.importChain) {
+            const part = this.image.chainPart();
+            if (part) {
+                const got = await h.importChain(part.chain, part.objects);
+                this.say("router", `host chain: ${got.added ?? 0} headers imported from the image chain (tip ${got.height}, ${Date.now() - t0} ms)`);
+                return;
+            }
+        }
+        let start = 0;
+        if (tip) {
+            const held = this.image.headerAt(tip.height);
+            start = held && hashOf(held) === tip.hash ? tip.height + 1 : Math.max(0, tip.height - 6);
+            if (from !== undefined)
+                start = Math.min(start, from);
+        }
+        if (start > top)
+            return;
+        let n = 0;
+        for (const run of this.image.runs(start, BACKFILL_RUN)) {
+            await h.headers(run);
+            n += run.length;
+            if (top - start > 10 * BACKFILL_RUN && n % (50 * BACKFILL_RUN) < BACKFILL_RUN)
+                this.say("router", `host chain: ${start + n}/${top + 1} from the image chain`);
+        }
+        if (n > BACKFILL_RUN)
+            this.say("router", `host chain: ${n} headers from the image chain (to ${top}, ${Date.now() - t0} ms)`);
+    }
     /**
      * The network the host chain is on: SKEIN_NETWORK (`network`), else the genesis defaults'
      * `walletNetwork` (a test host's), else the kernel's default (main).
@@ -955,6 +1108,9 @@ export class Router {
             await kernel.start();
             // #156: after the start (the genesis processed: its tree is the head `main`), before any event is admitted.
             await this.chainBootstrap(handle, kernel);
+            // #156: a serve kernel's own host chain still empty (born before the image held a chain): from the image chain now.
+            if (kernel instanceof Kernel)
+                await this.syncHolder(handle, { tip: () => kernel.chainTip(), headers: (raws) => kernel.chainHeaders(raws) }).catch((e) => this.say(handle, `host chain: not synced from the image chain: ${e.message}`));
             await kernel.running(identity);
             void kernel.idle().catch(() => { }); // busy until what start resumed is done; a kernel that exits meanwhile is reported by its exit
             if (!row.identity)
@@ -1188,7 +1344,14 @@ export class Router {
             // operator's key, `appConfig` merged.
             // #162: any instance's app config at birth (a create's `config`), the host skein's owner at birth.
             // #164: an owned instance's genesis names `owner` root (boot.ts then seeds no claim route), as the host skein's does.
-            await this.bootRow(handle, await this.namedImage(o.image, o.host), { image: true, manager: o.host, ...(owned ? { owner } : {}), ...(o.appConfig ? { appConfig: o.appConfig } : {}) });
+            // The image read from its root to the last object walked: a header taken meanwhile keeps the objects it replaces.
+            const release = this.image.hold();
+            try {
+                await this.bootRow(handle, await this.namedImage(o.image, o.host), { image: true, manager: o.host, ...(owned ? { owner } : {}), ...(o.appConfig ? { appConfig: o.appConfig } : {}) });
+            }
+            finally {
+                release();
+            }
             const l = await this.hydrate(handle);
             if (claim) {
                 // #127: the owner's own message, forwarded as signed; the host signs nothing for the owner. #143: one the door
@@ -1399,7 +1562,7 @@ export class Router {
      * handled and unhandled) or stopped. A failure is logged, not fatal.
      */
     async syncDispatch(l, first = false) {
-        if ((!this.p2p && !this.o.headersFeed && !this.events.open(l.row.handle)) || l.kernel.gone)
+        if ((!this.p2p && !this.headersKey() && !this.events.open(l.row.handle)) || l.kernel.gone)
             return;
         const d = await l.kernel.dispatch().catch((e) => { this.say(l.row.handle, `the dispatch table not read: ${e.message}`); return undefined; });
         if (d === undefined)
@@ -1720,7 +1883,7 @@ export class Router {
      * else not. One line when that changes.
      */
     followHeaders(handle, rows) {
-        const url = this.o.headersFeed;
+        const url = this.headersKey();
         if (!url)
             return;
         const want = this.o.db.get(handle)?.status === "enabled" && rows.some((r) => r.address === DEFAULT_BOX && takesEvent(r, DEFAULT_BOX));
@@ -1749,7 +1912,7 @@ export class Router {
     }
     /** Follow the host's headers feed for an instance whose status changed (#102): its table read again. */
     async refollowHeaders(handle) {
-        if (!this.o.headersFeed)
+        if (!this.headersKey())
             return;
         const l = this.loaded.get(handle);
         const d = l && !l.kernel.gone ? await l.kernel.dispatch().catch((e) => { this.say(handle, `the dispatch table not read: ${e.message}`); return undefined; }) : undefined;
@@ -2072,6 +2235,12 @@ export class Router {
             // #148: a stream, served on a socket only (handler).
             if (req.method === "GET" && EVENTS_ROUTE.test(x.route))
                 return json(400, { status: "error", code: "ERR_BAD_REQUEST", description: "an event stream (text/event-stream) is served on a connection of its own, not in process" });
+            // An application host's own reads (appRead): before the instance's program; undefined falls through.
+            if (req.method === "GET" && this.o.appRead) {
+                const a = await this.o.appRead(x.handle, x.route, url, req);
+                if (a)
+                    return a;
+            }
             return await this.held(this.forward(x.handle, x.route, url, req));
         };
         if (t)
@@ -2079,6 +2248,16 @@ export class Router {
         const path = url.pathname;
         if (req.method === "GET" && path === "/skeins")
             return await this.skeinsHeldBy(url.searchParams.get("key") ?? "");
+        if (path === `${ARC_ROUTE}/callback`)
+            return await this.arcRequest(req, path);
+        // SKEIN_ROOT_INSTANCE: a hostname that is no instance's serves the root instance (the standalone's one app at
+        // localhost:<port>/) — every path but the router's own three above (.well-known/skein-host, /skeins, the ARC
+        // callback). The BRC-169 set below is the host skein's only when no root instance takes the hostname: with one,
+        // POST /.well-known/auth must shake hands with the root instance (its sessions are its own, #68) and
+        // GET /manifest.json is the app's (its grouped permissions), as at an instance's own hostname.
+        const root = this.o.rootInstance;
+        if (root && this.o.db.get(root)?.status === "enabled")
+            return await serve({ handle: root, route: path });
         // #113: the host's own origin's BRC-169 requests are the host skein's (its onboarding app's routes).
         const route = DISCOVERY[`${req.method} ${path}`] ?? (req.method === "GET" && path.startsWith(PAYMAIL_PREFIX) ? `/${ONBOARD_APP}${path}` : undefined);
         if (route) {
@@ -2090,13 +2269,6 @@ export class Router {
                 return f;
             return json(404, { status: "error", code: "ERR_NOT_FOUND", description: "this host has no host skein (skein-host init): no BRC-169 here" });
         }
-        if (path === `${ARC_ROUTE}/callback`)
-            return await this.arcRequest(req, path);
-        // SKEIN_ROOT_INSTANCE: a hostname that is no instance's serves the root instance (the standalone's one app at localhost:<port>/),
-        // after the host's own paths above.
-        const root = this.o.rootInstance;
-        if (root && this.o.db.get(root)?.status === "enabled")
-            return await serve({ handle: root, route: path });
         return json(404, { status: "error", code: "ERR_NOT_FOUND", description: "no instance here: an instance is at http://<handle>.localhost:<port>/ or /@<handle>/" });
     }
     /**
@@ -2322,7 +2494,28 @@ export class Router {
             req.on("end", () => {
                 const url = `http://${req.headers.host ?? `127.0.0.1:${this.port}`}${req.url ?? "/"}`;
                 this.dispatch({ method: req.method ?? "GET", url, headers: headerMap(req.headers), body: new Uint8Array(Buffer.concat(chunks)) })
-                    .then((r) => { res.writeHead(r.status, { ...cors, ...r.headers, "content-length": String(r.body.length) }); res.end(r.body); })
+                    .then((r) => {
+                    if (!r.stream) {
+                        res.writeHead(r.status, { ...cors, ...r.headers, "content-length": String(r.body.length) });
+                        res.end(r.body);
+                        return;
+                    }
+                    // A response held open (an appRead's event stream): written as it comes, until either side ends it.
+                    if (res.destroyed)
+                        return;
+                    res.writeHead(r.status, { ...cors, ...r.headers });
+                    res.flushHeaders();
+                    let forget;
+                    let gone = false;
+                    const leave = () => { if (gone)
+                        return; gone = true; forget?.(); };
+                    res.on("close", leave);
+                    forget = r.stream((chunk) => { if (!gone)
+                        res.write(chunk); }, () => { if (!gone)
+                        res.end(); });
+                    if (gone)
+                        forget();
+                })
                     .catch((e) => { if (!res.headersSent) {
                     res.writeHead(500, { ...cors, "content-type": "application/json" });
                     res.end(JSON.stringify({ status: "error", code: "ERR_INTERNAL", description: e.message }));

package/dist/host/signer.d.ts

@@ -32,6 +32,12 @@ export declare class Signer {
      * (p2p.ts peerIdOf).
      */
     peerKey(id: string): PrivateKey;
+    /**
+     * The host node's peer key (#156): the key's child [2, "skein instance"], key ID `libp2p`, self —
+     * in one-key mode every instance's peer key (peerKey), so they share the host node; in master-key
+     * mode the same rule over the master (its own node beside the instances').
+     */
+    hostPeerKey(): PrivateKey;
     /** The instance's identity key (hex): what `skein-host add` records. */
     identity(id: string): string;
     /** The instance's signer: a ProtoWallet over its root key (a WalletInterface whose actions reject). */

package/dist/host/signer.js

@@ -137,6 +137,14 @@ export class Signer {
             return this.deriver.derivePrivateKey(INSTANCE_PROTOCOL, "libp2p", "self");
         return new KeyDeriver(this.instanceKey(id)).derivePrivateKey(INSTANCE_PROTOCOL, `libp2p:${id}`, "self");
     }
+    /**
+     * The host node's peer key (#156): the key's child [2, "skein instance"], key ID `libp2p`, self —
+     * in one-key mode every instance's peer key (peerKey), so they share the host node; in master-key
+     * mode the same rule over the master (its own node beside the instances').
+     */
+    hostPeerKey() {
+        return this.deriver.derivePrivateKey(INSTANCE_PROTOCOL, "libp2p", "self");
+    }
     /** The instance's identity key (hex): what `skein-host add` records. */
     identity(id) {
         return this.instanceKey(id).toPublicKey().toString();

package/dist/host/teranode.d.ts

@@ -0,0 +1,132 @@
+import type { Libp2p } from "@libp2p/interface";
+import { type P2PHost } from "./p2p.ts";
+/** The network's bootstrap peers (go-teranode-p2p-client config.go); regtest has none. */
+export declare const TERANODE_BOOTSTRAP: Record<string, string[]>;
+/** The default data hub per network (first sync; the gorillanode Teranode asset API); regtest has none. */
+export declare const DEFAULT_HUBS: Record<string, string>;
+/** The block topic of a network (SKEIN_NETWORK's names, or the topic's own). */
+export declare function blockTopic(network: string): string;
+/** A header's hash, display hex. */
+export declare const hashOf: (raw: Uint8Array) => string;
+/** A header's parent's hash, display hex. */
+export declare const prevOf: (raw: Uint8Array) => string;
+/** One block announcement, as the tracker reads it. */
+export interface Announcement {
+    hash: string;
+    height: number;
+    header: Uint8Array;
+    hub?: string;
+    peer?: string;
+}
+/**
+ * A block announcement out of a message on the block topic: the bus's envelope or a bare
+ * BlockMessage. Undefined (dropped) when it is neither, its header is not 80 bytes of hex, or the
+ * header's hash is not the `Hash` it names.
+ */
+export declare function parseAnnouncement(data: Uint8Array): Announcement | undefined;
+export interface HubOptions {
+    fetch?: typeof fetch;
+    timeoutMs?: number;
+}
+/** The hub's best header (80 bytes) and, when it says, its height (`bestblockheader/json`). */
+export declare function hubBest(hub: string, o?: HubOptions): Promise<{
+    header: Uint8Array;
+    hash: string;
+    height?: number;
+}>;
+/**
+ * Up to `n` headers from `hash` back (child to parent, `hash` first): each one's proof of work and
+ * every link checked; a page that does not start at `hash` or does not link is refused.
+ */
+export declare function hubBack(hub: string, hash: string, n: number, o?: HubOptions): Promise<Uint8Array[]>;
+/** What the tracker reads and writes: the host's image chain and the intake behind it (Router). */
+export interface TrackerChain {
+    /** The tip held (undefined: nothing held). */
+    tip(): {
+        height: number;
+        hash: string;
+    } | undefined;
+    /** The height of a header held near the tip, by hash (display hex). */
+    find(hash: string): number | undefined;
+    /** A run of headers, parents first, at `height` (its first one's): into the image chain and on to the host chain and the instances. */
+    take(height: number, raws: Uint8Array[]): Promise<void>;
+}
+export interface TrackerOptions {
+    network: string;
+    /** Bootstrap multiaddrs (dnsaddr resolved by libp2p): the network's and SKEIN_TERANODE_BOOTSTRAP's; dialled while the host node has few peers. */
+    bootstrap: string[];
+    /** The host's libp2p (#156): the tracker rides its host node. None: no mesh (a hub-only first sync). */
+    p2p?: P2PHost;
+    /** The hub for the first sync (DEFAULT_HUBS, SKEIN_HEADERS_HUB); none: the first announcement's. */
+    hub?: string;
+    /**
+     * Where the peers met are kept between runs (go-p2p-message-bus's peer cache): dialled at start
+     * beside the bootstrap peers, so a restart does not hang on the bootstrap; none: not kept.
+     */
+    peersFile?: string;
+    chain: TrackerChain;
+    /** Headers per hub request: a gap's crawl (default 2 000) and the first sync's (default 10 000; a hub may answer fewer). */
+    page?: number;
+    bulkPage?: number;
+    /** The most headers a gap's crawl walks back before it gives up (default 20 160). */
+    maxCrawl?: number;
+    /** Peer upkeep: bootstrap redialled while too few are on the block topic (ms, default 30 000). */
+    discoveryMs?: number;
+    hub_?: HubOptions;
+    log?(line: string): void;
+}
+export declare class TeranodeTracker {
+    readonly o: TrackerOptions;
+    readonly topic: string;
+    node?: Libp2p;
+    /** Settles when the first sync is done (or there was nothing to do); rejects never. */
+    ready: Promise<void>;
+    private queue;
+    private stopped;
+    private timer?;
+    private syncing;
+    /** The newest announcement seen while the first sync ran (caught up from after it). */
+    private pending?;
+    private peers;
+    private lastBootErr;
+    /** Counts for the logs and tests. */
+    readonly stats: {
+        announcements: number;
+        dropped: number;
+        taken: number;
+        crawled: number;
+        known: number;
+    };
+    constructor(o: TrackerOptions);
+    private say;
+    /** Join the mesh (on the host node) and start the first sync when the image chain is empty. */
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /** The block topic on the host node: each message judged here (an announcement Accept and followed, anything else Reject); upkeep every round. */
+    private join;
+    /** The bootstrap peers dialled while too few are on the block topic. */
+    private upkeep;
+    /** The host node's DHT routing table's size, the skein DHT's (undefined: the DHT off). */
+    tables(): {
+        skein?: number;
+    };
+    /** The peers on the block topic the host node knows (GossipSub's view). */
+    onTopic(): number;
+    /** The peers kept from earlier runs (their dialable addresses). */
+    private cached;
+    /** The peers connected now — the block topic's first — kept for the next run (at most 32). */
+    private keep;
+    /** An announcement: in turn with the others (one change to the chain at a time). */
+    announce(a: Announcement): void;
+    /** go-chaintracks processBlockMessage: known, a child of a header near the tip, or a crawl back from its hub. */
+    private follow;
+    /**
+     * go-chaintracks SyncFromRemoteTip: back from `hash` through the hub until a header held (its
+     * height `base`), or the genesis header (base -1: the run starts at 0); the run parents first.
+     */
+    private crawl;
+    /** The first sync: an empty chain filled from a hub (the default, else the first announcement's). */
+    private initial;
+    /** The whole chain, from `best` back to genesis through `hub`, written in height order. */
+    private bulk;
+}