@dcasegr/skein 0.0.0-stage → 0.1.1

package/dist/host/feeds.js

@@ -0,0 +1,422 @@
+// Router-held feeds (#33 part 2, #29's revision): long-lived subscriptions the
+// router keeps on behalf of instances. A headers feed's headers go to the
+// host chain (#156): the header chain the host keeps once, in its kernel
+// process (`headers` frames, kernel.ts; a serve instance's in its own store);
+// on a changed record the subscribed instance is sent the event
+//
+//   chain {kind: "chain", root: <the host chain record>, height}
+//
+// in its box (Router.chainHeaders: one `headers` run per SSE event, the
+// backfill's in runs of 2016; the per-instance `header` events are gone).
+//
+// (The broadcaster's proofs travel the same way, #65: {kind: "proof",
+// subject, txid, path, …} in box `chain`, arc.ts; its other statuses are
+// signed messages from the status provider. `statusOf` reads Arcade's JSON
+// for it: {kind: "status", subject: <tx CID>, txid, txStatus, merklePath?:
+// bytes, blockHeight?, blockHash?, extraInfo?}.)
+//
+// An instance declares its header feeds in its config (etc/config.json,
+// `feeds`), which its genesis carries:
+//
+//   {kind: "headers", url, box?}          an SSE stream of block headers (ChainTracks-style):
+//                                         each event's data is the header's hex (160 digits), or
+//                                         JSON {header | raw | hex: "<hex>"}, or chaintracks' JSON
+//                                         {version, previousHash, merkleRoot, time, bits, nonce, height?, hash?}
+//                                         (Arcade's /chaintracks/v2/tip/stream; the hashes in display
+//                                         order: the 80 bytes serialized from the fields, and a `hash`
+//                                         that is not their sha256d is logged and dropped), or an array of them
+//
+// The host has one headers feed of its own besides (#102, SKEIN_HEADERS_URL;
+// the Router's `headersFeed`): every enabled instance whose dispatch table
+// takes events in box `chain` is subscribed to it, the others not; the router
+// keeps that current as dispatch tables change (`host`). A genesis's `feeds`
+// are in addition. The host listens to it itself too (#132, `listen`): every
+// header it brings grows the default image's chain part (image-chain.ts).
+//
+// `box` defaults to "chain" (the wallet's sender-less subscription). Nothing
+// here judges an item: the kernel's host chain tracker validates headers (and
+// the chain app proofs). One SSE connection per URL, fanned out to every subscriber;
+// reconnects with exponential backoff (and `Last-Event-ID`); each instance's
+// queue of items not yet admitted holds at most `maxQueue` (the oldest go
+// first, logged).
+//
+// The transaction half is the host's broadcaster (#58, #65, arc.ts): one
+// Arcade subscription for the whole host over the same SSE client
+// (SseStream), each proof and status routed to every instance that holds the
+// transaction. The
+// per-instance `arc-callback` feed is gone: a genesis that still declares one
+// has it ignored.
+import { Hash, Utils } from "@bsv/sdk";
+import { CID } from "multiformats/cid";
+import * as Digest from "multiformats/hashes/digest";
+export const DEFAULT_BOX = "chain";
+/** A transaction's CID: bitcoin-tx (0xb1), dbl-sha2-256 (0x56) over the txid in internal byte order. */
+export function txCid(txid) {
+    return CID.createV1(0xb1, Digest.create(0x56, Uint8Array.from(Buffer.from(txid, "hex").reverse())));
+}
+/** The feeds a genesis declares (`feeds`), the malformed ones (and a pre-#58 `arc-callback`) left out. */
+export function feedsOf(g) {
+    const fs = g?.feeds;
+    if (!Array.isArray(fs))
+        return [];
+    const out = [];
+    for (const f of fs) {
+        const box = typeof f?.box === "string" && f.box ? f.box : undefined;
+        if (f?.kind === "headers" && typeof f.url === "string")
+            out.push({ kind: "headers", url: f.url, ...(box ? { box } : {}) });
+    }
+    return out;
+}
+/**
+ * The 80 bytes of a header given as chaintracks' JSON (Arcade's tip stream):
+ * version, previousHash, merkleRoot (display order: reversed into raw),
+ * time, bits, nonce — and, with `hash`, checked against their sha256d. Not
+ * that form: undefined. A `hash` that does not match: an Error saying so.
+ */
+export function headerOfFields(o) {
+    const u32 = (x) => typeof x === "number" && Number.isInteger(x) && x >= 0 && x <= 0xffffffff;
+    const h32 = (x) => typeof x === "string" && /^[0-9a-fA-F]{64}$/.test(x);
+    if (!u32(o.version) || !h32(o.previousHash) || !h32(o.merkleRoot) || !u32(o.time) || !u32(o.bits) || !u32(o.nonce))
+        return undefined;
+    const w = new Utils.Writer();
+    w.writeUInt32LE(o.version);
+    w.writeReverse(Utils.toArray(o.previousHash, "hex"));
+    w.writeReverse(Utils.toArray(o.merkleRoot, "hex"));
+    w.writeUInt32LE(o.time).writeUInt32LE(o.bits).writeUInt32LE(o.nonce);
+    const raw = w.toArray();
+    if (o.hash !== undefined) {
+        const hash = Utils.toHex(Hash.hash256(raw).reverse());
+        if (typeof o.hash !== "string" || o.hash.toLowerCase() !== hash)
+            return new Error(`header ${String(o.height ?? "?")}: its hash ${String(o.hash)} is not its fields' (${hash}): dropped`);
+    }
+    return Uint8Array.from(raw);
+}
+/**
+ * Header bytes out of one SSE event's data: hex, JSON with header/raw/hex,
+ * chaintracks' fields (headerOfFields), or an array of those. A header whose
+ * `hash` is not its fields' is left out, and `drop` told why.
+ */
+export function headersOf(data, drop) {
+    const hex = (s) => typeof s === "string" && /^[0-9a-fA-F]{160}$/.test(s) ? Uint8Array.from(Buffer.from(s, "hex")) : undefined;
+    const t = data.trim();
+    const direct = hex(t);
+    if (direct)
+        return [direct];
+    let v;
+    try {
+        v = JSON.parse(t);
+    }
+    catch {
+        return [];
+    } // neither hex nor JSON: no header; the caller logs an event with none
+    const one = (x) => {
+        if (typeof x === "string")
+            return hex(x);
+        const o = x;
+        if (!o || typeof o !== "object")
+            return undefined;
+        const h = hex(o.header) ?? hex(o.raw) ?? hex(o.hex);
+        if (h)
+            return h;
+        const f = headerOfFields(o);
+        if (f instanceof Error) {
+            drop?.(f.message);
+            return undefined;
+        }
+        return f;
+    };
+    const list = Array.isArray(v) ? v : Array.isArray(v?.headers) ? v.headers : [v];
+    return list.map(one).filter((x) => !!x);
+}
+/** Arcade's (and ARC's) status JSON — an SSE event's data, a webhook's body — as a `status` event record, or why not. */
+export function statusOf(v) {
+    const o = v;
+    if (!o || typeof o !== "object")
+        return "not a JSON object";
+    const txid = o.txid;
+    if (typeof txid !== "string" || !/^[0-9a-f]{64}$/i.test(txid))
+        return "no txid";
+    if (typeof o.txStatus !== "string" || !o.txStatus)
+        return "no txStatus";
+    const ev = { kind: "status", subject: txCid(txid.toLowerCase()), txid: txid.toLowerCase(), txStatus: o.txStatus };
+    if (typeof o.merklePath === "string" && o.merklePath) {
+        if (!/^([0-9a-f]{2})+$/i.test(o.merklePath))
+            return "merklePath is not hex";
+        ev.merklePath = Uint8Array.from(Buffer.from(o.merklePath, "hex"));
+    }
+    if (typeof o.blockHeight === "number")
+        ev.blockHeight = o.blockHeight;
+    if (typeof o.blockHash === "string" && o.blockHash)
+        ev.blockHash = o.blockHash;
+    if (typeof o.extraInfo === "string" && o.extraInfo)
+        ev.extraInfo = o.extraInfo;
+    return ev;
+}
+/**
+ * One SSE subscription held open (#33, #58): connects, parses the stream,
+ * hands each event to `onEvent` — awaited before the next one is read and
+ * before its id counts as seen, so a caller that remembers `lastId` after it
+ * never skips an event — and reconnects with backoff, sending the last id
+ * seen as `Last-Event-ID`.
+ */
+export class SseStream {
+    url;
+    lastId;
+    abort = new AbortController();
+    stopped = false;
+    o;
+    onEvent;
+    running;
+    constructor(url, onEvent, o = {}) {
+        this.url = url;
+        this.onEvent = onEvent;
+        this.o = o;
+        this.lastId = o.lastId;
+    }
+    start() { this.running ??= this.connect(); return this; }
+    /** Close the connection; no reconnect. Resolves once the loop has ended (an event being handled finishes first). */
+    async stop() { this.stopped = true; this.abort.abort(); await this.running; }
+    say(line) { this.o.log?.(line); }
+    async connect() {
+        const { min, max } = this.o.backoff ?? { min: 500, max: 30_000 };
+        let wait = min;
+        while (!this.stopped) {
+            try {
+                const res = await (this.o.fetch ?? fetch)(this.url, {
+                    headers: { ...this.o.headers, accept: "text/event-stream", ...(this.lastId !== undefined ? { "last-event-id": this.lastId } : {}) },
+                    signal: this.abort.signal,
+                });
+                if (!res.ok || !res.body)
+                    throw new Error(`HTTP ${res.status}`);
+                this.say(`connected${this.lastId !== undefined ? ` (after event ${this.lastId})` : ""}`);
+                await this.read(res.body, () => { wait = min; });
+                if (this.stopped)
+                    return;
+                this.say(`closed; reconnecting in ${wait} ms`);
+            }
+            catch (e) {
+                if (this.stopped)
+                    return;
+                this.say(`${e.message}; reconnecting in ${wait} ms`);
+            }
+            await new Promise((r) => { const t = setTimeout(r, wait); this.abort.signal.addEventListener("abort", () => { clearTimeout(t); r(); }, { once: true }); });
+            wait = Math.min(max, wait * 2);
+        }
+    }
+    /** Parse the event stream (text/event-stream): `data:` lines until a blank line make one event; `id:` and `event:` go with it. */
+    async read(body, onData) {
+        const dec = new TextDecoder();
+        let buf = "";
+        let data = [];
+        let id;
+        let event;
+        const reader = body.getReader();
+        try {
+            for (;;) {
+                const { value, done } = await reader.read();
+                if (done)
+                    return;
+                buf += dec.decode(value, { stream: true });
+                let nl;
+                while ((nl = buf.search(/\r\n|\n|\r/)) >= 0) {
+                    const line = buf.slice(0, nl);
+                    buf = buf.slice(nl + (buf.startsWith("\r\n", nl) ? 2 : 1));
+                    if (line === "") {
+                        if (data.length) {
+                            onData();
+                            try {
+                                await this.onEvent({ data: data.join("\n"), ...(id !== undefined ? { id } : {}), ...(event !== undefined ? { event } : {}) });
+                            }
+                            catch (e) {
+                                this.say(`an event not taken: ${e.message}`);
+                            }
+                        }
+                        if (this.stopped)
+                            return;
+                        if (id !== undefined)
+                            this.lastId = id;
+                        data = [];
+                        id = undefined;
+                        event = undefined;
+                        continue;
+                    }
+                    if (line.startsWith(":"))
+                        continue;
+                    const c = line.indexOf(":");
+                    const field = c < 0 ? line : line.slice(0, c);
+                    const v = c < 0 ? "" : line.slice(c + 1).replace(/^ /, "");
+                    if (field === "data")
+                        data.push(v);
+                    else if (field === "id")
+                        id = v;
+                    else if (field === "event")
+                        event = v;
+                }
+            }
+        }
+        finally {
+            reader.releaseLock?.();
+        }
+    }
+}
+/** Headers per backfill run (#141): one `headers` frame each, parents first (#156). */
+export const BACKFILL_RUN = 2016;
+export class Feeds {
+    o;
+    sse = new Map();
+    declared = new Map();
+    hosted = new Map();
+    listeners = new Map();
+    queues = new Map();
+    stopped = false;
+    constructor(o) { this.o = o; }
+    say(source, line) { this.o.log?.(source, line); }
+    /** `handle`'s feeds are these (replacing what it declared before). */
+    declare(handle, specs) {
+        if (this.stopped)
+            return;
+        this.declared.set(handle, specs);
+        this.subscribe(handle);
+    }
+    /**
+     * The host's own feed (#102) for `handle`: subscribed to it (`spec`), or not (undefined); a genesis's feeds are kept.
+     * `backfill` (#141): on subscribing, the headers the instance lacks up to the host's tip — read after the
+     * subscription is in place; until they are queued, the live items wait behind them (held), and a live header
+     * the backfill carries too is dropped. Admitted first, in runs of BACKFILL_RUN, then the live stream.
+     */
+    host(handle, spec, backfill) {
+        if (this.stopped)
+            return Promise.resolve();
+        if (spec)
+            this.hosted.set(handle, spec);
+        else
+            this.hosted.delete(handle);
+        if (!spec || !backfill) {
+            this.subscribe(handle);
+            return Promise.resolve();
+        }
+        const q = this.queue(handle);
+        q.held = true;
+        this.subscribe(handle);
+        return backfill().then((raws) => {
+            if (!raws.length)
+                return;
+            const have = new Set(raws.map((r) => Buffer.from(r).toString("hex")));
+            q.items = q.items.map((x) => ({ ...x, raws: x.raws.filter((r) => !have.has(Buffer.from(r).toString("hex"))) })).filter((x) => x.raws.length > 0);
+            const runs = [];
+            for (let i = 0; i < raws.length; i += BACKFILL_RUN)
+                runs.push({ box: spec.box ?? DEFAULT_BOX, raws: raws.slice(i, i + BACKFILL_RUN) });
+            q.items.unshift(...runs);
+            this.say(handle, `feed: ${raws.length} header${raws.length === 1 ? "" : "s"} it lacks pushed first (the backfill), then the live stream`);
+        }).catch((e) => this.say(handle, `feed: the backfill not read: ${e.message}`)).finally(() => {
+            q.held = false;
+            if (!q.running && q.items.length)
+                void this.drain(handle, q);
+        });
+    }
+    queue(handle) {
+        let q = this.queues.get(handle);
+        if (!q)
+            this.queues.set(handle, (q = { items: [], running: false, dropped: 0 }));
+        return q;
+    }
+    /**
+     * The host's own listener on a headers feed (#132: the default image's
+     * chain part grows by it): every header the stream brings, as well as the
+     * subscribers'. The connection stays open while a listener is on it.
+     */
+    listen(url, fn) {
+        if (this.stopped)
+            return;
+        this.listeners.set(url, fn);
+        this.connect(url);
+    }
+    /** Whether `handle` is subscribed to the host's own feed. */
+    hosts(handle) { return this.hosted.has(handle); }
+    /** Subscribe `handle` to what it declared and the host's feed for it, and to nothing else. */
+    subscribe(handle) {
+        const h = this.hosted.get(handle);
+        const specs = [...(this.declared.get(handle) ?? []), ...(h ? [h] : [])];
+        for (const [url, s] of this.sse) {
+            if (s.subscribers.has(handle) && !specs.some((f) => f.kind === "headers" && f.url === url))
+                s.subscribers.delete(handle);
+            if (!s.subscribers.size && !this.listeners.has(url)) {
+                void s.stream.stop();
+                this.sse.delete(url);
+            }
+        }
+        for (const f of specs) {
+            if (f.kind !== "headers")
+                continue;
+            this.connect(f.url).subscribers.set(handle, f.box ?? DEFAULT_BOX);
+        }
+    }
+    /** The one connection to `url`, opened if it is not. */
+    connect(url) {
+        let s = this.sse.get(url);
+        if (!s) {
+            const subscribers = new Map();
+            const stream = new SseStream(url, (ev) => this.dispatch(url, subscribers, ev.data), {
+                backoff: this.o.backoff, fetch: this.o.fetch, log: (l) => this.say("router", `feed ${url}: ${l}`),
+            });
+            s = { stream, subscribers };
+            this.sse.set(url, s);
+            stream.start();
+        }
+        return s;
+    }
+    /** The declared feeds of `handle`. */
+    of(handle) { return this.declared.get(handle) ?? []; }
+    /** Close every connection (no reconnect); resolves once their loops have ended. */
+    async stop() {
+        this.stopped = true;
+        const streams = [...this.sse.values()].map((s) => s.stream.stop());
+        this.sse.clear();
+        await Promise.all(streams);
+    }
+    /** Queue a run of headers for `handle` (bounded), and drain the queue serially. */
+    push(handle, box, raws) {
+        const q = this.queue(handle);
+        q.items.push({ box, raws });
+        const max = this.o.maxQueue ?? 1000;
+        if (q.items.length > max) {
+            const n = q.items.length - max;
+            q.items.splice(0, n);
+            q.dropped += n;
+            this.say(handle, `feed: queue full (${max}): dropped the ${n} oldest (${q.dropped} so far)`);
+        }
+        if (!q.running && !q.held)
+            void this.drain(handle, q);
+    }
+    /** How many items wait for `handle` (tests). */
+    pending(handle) { return this.queues.get(handle)?.items.length ?? 0; }
+    async drain(handle, q) {
+        q.running = true;
+        try {
+            while (q.items.length && !this.stopped && !q.held) {
+                const { box, raws } = q.items.shift();
+                try {
+                    await this.o.headers(handle, box, raws);
+                }
+                catch (e) {
+                    this.say(handle, `feed: ${raws.length} header${raws.length === 1 ? "" : "s"} not taken: ${e.message}`);
+                }
+            }
+        }
+        finally {
+            q.running = false;
+        }
+    }
+    dispatch(url, subscribers, data) {
+        let dropped = false;
+        const hs = headersOf(data, (why) => { dropped = true; this.say("router", `feed ${url}: ${why}`); });
+        if (!hs.length) {
+            if (!dropped)
+                this.say("router", `feed ${url}: an event with no header in it: ignored`);
+            return;
+        }
+        this.listeners.get(url)?.(hs);
+        for (const [handle, box] of subscribers)
+            this.push(handle, box, hs);
+    }
+}

package/dist/host/frontdoor.d.ts

@@ -0,0 +1,92 @@
+import type { CID } from "multiformats/cid";
+import type { Stamp } from "../runtime/syscalls.ts";
+import { type DoorAnswer } from "./door-answer.ts";
+import type { KernelSurface as Kernel, RequestAnswer } from "./kernel.ts";
+/** A raw request as the host appends it: `path` as the client signed it, `route` what the routes table sees. */
+export interface FrontRequest {
+    method: string;
+    path: string;
+    route?: string;
+    query: string;
+    headers: Record<string, string>;
+    body: Uint8Array;
+}
+export interface FrontAnswer {
+    status: number;
+    headers: Record<string, string>;
+    body: Uint8Array;
+    /** The request's entry and the thread it launched (absent: not processed within the wait). */
+    entry?: CID;
+    thread?: CID;
+    /** Fuel of the door's filters and the answering call, when the answer was the door's or a read (the steps' fuel is on their updates). */
+    fuel?: number;
+    /**
+     * The caller (hex) of an answer the host metered: a read after the request's thread (its
+     * answer's `read.caller`), or the principal the door's filters yielded for an answer it gave
+     * without an entry. Absent when the request named none.
+     */
+    caller?: string;
+}
+/** How long a synchronous client waits on its request's thread by default (ms): SKEIN_ANSWER_WAIT_MS, else two minutes. */
+export declare const ANSWER_WAIT_MS: number;
+/** The Retry-After (s) a client that waited past the bound, or met a host shutting down, is told. */
+export declare const RETRY_AFTER_S = 5;
+/** 503 + Retry-After: the wait's bound passed (the thread goes on), or the host is shutting down. */
+export declare function unavailable(description: string, extra?: Partial<FrontAnswer>): FrontAnswer;
+/** Lower-cased header names, one value each (joined with ", "). */
+export declare function headerMap(h: Record<string, string | string[] | undefined> | Headers): Record<string, string>;
+/**
+ * Append a package as received (#68): one `request` entry naming it and its
+ * transport. The record goes with the frame, not into the store (#121): the
+ * kernel's door matches the route, runs its filters and the gate, and puts
+ * the package as it hands it back (a BEEF's bytes never stored), so the entry
+ * written — its CID, returned — may name another record than this. A package
+ * the door turns away or answers writes nothing (#143): DoorAnswered.
+ */
+export declare function appendRequest(k: Kernel, transport: string, record: Record<string, unknown>, time?: Stamp): Promise<CID>;
+/** An HTTP request's record (log.zig: {kind: "http", method, path, route, query, headers, body}). */
+export declare function httpRecord(req: FrontRequest): Record<string, unknown>;
+/**
+ * One HTTP request through the instance: appended, then waited on (#66)
+ * until its thread comes to rest or `waitMs` passes (503 + Retry-After).
+ * `stop`, when it settles first, answers 503 too (the host shutting down).
+ */
+export declare function frontDoor(k: Kernel, req: FrontRequest, o?: {
+    now?: Stamp;
+    waitMs?: number;
+    stop?: Promise<unknown>;
+}): Promise<FrontAnswer>;
+/**
+ * A request thread's answer as the HTTP response: its last step's, a read after it, or 500/503.
+ * `req`, the request as received: a door refusal of a signed one is answered by the front door.
+ */
+export declare function httpAnswer(k: Kernel, entry: CID, a: RequestAnswer, req?: FrontRequest): Promise<FrontAnswer>;
+/**
+ * The door's answer without an entry (#143: a filter's rejection or answer — a read route's — or the
+ * gate's refusal), as the HTTP response: a signed request gets it signed on its session — a
+ * front-door call (fn "respond") with the request as received, at the request's time — and an
+ * unsigned one, or one whose session does not verify (unknown, expired: the stock client shakes
+ * hands again), gets it plain.
+ */
+export declare function doorAnswer(k: Kernel, d: DoorAnswer, req: FrontRequest, now?: Stamp): Promise<FrontAnswer>;
+/** The BRC-104 handshake's path: a route of its own (the front door's fn "handshake"). */
+export declare const HANDSHAKE_PATH = "/.well-known/auth";
+/** Whether a request carries BRC-104 headers (x-bsv-auth-*): a signed request. */
+export declare const isSigned: (headers: Record<string, string>) => boolean;
+/**
+ * One HTTP request through the kernel's door (#143): its route's filters decide — admitted and
+ * waited on, or answered at once with nothing logged (a read route; a rejection; the gate's refusal).
+ */
+export declare function serveHttp(k: Kernel, req: FrontRequest, o?: {
+    now?: Stamp;
+    waitMs?: number;
+    stop?: Promise<unknown>;
+}): Promise<FrontAnswer>;
+/** A fetch over one instance's door (serveHttp), no socket: for a client in a test, or a URL of the host's own. */
+export declare function frontDoorFetch(k: Kernel, o?: {
+    now?: () => Stamp;
+    route?: (path: string) => string;
+    onAnswer?: (a: FrontAnswer) => void;
+    waitMs?: number;
+}): typeof fetch;
+export declare const cidOf: (v: unknown) => CID;