@dcasegr/skein 0.0.0-stage → 0.1.1

package/dist/host/stores.js

@@ -0,0 +1,342 @@
+// The host's opener of instance stores (#96, #133): a SQLite file through
+// src/runtime/index-store.ts, or a set of the host's LMDB environment,
+// `lmdb:<env dir>#<set>` (index-store.ts `storeSpec`), read here directly,
+// read only, through lmdb-js — with the kernel's member gate
+// (kernel-zig/src/lmdb_store.zig): a block is there if the set holds it, or
+// the chain's set does. The disk is the host's (src/runtime reads none).
+//
+// A set of the one-process kernel's RocksDB database, `rocksdb:<db dir>#<set>`
+// (#158), is read through that process — RocksDB lets no second process see
+// the live database — over its host-read socket (`<db dir>/kernel.sock`,
+// kernel-zig/src/host.zig): `get`/`has` by CID through the set's member gate
+// (the frame names the `set`: a block only another instance holds is absent,
+// as for an LMDB set), the set's `state` pointer, and `members` (the CIDs it
+// holds: a checkpoint's extras), the frames its stdin takes. The index reader here is
+// synchronous, so the socket is spoken from a worker thread the caller waits
+// on (Atomics.wait, the answer taken with receiveMessageOnPort). With no
+// kernel process running there, a read is refused: the host must run.
+import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { MessageChannel, receiveMessageOnPort, Worker } from "node:worker_threads";
+import { decode, encode, fromBytes } from "../runtime/cid.js";
+import { CHAIN_SET, indexStoreOver, openStoreFile as openFile, parseLmdbSpec, parseRocksSpec, parseSqliteSpec } from "../runtime/index-store.js";
+import { openHostDb, setHasState } from "./store-sqlite.js";
+import { KernelHost } from "./kernel.js";
+/** A store, whichever backend: a file (index-store.ts openStoreFile), or an LMDB set, read only. */
+export function openStoreFile(spec, o = {}) {
+    const q = parseSqliteSpec(spec);
+    if (q) {
+        if (o.readOnly === false)
+            throw new Error(`${spec}: a SQLite set is read only here (the kernel writes it)`);
+        return openSqliteSetStore(q.db, q.set);
+    }
+    const r = parseRocksSpec(spec);
+    if (r) {
+        if (o.readOnly === false)
+            throw new Error(`${spec}: a RocksDB set is read only here (the kernel writes it; the host never does)`);
+        return openRocksStore(r.dir, r.set);
+    }
+    const l = parseLmdbSpec(spec);
+    if (!l)
+        return openFile(spec, o);
+    if (o.readOnly === false)
+        throw new Error(`${spec}: an LMDB set is read only here (the kernel writes it; the host never does)`);
+    return openLmdbStore(l.dir, l.set);
+}
+/** Whether a store is there to read: the file; for an LMDB set, its state pointer (the kernel has committed). */
+export function storeExists(spec) {
+    const q = parseSqliteSpec(spec);
+    if (q) {
+        if (!existsSync(q.db))
+            return false;
+        const db = new DatabaseSync(q.db, { readOnly: true });
+        try {
+            return setHasState(db, q.set);
+        }
+        finally {
+            db.close();
+        }
+    }
+    const r = parseRocksSpec(spec);
+    if (r)
+        return rocksReader(r.dir).request({ op: "state", h: r.set }) != null;
+    const l = parseLmdbSpec(spec);
+    if (!l)
+        return existsSync(spec);
+    if (!existsSync(join(l.dir, "data.mdb")))
+        return false;
+    const e = lmdbEnv(l.dir);
+    e.pointers.resetReadTxn();
+    return e.pointers.doesExist(Buffer.concat([setPrefix(l.set), Buffer.from("state")]));
+}
+const envs = new Map();
+/**
+ * The environment at `dir`, opened read only once per process (LMDB: one open per process) and
+ * kept. lmdb-js must link LMDB 0.9.x (its data-v1 build: `LMDB_DATA_V1=true npm rebuild lmdb
+ * --build-from-source`, kernel-zig/README.md): its prebuilt binaries link LMDB 0.9.90 (data format
+ * 2, LMDB's 1.0 line), which cannot open the kernel's 0.9.36 environment.
+ */
+function lmdbEnv(dir) {
+    const had = envs.get(dir);
+    if (had)
+        return had;
+    const lmdb = createRequire(import.meta.url)("lmdb");
+    const v = lmdb.version;
+    if (v.major !== 0 || v.minor !== 9 || v.patch >= 90)
+        throw new Error(`lmdb-js links LMDB ${v.major}.${v.minor}.${v.patch}, not 0.9.x: it cannot read the kernel's environment (rebuild it: LMDB_DATA_V1=true npm rebuild lmdb --build-from-source)`);
+    const env = lmdb.open({ path: dir, readOnly: true, noSubdir: false, maxDbs: 4, maxReaders: 4096, mapSize: 2 ** 40 });
+    const db = (name) => env.openDB({ name, encoding: "binary", keyEncoding: "binary" });
+    const e = { blocks: db("blocks"), members: db("members"), pointers: db("pointers") };
+    envs.set(dir, e);
+    return e;
+}
+const setPrefix = (set) => Buffer.concat([Buffer.from(set), Buffer.from([0])]);
+const prefixEnd = (p) => { const b = Buffer.from(p); b[b.length - 1]++; return b; };
+/**
+ * A set of an LMDB environment through the index (#96), read only: the same reader as a SQLite
+ * file's, with the kernel's gate — a block is there if the set holds it, or the chain's set does
+ * (#133); one only another set holds is absent. Each state read takes a fresh snapshot.
+ */
+export function openLmdbStore(dir, set) {
+    const e = lmdbEnv(dir);
+    const own = setPrefix(set), chain = setPrefix(CHAIN_SET);
+    const member = (c) => e.members.doesExist(Buffer.concat([own, c])) || e.members.doesExist(Buffer.concat([chain, c]));
+    return indexStoreOver({
+        get: (c) => (member(c) ? e.blocks.get(c) : undefined),
+        has: (c) => member(c) && e.blocks.doesExist(c),
+        pointer: () => { e.pointers.resetReadTxn(); return e.pointers.get(Buffer.concat([own, Buffer.from("state")])); },
+        close: () => { },
+    });
+}
+/**
+ * A set of the host's SQLite database (the settled split), read only, through the index: the same
+ * reader as a file's, with the gate the wasm host's store keeps (store-sqlite.ts) — a block is there
+ * if the set holds it, or the `host/chain` set does. Its own read-only connection, closed with it.
+ */
+export function openSqliteSetStore(path, set) {
+    const db = openHostDb(path, { readOnly: true });
+    const get = db.prepare(`SELECT b.bytes FROM members m JOIN blocks b ON b.cid = m.cid WHERE m.cid = ?3 AND (m."set" = ?1 OR m."set" = ?2) LIMIT 1`);
+    const has = db.prepare(`SELECT 1 FROM members m WHERE m.cid = ?3 AND (m."set" = ?1 OR m."set" = ?2) LIMIT 1`);
+    const ptr = db.prepare(`SELECT cid FROM pointers WHERE "set" = ? AND name = 'state'`);
+    return indexStoreOver({
+        get: (c) => { const r = get.get(set, CHAIN_SET, c); return r ? new Uint8Array(r.bytes) : undefined; },
+        has: (c) => has.get(set, CHAIN_SET, c) !== undefined,
+        pointer: () => { const r = ptr.get(set); return r ? new Uint8Array(r.cid) : undefined; },
+        close: () => db.close(),
+    });
+}
+/**
+ * The host chain a store sees (#156): a RocksDB database's (the kernel process's `chain` frame over
+ * its host-read socket), or a SQLite file's or an LMDB set's own (a serve instance's: the pointer
+ * `host/chain` and the record it names, read directly).
+ */
+export function readHostChain(spec) {
+    const q = parseSqliteSpec(spec);
+    if (q) {
+        // The host's database: the `host/chain` set's pointer `host/chain` (host_chain.zig), and the record it names.
+        if (!existsSync(q.db))
+            return undefined;
+        const db = new DatabaseSync(q.db, { readOnly: true });
+        try {
+            const p = db.prepare(`SELECT cid FROM pointers WHERE "set" = ? AND name = 'host/chain'`).get(CHAIN_SET);
+            if (!p)
+                return undefined;
+            const b = db.prepare("SELECT bytes FROM blocks WHERE cid = ?").get(p.cid);
+            if (!b)
+                return undefined;
+            const v = decode(new Uint8Array(b.bytes));
+            return v.tip ? { root: fromBytes(new Uint8Array(p.cid)), height: Number(v.tip.height), hash: v.tip.hash } : undefined;
+        }
+        catch {
+            return undefined;
+        }
+        finally {
+            db.close();
+        }
+    }
+    const r = parseRocksSpec(spec);
+    if (r) {
+        const v = rocksReader(r.dir).request({ op: "chain" });
+        return v ? { root: v.root, height: Number(v.height), hash: v.hash } : undefined;
+    }
+    let ptr;
+    let get;
+    const l = parseLmdbSpec(spec);
+    if (l) {
+        if (!existsSync(join(l.dir, "data.mdb")))
+            return undefined;
+        const e = lmdbEnv(l.dir);
+        e.pointers.resetReadTxn();
+        ptr = e.pointers.get(Buffer.concat([setPrefix(l.set), Buffer.from("host/chain")]));
+        get = (c) => e.blocks.get(c);
+    }
+    else {
+        if (!existsSync(spec))
+            return undefined;
+        const db = new DatabaseSync(spec, { readOnly: true });
+        try {
+            const t = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'pointers'").get();
+            ptr = t ? db.prepare("SELECT cid FROM pointers WHERE name = 'host/chain'").get()?.cid : undefined;
+            if (!ptr)
+                return undefined;
+            const b = db.prepare("SELECT bytes FROM blocks WHERE cid = ?").get(ptr)?.bytes;
+            get = () => b;
+        }
+        finally {
+            db.close();
+        }
+    }
+    if (!ptr)
+        return undefined;
+    const bytes = get(ptr);
+    if (!bytes)
+        return undefined;
+    const v = decode(bytes);
+    if (!v.tip)
+        return undefined;
+    return { root: fromBytes(ptr), height: Number(v.tip.height), hash: v.tip.hash };
+}
+/** Every block a store holds (a SQLite file's `blocks`; an LMDB set's members), for a checkpoint's extras. */
+export function* storeBlocks(spec) {
+    const q = parseSqliteSpec(spec);
+    if (q) {
+        const db = new DatabaseSync(q.db, { readOnly: true });
+        try {
+            for (const r of db.prepare(`SELECT m.cid, b.bytes FROM members m JOIN blocks b ON b.cid = m.cid WHERE m."set" = ?`).all(q.set))
+                yield { cid: fromBytes(new Uint8Array(r.cid)), bytes: new Uint8Array(r.bytes) };
+        }
+        finally {
+            db.close();
+        }
+        return;
+    }
+    const r = parseRocksSpec(spec);
+    if (r) {
+        // #158: the set's members, listed by the kernel; each block read through it.
+        const rd = rocksReader(r.dir);
+        for (const c of rd.request({ op: "members", set: r.set })) {
+            const bytes = rd.request({ op: "get", set: r.set, v: c });
+            if (bytes instanceof Uint8Array)
+                yield { cid: c, bytes };
+        }
+        return;
+    }
+    const l = parseLmdbSpec(spec);
+    if (l) {
+        const e = lmdbEnv(l.dir);
+        e.members.resetReadTxn();
+        const own = setPrefix(l.set);
+        for (const k of e.members.getKeys({ start: own, end: prefixEnd(own) })) {
+            const c = new Uint8Array(k.subarray(own.length));
+            const bytes = e.blocks.get(c);
+            if (bytes)
+                yield { cid: fromBytes(c), bytes: new Uint8Array(bytes) };
+        }
+        return;
+    }
+    const db = new DatabaseSync(spec, { readOnly: true });
+    try {
+        for (const r of db.prepare("SELECT cid, bytes FROM blocks").all())
+            yield { cid: fromBytes(new Uint8Array(r.cid)), bytes: new Uint8Array(r.bytes) };
+    }
+    finally {
+        db.close();
+    }
+}
+// ---------------------------------------------------------------- RocksDB, through the kernel (#158)
+/** The worker thread: speaks the host-read socket for a waiting caller (one request at a time). */
+const READER_WORKER = `
+const { workerData } = require("node:worker_threads");
+const net = require("node:net");
+const { sock, flag, port } = workerData;
+const i32 = new Int32Array(flag);
+let conn, buf = Buffer.alloc(0), pending = false;
+const done = (m) => { port.postMessage(m); Atomics.store(i32, 0, 1); Atomics.notify(i32, 0); };
+function connect(until) {
+  return new Promise((resolve, reject) => {
+    const c = net.createConnection(sock);
+    c.once("connect", () => {
+      c.removeAllListeners("error");
+      c.on("error", () => {});
+      c.on("close", () => { if (conn === c) conn = undefined; if (pending) { pending = false; done({ error: "the kernel's socket closed" }); } });
+      c.on("data", onData);
+      resolve(c);
+    });
+    c.once("error", (e) => { if (Date.now() < until) setTimeout(() => connect(until).then(resolve, reject), 50); else reject(e); });
+  });
+}
+function onData(d) {
+  buf = Buffer.concat([buf, d]);
+  while (buf.length >= 4) {
+    const n = buf.readUInt32BE(0);
+    if (buf.length < 4 + n) return;
+    const f = new Uint8Array(buf.subarray(4, 4 + n));
+    buf = buf.subarray(4 + n);
+    if (pending) { pending = false; done({ frame: f }); }
+  }
+}
+port.on("message", async ({ bytes, waitMs }) => {
+  try {
+    if (!conn) { buf = Buffer.alloc(0); conn = await connect(Date.now() + waitMs); }
+    pending = true;
+    const h = Buffer.alloc(4); h.writeUInt32BE(bytes.length);
+    conn.write(Buffer.concat([h, Buffer.from(bytes)]));
+  } catch (e) { done({ error: (e && e.code) || String(e) }); }
+});
+`;
+class RocksReader {
+    flag = new Int32Array(new SharedArrayBuffer(4));
+    port;
+    worker;
+    nextId = 1;
+    dir;
+    constructor(dir) {
+        this.dir = dir;
+        const { port1, port2 } = new MessageChannel();
+        this.port = port1;
+        this.worker = new Worker(READER_WORKER, { eval: true, workerData: { sock: join(dir, "kernel.sock"), flag: this.flag.buffer, port: port2 }, transferList: [port2] });
+        this.worker.unref();
+        this.port.unref();
+    }
+    /** One frame to the kernel and its `ok`, synchronously. */
+    request(f) {
+        // The process this node process runs may still be starting: wait for its socket a while; else not at all.
+        const waitMs = KernelHost.running(this.dir) ? 15_000 : 0;
+        Atomics.store(this.flag, 0, 0);
+        this.port.postMessage({ bytes: encode({ id: this.nextId++, ...f }).bytes, waitMs });
+        if (Atomics.wait(this.flag, 0, 0, waitMs + 30_000) === "timed-out")
+            throw new Error(`${this.dir}: the kernel's host-read socket did not answer`);
+        const m = receiveMessageOnPort(this.port)?.message;
+        if (!m || m.error !== undefined)
+            throw new Error(`${this.dir}: a RocksDB set is read through the host's kernel process (skein-kernel host), and none answers at ${join(this.dir, "kernel.sock")} (${m?.error ?? "no answer"}): start the host (skein-host run)`);
+        const a = decode(m.frame);
+        if (typeof a.error === "string")
+            throw new Error(`${this.dir}: ${a.error}`);
+        return a.ok;
+    }
+}
+const readers = new Map();
+function rocksReader(dir) {
+    let r = readers.get(dir);
+    if (!r) {
+        r = new RocksReader(dir);
+        readers.set(dir, r);
+    }
+    return r;
+}
+/**
+ * A set of the one-process kernel's RocksDB database (#158), read only, through the kernel: blocks by
+ * CID through the set's gate (what the instance holds), the state pointer read on every state read.
+ */
+export function openRocksStore(dir, set) {
+    const r = rocksReader(dir);
+    const asBytes = (v) => (v instanceof Uint8Array ? v : undefined);
+    return indexStoreOver({
+        get: (c) => asBytes(r.request({ op: "get", set, v: fromBytes(c) })),
+        has: (c) => r.request({ op: "has", set, v: fromBytes(c) }) === true,
+        pointer: () => { const p = r.request({ op: "state", h: set }); return p ? p.bytes : undefined; },
+        close: () => { },
+    });
+}

package/dist/host/sync.d.ts

@@ -0,0 +1,121 @@
+import { type InboundAnswer } from "./p2p.ts";
+import type { LiveBeat } from "./liveness.ts";
+export declare const SYNC_PROTOCOL = "/skein/overlay/sync/1.0.0";
+/** The overlay's receive row (skein-overlay etc/app.json): the host's own entry, one Atomic BEEF per frame; never served to peers. */
+export declare const BEEF_PROTOCOL = "/skein/overlay/beef/1.0.0";
+export interface SyncOptions {
+    /** The app that beats on `<topic>-live` for `handle` (its overlay), if any. */
+    overlayOf(handle: string, liveTopic: string): string | undefined;
+    /**
+     * Our own overlay's read (shruggr/skein#154): the engine's function `fn` (`anchors`, `admitted`,
+     * `holds`) over `handle`'s current state, with its read route's query; its JSON answer. Throws on
+     * a status other than 200.
+     */
+    local(handle: string, app: string, fn: "anchors" | "admitted" | "holds", query: Record<string, string>): Promise<unknown>;
+    /** One BEEF into the overlay's receive row, as from `peer` (its peer ID). */
+    deliver(handle: string, peer: string, beef: Uint8Array): Promise<InboundAnswer>;
+    /** The chain state's header at `height`: its hash and merkle root (display hex). */
+    header(handle: string, height: number): Promise<{
+        hash: string;
+        root: string;
+    } | undefined>;
+    /** One request frame to `peer` on the sync protocol: its answer frame. */
+    request(handle: string, peer: string, body: Uint8Array): Promise<Uint8Array>;
+    /** The live beats on a liveness topic at `handle` (the liveness tool's: the latest per sender). */
+    beats(handle: string, liveTopic: string): LiveBeat[];
+    /** Our own beacon's body on a liveness topic (what our overlay declared last), if any. */
+    ownBeat(handle: string, liveTopic: string): Uint8Array | undefined;
+    /** `handle`'s own peer ID. */
+    self(handle: string): string;
+    log?(source: string, line: string): void;
+    /** The heights per ranged pull; default 1 000. */
+    rangeHeights?: number;
+}
+/** A peer ID's string from its multihash bytes. */
+export declare const peerOf: (mh: Uint8Array) => string;
+/**
+ * An `admit` announcement on `<topic>` (skein-overlay gossip.zig: dag-cbor `{kind: "admit", txid,
+ * topics: {<topic>: {outputsToAdmit, coinsToRetain}}}`) → its txid, or undefined when the body is
+ * not one naming `topic`.
+ */
+export declare function admitOf(topic: string, body: Uint8Array): string | undefined;
+export declare class OverlaySync {
+    private readonly o;
+    /** `${handle} ${txid}` → the announced transaction, while it is not taken. */
+    private announced;
+    /** Reconciles running: `${handle} ${topic} ${peer}`. */
+    private running;
+    /** The beat body (hex) that started, or ended, the last reconcile per `${handle} ${topic} ${peer}`. */
+    private seen;
+    /** Peers caught serving a list that does not check: per handle. */
+    private dropped;
+    private inflight;
+    private stopped;
+    constructor(o: SyncOptions);
+    private say;
+    private track;
+    /** Whether anything is under way (a reconcile, a fetch). */
+    busy(): boolean;
+    idle(): Promise<void>;
+    stop(): void;
+    /** The announced transactions not taken yet, per handle (tests). */
+    pending(handle: string): Array<{
+        txid: string;
+        peers: string[];
+        fetching: boolean;
+    }>;
+    private isDropped;
+    /**
+     * A peer's `admit` for `txid` on `topic` (its announcement), at `handle` whose overlay `app` runs
+     * the topic: fetched from that peer unless the instance holds it or a fetch of it is under way;
+     * the peer joins the transaction's set either way.
+     */
+    admit(handle: string, app: string, topic: string, txid: string, peer: string): void;
+    /** `admit`'s work: the fetch it started, if any (the tail awaits it). */
+    private announce;
+    /** Whether `handle`'s overlay state reaches `txid` (its engine's `holds`). */
+    private holds;
+    /** Fetch an announced transaction from its peers, one after another, until one delivery is taken or none is left. */
+    private pursue;
+    /** A beat newly kept on `liveTopic` from another peer: a reconcile from it when its body changed since the last. */
+    beat(handle: string, liveTopic: string, b: LiveBeat): void;
+    /**
+     * A start (router.ts, the persisted peers): each peer that beat on the topic before asked at once,
+     * the locator and the tail, without a beat. One reconcile per peer, as for a beat.
+     */
+    reconnect(handle: string, app: string, topic: string, peer: string): void;
+    private start;
+    private anchors;
+    private ask;
+    /** The beat body's sync fields, or undefined when it does not decode. */
+    private static fields;
+    /**
+     * One reconcile with `peer` on `topic`: its mined history (the locator and the pulls) when its
+     * tip anchor is not ours, then its unmined tail when the mined side is equal and its tail digest
+     * is not ours. No beat (a start): both asked.
+     */
+    private reconcile;
+    /** The peer's unmined tail: each txid we do not hold fetched as an announced one from that peer. */
+    private tail;
+    /**
+     * Catch `topic` up from `peer`: the locator, then the pulls. `tip`: the tip its beat named (the
+     * ranges are spread across the peers whose last beat names the same); none (a start): that peer
+     * alone. → whether every range was taken (a height we hold no header for, or a list that fails,
+     * leaves it unfinished).
+     */
+    private catchUp;
+    /** The peer's admitted lists for `from..to`, every page. */
+    private pullRange;
+    /**
+     * Each block against our headers before any transaction is fetched: its hash ours at its height,
+     * the BUMP at that height and every listed txid in it with our merkle root. A failure drops the
+     * peer; a height we hold no header for yet stops the range (the next changed beat resumes it).
+     * → whether all checked.
+     */
+    private checked;
+    private problem;
+    /** One transaction asked of `peer` and handed to the receive row as from it. → whether it was accepted. */
+    private fetchInto;
+    /** Until the topic lists `txids` at `height` (admitted: each delivery is judged on its own thread), or 10 s. */
+    private landed;
+}