@serfab/cadre-host 1.13.0 → 1.15.0

package/dist/hosted/hosted-node-supervisor.js

@@ -0,0 +1,396 @@
+import debug from 'debug';
+import { errorMessage } from './node-status.js';
+const log = debug('cadre:host:hosted-node-supervisor');
+/**
+ * Unit of the doubling respawn backoff: the wait after `n` consecutive failed
+ * attempts is `base * 2^n`, capped at {@link HOSTED_NODE_RESPAWN_BACKOFF_MAX_MS}.
+ * So the first retry waits 10s (one attempt is already recorded by then), the
+ * second 20s, and so on. A node with no recorded attempt is respawned
+ * immediately.
+ */
+export const HOSTED_NODE_RESPAWN_BACKOFF_BASE_MS = 5000;
+/**
+ * Ceiling on the doubling respawn backoff. 5 minutes.
+ *
+ * NOTE: not reachable at the current {@link HOSTED_NODE_RESPAWN_MAX_ATTEMPTS} of 5
+ * — the longest wait actually served is the one before the 5th attempt,
+ * `5s * 2^4` = 80s, and once that attempt fails or its node dies the supervisor
+ * gives up rather than waiting again. The cap only starts clamping
+ * once a record can reach 6 recorded attempts (`5s * 2^6` = 320s), so raising
+ * the attempt cap without revisiting this one changes nothing.
+ */
+export const HOSTED_NODE_RESPAWN_BACKOFF_MAX_MS = 5 * 60000;
+/**
+ * Consecutive respawn attempts after which the host stops trying and marks the
+ * node `error` — a node that will not stay up is a host-side fault its cadre
+ * cannot see, so surface it instead of spinning forever.
+ *
+ * Every attempt counts, whether its spawn failed or succeeded: a node that spawns
+ * and then dies before {@link HOSTED_NODE_RESPAWN_HEALTHY_MS} refills the budget is
+ * as much a crash loop as one that never spawns. Give-up happens on whichever
+ * comes first — an attempt that throws with the cap reached, or a pass that finds
+ * the node down with the cap already reached.
+ */
+export const HOSTED_NODE_RESPAWN_MAX_ATTEMPTS = 5;
+/**
+ * How long a respawned node must stay up before its attempt budget is refilled.
+ * The last respawn produced something that survived, so the next crash starts
+ * from a fresh budget rather than inheriting an old crash loop's count.
+ */
+export const HOSTED_NODE_RESPAWN_HEALTHY_MS = 10 * 60000;
+/** How often the supervisor sweep runs while cadre-host is up. 1 minute. */
+export const HOSTED_NODE_RESPAWN_SWEEP_MS = 60000;
+/**
+ * Statuses the supervisor considers. `spawning` is a join still in flight (its
+ * own code path owns the child); `error` is a node the host gave up on, or whose
+ * invitation failed, which comes back only by an explicit Retry or reset.
+ */
+const SUPERVISED_STATUSES = new Set([
+    'unclaimed',
+    'joining',
+    'joined',
+]);
+/**
+ * HostedNodeSupervisor — owns the invariant *an `unclaimed`, `joining` or `joined`
+ * node is expected to be running*.
+ *
+ * Nothing else re-spawns a hosted node: `/api/nodes` is read-only. So without
+ * this class a crashed, OOM-killed, or reboot-killed node stays dead with its
+ * record still reading `joined`, silently costing its cadre a node.
+ *
+ * One code path ({@link reconcile}) behind three triggers:
+ *
+ * - **host startup** — one pass after `orchestrator.init()`, which catches every
+ *   node that died while the host was down;
+ * - **exit signal** — the orchestrator's `onStateChange`, so a crash is noticed
+ *   in milliseconds rather than at the next sweep;
+ * - **periodic sweep** — the backstop for deaths no exit event covers (host was
+ *   down, listener missed, orchestrator re-attached to an already-dead pid).
+ *
+ * Passes are serialized: `HostedNodeService.respawn` is not itself serialized, and
+ * two overlapping respawns of one id both spawn a child while the second drops
+ * the first's orchestrator handle (see its docstring). Every pass, and a deliberate
+ * {@link restart}, runs on the service's respawn queue
+ * (`HostedNodeService.serializeRespawns`), which the service's Retry shares.
+ *
+ * The supervisor reconciles records to handles, never handles to records: a
+ * handle no record names (`hosted-nodes.json` lost, or a handle an older build
+ * persisted) is logged at startup and left running, since stopping it could end
+ * a node that is serving its cadre. `cadre-host node remove <id>` reclaims it.
+ */
+export class HostedNodeSupervisor {
+    constructor(opts) {
+        /** At most one exit-triggered pass queued at a time (crash storms coalesce). */
+        this.exitPassQueued = false;
+        /** Set by `stop()` so an in-flight pass abandons its remaining records. */
+        this.disposed = false;
+        this.service = opts.service;
+        this.store = opts.store;
+        this.orchestrator = opts.orchestrator;
+        this.now = opts.now ?? (() => new Date());
+        log('HostedNodeSupervisor initialized');
+    }
+    /**
+     * One reconcile pass over the store. Serialized against every other pass, so
+     * a caller may get a queued pass rather than an immediate one. Returns the ids
+     * it respawned in *its own* pass.
+     */
+    reconcile() {
+        return this.service.serializeRespawns(() => this.reconcileOnce());
+    }
+    /**
+     * Stop a running node and start it again, for a change the node picks up
+     * only at start — its public addresses (`NatService.onNodeAddressesStale`).
+     * Serialized with the reconcile passes, so it never overlaps a respawn or a
+     * give-up of the same record, and it re-reads the record first: a record that
+     * is no longer supervised, has no handle, or whose node is not
+     * running is left alone (a node that is down is the crash path's).
+     *
+     * Does not spend the record's respawn budget. A restart whose respawn throws
+     * rejects; the node is then down, and the exit event its stop raised has queued
+     * a reconcile pass that takes it over through the crash path, with its counting.
+     */
+    restart(id) {
+        return this.service.serializeRespawns(() => this.restartOnce(id));
+    }
+    /**
+     * Subscribe to child exits, start the periodic sweep, and run the startup
+     * pass. Idempotent — a second call while started is a no-op.
+     */
+    start() {
+        if (this.timer)
+            return;
+        this.disposed = false;
+        // Subscribe before the first pass so an exit during it is not missed.
+        this.unsubscribe = this.orchestrator.onStateChange((info) => { this.onNodeState(info); });
+        this.timer = setInterval(() => { this.sweep('timer'); }, HOSTED_NODE_RESPAWN_SWEEP_MS);
+        this.timer.unref();
+        this.logOrphanHandles();
+        this.sweep('startup');
+    }
+    /**
+     * Stop the timer and unsubscribe. A pass already in flight abandons its
+     * remaining records rather than being awaited — every record it has not
+     * reached is picked up by the next host start's startup pass.
+     */
+    stop() {
+        this.disposed = true;
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        this.unsubscribe?.();
+        this.unsubscribe = undefined;
+        log('HostedNodeSupervisor stopped');
+    }
+    /**
+     * Name every orchestrator handle that no record claims, once at startup. Not
+     * reclaimed (see the class docstring): the admin ends it with
+     * `cadre-host node remove <id>`.
+     */
+    logOrphanHandles() {
+        let known;
+        try {
+            known = new Set(this.store.list().map((n) => n.id));
+        }
+        catch (err) {
+            log('orphan check could not list hosted nodes: %s', errorMessage(err));
+            return;
+        }
+        for (const handle of this.orchestrator.listNodes()) {
+            if (known.has(handle.id))
+                continue;
+            console.error(`hosted node ${handle.id} (${handle.status}) has an orchestrator handle but no record in hosted-nodes.json; `
+                + 'it is left as it is. Run `cadre-host node remove ' + handle.id + '` to stop it and delete its data.');
+        }
+    }
+    /** Fire-and-forget pass for the timer / exit / startup triggers. */
+    sweep(trigger) {
+        void this.reconcile()
+            .then((ids) => {
+            if (ids.length)
+                log('%s sweep respawned %d hosted node(s): %o', trigger, ids.length, ids);
+        })
+            .catch((err) => {
+            console.error(`hosted node respawn sweep failed: ${errorMessage(err)}`);
+        });
+    }
+    /**
+     * Exit-event trigger. Deliberately cheap: it does not map the handle back to a
+     * record (that needs a store read per event), it just asks for a pass.
+     */
+    onNodeState(info) {
+        if (info.status !== 'stopped')
+            return;
+        if (this.exitPassQueued)
+            return;
+        this.exitPassQueued = true;
+        void this.reconcile()
+            .catch((err) => { console.error(`hosted node respawn sweep failed: ${errorMessage(err)}`); })
+            .finally(() => { this.exitPassQueued = false; });
+    }
+    /** The unserialized body of one pass. */
+    async reconcileOnce() {
+        const respawned = [];
+        let candidates;
+        try {
+            candidates = this.store.list().filter((n) => SUPERVISED_STATUSES.has(n.status));
+        }
+        catch (err) {
+            // A malformed hosted-nodes.json throws on every load; log rather than let an
+            // unhandled rejection escape the timer.
+            log('reconcile could not list hosted nodes: %s', errorMessage(err));
+            return respawned;
+        }
+        for (const node of candidates) {
+            if (this.disposed)
+                break;
+            try {
+                const id = await this.reconcileOne(node);
+                if (id)
+                    respawned.push(id);
+            }
+            catch (err) {
+                // One bad record must never end the sweep — the others are still down.
+                log('reconcile of hosted node %s failed: %s', node.id, errorMessage(err));
+            }
+        }
+        return respawned;
+    }
+    async restartOnce(id) {
+        if (this.disposed)
+            return;
+        const node = this.store.get(id);
+        if (!node?.dockerId || !SUPERVISED_STATUSES.has(node.status))
+            return;
+        if (!(await this.orchestrator.isRunning(node.dockerId)))
+            return;
+        await this.orchestrator.stopContainer(node.dockerId);
+        const result = await this.service.respawn(id, { countAttempt: false });
+        log('restarted hosted node %s: %s', id, result.outcome);
+    }
+    /** Reconcile one record. Returns its id when this pass respawned it. */
+    async reconcileOne(node) {
+        // No handle means nothing was ever spawned — `join` owns that record until
+        // it writes one, and replaying it here would race that join.
+        if (!node.dockerId)
+            return undefined;
+        if (await this.orchestrator.isRunning(node.dockerId)) {
+            this.refillBudgetIfHealthy(node);
+            return undefined;
+        }
+        // The budget is spent, and the last attempt did not stay up long enough to
+        // refill it. Checked here and not only when `respawn` throws: a spawn that
+        // succeeds and then dies at once (a crash on boot, a port clash) never
+        // throws, so a catch-only check would respawn it forever.
+        // NOTE: a record at the cap whose last respawn was still up but not yet
+        // refilled when the host went down is given up on host restart rather than
+        // tried once more; if that is ever seen, skip this check when the host
+        // started after `respawn.lastAttemptAt`.
+        const attempts = node.respawn?.attempts ?? 0;
+        if (attempts >= HOSTED_NODE_RESPAWN_MAX_ATTEMPTS) {
+            await this.giveUp(node.id, attempts, 'the node did not stay running after its last respawn');
+            return undefined;
+        }
+        if (!this.backoffElapsed(node)) {
+            log('hosted node %s is down but still inside its respawn backoff', node.id);
+            return undefined;
+        }
+        return this.attemptRespawn(node);
+    }
+    /**
+     * Whether enough time has passed since the last attempt. A record with no
+     * attempt history (or an unparsable timestamp — never wedge a node on bad
+     * data) is eligible immediately.
+     */
+    backoffElapsed(node) {
+        const respawn = node.respawn;
+        if (!respawn)
+            return true;
+        const last = Date.parse(respawn.lastAttemptAt);
+        if (Number.isNaN(last))
+            return true;
+        const delay = Math.min(HOSTED_NODE_RESPAWN_BACKOFF_BASE_MS * 2 ** respawn.attempts, HOSTED_NODE_RESPAWN_BACKOFF_MAX_MS);
+        return this.now().getTime() - last >= delay;
+    }
+    /**
+     * Clear the attempt count once a respawned node has been up long enough to
+     * count as healthy, so the next crash gets a full budget instead of inheriting
+     * the previous crash loop's.
+     */
+    refillBudgetIfHealthy(node) {
+        if (!this.budgetRefillDue(node))
+            return;
+        // Re-read before writing: `node` comes from the snapshot this pass started
+        // from, which can be several awaits old, and `store.put` replaces the whole
+        // row — writing the stale copy back would undo a claim or a removal that
+        // landed mid-pass.
+        const current = this.store.get(node.id);
+        if (!current?.respawn || !this.budgetRefillDue(current))
+            return;
+        try {
+            this.store.put({ ...current, respawn: { ...current.respawn, attempts: 0 } });
+            log('hosted node %s is healthy again — respawn budget refilled', node.id);
+        }
+        catch (err) {
+            // Costs one attempt off the next crash's budget, nothing more.
+            log('failed to refill respawn budget for %s: %s', node.id, errorMessage(err));
+        }
+    }
+    /**
+     * Whether a record carries a crash-loop count old enough to clear. An
+     * unparsable timestamp is treated as not-due — unlike the backoff check,
+     * being wrong here would erase real attempt history rather than free a node.
+     */
+    budgetRefillDue(node) {
+        const respawn = node.respawn;
+        if (!respawn || respawn.attempts === 0)
+            return false;
+        const last = Date.parse(respawn.lastAttemptAt);
+        if (Number.isNaN(last))
+            return false;
+        return this.now().getTime() - last > HOSTED_NODE_RESPAWN_HEALTHY_MS;
+    }
+    /**
+     * One respawn attempt, with the give-up check on failure. (The check for an
+     * attempt that spawned but then died lives in {@link reconcileOne}.)
+     */
+    async attemptRespawn(node) {
+        try {
+            const result = await this.service.respawn(node.id);
+            switch (result.outcome) {
+                case 'abandoned':
+                    // The node was removed (or given up on) while the spawn was in flight
+                    // and the ending won. Nothing threw, so no attempt is persisted and the
+                    // give-up path is never entered — correct: this is not a failed attempt.
+                    log('respawn of hosted node %s was abandoned (record went %s mid-spawn)', node.id, result.status ?? 'missing');
+                    return undefined;
+                case 'respawned':
+                    log('respawned hosted node %s → %s', node.id, result.node.dockerId);
+                    return node.id;
+            }
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            // `respawn` persists the incremented counter; re-read it rather than
+            // recomputing, so a give-up decision is made on what is actually on disk.
+            const attempts = this.store.get(node.id)?.respawn?.attempts ?? 0;
+            log('respawn attempt %d for hosted node %s failed: %s', attempts, node.id, message);
+            if (attempts >= HOSTED_NODE_RESPAWN_MAX_ATTEMPTS) {
+                await this.giveUp(node.id, attempts, message);
+            }
+            return undefined;
+        }
+    }
+    /**
+     * Stop trying: mark the node `error` and stop whatever child the record
+     * still names.
+     *
+     * The record is written BEFORE the stop for the same reason `remove` does
+     * it in that order — the stop fires `onStateChange`, and a pass triggered by
+     * it must not see "node gone, record still `joined`" and start respawning
+     * again.
+     *
+     * The child is stopped but NOT removed: `removeContainer` deletes the workdir,
+     * and the workdir holds the identity key the node's cadre approved. A later
+     * `remove()` (or `reset()`) on an `error` record reclaims it.
+     *
+     * NOTE: an `error`-after-give-up record therefore keeps its workdir until
+     * someone removes it. If stale `error` workdirs ever pile up on real hosts,
+     * extend the reap sweep in `hosted-node-service.ts` to cover them.
+     */
+    async giveUp(id, attempts, lastError) {
+        const node = this.store.get(id);
+        if (!node)
+            return;
+        // The record may have been removed or given up on while the failing attempt
+        // was in flight — a `remove` mid-pass is exactly why `respawn` threw. A stop
+        // here would fire against a child `remove` already reclaimed.
+        if (!SUPERVISED_STATUSES.has(node.status)) {
+            log('hosted node %s went %s before give-up — leaving it alone', id, node.status);
+            return;
+        }
+        this.store.put({
+            ...node,
+            status: 'error',
+            error: `respawn gave up after ${attempts} attempts: ${lastError}`,
+            updatedAt: this.now().toISOString(),
+        });
+        log('gave up respawning hosted node %s after %d attempts: %s', id, attempts, lastError);
+        this.service.emit({ kind: 'changed', id });
+        if (node.dockerId) {
+            try {
+                await this.orchestrator.stopContainer(node.dockerId);
+            }
+            catch (err) {
+                // The record names a live handle in the ordinary case — a failed respawn
+                // restores the handle it dropped, and a respawn that spawned but failed
+                // to record writes the new one onto the record. Still guarded: a handle
+                // can be lost to a host restart (state.json gone, or the child already
+                // reaped), and the child is dead either way, which is why we are here.
+                log('failed to stop container %s while giving up: %s', node.dockerId, errorMessage(err));
+            }
+        }
+    }
+}
+//# sourceMappingURL=hosted-node-supervisor.js.map

package/dist/hosted/hosted-node-supervisor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hosted-node-supervisor.js","sourceRoot":"","sources":["../../src/hosted/hosted-node-supervisor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAO1B,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGhD,MAAM,GAAG,GAAG,KAAK,CAAC,mCAAmC,CAAC,CAAC;AAEvD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,IAAK,CAAC;AAEzD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,KAAM,CAAC;AAE7D;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,KAAM,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,CAAC,MAAM,4BAA4B,GAAG,KAAM,CAAC;AAEnD;;;;GAIG;AACH,MAAM,mBAAmB,GAAkC,IAAI,GAAG,CAAmB;IACnF,WAAW;IACX,SAAS;IACT,QAAQ;CACT,CAAC,CAAC;AAyBH;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,oBAAoB;IAa/B,YAAY,IAAiC;QAL7C,gFAAgF;QACxE,mBAAc,GAAG,KAAK,CAAC;QAC/B,2EAA2E;QACnE,aAAQ,GAAG,KAAK,CAAC;QAGvB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAC1C,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IACpE,CAAC;IAED;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,EAAU;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,sEAAsE;QACtE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,4BAA4B,CAAC,CAAC;QACvF,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACH,IAAI;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;QAC7B,GAAG,CAAC,8BAA8B,CAAC,CAAC;IACtC,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACtB,IAAI,KAAkB,CAAC;QACvB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,8CAA8C,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACvE,OAAO;QACT,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAE,SAAS;YACnC,OAAO,CAAC,KAAK,CACX,eAAe,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,MAAM,mEAAmE;kBAC3G,mDAAmD,GAAG,MAAM,CAAC,EAAE,GAAG,mCAAmC,CACxG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IAC5D,KAAK,CAAC,OAAe;QAC3B,KAAK,IAAI,CAAC,SAAS,EAAE;aAClB,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YACZ,IAAI,GAAG,CAAC,MAAM;gBAAE,GAAG,CAAC,0CAA0C,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC5F,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,OAAO,CAAC,KAAK,CAAC,qCAAqC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,IAAqB;QACvC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO;QACtC,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO;QAChC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,KAAK,IAAI,CAAC,SAAS,EAAE;aAClB,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,qCAAqC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5F,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,yCAAyC;IACjC,KAAK,CAAC,aAAa;QACzB,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,UAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6EAA6E;YAC7E,wCAAwC;YACxC,GAAG,CAAC,2CAA2C,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACpE,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,QAAQ;gBAAE,MAAM;YACzB,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,EAAE;oBAAE,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,uEAAuE;gBACvE,GAAG,CAAC,wCAAwC,EAAE,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,EAAU;QAClC,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,EAAE,QAAQ,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO;QACrE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,GAAG,CAAC,8BAA8B,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,wEAAwE;IAChE,KAAK,CAAC,YAAY,CAAC,IAAgB;QACzC,2EAA2E;QAC3E,6DAA6D;QAC7D,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAC;QAErC,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,2EAA2E;QAC3E,2EAA2E;QAC3E,uEAAuE;QACvE,0DAA0D;QAC1D,wEAAwE;QACxE,2EAA2E;QAC3E,uEAAuE;QACvE,yCAAyC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,IAAI,gCAAgC,EAAE,CAAC;YACjD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,sDAAsD,CAAC,CAAC;YAC7F,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,6DAA6D,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACK,cAAc,CAAC,IAAgB;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,mCAAmC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,EAC3D,kCAAkC,CACnC,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC;IAC9C,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,IAAgB;QAC5C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;YAAE,OAAO;QACxC,2EAA2E;QAC3E,4EAA4E;QAC5E,yEAAyE;QACzE,mBAAmB;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;YAAE,OAAO;QAChE,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7E,GAAG,CAAC,2DAA2D,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,+DAA+D;YAC/D,GAAG,CAAC,4CAA4C,EAAE,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,IAAgB;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACrC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,8BAA8B,CAAC;IACtE,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CAAC,IAAgB;QAC3C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnD,QAAQ,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvB,KAAK,WAAW;oBACd,sEAAsE;oBACtE,wEAAwE;oBACxE,yEAAyE;oBACzE,GAAG,CAAC,oEAAoE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;oBAC/G,OAAO,SAAS,CAAC;gBACnB,KAAK,WAAW;oBACd,GAAG,CAAC,+BAA+B,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACpE,OAAO,IAAI,CAAC,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,qEAAqE;YACrE,0EAA0E;YAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC;YACjE,GAAG,CAAC,kDAAkD,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACpF,IAAI,QAAQ,IAAI,gCAAgC,EAAE,CAAC;gBACjD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACK,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,QAAgB,EAAE,SAAiB;QAClE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,4EAA4E;QAC5E,6EAA6E;QAC7E,8DAA8D;QAC9D,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1C,GAAG,CAAC,0DAA0D,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACjF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;YACb,GAAG,IAAI;YACP,MAAM,EAAE,OAAO;YACf,KAAK,EAAE,yBAAyB,QAAQ,cAAc,SAAS,EAAE;YACjE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QACH,GAAG,CAAC,yDAAyD,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QACxF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,yEAAyE;gBACzE,wEAAwE;gBACxE,wEAAwE;gBACxE,uEAAuE;gBACvE,uEAAuE;gBACvE,GAAG,CAAC,iDAAiD,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/hosted/hosted-node-watcher.d.ts

@@ -0,0 +1,95 @@
+import type { NodeStateListener } from '../orchestrator/types.js';
+import type { HostedNodeStore } from './hosted-node-store.js';
+import type { HostedNodeChange } from './types.js';
+/** How often an unclaimed node's `/status` is read for the claim, and a joining node's for its redemption. */
+export declare const HOSTED_NODE_CLAIM_POLL_MS = 2000;
+/** How often a joined node's `/status` is read for its connection count. */
+export declare const HOSTED_NODE_CONNECTED_POLL_MS = 15000;
+/**
+ * The orchestrator surface the watcher needs: the exit/start signal, when the orchestrator
+ * has one, and the stop of a node whose invitation failed or that an owner removed.
+ */
+export interface WatchedOrchestrator {
+    onStateChange?(listener: NodeStateListener): () => void;
+    stopContainer(dockerId: string): Promise<void>;
+}
+export interface HostedNodeWatcherOptions {
+    store: HostedNodeStore;
+    orchestrator: WatchedOrchestrator;
+    now: () => Date;
+    /** Where a claim or a liveness change is reported (`HostedNodeService.emit`). */
+    emit: (change: HostedNodeChange) => void;
+}
+/**
+ * Follows each hosted node's `/status`: an `unclaimed` record becomes `joined` when
+ * the node reports a finished claim; a `joining` record becomes `joined` when a member
+ * accepted its invitation, or `error` when the redemption failed; a `joined`
+ * record's `connected` follows the node's control-connection count; and any of them
+ * becomes `error` when the node reports that an owner removed it from its cadre.
+ * Best-effort: a poll that fails is retried on the next tick, and nothing here throws
+ * out of the timer.
+ *
+ * One timer at the claim cadence drives all three: every tick polls every unclaimed and
+ * joining node, and each joined node every {@link HOSTED_NODE_CONNECTED_POLL_MS}, or at once when it
+ * was poked — the orchestrator's state change on a respawn pokes the node, and the
+ * poke stands until a poll succeeds, so a respawned child is noticed on its first
+ * answer rather than at the next 15 s mark. Polls are serialized on one promise
+ * tail so a timer pass and a poked pass cannot both write one record.
+ *
+ * NOTE: the 2 s timer ticks while nothing is unclaimed (one cached store read, no
+ * I/O per tick); if it ever shows up in a profile, stop it when no record is
+ * unclaimed and restart it from `poke`.
+ */
+export declare class HostedNodeWatcher {
+    private readonly store;
+    private readonly orchestrator;
+    private readonly now;
+    private readonly emit;
+    private tail;
+    private timer?;
+    private unsubscribe?;
+    /** When each joined node's `/status` last answered (ms since epoch). */
+    private readonly lastPolledAt;
+    /** Joined nodes to poll on the next pass regardless of cadence. */
+    private readonly due;
+    constructor(opts: HostedNodeWatcherOptions);
+    /** Start the timer and the state-change subscription, and poll once. Idempotent. */
+    start(): void;
+    stop(): void;
+    /** Ask for a pass now; `id` names a joined node to poll ahead of its cadence. Fire-and-forget. */
+    poke(id?: string): void;
+    /** One pass over every unclaimed, joining and due joined record, serialized against the others. */
+    poll(): Promise<void>;
+    private pollOnce;
+    /** Drop the cadence and poke bookkeeping of nodes no longer on record, so neither map grows with removals. */
+    private forgetRemoved;
+    private shouldPoll;
+    private pollOne;
+    /** An unclaimed node that reports a finished claim is `joined`, with the party and owner the claim named. */
+    private applyUnclaimed;
+    /**
+     * A joining node whose redemption settled: `joined` with the member that admitted it and
+     * the invitation's issuer as its owner, or `error` with the reason and whether Retry may
+     * help. A failed node's child is stopped after the write, as a give-up's is, and kept
+     * with its workdir for Retry; the write comes first so the exit the stop raises finds
+     * a record the supervisor leaves alone. Still `pending`: nothing.
+     */
+    private applyJoining;
+    /**
+     * A node that reports an owner removed it from its cadre: `error`, not retryable, its child
+     * stopped after the write as a failed invitation's is, and its workdir kept for Remove (or a
+     * claim node's Reset). The child had already stopped its node, and stays up only to report
+     * the removal, so stopping it loses nothing. The party and owner come from the report, for an
+     * `unclaimed` record whose claim this watcher never saw: its own party is the placeholder.
+     */
+    private applyRemoved;
+    /** Stop a node's child after its record went `error`; a failed stop is logged, the record stands. */
+    private stopChild;
+    /**
+     * A joined node's `connected` follows its control-connection count; published only on a
+     * change. Its `/status` keeps reporting the invitation for as long as the child that
+     * redeemed it runs, and whatever it says there is ignored: the node is a member, and the
+     * record says so.
+     */
+    private applyJoined;
+}