@north-light/api 0.3.560 → 0.3.562

package/dist/core/scoped-state/materialized-plugins.js

@@ -0,0 +1,103 @@
+// materialized-plugins.ts — the plugin bytes a cloud-role process holds for the people it serves (D84). A person's enabled plugins are
+// rows (`scope_plugins`, `scope_marketplaces`); a fresh API process has no plugin directories, so before it reads plugin manifests it
+// installs each enabled plugin from its marketplace into a process-local directory keyed by (person, plugins_revision). Leaf-safe:
+// the installed-plugin reader imports this, so it imports no database. `loadPersonSettings` (person-rows.ts) fills it.
+import { cpSync, existsSync, mkdirSync, renameSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { isAbsolute, join, relative, resolve } from 'node:path';
+import { CrtrError } from '../errors.js';
+import { gitAsync } from '../git.js';
+import { readMarketplaceManifest } from '../manifest.js';
+const materialized = new Map();
+const inFlight = new Map();
+/** The directory holding this person's materialized plugins as `<dir>/<plugin name>`; undefined when none were materialized. */
+export function materializedPluginsDir(person) {
+    const held = materialized.get(person);
+    if (held !== undefined && 'failure' in held)
+        throw held.failure;
+    return held?.dir;
+}
+function cacheRoot(person) { return join(tmpdir(), 'crtr-plugin-cache', person.replace(/[^A-Za-z0-9._-]/g, '_')); }
+async function cloneInto(url, ref, dest, what) {
+    const res = await gitAsync(['clone', '--depth', '1', ...(ref ? ['--branch', ref] : []), url, dest]);
+    if (res.status !== 0)
+        throw new CrtrError('plugin_install_failed', `${what}: git clone ${url} failed: ${res.stderr.trim()}`);
+}
+async function installAll(staging, installs) {
+    const marketRoots = new Map();
+    mkdirSync(join(staging, 'plugins'), { recursive: true });
+    for (const plugin of installs) {
+        if (plugin.sourcePath !== null)
+            throw new CrtrError('plugin_source_unreachable', `${plugin.name}: its source is ${plugin.sourcePath} on the machine that registered it, which the control plane cannot read`);
+        const market = plugin.marketplace;
+        if (market === null)
+            throw new CrtrError('plugin_source_unreachable', `${plugin.name}: it names neither a marketplace nor a path`);
+        if (market.url === null || market.url === '')
+            throw new CrtrError('plugin_source_unreachable', `${plugin.name}: its marketplace ${market.name} has no url`);
+        const key = `${market.url}\0${market.ref ?? ''}`;
+        let root = marketRoots.get(key);
+        if (root === undefined) {
+            root = join(staging, 'marketplaces', String(marketRoots.size));
+            await cloneInto(market.url, market.ref, root, `${plugin.name}: marketplace ${market.name}`);
+            marketRoots.set(key, root);
+        }
+        const entry = readMarketplaceManifest(root)?.plugins.find((p) => p.name === plugin.name);
+        if (entry === undefined)
+            throw new CrtrError('plugin_install_failed', `${plugin.name}: marketplace ${market.name} (${market.url}) does not list it`);
+        const dest = join(staging, 'plugins', plugin.name);
+        if (isAbsolute(entry.source) || /^[a-z][a-z0-9+.-]*:|^git@/i.test(entry.source) || entry.source.endsWith('.git')) {
+            await cloneInto(entry.source, null, dest, plugin.name);
+            continue;
+        }
+        const from = resolve(root, entry.source);
+        const inside = relative(root, from);
+        if (inside.startsWith('..') || isAbsolute(inside))
+            throw new CrtrError('plugin_install_failed', `${plugin.name}: its source ${entry.source} leaves the marketplace`);
+        if (!existsSync(from))
+            throw new CrtrError('plugin_install_failed', `${plugin.name}: its source ${entry.source} is not in marketplace ${market.name}`);
+        cpSync(from, dest, { recursive: true });
+    }
+    rmSync(join(staging, 'marketplaces'), { recursive: true, force: true });
+}
+/** Install `installs` for `person` at `revision` into this process's plugin cache, once per (person, revision). When a plugin cannot
+ *  be installed — a source on another machine, a marketplace that does not list it, a failed clone — the failure is held for that person
+ *  and thrown by the next read of their plugins (`materializedPluginsDir`), so a request that never reads plugins, such as the one that
+ *  fixes the settings, still works; the next load retries. */
+export async function materializePlugins(person, revision, installs) {
+    const held = materialized.get(person);
+    if (held?.revision === revision && !('failure' in held))
+        return;
+    const final = join(cacheRoot(person), String(revision));
+    const key = `${person}\0${revision}`;
+    let pending = inFlight.get(key);
+    if (pending === undefined) {
+        pending = (async () => {
+            if (!existsSync(join(final, 'plugins'))) {
+                const staging = join(cacheRoot(person), `${revision}.${process.pid}.${Date.now()}`);
+                try {
+                    await installAll(staging, installs);
+                    try {
+                        renameSync(staging, final);
+                    }
+                    catch (error) {
+                        if (!existsSync(join(final, 'plugins')))
+                            throw error;
+                    }
+                }
+                finally {
+                    rmSync(staging, { recursive: true, force: true });
+                }
+            }
+            return join(final, 'plugins');
+        })().finally(() => inFlight.delete(key));
+        inFlight.set(key, pending);
+    }
+    try {
+        materialized.set(person, { revision, dir: await pending });
+    }
+    catch (error) {
+        if (!(error instanceof CrtrError))
+            throw error;
+        materialized.set(person, { revision, failure: error });
+    }
+}

package/dist/core/scoped-state/person-store.d.ts

@@ -0,0 +1,72 @@
+import type { ProfileManifest, ScopeState } from '../../types.js';
+export type StoredProfile = {
+    profileId: string;
+    manifest: ProfileManifest;
+};
+/** Everything one person's rows hold, in the shape the synchronous readers want. `settings` is keyed by owner (`user`,
+ *  `profile:<id>`) and carries the rebuilt `plugins` and `marketplaces` blocks; `state` carries the bookkeeping. */
+export interface PersonSnapshot {
+    profiles: StoredProfile[];
+    defaults: Record<string, string>;
+    settings: Record<string, Record<string, unknown>>;
+    state: Record<string, ScopeState>;
+}
+export type SettingsOp = {
+    op: 'settings.replace';
+    owner: string;
+    raw: Record<string, unknown>;
+    ifRevision: string;
+} | {
+    op: 'state.write';
+    owner: string;
+    state: ScopeState;
+} | {
+    op: 'profile.insert';
+    profile: StoredProfile;
+} | {
+    op: 'profile.update';
+    profile: StoredProfile;
+} | {
+    op: 'profile.delete';
+    profileId: string;
+} | {
+    op: 'defaults.replace';
+    defaults: Record<string, string>;
+};
+export declare class SettingsRevisionConflictError extends Error {
+    constructor();
+}
+export declare class PersonSettingsNotLoadedError extends Error {
+    constructor(person: string);
+}
+declare const BLOCKS: readonly ["plugins", "marketplaces"];
+export declare function revisionOf(raw: Record<string, unknown> | undefined): string;
+export declare function installPersonSnapshot(person: string, snapshot: PersonSnapshot): void;
+export declare function heldPersons(): string[];
+export declare function dropPersonSnapshots(): void;
+/** Before the install's person is resolved (a Mac's first boot reads its config to resolve it), reads answer from this snapshot of
+ *  the old local store; `loadPersonSettings` replaces it once the person is known. */
+export declare function installPreludeSnapshot(snapshot: PersonSnapshot | undefined): void;
+/** The current person's snapshot. A miss is an error: a person whose rows exist never reads as defaults. A cloud role outside a
+ *  request has no person and no install settings: it reads defaults. */
+export declare function currentSnapshot(): PersonSnapshot;
+export declare function personHasSnapshot(): boolean;
+export type SettingsTransport = (person: string, ops: SettingsOp[]) => Promise<PersonSnapshot>;
+export declare function setSettingsTransport(next: SettingsTransport): void;
+/** Compute ops from the cached snapshot and send them; on a revision conflict reload (the transport returned or the rows hold a newer
+ *  snapshot) and compute again, up to three times. A failed write throws and leaves the cache as it was. */
+export declare function writeSettings(compute: (snapshot: PersonSnapshot) => SettingsOp[]): Promise<PersonSnapshot>;
+export declare function setPersonSnapshot(person: string, snapshot: PersonSnapshot): void;
+export declare function setSettingsReloader(next: (person: string) => Promise<PersonSnapshot>): void;
+export declare function ownerKeyOf(profileId?: string): string;
+export declare function readCachedSettings(owner: string): {
+    settings: Record<string, unknown> | null;
+    revision: string;
+};
+export declare function readCachedState(owner: string): ScopeState;
+export { BLOCKS as SETTINGS_ROW_BLOCKS };
+export declare function listCachedProfiles(): StoredProfile[];
+export declare function getCachedProfile(profileId: string): StoredProfile | null;
+export declare function cachedProfilePaused(profileId: string): boolean;
+export declare function cachedPausedProfileIds(): Set<string>;
+export declare function cachedDefaults(): Record<string, string>;

package/dist/core/scoped-state/person-store.js

@@ -0,0 +1,112 @@
+// person-store.ts — profiles, scope settings, scope bookkeeping and the plugin/marketplace install list as Postgres rows
+// keyed by person (002-fleet.sql). Reads stay synchronous over a per-process cache of one snapshot per person; a write is
+// asynchronous: rows first, then the cache is reloaded from the rows. A failed write throws and leaves the cache.
+// Two transports apply the same declarative ops: `applyOpsToRows` (a process that holds the database) and `PUT /v1/settings`
+// (a CLI or viewer process, installed by `loadPersonSettingsViaApi`). Project-scope settings are files and never come here.
+import { createHash } from 'node:crypto';
+import { PersonOutsideRequestError, PersonUnresolvedError, personId } from '../canvas/person-state.js';
+export class SettingsRevisionConflictError extends Error {
+    constructor() { super('settings changed since the supplied revision'); this.name = 'SettingsRevisionConflictError'; }
+}
+export class PersonSettingsNotLoadedError extends Error {
+    constructor(person) { super(`settings for ${person} are not loaded in this process`); this.name = 'PersonSettingsNotLoadedError'; }
+}
+const BLOCKS = ['plugins', 'marketplaces'];
+const caches = new Map();
+export function revisionOf(raw) {
+    return raw === undefined ? 'absent' : createHash('sha256').update(JSON.stringify(raw)).digest('hex');
+}
+// ------------------------------------------------------------------ cache
+export function installPersonSnapshot(person, snapshot) { caches.set(person, snapshot); }
+export function heldPersons() { return [...caches.keys()]; }
+export function dropPersonSnapshots() { caches.clear(); }
+const emptySnapshot = () => ({ profiles: [], defaults: {}, settings: {}, state: {} });
+let prelude;
+/** Before the install's person is resolved (a Mac's first boot reads its config to resolve it), reads answer from this snapshot of
+ *  the old local store; `loadPersonSettings` replaces it once the person is known. */
+export function installPreludeSnapshot(snapshot) { prelude = snapshot; }
+/** The current person's snapshot. A miss is an error: a person whose rows exist never reads as defaults. A cloud role outside a
+ *  request has no person and no install settings: it reads defaults. */
+export function currentSnapshot() {
+    let person;
+    try {
+        person = personId();
+    }
+    catch (error) {
+        if (error instanceof PersonOutsideRequestError)
+            return emptySnapshot();
+        // Resolving the install's person reads the user config (`runtime.issuer`) before any person exists: a process with no
+        // prelude and no person holds no settings, so it reads defaults, as a cloud role outside a request does.
+        if (error instanceof PersonUnresolvedError)
+            return prelude ?? emptySnapshot();
+        throw error;
+    }
+    const held = caches.get(person);
+    if (held !== undefined)
+        return held;
+    if (prelude !== undefined)
+        return prelude;
+    throw new PersonSettingsNotLoadedError(person);
+}
+export function personHasSnapshot() {
+    try {
+        return caches.has(personId());
+    }
+    catch {
+        return false;
+    }
+}
+/** The default transport and reloader are the database's (`person-rows.ts` registers them in a process that holds one); a CLI or viewer
+ *  process installs the `/v1/settings` pair. Neither being installed is a wiring error. */
+let transport = async () => { throw new Error('no settings transport installed in this process'); };
+export function setSettingsTransport(next) { transport = next; }
+/** Compute ops from the cached snapshot and send them; on a revision conflict reload (the transport returned or the rows hold a newer
+ *  snapshot) and compute again, up to three times. A failed write throws and leaves the cache as it was. */
+export async function writeSettings(compute) {
+    const person = personId();
+    for (let attempt = 0;; attempt++) {
+        const ops = compute(currentSnapshot());
+        if (ops.length === 0)
+            return currentSnapshot();
+        try {
+            const next = await transport(person, ops);
+            caches.set(person, next);
+            return next;
+        }
+        catch (error) {
+            if (!(error instanceof SettingsRevisionConflictError) || attempt >= 2)
+                throw error;
+            await reloadViaTransport(person);
+        }
+    }
+}
+export function setPersonSnapshot(person, snapshot) { caches.set(person, snapshot); prelude = undefined; }
+let reloader;
+export function setSettingsReloader(next) { reloader = next; }
+async function reloadViaTransport(person) {
+    caches.set(person, await (reloader ?? (async () => { throw new Error('no settings reloader installed in this process'); }))(person));
+}
+// ------------------------------------------------------------------ synchronous readers over the cache
+export function ownerKeyOf(profileId) { return profileId === undefined ? 'user' : `profile:${profileId}`; }
+export function readCachedSettings(owner) {
+    const raw = currentSnapshot().settings[owner];
+    return { settings: raw === undefined ? null : structuredClone(raw), revision: revisionOf(raw) };
+}
+export function readCachedState(owner) {
+    const st = currentSnapshot().state[owner];
+    return st === undefined ? { marketplaces: {}, plugins: {}, activeCanvas: null } : structuredClone(st);
+}
+export { BLOCKS as SETTINGS_ROW_BLOCKS };
+// ------------------------------------------------------------------ synchronous profile readers over the cache
+export function listCachedProfiles() { return structuredClone(currentSnapshot().profiles); }
+export function getCachedProfile(profileId) {
+    const found = currentSnapshot().profiles.find((p) => p.profileId === profileId);
+    return found === undefined ? null : structuredClone(found);
+}
+export function cachedProfilePaused(profileId) {
+    return currentSnapshot().profiles.find((p) => p.profileId === profileId)?.manifest.paused_at != null;
+}
+export function cachedPausedProfileIds() {
+    return new Set(currentSnapshot().profiles.filter((p) => p.manifest.paused_at !== null).map((p) => p.profileId));
+}
+export function cachedDefaults() { return { ...currentSnapshot().defaults }; }

package/dist/core/user-settings.js

@@ -469,7 +469,7 @@ export const CONFIG_SETTINGS_COVERAGE = {
     remoteCanvas: { surface: 'specialized', owner: 'canvas config commands' },
     api: { surface: 'specialized', owner: 'sys connect' },
     paths: { surface: 'specialized', owner: 'runtime layout' },
-    ledger: { surface: 'specialized', owner: 'money ledger (take_pct)' },
+    ledger: { surface: 'specialized', owner: 'money ledger (take_pct, sandbox rates)' },
     store: { surface: 'specialized', owner: 'daemon conversation store' },
     runtime: { surface: 'specialized', owner: 'app-facing listener' },
     spawnEnv: { surface: 'specialized', owner: 'spawn environment allow-list' },

package/dist/daemon/broker-signals/bus.d.ts

@@ -1,35 +1,72 @@
 import type { BrokerExtensionNodeDTO } from '../../api/dto/broker-ops.js';
 /** Node-owned mutable fields in extensionNode's projection. Keep this list and
  * the typed cache view together; object/worktree-owned fields are never cached. */
-export declare const BROKER_SIGNAL_NODE_COLUMNS: readonly ["description", "title", "icon", "mode", "lifecycle", "status", "cwd", "parent", "fork_from", "profile_id", "review_binding_json", "persona_ack_json", "intent"];
+export declare const BROKER_SIGNAL_NODE_COLUMNS: readonly ["description", "title", "icon", "mode", "lifecycle", "status", "cwd", "parent", "fork_from", "profile_id", "review_binding_json", "persona_ack_json", "intent", "situational_card"];
 type ProjectedNodeKey<C extends string> = C extends 'review_binding_json' ? 'review_binding' : C extends 'persona_ack_json' ? 'persona_ack' : C;
-export type BrokerSignalCachedNode = Pick<BrokerExtensionNodeDTO, Extract<ProjectedNodeKey<(typeof BROKER_SIGNAL_NODE_COLUMNS)[number]>, keyof BrokerExtensionNodeDTO>>;
+export type BrokerSignalCachedNode = Pick<BrokerExtensionNodeDTO, Extract<ProjectedNodeKey<Exclude<(typeof BROKER_SIGNAL_NODE_COLUMNS)[number], 'situational_card'>>, keyof BrokerExtensionNodeDTO>>;
 /** Only node-owned columns are suitable for P4's cache. The same columns
- * appear in `crtr_sig_node_upd` in schema-pg.sql; keep the two lists together. */
+ * appear in `crtr_sig_node_upd` (002-fleet.sql recreates it with `situational_card`); keep the two lists together. */
 export declare function cachedBrokerNodeFields(node: BrokerExtensionNodeDTO): BrokerSignalCachedNode;
-export type BrokerSignalKind = 'mail' | 'node';
-export type BrokerSignalSubscriber = (nodeId: string, kind: BrokerSignalKind) => void;
-/** Hints, not delivery state: a rolled-back write may mark a node. The consumer
- * always re-reads durable state. Pending marks are bounded by distinct node ids. */
+/** What a `crtr_signal` notification names. `mail` and `node` carry a node id; the others name the row that changed
+ *  (D39): a grant id, run or node id, structured request, queue, limit, or environment object. `fault_retry` carries a node id: the
+ *  fault-retry backstop publishes it in the commit that claims the nudge, and the API process holding that node's stream turns it
+ *  into a `fault-retry` line. The writers fire them (the
+ *  schema's triggers, or the unit that owns the writer); the bus only delivers. */
+export declare const BROKER_SIGNAL_KINDS: readonly ["mail", "node", "grant", "run_event", "report", "structured_request", "queue", "spend_limits", "environment_image", "fault_retry", "settings"];
+export type BrokerSignalKind = (typeof BROKER_SIGNAL_KINDS)[number];
+export type BrokerSignalSubscriber = (id: string, kind: BrokerSignalKind) => void;
+export interface BrokerSignalSubscription {
+    /** The kinds this subscriber hears. Default `mail` and `node`, the two a node's broker stream reads. */
+    kinds?: readonly BrokerSignalKind[];
+    /** Called each time the listen connection starts or restarts (LISTEN is active). Notifications sent while it was down
+     *  are lost, so a subscriber re-reads its durable state here; that is its catch-up. A subscriber added while the bus
+     *  is already listening is called once, soon after subscribing. */
+    onListenStart?: () => void;
+}
+export interface BrokerSignalBusOptions {
+    /** How often the listen connection is checked with `SELECT 1`. Default 30 s. */
+    checkIntervalMs?: number;
+    /** How long a check may take before the connection counts as lost. Default 10 s. */
+    checkTimeoutMs?: number;
+    /** Delay before a lost listen connection reconnects. Default 1 s. */
+    reconnectMs?: number;
+}
+/** Hints, not delivery state: a rolled-back write may mark an id. The consumer always re-reads durable state. Pending
+ * marks are bounded by distinct (kind, id) pairs and delivered once per flush. */
 export declare class BrokerSignalBus {
     private readonly dirty;
     private readonly subscribers;
     private scheduled;
-    subscribe(listener: BrokerSignalSubscriber): Promise<() => void>;
+    private readonly checkIntervalMs;
+    private readonly checkTimeoutMs;
+    private readonly reconnectMs;
+    constructor(options?: BrokerSignalBusOptions);
+    subscribe(listener: BrokerSignalSubscriber, subscription?: BrokerSignalSubscription): Promise<() => void>;
     markMail(nodeId: string): void;
     markNode(nodeId: string): void;
+    /** Hint any kind, as a notification of that kind would. */
+    markSignal(kind: BrokerSignalKind, id: string): void;
     private mark;
     private flush;
+    private callListenStart;
     /** Start listening on `crtr_signal` over a second, listen-only connection to `url` (the schema's `pg_notify`
-     * triggers publish `mail:<node id>` and `node:<node id>` at commit). Resolves once LISTEN is active, so a write
-     * committed after it returns is heard. A lost connection reconnects with backoff; signals sent while it was down
-     * are lost, which is safe because they are hints and every consumer re-reads durable state. */
+     * triggers publish `<kind>:<id>` at commit). Resolves once LISTEN is active, so a write committed after it returns is
+     * heard. A lost connection (an error, an end, or a failed `SELECT 1` check) reconnects with a fixed delay and calls every
+     * subscriber's `onListenStart` once LISTEN is active again. Signals sent while it was down are lost, which is safe because
+     * they are hints and every consumer re-reads durable state, at `onListenStart` and otherwise. */
     listen(url: string): Promise<void>;
     private listener;
     private listening;
     private url;
     private reconnect;
+    private check;
     private connectListener;
+    /** `SELECT 1` on the listen connection every interval. A dead TCP connection (a dropped route, a failover) raises no
+     *  error by itself, so a failed or unanswered check is what tells the bus the connection is lost. */
+    private startChecks;
+    private stopChecks;
+    private dropListener;
+    private scheduleReconnect;
     /** Stop listening and drop pending marks. The returned promise resolves once the connection has closed. */
     close(): Promise<void>;
 }

package/dist/daemon/broker-signals/bus.js

@@ -4,55 +4,86 @@ import pg from 'pg';
 export const BROKER_SIGNAL_NODE_COLUMNS = [
     'description', 'title', 'icon', 'mode', 'lifecycle', 'status', 'cwd',
     'parent', 'fork_from', 'profile_id', 'review_binding_json',
-    'persona_ack_json', 'intent',
+    'persona_ack_json', 'intent', 'situational_card',
 ];
 const nodeField = (column) => column === 'review_binding_json' ? 'review_binding' : column === 'persona_ack_json' ? 'persona_ack' : column;
 /** Only node-owned columns are suitable for P4's cache. The same columns
- * appear in `crtr_sig_node_upd` in schema-pg.sql; keep the two lists together. */
+ * appear in `crtr_sig_node_upd` (002-fleet.sql recreates it with `situational_card`); keep the two lists together. */
 export function cachedBrokerNodeFields(node) {
     const picked = {};
     for (const column of BROKER_SIGNAL_NODE_COLUMNS) {
+        if (column === 'situational_card')
+            continue;
         const field = nodeField(column);
         picked[field] = node[field];
     }
     return picked;
 }
-/** Hints, not delivery state: a rolled-back write may mark a node. The consumer
- * always re-reads durable state. Pending marks are bounded by distinct node ids. */
+/** What a `crtr_signal` notification names. `mail` and `node` carry a node id; the others name the row that changed
+ *  (D39): a grant id, run or node id, structured request, queue, limit, or environment object. `fault_retry` carries a node id: the
+ *  fault-retry backstop publishes it in the commit that claims the nudge, and the API process holding that node's stream turns it
+ *  into a `fault-retry` line. The writers fire them (the
+ *  schema's triggers, or the unit that owns the writer); the bus only delivers. */
+export const BROKER_SIGNAL_KINDS = ['mail', 'node', 'grant', 'run_event', 'report', 'structured_request', 'queue', 'spend_limits', 'environment_image', 'fault_retry', 'settings'];
+const KNOWN_KINDS = new Set(BROKER_SIGNAL_KINDS);
+/** Hints, not delivery state: a rolled-back write may mark an id. The consumer always re-reads durable state. Pending
+ * marks are bounded by distinct (kind, id) pairs and delivered once per flush. */
 export class BrokerSignalBus {
     dirty = new Map();
     subscribers = new Set();
     scheduled;
-    async subscribe(listener) {
-        this.subscribers.add(listener);
-        return () => { this.subscribers.delete(listener); };
+    checkIntervalMs;
+    checkTimeoutMs;
+    reconnectMs;
+    constructor(options = {}) {
+        this.checkIntervalMs = options.checkIntervalMs ?? 30_000;
+        this.checkTimeoutMs = options.checkTimeoutMs ?? 10_000;
+        this.reconnectMs = options.reconnectMs ?? 1000;
     }
-    markMail(nodeId) { this.mark(nodeId, 1); }
-    markNode(nodeId) { this.mark(nodeId, 2); }
-    mark(nodeId, flag) {
-        this.dirty.set(nodeId, (this.dirty.get(nodeId) ?? 0) | flag);
+    async subscribe(listener, subscription = {}) {
+        const subscriber = { listener, kinds: new Set(subscription.kinds ?? ['mail', 'node']), onListenStart: subscription.onListenStart };
+        this.subscribers.add(subscriber);
+        if (this.listener !== undefined && subscriber.onListenStart !== undefined)
+            queueMicrotask(() => { if (this.subscribers.has(subscriber))
+                this.callListenStart(subscriber); });
+        return () => { this.subscribers.delete(subscriber); };
+    }
+    markMail(nodeId) { this.mark('mail', nodeId); }
+    markNode(nodeId) { this.mark('node', nodeId); }
+    /** Hint any kind, as a notification of that kind would. */
+    markSignal(kind, id) { this.mark(kind, id); }
+    mark(kind, id) {
+        const key = `${kind}:${id}`;
+        if (!this.dirty.has(key))
+            this.dirty.set(key, { id, kind });
         this.scheduled ??= setImmediate(() => this.flush());
     }
     flush() {
         this.scheduled = undefined;
-        const batch = [...this.dirty];
+        const batch = [...this.dirty.values()];
         this.dirty.clear();
-        for (const [id, flags] of batch) {
-            for (const listener of this.subscribers) {
+        for (const { id, kind } of batch) {
+            for (const subscriber of this.subscribers) {
+                if (!subscriber.kinds.has(kind))
+                    continue;
                 try {
-                    if (flags & 1)
-                        listener(id, 'mail');
-                    if (flags & 2)
-                        listener(id, 'node');
+                    subscriber.listener(id, kind);
                 }
                 catch { /* a subscriber never interrupts another subscriber */ }
             }
         }
     }
+    callListenStart(subscriber) {
+        try {
+            subscriber.onListenStart?.();
+        }
+        catch { /* a subscriber never interrupts another subscriber */ }
+    }
     /** Start listening on `crtr_signal` over a second, listen-only connection to `url` (the schema's `pg_notify`
-     * triggers publish `mail:<node id>` and `node:<node id>` at commit). Resolves once LISTEN is active, so a write
-     * committed after it returns is heard. A lost connection reconnects with backoff; signals sent while it was down
-     * are lost, which is safe because they are hints and every consumer re-reads durable state. */
+     * triggers publish `<kind>:<id>` at commit). Resolves once LISTEN is active, so a write committed after it returns is
+     * heard. A lost connection (an error, an end, or a failed `SELECT 1` check) reconnects with a fixed delay and calls every
+     * subscriber's `onListenStart` once LISTEN is active again. Signals sent while it was down are lost, which is safe because
+     * they are hints and every consumer re-reads durable state, at `onListenStart` and otherwise. */
     async listen(url) {
         if (this.listener !== undefined || this.listening !== undefined)
             return;
@@ -64,8 +95,9 @@ export class BrokerSignalBus {
     listening;
     url;
     reconnect;
+    check;
     async connectListener() {
-        const client = new pg.Client({ connectionString: this.url, application_name: `crtr-signal-${process.pid}` });
+        const client = new pg.Client({ connectionString: this.url, application_name: `crtr-signal-${process.pid}`, keepAlive: true });
         client.on('notification', message => {
             // A hint must never cause an exception for an unexpected payload.
             const payload = message.payload ?? '';
@@ -73,39 +105,74 @@ export class BrokerSignalBus {
             if (colon < 0)
                 return;
             const kind = payload.slice(0, colon);
-            const id = payload.slice(colon + 1);
-            if (kind === 'mail')
-                this.markMail(id);
-            else if (kind === 'node')
-                this.markNode(id);
+            if (KNOWN_KINDS.has(kind))
+                this.mark(kind, payload.slice(colon + 1));
         });
         const lost = () => {
             if (this.listener !== client)
                 return;
-            this.listener = undefined;
+            this.dropListener(client);
+            this.scheduleReconnect();
+        };
+        client.on('error', lost);
+        client.on('end', lost);
+        try {
+            await client.connect();
+            await client.query('LISTEN crtr_signal');
+        }
+        catch (error) {
             client.removeAllListeners();
             client.on('error', () => { });
             client.end().catch(() => { });
-            if (this.url === undefined)
+            throw error;
+        }
+        if (this.url === undefined) { // closed while connecting
+            client.removeAllListeners();
+            client.on('error', () => { });
+            client.end().catch(() => { });
+            return;
+        }
+        this.listener = client;
+        this.startChecks(client, lost);
+        for (const subscriber of [...this.subscribers])
+            this.callListenStart(subscriber);
+    }
+    /** `SELECT 1` on the listen connection every interval. A dead TCP connection (a dropped route, a failover) raises no
+     *  error by itself, so a failed or unanswered check is what tells the bus the connection is lost. */
+    startChecks(client, lost) {
+        this.stopChecks();
+        this.check = setInterval(() => {
+            if (this.listener !== client)
                 return;
-            this.reconnect = setTimeout(() => {
-                this.reconnect = undefined;
-                this.connectListener().catch(() => lostRetry());
-            }, 1000);
-        };
-        const lostRetry = () => {
-            if (this.url === undefined || this.reconnect !== undefined)
+            let timer;
+            const unanswered = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error('listen connection check timed out')), this.checkTimeoutMs); });
+            Promise.race([client.query('SELECT 1'), unanswered])
+                .catch(() => lost())
+                .finally(() => clearTimeout(timer));
+        }, this.checkIntervalMs);
+        this.check.unref();
+    }
+    stopChecks() {
+        if (this.check !== undefined)
+            clearInterval(this.check);
+        this.check = undefined;
+    }
+    dropListener(client) {
+        this.listener = undefined;
+        this.stopChecks();
+        client.removeAllListeners();
+        client.on('error', () => { });
+        client.end().catch(() => { });
+    }
+    scheduleReconnect() {
+        if (this.url === undefined || this.reconnect !== undefined)
+            return;
+        this.reconnect = setTimeout(() => {
+            this.reconnect = undefined;
+            if (this.url === undefined)
                 return;
-            this.reconnect = setTimeout(() => {
-                this.reconnect = undefined;
-                this.connectListener().catch(() => lostRetry());
-            }, 1000);
-        };
-        client.on('error', lost);
-        client.on('end', lost);
-        await client.connect();
-        await client.query('LISTEN crtr_signal');
-        this.listener = client;
+            this.listening = this.connectListener().catch(() => this.scheduleReconnect()).finally(() => { this.listening = undefined; });
+        }, this.reconnectMs);
     }
     /** Stop listening and drop pending marks. The returned promise resolves once the connection has closed. */
     close() {
@@ -113,6 +180,7 @@ export class BrokerSignalBus {
         if (this.reconnect !== undefined)
             clearTimeout(this.reconnect);
         this.reconnect = undefined;
+        this.stopChecks();
         const listener = this.listener;
         this.listener = undefined;
         if (this.scheduled !== undefined)
@@ -123,6 +191,8 @@ export class BrokerSignalBus {
         if (listener === undefined)
             return Promise.resolve();
         listener.removeAllListeners('notification');
+        listener.removeAllListeners('error');
+        listener.on('error', () => { });
         return listener.end().catch(() => undefined);
     }
 }

package/dist/shared/inbox-entry-body.js

@@ -83,7 +83,7 @@ function renderPush(entry, push) {
         case 'messaged': return `${actor} messaged ${obj}.`;
         case 'sent': return `${actor} sent a message as ${obj}.`;
         case 'configured': return `${actor} changed the settings of ${obj}${typeof a['changed'] === 'string' ? `: ${a['changed']}` : ''}.`;
-        case 'exited': return `${obj[0].toUpperCase()}${obj.slice(1)} exited with code ${String(a['exit-code'])}. Log: ${String(a['log-path'])}`;
+        case 'exited': return `${obj[0].toUpperCase()}${obj.slice(1)} exited with code ${String(a['exit-code'])}. Output: ${String(a['output-key'])}`;
         case 'silent': {
             const files = typeof a['files'] === 'string' ? a['files'] : '';
             return `${obj[0].toUpperCase()}${obj.slice(1)} has been silent for ${String(a['silent-for'])}: its processes used almost no CPU and none of the files they hold open for writing changed. It may be hung. Watched: ${files === '' ? 'the job log only' : files}. Inspect those files; stop it with \`crtr node bash kill ${push.object.id}\`; or, if it is meant to sit idle, stop these messages with \`crtr node bash quiet ${push.object.id}\`.`;
@@ -94,7 +94,7 @@ function renderPush(entry, push) {
             const claim = a['reason'] === 'deadline'
                 ? ` It was canceled at its deadline; claim it with \`crtr node bash extend ${push.object.id} --for <duration>\` next time.`
                 : '';
-            return `${obj[0].toUpperCase()}${obj.slice(1)} was killed${by}${why}.${claim} Log: ${String(a['log-path'])}`;
+            return `${obj[0].toUpperCase()}${obj.slice(1)} was killed${by}${why}.${claim} Output: ${String(a['output-key'])}`;
         }
         default: return entry.label;
     }