@bookedsolid/rea 0.8.0 → 0.9.0

package/dist/cli/serve.d.ts

@@ -5,10 +5,18 @@
  * later `rea serve` that has raced in and rewritten the breadcrumbs
  * is never unexpectedly unlinked.
  */
+/**
+ * Serve-state file shape. 0.9.0 added the `downstreams` block; older code
+ * that reads the state file treats a missing `downstreams` as "no live
+ * view available" and falls back to the pre-0.9 fields. `session_id` is
+ * the ownership key used by `cleanupStateIfOwned` during shutdown.
+ */
 interface ServeState {
     session_id: string;
     started_at: string;
     metrics_port: number | null;
+    /** 0.9.0 — populated after the gateway starts; absent on this initial write. */
+    downstreams?: unknown[];
 }
 /**
  * Atomic file write: stage to a per-pid temp name, then rename(2). The

package/dist/cli/serve.js

@@ -249,12 +249,30 @@ export async function runServe() {
         console.error('');
         process.exit(1);
     }
+    // Metadata we'll also stamp into the state file below so `rea status`
+    // sees the session-id and start time alongside the new downstream block.
+    const startedAt = new Date().toISOString();
+    const statePath = reaPath(baseDir, SERVE_STATE_FILE);
     const handle = createGateway({
         baseDir,
         policy,
         registry: gatedRegistry,
         logger,
         metrics: metricsRegistry,
+        // 0.9.0 — let the gateway own live writes to serve.state.json so
+        // circuit-breaker transitions and supervisor events are reflected on
+        // disk for `rea status --json`. Legacy shape (session_id, started_at,
+        // metrics_port) is preserved for backward compatibility.
+        liveStateFilePath: statePath,
+        liveStateSessionId: sessionId,
+        liveStateStartedAt: startedAt,
+        liveStateMetricsPort: metricsServer?.port() ?? null,
+        // 0.9.0 pass-7 — reuse the gateway log redactor so downstream error
+        // strings are scrubbed for secret-shaped content BEFORE hitting
+        // serve.state.json or the operator's terminal via `rea status`.
+        // The redactor already incorporates SECRET_PATTERNS plus any
+        // operator-defined policy.redact.patterns loaded above.
+        liveStateLastErrorRedactor: logRedactor,
     });
     // ── HALT acknowledgement at startup (G5) ─────────────────────────────────
     const haltPath = reaPath(baseDir, HALT_FILE);
@@ -280,13 +298,21 @@ export async function runServe() {
         codexProbe.start();
     }
     // ── Pidfile + state (AFTER metrics boot so we persist the real port) ─────
-    const startedAt = new Date().toISOString();
+    //
+    // 0.9.0: the gateway's LiveStatePublisher owns all writes to
+    // serve.state.json, including the boot-time snapshot. Earlier drafts
+    // used the legacy `writeStateFile()` here to cover the bootstrap window
+    // between now and `handle.start()`'s first flush, but that write
+    // bypassed the sidecar-lock protocol and reintroduced the TOCTOU race
+    // P2b was designed to close (Codex 0.9.0 pass-3 P1: an overlapping
+    // older `rea serve` could clobber this unprotected write and the
+    // newer instance would later cleanup its own file during shutdown).
+    //
+    // Routing the boot write through `handle.livePublisher.flushNow()`
+    // means the boot snapshot is guarded by the same lock as every
+    // subsequent flush; overlapping gateways serialize cleanly.
     const pidPath = writePidfile(baseDir);
-    const statePath = writeStateFile(baseDir, {
-        session_id: sessionId,
-        started_at: startedAt,
-        metrics_port: metricsServer?.port() ?? null,
-    });
+    handle.livePublisher?.flushNow();
     let shuttingDown = false;
     const shutdown = async (signal) => {
         // A second signal (e.g. SIGTERM then SIGINT) must NOT re-enter cleanup —

package/dist/cli/status.d.ts

@@ -47,6 +47,24 @@ export declare function sanitizeForTerminal(value: string): string;
 export interface StatusOptions {
     json?: boolean | undefined;
 }
+/**
+ * Per-downstream live state surfaced in both JSON and pretty outputs
+ * (0.9.0, BUG-005). Mirrors `LiveDownstreamState` in
+ * `src/gateway/live-state.ts`; duplicated here to keep the CLI surface
+ * independent of gateway internals (the CLI can be built without the
+ * gateway module in a trimmed install).
+ */
+export interface LiveDownstreamSnapshot {
+    name: string;
+    connected: boolean;
+    healthy: boolean;
+    circuit_state: 'closed' | 'open' | 'half-open';
+    retry_at: string | null;
+    last_error: string | null;
+    tools_count: number | null;
+    open_transitions: number;
+    session_blocker_emitted: boolean;
+}
 interface ServeLiveness {
     running: boolean;
     pid: number | null;
@@ -56,6 +74,13 @@ interface ServeLiveness {
     session_id: string | null;
     started_at: string | null;
     metrics_port: number | null;
+    /**
+     * 0.9.0 — per-downstream live block, or `null` when the state file was
+     * written by an older gateway version that did not include it. A
+     * zero-length array means "gateway is running with no downstreams
+     * configured", which is a distinct signal from "unknown".
+     */
+    downstreams: LiveDownstreamSnapshot[] | null;
 }
 interface AuditStats {
     present: boolean;

package/dist/cli/status.js

@@ -96,21 +96,64 @@ function readPidfile(baseDir) {
         return null;
     }
 }
+/**
+ * Parse a single downstream entry from `serve.state.json`. Every field is
+ * validated — an unexpected type yields a null for that field rather than
+ * poisoning the whole entry, because the state file is touched on a hot
+ * path and we would rather surface a half-useful snapshot than a
+ * "corrupt, try again" error to the operator.
+ *
+ * Returns `null` when the entry's `name` is missing or not a string, since
+ * a downstream with no name is unusable for display.
+ */
+function parseDownstreamEntry(raw) {
+    if (typeof raw !== 'object' || raw === null)
+        return null;
+    const r = raw;
+    if (typeof r.name !== 'string' || r.name.length === 0)
+        return null;
+    const circuit = r.circuit_state === 'open' || r.circuit_state === 'half-open' || r.circuit_state === 'closed'
+        ? r.circuit_state
+        : 'closed';
+    return {
+        name: r.name,
+        connected: typeof r.connected === 'boolean' ? r.connected : false,
+        healthy: typeof r.healthy === 'boolean' ? r.healthy : false,
+        circuit_state: circuit,
+        retry_at: typeof r.retry_at === 'string' ? r.retry_at : null,
+        last_error: typeof r.last_error === 'string' ? r.last_error : null,
+        tools_count: typeof r.tools_count === 'number' && Number.isInteger(r.tools_count) ? r.tools_count : null,
+        open_transitions: typeof r.open_transitions === 'number' && Number.isInteger(r.open_transitions)
+            ? r.open_transitions
+            : 0,
+        session_blocker_emitted: typeof r.session_blocker_emitted === 'boolean' ? r.session_blocker_emitted : false,
+    };
+}
 function readServeState(baseDir) {
     const p = reaPath(baseDir, SERVE_STATE_FILE);
     try {
         const raw = fs.readFileSync(p, 'utf8');
         const parsed = JSON.parse(raw);
+        let downstreams = null;
+        if (Array.isArray(parsed.downstreams)) {
+            downstreams = [];
+            for (const entry of parsed.downstreams) {
+                const ds = parseDownstreamEntry(entry);
+                if (ds !== null)
+                    downstreams.push(ds);
+            }
+        }
         return {
             session_id: typeof parsed.session_id === 'string' ? parsed.session_id : null,
             started_at: typeof parsed.started_at === 'string' ? parsed.started_at : null,
             metrics_port: typeof parsed.metrics_port === 'number' && Number.isInteger(parsed.metrics_port)
                 ? parsed.metrics_port
                 : null,
+            downstreams,
         };
     }
     catch {
-        return { session_id: null, started_at: null, metrics_port: null };
+        return { session_id: null, started_at: null, metrics_port: null, downstreams: null };
     }
 }
 function probeServe(baseDir) {
@@ -124,6 +167,7 @@ function probeServe(baseDir) {
             session_id: null,
             started_at: null,
             metrics_port: null,
+            downstreams: null,
         };
     }
     const alive = isProcessAlive(pid);
@@ -135,6 +179,7 @@ function probeServe(baseDir) {
         session_id: state.session_id,
         started_at: state.started_at,
         metrics_port: state.metrics_port,
+        downstreams: state.downstreams,
     };
 }
 /**
@@ -356,6 +401,46 @@ function printPretty(payload) {
         }
     }
     console.log('');
+    // 0.9.0 — per-downstream block. Only shown when the serve process is
+    // believed to be running AND the state file carried the new array. An
+    // older gateway version that predates the publisher leaves `downstreams`
+    // null; we print an explanatory hint instead of rendering an empty
+    // table that looks like "zero downstreams".
+    if (s.running) {
+        console.log('  Downstreams');
+        if (s.downstreams === null) {
+            console.log(`    (state file has no downstream block — upgrade gateway to ≥0.9.0)`);
+        }
+        else if (s.downstreams.length === 0) {
+            console.log(`    (no downstream servers declared in .rea/registry.yaml)`);
+        }
+        else {
+            for (const d of s.downstreams) {
+                const name = sanitizeForTerminal(d.name);
+                const lastErr = safePretty(d.last_error);
+                const retryAt = safePretty(d.retry_at);
+                const healthToken = d.healthy ? (d.connected ? 'healthy' : 'connecting') : 'UNHEALTHY';
+                const circuit = d.circuit_state.toUpperCase();
+                console.log(`    ${name}`);
+                console.log(`      Health:           ${healthToken}`);
+                console.log(`      Circuit:          ${circuit}`);
+                if (retryAt !== null && d.circuit_state === 'open') {
+                    console.log(`      Retry at:         ${retryAt}`);
+                }
+                if (d.tools_count !== null) {
+                    console.log(`      Tools advertised: ${d.tools_count}`);
+                }
+                if (d.open_transitions > 0) {
+                    const blockerSuffix = d.session_blocker_emitted ? ' (SESSION_BLOCKER fired)' : '';
+                    console.log(`      Open transitions: ${d.open_transitions}${blockerSuffix}`);
+                }
+                if (lastErr !== null) {
+                    console.log(`      Last error:       ${lastErr}`);
+                }
+            }
+        }
+        console.log('');
+    }
     console.log('  Audit log');
     if (!a.present) {
         console.log(`    State:              not yet written`);

package/dist/gateway/circuit-breaker.d.ts

@@ -25,7 +25,14 @@ export interface CircuitStatus {
     serverName: string;
     retryAt?: string;
 }
-interface CircuitEntry {
+/**
+ * Internal per-server circuit state. Exported so observability consumers
+ * (live-state publisher, tests) can read `openedAt` and `cooldownMs` to
+ * compute a `retry_at` timestamp without duplicating the arithmetic.
+ * Treat fields as read-only from outside the breaker — mutating them
+ * breaks the invariants `recordSuccess` / `recordFailure` enforce.
+ */
+export interface CircuitEntry {
     state: CircuitState;
     consecutiveFailures: number;
     openedAt: number | null;
@@ -57,4 +64,3 @@ export declare class CircuitBreaker {
     recordFailure(serverName: string): void;
     getCircuit(serverName: string): CircuitEntry | undefined;
 }
-export {};

package/dist/gateway/downstream-pool.d.ts

@@ -5,7 +5,7 @@
  * The gateway splits on the FIRST `__` — downstream tools that themselves
  * contain `__` in their name continue to work because the split is one-shot.
  */
-import { DownstreamConnection, type DownstreamToolInfo } from './downstream.js';
+import { DownstreamConnection, type DownstreamSupervisorEvent, type DownstreamToolInfo } from './downstream.js';
 import type { Registry } from '../registry/types.js';
 import type { Logger } from './log.js';
 export interface PrefixedTool extends DownstreamToolInfo {
@@ -44,7 +44,19 @@ export declare class DownstreamPool {
      * or is skipped. Stale but truthful > absent.
      */
     private readonly lastToolsCount;
+    /**
+     * Optional supervisor event listener wired by the gateway. The pool
+     * re-emits per-connection events through this single sink so the
+     * SESSION_BLOCKER tracker + state publisher only need to subscribe once.
+     */
+    private supervisorListener;
     constructor(registry: Registry, logger?: Logger);
+    /**
+     * Register a supervisor-event sink. Replaces any previously registered
+     * listener. Intended for the gateway to wire the SESSION_BLOCKER tracker
+     * and live state publisher.
+     */
+    onSupervisorEvent(listener: ((event: DownstreamSupervisorEvent) => void) | null): void;
     get size(): number;
     connectAll(): Promise<void>;
     /**

package/dist/gateway/downstream-pool.js

@@ -5,7 +5,7 @@
  * The gateway splits on the FIRST `__` — downstream tools that themselves
  * contain `__` in their name continue to work because the split is one-shot.
  */
-import { DownstreamConnection } from './downstream.js';
+import { DownstreamConnection, } from './downstream.js';
 export class DownstreamPool {
     connections = new Map();
     /**
@@ -15,13 +15,31 @@ export class DownstreamPool {
      * or is skipped. Stale but truthful > absent.
      */
     lastToolsCount = new Map();
+    /**
+     * Optional supervisor event listener wired by the gateway. The pool
+     * re-emits per-connection events through this single sink so the
+     * SESSION_BLOCKER tracker + state publisher only need to subscribe once.
+     */
+    supervisorListener = null;
     constructor(registry, logger) {
         for (const server of registry.servers) {
             if (!server.enabled)
                 continue;
-            this.connections.set(server.name, new DownstreamConnection(server, logger));
+            const conn = new DownstreamConnection(server, logger);
+            conn.onSupervisorEvent((event) => {
+                this.supervisorListener?.(event);
+            });
+            this.connections.set(server.name, conn);
         }
     }
+    /**
+     * Register a supervisor-event sink. Replaces any previously registered
+     * listener. Intended for the gateway to wire the SESSION_BLOCKER tracker
+     * and live state publisher.
+     */
+    onSupervisorEvent(listener) {
+        this.supervisorListener = listener;
+    }
     get size() {
         return this.connections.size;
     }
@@ -52,7 +70,17 @@ export class DownstreamPool {
                 continue;
             try {
                 const tools = await conn.listTools();
+                // Codex 0.9.0 pass-2 P2a: emit a `health_changed` supervisor event
+                // whenever the cached tools count actually changes. Without this,
+                // a successful listTools would update the value in memory but the
+                // live-state publisher would never flush the change — `rea status`
+                // would keep reporting a stale `tools_count` until some unrelated
+                // circuit/respawn event flushed a snapshot.
+                const prev = this.lastToolsCount.get(server);
                 this.lastToolsCount.set(server, tools.length);
+                if (prev !== tools.length) {
+                    this.supervisorListener?.({ kind: 'health_changed', server });
+                }
                 for (const t of tools) {
                     const prefixed = {
                         ...t,

package/dist/gateway/downstream.d.ts

@@ -30,6 +30,26 @@
  * successful reconnect — in that case we mark the connection unhealthy and
  * let the circuit breaker take over.
  *
+ * ## Supervisor / child-death detection (0.9.0, BUG-002..003)
+ *
+ * The SDK `StdioClientTransport` exposes `onclose` + `onerror` callbacks that
+ * fire when the child process exits or the stdio pipe errors outside a
+ * caller-initiated `close()`. We wire both and treat an unexpected close as
+ * "child is dead" — the next `callTool` must force a fresh connect rather
+ * than calling into a stale `Client` that will reply `Not connected`.
+ *
+ * Before 0.9.0 the supervisor was reactive only: a dead child was not noticed
+ * until the NEXT tool call tried to use it, at which point the circuit could
+ * flap open → half-open → open with the child still dead because the
+ * half-open probe re-used the zombie client. 0.9.0 makes death detection
+ * eager: `onclose` nulls `this.client` so the very next call takes the
+ * `connect()` branch and actually respawns the child.
+ *
+ * "Not connected" error messages from the SDK (our in-flight fallback) are
+ * now also treated as fatal for the current client — we null it before the
+ * one-shot reconnect path so we spawn fresh rather than retrying with the
+ * same dead handle.
+ *
  * ## Why not request-level retries
  *
  * MCP tool calls are not idempotent by default. Retrying `send_message` after
@@ -74,6 +94,42 @@ export interface BuiltChildEnv {
     secretKeys: string[];
 }
 export declare function buildChildEnv(config: RegistryServer, hostEnv?: NodeJS.ProcessEnv): BuiltChildEnv;
+/**
+ * Event emitted by {@link DownstreamConnection} when the supervisor observes
+ * a lifecycle transition worth surfacing. Consumers (the pool, the
+ * SESSION_BLOCKER tracker, observability sinks) subscribe via
+ * {@link DownstreamConnection.onSupervisorEvent}.
+ *
+ * The `kind` is a narrow closed set so sinks can switch exhaustively. `reason`
+ * carries the operator-readable detail; it is already bounded by
+ * `boundedDiagnosticString` at the call site.
+ */
+export type DownstreamSupervisorEvent = {
+    kind: 'child_died_unexpectedly';
+    server: string;
+    reason: string;
+} | {
+    kind: 'respawned';
+    server: string;
+} | {
+    /**
+     * A non-transition health change. Fires whenever a visible field in
+     * {@link DownstreamHealth} (health, last_error, tools_count) mutates
+     * WITHOUT being accompanied by a breaker transition or respawn event.
+     *
+     * Codex 0.9.0 pass-2 P2a: without this event, the first failed call/
+     * reconnect below the breaker threshold (or a successful `listTools`
+     * that updates the cached tool count) never reaches the live state
+     * publisher, so `rea status` would show stale data until some later,
+     * unrelated circuit or respawn event finally flushed a snapshot.
+     *
+     * Firing is best-effort from the connection class; the pool additionally
+     * emits this kind after `listAllTools` updates `lastToolsCount` so a
+     * tool-catalog change is always visible in the next debounced snapshot.
+     */
+    kind: 'health_changed';
+    server: string;
+};
 export declare class DownstreamConnection {
     #private;
     private readonly config;
@@ -84,6 +140,29 @@ export declare class DownstreamConnection {
      */
     private readonly logger?;
     private client;
+    /**
+     * Handle to the currently active transport, so our `onclose`/`onerror`
+     * hooks can tell "this is the transport we care about" vs "a stale callback
+     * firing after we already swapped to a new transport". Cleared in `close()`
+     * BEFORE we invoke `client.close()` so our own tear-down does not race the
+     * supervisor path.
+     */
+    private activeTransport;
+    /**
+     * Set of transports currently being torn down by an in-flight `close()`.
+     * `onclose` / `onerror` callbacks that fire for a transport in this set
+     * must NOT be promoted to an "unexpected child death" — they are our own
+     * tear-down signal.
+     *
+     * Codex P2 (0.9.0 review): the earlier `closingIntentionally` boolean was
+     * connection-wide. Under concurrent calls, one call's `await this.close()`
+     * could overlap with another call's reconnect that had already installed
+     * a NEW transport. A genuine `onclose` from the new transport would hit
+     * the boolean guard and be silently ignored, reintroducing the stale-
+     * handle bug the patch targeted. Per-transport scoping eliminates the
+     * race: only the exact transport we asked to close is silenced.
+     */
+    private readonly closingTransports;
     /**
      * Whether a reconnect has already been attempted in the CURRENT failure
      * episode. Resets to `false` after a reconnect succeeds (so a later,
@@ -93,7 +172,30 @@ export declare class DownstreamConnection {
     private reconnectAttempted;
     /** Epoch ms of the last successful reconnect. Used by the flapping guard. */
     private lastReconnectAt;
+    /**
+     * Epoch ms of the most recent unexpected child-death event. Stamped by
+     * `handleUnexpectedClose()`. 0 means "never died unexpectedly".
+     *
+     * Codex 0.9.0 pass-5 P2b: when `handleUnexpectedClose` nulls `this.client`,
+     * the very next `callTool` takes the top-level `client === null` branch,
+     * which normally bypasses the flap-window check entirely (that check lives
+     * in the catch branch below, conditioned on `lastReconnectAt`). A downstream
+     * that crashes immediately after every spawn would therefore be respawned
+     * unconditionally on every incoming call — exactly the loop the flap
+     * window is supposed to suppress. Consulting this timestamp in the
+     * `client === null` branch lets us refuse the respawn when the previous
+     * death is within the flap window, and the caller gets a clear error
+     * instead of watching the child die again.
+     */
+    private unexpectedDeathAt;
     private health;
+    /**
+     * Optional supervisor-event listener. Set via
+     * {@link onSupervisorEvent}. A single subscriber is sufficient — the pool
+     * is the one consumer. Listener failures are swallowed; a broken consumer
+     * must never break the connection lifecycle.
+     */
+    private supervisorListener;
     constructor(config: RegistryServer, 
     /**
      * Optional structured logger (G5). When omitted, connection lifecycle
@@ -105,6 +207,61 @@ export declare class DownstreamConnection {
     get isHealthy(): boolean;
     /** True iff the underlying MCP client is currently connected. */
     get isConnected(): boolean;
+    /**
+     * Register a supervisor-event listener. Intended for the pool to wire up
+     * SESSION_BLOCKER tracking + observability hooks without the connection
+     * class having to know about either. Only one listener is supported — a
+     * second call replaces the first. Pass `null` to detach.
+     */
+    onSupervisorEvent(listener: ((event: DownstreamSupervisorEvent) => void) | null): void;
+    /**
+     * Invoke the supervisor listener if registered. Swallows listener errors —
+     * a broken observer must never break the connection state machine.
+     */
+    private emitSupervisorEvent;
+    /**
+     * Emit a `health_changed` event. Called from every site that mutates a
+     * health/last_error/tools_count-visible field WITHOUT firing one of the
+     * louder supervisor events (`child_died_unexpectedly` / `respawned`).
+     * Addresses Codex 0.9.0 pass-2 P2a — live-state was only scheduled from
+     * breaker transitions and respawns, so transient errors below the breaker
+     * threshold would leave `rea status` showing stale data.
+     */
+    private emitHealthChanged;
+    /**
+     * Handle an unexpected transport close. Fires when the child process exits
+     * outside a caller-initiated `close()`, or when the stdio pipe errors in a
+     * way the SDK surfaces as a close event.
+     *
+     * Contract:
+     *   - Only runs for the currently-active transport (stale callbacks from
+     *     an already-swapped transport are ignored).
+     *   - Does NOT run when WE initiated the close (the transport is a member
+     *     of `closingTransports` for the duration of our own `close()` call).
+     *   - Nulls `this.client` so the next `callTool` takes the `connect()`
+     *     branch and actually respawns the child.
+     *   - Marks the connection unhealthy so the pool knows not to route
+     *     traffic to it while we wait for the next call.
+     *   - Emits a `child_died_unexpectedly` supervisor event so the pool's
+     *     SESSION_BLOCKER tracker can count this even though no callTool has
+     *     failed yet (the child may die mid-idle).
+     */
+    private handleUnexpectedClose;
+    /**
+     * Handle a transport-layer protocol error. onerror does NOT always imply
+     * close — the SDK emits it for protocol errors too. We record the error
+     * text but leave connection invalidation to the eventual onclose callback,
+     * which is guaranteed to follow a fatal transport error on stdio.
+     *
+     * Codex 0.9.0 pass-6 P2: filter stale/intentional-close callbacks the
+     * same way `handleUnexpectedClose` does. Without this, a delayed
+     * onerror from a PREVIOUSLY-ACTIVE transport (one we've already torn
+     * down or replaced) can clobber the HEALTHY replacement connection's
+     * last_error and emit a spurious health_changed, leaving `rea status`
+     * showing a stale error on a perfectly live child. The `onclose`
+     * hook already enforced this filter; the `onerror` hook did not.
+     */
+    private handleTransportError;
     /**
      * Last error observed, or null if the connection has never failed (or fully
      * recovered).