@chorus-aidlc/chorus 0.14.2 → 0.14.3

package/cli/daemon-rest-client.mjs

@@ -16,6 +16,8 @@
 //                                                       startedAt|null }] }
 //   reportInterrupt  → POST /api/daemon/report-interrupt
 //                      { connectionUuid, entityType, entityUuid, reason }
+//   heartbeat        → POST /api/daemon/connection-heartbeat
+//                      { connectionUuid, connectedAt }
 //   readPendingTurns → GET  /api/daemon/pending-turns?connectionUuid=…
 //                      → { turns: [{ turnUuid, sessionId, directIdeaUuid, trigger,
 //                                    promptText }] }
@@ -72,6 +74,7 @@ const NOOP_LOGGER = { info() {}, warn() {}, error() {} };
  *   transcript: (p: { sessionId: string, messages: Array<{ role: string, text: string }> }) => Promise<DaemonRestResult>,
  *   executionState: (p: { executions: Array<Record<string, unknown>> }) => Promise<DaemonRestResult>,
  *   reportInterrupt: (p: { entityType: string, entityUuid: string, reason: string }) => Promise<DaemonRestResult>,
+ *   heartbeat: (p: { connectionUuid: string, connectedAt: string }) => Promise<DaemonRestResult>,
  *   readPendingTurns: () => Promise<DaemonRestResult>,
  * }}
  */
@@ -136,16 +139,25 @@ export function createDaemonRestClient(opts) {
      * server resolves the turn by the session BUSINESS KEY (`sessionId`); the optional
      * `entityType`/`entityUuid` stamp the weak executionUuid link. An `interrupted`
      * status may carry `interruptedReason` (`user`/`crash`/`shutdown` — the server
-     * rejects `offline`, which is its own reconcile verdict). Requires the
-     * connectionUuid (the server addresses the turn against a connection the agent owns).
+     * rejects `offline`, which is its own reconcile verdict). A terminal edge may also
+     * carry `transcriptRelayError` — the daemon-known reason its transcript upload
+     * finally failed (fix #444 follow-up), persisted as a turn annotation — and `usage`,
+     * the whole normalized per-turn TokenUsage object (daemon-token-usage), persisted as
+     * the turn's usage. Both ride the terminal edge only. Requires the connectionUuid (the
+     * server addresses the turn against a connection the agent owns).
      */
-    async turnAdvance({ sessionId, status, entityType, entityUuid, interruptedReason }) {
+    async turnAdvance({ sessionId, status, entityType, entityUuid, interruptedReason, transcriptRelayError, usage }) {
       const connectionUuid = getConnectionUuid();
       if (!connectionUuid) {
         const error = `cannot advance turn for session ${sessionId} → ${status} — no connection uuid yet`;
         logger.warn(`[Chorus] ${error}`);
         return { ok: false, status: null, error, skipped: true };
       }
+      // Per-turn token usage (daemon-token-usage) is meaningful only on a TERMINAL edge —
+      // the daemon knows it at subprocess exit, exactly like transcriptRelayError. Guard on
+      // the terminal status here so a stray usage on → running is never sent (the server
+      // ignores it there anyway; this keeps the wire honest).
+      const isTerminal = status === "ended" || status === "interrupted";
       const body = {
         connectionUuid,
         sessionId,
@@ -154,6 +166,11 @@ export function createDaemonRestClient(opts) {
         ...(entityType && entityUuid ? { entityType, entityUuid } : {}),
         // Only meaningful alongside status=interrupted; never sent otherwise.
         ...(status === "interrupted" && interruptedReason ? { interruptedReason } : {}),
+        // Transcript-relay failure annotation (fix #444 follow-up): only sent when the
+        // daemon actually knows the upload failed (a truthy reason on a terminal edge).
+        ...(transcriptRelayError ? { transcriptRelayError } : {}),
+        // The whole normalized TokenUsage object, nested under `usage`, only on a terminal edge.
+        ...(usage && isTerminal ? { usage } : {}),
       };
       return post(
         "turn-advance",
@@ -223,6 +240,20 @@ export function createDaemonRestClient(opts) {
       );
     },
 
+    /**
+     * POST /api/daemon/connection-heartbeat — acknowledge receipt of one SSE heartbeat.
+     * The values come directly from the active stream's registration handshake; unlike
+     * other connection-scoped operations this must not resolve the uuid lazily, because
+     * a delayed acknowledgment must retain its original generation fence.
+     */
+    async heartbeat({ connectionUuid, connectedAt }) {
+      return post(
+        "connection-heartbeat",
+        "/api/daemon/connection-heartbeat",
+        { connectionUuid, connectedAt },
+      );
+    },
+
     /**
      * GET /api/daemon/pending-turns?connectionUuid=… — read this connection's unstarted
      * (pending) turns from the turn table (the reconnect-backfill / deliver_turn source).

package/cli/daemon.mjs

@@ -43,6 +43,7 @@ import {
 import { WAKE_ACTIONS } from "./prompts.mjs";
 import { createInterruptReporter } from "./interrupt-reporter.mjs";
 import { createTurnReporter } from "./turn-reporter.mjs";
+import { createDaemonRestClient } from "./daemon-rest-client.mjs";
 import { createControlHandler } from "./control-handler.mjs";
 import { resolveSigintTimeoutMs, resolveDaemonCwds } from "./daemon-config.mjs";
 import {
@@ -205,6 +206,13 @@ export function buildDaemon(creds, deps = {}) {
     // doesn't matter. Per-connection so each path's wakes report against the right row.
     /** @type {{ connectionUuid: string|null }} */
     const connectionState = { connectionUuid: null };
+    const daemonRestClient = createDaemonRestClient({
+      url: creds.url,
+      apiKey: creds.apiKey,
+      getConnectionUuid: () => connectionState.connectionUuid,
+      logger,
+      fetchImpl: deps.fetchImpl,
+    });
 
     // Per-connection terminal-outcome bookkeeping (add-daemon-connection-conflict-skip).
     // `resolved` guards recordConnectionOutcome against double-counting a reconnect's
@@ -389,6 +397,7 @@ export function buildDaemon(creds, deps = {}) {
           recordConnectionOutcome(outcome, false);
         },
         onControl,
+        acknowledgeHeartbeat: (registration) => daemonRestClient.heartbeat(registration),
         onReconnect: backfill,
         logger,
       });

package/cli/sse-listener.mjs

@@ -29,6 +29,7 @@ import { fileURLToPath } from "node:url";
 
 const INITIAL_DELAY_MS = 1_000;
 const MAX_DELAY_MS = 30_000;
+const WATCHDOG_TIMEOUT_MS = 75_000;
 
 const NOOP_LOGGER = { info() {}, warn() {}, error() {} };
 
@@ -78,6 +79,8 @@ const PROCESS_STARTED_AT = new Date();
  *   The event carries `{ host, cwd }` of the conflicting identity.
  * @property {() => Promise<void>} [onReconnect]  Called after a reconnect so the
  *   caller can back-fill notifications missed during the gap.
+ * @property {(registration: {connectionUuid:string,connectedAt:string}) => Promise<unknown>} [acknowledgeHeartbeat]
+ *   Best-effort REST acknowledgment for a parsed heartbeat comment.
  * @property {string} [cwd]  The working directory THIS connection serves (T3 — 单
  *   daemon 多路径). A multi-path daemon runs one SseListener per declared path, each
  *   self-reporting its own cwd so the server registers them as distinct connections
@@ -88,6 +91,7 @@ const PROCESS_STARTED_AT = new Date();
  * @property {typeof fetch} [fetchImpl]  Injectable for tests.
  * @property {number} [initialDelayMs]
  * @property {number} [maxDelayMs]
+ * @property {number} [watchdogTimeoutMs] Injectable for deterministic tests only.
  */
 
 export class SseListener {
@@ -100,12 +104,14 @@ export class SseListener {
     this.onControl = opts.onControl ?? (() => {});
     this.onConflict = opts.onConflict ?? (() => {});
     this.onReconnect = opts.onReconnect ?? (async () => {});
+    this.acknowledgeHeartbeat = opts.acknowledgeHeartbeat ?? (async () => {});
     /** @type {string|null} The connection this stream registered as (once reported). */
     this.connectionUuid = null;
     this.logger = opts.logger ?? NOOP_LOGGER;
     this.fetchImpl = opts.fetchImpl ?? globalThis.fetch;
     this.initialDelayMs = opts.initialDelayMs ?? INITIAL_DELAY_MS;
     this.maxDelayMs = opts.maxDelayMs ?? MAX_DELAY_MS;
+    this.watchdogTimeoutMs = opts.watchdogTimeoutMs ?? WATCHDOG_TIMEOUT_MS;
 
     // Build the self-reporting endpoint URL once and reuse it across every
     // (re)connect, so the reconnect path always re-sends the same params.
@@ -128,6 +134,7 @@ export class SseListener {
       host: hostname(),
       cwd: this.cwd,
       startedAt: PROCESS_STARTED_AT.toISOString(),
+      livenessAck: "v1",
     });
     this.endpoint = `${this.url}/api/events/notifications?${params.toString()}`;
 
@@ -136,14 +143,20 @@ export class SseListener {
     this.abortController = null;
     /** @type {ReturnType<typeof setTimeout> | null} */
     this.reconnectTimer = null;
+    /** @type {ReturnType<typeof setTimeout> | null} */
+    this.watchdogTimer = null;
+    /** @type {{connectionUuid:string,connectedAt:string}|null} */
+    this.activeRegistration = null;
     this.reconnectDelay = this.initialDelayMs;
   }
 
   /** Open the SSE connection. On failure, schedules a backoff reconnect. */
   async connect() {
     this.#clearReconnectTimer();
+    this.#clearWatchdog();
     const abortController = new AbortController();
     this.abortController = abortController;
+    this.activeRegistration = null;
 
     let response;
     try {
@@ -173,6 +186,7 @@ export class SseListener {
     this.status = "connected";
     this.reconnectDelay = this.initialDelayMs;
     this.logger.info("[Chorus] SSE connection established");
+    this.#refreshWatchdog(abortController);
 
     if (wasReconnect) {
       try {
@@ -182,22 +196,25 @@ export class SseListener {
       }
     }
 
-    this.#consumeStream(response.body, abortController.signal);
+    this.#consumeStream(response.body, abortController);
   }
 
   /** Gracefully close the connection (no reconnect). */
   disconnect() {
     this.#clearReconnectTimer();
+    this.#clearWatchdog();
     if (this.abortController) {
       this.abortController.abort();
       this.abortController = null;
     }
     this.status = "disconnected";
+    this.activeRegistration = null;
     this.logger.info("[Chorus] SSE connection closed");
   }
 
-  /** @param {ReadableStream<Uint8Array>} body @param {AbortSignal} signal */
-  async #consumeStream(body, signal) {
+  /** @param {ReadableStream<Uint8Array>} body @param {AbortController} abortController */
+  async #consumeStream(body, abortController) {
+    const signal = abortController.signal;
     const decoder = new TextDecoder();
     const reader = body.getReader();
     let buffer = "";
@@ -205,6 +222,12 @@ export class SseListener {
       while (true) {
         const { done, value } = await reader.read();
         if (done || signal.aborted) break;
+        // Transport activity is byte arrival, not successful SSE parsing. Refresh before
+        // decode so comments, partial frames, and malformed payloads all prove the stream
+        // is still delivering bytes.
+        if (value && value.byteLength > 0) {
+          this.#refreshWatchdog(abortController);
+        }
         // Strip CR on ingest so CRLF transports (`\r\n\r\n` boundaries) parse the
         // same as LF — the `\n\n` boundary scan below would otherwise never match.
         buffer += decoder.decode(value, { stream: true }).replace(/\r/g, "");
@@ -219,6 +242,7 @@ export class SseListener {
       if (signal.aborted) return;
       this.logger.warn(`[Chorus] SSE stream error: ${err}`);
     } finally {
+      if (this.abortController === abortController) this.#clearWatchdog();
       try {
         reader.releaseLock();
       } catch {
@@ -235,7 +259,24 @@ export class SseListener {
   #processMessage(raw) {
     for (const line of raw.split("\n")) {
       // CR already stripped on ingest. Heartbeat / comment lines start with ":".
-      if (line.startsWith(":")) continue;
+      if (line.startsWith(":")) {
+        if (line.slice(1).trim() === "heartbeat" && this.activeRegistration) {
+          const registration = this.activeRegistration;
+          Promise.resolve()
+            .then(() => this.acknowledgeHeartbeat(registration))
+            .then((result) => {
+              if (result && typeof result === "object" && result.ok === false) {
+                // The REST client already logged the transport cause. Keep listener
+                // behavior best-effort and avoid adding a retry loop here.
+                return;
+              }
+            })
+            .catch((err) => {
+              this.logger.warn(`[Chorus] heartbeat acknowledgment failed: ${err}`);
+            });
+        }
+        continue;
+      }
       if (line.startsWith("data: ")) {
         const jsonStr = line.slice(6);
         let event;
@@ -248,8 +289,16 @@ export class SseListener {
         // The server's first data event tells us which DaemonConnection this
         // stream registered as. Capture it (for execution-state attribution) and
         // do NOT forward it as a notification — it isn't one.
-        if (event && event.type === "connection_registered" && typeof event.connectionUuid === "string") {
+        if (
+          event &&
+          event.type === "connection_registered" &&
+          typeof event.connectionUuid === "string"
+        ) {
           this.connectionUuid = event.connectionUuid;
+          this.activeRegistration =
+            typeof event.connectedAt === "string"
+              ? { connectionUuid: event.connectionUuid, connectedAt: event.connectedAt }
+              : null;
           try {
             this.onConnectionId(event.connectionUuid);
           } catch (err) {
@@ -305,4 +354,32 @@ export class SseListener {
       this.reconnectTimer = null;
     }
   }
+
+  /** @param {AbortController|null} abortController */
+  #refreshWatchdog(abortController) {
+    this.#clearWatchdog();
+    if (!abortController || abortController.signal.aborted || this.abortController !== abortController) return;
+    this.watchdogTimer = setTimeout(() => {
+      this.watchdogTimer = null;
+      if (
+        this.abortController !== abortController ||
+        abortController.signal.aborted ||
+        this.status !== "connected"
+      ) {
+        return;
+      }
+      this.logger.warn(
+        `[Chorus] SSE stream received no bytes for ${this.watchdogTimeoutMs}ms; reconnecting`
+      );
+      abortController.abort();
+      this.#scheduleReconnect();
+    }, this.watchdogTimeoutMs);
+  }
+
+  #clearWatchdog() {
+    if (this.watchdogTimer !== null) {
+      clearTimeout(this.watchdogTimer);
+      this.watchdogTimer = null;
+    }
+  }
 }

package/cli/turn-reporter.mjs

@@ -46,7 +46,7 @@ export const TURN_INTERRUPT_REASONS = new Set(["user", "crash", "shutdown"]);
  *   logger?: { info(m:string):void, warn(m:string):void, error(m:string):void },
  *   fetchImpl?: typeof fetch,             Injectable for tests.
  * }} opts
- * @returns {(params: { sessionId: string, status: "running"|"ended"|"interrupted", entityType?: string|null, entityUuid?: string|null, interruptedReason?: "user"|"crash"|"shutdown"|null }) => Promise<void>}
+ * @returns {(params: { sessionId: string, status: "running"|"ended"|"interrupted", entityType?: string|null, entityUuid?: string|null, interruptedReason?: "user"|"crash"|"shutdown"|null, transcriptRelayError?: string|null, usage?: import("./upload-hooks.mjs").TokenUsage|null }) => Promise<void>}
  */
 export function createTurnReporter(opts) {
   const logger = opts.logger ?? NOOP_LOGGER;
@@ -67,6 +67,8 @@ export function createTurnReporter(opts) {
     entityType,
     entityUuid,
     interruptedReason,
+    transcriptRelayError,
+    usage,
   }) {
     if (typeof sessionId !== "string" || !sessionId || !TURN_STATUSES.has(status)) {
       logger.warn(
@@ -88,9 +90,19 @@ export function createTurnReporter(opts) {
     }
     // The client resolves the connectionUuid lazily, validates it (logging
     // "no connection uuid yet" when absent), POSTs the exact server payload — sending
-    // entityType/entityUuid only when both are present — and logs the failure cause on a
-    // network error / non-2xx. We swallow the structured result so a failed report can
-    // never crash the wake path.
-    await client.turnAdvance({ sessionId, status, entityType, entityUuid, interruptedReason });
+    // entityType/entityUuid only when both are present, and transcriptRelayError only when
+    // truthy (fix #444 follow-up) — and logs the failure cause on a network error / non-2xx.
+    // We swallow the structured result so a failed report can never crash the wake path.
+    await client.turnAdvance({
+      sessionId,
+      status,
+      entityType,
+      entityUuid,
+      interruptedReason,
+      transcriptRelayError,
+      // Per-turn token usage (daemon-token-usage): the client spreads it into the body
+      // only on a terminal edge. No validation here — usage is an opaque nested object.
+      usage,
+    });
   };
 }