@animalabs/agent-framework 0.5.3 → 0.5.6

package/src/framework.ts

@@ -1,6 +1,7 @@
 import { join } from 'node:path';
 import { JsStore } from '@animalabs/chronicle';
 import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolResult as MembraneToolResult } from '@animalabs/membrane';
+import { MembraneError } from '@animalabs/membrane';
 import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager';
 import type {
   MessageId,
@@ -108,13 +109,28 @@ class DefaultInferencePolicy implements InferencePolicy {
 
 /**
  * Default error policy - retry with exponential backoff.
+ *
+ * Respects MembraneError.retryable: non-retryable errors (400 invalid_request,
+ * 401 auth, context_length, safety) are terminal on attempt 0. Retrying them
+ * burns API quota without any chance of success — the payload doesn't change
+ * between attempts. Production traces showed clerk + reviewer wasting 4
+ * inferences on each 400 due to blind retries.
+ *
+ * For retryable errors, honors retryAfterMs when present (rate limits),
+ * otherwise falls back to exponential backoff capped at maxRetries.
  */
 class DefaultErrorPolicy implements ErrorPolicy {
   maxRetries = 3;
 
   onInferenceError(error: Error, _agentName: string, attempt: number): ErrorAction {
+    if (error instanceof MembraneError && !error.retryable) {
+      return { retry: false };
+    }
     if (attempt < this.maxRetries) {
-      return { retry: true, delayMs: Math.pow(2, attempt) * 1000 };
+      const delayMs = error instanceof MembraneError && error.retryAfterMs !== undefined
+        ? error.retryAfterMs
+        : Math.pow(2, attempt) * 1000;
+      return { retry: true, delayMs };
     }
     return { retry: false };
   }
@@ -680,13 +696,22 @@ export class AgentFramework {
       let toolCallsCount = 0;
       let settled = false;
       let inferenceStarted = false;
+      let lastActivity = Date.now();
 
       const STARTUP_TIMEOUT_MS = 30_000;
+      // After inference has started, give it 15 minutes of activity-bounded
+      // life. Each addressed trace event refreshes the deadline; only
+      // sustained silence trips it. Subagents that legitimately stream for
+      // 10+ minutes (e.g. long fork investigations) get the extra slack but
+      // stalled streams that emit `inference:completed` and then go quiet —
+      // the exact zombie shape we saw in production — are caught here.
+      const COMPLETION_IDLE_TIMEOUT_MS = 15 * 60_000;
 
       const cleanup = () => {
         if (settled) return;
         settled = true;
         clearTimeout(startupWatchdog);
+        clearInterval(completionWatchdog);
         this.offTrace(traceListener);
         this.agents.delete(agent.name);
       };
@@ -705,12 +730,34 @@ export class AgentFramework {
         }
       }, STARTUP_TIMEOUT_MS);
 
+      // Completion watchdog: once inference has started, reject if no trace
+      // activity for COMPLETION_IDLE_TIMEOUT_MS. Polls every 30s. Without
+      // this, an ephemeral that loses its terminal event (e.g. due to a
+      // future ordering regression or an upstream stream drop) hangs forever
+      // and keeps its concurrency slot pinned.
+      const completionWatchdog = setInterval(() => {
+        if (settled || !inferenceStarted) return;
+        const idle = Date.now() - lastActivity;
+        if (idle > COMPLETION_IDLE_TIMEOUT_MS) {
+          cleanup();
+          reject(new Error(
+            `Ephemeral agent "${agent.name}" stalled: no trace activity for ` +
+            `${Math.round(idle / 1000)}s after inference started (threshold ` +
+            `${Math.round(COMPLETION_IDLE_TIMEOUT_MS / 1000)}s). Stream likely ` +
+            `dropped or terminal event was lost.`
+          ));
+        }
+      }, 30_000);
+
       const traceListener = (event: TraceEvent) => {
         if (settled) return;
         // Only track events for our ephemeral agent
         const agentName = 'agentName' in event ? (event as { agentName: string }).agentName : null;
         if (agentName !== agent.name) return;
 
+        // Any addressed event counts as liveness for the completion watchdog.
+        lastActivity = Date.now();
+
         switch (event.type) {
           case 'inference:started':
             inferenceStarted = true;
@@ -732,13 +779,23 @@ export class AgentFramework {
             speech = '';
             break;
           case 'inference:completed': {
-            // Check if agent is idle (no pending tool calls = truly done)
-            // If tools were called, the agent will cycle through waiting_for_tools → ready → streaming
-            // and eventually complete again. We only resolve when there are no more tool calls.
-            if (agent.state.status === 'idle') {
-              cleanup();
-              resolve({ speech, toolCallsCount });
-            }
+            // `inference:completed` is terminal — driveStream only emits it
+            // from `case 'complete'`, which fires after the full tool
+            // round-trip has finished and the model has stopped. Intermediate
+            // tool-cycle transitions use `inference:stream_resumed`, not
+            // `inference:completed`. So we resolve unconditionally here.
+            //
+            // Previously this branch gated on `agent.state.status === 'idle'`,
+            // which was correct in intent but order-fragile: the framework
+            // emitted `inference:completed` BEFORE calling `agent.reset()`,
+            // so synchronous listeners observed status === 'streaming' at
+            // the trace boundary. The gate failed silently and the promise
+            // hung forever — every production zombie subagent originated
+            // here. The reset+emit order has been fixed in driveStream's
+            // `case 'complete'` handler; the gate is dropped here as
+            // belt-and-suspenders.
+            cleanup();
+            resolve({ speech, toolCallsCount });
             break;
           }
           case 'inference:turn_ended': {
@@ -1770,8 +1827,22 @@ export class AgentFramework {
               type: 'inference:tokens',
               agentName: agent.name,
               content: event.content,
+              blockType: event.meta.type,
+              blockIndex: event.meta.blockIndex,
+            });
+            break;
+
+          case 'block': {
+            const { event: phase, index, block } = event.event;
+            this.emitTrace({
+              type: 'inference:content_block',
+              agentName: agent.name,
+              phase,
+              blockType: block.type,
+              blockIndex: index,
             });
             break;
+          }
 
           case 'tool-calls': {
             hadToolCalls = true;
@@ -1920,6 +1991,19 @@ export class AgentFramework {
                   cacheRead: du.cacheReadTokens,
                 }
               : undefined;
+
+            // Reset agent state BEFORE emitting inference:completed so that
+            // synchronous listeners (e.g. runEphemeralToCompletion) observe
+            // status === 'idle' the moment the event fires. Previously
+            // agent.reset() ran after `await dispatchSpeech`, which meant the
+            // ephemeral-completion listener saw status === 'streaming' at the
+            // trace boundary, failed its idle gate, and the promise hung
+            // forever — every "zombie subagent" in production tracked back
+            // here. Speech dispatch happens after but doesn't depend on the
+            // status field.
+            agent.reset();
+            this.eventGate?.onInferenceEnded(agent.name);
+
             this.emitTrace({
               type: 'inference:completed',
               agentName: agent.name,
@@ -1968,9 +2052,6 @@ export class AgentFramework {
               );
             }
 
-            // Done — reset to idle
-            agent.reset();
-            this.eventGate?.onInferenceEnded(agent.name);
             break;
           }
 

package/src/modules/health/index.ts

@@ -0,0 +1,275 @@
+/**
+ * HealthModule — minimal framework self-introspection.
+ *
+ * Exposes a single tool, `health--snapshot`, that returns a structured
+ * snapshot of recent framework activity. Useful when an agent gets a
+ * concurrency timeout, a 400 from the API, or wants to verify it's not
+ * silently failing.
+ *
+ * Sources:
+ *  - `framework/inference-log` (queryInferenceLogs) — recent inference
+ *    success/error counts, last N errors, per-agent token totals.
+ *  - `modules/subagent/state` — best-effort. The slot is owned by
+ *    `connectome-host`'s SubagentModule, not by this module. We read it
+ *    directly from the chronicle store so the tool works even when the
+ *    HealthModule has no awareness of SubagentModule's internals.
+ *  - Registered module names.
+ *
+ * Kept deliberately read-only: no side effects, no kill switches.
+ * Reaping/cancellation lives in the SubagentModule.
+ */
+
+import type { JsStore } from '@animalabs/chronicle';
+import type { Module, ModuleContext } from '../../types/module.js';
+import type { ToolDefinition, ToolCall, ToolResult, ProcessEvent } from '../../types/events.js';
+import type { EventResponse, ProcessState } from '../../types/module.js';
+import type { AgentFramework } from '../../framework.js';
+
+export interface HealthModuleConfig {
+  /**
+   * Default number of recent inferences to summarize when the tool is
+   * called without an explicit limit. Default: 20.
+   */
+  defaultLookback?: number;
+
+  /**
+   * State slot to read for the subagent registry. Allows hosts that name
+   * their subagent module differently to point this at the right slot.
+   * Default: `modules/subagent/state`.
+   */
+  subagentStateId?: string;
+}
+
+interface SnapshotInput {
+  /** How many recent inferences to summarize. Default config.defaultLookback. */
+  lookback?: number;
+  /** Include per-agent token totals across all-time. Default true. */
+  includeTokens?: boolean;
+  /** Include subagent registry summary. Default true. */
+  includeSubagents?: boolean;
+}
+
+interface PersistedSubagentRecord {
+  name: string;
+  type?: string;
+  status?: string;
+  startedAt?: number;
+  lastActivityAt?: number;
+  completedAt?: number;
+  toolCallsCount?: number;
+  statusMessage?: string;
+}
+
+export class HealthModule implements Module {
+  readonly name = 'health';
+
+  private ctx: ModuleContext | null = null;
+  private framework: AgentFramework | null = null;
+  private store: JsStore | null = null;
+  private config: Required<HealthModuleConfig>;
+
+  constructor(config: HealthModuleConfig = {}) {
+    this.config = {
+      defaultLookback: config.defaultLookback ?? 20,
+      subagentStateId: config.subagentStateId ?? 'modules/subagent/state',
+    };
+  }
+
+  /**
+   * Wire framework + store. Host calls this after AgentFramework.create()
+   * so the module can query inference logs and chronicle state.
+   */
+  bind(framework: AgentFramework, store: JsStore): void {
+    this.framework = framework;
+    this.store = store;
+  }
+
+  async start(ctx: ModuleContext): Promise<void> {
+    this.ctx = ctx;
+  }
+
+  async stop(): Promise<void> {
+    this.ctx = null;
+  }
+
+  getTools(): ToolDefinition[] {
+    return [
+      {
+        name: 'snapshot',
+        description:
+          'Return a structured snapshot of framework health: recent inference success/error counts, last errors, per-agent token totals, active subagent registry, and registered modules. Read-only. Useful when diagnosing why a spawn timed out, why an inference failed, or whether the framework is silently degraded.',
+        inputSchema: {
+          type: 'object' as const,
+          properties: {
+            lookback: {
+              type: 'number',
+              description: `How many recent inferences to summarize (default ${this.config.defaultLookback}).`,
+            },
+            includeTokens: {
+              type: 'boolean',
+              description: 'Include per-agent token totals (default true).',
+            },
+            includeSubagents: {
+              type: 'boolean',
+              description: 'Include subagent registry summary (default true).',
+            },
+          },
+        },
+      },
+    ];
+  }
+
+  async handleToolCall(call: ToolCall): Promise<ToolResult> {
+    if (call.name !== 'snapshot') {
+      return { success: false, isError: true, error: `Unknown tool: ${call.name}` };
+    }
+    const input = (call.input ?? {}) as SnapshotInput;
+    const lookback = input.lookback ?? this.config.defaultLookback;
+    const includeTokens = input.includeTokens ?? true;
+    const includeSubagents = input.includeSubagents ?? true;
+
+    try {
+      const snapshot = this.buildSnapshot(lookback, includeTokens, includeSubagents);
+      return { success: true, data: snapshot };
+    } catch (error) {
+      return {
+        success: false,
+        isError: true,
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+
+  async onProcess(_event: ProcessEvent, _state: ProcessState): Promise<EventResponse> {
+    return {};
+  }
+
+  private buildSnapshot(
+    lookback: number,
+    includeTokens: boolean,
+    includeSubagents: boolean,
+  ): Record<string, unknown> {
+    if (!this.framework || !this.store) {
+      throw new Error(
+        'HealthModule not bound — host must call bind(framework, store) before tool dispatch.',
+      );
+    }
+
+    // ── Inference log summary ───────────────────────────────────────────
+    // queryInferenceLogs returns InferenceLogEntryWithId. Prefer the
+    // `summary` field (no blob deref) but fall back to `entry` for older
+    // records or if the summarizer wasn't populated.
+    const recent = this.framework.queryInferenceLogs({ limit: lookback });
+    const recentEntries = recent.entries ?? [];
+
+    const projectSummary = (e: typeof recentEntries[number]) => {
+      const s = e.summary ?? e.entry;
+      return {
+        timestamp: s.timestamp,
+        agentName: s.agentName,
+        success: s.success,
+        error: s.error,
+        requestId: s.requestId,
+        tokenUsage: s.tokenUsage,
+      };
+    };
+
+    const recentErrors = recentEntries
+      .map(projectSummary)
+      .filter((s) => s.success === false)
+      .slice(0, 10)
+      .map((s) => ({
+        timestamp: s.timestamp,
+        agentName: s.agentName,
+        error: typeof s.error === 'string' ? s.error.slice(0, 400) : s.error,
+        requestId: s.requestId,
+      }));
+
+    let tokenTotals: Record<string, { input: number; output: number; cacheRead: number; inferences: number }> | undefined;
+    if (includeTokens) {
+      // Pull a larger window for totals — defaults to 1000 entries which is
+      // enough to characterize ongoing usage without dragging the entire log.
+      const wide = this.framework.queryInferenceLogs({ limit: 1000 });
+      tokenTotals = {};
+      for (const raw of wide.entries ?? []) {
+        const s = projectSummary(raw);
+        const agent = s.agentName ?? 'unknown';
+        const t = tokenTotals[agent] ?? { input: 0, output: 0, cacheRead: 0, inferences: 0 };
+        t.inferences++;
+        const u = s.tokenUsage;
+        if (u) {
+          t.input += u.input ?? 0;
+          t.output += u.output ?? 0;
+          t.cacheRead += u.cacheRead ?? 0;
+        }
+        tokenTotals[agent] = t;
+      }
+    }
+
+    // ── Subagent registry summary (best-effort) ────────────────────────
+    let subagents: Array<Record<string, unknown>> | string | undefined;
+    if (includeSubagents) {
+      try {
+        const raw = this.store.getStateJson(this.config.subagentStateId);
+        if (raw) {
+          const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw;
+          const agents = (parsed as { agents?: Record<string, PersistedSubagentRecord> })?.agents
+            ?? {};
+          const now = Date.now();
+          subagents = Object.entries(agents).map(([key, sa]) => {
+            const startedAt = sa.startedAt ?? 0;
+            const lastActivityAt = sa.lastActivityAt ?? startedAt;
+            return {
+              key,
+              name: sa.name,
+              type: sa.type,
+              status: sa.status,
+              startedAt: startedAt > 0 ? new Date(startedAt).toISOString() : null,
+              lastActivityAt: lastActivityAt > 0 ? new Date(lastActivityAt).toISOString() : null,
+              runtimeSeconds: startedAt > 0 ? Math.floor((now - startedAt) / 1000) : null,
+              silentSeconds: lastActivityAt > 0 ? Math.floor((now - lastActivityAt) / 1000) : null,
+              toolCallsCount: sa.toolCallsCount,
+              statusMessage: sa.statusMessage,
+            };
+          });
+        } else {
+          subagents = 'no subagent state registered';
+        }
+      } catch (error) {
+        subagents = `error reading subagent state: ${error instanceof Error ? error.message : String(error)}`;
+      }
+    }
+
+    // ── Module list ────────────────────────────────────────────────────
+    const modules = this.ctx ? this.listModulesViaContext() : [];
+
+    return {
+      timestamp: new Date().toISOString(),
+      window: {
+        lookback,
+        inferencesInWindow: recentEntries.length,
+      },
+      inferences: {
+        successCount: recentEntries.filter((e) => projectSummary(e).success).length,
+        errorCount: recentEntries.filter((e) => projectSummary(e).success === false).length,
+        recentErrors,
+      },
+      tokenTotalsByAgent: tokenTotals,
+      subagents,
+      modules,
+    };
+  }
+
+  private listModulesViaContext(): string[] {
+    // ModuleContext.getActiveTools returns flat tool names; derive module set
+    // from the prefix before '--'. This avoids needing a dedicated module-list
+    // accessor on ModuleContext (kept narrow on purpose).
+    const tools = this.ctx?.getActiveTools() ?? [];
+    const set = new Set<string>();
+    for (const t of tools) {
+      const i = t.name.indexOf('--');
+      if (i > 0) set.add(t.name.slice(0, i));
+    }
+    return [...set].sort();
+  }
+}

package/src/modules/index.ts

@@ -5,6 +5,9 @@
 export { ApiModule } from './api/index.js';
 export type { ApiEvent } from './api/index.js';
 
+export { HealthModule } from './health/index.js';
+export type { HealthModuleConfig } from './health/index.js';
+
 export { WorkspaceModule } from './workspace/index.js';
 export type {
   WorkspaceConfig,

package/src/types/trace.ts

@@ -67,6 +67,27 @@ export type TraceEvent =
       type: 'inference:tokens';
       agentName: string;
       content: string;
+      /**
+       * Which membrane block produced this chunk. Carried verbatim from the
+       * membrane's ChunkMeta. Consumers that want to render thinking content
+       * distinctly from regular text should switch on this.
+       */
+      blockType: 'text' | 'thinking' | 'tool_call' | 'tool_result';
+      /** 0-indexed block position in the current assistant turn. */
+      blockIndex: number;
+    })
+  | (TraceEventBase & {
+      /**
+       * Structural boundary inside the assistant turn — emitted on every
+       * block_start / block_complete from the membrane. Use this to switch
+       * render lanes (e.g. open a dim "thinking" element on block_start
+       * with blockType='thinking', close it on block_complete).
+       */
+      type: 'inference:content_block';
+      agentName: string;
+      phase: 'block_start' | 'block_complete';
+      blockType: 'text' | 'thinking' | 'tool_call' | 'tool_result';
+      blockIndex: number;
     })
   | (TraceEventBase & {
       type: 'inference:tool_calls_yielded';