@animalabs/agent-framework 0.5.7 → 0.6.2

package/src/runtime/liveness-watchdog-worker.ts

@@ -0,0 +1,66 @@
+/**
+ * Liveness watchdog worker — runs on its own thread, so it keeps ticking even
+ * when the main thread is wedged. Watches the shared heartbeat; if it goes stale
+ * past the threshold, writes a diagnostic and kills the whole process.
+ *
+ * Killing from a worker: process.kill(process.pid, signal) signals the whole
+ * process. SIGABRT → core dump (if enabled) + terminate; SIGKILL → immediate.
+ * Catchable signals (SIGTERM) are avoided — their handler would run on the
+ * wedged main thread and never fire.
+ */
+import { workerData } from 'node:worker_threads';
+import { writeFileSync } from 'node:fs';
+
+interface Data {
+  sab: SharedArrayBuffer;
+  thresholdMs: number;
+  checkMs: number;
+  action: 'abort' | 'exit';
+  reportPath: string | null;
+}
+const { sab, thresholdMs, checkMs, action, reportPath } = workerData as Data;
+
+const hb = new Float64Array(sab, 0, 1);
+const phaseBuf = new Uint8Array(sab, 16, 48);
+const dec = new TextDecoder();
+
+function currentPhase(): string {
+  return dec.decode(phaseBuf).replace(/\0+$/, '') || 'unknown';
+}
+
+function check(): void {
+  const last = hb[0];
+  const age = Date.now() - last;
+  if (last > 0 && age > thresholdMs) {
+    const phase = currentPhase();
+    const report = {
+      ts: new Date().toISOString(),
+      event: 'main-thread-wedged',
+      ageMs: Math.round(age),
+      thresholdMs,
+      phase,
+      action,
+      pid: process.pid,
+    };
+    const line = `[liveness-watchdog] main thread unresponsive ${Math.round(age)}ms ` +
+      `(phase=${phase}) — failing hard via ${action}`;
+    try {
+      console.error(line);
+    } catch {
+      /* ignore */
+    }
+    if (reportPath) {
+      try {
+        writeFileSync(reportPath, JSON.stringify(report) + '\n', { flag: 'a' });
+      } catch {
+        /* best effort */
+      }
+    }
+    // Kill the whole process. The supervisor restarts a fresh one.
+    process.kill(process.pid, action === 'abort' ? 'SIGABRT' : 'SIGKILL');
+    return; // process is dying
+  }
+  setTimeout(check, checkMs);
+}
+
+setTimeout(check, checkMs);

package/src/runtime/liveness-watchdog.ts

@@ -0,0 +1,122 @@
+/**
+ * LivenessWatchdog — detects a wedged (unresponsive) main thread and fails hard.
+ *
+ * The agent's event loop is single-threaded. A synchronous infinite loop or a
+ * self-rescheduling microtask flood (e.g. a pathological compression drain) can
+ * peg the thread for minutes: no events, no heartbeat, no recovery — the agent
+ * goes silently deaf. A same-thread timer can't catch this (it can't fire while
+ * the loop is blocked), so detection must live off-thread.
+ *
+ * Mechanism: the main thread stamps a heartbeat (and an optional phase label)
+ * into shared memory on a timer. A worker thread — which keeps running even when
+ * main is wedged — watches the heartbeat; if it goes stale past a threshold it
+ * writes a diagnostic and kills the whole process:
+ *   - 'abort' → SIGABRT (core dump if enabled, for an offline stack) then dies
+ *   - 'exit'  → SIGKILL (clean, quiet; rely on the supervisor to restart)
+ * Either way the supervisor (launchd/systemd) restarts a fresh process instead
+ * of leaving a deaf one running.
+ */
+import { Worker } from 'node:worker_threads';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { phaseChannel } from '@animalabs/context-manager';
+
+const WORKER_PATH = join(dirname(fileURLToPath(import.meta.url)), 'liveness-watchdog-worker.js');
+
+const HEARTBEAT_OFFSET = 0; // Float64: last heartbeat (ms since epoch)
+const PHASE_OFFSET = 16; // Uint8[48]: current phase label (UTF-8)
+const PHASE_LEN = 48;
+const SAB_SIZE = 64;
+
+export interface WatchdogOptions {
+  /** Wedge threshold: kill if no heartbeat for this long. Default 30000ms. */
+  thresholdMs?: number;
+  /** Heartbeat cadence. Default 1000ms. */
+  heartbeatMs?: number;
+  /** 'abort' → SIGABRT (core dump for debugging); 'exit' → SIGKILL. Default 'abort'. */
+  action?: 'abort' | 'exit';
+  /** Optional file the worker writes a JSON wedge report to before killing. */
+  reportPath?: string;
+  /** Disable entirely. */
+  enabled?: boolean;
+}
+
+export class LivenessWatchdog {
+  private readonly sab = new SharedArrayBuffer(SAB_SIZE);
+  private readonly hb = new Float64Array(this.sab, HEARTBEAT_OFFSET, 1);
+  private readonly phaseBuf = new Uint8Array(this.sab, PHASE_OFFSET, PHASE_LEN);
+  private readonly enc = new TextEncoder();
+  private timer?: ReturnType<typeof setInterval>;
+  private worker?: Worker;
+  private started = false;
+
+  constructor(private readonly opts: WatchdogOptions = {}) {}
+
+  start(): void {
+    if (this.started || this.opts.enabled === false) return;
+    this.started = true;
+    const heartbeatMs = this.opts.heartbeatMs ?? 1000;
+    this.setPhase('starting');
+    this.beat();
+    this.timer = setInterval(() => this.beat(), heartbeatMs);
+    this.timer.unref?.();
+    try {
+      this.worker = new Worker(WORKER_PATH, {
+        workerData: {
+          sab: this.sab,
+          thresholdMs: this.opts.thresholdMs ?? 30_000,
+          checkMs: Math.min(heartbeatMs, 2000),
+          action: this.opts.action ?? 'abort',
+          reportPath: this.opts.reportPath ?? null,
+        },
+      });
+      this.worker.unref();
+      this.worker.on('error', (e) => console.error('[liveness-watchdog] worker error:', e.message));
+      // Wire the phase channel so labelled synchronous spans (compression,
+      // context-build, merge-graph walks) name themselves in a wedge report.
+      phaseChannel.report = (label: string) => this.setPhase(label);
+      console.error(
+        `[liveness-watchdog] armed: threshold=${this.opts.thresholdMs ?? 30_000}ms ` +
+          `action=${this.opts.action ?? 'abort'}`,
+      );
+    } catch (e) {
+      console.error('[liveness-watchdog] failed to start:', (e as Error).message);
+    }
+    this.setPhase('idle');
+  }
+
+  /** Stamp liveness. Called on a timer; the worker watches for staleness. */
+  private beat(): void {
+    this.hb[0] = Date.now();
+  }
+
+  /**
+   * Label the operation the main thread is about to run synchronously, so a
+   * wedge report names it (e.g. 'compression-tick', 'context-build'). Keep it
+   * short (truncated to 48 bytes). Reset to 'idle' when done.
+   */
+  setPhase(label: string): void {
+    this.phaseBuf.fill(0);
+    const bytes = this.enc.encode(label);
+    this.phaseBuf.set(bytes.subarray(0, PHASE_LEN));
+  }
+
+  /** Run a synchronous section under a named phase, restoring the prior phase. */
+  withPhase<T>(label: string, fn: () => T): T {
+    this.setPhase(label);
+    try {
+      return fn();
+    } finally {
+      this.setPhase('idle');
+    }
+  }
+
+  stop(): void {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = undefined;
+    phaseChannel.report = () => {};
+    void this.worker?.terminate();
+    this.worker = undefined;
+    this.started = false;
+  }
+}

package/src/types/agent.ts

@@ -45,6 +45,22 @@ export interface AgentConfig {
   /** Per-agent context compile budget (input tokens). When unset, the
    *  ContextManager's built-in default (100k) applies. */
   contextBudgetTokens?: number;
+
+  /**
+   * Prompt-cache TTL forwarded to the provider (Anthropic `cache_control.ttl`).
+   * '5m' (provider default) or '1h'. When unset, Membrane's default applies.
+   *
+   * Why you'd set '1h': for persistent agents whose conversational cadence is
+   * slower than 5 minutes, the default TTL expires between turns and the full
+   * context is re-WRITTEN to cache on nearly every call. Cache writes carry a
+   * premium (1.25x base input for 5m, 2x for 1h) while reads cost 0.1x — so a
+   * chatty-but-not-rapid agent pays the write premium over and over. With '1h'
+   * the write happens once per idle-hour and subsequent turns hit cache reads;
+   * in practice cache writes can dominate spend for slow-cadence agents.
+   * Keep '5m' (or unset) for high-frequency loops with sub-5-minute cadence,
+   * where the cheaper write premium wins.
+   */
+  cacheTtl?: '5m' | '1h';
   /**
    * Extended thinking config. When `enabled: true`, the agent runs with
    * Anthropic's native extended thinking; responses include `thinking` blocks
@@ -63,6 +79,28 @@ export interface AgentConfig {
      */
     display?: 'summarized' | 'omitted';
   };
+  /**
+   * How to handle a model content-policy refusal (`stop_reason: refusal`).
+   * When `autoRewind` is on, a refused turn triggers an automatic rewind: the
+   * framework redacts the triggering turn (the tool result or message that
+   * tripped the classifier), injects a metadata-only marker in its place (which
+   * carries none of the offending content, so it cannot itself re-trip), and
+   * re-runs — up to `maxRewinds` times before giving up. This keeps the agent
+   * on its own model (no fallback-model substitution) while self-healing around
+   * a poison turn. Default: off (a refusal just surfaces a marker + reaction).
+   */
+  refusalHandling?: {
+    /** Auto-rewind the triggering turn on refusal and retry. Default false. */
+    autoRewind?: boolean;
+    /** Max consecutive rewinds before giving up a turn. Default 3. */
+    maxRewinds?: number;
+    /**
+     * When the rewound turn is a *human* message (not a machine tool result),
+     * announce the withholding on the conversational surface (Discord) rather
+     * than dropping it silently. Default true.
+     */
+    announceHumanTurns?: boolean;
+  };
 }
 
 /**
@@ -138,4 +176,14 @@ export interface InferenceRequest {
   reason: string;
   source: string;
   timestamp: number;
+  /**
+   * The MCPL channel whose message triggered this inference, if any (composite
+   * id, e.g. `discord:guild:channel` / `discord:dm:id`). The framework routes
+   * the turn's auto-published plain-text speech here so a single TRUNK agent
+   * replies in the channel it is answering — not the process-global most-recent-
+   * inbound locus, which a concurrent message elsewhere can hijack (item-3
+   * redux). Undefined for non-channel wakes (heartbeat, timers, module events),
+   * which correctly fall back to the global default channel.
+   */
+  channelId?: string;
 }

package/src/types/framework.ts

@@ -68,6 +68,21 @@ export interface FrameworkConfig {
   /** EventGate config. If omitted, all events trigger inference (unchanged default). */
   gate?: GateOptions;
 
+  /**
+   * Liveness watchdog: fail hard (and let the supervisor restart) if the main
+   * thread wedges — a synchronous infinite loop or microtask flood that would
+   * otherwise leave the agent silently deaf. Off unless `enabled` is set.
+   */
+  watchdog?: {
+    enabled?: boolean;
+    /** Kill if no heartbeat for this long (ms). Default 30000. */
+    thresholdMs?: number;
+    /** 'abort' → SIGABRT (core dump for debugging); 'exit' → SIGKILL. Default 'abort'. */
+    action?: 'abort' | 'exit';
+    /** Optional path for a JSON wedge report written before the kill. */
+    reportPath?: string;
+  };
+
   /**
    * Per-channel conversation routing. When set, incoming MCPL channel
    * messages route to per-channel fork agents spawned from the template

package/src/types/module.ts

@@ -36,6 +36,15 @@ export interface Module {
   /** Unique name, used for tool namespacing and state storage */
   readonly name: string;
 
+  /**
+   * Per-module budget (ms) for gatherContext() before the framework skips this
+   * module's injection for the turn (fail-open). Defaults to the registry-wide
+   * default (15s) when omitted. Declare a larger budget when gatherContext
+   * does real work — e.g. sequential LLM calls with provider backoff — so the
+   * injection isn't silently dropped every turn.
+   */
+  readonly contextTimeoutMs?: number;
+
   /**
    * Start the module.
    * Called when the framework starts or when the module is added.
@@ -101,9 +110,52 @@ export interface ModuleContext {
 
   /**
    * Set persistent state for this module.
+   *
+   * This rewrites the module's full state object every call — appropriate for
+   * small, bounded state only. For accumulating collections (registries,
+   * corpora, histories) use a log state (`registerLogState`/`appendToLog`)
+   * instead: a setState of an ever-growing object makes every persisted record
+   * proportional to accumulated content, i.e. O(n²) aggregate disk.
    */
   setState<T>(state: T): void;
 
+  /**
+   * Register an auxiliary append-log state for this module, stored as
+   * `modules/<module>/<name>`. Idempotent — safe to call on every start().
+   * Items are appended with `appendToLog` (one O(item) record per append) and
+   * point-edited with `editLogItem`; Chronicle's append_log snapshot cadence
+   * handles consolidation. `name` must not be 'state' (reserved for the main
+   * snapshot state) and must not contain '/'.
+   */
+  registerLogState(
+    name: string,
+    opts?: { deltaSnapshotEvery?: number; fullSnapshotEvery?: number }
+  ): void;
+
+  /**
+   * Append one item to a log state registered via registerLogState.
+   * Record size is O(item), independent of accumulated log length.
+   */
+  appendToLog<T>(name: string, item: T): void;
+
+  /**
+   * Overwrite the item at `index` in a log state. Indices are stable —
+   * appends assign the next index and nothing is ever removed.
+   */
+  editLogItem<T>(name: string, index: number, item: T): void;
+
+  /**
+   * Read the full contents of a log state (empty array if never written).
+   * Intended for one-shot reconstruction at start(); keep hot-path reads
+   * against the in-memory copy.
+   */
+  getLog<T>(name: string): T[];
+
+  /**
+   * Number of items in a log state (0 if never written).
+   */
+  getLogLength(name: string): number;
+
   /**
    * Process queue for pushing events from external listeners.
    * @deprecated Use pushEvent() instead — it also emits process:received traces.
@@ -186,6 +238,19 @@ export interface ModuleContext {
    */
   unregisterSpeechHandler(): void;
 
+  /**
+   * Execute a tool call autonomously, without agent mediation.
+   *
+   * Routes through the framework's normal tool dispatch — MCPL-prefixed names
+   * (e.g. `mcpl--discord--unsubscribe_channel`) hit the MCPL subsystem; module
+   * names hit the owning module. Unlike pushing a `tool-call` event, this does
+   * NOT record tool_use/tool_result into any agent's context — it's a pure
+   * side-effecting call whose result is returned to the caller. Used by modules
+   * that need to act on their own (e.g. subscription GC unsubscribing a noisy
+   * channel) rather than asking the agent to call the tool.
+   */
+  callTool(call: ToolCall): Promise<ToolResult>;
+
   /**
    * Subscribe to the framework's trace event stream for observability.
    *

package/src/types/trace.ts

@@ -60,6 +60,13 @@ export type TraceEvent =
       type: 'inference:exhausted';
       agentName: string;
       error: string;
+      /** Whether membrane classified the failure as retryable (observability). */
+      retryable?: boolean;
+      /**
+       * The membrane error type (e.g. 'invalid_request', 'auth', 'context_length').
+       * The poison-history breaker fires only on 'invalid_request'.
+       */
+      errorType?: string;
     })
 
   // Streaming inference lifecycle
@@ -270,6 +277,50 @@ export type TraceEvent =
       type: 'mcpl:conversation-disposed';
       agentName: string;
       channelId?: string;
+    })
+
+  // MCPL server connection lifecycle (spawn / handshake / reconnect health).
+  // Previously these outcomes were visible only on the host process's own
+  // stderr, so a server that lost the boot handshake race just went missing.
+  | (TraceEventBase & {
+      type: 'mcpl:server-connect-failed';
+      serverId: string;
+      error: string;
+      /** Ordinal of the failed attempt: 0 is the initial connect, N is the Nth background retry. */
+      attempt: number;
+      /** True when a background reconnect loop will keep retrying. */
+      willRetry: boolean;
+    })
+  | (TraceEventBase & {
+      type: 'mcpl:server-reconnected';
+      serverId: string;
+      /** How many attempts the reconnect loop needed (1 = first retry succeeded). */
+      attempts: number;
+    })
+  | (TraceEventBase & {
+      type: 'mcpl:server-closed';
+      serverId: string;
+      /** Child process exit code, or null when killed by signal / closed explicitly. */
+      code: number | null;
+      /** Signal that terminated the child, if any. */
+      signal: string | null;
+      /** True when a background reconnect loop will revive the connection. */
+      willReconnect: boolean;
+    })
+  | (TraceEventBase & {
+      type: 'mcpl:server-error';
+      serverId: string;
+      error: string;
+    })
+  | (TraceEventBase & {
+      type: 'mcpl:orphaned-response';
+      serverId: string;
+      /** The JSON-RPC id of the late response whose pending request already timed out. */
+      responseId: string | number;
+      /** True when the dropped response carried host-managed `state`. */
+      hadState: boolean;
+      /** True when the dropped response carried a server-managed `checkpoint`. */
+      hadCheckpoint: boolean;
     });
 
 /**