@animalabs/agent-framework 0.6.2 → 0.6.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animalabs/agent-framework",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Multi-agent framework with pluggable modules, persistent state, and concurrent tool execution",
   "repository": {
     "type": "git",

package/src/framework.ts

@@ -40,6 +40,9 @@ import type {
   AgentState,
   Module,
   SpeechContext,
+  ContextMaintenanceRun,
+  ContextMaintenanceAgentRun,
+  ContextMaintenanceSnapshot,
 } from './types/index.js';
 import { ProcessQueueImpl } from './queue.js';
 import { Agent } from './agent.js';
@@ -144,6 +147,10 @@ class DefaultErrorPolicy implements ErrorPolicy {
 
 /** Default sync interval in milliseconds */
 const DEFAULT_SYNC_INTERVAL_MS = 1000;
+/** Poll pending context-strategy maintenance independently of user activity. */
+const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
+/** Bound one pass so a large backlog yields to inference and other agents. */
+const MAINTENANCE_TICKS_PER_PASS = 8;
 
 /**
  * Extract fields the EventGate cares about from a ProcessEvent. The set of
@@ -253,6 +260,12 @@ export class AgentFramework {
   private traceListeners: TraceEventListener[] = [];
   private syncIntervalMs: number;
   private syncTimer: ReturnType<typeof setInterval> | null = null;
+  private maintenanceIntervalMs: number;
+  private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
+  private maintenancePass: Promise<void> | null = null;
+  private maintenanceRunId = 0;
+  private currentMaintenanceRun: ContextMaintenanceRun | null = null;
+  private maintenanceHistory: ContextMaintenanceRun[] = [];
   /** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
   private staleWarnAt = new Map<string, number>();
   /** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
@@ -267,6 +280,19 @@ export class AgentFramework {
   /** N consecutive failed inferences ⇒ the agent is treated as hard-down and
    *  escalated loudly to stderr. */
   private readonly inferenceFailureEscalationThreshold = 3;
+  /** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
+   *  opsAlert() so a persistent failure re-posts once per cooldown window
+   *  instead of on every occurrence. */
+  private opsAlertLastSent: Map<string, number> = new Map();
+  /** Cooldown between webhook posts for the same (agent, kind). */
+  private readonly opsAlertCooldownMs = 15 * 60_000;
+  /** Per-agent refusal bookkeeping for observability — exposed via
+   *  healthSnapshot() and mirrored to failures.log / ops alerts. */
+  private refusalStats: Map<string, { total: number; byCategory: Record<string, number>; lastAt: number; lastCategory: string }> = new Map();
+  /** Per-agent count of consecutive refusals (reset on any non-refusal
+   *  completion). Distinct from refusalRewinds, which budgets the auto-rewind
+   *  loop — this one drives ops alerting. */
+  private refusalStreak: Map<string, number> = new Map();
   /** Per-agent count of consecutive refusal-driven rewinds in the current turn
    *  chain (reset when a turn completes without a refusal). Bounds the auto
    *  rewind loop — see refusalHandling + rewindTriggeringTurn. */
@@ -356,6 +382,7 @@ export class AgentFramework {
     inferencePolicy: InferencePolicy,
     errorPolicy: ErrorPolicy,
     syncIntervalMs: number,
+    maintenanceIntervalMs: number,
     processLoggingPersist: boolean,
     processLoggingBroadcast: boolean
   ) {
@@ -365,6 +392,7 @@ export class AgentFramework {
     this.inferencePolicy = inferencePolicy;
     this.errorPolicy = errorPolicy;
     this.syncIntervalMs = syncIntervalMs;
+    this.maintenanceIntervalMs = maintenanceIntervalMs;
     this.processLoggingPersist = processLoggingPersist;
     this.processLoggingBroadcast = processLoggingBroadcast;
     this.queue = new ProcessQueueImpl();
@@ -455,6 +483,7 @@ export class AgentFramework {
       config.inferencePolicy ?? new DefaultInferencePolicy(),
       config.errorPolicy ?? new DefaultErrorPolicy(),
       config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS,
+      config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS,
       processLoggingPersist,
       processLoggingBroadcast
     );
@@ -610,6 +639,16 @@ export class AgentFramework {
         }
       }, this.syncIntervalMs);
     }
+
+    if (this.maintenanceIntervalMs > 0) {
+      this.maintenanceTimer = setInterval(() => {
+        this.startQueuedMaintenance();
+      }, this.maintenanceIntervalMs);
+      this.maintenanceTimer.unref?.();
+      // Do not make a restored queue wait a full interval before its first
+      // attempt. MCPL/module initialization has completed before start().
+      this.startQueuedMaintenance();
+    }
   }
 
   /**
@@ -638,6 +677,14 @@ export class AgentFramework {
       this.syncTimer = null;
     }
 
+    if (this.maintenanceTimer) {
+      clearInterval(this.maintenanceTimer);
+      this.maintenanceTimer = null;
+    }
+    if (this.maintenancePass) {
+      await this.maintenancePass;
+    }
+
     if (this.loopPromise) {
       await this.loopPromise;
     }
@@ -668,6 +715,103 @@ export class AgentFramework {
     }
   }
 
+  /**
+   * Advance queued ContextManager work without requiring a new message.
+   *
+   * Tool definitions are refreshed first because compression replays stored
+   * tool cycles and providers reject those requests when the corresponding
+   * schemas are absent. Passes never overlap; each agent gets a bounded drain
+   * so maintenance cannot monopolize the framework event loop.
+   */
+  private startQueuedMaintenance(): void {
+    if (this.maintenancePass || !this.running) return;
+    const pass = this.runQueuedMaintenance().catch((error) => {
+      console.error(
+        '[context-maintenance] pass failed:',
+        error instanceof Error ? error.message : error,
+      );
+    });
+    this.maintenancePass = pass;
+    void pass.finally(() => {
+      if (this.maintenancePass === pass) this.maintenancePass = null;
+    });
+  }
+
+  private async runQueuedMaintenance(): Promise<void> {
+    const allTools = this.getAllTools();
+    const queued = [...this.agents.values()].flatMap((agent) => {
+      const cm = agent.getContextManager();
+      const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
+      cm.setToolDefinitions(tools);
+      if (cm.isReady()) return [];
+      const pending = cm.getPendingWork()?.description;
+      const progress = this.contextProgress(cm);
+      const record: ContextMaintenanceAgentRun = {
+        agentName: agent.name,
+        startedAt: Date.now(),
+        ticks: 0,
+        readyBefore: false,
+        ...(pending ? { pendingBefore: pending } : {}),
+        ...(progress ? { progressBefore: progress } : {}),
+      };
+      return [{
+        agent,
+        cm,
+        record,
+      }];
+    });
+    if (queued.length === 0) return;
+
+    const run: ContextMaintenanceRun = {
+      id: ++this.maintenanceRunId,
+      startedAt: Date.now(),
+      agents: queued.map(item => item.record),
+    };
+    this.currentMaintenanceRun = run;
+
+    try {
+      await Promise.all(queued.map(async ({ agent, cm, record }) => {
+        try {
+          for (
+            let i = 0;
+            i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady();
+            i++
+          ) {
+            await cm.tick();
+            record.ticks++;
+          }
+        } catch (error) {
+          record.error = error instanceof Error ? error.message : String(error);
+          console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
+        } finally {
+          const pending = cm.getPendingWork()?.description;
+          record.finishedAt = Date.now();
+          record.readyAfter = cm.isReady();
+          if (pending) record.pendingAfter = pending;
+          const progress = this.contextProgress(cm);
+          if (progress) record.progressAfter = progress;
+        }
+      }));
+    } finally {
+      run.finishedAt = Date.now();
+      run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
+      this.currentMaintenanceRun = null;
+      this.maintenanceHistory.unshift(run);
+      if (this.maintenanceHistory.length > 50) this.maintenanceHistory.length = 50;
+    }
+  }
+
+  private contextProgress(cm: ContextManager): Record<string, unknown> | undefined {
+    const strategy = cm.getStrategy() as {
+      getProgressSnapshot?: () => unknown;
+    };
+    if (typeof strategy.getProgressSnapshot !== 'function') return undefined;
+    const progress = strategy.getProgressSnapshot();
+    return progress && typeof progress === 'object'
+      ? progress as Record<string, unknown>
+      : undefined;
+  }
+
   /**
    * Push a process event to the queue.
    */
@@ -790,6 +934,28 @@ export class AgentFramework {
     return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
   }
 
+  /** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
+  getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot {
+    const agents = [...this.agents.values()].map((agent) => {
+      const cm = agent.getContextManager();
+      const pending = cm.getPendingWork()?.description;
+      const progress = this.contextProgress(cm);
+      return {
+        agentName: agent.name,
+        ready: cm.isReady(),
+        ...(pending ? { pending } : {}),
+        ...(progress ? { progress } : {}),
+      };
+    });
+    return structuredClone({
+      intervalMs: this.maintenanceIntervalMs,
+      ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
+      current: this.currentMaintenanceRun,
+      history: this.maintenanceHistory,
+      agents,
+    });
+  }
+
   /**
    * Build the membrane-normalized request that WOULD be emitted if `agentName`
    * were activated right now — WITHOUT running inference, opening a stream, or
@@ -3544,6 +3710,8 @@ export class AgentFramework {
                   (stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''),
               );
 
+              this.noteRefusal(agent.name, category, tokenUsage);
+
               // Rewind: excise the turn that fed the refusal, drop a
               // metadata-only marker, and retry — keeping the agent on its own
               // model instead of substituting a fallback. Driven either by an
@@ -3621,6 +3789,7 @@ export class AgentFramework {
                 this.refusalRewinds.set(agent.name, 0);
               }
               this.rewindEpisode.delete(agent.name);
+              this.refusalStreak.delete(agent.name);
             }
 
             // Dispatch speech (and thoughts if any)
@@ -4390,10 +4559,108 @@ export class AgentFramework {
         status: agent.state.status,
         consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
         lastInference: this.lastInferenceAt.get(name) ?? null,
+        refusalStats: this.refusalStats.get(name) ?? null,
       })),
     };
   }
 
+  /** Append a structured JSONL record to logs/failures.log (best-effort).
+   *  Durable and independent of journald/unit log redirects — this is what
+   *  connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
+   *  records carry {at, agent, consecutive, reason}; new records add `kind`
+   *  and kind-specific fields (additive only — doctor parses by regex). */
+  private logFailure(record: Record<string, unknown>): void {
+    try {
+      mkdirSync('logs', { recursive: true });
+      appendFileSync(
+        'logs/failures.log',
+        JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n',
+      );
+    } catch { /* best-effort */ }
+  }
+
+  /**
+   * Ops alert — the one escalation path for "a human should hear about this":
+   * (1) durable failures.log record (unless the caller already wrote one),
+   * (2) an `ops:alert` trace so authorized observers get it on the wire,
+   * (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
+   *     (agent, kind) per cooldown window — a persistent failure re-posts
+   *     every ~15 min instead of flooding the channel on every occurrence.
+   * See connectome docs/observability.md.
+   */
+  private opsAlert(
+    kind: string,
+    agentName: string,
+    message: string,
+    opts?: { data?: Record<string, unknown>; skipLog?: boolean },
+  ): void {
+    if (!opts?.skipLog) {
+      this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
+    }
+    this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
+
+    const hook = process.env.CONNECTOME_OPS_WEBHOOK;
+    if (!hook) return;
+    const key = `${agentName}:${kind}`;
+    const now = Date.now();
+    if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs) return;
+    // Stamp BEFORE the async post so a burst can't race past the cooldown.
+    this.opsAlertLastSent.set(key, now);
+    fetch(hook, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
+      }),
+    }).catch((err) => console.error(
+      `[ops-alert] webhook post failed (${key}):`,
+      err instanceof Error ? err.message : err,
+    ));
+  }
+
+  /**
+   * Record a model refusal for observability: per-agent stats + a
+   * consecutive-refusal streak (exposed via healthSnapshot), a durable
+   * failures.log record for EVERY refusal (previously invisible outside
+   * stderr), and a throttled ops alert once refusals repeat — one refusal is
+   * often survivable (auto-rewind may clear it); two in a row means stuck.
+   * The streak is reset by any non-refusal completion in the stream driver.
+   */
+  private noteRefusal(
+    agentName: string,
+    category: string,
+    tokens?: { input: number; output: number },
+  ): number {
+    const stats = this.refusalStats.get(agentName)
+      ?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
+    stats.total += 1;
+    stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
+    stats.lastAt = Date.now();
+    stats.lastCategory = category;
+    this.refusalStats.set(agentName, stats);
+
+    const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
+    this.refusalStreak.set(agentName, streak);
+
+    this.logFailure({
+      agent: agentName,
+      kind: 'refusal',
+      category,
+      streak,
+      locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
+      tokens: tokens ?? null,
+    });
+    if (streak >= 2) {
+      this.opsAlert(
+        'refusal',
+        agentName,
+        `refusal streak ${streak}, category=${category}`,
+        { skipLog: true, data: { category, streak } },
+      );
+    }
+    return streak;
+  }
+
   private noteInferenceExhausted(
     agentName: string,
     reason: string,
@@ -4409,14 +4676,8 @@ export class AgentFramework {
 
     // (1b) Machine-greppable durable record, independent of journald/unit log
     // redirects: logs/failures.log under the host's working directory. This is
-    // what connectome-doctor reads.
-    try {
-      mkdirSync('logs', { recursive: true });
-      appendFileSync(
-        'logs/failures.log',
-        JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n',
-      );
-    } catch { /* best-effort */ }
+    // what connectome-doctor reads. Legacy fields kept; `kind` is additive.
+    this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
 
     // (2) Agent-facing chronicle marker (no inference triggered → no loop).
     const agent = this.agents.get(agentName);
@@ -4447,17 +4708,15 @@ export class AgentFramework {
         `inferences — it cannot complete a turn. Last reason: ${reason}`,
       );
 
-      // Optional ops alert: set CONNECTOME_OPS_WEBHOOK to a Discord webhook URL.
-      const hook = process.env.CONNECTOME_OPS_WEBHOOK;
-      if (hook) {
-        fetch(hook, {
-          method: 'POST',
-          headers: { 'content-type': 'application/json' },
-          body: JSON.stringify({
-            content: `\u{1F6A8} **${agentName}** hard-down: ${streak} consecutive inference failures.\nLast reason: ${reason.slice(0, 500)}`,
-          }),
-        }).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
-      }
+      // Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
+      // a hard-down agent re-posts every ~15 min, not on every failed retry.
+      // failures.log already got the per-exhaustion record above.
+      this.opsAlert(
+        'hard-down',
+        agentName,
+        `${streak} consecutive inference failures. Last reason: ${reason}`,
+        { skipLog: true },
+      );
 
       // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
       // rejection of the history itself. Deliberately NOT keyed on
@@ -5030,6 +5289,17 @@ export class AgentFramework {
         attempt: params.attempt,
         willRetry: connection.willReconnect,
       });
+      // The reconnect loop never gives up (backoff caps at ~300s), so
+      // "the server is effectively down" is an attempt-count judgment:
+      // 5 failed attempts ≈ a few minutes of outage. Throttled per
+      // (serverId, kind), so a long outage re-posts every ~15 min.
+      if (params.attempt >= 5) {
+        this.opsAlert(
+          'mcpl-down',
+          connection.id,
+          `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`,
+        );
+      }
     });
 
     // A late response to a tools/call that already timed out, carrying

package/src/types/framework.ts

@@ -56,6 +56,13 @@ export interface FrameworkConfig {
   /** Interval for periodic store sync in milliseconds (default: 1000ms, 0 to disable) */
   syncIntervalMs?: number;
 
+  /**
+   * Interval for queued context-maintenance polling in milliseconds
+   * (default: 5000ms, 0 to disable). Each pass refreshes the agent's live tool
+   * definitions, then advances pending compression/indexing work.
+   */
+  maintenanceIntervalMs?: number;
+
   /** Process logging configuration (default: disabled) */
   processLogging?: ProcessLoggingConfig;
 
@@ -92,6 +99,43 @@ export interface FrameworkConfig {
   conversations?: ConversationRouterConfig;
 }
 
+/** One agent's outcome within a periodic context-maintenance pass. */
+export interface ContextMaintenanceAgentRun {
+  agentName: string;
+  startedAt: number;
+  finishedAt?: number;
+  ticks: number;
+  readyBefore: boolean;
+  readyAfter?: boolean;
+  pendingBefore?: string;
+  pendingAfter?: string;
+  progressBefore?: Record<string, unknown>;
+  progressAfter?: Record<string, unknown>;
+  error?: string;
+}
+
+/** A bounded periodic context-maintenance pass. */
+export interface ContextMaintenanceRun {
+  id: number;
+  startedAt: number;
+  finishedAt?: number;
+  agents: ContextMaintenanceAgentRun[];
+}
+
+/** Debug-safe maintenance state. Contains counts and errors, never messages. */
+export interface ContextMaintenanceSnapshot {
+  intervalMs: number;
+  ticksPerPass: number;
+  current: ContextMaintenanceRun | null;
+  history: ContextMaintenanceRun[];
+  agents: Array<{
+    agentName: string;
+    ready: boolean;
+    pending?: string;
+    progress?: Record<string, unknown>;
+  }>;
+}
+
 /**
  * Policy for deciding when to run inference.
  */

package/src/types/trace.ts

@@ -321,6 +321,22 @@ export type TraceEvent =
       hadState: boolean;
       /** True when the dropped response carried a server-managed `checkpoint`. */
       hadCheckpoint: boolean;
+    })
+
+  // Ops / fleet observability. Everything the framework's opsAlert() escalates
+  // (refusals, hard-down, MCPL unreachable, …) mirrored onto the trace bus so
+  // authorized observers see alerts on the same wire as the rest of the
+  // agent's internal life. Discord (CONNECTOME_OPS_WEBHOOK) is one sink of
+  // this stream, not a separate system. See connectome docs/observability.md.
+  | (TraceEventBase & {
+      type: 'ops:alert';
+      /** Alert kind — open set: 'refusal' | 'hard-down' | 'mcpl-down' | … */
+      kind: string;
+      /** Agent name, or a server/component id for non-agent alerts (e.g. mcpl-down). */
+      agentName: string;
+      message: string;
+      /** Kind-specific structured payload (mirrors the failures.log record). */
+      data?: Record<string, unknown>;
     });
 
 /**