@animalabs/agent-framework 0.6.2 → 0.6.4

package/dist/src/framework.js

@@ -21,6 +21,7 @@ import { CheckpointManager } from './mcpl/checkpoint-manager.js';
 import { isToolAllowed } from './mcpl/tool-policy.js';
 import { EventGate } from './gate/event-gate.js';
 import { UsageTracker } from './usage/usage-tracker.js';
+import { formatZonedDateTime, resolveTimeZone } from './timezone.js';
 const FRAMEWORK_STATE_ID = 'framework/state';
 const CONVERSATION_ROUTER_STATE_ID = 'framework/conversation-router';
 const INFERENCE_LOG_ID = 'framework/inference-log';
@@ -66,6 +67,10 @@ class DefaultErrorPolicy {
 }
 /** Default sync interval in milliseconds */
 const DEFAULT_SYNC_INTERVAL_MS = 1000;
+/** Poll pending context-strategy maintenance independently of user activity. */
+const DEFAULT_MAINTENANCE_INTERVAL_MS = 5000;
+/** Bound one pass so a large backlog yields to inference and other agents. */
+const MAINTENANCE_TICKS_PER_PASS = 8;
 /**
  * Extract fields the EventGate cares about from a ProcessEvent. The set of
  * event variants that carry `content`/`mount`/`paths`/`metadata` is open-ended
@@ -128,6 +133,12 @@ export class AgentFramework {
     traceListeners = [];
     syncIntervalMs;
     syncTimer = null;
+    maintenanceIntervalMs;
+    maintenanceTimer = null;
+    maintenancePass = null;
+    maintenanceRunId = 0;
+    currentMaintenanceRun = null;
+    maintenanceHistory = [];
     /** Last time we console-warned about stale (busy-requeued) inference requests, per agent. */
     staleWarnAt = new Map();
     /** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */
@@ -142,6 +153,19 @@ export class AgentFramework {
     /** N consecutive failed inferences ⇒ the agent is treated as hard-down and
      *  escalated loudly to stderr. */
     inferenceFailureEscalationThreshold = 3;
+    /** Per-(agent,kind) timestamp of the last ops webhook post. Throttles
+     *  opsAlert() so a persistent failure re-posts once per cooldown window
+     *  instead of on every occurrence. */
+    opsAlertLastSent = new Map();
+    /** Cooldown between webhook posts for the same (agent, kind). */
+    opsAlertCooldownMs = 15 * 60_000;
+    /** Per-agent refusal bookkeeping for observability — exposed via
+     *  healthSnapshot() and mirrored to failures.log / ops alerts. */
+    refusalStats = new Map();
+    /** Per-agent count of consecutive refusals (reset on any non-refusal
+     *  completion). Distinct from refusalRewinds, which budgets the auto-rewind
+     *  loop — this one drives ops alerting. */
+    refusalStreak = new Map();
     /** Per-agent count of consecutive refusal-driven rewinds in the current turn
      *  chain (reset when a turn completes without a refusal). Bounds the auto
      *  rewind loop — see refusalHandling + rewindTriggeringTurn. */
@@ -211,15 +235,19 @@ export class AgentFramework {
     eventGate = null;
     // Session-level token usage tracking (always-on)
     usageTracker;
-    constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, processLoggingPersist, processLoggingBroadcast) {
+    /** Presentation-only wall-clock zone; persistence remains UTC/epoch. */
+    timeZone;
+    constructor(store, ownsStore, membrane, inferencePolicy, errorPolicy, syncIntervalMs, maintenanceIntervalMs, processLoggingPersist, processLoggingBroadcast, timeZone) {
         this.store = store;
         this.ownsStore = ownsStore;
         this.membrane = membrane;
         this.inferencePolicy = inferencePolicy;
         this.errorPolicy = errorPolicy;
         this.syncIntervalMs = syncIntervalMs;
+        this.maintenanceIntervalMs = maintenanceIntervalMs;
         this.processLoggingPersist = processLoggingPersist;
         this.processLoggingBroadcast = processLoggingBroadcast;
+        this.timeZone = timeZone;
         this.queue = new ProcessQueueImpl();
         this.usageTracker = new UsageTracker({
             emitTrace: (e) => this.emitTrace({ ...e }),
@@ -299,7 +327,7 @@ export class AgentFramework {
                 // Already registered
             }
         }
-        const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast);
+        const framework = new AgentFramework(store, ownsStore, config.membrane, config.inferencePolicy ?? new DefaultInferencePolicy(), config.errorPolicy ?? new DefaultErrorPolicy(), config.syncIntervalMs ?? DEFAULT_SYNC_INTERVAL_MS, config.maintenanceIntervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS, processLoggingPersist, processLoggingBroadcast, resolveTimeZone(config.timeZone));
         // Restore persisted usage data (if any) from prior session
         framework.restoreUsageState();
         // Create agents
@@ -433,6 +461,15 @@ export class AgentFramework {
                 }
             }, this.syncIntervalMs);
         }
+        if (this.maintenanceIntervalMs > 0) {
+            this.maintenanceTimer = setInterval(() => {
+                this.startQueuedMaintenance();
+            }, this.maintenanceIntervalMs);
+            this.maintenanceTimer.unref?.();
+            // Do not make a restored queue wait a full interval before its first
+            // attempt. MCPL/module initialization has completed before start().
+            this.startQueuedMaintenance();
+        }
     }
     /**
      * Stop the event loop.
@@ -456,6 +493,13 @@ export class AgentFramework {
             clearInterval(this.syncTimer);
             this.syncTimer = null;
         }
+        if (this.maintenanceTimer) {
+            clearInterval(this.maintenanceTimer);
+            this.maintenanceTimer = null;
+        }
+        if (this.maintenancePass) {
+            await this.maintenancePass;
+        }
         if (this.loopPromise) {
             await this.loopPromise;
         }
@@ -481,6 +525,100 @@ export class AgentFramework {
             this.store.close();
         }
     }
+    /**
+     * Advance queued ContextManager work without requiring a new message.
+     *
+     * Tool definitions are refreshed first because compression replays stored
+     * tool cycles and providers reject those requests when the corresponding
+     * schemas are absent. Passes never overlap; each agent gets a bounded drain
+     * so maintenance cannot monopolize the framework event loop.
+     */
+    startQueuedMaintenance() {
+        if (this.maintenancePass || !this.running)
+            return;
+        const pass = this.runQueuedMaintenance().catch((error) => {
+            console.error('[context-maintenance] pass failed:', error instanceof Error ? error.message : error);
+        });
+        this.maintenancePass = pass;
+        void pass.finally(() => {
+            if (this.maintenancePass === pass)
+                this.maintenancePass = null;
+        });
+    }
+    async runQueuedMaintenance() {
+        const allTools = this.getAllTools();
+        const queued = [...this.agents.values()].flatMap((agent) => {
+            const cm = agent.getContextManager();
+            const tools = allTools.filter((tool) => agent.canUseTool(tool.name));
+            cm.setToolDefinitions(tools);
+            if (cm.isReady())
+                return [];
+            const pending = cm.getPendingWork()?.description;
+            const progress = this.contextProgress(cm);
+            const record = {
+                agentName: agent.name,
+                startedAt: Date.now(),
+                ticks: 0,
+                readyBefore: false,
+                ...(pending ? { pendingBefore: pending } : {}),
+                ...(progress ? { progressBefore: progress } : {}),
+            };
+            return [{
+                    agent,
+                    cm,
+                    record,
+                }];
+        });
+        if (queued.length === 0)
+            return;
+        const run = {
+            id: ++this.maintenanceRunId,
+            startedAt: Date.now(),
+            agents: queued.map(item => item.record),
+        };
+        this.currentMaintenanceRun = run;
+        try {
+            await Promise.all(queued.map(async ({ agent, cm, record }) => {
+                try {
+                    for (let i = 0; i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady(); i++) {
+                        await cm.tick();
+                        record.ticks++;
+                    }
+                }
+                catch (error) {
+                    record.error = error instanceof Error ? error.message : String(error);
+                    console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error);
+                }
+                finally {
+                    const pending = cm.getPendingWork()?.description;
+                    record.finishedAt = Date.now();
+                    record.readyAfter = cm.isReady();
+                    if (pending)
+                        record.pendingAfter = pending;
+                    const progress = this.contextProgress(cm);
+                    if (progress)
+                        record.progressAfter = progress;
+                }
+            }));
+        }
+        finally {
+            run.finishedAt = Date.now();
+            run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName));
+            this.currentMaintenanceRun = null;
+            this.maintenanceHistory.unshift(run);
+            if (this.maintenanceHistory.length > 50)
+                this.maintenanceHistory.length = 50;
+        }
+    }
+    contextProgress(cm) {
+        const strategy = cm.getStrategy();
+        if (typeof strategy.getProgressSnapshot !== 'function')
+            return undefined;
+        const progress = strategy.getProgressSnapshot();
+        return progress && typeof progress === 'object'
+            ? progress
+            : undefined;
+    }
     /**
      * Push a process event to the queue.
      */
@@ -592,6 +730,27 @@ export class AgentFramework {
         }
         return [...moduleTools, ...this.mcplTools, ...channelTools, ...gateTools];
     }
+    /** Counts-only context-maintenance diagnostics for authenticated debug UIs. */
+    getContextMaintenanceSnapshot() {
+        const agents = [...this.agents.values()].map((agent) => {
+            const cm = agent.getContextManager();
+            const pending = cm.getPendingWork()?.description;
+            const progress = this.contextProgress(cm);
+            return {
+                agentName: agent.name,
+                ready: cm.isReady(),
+                ...(pending ? { pending } : {}),
+                ...(progress ? { progress } : {}),
+            };
+        });
+        return structuredClone({
+            intervalMs: this.maintenanceIntervalMs,
+            ticksPerPass: MAINTENANCE_TICKS_PER_PASS,
+            current: this.currentMaintenanceRun,
+            history: this.maintenanceHistory,
+            agents,
+        });
+    }
     /**
      * Build the membrane-normalized request that WOULD be emitted if `agentName`
      * were activated right now — WITHOUT running inference, opening a stream, or
@@ -2970,6 +3129,7 @@ export class AgentFramework {
                             const category = stopDetails?.category ?? 'unknown';
                             console.error(`[inference-refusal] agent=${agent.name} category=${category}` +
                                 (stopDetails?.explanation ? ` explanation=${stopDetails.explanation}` : ''));
+                            this.noteRefusal(agent.name, category, tokenUsage);
                             // Rewind: excise the turn that fed the refusal, drop a
                             // metadata-only marker, and retry — keeping the agent on its own
                             // model instead of substituting a fallback. Driven either by an
@@ -3037,6 +3197,7 @@ export class AgentFramework {
                                 this.refusalRewinds.set(agent.name, 0);
                             }
                             this.rewindEpisode.delete(agent.name);
+                            this.refusalStreak.delete(agent.name);
                         }
                         // Dispatch speech (and thoughts if any)
                         if (speechContent.length > 0 || thoughts.length > 0) {
@@ -3740,9 +3901,84 @@ export class AgentFramework {
                 status: agent.state.status,
                 consecutiveInferenceFailures: this.consecutiveInferenceFailures.get(name) ?? 0,
                 lastInference: this.lastInferenceAt.get(name) ?? null,
+                refusalStats: this.refusalStats.get(name) ?? null,
             })),
         };
     }
+    /** Append a structured JSONL record to logs/failures.log (best-effort).
+     *  Durable and independent of journald/unit log redirects — this is what
+     *  connectome-doctor and fleet tooling read. Always stamps `at`. Legacy
+     *  records carry {at, agent, consecutive, reason}; new records add `kind`
+     *  and kind-specific fields (additive only — doctor parses by regex). */
+    logFailure(record) {
+        try {
+            mkdirSync('logs', { recursive: true });
+            appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), ...record }) + '\n');
+        }
+        catch { /* best-effort */ }
+    }
+    /**
+     * Ops alert — the one escalation path for "a human should hear about this":
+     * (1) durable failures.log record (unless the caller already wrote one),
+     * (2) an `ops:alert` trace so authorized observers get it on the wire,
+     * (3) if CONNECTOME_OPS_WEBHOOK is set, a Discord post throttled to one per
+     *     (agent, kind) per cooldown window — a persistent failure re-posts
+     *     every ~15 min instead of flooding the channel on every occurrence.
+     * See connectome docs/observability.md.
+     */
+    opsAlert(kind, agentName, message, opts) {
+        if (!opts?.skipLog) {
+            this.logFailure({ agent: agentName, kind, reason: message, ...opts?.data });
+        }
+        this.emitTrace({ type: 'ops:alert', kind, agentName, message, data: opts?.data });
+        const hook = process.env.CONNECTOME_OPS_WEBHOOK;
+        if (!hook)
+            return;
+        const key = `${agentName}:${kind}`;
+        const now = Date.now();
+        if (now - (this.opsAlertLastSent.get(key) ?? 0) < this.opsAlertCooldownMs)
+            return;
+        // Stamp BEFORE the async post so a burst can't race past the cooldown.
+        this.opsAlertLastSent.set(key, now);
+        fetch(hook, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({
+                content: `\u{1F6A8} **${agentName}** ${kind}: ${message.slice(0, 500)}`,
+            }),
+        }).catch((err) => console.error(`[ops-alert] webhook post failed (${key}):`, err instanceof Error ? err.message : err));
+    }
+    /**
+     * Record a model refusal for observability: per-agent stats + a
+     * consecutive-refusal streak (exposed via healthSnapshot), a durable
+     * failures.log record for EVERY refusal (previously invisible outside
+     * stderr), and a throttled ops alert once refusals repeat — one refusal is
+     * often survivable (auto-rewind may clear it); two in a row means stuck.
+     * The streak is reset by any non-refusal completion in the stream driver.
+     */
+    noteRefusal(agentName, category, tokens) {
+        const stats = this.refusalStats.get(agentName)
+            ?? { total: 0, byCategory: {}, lastAt: 0, lastCategory: '' };
+        stats.total += 1;
+        stats.byCategory[category] = (stats.byCategory[category] ?? 0) + 1;
+        stats.lastAt = Date.now();
+        stats.lastCategory = category;
+        this.refusalStats.set(agentName, stats);
+        const streak = (this.refusalStreak.get(agentName) ?? 0) + 1;
+        this.refusalStreak.set(agentName, streak);
+        this.logFailure({
+            agent: agentName,
+            kind: 'refusal',
+            category,
+            streak,
+            locus: this.channelRegistry?.buildChannelContext()?.incoming?.channelId ?? null,
+            tokens: tokens ?? null,
+        });
+        if (streak >= 2) {
+            this.opsAlert('refusal', agentName, `refusal streak ${streak}, category=${category}`, { skipLog: true, data: { category, streak } });
+        }
+        return streak;
+    }
     noteInferenceExhausted(agentName, reason, retryable, errorType) {
         const streak = (this.consecutiveInferenceFailures.get(agentName) ?? 0) + 1;
         this.consecutiveInferenceFailures.set(agentName, streak);
@@ -3751,12 +3987,8 @@ export class AgentFramework {
         console.error(`[inference-failed] agent=${agentName} consecutive=${streak}: ${reason}`);
         // (1b) Machine-greppable durable record, independent of journald/unit log
         // redirects: logs/failures.log under the host's working directory. This is
-        // what connectome-doctor reads.
-        try {
-            mkdirSync('logs', { recursive: true });
-            appendFileSync('logs/failures.log', JSON.stringify({ at: new Date().toISOString(), agent: agentName, consecutive: streak, reason }) + '\n');
-        }
-        catch { /* best-effort */ }
+        // what connectome-doctor reads. Legacy fields kept; `kind` is additive.
+        this.logFailure({ agent: agentName, consecutive: streak, reason, kind: 'inference-exhausted' });
         // (2) Agent-facing chronicle marker (no inference triggered → no loop).
         const agent = this.agents.get(agentName);
         if (agent && process.env.SUPPRESS_INFERENCE_FAILED_MARKER !== '1') {
@@ -3778,17 +4010,10 @@ export class AgentFramework {
         if (streak >= this.inferenceFailureEscalationThreshold) {
             console.error(`[inference-hard-down] agent=${agentName} has FAILED ${streak} consecutive ` +
                 `inferences — it cannot complete a turn. Last reason: ${reason}`);
-            // Optional ops alert: set CONNECTOME_OPS_WEBHOOK to a Discord webhook URL.
-            const hook = process.env.CONNECTOME_OPS_WEBHOOK;
-            if (hook) {
-                fetch(hook, {
-                    method: 'POST',
-                    headers: { 'content-type': 'application/json' },
-                    body: JSON.stringify({
-                        content: `\u{1F6A8} **${agentName}** hard-down: ${streak} consecutive inference failures.\nLast reason: ${reason.slice(0, 500)}`,
-                    }),
-                }).catch((err) => console.error('[inference-hard-down] webhook post failed:', err instanceof Error ? err.message : err));
-            }
+            // Ops alert (webhook + ops:alert trace), throttled per (agent, kind) —
+            // a hard-down agent re-posts every ~15 min, not on every failed retry.
+            // failures.log already got the per-exhaustion record above.
+            this.opsAlert('hard-down', agentName, `${streak} consecutive inference failures. Last reason: ${reason}`, { skipLog: true });
             // (4) Poison-history breaker: ONLY for `invalid_request` — a 400-class
             // rejection of the history itself. Deliberately NOT keyed on
             // `retryable === false`, which membrane also returns for auth (expired
@@ -4233,6 +4458,13 @@ export class AgentFramework {
                 attempt: params.attempt,
                 willRetry: connection.willReconnect,
             });
+            // The reconnect loop never gives up (backoff caps at ~300s), so
+            // "the server is effectively down" is an attempt-count judgment:
+            // 5 failed attempts ≈ a few minutes of outage. Throttled per
+            // (serverId, kind), so a long outage re-posts every ~15 min.
+            if (params.attempt >= 5) {
+                this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`);
+            }
         });
         // A late response to a tools/call that already timed out, carrying
         // stateful data the server advanced to. The host can't re-inject it (the
@@ -4657,7 +4889,16 @@ export class AgentFramework {
         }
         console.error(`[sleep] agent=${agentName} seconds=${seconds} announce=${announce} until=${new Date(until).toISOString()}`);
         // endTurn: the agent stops here and goes idle for the duration.
-        finish({ success: true, data: { sleepingFor: human, until }, endTurn: true });
+        finish({
+            success: true,
+            data: {
+                sleepingFor: human,
+                until,
+                untilLocal: formatZonedDateTime(until, this.timeZone),
+                timeZone: this.timeZone,
+            },
+            endTurn: true,
+        });
     }
     /** Aggregate the event-tag vocabulary: reserved chat:* core + each connected
      *  server's declared tag ontology + gate.js status. */