@animalabs/agent-framework 0.16.0 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@animalabs/agent-framework",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Multi-agent framework with pluggable modules, persistent state, and concurrent tool execution",
   "repository": {
     "type": "git",

package/src/framework.ts

@@ -21,6 +21,7 @@ import type {
   FrameworkConfig,
   InferencePolicy,
   ErrorPolicy,
+  ProviderHoldHook,
   ErrorAction,
   FrameworkState,
   TraceEvent,
@@ -744,6 +745,12 @@ interface ProviderAccelerationCooldown {
   heldRequests: InferenceRequest[];
   reason: string;
   failures: number;
+  /** Set on a host-requested hold (config.providerHold): the error to
+   *  re-present to the hook when this slice expires. */
+  hostHoldError?: Error;
+  /** Armed by an auxiliary (compression) failure with no primary involved:
+   *  release reopens auxiliary admission without synthesising an inference. */
+  auxiliaryOrigin?: boolean;
 }
 interface ProviderAccelerationRecovery {
   startedAt: number;
@@ -948,6 +955,7 @@ export class AgentFramework {
   private providerAccelerationLastRecovery: Map<string, ProviderAccelerationReceipt> = new Map();
   private providerAccelerationDefaultCooldownMs = PROVIDER_ACCELERATION_DEFAULT_COOLDOWN_MS;
   private providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS;
+  private providerHoldHook: ProviderHoldHook | undefined;
   private providerAdmissionClosed = false;
   /** Last time we reported stale (busy-requeued) inference requests, per agent. */
   private staleWarnAt = new Map<string, number>();
@@ -1452,6 +1460,7 @@ export class AgentFramework {
       normalizeDiscordAwarenessDeadline(config.discordAwarenessDeadlineMs),
       new OperatorLog(operatorLogPath),
     );
+    framework.providerHoldHook = config.providerHold;
 
     // If an offline recovery process crashed after switching Chronicle but
     // before committing its prepared marker batch, the active branch is the
@@ -1915,6 +1924,14 @@ export class AgentFramework {
     }
     gate.auxiliaryInFlight++;
     try { return await run(); }
+    catch (error) {
+      // Compression can be the first call to find a spent quota (idle agent,
+      // scheduled maintenance). Arm the host hold so later passes wait at the
+      // gate instead of retrying into it; this pass still fails to its caller.
+      const agent = this.agents.get(agentName);
+      if (agent && error instanceof Error) this.holdProviderAcceleration(agent, error, undefined, true);
+      throw error;
+    }
     finally {
       gate.auxiliaryInFlight = Math.max(0, gate.auxiliaryInFlight - 1);
       if (gate.auxiliaryInFlight === 0) for (const resolve of gate.idleWaiters.splice(0)) resolve();
@@ -1941,27 +1958,75 @@ export class AgentFramework {
     return a.agentName === b.agentName && a.reason === b.reason && a.source === b.source &&
       a.timestamp === b.timestamp && a.channelId === b.channelId;
   }
-  private holdProviderAcceleration(agent: Agent, error: Error, trigger?: InferenceRequest): boolean {
-    if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name) || !isOrganizationAccelerationRateLimit(error)) return false;
-    const now = Date.now(); const delayMs = this.accelerationCooldownMs(agent.name, error);
+  private consultProviderHold(error: Error, agentName: string): { holdMs: number; reason: string } | undefined {
+    if (!this.providerHoldHook) return undefined;
+    try {
+      const hold = this.providerHoldHook(error, agentName, { model: this.agents.get(agentName)?.model });
+      if (!hold || !Number.isFinite(hold.holdMs) || hold.holdMs <= 0) return undefined;
+      return { holdMs: Math.min(PROVIDER_ACCELERATION_MAX_COOLDOWN_MS, Math.max(1_000, hold.holdMs)),
+        reason: typeof hold.reason === 'string' && hold.reason ? hold.reason : error.message };
+    } catch (err) {
+      console.error(`[provider-cooldown] providerHold hook threw for ${agentName}; treating as no hold:`, err);
+      return undefined;
+    }
+  }
+  /** A host hold whose slice expired: ask the host again before spending an
+   *  inference on it. True = extended in place (held requests stay held). */
+  private extendHostProviderHold(agentName: string, cooldown: ProviderAccelerationCooldown): boolean {
+    if (!cooldown.hostHoldError) return false;
+    const hold = this.consultProviderHold(cooldown.hostHoldError, agentName);
+    if (!hold) return false;
+    clearTimeout(cooldown.timer);
+    cooldown.until = Date.now() + hold.holdMs;
+    cooldown.timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agentName), hold.holdMs);
+    cooldown.timer.unref?.();
+    if (hold.reason !== cooldown.reason) {
+      cooldown.reason = hold.reason;
+      console.error(`[provider-cooldown] agent=${agentName} host hold extended — ${hold.reason}`);
+    }
+    return true;
+  }
+  private holdProviderAcceleration(agent: Agent, error: Error, trigger?: InferenceRequest, auxiliary = false): boolean {
+    if (this.ephemeralRuns.has(agent.name) || this.conversationAgentHomes.has(agent.name)) return false;
+    // The host knows more than the message wording does: ask it first. An
+    // auxiliary failure arms host holds only — the built-in acceleration
+    // cooldown stays a primary-path mechanism.
+    const hostHold = this.consultProviderHold(error, agent.name);
+    const acceleration = !hostHold && !auxiliary && isOrganizationAccelerationRateLimit(error);
+    if (!acceleration && !hostHold) return false;
+    const now = Date.now(); const delayMs = hostHold ? hostHold.holdMs : this.accelerationCooldownMs(agent.name, error as MembraneError);
     const existing = this.providerAccelerationCooldowns.get(agent.name);
     const held = existing?.heldRequests ?? [];
     if (trigger && !held.some((r) => this.sameInferenceRequest(r, trigger))) held.push(trigger);
     if (existing) clearTimeout(existing.timer);
     const timer = setTimeout(() => this.releaseProviderAccelerationCooldown(agent.name), delayMs); timer.unref?.();
     this.providerAccelerationCooldowns.set(agent.name, { startedAt: existing?.startedAt ?? now,
-      until: now + delayMs, timer, heldRequests: held, reason: error.message,
-      failures: (existing?.failures ?? 0) + 1 });
+      until: now + delayMs, timer, heldRequests: held, reason: hostHold?.reason ?? error.message,
+      failures: (existing?.failures ?? 0) + 1, ...(hostHold ? { hostHoldError: error } : {}),
+      ...(auxiliary && (!existing || existing.auxiliaryOrigin) ? { auxiliaryOrigin: true } : {}) });
+    if (auxiliary && !existing) {
+      console.error(`[provider-cooldown] agent=${agent.name} host hold (${hostHold!.reason}) from an auxiliary call — ` +
+        `holding primary/auxiliary for ${delayMs}ms`);
+      return true;
+    }
     const gate = this.providerGate(agent.name); gate.primaryPending = true;
     this.providerAccelerationRecoveries.set(agent.name, { startedAt: existing?.startedAt ?? now,
       failures: (existing?.failures ?? 0) + 1, heldRequests: held.length, reason: error.message });
-    console.error(`[provider-cooldown] agent=${agent.name} organization acceleration 429 — ` +
+    console.error(`[provider-cooldown] agent=${agent.name} ` +
+      `${hostHold ? `host hold (${hostHold.reason})` : 'organization acceleration 429'} — ` +
       `holding primary/auxiliary for ${delayMs}ms; ${held.length} request(s) retained`);
     return true;
   }
   private releaseProviderAccelerationCooldown(agentName: string): void {
     const cooldown = this.providerAccelerationCooldowns.get(agentName); if (!cooldown) return;
+    if (this.extendHostProviderHold(agentName, cooldown)) return;
     clearTimeout(cooldown.timer); this.providerAccelerationCooldowns.delete(agentName);
+    if (cooldown.auxiliaryOrigin && cooldown.heldRequests.length === 0) {
+      // Nobody asked for an inference while parked: just reopen the gate.
+      console.error(`[provider-cooldown] agent=${agentName} auxiliary hold released after ${Date.now() - cooldown.startedAt}ms`);
+      this.flushAuxiliaryAdmission(agentName);
+      return;
+    }
     const recovery = this.providerAccelerationRecoveries.get(agentName);
     if (recovery) { recovery.releasedAt = Date.now(); recovery.heldRequests = cooldown.heldRequests.length; }
     const requests = cooldown.heldRequests.length > 0 ? cooldown.heldRequests : [{ agentName,
@@ -7322,7 +7387,7 @@ export class AgentFramework {
 
       const providerCooldown = this.providerAccelerationCooldowns?.get(agentName);
       if (providerCooldown) {
-        if (now < providerCooldown.until) {
+        if (now < providerCooldown.until || this.extendHostProviderHold(agentName, providerCooldown)) {
           for (const req of requests) {
             if (!providerCooldown.heldRequests.some((r) => this.sameInferenceRequest(r, req))) {
               providerCooldown.heldRequests.push(req);
@@ -11349,6 +11414,8 @@ export class AgentFramework {
             auxiliaryInFlight: gate?.auxiliaryInFlight ?? 0,
             auxiliaryDeferred: gate?.deferredAuxiliary ?? 0,
             cooldownUntil: cooldown?.until ?? null,
+            cooldownReason: cooldown?.reason ?? null,
+            hostHold: cooldown?.hostHoldError !== undefined,
             heldRequests: cooldown?.heldRequests.length ?? 0,
             lastRecovery: this.providerAccelerationLastRecovery?.get(name) ?? null,
           };

package/src/types/framework.ts

@@ -70,6 +70,21 @@ export interface CodeExecutionConfig {
   maxWakesPerScript?: number;
 }
 
+/** See `FrameworkConfig.providerHold`. */
+export interface ProviderHold {
+  holdMs: number;
+  /** Operator-facing explanation, logged with the hold. */
+  reason?: string;
+}
+/** What the framework knows about the failing call. */
+export interface ProviderHoldContext {
+  /** Model of the agent whose call failed — a quota window may be model-scoped. */
+  model?: string;
+}
+export type ProviderHoldHook = (
+  error: Error, agentName: string, context: ProviderHoldContext,
+) => ProviderHold | undefined;
+
 export interface FrameworkConfig {
   /**
    * The subconscious resident (issue #77): a persistent side-agent that
@@ -150,6 +165,27 @@ export interface FrameworkConfig {
   /** Custom error policy */
   errorPolicy?: ErrorPolicy;
 
+  /**
+   * Host verdict on a failed inference: "this cannot succeed until later".
+   * Consulted before the error policy and before the built-in
+   * organization-acceleration classification, for a persistent agent's
+   * primary inference and for its auxiliary (compression) calls. Ephemeral
+   * runs (subagents) and conversation forks are NOT covered: they have no
+   * provider-admission gate and keep the ordinary error policy. A hold parks the agent's provider
+   * admission (primary and auxiliary) instead of retrying, and — like the
+   * built-in organization-acceleration cooldown — records no failed turn and
+   * does not feed the hard-down streak. The motivating case is a subscription
+   * credential whose quota window is spent: its 429 looks like a throttle,
+   * but retrying cannot help until the window resets.
+   *
+   * Holds are served in slices of at most 10 minutes. When a slice expires
+   * the hook is consulted again with the same error, before any inference is
+   * attempted; it extends the hold by returning another, or releases it by
+   * returning undefined. Must be synchronous and cheap; a throwing hook is
+   * treated as "no hold".
+   */
+  providerHold?: ProviderHoldHook;
+
   /** Interval for periodic store sync in milliseconds (default: 1000ms, 0 to disable) */
   syncIntervalMs?: number;