@desplega.ai/agent-swarm 1.56.5 → 1.56.6

package/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Agent Swarm API",
-    "version": "1.56.4",
+    "version": "1.56.5",
     "description": "Multi-agent orchestration API for Claude Code, Codex, and Gemini CLI. Enables task distribution, agent communication, and service discovery.\n\nMCP tools are documented separately in [MCP.md](./MCP.md)."
   },
   "servers": [
@@ -1415,6 +1415,237 @@
         }
       }
     },
+    "/api/keys/report-usage": {
+      "post": {
+        "summary": "Record which API key was used for a task",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "keyType": {
+                    "type": "string"
+                  },
+                  "keySuffix": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 10
+                  },
+                  "keyIndex": {
+                    "type": "integer",
+                    "minimum": 0
+                  },
+                  "taskId": {
+                    "type": "string",
+                    "format": "uuid"
+                  },
+                  "scope": {
+                    "type": "string"
+                  },
+                  "scopeId": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "keyType",
+                  "keySuffix",
+                  "keyIndex"
+                ]
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Usage recorded"
+          },
+          "400": {
+            "description": "Validation error"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
+    "/api/keys/report-rate-limit": {
+      "post": {
+        "summary": "Mark an API key as rate-limited",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "keyType": {
+                    "type": "string"
+                  },
+                  "keySuffix": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 10
+                  },
+                  "keyIndex": {
+                    "type": "integer",
+                    "minimum": 0
+                  },
+                  "rateLimitedUntil": {
+                    "type": "string",
+                    "format": "date-time"
+                  },
+                  "scope": {
+                    "type": "string"
+                  },
+                  "scopeId": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "keyType",
+                  "keySuffix",
+                  "keyIndex",
+                  "rateLimitedUntil"
+                ]
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Key marked as rate-limited"
+          },
+          "400": {
+            "description": "Validation error"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
+    "/api/keys/available": {
+      "get": {
+        "summary": "Get available (non-rate-limited) key indices for a credential type",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "parameters": [
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": true,
+            "name": "keyType",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "integer",
+              "minimum": 1
+            },
+            "required": true,
+            "name": "totalKeys",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scope",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scopeId",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "List of available key indices"
+          },
+          "400": {
+            "description": "Validation error"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
+    "/api/keys/status": {
+      "get": {
+        "summary": "Get all API key status records",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "parameters": [
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "keyType",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scope",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scopeId",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "List of key status records"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
     "/api/events": {
       "post": {
         "summary": "Store a single event",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@desplega.ai/agent-swarm",
-  "version": "1.56.5",
+  "version": "1.56.6",
   "description": "Multi-agent orchestration for Claude Code, Codex, Gemini CLI, and other AI coding assistants",
   "license": "MIT",
   "author": "desplega.sh <contact@desplega.sh>",

package/src/be/db.ts

@@ -7487,3 +7487,157 @@ export function getContextSummaryByTaskId(taskId: string): ContextSummary {
     snapshotCount: countRow?.cnt ?? 0,
   };
 }
+
+// ─── API Key Pool Tracking ───────────────────────────────────────────────────
+
+export interface ApiKeyStatus {
+  id: string;
+  keyType: string;
+  keySuffix: string;
+  keyIndex: number;
+  scope: string;
+  scopeId: string | null;
+  status: string;
+  rateLimitedUntil: string | null;
+  lastUsedAt: string | null;
+  lastRateLimitAt: string | null;
+  totalUsageCount: number;
+  rateLimitCount: number;
+  createdAt: string;
+  updatedAt: string;
+}
+
+/**
+ * Get available (non-rate-limited) key indices for a credential type.
+ * Automatically clears expired rate limits before returning.
+ */
+export function getAvailableKeyIndices(
+  keyType: string,
+  totalKeys: number,
+  scope = "global",
+  scopeId: string | null = null,
+): number[] {
+  const now = new Date().toISOString();
+  const db = getDb();
+  const effectiveScopeId = scopeId ?? "";
+
+  // Auto-clear expired rate limits
+  db.prepare(
+    `UPDATE api_key_status
+     SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
+     WHERE keyType = ? AND scope = ? AND scopeId = ?
+       AND status = 'rate_limited' AND rateLimitedUntil IS NOT NULL AND rateLimitedUntil <= ?`,
+  ).run(now, keyType, scope, effectiveScopeId, now);
+
+  // Get currently rate-limited key indices
+  const rateLimited = db
+    .prepare<{ keyIndex: number }, [string, string, string]>(
+      `SELECT keyIndex FROM api_key_status
+       WHERE keyType = ? AND scope = ? AND scopeId = ?
+         AND status = 'rate_limited'`,
+    )
+    .all(keyType, scope, effectiveScopeId);
+
+  const blockedIndices = new Set(rateLimited.map((r) => r.keyIndex));
+  const available: number[] = [];
+  for (let i = 0; i < totalKeys; i++) {
+    if (!blockedIndices.has(i)) available.push(i);
+  }
+  return available;
+}
+
+/**
+ * Record that a key was used for a task (upsert key status + update task).
+ */
+export function recordKeyUsage(
+  keyType: string,
+  keySuffix: string,
+  keyIndex: number,
+  taskId: string | null,
+  scope = "global",
+  scopeId: string | null = null,
+): void {
+  const now = new Date().toISOString();
+  const db = getDb();
+  const effectiveScopeId = scopeId ?? "";
+
+  // Upsert key status record
+  db.prepare(
+    `INSERT INTO api_key_status (keyType, keySuffix, keyIndex, scope, scopeId, lastUsedAt, totalUsageCount, updatedAt)
+     VALUES (?, ?, ?, ?, ?, ?, 1, ?)
+     ON CONFLICT(keyType, keySuffix, scope, scopeId)
+     DO UPDATE SET
+       lastUsedAt = excluded.lastUsedAt,
+       totalUsageCount = totalUsageCount + 1,
+       keyIndex = excluded.keyIndex,
+       updatedAt = excluded.updatedAt`,
+  ).run(keyType, keySuffix, keyIndex, scope, effectiveScopeId, now, now);
+
+  // Record which key was used on the task
+  if (taskId) {
+    db.prepare("UPDATE agent_tasks SET credentialKeySuffix = ? WHERE id = ?").run(
+      keySuffix,
+      taskId,
+    );
+  }
+}
+
+/**
+ * Mark a key as rate-limited with a retry-after timestamp.
+ */
+export function markKeyRateLimited(
+  keyType: string,
+  keySuffix: string,
+  keyIndex: number,
+  rateLimitedUntil: string,
+  scope = "global",
+  scopeId: string | null = null,
+): void {
+  const now = new Date().toISOString();
+  const effectiveScopeId = scopeId ?? "";
+  getDb()
+    .prepare(
+      `INSERT INTO api_key_status (keyType, keySuffix, keyIndex, scope, scopeId, status, rateLimitedUntil, lastRateLimitAt, rateLimitCount, updatedAt)
+       VALUES (?, ?, ?, ?, ?, 'rate_limited', ?, ?, 1, ?)
+       ON CONFLICT(keyType, keySuffix, scope, scopeId)
+       DO UPDATE SET
+         status = 'rate_limited',
+         rateLimitedUntil = excluded.rateLimitedUntil,
+         lastRateLimitAt = excluded.lastRateLimitAt,
+         rateLimitCount = rateLimitCount + 1,
+         keyIndex = excluded.keyIndex,
+         updatedAt = excluded.updatedAt`,
+    )
+    .run(keyType, keySuffix, keyIndex, scope, effectiveScopeId, rateLimitedUntil, now, now);
+}
+
+/**
+ * Get all key status records for a credential type.
+ */
+export function getKeyStatuses(
+  keyType?: string,
+  scope?: string,
+  scopeId?: string | null,
+): ApiKeyStatus[] {
+  const db = getDb();
+  const conditions: string[] = [];
+  const params: string[] = [];
+
+  if (keyType) {
+    conditions.push("keyType = ?");
+    params.push(keyType);
+  }
+  if (scope) {
+    conditions.push("scope = ?");
+    params.push(scope);
+    if (scopeId !== undefined) {
+      conditions.push("scopeId = ?");
+      params.push(scopeId ?? "");
+    }
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  return db
+    .prepare<ApiKeyStatus, string[]>(`SELECT * FROM api_key_status ${where} ORDER BY keyIndex`)
+    .all(...params);
+}

package/src/be/migrations/028_api_key_tracking.sql

@@ -0,0 +1,38 @@
+-- Track API key pool status for rate limit awareness and automatic rotation
+CREATE TABLE IF NOT EXISTS api_key_status (
+  id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+  -- Which credential type: 'CLAUDE_CODE_OAUTH_TOKEN' or 'ANTHROPIC_API_KEY'
+  keyType TEXT NOT NULL,
+  -- Last 5 characters of the key (for identification without storing secrets)
+  keySuffix TEXT NOT NULL,
+  -- Position in the comma-separated credential pool (0-based)
+  keyIndex INTEGER NOT NULL,
+  -- Scope mirrors swarm_config (global, agent, repo)
+  scope TEXT NOT NULL DEFAULT 'global',
+  scopeId TEXT NOT NULL DEFAULT '',
+  -- Current status
+  status TEXT NOT NULL DEFAULT 'available' CHECK(status IN ('available', 'rate_limited')),
+  -- When the rate limit expires (ISO timestamp from retry-after)
+  rateLimitedUntil TEXT,
+  -- Tracking timestamps
+  lastUsedAt TEXT,
+  lastRateLimitAt TEXT,
+  -- Counters
+  totalUsageCount INTEGER NOT NULL DEFAULT 0,
+  rateLimitCount INTEGER NOT NULL DEFAULT 0,
+  -- Metadata
+  createdAt TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  updatedAt TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  -- Unique constraint: one entry per key type + suffix + scope.
+  -- Known limitation: two different keys sharing the same last 5 chars will collide,
+  -- causing the ON CONFLICT upsert to overwrite keyIndex. In practice this is rare
+  -- (1 in ~60M for random alphanumeric suffixes) and the impact is minor (wrong index
+  -- logged, but rate-limit tracking still works since suffix is the real identifier).
+  UNIQUE(keyType, keySuffix, scope, scopeId)
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_key_status_lookup
+  ON api_key_status(keyType, scope, scopeId, status);
+
+-- Track which key was used per task (last 5 chars only)
+ALTER TABLE agent_tasks ADD COLUMN credentialKeySuffix TEXT;

package/src/commands/runner.ts

@@ -18,7 +18,8 @@ import {
   type ProviderSessionConfig,
 } from "../providers/index.ts";
 import { getContextWindowSize } from "../utils/context-window.ts";
-import { resolveCredentialPools } from "../utils/credentials.ts";
+import { type CredentialSelection, resolveCredentialPools } from "../utils/credentials.ts";
+import { parseRateLimitResetTime } from "../utils/error-tracker.ts";
 import { prettyPrintLine, prettyPrintStderr } from "../utils/pretty-print.ts";
 import { detectVcsProvider } from "../vcs/index.ts";
 import { interpolate } from "../workflows/template.ts";
@@ -178,12 +179,17 @@ async function closeAgent(config: ApiConfig, role: string): Promise<void> {
  * Falls back to baseEnv on any error (network, parse, etc).
  * Credential env vars with comma-separated values get one randomly selected.
  */
+interface ResolvedEnvResult {
+  env: Record<string, string | undefined>;
+  credentialSelections: CredentialSelection[];
+}
+
 async function fetchResolvedEnv(
   apiUrl: string,
   apiKey: string,
   agentId: string,
   baseEnv: Record<string, string | undefined> = process.env,
-): Promise<Record<string, string | undefined>> {
+): Promise<ResolvedEnvResult> {
   const env: Record<string, string | undefined> = { ...baseEnv };
 
   if (apiUrl && agentId) {
@@ -213,8 +219,9 @@ async function fetchResolvedEnv(
     }
   }
 
-  resolveCredentialPools(env);
-  return env;
+  const credentialSelections = await resolveCredentialPools(env, { apiUrl, apiKey });
+
+  return { env, credentialSelections };
 }
 
 /** Tools that produce noise — skip auto-progress for these */
@@ -545,6 +552,64 @@ async function ensureTaskFinished(
   }
 }
 
+/** Report key usage to the API (fire-and-forget) */
+async function reportKeyUsage(
+  apiUrl: string,
+  apiKey: string,
+  keyType: string,
+  selection: CredentialSelection,
+  taskId?: string,
+): Promise<void> {
+  try {
+    await fetch(`${apiUrl}/api/keys/report-usage`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        keyType,
+        keySuffix: selection.keySuffix,
+        keyIndex: selection.index,
+        taskId,
+      }),
+    });
+  } catch {
+    // Non-blocking
+  }
+}
+
+/** Report a rate-limited key to the API (fire-and-forget) */
+async function reportKeyRateLimit(
+  apiUrl: string,
+  apiKey: string,
+  keyType: string,
+  keySuffix: string,
+  keyIndex: number,
+  rateLimitedUntil: string,
+): Promise<void> {
+  try {
+    await fetch(`${apiUrl}/api/keys/report-rate-limit`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        keyType,
+        keySuffix,
+        keyIndex,
+        rateLimitedUntil,
+      }),
+    });
+    console.log(
+      `[credentials] Reported key ...${keySuffix} as rate-limited until ${rateLimitedUntil}`,
+    );
+  } catch {
+    // Non-blocking
+  }
+}
+
 /**
  * Pause a task via the API (for graceful shutdown).
  * Unlike marking as failed, paused tasks can be resumed after container restart.
@@ -775,6 +840,12 @@ interface RunningTask {
   cursorUpdates?: Array<{ channelId: string; ts: string }>;
   /** Resolved working directory for VCS detection */
   workingDir?: string;
+  /** Credential tracking: which key was used for this task */
+  credentialInfo?: {
+    keyType: string;
+    keySuffix: string;
+    keyIndex: number;
+  };
 }
 
 /** Runner state for tracking concurrent tasks */
@@ -1395,7 +1466,18 @@ async function spawnProviderProcess(
   const effectiveTaskId = realTaskId || crypto.randomUUID();
 
   // Resolve env first so we can use MODEL_OVERRIDE from config
-  const freshEnv = await fetchResolvedEnv(opts.apiUrl, opts.apiKey, opts.agentId);
+  const { env: freshEnv, credentialSelections } = await fetchResolvedEnv(
+    opts.apiUrl,
+    opts.apiKey,
+    opts.agentId,
+  );
+
+  // Report which key was selected for this task (fire-and-forget)
+  if (credentialSelections.length > 0 && realTaskId) {
+    for (const sel of credentialSelections) {
+      reportKeyUsage(opts.apiUrl, opts.apiKey, sel.keyType, sel, realTaskId).catch(() => {});
+    }
+  }
 
   // Propagate agent-fs config to process.env so getBasePrompt() can read them
   // (fetchResolvedEnv returns a new object, doesn't update process.env)
@@ -1729,6 +1811,16 @@ async function spawnProviderProcess(
     return result;
   });
 
+  // Build credential info for rate limit tracking
+  const primarySelection = credentialSelections[0];
+  const credentialInfo = primarySelection
+    ? {
+        keyType: primarySelection.keyType,
+        keySuffix: primarySelection.keySuffix,
+        keyIndex: primarySelection.index,
+      }
+    : undefined;
+
   const runningTask: RunningTask = {
     taskId: effectiveTaskId,
     session,
@@ -1736,6 +1828,7 @@ async function spawnProviderProcess(
     startTime: new Date(),
     promise,
     result: null,
+    credentialInfo,
   };
 
   // Non-blocking completion tracking
@@ -1766,7 +1859,7 @@ async function runProviderIteration(
     cwd?: string;
   },
 ): Promise<ProviderResult> {
-  const freshEnv = await fetchResolvedEnv(opts.apiUrl, opts.apiKey, opts.agentId);
+  const { env: freshEnv } = await fetchResolvedEnv(opts.apiUrl, opts.apiKey, opts.agentId);
   const model = (freshEnv.MODEL_OVERRIDE as string) || "";
 
   const config: ProviderSessionConfig = {
@@ -1811,6 +1904,7 @@ async function checkCompletedProcesses(
     triggerType?: string;
     cursorUpdates?: Array<{ channelId: string; ts: string }>;
     workingDir?: string;
+    credentialInfo?: RunningTask["credentialInfo"];
   }> = [];
 
   for (const [taskId, task] of state.activeTasks) {
@@ -1825,12 +1919,13 @@ async function checkCompletedProcesses(
         triggerType: task.triggerType,
         cursorUpdates: task.cursorUpdates,
         workingDir: task.workingDir,
+        credentialInfo: task.credentialInfo,
       });
     }
   }
 
   // Remove completed tasks from the map and ensure they're marked as finished
-  for (const { taskId, result, cursorUpdates, workingDir } of completedTasks) {
+  for (const { taskId, result, cursorUpdates, workingDir, credentialInfo } of completedTasks) {
     state.activeTasks.delete(taskId);
 
     if (apiConfig) {
@@ -1849,6 +1944,28 @@ async function checkCompletedProcesses(
       if (result.exitCode !== 0 && result.failureReason) {
         failureReason = result.failureReason;
         console.log(`[${role}] Detected error for task ${taskId.slice(0, 8)}: ${failureReason}`);
+
+        // If rate-limited and we know which key was used, report it
+        if (credentialInfo && /rate.?limit/i.test(failureReason)) {
+          // Try to extract reset time from the error message (e.g. "resets 3pm (UTC)")
+          const parsedResetTime = parseRateLimitResetTime(failureReason);
+          const defaultCooldownMs = 5 * 60 * 1000;
+          const rateLimitedUntil =
+            parsedResetTime ?? new Date(Date.now() + defaultCooldownMs).toISOString();
+          if (parsedResetTime) {
+            console.log(
+              `[credentials] Parsed rate limit reset time from error: ${parsedResetTime}`,
+            );
+          }
+          reportKeyRateLimit(
+            apiConfig.apiUrl,
+            apiConfig.apiKey,
+            credentialInfo.keyType,
+            credentialInfo.keySuffix,
+            credentialInfo.keyIndex,
+            rateLimitedUntil,
+          ).catch(() => {});
+        }
       }
       await ensureTaskFinished(apiConfig, role, taskId, result.exitCode, failureReason);