@desplega.ai/agent-swarm 1.56.5 → 1.57.0

package/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Agent Swarm API",
-    "version": "1.56.4",
+    "version": "1.56.5",
     "description": "Multi-agent orchestration API for Claude Code, Codex, and Gemini CLI. Enables task distribution, agent communication, and service discovery.\n\nMCP tools are documented separately in [MCP.md](./MCP.md)."
   },
   "servers": [
@@ -1415,6 +1415,237 @@
         }
       }
     },
+    "/api/keys/report-usage": {
+      "post": {
+        "summary": "Record which API key was used for a task",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "keyType": {
+                    "type": "string"
+                  },
+                  "keySuffix": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 10
+                  },
+                  "keyIndex": {
+                    "type": "integer",
+                    "minimum": 0
+                  },
+                  "taskId": {
+                    "type": "string",
+                    "format": "uuid"
+                  },
+                  "scope": {
+                    "type": "string"
+                  },
+                  "scopeId": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "keyType",
+                  "keySuffix",
+                  "keyIndex"
+                ]
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Usage recorded"
+          },
+          "400": {
+            "description": "Validation error"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
+    "/api/keys/report-rate-limit": {
+      "post": {
+        "summary": "Mark an API key as rate-limited",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "keyType": {
+                    "type": "string"
+                  },
+                  "keySuffix": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 10
+                  },
+                  "keyIndex": {
+                    "type": "integer",
+                    "minimum": 0
+                  },
+                  "rateLimitedUntil": {
+                    "type": "string",
+                    "format": "date-time"
+                  },
+                  "scope": {
+                    "type": "string"
+                  },
+                  "scopeId": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "keyType",
+                  "keySuffix",
+                  "keyIndex",
+                  "rateLimitedUntil"
+                ]
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Key marked as rate-limited"
+          },
+          "400": {
+            "description": "Validation error"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
+    "/api/keys/available": {
+      "get": {
+        "summary": "Get available (non-rate-limited) key indices for a credential type",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "parameters": [
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": true,
+            "name": "keyType",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "integer",
+              "minimum": 1
+            },
+            "required": true,
+            "name": "totalKeys",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scope",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scopeId",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "List of available key indices"
+          },
+          "400": {
+            "description": "Validation error"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
+    "/api/keys/status": {
+      "get": {
+        "summary": "Get all API key status records",
+        "tags": [
+          "API Keys"
+        ],
+        "security": [
+          {
+            "bearerAuth": []
+          }
+        ],
+        "parameters": [
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "keyType",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scope",
+            "in": "query"
+          },
+          {
+            "schema": {
+              "type": "string"
+            },
+            "required": false,
+            "name": "scopeId",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "List of key status records"
+          },
+          "401": {
+            "description": "Unauthorized"
+          }
+        }
+      }
+    },
     "/api/events": {
       "post": {
         "summary": "Store a single event",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@desplega.ai/agent-swarm",
-  "version": "1.56.5",
+  "version": "1.57.0",
   "description": "Multi-agent orchestration for Claude Code, Codex, Gemini CLI, and other AI coding assistants",
   "license": "MIT",
   "author": "desplega.sh <contact@desplega.sh>",

package/src/be/db.ts

@@ -727,6 +727,8 @@ type AgentTaskRow = {
   totalContextTokensUsed: number | null;
   contextWindowSize: number | null;
   was_paused: number;
+  credentialKeySuffix: string | null;
+  credentialKeyType: string | null;
 };
 
 function rowToAgentTask(row: AgentTaskRow): AgentTask {
@@ -780,6 +782,8 @@ function rowToAgentTask(row: AgentTaskRow): AgentTask {
     output: row.output ?? undefined,
     progress: row.progress ?? undefined,
     wasPaused: !!row.was_paused,
+    credentialKeySuffix: row.credentialKeySuffix ?? undefined,
+    credentialKeyType: row.credentialKeyType ?? undefined,
   };
 }
 
@@ -7487,3 +7491,196 @@ export function getContextSummaryByTaskId(taskId: string): ContextSummary {
     snapshotCount: countRow?.cnt ?? 0,
   };
 }
+
+// ─── API Key Pool Tracking ───────────────────────────────────────────────────
+
+export interface ApiKeyStatus {
+  id: string;
+  keyType: string;
+  keySuffix: string;
+  keyIndex: number;
+  scope: string;
+  scopeId: string | null;
+  status: string;
+  rateLimitedUntil: string | null;
+  lastUsedAt: string | null;
+  lastRateLimitAt: string | null;
+  totalUsageCount: number;
+  rateLimitCount: number;
+  createdAt: string;
+  updatedAt: string;
+}
+
+/**
+ * Get available (non-rate-limited) key indices for a credential type.
+ * Automatically clears expired rate limits before returning.
+ */
+export function getAvailableKeyIndices(
+  keyType: string,
+  totalKeys: number,
+  scope = "global",
+  scopeId: string | null = null,
+): number[] {
+  const now = new Date().toISOString();
+  const db = getDb();
+  const effectiveScopeId = scopeId ?? "";
+
+  // Auto-clear expired rate limits
+  db.prepare(
+    `UPDATE api_key_status
+     SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
+     WHERE keyType = ? AND scope = ? AND scopeId = ?
+       AND status = 'rate_limited' AND rateLimitedUntil IS NOT NULL AND rateLimitedUntil <= ?`,
+  ).run(now, keyType, scope, effectiveScopeId, now);
+
+  // Get currently rate-limited key indices
+  const rateLimited = db
+    .prepare<{ keyIndex: number }, [string, string, string]>(
+      `SELECT keyIndex FROM api_key_status
+       WHERE keyType = ? AND scope = ? AND scopeId = ?
+         AND status = 'rate_limited'`,
+    )
+    .all(keyType, scope, effectiveScopeId);
+
+  const blockedIndices = new Set(rateLimited.map((r) => r.keyIndex));
+  const available: number[] = [];
+  for (let i = 0; i < totalKeys; i++) {
+    if (!blockedIndices.has(i)) available.push(i);
+  }
+  return available;
+}
+
+/**
+ * Record that a key was used for a task (upsert key status + update task).
+ */
+export function recordKeyUsage(
+  keyType: string,
+  keySuffix: string,
+  keyIndex: number,
+  taskId: string | null,
+  scope = "global",
+  scopeId: string | null = null,
+): void {
+  const now = new Date().toISOString();
+  const db = getDb();
+  const effectiveScopeId = scopeId ?? "";
+
+  // Upsert key status record
+  db.prepare(
+    `INSERT INTO api_key_status (keyType, keySuffix, keyIndex, scope, scopeId, lastUsedAt, totalUsageCount, updatedAt)
+     VALUES (?, ?, ?, ?, ?, ?, 1, ?)
+     ON CONFLICT(keyType, keySuffix, scope, scopeId)
+     DO UPDATE SET
+       lastUsedAt = excluded.lastUsedAt,
+       totalUsageCount = totalUsageCount + 1,
+       keyIndex = excluded.keyIndex,
+       updatedAt = excluded.updatedAt`,
+  ).run(keyType, keySuffix, keyIndex, scope, effectiveScopeId, now, now);
+
+  // Record which key was used on the task
+  if (taskId) {
+    db.prepare(
+      "UPDATE agent_tasks SET credentialKeySuffix = ?, credentialKeyType = ? WHERE id = ?",
+    ).run(keySuffix, keyType, taskId);
+  }
+}
+
+/**
+ * Mark a key as rate-limited with a retry-after timestamp.
+ */
+export function markKeyRateLimited(
+  keyType: string,
+  keySuffix: string,
+  keyIndex: number,
+  rateLimitedUntil: string,
+  scope = "global",
+  scopeId: string | null = null,
+): void {
+  const now = new Date().toISOString();
+  const effectiveScopeId = scopeId ?? "";
+  getDb()
+    .prepare(
+      `INSERT INTO api_key_status (keyType, keySuffix, keyIndex, scope, scopeId, status, rateLimitedUntil, lastRateLimitAt, rateLimitCount, updatedAt)
+       VALUES (?, ?, ?, ?, ?, 'rate_limited', ?, ?, 1, ?)
+       ON CONFLICT(keyType, keySuffix, scope, scopeId)
+       DO UPDATE SET
+         status = 'rate_limited',
+         rateLimitedUntil = excluded.rateLimitedUntil,
+         lastRateLimitAt = excluded.lastRateLimitAt,
+         rateLimitCount = rateLimitCount + 1,
+         keyIndex = excluded.keyIndex,
+         updatedAt = excluded.updatedAt`,
+    )
+    .run(keyType, keySuffix, keyIndex, scope, effectiveScopeId, rateLimitedUntil, now, now);
+}
+
+/**
+ * Get all key status records for a credential type.
+ */
+export function getKeyStatuses(
+  keyType?: string,
+  scope?: string,
+  scopeId?: string | null,
+): ApiKeyStatus[] {
+  const db = getDb();
+  const conditions: string[] = [];
+  const params: string[] = [];
+
+  if (keyType) {
+    conditions.push("keyType = ?");
+    params.push(keyType);
+  }
+  if (scope) {
+    conditions.push("scope = ?");
+    params.push(scope);
+    if (scopeId !== undefined) {
+      conditions.push("scopeId = ?");
+      params.push(scopeId ?? "");
+    }
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  return db
+    .prepare<ApiKeyStatus, string[]>(`SELECT * FROM api_key_status ${where} ORDER BY keyIndex`)
+    .all(...params);
+}
+
+export interface KeyCostSummary {
+  keyType: string;
+  keySuffix: string;
+  totalCost: number;
+  totalInputTokens: number;
+  totalOutputTokens: number;
+  taskCount: number;
+}
+
+/**
+ * Aggregate cost data per API key by joining session_costs through agent_tasks.
+ */
+export function getKeyCostSummary(keyType?: string): KeyCostSummary[] {
+  const db = getDb();
+  const conditions = ["t.credentialKeySuffix IS NOT NULL"];
+  const params: string[] = [];
+
+  if (keyType) {
+    conditions.push("t.credentialKeyType = ?");
+    params.push(keyType);
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  return db
+    .prepare<KeyCostSummary, string[]>(
+      `SELECT
+        t.credentialKeyType as keyType,
+        t.credentialKeySuffix as keySuffix,
+        COALESCE(SUM(sc.totalCostUsd), 0) as totalCost,
+        COALESCE(SUM(sc.inputTokens), 0) as totalInputTokens,
+        COALESCE(SUM(sc.outputTokens), 0) as totalOutputTokens,
+        COUNT(DISTINCT sc.taskId) as taskCount
+      FROM session_costs sc
+      JOIN agent_tasks t ON sc.taskId = t.id
+      ${where}
+      GROUP BY t.credentialKeyType, t.credentialKeySuffix`,
+    )
+    .all(...params);
+}

package/src/be/migrations/028_api_key_tracking.sql

@@ -0,0 +1,38 @@
+-- Track API key pool status for rate limit awareness and automatic rotation
+CREATE TABLE IF NOT EXISTS api_key_status (
+  id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
+  -- Which credential type: 'CLAUDE_CODE_OAUTH_TOKEN' or 'ANTHROPIC_API_KEY'
+  keyType TEXT NOT NULL,
+  -- Last 5 characters of the key (for identification without storing secrets)
+  keySuffix TEXT NOT NULL,
+  -- Position in the comma-separated credential pool (0-based)
+  keyIndex INTEGER NOT NULL,
+  -- Scope mirrors swarm_config (global, agent, repo)
+  scope TEXT NOT NULL DEFAULT 'global',
+  scopeId TEXT NOT NULL DEFAULT '',
+  -- Current status
+  status TEXT NOT NULL DEFAULT 'available' CHECK(status IN ('available', 'rate_limited')),
+  -- When the rate limit expires (ISO timestamp from retry-after)
+  rateLimitedUntil TEXT,
+  -- Tracking timestamps
+  lastUsedAt TEXT,
+  lastRateLimitAt TEXT,
+  -- Counters
+  totalUsageCount INTEGER NOT NULL DEFAULT 0,
+  rateLimitCount INTEGER NOT NULL DEFAULT 0,
+  -- Metadata
+  createdAt TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  updatedAt TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  -- Unique constraint: one entry per key type + suffix + scope.
+  -- Known limitation: two different keys sharing the same last 5 chars will collide,
+  -- causing the ON CONFLICT upsert to overwrite keyIndex. In practice this is rare
+  -- (1 in ~60M for random alphanumeric suffixes) and the impact is minor (wrong index
+  -- logged, but rate-limit tracking still works since suffix is the real identifier).
+  UNIQUE(keyType, keySuffix, scope, scopeId)
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_key_status_lookup
+  ON api_key_status(keyType, scope, scopeId, status);
+
+-- Track which key was used per task (last 5 chars only)
+ALTER TABLE agent_tasks ADD COLUMN credentialKeySuffix TEXT;

package/src/be/migrations/029_task_credential_key_type.sql

@@ -0,0 +1,2 @@
+-- Store which credential type (env var name) was used per task
+ALTER TABLE agent_tasks ADD COLUMN credentialKeyType TEXT;