@desplega.ai/agent-swarm 1.92.2 → 1.93.0

package/src/be/skill-sync.ts

@@ -6,85 +6,19 @@
  * so Claude Code, Pi, and Codex discover them natively.
  *
  * This runs on the API side — workers call it via POST /api/skills/sync-filesystem.
+ * The actual FS write logic lives in the worker-safe src/utils/skill-fs-writer.ts
+ * so workers can also call it locally with their own homedir().
  */
 
-import type { Dirent } from "node:fs";
-import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
-import { dirname, join } from "node:path";
+import {
+  type SkillFsEntry,
+  type SkillSyncResult,
+  writeSkillsToFilesystem,
+} from "../utils/skill-fs-writer";
 import { getAgentSkills, getSkillFiles } from "./db";
 
-export interface SkillSyncResult {
-  synced: number;
-  removed: number;
-  errors: string[];
-}
-
-/**
- * Marker file written into every swarm-managed skill directory. Cleanup
- * only ever removes directories that contain this marker, so unrelated
- * personal skills the user installed via the harness's own tooling (e.g.
- * `codex skills add ...` writing into `~/.codex/skills/<name>/`) are left
- * untouched even when the API server shares a HOME with the worker (local
- * dev). See `~/.codex/skills` blast-radius note in PR #555.
- */
-const SWARM_MARKER_FILE = ".swarm-managed";
-
-function reconcileManagedSkillFiles(skillDir: string, currentRelativeFiles: Set<string>): number {
-  if (!existsSync(join(skillDir, SWARM_MARKER_FILE))) return 0;
-
-  let removed = 0;
-
-  const walk = (dir: string, relativeDir = ""): boolean => {
-    let entries: Dirent[];
-    try {
-      entries = readdirSync(dir, { withFileTypes: true });
-    } catch {
-      return false;
-    }
-
-    let hasEntries = false;
-    for (const entry of entries) {
-      const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
-      const fullPath = join(dir, entry.name);
-
-      if (entry.isDirectory()) {
-        const childHasEntries = walk(fullPath, relativePath);
-        if (!childHasEntries) {
-          try {
-            rmSync(fullPath, { recursive: true, force: true });
-          } catch {
-            hasEntries = true;
-          }
-        } else {
-          hasEntries = true;
-        }
-        continue;
-      }
-
-      if (
-        relativePath === "SKILL.md" ||
-        relativePath === SWARM_MARKER_FILE ||
-        currentRelativeFiles.has(relativePath)
-      ) {
-        hasEntries = true;
-        continue;
-      }
-
-      try {
-        rmSync(fullPath, { force: true });
-        removed++;
-      } catch {
-        hasEntries = true;
-      }
-    }
-
-    return hasEntries;
-  };
-
-  walk(skillDir);
-  return removed;
-}
+export type { SkillSyncResult };
 
 /**
  * Sync agent's installed skills to the filesystem.
@@ -92,6 +26,9 @@ function reconcileManagedSkillFiles(skillDir: string, currentRelativeFiles: Set<
  * For simple skills (content in DB): writes SKILL.md to ~/.claude/skills/<name>/
  * For DB-backed complex skills: writes SKILL.md plus bundled skill_files rows.
  * Legacy complex skills without skill_files remain handled by npx in entrypoint.
+ *
+ * API-side adapter: fetches skill data from DB, builds SkillFsEntry[], then
+ * delegates all FS writes to writeSkillsToFilesystem() from skill-fs-writer.ts.
  */
 export function syncSkillsToFilesystem(
   agentId: string,
@@ -100,112 +37,24 @@ export function syncSkillsToFilesystem(
 ): SkillSyncResult {
   const skills = getAgentSkills(agentId);
   const home = homeOverride ?? homedir();
-  const errors: string[] = [];
-  let synced = 0;
-  let removed = 0;
-
-  // Directories to write to
-  const skillDirs: string[] = [];
-  if (harnessType === "claude" || harnessType === "all") {
-    skillDirs.push(join(home, ".claude", "skills"));
-  }
-  if (harnessType === "pi" || harnessType === "all") {
-    skillDirs.push(join(home, ".pi", "agent", "skills"));
-  }
-  if (harnessType === "codex" || harnessType === "all") {
-    skillDirs.push(join(home, ".codex", "skills"));
-  }
-
-  // Ensure base dirs exist
-  for (const dir of skillDirs) {
-    mkdirSync(dir, { recursive: true });
-  }
-
-  // Track which skill names we write (for cleanup)
-  const writtenNames = new Set<string>();
-
-  for (const skill of skills) {
-    if (!skill.isActive || !skill.isEnabled) continue;
-    const bundledFiles = skill.isComplex ? getSkillFiles(skill.id) : [];
-    if (skill.isComplex && bundledFiles.length === 0) continue; // Legacy complex skills handled by npx
-    if (!skill.content) continue;
-
-    // Sanitize skill name to prevent path traversal (strip /, .., and non-safe chars)
-    const safeName = skill.name.replace(/[^a-zA-Z0-9_-]/g, "_");
-    if (!safeName) continue;
-
-    writtenNames.add(safeName);
-    const currentBundledFilePaths = new Set(
-      bundledFiles.filter((file) => !file.isBinary).map((file) => file.path),
-    );
-
-    for (const baseDir of skillDirs) {
-      const skillDir = join(baseDir, safeName);
-      const skillFile = join(skillDir, "SKILL.md");
-      const markerFile = join(skillDir, SWARM_MARKER_FILE);
-
-      try {
-        mkdirSync(skillDir, { recursive: true });
-        removed += reconcileManagedSkillFiles(skillDir, currentBundledFilePaths);
-        writeFileSync(skillFile, skill.content, "utf-8");
-        writeFileSync(markerFile, "", "utf-8");
-        synced++;
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : "Unknown error";
-        errors.push(`${skill.name} -> ${skillDir}: ${msg}`);
-        console.error(
-          `[skill-sync] Failed to write SKILL.md for ${skill.name} to ${skillDir}: ${msg}`,
-        );
-      }
-
-      for (const file of bundledFiles) {
-        if (file.isBinary) {
-          console.log(`[skill-sync] Skipping binary skill file ${skill.name}/${file.path}`);
-          continue;
-        }
-
-        const targetPath = join(skillDir, file.path);
-        try {
-          mkdirSync(dirname(targetPath), { recursive: true });
-          writeFileSync(targetPath, file.content, "utf-8");
-        } catch (err) {
-          const msg = err instanceof Error ? err.message : "Unknown error";
-          errors.push(`${skill.name}/${file.path} -> ${targetPath}: ${msg}`);
-          console.error(
-            `[skill-sync] Failed to write bundled file ${skill.name}/${file.path} to ${targetPath}: ${msg}`,
-          );
-        }
-      }
-    }
-  }
-
-  // Cleanup: only remove directories WE previously created (marker file
-  // present). Leaves user-installed personal skills alone — important on
-  // local dev where ~/.codex/skills holds skills the user installed
-  // outside the swarm.
-  for (const baseDir of skillDirs) {
-    if (!existsSync(baseDir)) continue;
-
-    try {
-      const existing = readdirSync(baseDir, { withFileTypes: true });
-      for (const entry of existing) {
-        if (!entry.isDirectory()) continue;
-        if (writtenNames.has(entry.name)) continue;
-        const skillDir = join(baseDir, entry.name);
-        if (!existsSync(join(skillDir, SWARM_MARKER_FILE))) continue;
-        try {
-          rmSync(skillDir, { recursive: true, force: true });
-          removed++;
-        } catch {
-          // Non-fatal — skip cleanup errors
-        }
-      }
-    } catch {
-      // Non-fatal — skip if we can't read the directory
-    }
-  }
 
-  return { synced, removed, errors };
+  const entries: SkillFsEntry[] = skills.map((skill) => ({
+    id: skill.id,
+    name: skill.name,
+    content: skill.content ?? null,
+    isComplex: skill.isComplex,
+    isEnabled: skill.isEnabled,
+    isActive: skill.isActive,
+    files: skill.isComplex
+      ? getSkillFiles(skill.id).map((f) => ({
+          path: f.path,
+          content: f.content,
+          isBinary: f.isBinary,
+        }))
+      : [],
+  }));
+
+  return writeSkillsToFilesystem(entries, harnessType, home);
 }
 
 export interface SkillsSignature {

package/src/commands/runner.ts

@@ -1110,6 +1110,35 @@ async function reportKeyRateLimit(
   }
 }
 
+/** Clear a stale rate-limit record after a successful task (fire-and-forget) */
+async function reportKeyClearRateLimit(
+  apiUrl: string,
+  apiKey: string,
+  keyType: string,
+  keySuffix: string,
+): Promise<void> {
+  try {
+    const resp = await fetch(`${apiUrl}/api/keys/clear-rate-limit`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({ keyType, keySuffix }),
+    });
+    if (resp.ok) {
+      const data = (await resp.json()) as { cleared?: boolean };
+      if (data.cleared) {
+        console.log(
+          `[credentials] Cleared stale rate-limit for ...${keySuffix} after successful task`,
+        );
+      }
+    }
+  } catch {
+    // Non-blocking
+  }
+}
+
 /**
  * Supersede a task via the API (for graceful shutdown / context-limit /
  * operator-triggered). Returns `{ ok: true, resumeTaskId }` on success.
@@ -1472,6 +1501,8 @@ interface RunningTask {
    * provider before it completed, and vice versa).
    */
   hasLocalEnvironment: boolean;
+  /** Harness variant captured on session_init (e.g. "bridge" or "stock") */
+  harnessVariant?: string;
 }
 
 /** Runner state for tracking concurrent tasks */
@@ -1590,6 +1621,8 @@ async function saveProviderSessionId(
   provider?: ProviderName,
   providerMeta?: Record<string, unknown>,
   model?: string,
+  harnessVariant?: string,
+  harnessVariantMeta?: Record<string, unknown>,
 ): Promise<void> {
   const headers: Record<string, string> = { "Content-Type": "application/json" };
   if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
@@ -1597,13 +1630,45 @@ async function saveProviderSessionId(
   if (provider !== undefined) body.provider = provider;
   if (providerMeta !== undefined) body.providerMeta = providerMeta;
   if (model !== undefined && model !== "") body.model = model;
-  await fetch(`${apiUrl}/api/tasks/${taskId}/claude-session`, {
+  if (harnessVariant !== undefined) body.harnessVariant = harnessVariant;
+  if (harnessVariantMeta !== undefined) body.harnessVariantMeta = harnessVariantMeta;
+  await fetch(`${apiUrl}/api/tasks/${taskId}/session`, {
     method: "PUT",
     headers,
     body: JSON.stringify(body),
   });
 }
 
+async function findBridgeFailureArtifact(cwd: string): Promise<string | undefined> {
+  try {
+    const bridgeDir = `${cwd}/.claude-bridge/runs`;
+    const dir = await Array.fromAsync(
+      new Bun.Glob("*/tmux-pane-final.txt").scan({ cwd: bridgeDir, absolute: true }),
+    );
+    if (dir.length === 0) return undefined;
+    dir.sort();
+    return dir[dir.length - 1];
+  } catch {
+    return undefined;
+  }
+}
+
+async function updateHarnessVariantMeta(
+  apiUrl: string,
+  apiKey: string,
+  taskId: string,
+  claudeSessionId: string,
+  meta: Record<string, unknown>,
+): Promise<void> {
+  const headers: Record<string, string> = { "Content-Type": "application/json" };
+  if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+  await fetch(`${apiUrl}/api/tasks/${taskId}/session`, {
+    method: "PUT",
+    headers,
+    body: JSON.stringify({ claudeSessionId, harnessVariantMeta: meta }),
+  });
+}
+
 /** Cache of tasks that already have VCS linked — prevents repeated gh pr list calls */
 const vcsDetectedTasks = new Set<string>();
 
@@ -2493,10 +2558,19 @@ async function spawnProviderProcess(
   // Resolve Codex OAuth pool slot BEFORE building ProviderSessionConfig so we
   // can pass codexSlot through and the adapter writes token refreshes back to
   // the correct slot key (codex_oauth_<slot>) instead of defaulting to slot 0.
+  //
+  // Always resolve for codex (not just when credentialSelections is empty) so
+  // that if the OPENAI_API_KEY credential is rate-limited we can fail over to
+  // a CODEX_OAUTH slot — even though the keyType differs.
   let oauthSelection: CredentialSelection | undefined;
-  if (adapter.name === "codex" && credentialSelections.length === 0) {
+  if (adapter.name === "codex") {
     oauthSelection = (await resolveCodexOAuthCredentialInfo(opts.apiUrl, opts.apiKey)) ?? undefined;
-    if (oauthSelection && realTaskId) {
+    const oauthIsPrimary =
+      credentialSelections.length === 0 ||
+      (credentialSelections[0]?.isRateLimitFallback &&
+        oauthSelection &&
+        !oauthSelection.isRateLimitFallback);
+    if (oauthSelection && realTaskId && oauthIsPrimary) {
       reportKeyUsage(
         opts.apiUrl,
         opts.apiKey,
@@ -2676,6 +2750,8 @@ async function spawnProviderProcess(
               event.provider,
               event.providerMeta,
               model,
+              event.harnessVariant,
+              event.harnessVariantMeta,
             ).catch((err) => console.warn(`[runner] Failed to save session ID: ${err}`));
           } else {
             // Pool task: save provider session ID on active session so it can be
@@ -3085,8 +3161,23 @@ async function spawnProviderProcess(
       }),
     );
 
-  // Build credential info for rate limit tracking
-  const primarySelection = credentialSelections[0] ?? oauthSelection;
+  // Build credential info for rate limit tracking.
+  // For codex: when OPENAI_API_KEY is rate-limited but CODEX_OAUTH has
+  // available slots (or vice versa), prefer the healthy credential.
+  let primarySelection: CredentialSelection | undefined;
+  const firstCred = credentialSelections[0];
+  if (firstCred && oauthSelection) {
+    if (firstCred.isRateLimitFallback && !oauthSelection.isRateLimitFallback) {
+      primarySelection = oauthSelection;
+      console.log(
+        `[credentials] Cross-keyType failover: ${firstCred.keyType} all rate-limited, using ${oauthSelection.keyType} [...${oauthSelection.keySuffix}]`,
+      );
+    } else {
+      primarySelection = firstCred;
+    }
+  } else {
+    primarySelection = firstCred ?? oauthSelection;
+  }
   const credentialInfo = primarySelection
     ? {
         keyType: primarySelection.keyType,
@@ -3160,7 +3251,14 @@ async function checkCompletedProcesses(
   }
 
   // Remove completed tasks from the map and ensure they're marked as finished
-  for (const { taskId, result, cursorUpdates, workingDir, credentialInfo } of completedTasks) {
+  for (const {
+    taskId,
+    result,
+    cursorUpdates,
+    workingDir,
+    credentialInfo,
+    harnessProvider,
+  } of completedTasks) {
     state.activeTasks.delete(taskId);
     vcsDetectedTasks.delete(taskId);
     vcsCheckTimestamps.delete(taskId);
@@ -3251,9 +3349,28 @@ async function checkCompletedProcesses(
         result.exitCode,
         failureReason,
         result.output,
-        state.harnessProvider,
+        harnessProvider,
       );
 
+      if (result.exitCode === 0 && credentialInfo) {
+        reportKeyClearRateLimit(
+          apiConfig.apiUrl,
+          apiConfig.apiKey,
+          credentialInfo.keyType,
+          credentialInfo.keySuffix,
+        ).catch(() => {});
+      }
+
+      if (result.exitCode !== 0 && harnessProvider === "claude" && workingDir && result.sessionId) {
+        const artifactPath = await findBridgeFailureArtifact(workingDir);
+        if (artifactPath) {
+          console.log(`[${role}] Bridge failure artifact found: ${artifactPath}`);
+          updateHarnessVariantMeta(apiConfig.apiUrl, apiConfig.apiKey, taskId, result.sessionId, {
+            failureArtifact: artifactPath,
+          }).catch((err) => console.warn(`[runner] Failed to update harness variant meta: ${err}`));
+        }
+      }
+
       ensure({
         id: "worker_process_finished",
         flow: "task",

package/src/http/api-keys.ts

@@ -1,6 +1,7 @@
 import type { IncomingMessage, ServerResponse } from "node:http";
 import { z } from "zod";
 import {
+  clearKeyRateLimit,
   getAvailableKeyIndices,
   getKeyCostSummary,
   getKeyStatuses,
@@ -134,6 +135,26 @@ const setKeyName = route({
   auth: { apiKey: true },
 });
 
+const clearRateLimitRoute = route({
+  method: "post",
+  path: "/api/keys/clear-rate-limit",
+  pattern: ["api", "keys", "clear-rate-limit"],
+  summary: "Clear rate-limited status for a key after a successful use proves it is healthy",
+  tags: ["API Keys"],
+  body: z.object({
+    keyType: z.string(),
+    keySuffix: z.string().min(1).max(10),
+    scope: z.string().optional(),
+    scopeId: z.string().optional(),
+  }),
+  responses: {
+    200: { description: "Rate limit cleared (or key was not rate-limited)" },
+    400: { description: "Validation error" },
+    401: { description: "Unauthorized" },
+  },
+  auth: { apiKey: true },
+});
+
 // ─── Handler ─────────────────────────────────────────────────────────────────
 
 export async function handleApiKeys(
@@ -242,5 +263,26 @@ export async function handleApiKeys(
     return true;
   }
 
+  // POST /api/keys/clear-rate-limit
+  if (clearRateLimitRoute.match(req.method, pathSegments)) {
+    const parsed = await clearRateLimitRoute.parse(req, res, pathSegments, queryParams);
+    if (!parsed) return true;
+
+    const { keyType, keySuffix, scope, scopeId } = parsed.body;
+    try {
+      const cleared = clearKeyRateLimit(keyType, keySuffix, scope, scopeId ?? null);
+      json(res, {
+        success: true,
+        cleared,
+        message: cleared
+          ? `Rate limit cleared for ...${keySuffix}`
+          : `Key ...${keySuffix} was not rate-limited`,
+      });
+    } catch (err) {
+      jsonError(res, err instanceof Error ? err.message : "Failed to clear rate limit", 500);
+    }
+    return true;
+  }
+
   return false;
 }

package/src/http/mcp-bridge.ts

@@ -16,7 +16,7 @@ function getBridgeServer(): McpServer {
 }
 
 type RegisteredTool = {
-  handler: Function;
+  handler: (argsOrExtra: unknown, extra?: unknown) => unknown | Promise<unknown>;
   inputSchema?: unknown;
   enabled?: boolean;
 };

package/src/http/memory.ts

@@ -406,6 +406,7 @@ export async function handleMemory(
 
     const { query, agentId, scope, source, sourcePath, limit, offset } = parsed.body;
     const store = getMemoryStore();
+    const pageLimit = Math.min(limit, 100);
     const pathNeedle = sourcePath?.trim().toLowerCase();
     const matchesPath = (p: string | null) =>
       !pathNeedle || (p?.toLowerCase().includes(pathNeedle) ?? false);
@@ -416,11 +417,14 @@ export async function handleMemory(
         const queryEmbedding = await provider.embed(query.trim());
 
         if (!queryEmbedding) {
-          json(res, { results: [], total: 0, mode: "semantic" });
+          json(res, { results: [], total: 0, limit: pageLimit, offset, mode: "semantic" });
           return true;
         }
 
-        const candidateLimit = Math.min(limit, 100) * CANDIDATE_SET_MULTIPLIER;
+        const candidateLimit = Math.min(
+          4096,
+          Math.max(offset + pageLimit, pageLimit) * CANDIDATE_SET_MULTIPLIER,
+        );
         let candidates = store.search(queryEmbedding, agentId ?? "", {
           scope,
           limit: candidateLimit,
@@ -433,10 +437,11 @@ export async function handleMemory(
         if (pathNeedle) {
           candidates = candidates.filter((c) => matchesPath(c.sourcePath));
         }
-        const ranked = rerank(candidates, { limit: Math.min(limit, 100) });
+        const ranked = rerank(candidates, { limit: candidates.length });
+        const page = ranked.slice(offset, offset + pageLimit);
 
         json(res, {
-          results: ranked.map((r) => ({
+          results: page.map((r) => ({
             id: r.id,
             name: r.name,
             content: r.content,
@@ -457,33 +462,25 @@ export async function handleMemory(
             totalChunks: r.totalChunks,
             tags: r.tags,
           })),
-          total: ranked.length,
+          total: candidates.length,
+          limit: pageLimit,
+          offset,
           mode: "semantic",
         });
         return true;
       }
 
-      // When filtering by sourcePath, over-fetch then post-filter so the visible
-      // page isn't gutted by the in-memory filter.
-      const fetchLimit = pathNeedle
-        ? Math.min(500, Math.max(limit * 10, 100))
-        : Math.min(limit, 100);
-      let rows = store.list(agentId ?? "", {
+      const listOptions = {
         scope,
-        limit: fetchLimit,
+        limit: pageLimit,
         offset,
         isLead: true,
-      });
-      if (agentId) {
-        rows = rows.filter((r) => r.agentId === agentId);
-      }
-      if (source) {
-        rows = rows.filter((r) => r.source === source);
-      }
-      if (pathNeedle) {
-        rows = rows.filter((r) => matchesPath(r.sourcePath));
-      }
-      rows = rows.slice(0, Math.min(limit, 100));
+        ownerAgentId: agentId,
+        source,
+        sourcePath: pathNeedle,
+      };
+      const rows = store.list(agentId ?? "", listOptions);
+      const total = store.count(agentId ?? "", listOptions);
 
       json(res, {
         results: rows.map((r) => ({
@@ -504,7 +501,9 @@ export async function handleMemory(
           totalChunks: r.totalChunks,
           tags: r.tags,
         })),
-        total: rows.length,
+        total,
+        limit: pageLimit,
+        offset,
         mode: "list",
       });
     } catch (err) {

package/src/http/tasks.ts

@@ -98,11 +98,11 @@ const createTask = route({
   },
 });
 
-const updateClaudeSession = route({
+const updateSession = route({
   method: "put",
-  path: "/api/tasks/{id}/claude-session",
-  pattern: ["api", "tasks", null, "claude-session"],
-  summary: "Update Claude session ID for a task",
+  path: "/api/tasks/{id}/session",
+  pattern: ["api", "tasks", null, "session"],
+  summary: "Update provider session ID and harness metadata for a task",
   tags: ["Tasks"],
   params: z.object({ id: z.string() }),
   body: z.union([
@@ -121,6 +121,8 @@ const updateClaudeSession = route({
       provider: ProviderNameSchema.exclude(["devin"]).optional(),
       model: z.string().optional(),
       providerMeta: z.object({}).optional(),
+      harnessVariant: z.string().optional(),
+      harnessVariantMeta: z.record(z.string(), z.unknown()).optional(),
     }),
   ]),
   responses: {
@@ -427,8 +429,8 @@ export async function handleTasks(
     return true;
   }
 
-  if (updateClaudeSession.match(req.method, pathSegments)) {
-    const parsed = await updateClaudeSession.parse(req, res, pathSegments, queryParams);
+  if (updateSession.match(req.method, pathSegments)) {
+    const parsed = await updateSession.parse(req, res, pathSegments, queryParams);
     if (!parsed) return true;
     const task = updateTaskClaudeSessionId(
       parsed.params.id,
@@ -436,6 +438,8 @@ export async function handleTasks(
       parsed.body.provider,
       parsed.body.providerMeta,
       parsed.body.model,
+      "harnessVariant" in parsed.body ? parsed.body.harnessVariant : undefined,
+      "harnessVariantMeta" in parsed.body ? parsed.body.harnessVariantMeta : undefined,
     );
     if (!task) {
       jsonError(res, "Task not found", 404);