@desplega.ai/agent-swarm 1.92.2 → 1.93.0

package/src/utils/context-window.ts

@@ -26,6 +26,8 @@
 export const CONTEXT_FORMULA = "input-cache-output" as const;
 
 const CONTEXT_WINDOW_DEFAULTS: Record<string, number> = {
+  // Anthropic Fable / Mythos tier
+  "claude-fable-5": 1_000_000,
   // Anthropic 4.x family
   "claude-opus-4-8": 1_000_000,
   "claude-opus-4-7": 1_000_000,
@@ -45,6 +47,7 @@ const CONTEXT_WINDOW_DEFAULTS: Record<string, number> = {
   "claude-3-sonnet": 200_000,
   "claude-3-haiku": 200_000,
   // Shortnames used by the local-CLI adapter and pi-mono OpenRouter mirror.
+  fable: 1_000_000,
   opus: 1_000_000,
   sonnet: 1_000_000,
   haiku: 200_000,

package/src/utils/credentials.ts

@@ -64,6 +64,8 @@ export interface CredentialSelection {
   keySuffix: string;
   /** Which credential pool env var this selection came from */
   keyType: string;
+  /** True when all indices for this keyType were rate-limited (best-effort pick) */
+  isRateLimitFallback: boolean;
 }
 
 /**
@@ -82,10 +84,19 @@ export function selectCredential(
     .filter(Boolean);
   if (credentials.length <= 1) {
     const selected = value.trim();
-    return { selected, index: 0, total: 1, keySuffix: selected.slice(-5), keyType };
+    const isRateLimitFallback = availableIndices !== undefined && availableIndices.length === 0;
+    return {
+      selected,
+      index: 0,
+      total: 1,
+      keySuffix: selected.slice(-5),
+      keyType,
+      isRateLimitFallback,
+    };
   }
 
   let index: number;
+  let isRateLimitFallback = false;
   if (availableIndices && availableIndices.length > 0) {
     // Pick randomly from available (non-rate-limited) indices
     const validIndices = availableIndices.filter((i) => i >= 0 && i < credentials.length);
@@ -94,17 +105,26 @@ export function selectCredential(
     } else {
       // All available indices out of range — fall back to random from all
       index = Math.floor(Math.random() * credentials.length);
+      isRateLimitFallback = true;
     }
   } else if (availableIndices && availableIndices.length === 0) {
     // All keys are rate-limited — pick randomly anyway (best effort)
     index = Math.floor(Math.random() * credentials.length);
+    isRateLimitFallback = true;
   } else {
     // No availability info — pure random (backward compatible)
     index = Math.floor(Math.random() * credentials.length);
   }
 
   const selected = credentials[index]!;
-  return { selected, index, total: credentials.length, keySuffix: selected.slice(-5), keyType };
+  return {
+    selected,
+    index,
+    total: credentials.length,
+    keySuffix: selected.slice(-5),
+    keyType,
+    isRateLimitFallback,
+  };
 }
 
 /**

package/src/utils/skill-fs-writer.ts

@@ -0,0 +1,220 @@
+/**
+ * Pure, DB-free filesystem writer for agent skills.
+ *
+ * Worker-safe: imports only node:fs / node:os / node:path — no be/db, no bun:sqlite.
+ *
+ * Shared by:
+ *   - API-side: syncSkillsToFilesystem (src/be/skill-sync.ts) which fetches
+ *     SkillFsEntry data from the DB then delegates here.
+ *   - Worker-side: refreshSkillsIfChanged (src/utils/skills-refresh.ts) which
+ *     fetches SkillFsEntry data over HTTP then calls writeSkillsToFilesystem
+ *     with the worker's own homedir(), writing SKILL.md files to the correct
+ *     machine instead of the API box.
+ */
+
+import type { Dirent } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+export interface SkillSyncResult {
+  synced: number;
+  removed: number;
+  errors: string[];
+}
+
+export interface SkillFsEntry {
+  id: string;
+  name: string;
+  content: string | null;
+  isComplex: boolean;
+  isEnabled: boolean;
+  isActive: boolean;
+  files: { path: string; content: string; isBinary: boolean }[];
+}
+
+/**
+ * Marker file written into every swarm-managed skill directory. Cleanup
+ * only ever removes directories that contain this marker, so unrelated
+ * personal skills the user installed via the harness's own tooling (e.g.
+ * `codex skills add ...` writing into `~/.codex/skills/<name>/`) are left
+ * untouched even when the API server shares a HOME with the worker (local
+ * dev). See `~/.codex/skills` blast-radius note in PR #555.
+ */
+export const SWARM_MARKER_FILE = ".swarm-managed";
+
+function reconcileManagedSkillFiles(skillDir: string, currentRelativeFiles: Set<string>): number {
+  if (!existsSync(join(skillDir, SWARM_MARKER_FILE))) return 0;
+
+  let removed = 0;
+
+  const walk = (dir: string, relativeDir = ""): boolean => {
+    let entries: Dirent[];
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return false;
+    }
+
+    let hasEntries = false;
+    for (const entry of entries) {
+      const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
+      const fullPath = join(dir, entry.name);
+
+      if (entry.isDirectory()) {
+        const childHasEntries = walk(fullPath, relativePath);
+        if (!childHasEntries) {
+          try {
+            rmSync(fullPath, { recursive: true, force: true });
+          } catch {
+            hasEntries = true;
+          }
+        } else {
+          hasEntries = true;
+        }
+        continue;
+      }
+
+      if (
+        relativePath === "SKILL.md" ||
+        relativePath === SWARM_MARKER_FILE ||
+        currentRelativeFiles.has(relativePath)
+      ) {
+        hasEntries = true;
+        continue;
+      }
+
+      try {
+        rmSync(fullPath, { force: true });
+        removed++;
+      } catch {
+        hasEntries = true;
+      }
+    }
+
+    return hasEntries;
+  };
+
+  walk(skillDir);
+  return removed;
+}
+
+/**
+ * Write skill entries to the filesystem under the given home directory.
+ *
+ * For simple skills (non-complex): writes SKILL.md only.
+ * For DB-backed complex skills: writes SKILL.md plus bundled files.
+ * Skips legacy complex skills with no files (handled by npx in entrypoint).
+ * Binary files are skipped.
+ * Stale swarm-managed skill directories are cleaned up.
+ */
+export function writeSkillsToFilesystem(
+  entries: SkillFsEntry[],
+  harnessType: "claude" | "pi" | "codex" | "all" = "all",
+  home: string,
+): SkillSyncResult {
+  const errors: string[] = [];
+  let synced = 0;
+  let removed = 0;
+
+  // Directories to write to
+  const skillDirs: string[] = [];
+  if (harnessType === "claude" || harnessType === "all") {
+    skillDirs.push(join(home, ".claude", "skills"));
+  }
+  if (harnessType === "pi" || harnessType === "all") {
+    skillDirs.push(join(home, ".pi", "agent", "skills"));
+  }
+  if (harnessType === "codex" || harnessType === "all") {
+    skillDirs.push(join(home, ".codex", "skills"));
+  }
+
+  // Ensure base dirs exist
+  for (const dir of skillDirs) {
+    mkdirSync(dir, { recursive: true });
+  }
+
+  // Track which skill names we write (for cleanup)
+  const writtenNames = new Set<string>();
+
+  for (const skill of entries) {
+    if (!skill.isActive || !skill.isEnabled) continue;
+    if (skill.isComplex && skill.files.length === 0) continue; // Legacy complex skills handled by npx
+    if (!skill.content) continue;
+
+    // Sanitize skill name to prevent path traversal (strip /, .., and non-safe chars)
+    const safeName = skill.name.replace(/[^a-zA-Z0-9_-]/g, "_");
+    if (!safeName) continue;
+
+    writtenNames.add(safeName);
+    const currentBundledFilePaths = new Set(
+      skill.files.filter((file) => !file.isBinary).map((file) => file.path),
+    );
+
+    for (const baseDir of skillDirs) {
+      const skillDir = join(baseDir, safeName);
+      const skillFile = join(skillDir, "SKILL.md");
+      const markerFile = join(skillDir, SWARM_MARKER_FILE);
+
+      try {
+        mkdirSync(skillDir, { recursive: true });
+        removed += reconcileManagedSkillFiles(skillDir, currentBundledFilePaths);
+        writeFileSync(skillFile, skill.content, "utf-8");
+        writeFileSync(markerFile, "", "utf-8");
+        synced++;
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : "Unknown error";
+        errors.push(`${skill.name} -> ${skillDir}: ${msg}`);
+        console.error(
+          `[skill-fs-writer] Failed to write SKILL.md for ${skill.name} to ${skillDir}: ${msg}`,
+        );
+      }
+
+      for (const file of skill.files) {
+        if (file.isBinary) {
+          console.log(`[skill-fs-writer] Skipping binary skill file ${skill.name}/${file.path}`);
+          continue;
+        }
+
+        const targetPath = join(skillDir, file.path);
+        try {
+          mkdirSync(dirname(targetPath), { recursive: true });
+          writeFileSync(targetPath, file.content, "utf-8");
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : "Unknown error";
+          errors.push(`${skill.name}/${file.path} -> ${targetPath}: ${msg}`);
+          console.error(
+            `[skill-fs-writer] Failed to write bundled file ${skill.name}/${file.path} to ${targetPath}: ${msg}`,
+          );
+        }
+      }
+    }
+  }
+
+  // Cleanup: only remove directories WE previously created (marker file
+  // present). Leaves user-installed personal skills alone — important on
+  // local dev where ~/.codex/skills holds skills the user installed
+  // outside the swarm.
+  for (const baseDir of skillDirs) {
+    if (!existsSync(baseDir)) continue;
+
+    try {
+      const existing = readdirSync(baseDir, { withFileTypes: true });
+      for (const entry of existing) {
+        if (!entry.isDirectory()) continue;
+        if (writtenNames.has(entry.name)) continue;
+        const skillDir = join(baseDir, entry.name);
+        if (!existsSync(join(skillDir, SWARM_MARKER_FILE))) continue;
+        try {
+          rmSync(skillDir, { recursive: true, force: true });
+          removed++;
+        } catch {
+          // Non-fatal — skip cleanup errors
+        }
+      }
+    } catch {
+      // Non-fatal — skip if we can't read the directory
+    }
+  }
+
+  return { synced, removed, errors };
+}

package/src/utils/skills-refresh.ts

@@ -2,15 +2,27 @@
  * Worker-side per-task skill refresh.
  *
  * Polls the cheap signature endpoint; on a hash mismatch, refetches the
- * full skill list and re-runs filesystem sync (claude/pi/codex dirs). The
- * worker stores the signature returned in the list response so the cached
- * hash always corresponds exactly to the snapshot it acted on — avoids a
- * stale-hash race between the signature and list endpoints.
+ * full skill list and writes SKILL.md files to the worker's local HOME via
+ * writeSkillsToFilesystem() from skill-fs-writer.ts. This ensures newly
+ * created/approved skills land on the worker disk mid-session — no container
+ * restart required.
+ *
+ * Previously Step 3 POSTed to /api/skills/sync-filesystem, which wrote to
+ * the API server's HOME instead of the worker disk. Now Step 3 builds
+ * SkillFsEntry[] from the already-fetched skill data and writes locally.
+ * For complex skills the worker fetches bundled files via N+1 HTTP calls
+ * (acceptable for v1 — simple skills need zero extra fetches).
+ *
+ * The /api/skills/sync-filesystem endpoint is retained for single-box local
+ * dev (where API and worker share a HOME). Workers no longer call it.
  *
  * Transient errors are swallowed (returned as `changed: false`) so a flaky
  * API can't churn the system prompt.
  */
 
+import { homedir } from "node:os";
+import { type SkillFsEntry, writeSkillsToFilesystem } from "./skill-fs-writer";
+
 export type SkillsRefreshContext = {
   apiUrl: string;
   swarmUrl: string;
@@ -27,6 +39,7 @@ export type SkillsRefreshResult = {
 export async function refreshSkillsIfChanged(
   ctx: SkillsRefreshContext,
   lastHashRef: { current: string | null },
+  homeOverride?: string,
 ): Promise<SkillsRefreshResult> {
   const { apiUrl, apiKey, agentId, role } = ctx;
   const authHeaders: Record<string, string> = { "X-Agent-ID": agentId };
@@ -52,70 +65,140 @@ export async function refreshSkillsIfChanged(
     return { changed: false };
   }
 
-  // Step 2: full fetch + sync (only reached when hash differs or first call)
-  let summary: { name: string; description: string }[] | undefined;
+  // Step 2: full fetch (only reached when hash differs or first call)
+  // Keep the full skill rows including content, id, isComplex — data is
+  // already on the wire, was previously discarded.
+  type SkillRow = {
+    id: string;
+    name: string;
+    description: string;
+    content: string | null;
+    isComplex: boolean;
+    isEnabled: boolean;
+    isActive: boolean;
+  };
+  let skillRows: SkillRow[] = [];
   let newHash: string | null = null;
+  let listFetchOk = false;
   try {
     const skillsResp = await fetch(`${apiUrl}/api/agents/${agentId}/skills`, {
       headers: authHeaders,
     });
     if (skillsResp.ok) {
       const skillsData = (await skillsResp.json()) as {
-        skills: { name: string; description: string; isActive: boolean; isEnabled: boolean }[];
+        skills: SkillRow[];
         signature?: string;
       };
-      summary = skillsData.skills
-        .filter((s) => s.isActive && s.isEnabled)
-        .map((s) => ({ name: s.name, description: s.description }));
+      skillRows = skillsData.skills;
       if (typeof skillsData.signature === "string") {
         newHash = skillsData.signature;
       }
+      listFetchOk = true;
     }
   } catch {
-    // Non-fatal — skills are optional
+    // Transient network / parse error — bail out without touching the local FS
+  }
+
+  // Guard: a failed list fetch must not proceed to writeSkillsToFilesystem.
+  // An empty entries array would wipe every swarm-managed skill directory from
+  // the worker disk, which is worse than leaving the cache stale.
+  if (!listFetchOk) {
+    return { changed: false };
   }
 
-  // Step 3: filesystem sync (claude/pi/codex dirs)
+  const summary = skillRows
+    .filter((s) => s.isActive && s.isEnabled)
+    .map((s) => ({ name: s.name, description: s.description }));
+
+  // Step 3: build SkillFsEntry[] and write to THIS worker's local HOME.
+  //
+  // For complex+enabled skills, fetch bundled files via N+1 HTTP calls
+  // (GET /api/skills/:id/files for manifest, then per non-binary file).
+  // Simple skills (the common case) need zero extra fetches.
   let syncOk = false;
   try {
-    const syncHeaders: Record<string, string> = {
-      "Content-Type": "application/json",
-      "X-Agent-ID": agentId,
-    };
-    if (apiKey) syncHeaders.Authorization = `Bearer ${apiKey}`;
-    const syncRes = await fetch(`${apiUrl}/api/skills/sync-filesystem`, {
-      method: "POST",
-      headers: syncHeaders,
-    });
-    if (syncRes.ok) {
-      const syncResult = (await syncRes.json()) as {
-        synced: number;
-        removed: number;
-        errors: string[];
-      };
-      console.log(
-        `[${role}] Skills synced: ${syncResult.synced} written, ${syncResult.removed} removed`,
-      );
-      if (syncResult.errors.length > 0) {
-        console.warn(`[${role}] Skill sync errors: ${syncResult.errors.join(", ")}`);
+    const entries: SkillFsEntry[] = [];
+
+    for (const skill of skillRows) {
+      if (!skill.isActive || !skill.isEnabled) continue;
+
+      const files: { path: string; content: string; isBinary: boolean }[] = [];
+
+      if (skill.isComplex) {
+        // Fetch manifest to know which files exist + which are binary
+        try {
+          const manifestResp = await fetch(`${apiUrl}/api/skills/${skill.id}/files`, {
+            headers: authHeaders,
+          });
+          if (manifestResp.ok) {
+            const manifestData = (await manifestResp.json()) as {
+              files: { path: string; isBinary: boolean }[];
+            };
+
+            // Fetch content for each non-binary file (N+1 — acceptable for v1)
+            for (const manifestEntry of manifestData.files) {
+              if (manifestEntry.isBinary) {
+                files.push({ path: manifestEntry.path, content: "", isBinary: true });
+                continue;
+              }
+              try {
+                const encodedPath = manifestEntry.path.split("/").map(encodeURIComponent).join("/");
+                const fileResp = await fetch(
+                  `${apiUrl}/api/skills/${skill.id}/files/${encodedPath}`,
+                  { headers: authHeaders },
+                );
+                if (fileResp.ok) {
+                  const fileData = (await fileResp.json()) as {
+                    file: { path: string; content: string; isBinary: boolean };
+                  };
+                  files.push({
+                    path: fileData.file.path,
+                    content: fileData.file.content,
+                    isBinary: fileData.file.isBinary,
+                  });
+                }
+              } catch {
+                // Non-fatal — skip this file
+              }
+            }
+          }
+        } catch {
+          // Non-fatal — treat as no files (will skip complex skill per writer logic)
+        }
       }
-      syncOk = true;
-    } else {
-      console.warn(`[${role}] Skill sync failed: HTTP ${syncRes.status}`);
+
+      entries.push({
+        id: skill.id,
+        name: skill.name,
+        content: skill.content ?? null,
+        isComplex: skill.isComplex,
+        isEnabled: skill.isEnabled,
+        isActive: skill.isActive,
+        files,
+      });
+    }
+
+    const writeResult = writeSkillsToFilesystem(entries, "all", homeOverride ?? homedir());
+    console.log(
+      `[${role}] Skills synced: ${writeResult.synced} written, ${writeResult.removed} removed`,
+    );
+    if (writeResult.errors.length > 0) {
+      console.warn(`[${role}] Skill sync errors: ${writeResult.errors.join(", ")}`);
     }
+    syncOk = true;
   } catch (err) {
     console.warn(`[${role}] Skill sync failed: ${(err as Error).message}`);
   }
 
-  if (summary === undefined && newHash === null) {
+  if (skillRows.length === 0 && newHash === null) {
     return { changed: false };
   }
 
-  // Only cache the new hash once the FS sync has actually succeeded —
-  // otherwise a transient sync failure would leave the cached hash matching
+  // Only cache the new hash once the local FS write has actually succeeded —
+  // otherwise a transient write failure would leave the cached hash matching
   // the current signature, causing later polls to short-circuit and the
-  // disk state to stay stale until an unrelated skill mutation. The next
-  // poll re-enters this code path (lastHashRef unchanged) and retries.
+  // disk state to stay stale forever. The next poll re-enters this code path
+  // (lastHashRef unchanged) and retries.
   if (syncOk && newHash !== null) {
     lastHashRef.current = newHash;
   }