@desplega.ai/agent-swarm 1.74.4 → 1.76.0

package/src/commands/runner.ts

@@ -9,6 +9,7 @@ import {
   generateDefaultSoulMd,
   generateDefaultToolsMd,
 } from "../prompts/defaults.ts";
+import { renderMemoriesPrompt } from "../prompts/memories.ts";
 import { configureHttpResolver, resolveTemplateAsync } from "../prompts/resolver.ts";
 import { authJsonToCredentialSelection } from "../providers/codex-oauth/auth-json.js";
 import {
@@ -24,10 +25,17 @@ import { computeBudgetBackoffMs } from "../utils/budget-backoff.ts";
 import { getContextWindowSize } from "../utils/context-window.ts";
 import { type CredentialSelection, resolveCredentialPools } from "../utils/credentials.ts";
 import { parseRateLimitResetTime } from "../utils/error-tracker.ts";
+import { resolveHarnessProvider } from "../utils/harness-provider.ts";
 import { prettyPrintLine, prettyPrintStderr } from "../utils/pretty-print.ts";
 import { scrubSecrets } from "../utils/secret-scrubber.ts";
 import { detectVcsProvider } from "../vcs/index.ts";
 import { interpolate } from "../workflows/template.ts";
+import { awaitCredentials, BootMaxWaitExceededError, EX_CONFIG } from "./credential-wait.ts";
+import {
+  buildCredStatusReport,
+  isCredCheckDisabled,
+  reportCredStatus,
+} from "./provider-credentials.ts";
 // Side-effect import: registers runner trigger/resumption templates
 import "./templates.ts";
 
@@ -202,6 +210,13 @@ async function closeAgent(config: ApiConfig, role: string): Promise<void> {
 interface ResolvedEnvResult {
   env: Record<string, string | undefined>;
   credentialSelections: CredentialSelection[];
+  /**
+   * Effective `HARNESS_PROVIDER` after layering swarm_config over the base
+   * env. Callers should prefer this over `process.env.HARNESS_PROVIDER` so
+   * that an operator's swarm_config row (repo > agent > global) actually
+   * takes effect on the worker.
+   */
+  resolvedProvider: ProviderName;
 }
 
 async function fetchResolvedEnv(
@@ -239,6 +254,8 @@ async function fetchResolvedEnv(
     }
   }
 
+  const resolvedProvider = resolveHarnessProvider(env, baseEnv);
+
   const credentialSelections = await resolveCredentialPools(env, {
     apiUrl,
     apiKey,
@@ -246,10 +263,13 @@ async function fetchResolvedEnv(
     // CLAUDE_CODE_OAUTH_TOKEN stamped on their task record (and vice
     // versa) just because both env vars happen to be set in the worker
     // container. See `PROVIDER_CREDENTIAL_VARS` in src/utils/credentials.ts.
-    provider: process.env.HARNESS_PROVIDER,
+    //
+    // Use the resolved provider (swarm_config > env) so an operator can flip
+    // the worker's harness from the dashboard without restarting the container.
+    provider: resolvedProvider,
   });
 
-  return { env, credentialSelections };
+  return { env, credentialSelections, resolvedProvider };
 }
 
 /** Tools that produce noise — skip auto-progress for these */
@@ -533,6 +553,12 @@ export async function ensureTaskFinished(
   exitCode: number,
   failureReason?: string,
   providerOutput?: string,
+  /**
+   * Active provider for this task. When provided, gates the structured-output
+   * fallback path correctly even if `process.env.HARNESS_PROVIDER` differs
+   * from the resolved swarm_config value. Falls back to env when omitted.
+   */
+  provider?: ProviderName,
 ): Promise<void> {
   const headers: Record<string, string> = {
     "X-Agent-ID": config.agentId,
@@ -558,7 +584,7 @@ export async function ensureTaskFinished(
     body.output = providerOutput;
   } else {
     // Try structured output fallback if the task has an outputSchema
-    const adapterType = process.env.HARNESS_PROVIDER || "claude";
+    const adapterType = provider ?? process.env.HARNESS_PROVIDER ?? "claude";
     const fallback = await handleStructuredOutputFallback(config, taskId, adapterType);
 
     console.log(`[${role}] Task ${taskId.slice(0, 8)} fallback result: ${fallback.kind}`);
@@ -890,7 +916,15 @@ function setupShutdownHandlers(
               console.warn(
                 `[${role}] Failed to pause task ${taskId.slice(0, 8)}, marking as failed instead`,
               );
-              await ensureTaskFinished(apiConfig, role, taskId, 1);
+              await ensureTaskFinished(
+                apiConfig,
+                role,
+                taskId,
+                1,
+                undefined,
+                undefined,
+                state.harnessProvider,
+              );
             }
           }
         }
@@ -958,6 +992,14 @@ interface RunningTask {
 interface RunnerState {
   activeTasks: Map<string, RunningTask>;
   maxConcurrent: number;
+  /**
+   * Effective harness provider for this worker boot session — resolved
+   * from `swarm_config` (overlay) > `process.env.HARNESS_PROVIDER` > "claude".
+   * Used by error / cleanup paths so the structured-output fallback runs the
+   * right adapter even when env disagrees with swarm_config. Section 4
+   * (per-task live re-resolution) will mutate this between tasks.
+   */
+  harnessProvider: ProviderName;
 }
 
 /** Buffer for session logs */
@@ -1335,6 +1377,13 @@ async function registerAgent(opts: {
   role?: string;
   capabilities?: string[];
   maxTasks?: number;
+  /**
+   * Resolved harness provider (swarm_config > env > "claude"). Sent as both
+   * the legacy `provider` field and the canonical `harness_provider` column.
+   * Defaults to `process.env.HARNESS_PROVIDER || "claude"` for callers that
+   * haven't migrated to passing it explicitly.
+   */
+  harnessProvider?: ProviderName;
 }): Promise<void> {
   const headers: Record<string, string> = {
     "Content-Type": "application/json",
@@ -1344,7 +1393,16 @@ async function registerAgent(opts: {
     headers.Authorization = `Bearer ${opts.apiKey}`;
   }
 
-  const provider = (process.env.HARNESS_PROVIDER || "claude") as ProviderName;
+  const provider: ProviderName =
+    opts.harnessProvider ?? ((process.env.HARNESS_PROVIDER || "claude") as ProviderName);
+
+  // Phase 1.5 (cloud-personalization): also push the canonical
+  // `harness_provider` field so the API can persist it in its own column
+  // (`agents.harness_provider`). Always send the resolved provider value
+  // (defaulting to "claude" when HARNESS_PROVIDER is unset) so agents that
+  // don't explicitly set the env var still self-report instead of leaving
+  // the column NULL — matches how `provider` already defaults above.
+  const harnessProvider: ProviderName = provider;
 
   const response = await fetch(`${opts.apiUrl}/api/agents`, {
     method: "POST",
@@ -1356,6 +1414,7 @@ async function registerAgent(opts: {
       capabilities: opts.capabilities,
       maxTasks: opts.maxTasks,
       provider,
+      harness_provider: harnessProvider,
     }),
   });
 
@@ -1543,6 +1602,7 @@ async function fetchRelevantMemories(
   apiKey: string,
   agentId: string,
   taskDescription: string,
+  taskId?: string,
 ): Promise<string | null> {
   try {
     const headers: Record<string, string> = {
@@ -1550,6 +1610,11 @@ async function fetchRelevantMemories(
       "X-Agent-ID": agentId,
     };
     if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+    // Memory rater v1.5: server uses this header to log `memory_retrieval`
+    // rows so server-side raters (ImplicitCitationRater) can score the
+    // memories they surface against this task's session_logs at completion.
+    // Plan: thoughts/taras/plans/2026-05-05-memory-rater-v1.5/step-2.md §2
+    if (taskId) headers["X-Source-Task-ID"] = taskId;
 
     const response = await fetch(`${apiUrl}/api/memory/search`, {
       method: "POST",
@@ -1563,14 +1628,7 @@ async function fetchRelevantMemories(
       results: Array<{ id: string; name: string; content: string; similarity: number }>;
     };
 
-    const useful = (data.results || []).filter((m) => m.similarity > 0.4);
-    if (useful.length === 0) return null;
-
-    const memoryContext = useful
-      .map((m) => `- **${m.name}** (id: ${m.id}): ${m.content.substring(0, 300)}`)
-      .join("\n");
-
-    return `\n\n### Relevant Past Knowledge\n\nThese memories from your previous sessions may be useful. Use \`memory-get\` with the memory ID to retrieve full details.\n\n${memoryContext}\n`;
+    return renderMemoriesPrompt(data.results || []);
   } catch {
     // Non-blocking — don't fail task start because of memory search
     return null;
@@ -2185,6 +2243,7 @@ async function checkCompletedProcesses(
         result.exitCode,
         failureReason,
         result.output,
+        state.harnessProvider,
       );
 
       ensure({
@@ -2295,9 +2354,6 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
   // Initialize Business-Use SDK for worker-side instrumentation
   initialize();
 
-  // Create provider adapter based on HARNESS_PROVIDER env var (default: claude)
-  const adapter = createProviderAdapter(process.env.HARNESS_PROVIDER || "claude");
-
   const sessionId = process.env.SESSION_ID || crypto.randomUUID().slice(0, 8);
   const baseLogDir = opts.logsDir || process.env.LOG_DIR || "/logs";
   const logDir = `${baseLogDir}/${sessionId}`;
@@ -2312,6 +2368,30 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
 
   const apiUrl = process.env.MCP_BASE_URL || `http://localhost:${process.env.PORT || "3013"}`;
   const swarmUrl = process.env.SWARM_URL || "localhost";
+  const apiKey = process.env.API_KEY || "";
+
+  // Resolve the boot harness provider from swarm_config (repo > agent > global,
+  // overlaid on top of `process.env`). This is what selects the adapter for
+  // this worker's lifetime. On a fresh worker (agentId="unknown") only global
+  // swarm_config applies; once registered, an operator writing an agent-scoped
+  // HARNESS_PROVIDER row takes effect on the next reconciliation cycle (Section 4)
+  // or worker restart.
+  //
+  // Failures (network, API down, malformed value) fall back to env then "claude"
+  // so a swarm_config outage cannot wedge boot.
+  let bootProvider: ProviderName;
+  try {
+    bootProvider = (await fetchResolvedEnv(apiUrl, apiKey, agentId)).resolvedProvider;
+  } catch (err) {
+    console.warn(`[runner] fetchResolvedEnv failed at boot, falling back to env: ${err}`);
+    bootProvider = resolveHarnessProvider({}, process.env);
+  }
+  console.log(`[runner] Resolved HARNESS_PROVIDER: ${bootProvider}`);
+
+  // Create provider adapter using the resolved value. `let` so the poll-loop
+  // reconciliation block (Section 4) can swap it live when an operator changes
+  // HARNESS_PROVIDER in swarm_config — call sites read the current binding.
+  let adapter = createProviderAdapter(bootProvider);
 
   // Configure HTTP-based template resolution (workers resolve via API, not local DB)
   if (process.env.API_KEY) {
@@ -2382,9 +2462,11 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
   // Slack context for current task (gates Slack instructions in prompt)
   let currentTaskSlackContext: BasePromptArgs["slackContext"] | undefined;
 
-  // Generate base prompt (identity fields injected after profile fetch below)
-  const { traits } = adapter;
+  // Generate base prompt (identity fields injected after profile fetch below).
+  // Traits are read fresh on each call so a live adapter swap (Section 4)
+  // produces a prompt matching the new provider's capabilities.
   const buildSystemPrompt = async () => {
+    const { traits } = adapter;
     return getBasePrompt({
       role,
       agentId,
@@ -2460,7 +2542,6 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
   console.log(`[${role}] Total system prompt length: ${resolvedSystemPrompt.length} chars`);
 
   const isAiLoop = opts.aiLoop || process.env.AI_LOOP === "true";
-  const apiKey = process.env.API_KEY || "";
 
   // Constants for polling
   const PollIntervalMs = 2000; // 2 seconds between polls
@@ -2508,11 +2589,27 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
     const state: RunnerState = {
       activeTasks: new Map(),
       maxConcurrent,
+      harnessProvider: bootProvider,
     };
 
     // Track tasks already signaled for cancellation to avoid repeated SIGTERM
     const cancelledSignaled = new Set<string>();
 
+    // Migration 055 — cache the harness_provider value used when we last
+    // built a `cred_status` snapshot. Re-runs the post-task check only when
+    // the resolved provider changes. Section 4 of the swarm_config-overrides-
+    // HARNESS_PROVIDER work makes this dynamic: state.harnessProvider is
+    // reconciled below from `swarm_config`, so an operator's change reaches
+    // here without a worker restart.
+    let cachedCredHarnessProvider: string | null = null;
+
+    // Throttle for live HARNESS_PROVIDER reconciliation. Each reconciliation
+    // calls `fetchResolvedEnv` which also re-resolves credential pools — we
+    // don't want that on every 2s poll. 10s gives operator changes a near-
+    // immediate effect from a UX perspective without hammering the API.
+    let lastHarnessReconcileAt = 0;
+    const HARNESS_RECONCILE_INTERVAL_MS = 10_000;
+
     // Create API config for ping/close
     const apiConfig: ApiConfig = { apiUrl, apiKey, agentId };
 
@@ -2534,6 +2631,7 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
         isLead,
         capabilities,
         maxTasks: maxConcurrent,
+        harnessProvider: bootProvider,
       });
       console.log(`[${role}] Registered as "${agentName}" (ID: ${agentId})`);
     } catch (error) {
@@ -2541,6 +2639,65 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
       process.exit(1);
     }
 
+    // Block until harness credentials are present in env. This loop replaces
+    // the old bash-level fail-fast in `docker-entrypoint.sh` — the worker is
+    // already registered (visible to the dashboard) and self-heals once
+    // creds appear in `swarm_config`. See plans/2026-05-06-worker-credential-safe-loop.md.
+    //
+    // CRED_CHECK_DISABLE=1 opts out entirely: the worker trusts the operator
+    // and starts polling immediately, with a NULL `cred_status` row that the
+    // dashboard surfaces as "unreported."
+    const harnessProvider = bootProvider;
+    cachedCredHarnessProvider = harnessProvider;
+    if (isCredCheckDisabled(process.env)) {
+      console.log(`[${role}] CRED_CHECK_DISABLE=1, skipping credential checks`);
+    } else {
+      try {
+        await awaitCredentials({
+          provider: harnessProvider,
+          refreshEnv: async () => {
+            const { env } = await fetchResolvedEnv(apiUrl, apiKey, agentId);
+            return env;
+          },
+          onTick: (status) => {
+            // Best-effort status report — the dispatcher uses it to route
+            // around blocked agents. Failures are non-fatal (the wait loop
+            // already swallows onTick exceptions). We do NOT include
+            // `cred_status` here — the live test runs once the worker is
+            // ready (below), and intermediate ticks are presence-only.
+            fetch(`${apiUrl}/api/agents/${encodeURIComponent(agentId)}/credential-status`, {
+              method: "PUT",
+              headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "X-Agent-ID": agentId,
+                "Content-Type": "application/json",
+              },
+              body: JSON.stringify({ ready: status.ready, missing: status.missing }),
+            }).catch(() => {
+              // Swallowed — Phase 2 wait loop logs every tick anyway.
+            });
+          },
+        });
+      } catch (err) {
+        if (err instanceof BootMaxWaitExceededError) {
+          console.error(`[${role}] ${err.message}`);
+          process.exit(EX_CONFIG);
+        }
+        throw err;
+      }
+
+      // Migration 055: build the full snapshot (presence + live test) once
+      // creds are ready and POST it to the agent row. Status endpoint reads
+      // this instead of running predicates server-side.
+      try {
+        const snapshot = await buildCredStatusReport(harnessProvider, process.env, {}, "boot");
+        await reportCredStatus(apiUrl, apiKey, agentId, snapshot);
+      } catch (err) {
+        // Non-fatal — worker proceeds even if reporting fails.
+        console.warn(`[${role}] cred_status boot report failed (non-fatal): ${err}`);
+      }
+    }
+
     // Clean up any stale active sessions from previous runs (crash recovery)
     await cleanupActiveSessions(apiConfig);
     console.log(`[${role}] Cleaned up stale active sessions`);
@@ -2865,6 +3022,7 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
             apiKey,
             agentId,
             task.task,
+            task.id,
           );
           if (resumeMemoryContext) {
             resumePrompt += resumeMemoryContext;
@@ -2970,7 +3128,15 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
             console.error(
               `[${role}] Failed to spawn process for resumed task ${task.id.slice(0, 8)}: ${errMsg}`,
             );
-            await ensureTaskFinished(apiConfig, role, task.id, 1, `Spawn failed: ${errMsg}`);
+            await ensureTaskFinished(
+              apiConfig,
+              role,
+              task.id,
+              1,
+              `Spawn failed: ${errMsg}`,
+              undefined,
+              state.harnessProvider,
+            );
             continue;
           }
 
@@ -3020,6 +3186,64 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
       // Check for completed processes first and ensure tasks are marked as finished
       await checkCompletedProcesses(state, role, apiConfig);
 
+      // Live HARNESS_PROVIDER reconciliation. Re-fetches `swarm_config` (overlaid
+      // on env) and swaps the adapter if the resolved provider changed —
+      // typically because an operator PATCH'd /api/agents/:id/harness-provider
+      // (which writes a swarm_config row) or upserted a config row directly.
+      //
+      // Safety: in-flight sessions hold their own `ProviderSession` references
+      // and continue on the old adapter unaffected. New spawns (below) read
+      // the current `adapter` binding and pick up the swap. `basePrompt` is
+      // rebuilt because traits (and therefore prompt content) may differ across
+      // providers.
+      if (Date.now() - lastHarnessReconcileAt > HARNESS_RECONCILE_INTERVAL_MS) {
+        lastHarnessReconcileAt = Date.now();
+        try {
+          const { resolvedProvider } = await fetchResolvedEnv(apiUrl, apiKey, agentId);
+          if (resolvedProvider !== state.harnessProvider) {
+            const previous = state.harnessProvider;
+            console.log(
+              `[${role}] [harness] Reconciling adapter: ${previous} → ${resolvedProvider}`,
+            );
+            try {
+              adapter = createProviderAdapter(resolvedProvider);
+              state.harnessProvider = resolvedProvider;
+              basePrompt = await buildSystemPrompt();
+              resolvedSystemPrompt = additionalSystemPrompt
+                ? `${basePrompt}\n\n${additionalSystemPrompt}`
+                : basePrompt;
+              // Force a fresh cred_status report below for the new provider.
+              cachedCredHarnessProvider = null;
+              console.log(
+                `[${role}] [harness] Swapped to ${resolvedProvider} (basePrompt rebuilt: ${basePrompt.length} chars)`,
+              );
+            } catch (err) {
+              console.warn(
+                `[${role}] [harness] Failed to swap to ${resolvedProvider} (staying on ${previous}): ${err}`,
+              );
+            }
+          }
+        } catch (err) {
+          console.warn(`[${role}] [harness] Reconcile fetch failed (non-fatal): ${err}`);
+        }
+      }
+
+      // Migration 055 — post-task credential refresh, cache-keyed on the
+      // *resolved* harness_provider. Re-runs the snapshot when the provider
+      // changes (boot, or after a live swap above) so the dashboard shows
+      // up-to-date credential status for the active adapter.
+      if (!isCredCheckDisabled(process.env)) {
+        const currentHarness = state.harnessProvider;
+        if (currentHarness !== cachedCredHarnessProvider) {
+          cachedCredHarnessProvider = currentHarness;
+          buildCredStatusReport(currentHarness, process.env, {}, "post_task")
+            .then((snap) => reportCredStatus(apiUrl, apiKey, agentId, snap))
+            .catch((err) =>
+              console.warn(`[${role}] cred_status post_task report failed (non-fatal): ${err}`),
+            );
+        }
+      }
+
       // Periodic VCS detection for running tasks (fire-and-forget, throttled per task)
       const now = Date.now();
       for (const [taskId, task] of state.activeTasks) {
@@ -3150,7 +3374,13 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
                 ? (trigger.task as { task: string; id?: string })
                 : null;
             if (task?.task) {
-              const memoryContext = await fetchRelevantMemories(apiUrl, apiKey, agentId, task.task);
+              const memoryContext = await fetchRelevantMemories(
+                apiUrl,
+                apiKey,
+                agentId,
+                task.task,
+                task.id,
+              );
               if (memoryContext) {
                 triggerPrompt += memoryContext;
                 console.log(`[${role}] Injected relevant memories into task prompt`);
@@ -3320,6 +3550,8 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
                 trigger.taskId,
                 1,
                 `Spawn failed: ${errMsg}`,
+                undefined,
+                state.harnessProvider,
               );
             }
             continue;