@desplega.ai/agent-swarm 1.149.1 → 1.151.0

package/src/heartbeat/heartbeat.ts

@@ -45,6 +45,8 @@ import {
   expireStaleRuntimeInstances,
 } from "../be/multi-runtime";
 import { promotePendingSteeringForTask } from "../be/steering";
+import type { HeartbeatAction, HeartbeatClassification } from "../extensions/contract";
+import { dispatchPre, extensionIdForAgent } from "../extensions/dispatcher";
 import { resolveTemplate } from "../prompts/resolver";
 import {
   createPoolStarvationDecisionTask,
@@ -159,6 +161,21 @@ export function maxResumeGenerations(): number {
 
 export const RESUME_BUDGET_EXHAUSTED_REASON = "resume_budget_exhausted";
 
+type StallClassification = HeartbeatClassification;
+type RemediationAction = HeartbeatAction;
+
+type RemediationDecision = { action: RemediationAction; reason: string };
+
+interface RemediationFacts {
+  skipAutoResume: boolean;
+  alreadyResumed: boolean;
+  resumeBudgetExhausted: boolean;
+  supersedeReason: string;
+  legacyFailReason: string;
+}
+
+const REMEDIATION_ACTIONS = new Set<RemediationAction>(["supersede-resume", "fail", "record"]);
+
 /**
  * Grace window (minutes) a crash-recovery resume pinned to its original agent
  * (DES-523 Phase 1) waits to be reclaimed before the reaper concludes the agent
@@ -206,6 +223,11 @@ const HEARTBEAT_CHECKLIST_DISABLE = Boolean(process.env.HEARTBEAT_CHECKLIST_DISA
 
 export interface HeartbeatFindings {
   stalledTasks: AgentTask[];
+  extensionSkipped: Array<{
+    taskId: string;
+    extension: { id: string; name: string };
+    reason: string;
+  }>;
   autoFailedTasks: Array<{ taskId: string; agentId: string; reason: string }>;
   autoResumedTasks: Array<{
     taskId: string;
@@ -296,6 +318,7 @@ export async function preflightGate(): Promise<boolean> {
 export async function codeLevelTriage(): Promise<HeartbeatFindings> {
   const findings: HeartbeatFindings = {
     stalledTasks: [],
+    extensionSkipped: [],
     autoFailedTasks: [],
     autoResumedTasks: [],
     pinnedResumes: [],
@@ -377,131 +400,173 @@ async function detectAndRemediateStalledTasks(findings: HeartbeatFindings): Prom
       continue;
     }
 
+    let classification: StallClassification | undefined;
+    let remediationOpts: RemediationOptions | undefined;
+    let proposed: RemediationDecision | undefined;
+
     if (!session) {
-      // Case A: No active session — worker is dead
       if (taskAgeMs >= stallThresholdNoSessionMin() * 60 * 1000) {
-        await remediateCrashedWorkerTask(findings, task, {
+        classification = "no-session";
+        remediationOpts = {
           supersedeReason:
             "Auto-superseded by heartbeat: worker session not found (no active session for task)",
           legacyFailReason:
             "Auto-failed by heartbeat: worker session not found (no active session for task)",
           shortLabel: "no active session",
-        });
+        };
+        proposed = await defaultRemediationDecision(task, remediationOpts);
       }
     } else {
       const isStaleHeartbeat =
         sessionHeartbeatAgeMs! >= stallThresholdStaleHeartbeatMin() * 60 * 1000;
 
       if (isStaleHeartbeat) {
-        // Case B: Session exists but heartbeat is stale — worker likely crashed
         if (taskAgeMs >= stallThresholdStaleHeartbeatMin() * 60 * 1000) {
-          await remediateCrashedWorkerTask(findings, task, {
+          classification = "stale-session";
+          remediationOpts = {
             supersedeReason:
               "Auto-superseded by heartbeat: worker session heartbeat is stale (likely crashed)",
             legacyFailReason:
               "Auto-failed by heartbeat: worker session heartbeat is stale (likely crashed)",
             shortLabel: "stale session heartbeat",
             cleanupActiveSession: true,
-          });
+          };
+          proposed = await defaultRemediationDecision(task, remediationOpts);
         }
       } else {
-        // Case C: Session exists and heartbeat is fresh — ambiguous
         if (taskAgeMs >= stallThresholdMinutes() * 60 * 1000) {
-          findings.stalledTasks.push(task);
+          classification = "fresh-stalled";
+          remediationOpts = {
+            supersedeReason:
+              "Auto-superseded by heartbeat: task stalled despite a fresh session heartbeat",
+            legacyFailReason:
+              "Auto-failed by heartbeat: task stalled despite a fresh session heartbeat",
+            shortLabel: "fresh session heartbeat",
+          };
+          proposed = {
+            action: "record",
+            reason: "Task stalled despite a fresh session heartbeat",
+          };
         }
       }
     }
+
+    if (!classification || !remediationOpts || !proposed) continue;
+
+    const dispatched = await dispatchPre(
+      "pre.heartbeat.remediate",
+      {
+        task,
+        ...(session ? { session } : {}),
+        classification,
+        proposedAction: proposed.action,
+        reason: proposed.reason,
+        taskAgeMs,
+        ...(sessionHeartbeatAgeMs === null ? {} : { sessionHeartbeatAgeMs }),
+      },
+      { skipExtensionId: extensionIdForAgent(task.creatorAgentId) },
+    );
+
+    if (dispatched.action === "block") {
+      findings.stalledTasks.push(task);
+      findings.extensionSkipped.push({
+        taskId: task.id,
+        extension: dispatched.extension,
+        reason: dispatched.reason,
+      });
+      continue;
+    }
+
+    if (dispatched.action === "modify") {
+      const action = dispatched.data.proposedAction;
+      if (isRemediationAction(action)) {
+        proposed = { ...proposed, action };
+      } else {
+        console.warn(
+          "[Heartbeat] Extension returned an invalid remediation action:",
+          scrubSecrets(String(action)),
+        );
+      }
+    }
+
+    if (proposed.action === "record") {
+      findings.stalledTasks.push(task);
+      continue;
+    }
+
+    await remediateCrashedWorkerTask(findings, task, remediationOpts, proposed);
   }
 }
 
+function isRemediationAction(value: unknown): value is RemediationAction {
+  return typeof value === "string" && REMEDIATION_ACTIONS.has(value as RemediationAction);
+}
+
+function decideRemediation(task: AgentTask, facts: RemediationFacts): RemediationDecision {
+  if (task.workflowRunStepId != null) {
+    return { action: "fail", reason: "superseded_workflow_task" };
+  }
+  if (facts.skipAutoResume || facts.alreadyResumed) {
+    return { action: "fail", reason: facts.legacyFailReason };
+  }
+  if (facts.resumeBudgetExhausted) {
+    return { action: "fail", reason: RESUME_BUDGET_EXHAUSTED_REASON };
+  }
+  return { action: "supersede-resume", reason: facts.supersedeReason };
+}
+
+async function defaultRemediationDecision(
+  task: AgentTask,
+  opts: RemediationOptions,
+): Promise<RemediationDecision> {
+  const skipAutoResume = SKIP_AUTO_RESUME_TYPES.has(task.taskType ?? "");
+  const isWorkflowStep = task.workflowRunStepId != null;
+  const alreadyResumed =
+    !skipAutoResume && !isWorkflowStep && (await hasNonTerminalResumeChild(task.id));
+  const resumeBudgetExhausted = getNextResumeGeneration(task) > maxResumeGenerations();
+  return decideRemediation(task, {
+    skipAutoResume,
+    alreadyResumed,
+    resumeBudgetExhausted,
+    supersedeReason: opts.supersedeReason,
+    legacyFailReason: opts.legacyFailReason,
+  });
+}
+
+interface RemediationOptions {
+  supersedeReason: string;
+  legacyFailReason: string;
+  shortLabel: string;
+  cleanupActiveSession?: boolean;
+}
+
 /**
- * Shared remediation for Cases A (no active session) and B (stale heartbeat) of the
- * stalled-task detector. Prefers the supersede → resume follow-up path (DES-523) so a
- * crashed worker's task gets a fresh "resume" sibling instead of being silently dropped.
+ * Applies a selected remediation action for a stalled task. Extensions can replace
+ * the selected action before this function runs.
  *
- * Falls back to the legacy `failTask` path when:
- *   - the task is a system task (heartbeat / boot-triage) — would loop forever,
- *   - a non-terminal child already exists — a prior sweep already created a resume,
- *   - `createResumeFollowUp` returns `workflow-skip` — workflow engine owns retries.
+ * The supersede and resume path gives crashed work a fresh sibling. The fail path keeps
+ * workflow ownership, excluded task types, idempotency, and resume-budget behavior.
  */
 async function remediateCrashedWorkerTask(
   findings: HeartbeatFindings,
   task: AgentTask,
-  opts: {
-    supersedeReason: string;
-    legacyFailReason: string;
-    shortLabel: string;
-    cleanupActiveSession?: boolean;
-  },
+  opts: RemediationOptions,
+  decision: RemediationDecision,
 ): Promise<void> {
   if (!task.agentId) return; // Type guard — caller already checked.
 
-  const skipAutoResume = SKIP_AUTO_RESUME_TYPES.has(task.taskType ?? "");
-  // Workflow-step tasks: skip supersede entirely so the engine's retry policy
-  // owns recovery. `createResumeFollowUp` would also bail with `workflow-skip`,
-  // but checking here avoids leaving the parent in `superseded` with a dangling
-  // dedicated-reason `failTask` no-op chasing it.
-  const isWorkflowStep = task.workflowRunStepId != null;
-  // Idempotency: if a non-terminal `resume` child already exists for this
-  // parent, a prior sweep already created the resume — fall back to the
-  // legacy fail path. We filter on `taskType = 'resume'` specifically (not
-  // any child task) because `send-task` auto-defaults `parentTaskId` to the
-  // caller's current task, so a crashed worker with delegated subtasks
-  // would otherwise be incorrectly skipped (PR #594 review).
-  const alreadyResumed =
-    !skipAutoResume && !isWorkflowStep && (await hasNonTerminalResumeChild(task.id));
-
-  if (isWorkflowStep) {
-    const failed = await failTask(task.id, "superseded_workflow_task");
+  if (decision.action === "fail") {
+    const failed = await failTask(task.id, decision.reason);
     if (failed) {
       findings.autoFailedTasks.push({
         taskId: task.id,
         agentId: task.agentId,
-        reason: "superseded_workflow_task",
+        reason: decision.reason,
       });
       if (opts.cleanupActiveSession) await deleteActiveSession(task.id);
-      console.log(
-        `[Heartbeat] Workflow-step task ${task.id.slice(0, 8)} failed — engine will handle retry (${opts.shortLabel})`,
-      );
-      const remaining = await getActiveTaskCount(task.agentId);
-      if (remaining === 0) await restoreAgentIdleAfterRemediation(task.agentId);
-    }
-    return;
-  }
-
-  if (skipAutoResume || alreadyResumed) {
-    const failed = await failTask(task.id, opts.legacyFailReason);
-    if (failed) {
-      findings.autoFailedTasks.push({
-        taskId: task.id,
-        agentId: task.agentId,
-        reason: opts.legacyFailReason,
-      });
-      if (opts.cleanupActiveSession) await deleteActiveSession(task.id);
-      console.log(
-        `[Heartbeat] Auto-failed task ${task.id.slice(0, 8)} — ${opts.shortLabel} (${
-          skipAutoResume ? "skipRetry taskType" : "resume already exists"
-        })`,
-      );
-      const remaining = await getActiveTaskCount(task.agentId);
-      if (remaining === 0) await restoreAgentIdleAfterRemediation(task.agentId);
-    }
-    return;
-  }
-
-  const nextResumeGeneration = getNextResumeGeneration(task);
-  if (nextResumeGeneration > maxResumeGenerations()) {
-    const failed = await failTask(task.id, RESUME_BUDGET_EXHAUSTED_REASON);
-    if (failed) {
-      findings.autoFailedTasks.push({
-        taskId: task.id,
-        agentId: task.agentId,
-        reason: RESUME_BUDGET_EXHAUSTED_REASON,
-      });
-      if (opts.cleanupActiveSession) await deleteActiveSession(task.id);
-      console.warn(
-        `[Heartbeat] Auto-failed task ${task.id.slice(0, 8)} — ${RESUME_BUDGET_EXHAUSTED_REASON} (${opts.shortLabel})`,
-      );
+      const message = `[Heartbeat] Auto-failed task ${task.id.slice(0, 8)}: ${decision.reason} (${opts.shortLabel})`;
+      if (decision.reason === RESUME_BUDGET_EXHAUSTED_REASON) console.warn(message);
+      else console.log(message);
       const remaining = await getActiveTaskCount(task.agentId);
       if (remaining === 0) await restoreAgentIdleAfterRemediation(task.agentId);
     }
@@ -511,7 +576,7 @@ async function remediateCrashedWorkerTask(
   await beforeHeartbeatSupersedeForTests?.(task);
 
   const superseded = await supersedeTask(task.id, {
-    reason: opts.supersedeReason,
+    reason: decision.reason,
     resumeTaskId: null,
   });
   if (!superseded) {
@@ -539,7 +604,7 @@ async function remediateCrashedWorkerTask(
       taskId: task.id,
       resumeTaskId: resume.task.id,
       agentId: task.agentId,
-      reason: opts.supersedeReason,
+      reason: decision.reason,
     });
     // Phase 1 (DES-523): when the resume pinned back to the original
     // (stable-ID) agent, record it so the sweep summary surfaces the pin
@@ -766,6 +831,7 @@ export async function runRebootSweep(): Promise<void> {
                 : preferredAgentId === task.agentId
                   ? "continuity"
                   : "reroute_fault",
+            routingSource: preferredAgentId === undefined ? undefined : "engine_default",
             tags,
             priority: task.priority,
             source: task.source,
@@ -1375,6 +1441,7 @@ export async function checkHeartbeatChecklist(): Promise<void> {
   await createTaskExtended(result.text, {
     agentId: lead.id,
     routingReason: "skill",
+    routingSource: "engine_default",
     taskType: "heartbeat-checklist",
     tags: ["checklist", "auto-generated"],
     priority: 60,
@@ -1401,6 +1468,7 @@ export async function runHeartbeatSweep(): Promise<void> {
     if (!(await preflightGate())) {
       const cleanupOnlyFindings: HeartbeatFindings = {
         stalledTasks: [],
+        extensionSkipped: [],
         autoFailedTasks: [],
         autoResumedTasks: [],
         pinnedResumes: [],
@@ -1467,6 +1535,9 @@ function logFindings(findings: HeartbeatFindings): void {
   if (findings.stalledTasks.length > 0) {
     parts.push(`stalled=${findings.stalledTasks.length}`);
   }
+  if (findings.extensionSkipped.length > 0) {
+    parts.push(`extension_skipped=${findings.extensionSkipped.length}`);
+  }
   if (findings.workerHealthFixes.length > 0) {
     parts.push(`health_fixes=${findings.workerHealthFixes.length}`);
   }
@@ -1592,6 +1663,7 @@ export async function createBootTriageTask(): Promise<void> {
   await createTaskExtended(result.text, {
     agentId: lead.id,
     routingReason: "skill",
+    routingSource: "engine_default",
     taskType: "boot-triage",
     tags: ["boot", "triage", "auto-generated"],
     priority: 70, // Higher than regular checklist (60)

package/src/http/agents.ts

@@ -399,7 +399,7 @@ const listAgentRuntimeInstances = route({
   pattern: ["api", "agents", null, "runtime-instances"],
   summary: "List runtime instances serving an agent",
   description:
-    "Read-only view of the worker processes currently registered for a logical agent. Rows exist only for multi-runtime registrations (MULTI_RUNTIME_ENABLED), so the list is empty in the default configuration. `isLive` combines `status` with `lastSeenAt` freshness against the server's staleness cutoff (`staleThresholdMinutes`); `reportedSlots` is each process's self-reported capacity, distinct from the agent's logical `maxTasks` policy.",
+    "Read-only view of the worker processes currently registered for a logical agent. Rows exist only for multi-runtime registrations (MULTI_RUNTIME_ENABLED), so the list is empty when the flag is disabled. `isLive` combines `status` with `lastSeenAt` freshness against the server's staleness cutoff (`staleThresholdMinutes`); `reportedSlots` is each process's self-reported capacity, distinct from the agent's logical `maxTasks` policy.",
   tags: ["Agents"],
   params: z.object({ id: z.string() }),
   responses: {

package/src/http/all-routes.ts

@@ -23,6 +23,7 @@ import "./ecosystem";
 
 import "./api-keys";
 import "./events";
+import "./extensions";
 import "./favorites";
 import "./fs";
 import "./heartbeat";

package/src/http/approval-requests.ts

@@ -386,6 +386,7 @@ export async function handleApprovalRequests(
         await createTaskExtended(taskText, {
           agentId: sourceTask.agentId,
           routingReason: sourceTask.agentId ? "continuity" : undefined,
+          routingSource: sourceTask.agentId ? "engine_default" : undefined,
           parentTaskId: updated.sourceTaskId,
           source: "system",
           taskType: "hitl-follow-up",

package/src/http/apps.ts

@@ -1410,15 +1410,20 @@ export async function handleApps(
       action_name: parsed.params.name,
       input_json: JSON.stringify(input),
     });
-    const task = await createTaskWithSiblingAwareness(taskPrompt.text, {
-      source: "api",
-      agentId: action.agentId ?? lead?.id,
-      routingReason: action.agentId ? "human_pinned" : lead ? "skill" : undefined,
-      // App-spawned tasks group under the app's asset namespace so a
-      // swarm-tasks source can pull them back via config.assetKey.
-      key: normalizeAssetKey(`shared/app:${app.id}/action:${parsed.params.name}/`),
-      ...(actor.startsWith("user:") ? { requestedByUserId: actor.slice("user:".length) } : {}),
-    });
+    const task = await createTaskWithSiblingAwareness(
+      taskPrompt.text,
+      {
+        source: "api",
+        agentId: action.agentId ?? lead?.id,
+        routingReason: action.agentId ? "human_pinned" : lead ? "skill" : undefined,
+        routingSource: action.agentId || lead ? "engine_default" : undefined,
+        // App-spawned tasks group under the app's asset namespace so a
+        // swarm-tasks source can pull them back via config.assetKey.
+        key: normalizeAssetKey(`shared/app:${app.id}/action:${parsed.params.name}/`),
+        ...(actor.startsWith("user:") ? { requestedByUserId: actor.slice("user:".length) } : {}),
+      },
+      { origin: "app" },
+    );
     runActionRoute.respond(res, 200, { ok: true, taskId: task.id, status: task.status });
     return true;
   }

package/src/http/config.ts

@@ -310,7 +310,10 @@ export async function handleConfig(
     }
     const presence: Record<string, boolean> = {};
     for (const key of keys) {
-      presence[key] = process.env[key] !== undefined && process.env[key] !== "";
+      // Organization identity is trimmed by both /status and telemetry.
+      // Match that definition without changing presence semantics for secrets.
+      const value = key === "SWARM_ORG_NAME" ? process.env[key]?.trim() : process.env[key];
+      presence[key] = value !== undefined && value !== "";
     }
     envPresence.respond(res, 200, { presence });
     return true;

package/src/http/core.ts

@@ -38,8 +38,10 @@ import { agentWithCapacity, getPathSegments, jsonError, parseQueryParams } from
 
 /**
  * Load global swarm_config entries into process.env.
- * When override=false (default, used at startup), existing env vars take precedence.
- * When override=true (used for reload), DB values overwrite process.env.
+ * When override=true (used at startup, on reload, and in createServer), DB
+ * values overwrite process.env, so a dashboard-saved value wins over the
+ * deployment env. When override=false, existing env vars take precedence
+ * (only used by callers that want env-first semantics; no boot path does).
  * Reserved keys are filtered before decryption because they must remain
  * environment-only, even if legacy rows still exist in the DB.
  * Returns the list of keys that were set/updated.
@@ -139,6 +141,13 @@ export async function reloadGlobalConfigsAndIntegrations(): Promise<ReloadConfig
 }
 
 async function reloadGlobalConfigsAndIntegrationsInner(): Promise<ReloadConfigResult> {
+  // Lazy import keeps http/core out of the memory module-init graph (pulls
+  // in the OpenAI SDK). Read the pre-reload configured state first — the
+  // provider memoizes its API key at construction, so this reflects whatever
+  // key (if any) was live before we hydrate the new env below.
+  const { getEmbeddingProvider, resetEmbeddingProvider } = await import("../be/memory");
+  const wasEmbeddingsConfigured = getEmbeddingProvider().isConfigured();
+
   const updated = await loadGlobalConfigsIntoEnv(true);
 
   // File-storage provider selection reads process.env once and memoizes; the
@@ -149,8 +158,35 @@ async function reloadGlobalConfigsAndIntegrationsInner(): Promise<ReloadConfigRe
   const { resetFileStorageProvider } = await import("../fs/registry");
   resetFileStorageProvider();
 
+  // Same reasoning as the file-storage provider above: the embedding
+  // provider captures EMBEDDING_API_KEY/OPENAI_API_KEY once at construction,
+  // so a key set (or rotated) via config reload stays inert until this reset.
+  resetEmbeddingProvider();
+  const isEmbeddingsConfigured = getEmbeddingProvider().isConfigured();
+
   const integrations: string[] = [];
 
+  if (isEmbeddingsConfigured) {
+    integrations.push("embeddings");
+  }
+
+  // Off-to-on transition: existing rows written while no key resolved (or
+  // with a stale dimension) never got embedded. Kick off the same backfill
+  // boot runs, fire-and-forget — it batches, swallows per-batch errors, and
+  // is idempotent, so it's safe to not await here.
+  if (!wasEmbeddingsConfigured && isEmbeddingsConfigured) {
+    import("../be/memory/boot-reembed")
+      .then(({ runBootReembed }) => runBootReembed())
+      .catch((err) => {
+        console.error("[config-reload] memory backfill failed (non-fatal):", err);
+      });
+    import("../be/scripts/boot-reembed")
+      .then(({ runBootReembedScripts }) => runBootReembedScripts())
+      .catch((err) => {
+        console.error("[config-reload] script backfill failed (non-fatal):", err);
+      });
+  }
+
   resetAgentMail();
   if (initAgentMail()) integrations.push("agentmail");