@desplega.ai/agent-swarm 2.7.5 → 2.7.7

package/src/slack/left-threads.ts

@@ -2,7 +2,8 @@ import { deleteKv, getKv, upsertKv } from "../be/db";
 import { isEnvFlagEnabled } from "../utils/env-flag";
 
 /**
- * Slack threads the swarm left via `slack-leave-thread`.
+ * Slack threads the swarm left, via the `slack-leave-thread` tool or a
+ * human's `!leave` in the thread.
  *
  * A left thread drops every inbound message that does not @mention the bot,
  * in code, before buffering, routing, or any AI call. A mention clears the
@@ -11,7 +12,7 @@ import { isEnvFlagEnabled } from "../utils/env-flag";
  */
 export const SLACK_LEFT_THREAD_KV_NAMESPACE = "slack-left-thread";
 
-/** Operator flag that turns the `slack-leave-thread` tool off. */
+/** Operator flag that turns leaving threads off: the tool and `!leave`. */
 export const SLACK_LEAVE_THREAD_DISABLE = "SLACK_LEAVE_THREAD_DISABLE";
 
 /**
@@ -22,19 +23,41 @@ export function isSlackLeaveThreadDisabled(): boolean {
   return isEnvFlagEnabled(SLACK_LEAVE_THREAD_DISABLE, false);
 }
 
+/** Ephemeral reply to the human who sent `!leave`. */
+export const SLACK_LEFT_THREAD_CONFIRMATION =
+  "Left this thread. @mention the swarm to bring it back.";
+
+/**
+ * True when a thread message is the `!leave` command: exactly `!leave`,
+ * optionally after @mentions of the bot. A message that mentions anyone else
+ * is not addressed to the swarm and does not match.
+ */
+export function isSlackLeaveThreadCommand(
+  text: string | undefined,
+  botUserId: string | null,
+): boolean {
+  const withoutBotMentions = botUserId
+    ? (text ?? "").replaceAll(`<@${botUserId}>`, "")
+    : (text ?? "");
+  return withoutBotMentions.trim() === "!leave";
+}
+
 function threadKey(channelId: string, threadTs: string): string {
   return `${channelId}:${threadTs}`;
 }
 
+/** Who left the thread: an agent via the tool, or a Slack user via `!leave`. */
+export type SlackThreadLeftBy = { agentId: string } | { slackUserId: string };
+
 export async function leaveSlackThread(
   channelId: string,
   threadTs: string,
-  agentId: string,
+  leftBy: SlackThreadLeftBy,
 ): Promise<void> {
   await upsertKv({
     namespace: SLACK_LEFT_THREAD_KV_NAMESPACE,
     key: threadKey(channelId, threadTs),
-    value: { agentId, leftAt: new Date().toISOString() },
+    value: { ...leftBy, leftAt: new Date().toISOString() },
     valueType: "json",
   });
 }

package/src/tasks/follow-up-gate.ts

@@ -0,0 +1,318 @@
+import { createLogEntry, getDbClient } from "../be/db";
+import { getScript } from "../be/scripts/db";
+import { runSavedScriptAsAgent } from "../be/scripts/run-saved";
+import type { AgentTask, ScriptRecord } from "../types";
+import {
+  isFollowUpGateScheduleId,
+  parseFollowUpGateScheduleIds,
+} from "../utils/follow-up-gate-schedule-ids";
+import { scrubSecrets } from "../utils/secret-scrubber";
+
+/**
+ * Follow-up gate, shadow mode. When a gated scheduled run's Lead follow-up is
+ * created, a configured verifier script checks the run after commit and the
+ * server records whether the follow-up could have been skipped. Shadow mode
+ * never skips or delays a follow-up.
+ */
+
+export const FOLLOW_UP_GATE_SHADOW_EVENT = "task_follow_up_gate_shadow" as const;
+
+export type FollowUpGateConfig = {
+  mode: "off" | "shadow";
+  scheduleIds: ReadonlySet<string>;
+  verifierScript: string;
+};
+
+export type GateIneligibleReason =
+  | "mode_off"
+  | "not_scheduled"
+  | "schedule_not_gated"
+  | "status_failed"
+  | "follow_up_instructions";
+
+/**
+ * Read the gate config on every call so a `swarm_config` reload applies. Any
+ * mode other than `shadow` reads as `off`, and malformed ids are ignored.
+ */
+export function readFollowUpGateConfig(env: NodeJS.ProcessEnv = process.env): FollowUpGateConfig {
+  const mode = env.FOLLOW_UP_GATE_MODE?.trim().toLowerCase() === "shadow" ? "shadow" : "off";
+  const scheduleIds = new Set(
+    parseFollowUpGateScheduleIds(env.FOLLOW_UP_GATE_SCHEDULE_IDS)
+      .filter(isFollowUpGateScheduleId)
+      .map((id) => id.toLowerCase()),
+  );
+  return { mode, scheduleIds, verifierScript: env.FOLLOW_UP_GATE_VERIFIER_SCRIPT?.trim() ?? "" };
+}
+
+/** Pure decision on whether a follow-up's worker task may be verified. */
+export function classifyFollowUpGate(input: {
+  task: Pick<AgentTask, "scheduleId" | "followUpConfig">;
+  status: "completed" | "failed";
+  config: FollowUpGateConfig;
+}): { eligible: true } | { eligible: false; reason: GateIneligibleReason } {
+  const { task, status, config } = input;
+  if (config.mode === "off") return { eligible: false, reason: "mode_off" };
+  if (!task.scheduleId) return { eligible: false, reason: "not_scheduled" };
+  if (!config.scheduleIds.has(task.scheduleId.toLowerCase())) {
+    return { eligible: false, reason: "schedule_not_gated" };
+  }
+  if (status === "failed") return { eligible: false, reason: "status_failed" };
+  if (task.followUpConfig?.onCompleted?.trim() || task.followUpConfig?.onFailed?.trim()) {
+    return { eligible: false, reason: "follow_up_instructions" };
+  }
+  return { eligible: true };
+}
+
+type VerifierScript = Pick<ScriptRecord, "name" | "version" | "contentHash">;
+
+export type FollowUpGateDeps = {
+  /** Resolve the global verifier script, or null when it does not exist. */
+  getVerifier: (name: string) => Promise<VerifierScript | null>;
+  /** Run the verifier as `agentId` and return its result, or throw. */
+  runVerifier: (
+    script: VerifierScript,
+    input: { taskId: string },
+    agentId: string,
+    signal: AbortSignal,
+  ) => Promise<unknown>;
+};
+
+const FOLLOW_UP_GATE_TIMEOUT_MS = 60_000;
+const FOLLOW_UP_GATE_MAX_IN_FLIGHT = 2;
+const TEXT_LIMIT = 300;
+const MAX_UNVERIFIED = 20;
+
+const defaultDeps: FollowUpGateDeps = {
+  getVerifier: (name) => getScript({ name, scope: "global" }),
+  runVerifier: async (script, input, agentId, signal) => {
+    const output = await runSavedScriptAsAgent({
+      script: script as ScriptRecord,
+      input,
+      agentId,
+      timeoutMs,
+      signal,
+    });
+    if (output.exitCode !== 0 || output.error || output.runtimeError) {
+      throw new Error(
+        output.runtimeError?.message ??
+          output.error ??
+          `script exited with code ${output.exitCode}`,
+      );
+    }
+    return output.result;
+  },
+};
+
+let timeoutMs = FOLLOW_UP_GATE_TIMEOUT_MS;
+let depsForTests: FollowUpGateDeps | null = null;
+let inFlight = 0;
+const pending = new Set<Promise<void>>();
+
+/** Replace the verifier lookup and runner. `null` restores the defaults. */
+export function setFollowUpGateDepsForTests(deps: FollowUpGateDeps | null): void {
+  depsForTests = deps;
+}
+
+/** Override the verifier deadline. `null` restores the 60s default. */
+export function setFollowUpGateTimeoutMsForTests(ms: number | null): void {
+  timeoutMs = ms ?? FOLLOW_UP_GATE_TIMEOUT_MS;
+}
+
+/** Wait until every scheduled shadow hook has written its row. */
+export async function drainFollowUpGateShadowForTest(): Promise<void> {
+  // Hooks run in FIFO order behind the client lock, so once this marker runs,
+  // every earlier hook has started and registered its promise.
+  await new Promise<void>((resolve) => getDbClient().afterCommit(resolve));
+  while (pending.size > 0) await Promise.allSettled([...pending]);
+}
+
+type ShadowArgs = {
+  task: Pick<AgentTask, "id" | "scheduleId" | "followUpConfig">;
+  status: "completed" | "failed";
+  followUpTaskId: string;
+  leadAgentId: string;
+};
+
+type ShadowRecord = {
+  decision: "would_skip" | "would_keep" | "error" | "ineligible";
+  reason: string | null;
+  verdict?: { ok: boolean | null; check: string | null; detail: string; controlsOk: boolean };
+  unverified?: string[];
+  verifier?: VerifierScript | null;
+  latencyMs?: number | null;
+  error?: string;
+};
+
+/**
+ * Record a shadow gate decision for a just-created follow-up. Returns
+ * synchronously: all work, including the row write, runs after commit. With
+ * the gate off or the schedule not gated, nothing is registered.
+ */
+export function scheduleFollowUpGateShadow(args: ShadowArgs): void {
+  const config = readFollowUpGateConfig();
+  const verdict = classifyFollowUpGate({ task: args.task, status: args.status, config });
+  if (
+    !verdict.eligible &&
+    (verdict.reason === "mode_off" ||
+      verdict.reason === "not_scheduled" ||
+      verdict.reason === "schedule_not_gated")
+  ) {
+    return;
+  }
+  const deps = depsForTests ?? defaultDeps;
+  getDbClient().afterCommit(() => {
+    const run = verdict.eligible
+      ? runShadow(args, config, deps)
+      : writeRecord(args, { decision: "ineligible", reason: verdict.reason });
+    pending.add(run);
+    const settle = () => {
+      pending.delete(run);
+    };
+    void run.then(settle, settle);
+  });
+}
+
+async function runShadow(
+  args: ShadowArgs,
+  config: FollowUpGateConfig,
+  deps: FollowUpGateDeps,
+): Promise<void> {
+  if (inFlight >= FOLLOW_UP_GATE_MAX_IN_FLIGHT) {
+    await writeRecord(args, { decision: "error", reason: "concurrency_cap" });
+    return;
+  }
+  inFlight++;
+  try {
+    await writeRecord(args, await verify(args, config, deps));
+  } finally {
+    inFlight--;
+  }
+}
+
+async function verify(
+  args: ShadowArgs,
+  config: FollowUpGateConfig,
+  deps: FollowUpGateDeps,
+): Promise<ShadowRecord> {
+  if (!config.verifierScript) return { decision: "error", reason: "no_verifier" };
+  const started = performance.now();
+  const latency = () => Math.round(performance.now() - started);
+  const controller = new AbortController();
+  const deadlineMs = timeoutMs;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let script: VerifierScript | null = null;
+  let timedOut = false;
+  try {
+    const run = (async () => {
+      script = await deps.getVerifier(config.verifierScript);
+      if (!script) return null;
+      return {
+        result: await deps.runVerifier(
+          script,
+          { taskId: args.task.id },
+          args.leadAgentId,
+          controller.signal,
+        ),
+      };
+    })();
+    // A run that settles after the deadline must not surface as an unhandled rejection.
+    run.catch(() => {});
+    const deadline = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => {
+        timedOut = true;
+        controller.abort();
+        reject(new Error(`verifier timed out after ${deadlineMs}ms`));
+      }, deadlineMs);
+    });
+    const outcome = await Promise.race([run, deadline]);
+    const verifier = verifierInfo(script);
+    if (!outcome) {
+      return { decision: "error", reason: "no_verifier", verifier, latencyMs: latency() };
+    }
+    const result = outcome.result;
+    if (!result || typeof result !== "object") {
+      return {
+        decision: "error",
+        reason: "verifier_error",
+        verifier,
+        latencyMs: latency(),
+        error: "verifier result is not an object",
+      };
+    }
+    return { ...mapVerdict(result as Record<string, unknown>), verifier, latencyMs: latency() };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return {
+      decision: "error",
+      reason: timedOut ? "timeout" : "verifier_error",
+      verifier: verifierInfo(script),
+      latencyMs: latency(),
+      error: clip(message),
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function verifierInfo(script: VerifierScript | null): VerifierScript | null {
+  return script
+    ? { name: script.name, version: script.version, contentHash: script.contentHash }
+    : null;
+}
+
+/** `ok:true` with passing controls is `would_skip`; anything else keeps the follow-up. */
+function mapVerdict(result: Record<string, unknown>): ShadowRecord {
+  const ok = typeof result.ok === "boolean" ? result.ok : null;
+  const controls = result.controls as { ok?: unknown } | null | undefined;
+  const controlsOk = controls?.ok === true;
+  const unverified = Array.isArray(result.unverified)
+    ? result.unverified
+        .filter((name): name is string => typeof name === "string")
+        .slice(0, MAX_UNVERIFIED)
+        .map((name) => name.slice(0, 100))
+    : [];
+  return {
+    decision: ok === true && controlsOk ? "would_skip" : "would_keep",
+    reason: null,
+    verdict: {
+      ok,
+      check: typeof result.check === "string" ? result.check.slice(0, 100) : null,
+      detail: typeof result.detail === "string" ? clip(result.detail) : "",
+      controlsOk,
+    },
+    unverified,
+  };
+}
+
+function clip(text: string): string {
+  return scrubSecrets(text).slice(0, TEXT_LIMIT);
+}
+
+async function writeRecord(args: ShadowArgs, record: ShadowRecord): Promise<void> {
+  try {
+    await createLogEntry({
+      eventType: FOLLOW_UP_GATE_SHADOW_EVENT,
+      taskId: args.task.id,
+      agentId: args.leadAgentId,
+      metadata: {
+        v: 1,
+        mode: "shadow",
+        followUpTaskId: args.followUpTaskId,
+        scheduleId: args.task.scheduleId ?? null,
+        status: args.status,
+        decision: record.decision,
+        reason: record.reason,
+        verdict: record.verdict ?? null,
+        unverified: record.unverified ?? [],
+        verifier: record.verifier ?? null,
+        latencyMs: record.latencyMs ?? null,
+        ...(record.error ? { error: record.error } : {}),
+      },
+    });
+  } catch (err) {
+    console.warn(
+      `[follow-up-gate] Could not record shadow decision for task ${args.task.id.slice(0, 8)}:`,
+      scrubSecrets(err instanceof Error ? err.message : String(err)),
+    );
+  }
+}

package/src/tasks/follow-up-redelegation-guard.ts

@@ -0,0 +1,155 @@
+/**
+ * Follow-up re-delegation guard, shared by every agent dispatch path that can
+ * run inside a follow-up session: MCP `send-task` and durable-run
+ * `ctx.step.agentTask` steps (`src/http/script-runs.ts`).
+ *
+ * When the dispatching session's source task is a `follow-up` in a Slack
+ * thread, two rules stop the cycle
+ * worker completes → follow-up → Lead re-delegates → repeat,
+ * while a distinct next step (step 2 of a chain, a fix round, a retry) passes:
+ *
+ * 1. same-work: the new task matches a completed task in the thread under the
+ *    dedup similarity rules. Bypassed when a task in the thread was cancelled
+ *    or failed after that completion: the work was interrupted or went wrong,
+ *    so re-dispatch is the correct response, not a deduped no-op.
+ * 2. loop-cap: the caller already made FOLLOW_UP_DISPATCH_CAP dispatches into
+ *    the thread within FOLLOW_UP_DISPATCH_WINDOW_MINUTES. A durable script run
+ *    counts once however many steps it dispatched. Independent of similarity,
+ *    so a loop that rewords each re-delegation still stops.
+ *
+ * Callers bypass both with `allowRedelegation` and log the override. Each
+ * caller evaluates the rules inside the write transaction that inserts the
+ * task, so two concurrent dispatches into one thread (send-task or a run's
+ * first step, in any mix) cannot both read the count below the cap.
+ */
+import {
+  countAgentDispatchesInThread,
+  findCompletedTasksInThread,
+  findRecentCancelledTaskInThread,
+  findRecentFailedTaskInThread,
+  getTaskById,
+} from "../be/db";
+import { type DedupMatch, matchSimilarTask } from "../tools/task-dedup";
+
+/** How far back the same-work rule looks for a completed task in the thread. */
+export const FOLLOW_UP_SAME_WORK_WINDOW_MINUTES = 2880; // 48 hours
+/**
+ * Loop cap: dispatches one agent may make into one Slack thread per window
+ * before a follow-up session needs allowRedelegation. A long legitimate chain
+ * (plan, implement, review, fix, re-review, fix, CI fix, retry) is about 8
+ * dispatches; a runaway loop makes one per worker run (minutes), so it hits
+ * 10 within a few hours and burns at most 10 worker sessions.
+ */
+export const FOLLOW_UP_DISPATCH_CAP = 10;
+export const FOLLOW_UP_DISPATCH_WINDOW_MINUTES = 360; // 6 hours
+
+/** Rows per page when the same-work rule scans a thread's completions. */
+export const FOLLOW_UP_SAME_WORK_PAGE_SIZE = 200;
+
+export type FollowUpThread = { channelId: string; threadTs: string; userId?: string };
+
+export type FollowUpRedelegationBlock = {
+  rule: "same-work" | "loop-cap";
+  message: string;
+};
+
+/**
+ * The Slack thread the guard applies to, or null when the source task is not
+ * a `follow-up` in a Slack thread. A `resume` source is intentionally exempt:
+ * a resume IS the legitimate re-dispatch.
+ */
+export async function followUpThreadForSourceTask(
+  sourceTaskId: string | null | undefined,
+): Promise<FollowUpThread | null> {
+  if (!sourceTaskId) return null;
+  const sourceTask = await getTaskById(sourceTaskId);
+  if (
+    sourceTask?.taskType !== "follow-up" ||
+    !sourceTask.slackChannelId ||
+    !sourceTask.slackThreadTs
+  ) {
+    return null;
+  }
+  return {
+    channelId: sourceTask.slackChannelId,
+    threadTs: sourceTask.slackThreadTs,
+    userId: sourceTask.slackUserId,
+  };
+}
+
+/**
+ * The most recent completed task in the thread's same-work window whose
+ * description matches this dispatch. Pages through every completion in the
+ * window (an older match can sit behind any number of newer unrelated ones),
+ * one bounded query per page, and stops at the first match.
+ */
+async function findMatchingCompletedWork(args: {
+  channelId: string;
+  threadTs: string;
+  taskDescription: string;
+  targetAgentId?: string;
+}): Promise<DedupMatch | null> {
+  let before: { lastUpdatedAt: string; id: string } | undefined;
+  for (;;) {
+    const page = await findCompletedTasksInThread(
+      args.channelId,
+      args.threadTs,
+      FOLLOW_UP_SAME_WORK_WINDOW_MINUTES,
+      FOLLOW_UP_SAME_WORK_PAGE_SIZE,
+      before,
+    );
+    for (const existing of page) {
+      const match = matchSimilarTask(existing, args);
+      if (match) return match;
+    }
+    const last = page.at(-1);
+    if (!last || page.length < FOLLOW_UP_SAME_WORK_PAGE_SIZE) return null;
+    before = { lastUpdatedAt: last.lastUpdatedAt, id: last.id };
+  }
+}
+
+/**
+ * The follow-up re-delegation rules. Returns the rule that blocks this
+ * dispatch, or null when it may proceed.
+ */
+export async function evaluateFollowUpRedelegation(args: {
+  channelId: string;
+  threadTs: string;
+  creatorAgentId: string;
+  taskDescription: string;
+  targetAgentId?: string;
+}): Promise<FollowUpRedelegationBlock | null> {
+  const { channelId, threadTs } = args;
+  const match = await findMatchingCompletedWork(args);
+  if (match) {
+    const existing = match.task;
+    const [cancelled, failed] = await Promise.all([
+      findRecentCancelledTaskInThread(channelId, threadTs, FOLLOW_UP_SAME_WORK_WINDOW_MINUTES),
+      findRecentFailedTaskInThread(channelId, threadTs, FOLLOW_UP_SAME_WORK_WINDOW_MINUTES),
+    ]);
+    const completedAt = new Date(existing.lastUpdatedAt).getTime();
+    const interruptedLater = [cancelled, failed].some(
+      (t) => t && new Date(t.lastUpdatedAt).getTime() > completedAt,
+    );
+    if (!interruptedLater) {
+      return {
+        rule: "same-work",
+        message: `Blocked by the follow-up re-delegation guard (same-work rule): this task matches completed task ${existing.id.slice(0, 8)} in this Slack thread (${match.reason}), so the work was already done. If it must run again, pass allowRedelegation: true.`,
+      };
+    }
+  }
+
+  const dispatched = await countAgentDispatchesInThread(
+    channelId,
+    threadTs,
+    args.creatorAgentId,
+    FOLLOW_UP_DISPATCH_WINDOW_MINUTES,
+  );
+  if (dispatched >= FOLLOW_UP_DISPATCH_CAP) {
+    return {
+      rule: "loop-cap",
+      message: `Blocked by the follow-up re-delegation guard (loop-cap rule): you already dispatched ${dispatched} times into this Slack thread in the last ${FOLLOW_UP_DISPATCH_WINDOW_MINUTES / 60} hours (cap ${FOLLOW_UP_DISPATCH_CAP}; a durable script run counts once). If this is a deliberate next step and not a re-delegation loop, pass allowRedelegation: true.`,
+    };
+  }
+  return null;
+}

package/src/tasks/worker-follow-up.ts

@@ -34,6 +34,7 @@ import {
 import { isEnvFlagEnabled } from "../utils/env-flag";
 import { taskAttachmentDisplayUrl } from "../utils/task-attachment-links";
 import { chatThreadOfContextKey } from "./context-key";
+import { scheduleFollowUpGateShadow } from "./follow-up-gate";
 import { taskModelPin } from "./model-pin";
 import { createTaskWithSiblingAwareness } from "./sibling-awareness";
 // Side-effect import: registers task lifecycle templates in the in-memory registry.
@@ -261,6 +262,9 @@ async function findDeferredWakeFor(taskId: string, agentId: string): Promise<Def
  * `all` wait with siblings still pending wakes nobody, so the follow-up stays.
  * A `followUpConfig.onCompleted` / `onFailed` instruction for this status also
  * keeps the follow-up, because the wake-up task does not carry that text.
+ *
+ * With `FOLLOW_UP_GATE_MODE=shadow`, a gated scheduled run also records a
+ * verifier's would-skip decision after commit; the follow-up is never skipped.
  */
 export async function createWorkerTaskFollowUp(args: {
   task: AgentTask;
@@ -392,7 +396,7 @@ export async function createWorkerTaskFollowUp(args: {
     );
     return null;
   }
-  return await createTaskWithSiblingAwareness(
+  const followUp = await createTaskWithSiblingAwareness(
     changes.description ?? followUpDescription,
     {
       agentId: followUpAgentId,
@@ -413,6 +417,22 @@ export async function createWorkerTaskFollowUp(args: {
     },
     { origin: "followUp" },
   );
+  if (followUp) {
+    try {
+      scheduleFollowUpGateShadow({
+        task,
+        status,
+        followUpTaskId: followUp.id,
+        leadAgentId: leadAgent.id,
+      });
+    } catch (err) {
+      console.warn(
+        `[worker-follow-up] Follow-up gate shadow failed for task ${task.id.slice(0, 8)}:`,
+        err instanceof Error ? err.message : String(err),
+      );
+    }
+  }
+  return followUp;
 }
 
 /** Hop cap for `deferredWorkChatContextKey`; a defer/resume chain is short. */