@desplega.ai/agent-swarm 2.5.2 → 2.5.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@desplega.ai/agent-swarm",
-  "version": "2.5.2",
+  "version": "2.5.4",
   "description": "agent-swarm.dev — an open-source operating system for AI work: a lead agent breaks goals into tasks, routes them to specialized workers such as Claude Code or Codex, runs each worker in an isolated container, and preserves shared memory, tools, schedules, and review gates so delegated work compounds across sessions.",
   "keywords": [
     "agent-swarm.dev",
@@ -54,7 +54,7 @@
   "config": {
     "name": "agent-swarm"
   },
-  "packageManager": "bun@1.4.3",
+  "packageManager": "bun@1.4.4",
   "workspaces": [
     "apps/ui",
     "apps/templates-ui",
@@ -180,8 +180,8 @@
     "@desplega/model-catalog": "workspace:*",
     "@desplega/model-routing": "workspace:*",
     "@faker-js/faker": "^10.4.0",
-    "@opencode/plugin": "2.1.3",
-    "@opencode/sdk": "2.1.3",
+    "@opencode/plugin": "2.1.4",
+    "@opencode/sdk": "2.1.4",
     "@oxlint/plugins": "1.78.0",
     "@types/bun": "latest",
     "dependency-cruiser": "^18.0.0",
@@ -199,10 +199,10 @@
     "bun": ">=1.3.12"
   },
   "dependencies": {
-    "@agentclientprotocol/sdk": "2.0.1",
+    "@agentclientprotocol/sdk": "2.0.2",
     "@ai-sdk/openai": "^3.0.41",
-    "@anthropic-ai/claude-agent-sdk": "0.3.309",
-    "@anthropic-ai/sdk": "^1.1.0",
+    "@anthropic-ai/claude-agent-sdk": "0.3.310",
+    "@anthropic-ai/sdk": "^1.3.0",
     "@asteasolutions/zod-to-openapi": "^8.0.0",
     "@aws-sdk/client-bedrock": "3.1048.0",
     "@chat-adapter/discord": "4.41.1",

package/plugin/commands/create-pr.md

@@ -31,8 +31,8 @@ You should be working in a repository cloned to `/workspace/personal/<repo-name>
    - **Show the change, do not narrate it.** Aim for about 250 prose words; code blocks, mermaid, tables and `<details>` do not count. No file-by-file changelog: that belongs in your task output.
    - **Review map**: build it from `git diff --stat <base>...HEAD`. One row per area, biggest risk first, with `(+N/-M)`, a depth (🔍 deep, 👀 skim, ⏭ skip for tests and generated files) and what could break.
    - **Change outline**: 1 or 2 views of the shape of the change. Pseudocode with `+`/`-` lines for logic, mermaid for flow, a tree with paths on the nodes for structure.
-   - **Swarm provenance** (when you run in the swarm): fill it from your task context. Link the task (`$APP_URL/tasks/<taskId>`) and its tree (`$APP_URL/sessions/<rootTaskId>`), the Slack permalink of the ask if the task came from Slack, any workflow run, and the plan or research as an `agent-fs share-create` link plus its durable `live.agent-fs.dev` path. These auth-gated links go in this section only.
-   - **Media**: images and mp4s as GitHub user attachments (`github-attach` skill), docs as `agent-fs share-create` links. Never a presigned agent-fs URL for docs or videos.
+   - **Swarm provenance** (when you run in the swarm): fill it from your task context. Link the task (`$APP_URL/tasks/<taskId>`) and its tree (`$APP_URL/sessions/<rootTaskId>`), the Slack permalink of the ask if the task came from Slack, any workflow run, and the plan or research as its agent-fs live app link (`${AGENT_FS_LIVE_URL}/file/~/<org>/<drive>/<path>`). These auth-gated links go in this section only.
+   - **Media**: images and mp4s as GitHub user attachments (`github-attach` skill), docs as the agent-fs live app link under Swarm provenance (an `agent-fs share-create` link only when the requester asks for one or a reader has no agent-fs login). Never a presigned agent-fs URL for docs or videos.
 6. **Create the PR/MR** using `gh pr create` or `glab mr create`. `gh pr create --body` does not apply the repo template, so write the filled description to a file and pass it with `--body-file <file>`.
 7. **Check CI status** — After creating the PR, wait ~30 seconds, then check CI with `gh pr checks <pr-number>` (GitHub) or `glab mr view --json pipelines` (GitLab). If any check is failing, investigate the failure, fix it, push the fix, and re-check. Repeat until CI is green.
 8. **Report** the PR/MR URL and CI status.

package/plugin/pi-skills/create-pr/SKILL.md

@@ -31,8 +31,8 @@ You should be working in a repository cloned to `/workspace/personal/<repo-name>
    - **Show the change, do not narrate it.** Aim for about 250 prose words; code blocks, mermaid, tables and `<details>` do not count. No file-by-file changelog: that belongs in your task output.
    - **Review map**: build it from `git diff --stat <base>...HEAD`. One row per area, biggest risk first, with `(+N/-M)`, a depth (🔍 deep, 👀 skim, ⏭ skip for tests and generated files) and what could break.
    - **Change outline**: 1 or 2 views of the shape of the change. Pseudocode with `+`/`-` lines for logic, mermaid for flow, a tree with paths on the nodes for structure.
-   - **Swarm provenance** (when you run in the swarm): fill it from your task context. Link the task (`$APP_URL/tasks/<taskId>`) and its tree (`$APP_URL/sessions/<rootTaskId>`), the Slack permalink of the ask if the task came from Slack, any workflow run, and the plan or research as an `agent-fs share-create` link plus its durable `live.agent-fs.dev` path. These auth-gated links go in this section only.
-   - **Media**: images and mp4s as GitHub user attachments (`github-attach` skill), docs as `agent-fs share-create` links. Never a presigned agent-fs URL for docs or videos.
+   - **Swarm provenance** (when you run in the swarm): fill it from your task context. Link the task (`$APP_URL/tasks/<taskId>`) and its tree (`$APP_URL/sessions/<rootTaskId>`), the Slack permalink of the ask if the task came from Slack, any workflow run, and the plan or research as its agent-fs live app link (`${AGENT_FS_LIVE_URL}/file/~/<org>/<drive>/<path>`). These auth-gated links go in this section only.
+   - **Media**: images and mp4s as GitHub user attachments (`github-attach` skill), docs as the agent-fs live app link under Swarm provenance (an `agent-fs share-create` link only when the requester asks for one or a reader has no agent-fs login). Never a presigned agent-fs URL for docs or videos.
 6. **Create the PR/MR** using `gh pr create` or `glab mr create`. `gh pr create --body` does not apply the repo template, so write the filled description to a file and pass it with `--body-file <file>`.
 7. **Check CI status** — After creating the PR, wait ~30 seconds, then check CI with `gh pr checks <pr-number>` (GitHub) or `glab mr view --json pipelines` (GitLab). If any check is failing, investigate the failure, fix it, push the fix, and re-check. Repeat until CI is green.
 8. **Report** the PR/MR URL and CI status.

package/src/be/approval-config-question.ts

@@ -0,0 +1,389 @@
+/**
+ * HITL `config` questions: ask a person to set one swarm config key (a secret
+ * or a plain value) from the dashboard. The answer is written to
+ * `swarm_config` by the respond handler, and only a {@link ConfigResult} is
+ * ever stored on the request or sent onward, so a secret never reaches the
+ * agent, the workflow run, the follow-up task or any notification.
+ */
+import * as z from "zod";
+import { getDbClient } from "./db";
+import { upsertSwarmConfigWithPolicyMirror } from "./multi-runtime";
+import {
+  internalConfigKeyError,
+  isApiOnlyConfigKey,
+  isInternalConfigKey,
+  isReservedConfigKey,
+  reservedKeyError,
+  validateConfigEntry,
+} from "./swarm-config-guard";
+
+export const ConfigQuestionSettingsSchema = z.object({
+  key: z
+    .string()
+    .min(1)
+    .max(255)
+    .regex(/^\S+$/, "must not contain whitespace")
+    .describe("The swarm config key to set, e.g. BITBUCKET_TOKEN"),
+  scope: z.enum(["global", "agent", "repo"]).describe("Config scope the value is written to"),
+  scopeId: z
+    .string()
+    .min(1)
+    .optional()
+    .describe(
+      "Agent or repo id for agent/repo scope. From request-human-input, agent scope defaults to the calling agent; pass it explicitly when asking on behalf of another agent.",
+    ),
+  isSecret: z
+    .boolean()
+    .optional()
+    .describe(
+      "Store encrypted and never show the value (default true). An existing secret row always stays secret.",
+    ),
+  helpUrl: z
+    .string()
+    .url()
+    .refine((url) => /^https?:\/\//i.test(url), "must be an http(s) URL")
+    .optional()
+    .describe("Link shown next to the field, e.g. where to create the token"),
+  options: z
+    .array(
+      z.object({
+        value: z.string(),
+        label: z.string(),
+        description: z.string().optional(),
+      }),
+    )
+    .optional()
+    .describe("Allowed values for a non-secret key, rendered as a choice"),
+  defaultValue: z.string().optional().describe("Prefilled value for a non-secret key"),
+});
+
+export type ConfigQuestionSettings = z.infer<typeof ConfigQuestionSettingsSchema>;
+
+/** Settings after create-time normalization: `isSecret` is always resolved. */
+export type NormalizedConfigSettings = ConfigQuestionSettings & { isSecret: boolean };
+
+/** What the dashboard sends for a config question (D6). */
+export const ConfigAnswerSchema = z.discriminatedUnion("action", [
+  z.object({ action: z.literal("set"), value: z.string() }),
+  z.object({ action: z.literal("keep") }),
+  z.object({ action: z.literal("skip") }),
+]);
+export type ConfigAnswer = z.infer<typeof ConfigAnswerSchema>;
+
+/** The only form of a config answer that is ever stored or sent (D7). */
+export const ConfigResultSchema = z.object({
+  key: z.string(),
+  scope: z.enum(["global", "agent", "repo"]),
+  scopeId: z.string().nullable(),
+  isSecret: z.boolean(),
+  status: z.enum(["set", "kept", "skipped"]),
+  updatedAt: z.string().nullable(),
+  value: z.string().optional(),
+});
+export type ConfigResult = z.infer<typeof ConfigResultSchema>;
+
+/** Live state of the key a config question targets (never persisted). */
+export const ConfigStateSchema = z.object({
+  exists: z.boolean(),
+  isSecret: z.boolean(),
+  updatedAt: z.string().nullable(),
+  value: z.string().optional().describe("Current value; present only for non-secret rows"),
+});
+export type ConfigState = z.infer<typeof ConfigStateSchema>;
+
+type ConfigQuestionInput = {
+  id: string;
+  type: string;
+  label?: string;
+  required?: boolean;
+  config?: ConfigQuestionSettings;
+};
+
+export type NormalizedConfigQuestion<Q extends ConfigQuestionInput> = Omit<Q, "config"> & {
+  label: string;
+  required: boolean;
+  config: NormalizedConfigSettings;
+};
+
+type ApproversLike = { policy: unknown };
+
+interface ConfigRowMeta {
+  isSecret: boolean;
+  updatedAt: string;
+  envPath: string | null;
+  description: string | null;
+  value?: string;
+}
+
+/** The exact `(scope, scopeId, key)` row, without decrypting anything. */
+async function readConfigRow(
+  settings: Pick<ConfigQuestionSettings, "scope" | "scopeId" | "key">,
+): Promise<ConfigRowMeta | null> {
+  const scopeId = settings.scope === "global" ? null : (settings.scopeId ?? null);
+  const row = await getDbClient().get<{
+    isSecret: number;
+    lastUpdatedAt: string;
+    envPath: string | null;
+    description: string | null;
+    plainValue: string | null;
+  }>(
+    `SELECT isSecret, lastUpdatedAt, envPath, description,
+            CASE WHEN isSecret = 1 THEN NULL ELSE value END AS plainValue
+       FROM swarm_config
+      WHERE scope = ? AND ${scopeId === null ? "scopeId IS NULL" : "scopeId = ?"} AND key = ?`,
+    scopeId === null ? [settings.scope, settings.key] : [settings.scope, scopeId, settings.key],
+  );
+  if (!row) return null;
+  // API-only keys are never readable over HTTP, so they count as secret here.
+  const isSecret = row.isSecret === 1 || isApiOnlyConfigKey(settings.key);
+  return {
+    isSecret,
+    updatedAt: row.lastUpdatedAt,
+    envPath: row.envPath,
+    description: row.description,
+    ...(isSecret || row.plainValue === null ? {} : { value: row.plainValue }),
+  };
+}
+
+export async function readConfigState(settings: ConfigQuestionSettings): Promise<ConfigState> {
+  const row = await readConfigRow(settings);
+  if (!row) return { exists: false, isSecret: settings.isSecret !== false, updatedAt: null };
+  const isSecret = row.isSecret || settings.isSecret !== false;
+  return {
+    exists: true,
+    isSecret,
+    updatedAt: row.updatedAt,
+    ...(!isSecret && row.value !== undefined ? { value: row.value } : {}),
+  };
+}
+
+/**
+ * Create-time rules for one config question (D2, D3, D5). Defaults `label` to
+ * the key and `required` to true, resolves the effective `isSecret` against
+ * the live row, and rejects keys and shapes the config API would refuse.
+ */
+export async function normalizeConfigQuestion<Q extends ConfigQuestionInput>(
+  question: Q,
+  ctx: { approvers: ApproversLike; callerAgentId?: string },
+): Promise<{ ok: true; question: NormalizedConfigQuestion<Q> } | { ok: false; error: string }> {
+  const fail = (reason: string) => ({
+    ok: false as const,
+    error: `config question "${question.id}": ${reason}`,
+  });
+  const parsed = ConfigQuestionSettingsSchema.safeParse(question.config);
+  if (!parsed.success) {
+    const issue = parsed.error.issues[0];
+    const path = issue?.path.length ? `config.${issue.path.map(String).join(".")}` : "config";
+    return fail(`${path}: ${issue?.message ?? "invalid"}`);
+  }
+  if (ctx.approvers.policy !== "any") {
+    return fail('a config question needs approvers.policy "any" (a config value has one author)');
+  }
+
+  const settings = { ...parsed.data };
+  if (settings.scope === "agent" && !settings.scopeId && ctx.callerAgentId) {
+    settings.scopeId = ctx.callerAgentId;
+  }
+  if (settings.scope === "global" && settings.scopeId) {
+    return fail("global scope must not have scopeId");
+  }
+  if (settings.scope !== "global" && !settings.scopeId) {
+    return fail(`${settings.scope} scope requires scopeId`);
+  }
+  if (isReservedConfigKey(settings.key)) return fail(reservedKeyError(settings.key).message);
+  if (isInternalConfigKey(settings.key)) return fail(internalConfigKeyError(settings.key).message);
+
+  const row = await readConfigRow(settings);
+  const isSecret =
+    settings.isSecret !== false || row?.isSecret === true || isApiOnlyConfigKey(settings.key);
+  if (isSecret && (settings.options || settings.defaultValue !== undefined)) {
+    return fail(
+      "options and defaultValue are only allowed when isSecret is false (and the key is not already stored as a secret)",
+    );
+  }
+  if (settings.options?.length === 0) return fail("options must not be empty");
+  if (
+    settings.options &&
+    settings.defaultValue !== undefined &&
+    !settings.options.some((option) => option.value === settings.defaultValue)
+  ) {
+    return fail("defaultValue must be one of options");
+  }
+
+  const label = question.label?.trim() ? question.label : settings.key;
+  return {
+    ok: true,
+    question: {
+      ...question,
+      label,
+      required: question.required ?? true,
+      config: { ...settings, isSecret },
+    },
+  };
+}
+
+type AnyQuestion = { id: string; type: string; label?: string; config?: ConfigQuestionSettings };
+
+/**
+ * Normalizes every config question in a list and checks that the others carry
+ * a label and no `config` block. Non-config questions pass through unchanged.
+ */
+export async function normalizeApprovalQuestions<Q extends AnyQuestion>(
+  questions: Q[],
+  ctx: { approvers: ApproversLike; callerAgentId?: string },
+): Promise<
+  | { ok: true; questions: ((Q & { label: string }) | NormalizedConfigQuestion<Q>)[] }
+  | { ok: false; error: string }
+> {
+  const out: ((Q & { label: string }) | NormalizedConfigQuestion<Q>)[] = [];
+  for (const question of questions) {
+    if (question.type !== "config") {
+      if (question.config !== undefined) {
+        return {
+          ok: false,
+          error: `question "${question.id}": the config block is only valid on type "config"`,
+        };
+      }
+      if (typeof question.label !== "string") {
+        return { ok: false, error: `question "${question.id}": label is required` };
+      }
+      out.push({ ...question, label: question.label });
+      continue;
+    }
+    const normalized = await normalizeConfigQuestion(question, ctx);
+    if (!normalized.ok) return normalized;
+    out.push(normalized.question);
+  }
+  return { ok: true, questions: out };
+}
+
+/** A stored config question, as read back from `approval_requests.questions`. */
+export type StoredConfigQuestion = {
+  id: string;
+  type: "config";
+  label: string;
+  required?: boolean;
+  config: NormalizedConfigSettings;
+};
+
+export function isConfigQuestion(question: {
+  type: string;
+  config?: unknown;
+}): question is StoredConfigQuestion {
+  return question.type === "config" && typeof question.config === "object" && !!question.config;
+}
+
+/**
+ * Validates one raw answer against D6 and the config API's rules. Errors name
+ * the question and the rule, never the value.
+ */
+export function parseConfigAnswer(
+  question: StoredConfigQuestion,
+  raw: unknown,
+  state: ConfigState,
+): { ok: true; answer: ConfigAnswer } | { ok: false; error: string } {
+  const fail = (reason: string) => ({
+    ok: false as const,
+    error: `config question "${question.id}": ${reason}`,
+  });
+  const required = question.required !== false;
+  if (raw === undefined || raw === null) {
+    return required ? fail("an answer is required") : { ok: true, answer: { action: "skip" } };
+  }
+  const parsed = ConfigAnswerSchema.safeParse(raw);
+  if (!parsed.success) {
+    return fail(
+      'answer must be { action: "set", value } , { action: "keep" } or { action: "skip" }',
+    );
+  }
+  const answer = parsed.data;
+  if (answer.action === "keep") {
+    return state.exists ? { ok: true, answer } : fail("cannot keep a key that is not set");
+  }
+  if (answer.action === "skip") {
+    return required ? fail("a required question cannot be skipped") : { ok: true, answer };
+  }
+  if (answer.value.trim().length === 0) return fail("value must not be empty");
+  const { config } = question;
+  const isSecret = config.isSecret || state.isSecret;
+  if (!isSecret && config.options && !config.options.some((o) => o.value === answer.value)) {
+    return fail("value must be one of options");
+  }
+  const error = validateConfigEntry({
+    scope: config.scope,
+    scopeId: config.scopeId ?? null,
+    key: config.key,
+    value: answer.value,
+  });
+  if (error) return fail(isSecret ? `value rejected by ${config.key} validation` : error);
+  return { ok: true, answer };
+}
+
+function result(
+  question: StoredConfigQuestion,
+  status: ConfigResult["status"],
+  isSecret: boolean,
+  updatedAt: string | null,
+  value?: string,
+): ConfigResult {
+  const { key, scope, scopeId } = question.config;
+  return {
+    key,
+    scope,
+    scopeId: scope === "global" ? null : (scopeId ?? null),
+    isSecret,
+    status,
+    updatedAt,
+    ...(!isSecret && value !== undefined ? { value } : {}),
+  };
+}
+
+export function skippedConfigResult(question: StoredConfigQuestion): ConfigResult {
+  return result(question, "skipped", question.config.isSecret, null);
+}
+
+/** `keep`: nothing is written. Null when the row disappeared since validation. */
+export async function keptConfigResult(
+  question: StoredConfigQuestion,
+): Promise<ConfigResult | null> {
+  const row = await readConfigRow(question.config);
+  if (!row) return null;
+  const isSecret = question.config.isSecret || row.isSecret;
+  return result(question, "kept", isSecret, row.updatedAt, row.value);
+}
+
+/**
+ * `set`: writes the value inside the caller's transaction. Carries the row's
+ * `envPath` and `description` forward (the upsert would null them) and keeps
+ * an existing secret secret. The caller runs the HTTP side effects after
+ * commit.
+ */
+export async function applyConfigAnswer(
+  question: StoredConfigQuestion,
+  value: string,
+): Promise<ConfigResult> {
+  const { config } = question;
+  const existing = await readConfigRow(config);
+  const isSecret = config.isSecret || existing?.isSecret === true;
+  const written = await upsertSwarmConfigWithPolicyMirror({
+    scope: config.scope,
+    scopeId: config.scope === "global" ? null : (config.scopeId ?? null),
+    key: config.key,
+    value,
+    isSecret,
+    envPath: existing?.envPath ?? null,
+    description: existing?.description ?? null,
+  });
+  return result(question, "set", isSecret, written.lastUpdatedAt, value);
+}
+
+/** One line for the hitl-follow-up task text. Never carries a secret. */
+export function formatConfigResult(value: unknown): string | null {
+  const parsed = ConfigResultSchema.safeParse(value);
+  if (!parsed.success) return null;
+  const r = parsed.data;
+  const where = `${r.scope}${r.scopeId ? `:${r.scopeId}` : ""}`;
+  const updated = r.updatedAt ? `, updated ${r.updatedAt}` : "";
+  const shown = !r.isSecret && r.value !== undefined ? ` = ${r.value}` : "";
+  return `${r.key} (${where}, ${r.isSecret ? "secret" : "plain"}): ${r.status}${updated}${shown}`;
+}

package/src/be/automation-preflight.ts

@@ -345,6 +345,29 @@ function utcDayBounds(now: Date): { start: string; end: string } {
   return { start: start.toISOString(), end: new Date(start.getTime() + 86_400_000).toISOString() };
 }
 
+/**
+ * True when `runId` is the workflow's only setup-failure run so far this UTC
+ * day, top-level or child. Call it in the transaction that inserted `runId`:
+ * transactions serialize on the write lock, so of several concurrent failures
+ * exactly one sees no other and alerts.
+ */
+export async function isFirstWorkflowPreflightFailureToday(
+  workflowId: string,
+  runId: string,
+  now: Date = new Date(),
+): Promise<boolean> {
+  const bounds = utcDayBounds(now);
+  const row = await getDbClient().get<{ found: number }>(
+    `SELECT 1 AS found FROM workflow_runs
+     WHERE workflowId = ? AND id != ? AND status = 'failed'
+       AND error LIKE 'needs_setup:%'
+       AND startedAt >= ? AND startedAt < ?
+     LIMIT 1`,
+    [workflowId, runId, bounds.start, bounds.end],
+  );
+  return row == null;
+}
+
 /** Record at most one copy of the same setup failure on a schedule per UTC day. */
 export async function recordSchedulePreflightFailure(
   scheduleId: string,
@@ -376,7 +399,11 @@ export async function recordSchedulePreflightFailure(
   });
 }
 
-/** Create and finish at most one failed preflight run per workflow and UTC day. */
+/**
+ * Create and finish at most one failed top-level preflight run per workflow
+ * and UTC day. `recorded` is true when the caller should alert: this call
+ * inserted the run and no child failure run of the workflow came first today.
+ */
 export async function recordWorkflowPreflightFailure(input: {
   workflowId: string;
   triggerType: "schedule" | "manual" | "event" | "api";
@@ -388,9 +415,11 @@ export async function recordWorkflowPreflightFailure(input: {
   const now = input.now ?? new Date();
   const bounds = utcDayBounds(now);
   return await getDbClient().transaction(async (tx) => {
+    // Child runs record their own linked failure (`startWorkflowExecution`);
+    // a top-level trigger never reuses one.
     const existing = await tx.get<{ id: string }>(
       `SELECT id FROM workflow_runs
-       WHERE workflowId = ? AND status = 'failed'
+       WHERE workflowId = ? AND status = 'failed' AND parentStepId IS NULL
          AND error LIKE 'needs_setup:%'
          AND startedAt >= ? AND startedAt < ?
        ORDER BY startedAt ASC LIMIT 1`,
@@ -417,6 +446,11 @@ export async function recordWorkflowPreflightFailure(input: {
         timestamp,
       ],
     );
-    return { runId, recorded: true };
+    // Claimed in this transaction, which also sees every child failure run
+    // committed before it: concurrent failures alert exactly once a day.
+    return {
+      runId,
+      recorded: await isFirstWorkflowPreflightFailureToday(input.workflowId, runId, now),
+    };
   });
 }

package/src/be/db/approvals.ts

@@ -162,3 +162,51 @@ export async function getPendingApprovalsForTasks(taskIds: string[]): Promise<
     taskIds,
   );
 }
+
+/** The claim key of one Slack thread's cancellation post: `<channel>:<messageTs>`. */
+export function approvalCancellationNotificationKey(target: string, messageTs: string): string {
+  return `${target}:${messageTs}`;
+}
+
+/**
+ * Ids of the cancelled requests of the given runs with a Slack cancellation
+ * post not yet delivered (never claimed, or a claim a failed post released).
+ * A delivered post never comes back, so a repeat cancel of a settled tree
+ * retries only what failed. One query per 500 runs.
+ */
+export async function listUndeliveredCancelledApprovalRequestIds(
+  runIds: string[],
+): Promise<string[]> {
+  const ids: string[] = [];
+  for (let i = 0; i < runIds.length; i += 500) {
+    const chunk = runIds.slice(i, i + 500);
+    const rows = await getDbClient().query<{
+      id: string;
+      notificationChannels: string;
+      claims: string | null;
+    }>(
+      `SELECT id, notificationChannels, cancellationNotificationClaims AS claims
+         FROM approval_requests
+        WHERE workflowRunId IN (${chunk.map(() => "?").join(", ")})
+          AND status = 'cancelled' AND notificationChannels IS NOT NULL`,
+      chunk,
+    );
+    for (const row of rows) {
+      const claims = row.claims ? (JSON.parse(row.claims) as Record<string, unknown>) : {};
+      const channels = JSON.parse(row.notificationChannels) as Array<{
+        channel?: string;
+        target?: string;
+        messageTs?: string;
+      }>;
+      const undelivered = channels.some(
+        (n) =>
+          n.channel === "slack" &&
+          n.target &&
+          n.messageTs &&
+          claims[approvalCancellationNotificationKey(n.target, n.messageTs)] !== "delivered",
+      );
+      if (undelivered) ids.push(row.id);
+    }
+  }
+  return ids;
+}