@desplega.ai/agent-swarm 1.158.0 → 1.160.0

package/src/be/db.ts

@@ -124,6 +124,12 @@ import {
   SERVER_GENERATED_ATTACHMENT_CAPABILITY,
   SessionCostModelBreakdownSchema,
 } from "../types";
+import {
+  CODEX_AUTH_FAILURE_BENCH_MS,
+  CODEX_AUTH_FAILURE_BENCH_THRESHOLD,
+  CODEX_AUTH_WATCH_NAMESPACE,
+  codexAuthBenchMarkerKey,
+} from "../utils/codex-auth-failure";
 import { deriveProviderFromKeyType } from "../utils/credentials";
 import type { RateLimitWindowTelemetry } from "../utils/error-tracker";
 import { extractGitHubPullRequestUrls } from "../utils/github-pull-request";
@@ -136,7 +142,11 @@ import {
 import { activeModelBlock, type ModelFamily } from "../utils/model-rate-limit-windows";
 import { getCurrentRequestUserId } from "../utils/request-auth-context";
 import { registerVolatileSecret, scrubSecrets } from "../utils/secret-scrubber";
-import { estimateClaudePlan, SUBSCRIPTION_KEY_TYPES } from "../utils/subscription-plans";
+import {
+  estimateClaudePlan,
+  planAllowsModelFamily,
+  SUBSCRIPTION_KEY_TYPES,
+} from "../utils/subscription-plans";
 import { auditAssetKeys } from "./asset-key-audit";
 import { decryptSecret, encryptSecret, getEncryptionKey } from "./crypto";
 import { normalizeDate, normalizeDateRequired } from "./date-utils";
@@ -161,6 +171,7 @@ import {
   getLatestContextVersion,
 } from "./db/context-versions";
 import { getDb, getDbClient } from "./db/runtime";
+import { classifyTaskHumanFree, reclassifyTaskHumanFree } from "./db/tasks/human-free";
 import {
   type AgentTaskRow,
   configureTaskReadDependencies,
@@ -187,6 +198,7 @@ export {
   getAgentById,
   getAgentDailyTaskCounts,
   getAgentHarnessProviders,
+  getAgentStatusCounts,
   getAllAgents,
   getLeadAgent,
   getRemainingCapacity,
@@ -212,6 +224,7 @@ export {
   updateAgentStatus,
   updateAgentStatusFromCapacity,
 } from "./db/agents";
+export { recordKeySeatMismatch } from "./db/api-keys";
 export { type ApprovalRequestSummary, listApprovalRequestSummaries } from "./db/approvals";
 export {
   computeContentHash,
@@ -260,6 +273,7 @@ export {
   getRecentlyFinishedWorkerTasks,
   getSlackTasksMissingTree,
   getSupersededTasksWithoutResume,
+  getSupersededTasksWithUnsettledDependents,
   getTaskById,
   getTaskStats,
   getTasksByAgentId,
@@ -291,6 +305,7 @@ export {
   pauseTask,
   resetOrphanedInProgressTasksForAgent,
   resumeTask,
+  settleSupersededTaskDependents,
   startTask,
   supersedeTask,
   updateTaskClaudeSessionId,
@@ -891,6 +906,44 @@ export async function getSlackTasksInThread(
   return rows.map(rowToAgentTask);
 }
 
+/**
+ * Slack threads with an open human request raised by one of their own tasks
+ * (`request-human-input`): the swarm cannot go on until a person answers.
+ * A request past its `expiresAt` is about to become a timeout follow-up, so
+ * it no longer counts as waiting.
+ */
+export async function listSlackThreadsAwaitingHuman(): Promise<
+  Array<{ channelId: string; threadTs: string }>
+> {
+  return getDbClient().query<{ channelId: string; threadTs: string }>(
+    `SELECT DISTINCT task.slackChannelId AS channelId, task.slackThreadTs AS threadTs
+       FROM approval_requests request
+       JOIN agent_tasks task ON task.id = request.sourceTaskId
+       WHERE request.status = 'pending'
+       AND (request.expiresAt IS NULL OR request.expiresAt > strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+       AND task.slackChannelId IS NOT NULL
+       AND task.slackThreadTs IS NOT NULL`,
+  );
+}
+
+export async function isSlackThreadAwaitingHuman(
+  channelId: string,
+  threadTs: string,
+): Promise<boolean> {
+  const row = await getDbClient().get<{ waiting: number }>(
+    `SELECT 1 AS waiting
+       FROM approval_requests request
+       JOIN agent_tasks task ON task.id = request.sourceTaskId
+       WHERE request.status = 'pending'
+       AND (request.expiresAt IS NULL OR request.expiresAt > strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+       AND task.slackChannelId = ?
+       AND task.slackThreadTs = ?
+       LIMIT 1`,
+    [channelId, threadTs],
+  );
+  return !!row;
+}
+
 export async function markTaskSlackReplySent(taskId: string): Promise<void> {
   await getDbClient().run(`UPDATE agent_tasks SET slackReplySent = 1 WHERE id = ?`, [taskId]);
 }
@@ -2703,6 +2756,16 @@ export async function createTaskExtended(
       logRoutingDecisionFailure(id, "capture", error);
     }
 
+    const isHumanFree = await classifyTaskHumanFree({
+      taskType: options?.taskType,
+      tags: JSON.stringify(options?.tags ?? []),
+      source: options?.source ?? "mcp",
+      requestedByUserId: options?.requestedByUserId,
+      requestedByUserIdInherited,
+      parentTaskId: options?.parentTaskId,
+      workflowRunId: options?.workflowRunId,
+    });
+
     const inserted = await getDbClient().get<AgentTaskRow>(
       `INSERT INTO agent_tasks (
         id, "key", agentId, creatorAgentId, task, status, source, routing_reason, routing_source, routing_note,
@@ -2712,8 +2775,8 @@ export async function createTaskExtended(
         vcsInstallationId, vcsNodeId,
         agentmailInboxId, agentmailMessageId, agentmailThreadId,
         mentionMessageId, mentionChannelId, dir, parentTaskId, model, modelTier, effort, scheduleId,
-        workflowRunId, workflowRunStepId, outputSchema, followUpConfig, requestedByUserId, requestedByUserIdInherited, contextKey, routingAffinity, swarmVersion, createdAt, lastUpdatedAt, created_by, updated_by
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
+        workflowRunId, workflowRunStepId, outputSchema, followUpConfig, requestedByUserId, requestedByUserIdInherited, contextKey, routingAffinity, swarmVersion, createdAt, lastUpdatedAt, created_by, updated_by, isHumanFree
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
       [
         id,
         assetKey,
@@ -2768,6 +2831,7 @@ export async function createTaskExtended(
         now,
         auditUserId,
         auditUserId,
+        isHumanFree ? 1 : 0,
       ],
     );
     if (!inserted) throw new Error("Failed to create task");
@@ -3318,9 +3382,15 @@ const ELIGIBILITY_SCAN_CAP = Number(process.env.ELIGIBILITY_SCAN_CAP) || 500;
  * JSON-parsing `routingAffinity` in SQL) until `limit` eligible tasks are
  * found or the pool is exhausted, capped at `ELIGIBILITY_SCAN_CAP` rows
  * scanned so a pool full of ineligible tasks can't turn every poll into an
- * unbounded scan.
+ * unbounded scan. `accept` adds a caller-side filter (e.g. harness model
+ * compatibility) that runs inside the scan, so rejected rows do not use up
+ * the `limit`.
  */
-export async function getUnassignedTaskIdsForAgent(agentId: string, limit = 10): Promise<string[]> {
+export async function getUnassignedTaskIdsForAgent(
+  agentId: string,
+  limit = 10,
+  accept?: (task: AgentTask) => boolean | Promise<boolean>,
+): Promise<string[]> {
   const agent = await getAgentById(agentId);
   if (!agent) return [];
 
@@ -3337,7 +3407,7 @@ export async function getUnassignedTaskIdsForAgent(agentId: string, limit = 10):
 
     for (const row of rows) {
       const task = rowToAgentTask(row);
-      if (isAgentEligibleForTask(agent, task)) {
+      if (isAgentEligibleForTask(agent, task) && (!accept || (await accept(task)))) {
         eligible.push(task.id);
         if (eligible.length >= limit) break;
       }
@@ -4893,60 +4963,12 @@ export interface SessionCostByUserRow {
 /** `opts.userId` sentinel selecting spend with no human requester. */
 export const UNATTRIBUTED_USER_ID = "unattributed";
 
-// Structurally-human-free tasks and their descendants. An explicitly supplied
-// requester is a human handoff and stops propagation; a requester copied from
-// the parent does not.
-const HUMAN_FREE_TASKS_CTE = `human_free_tasks(id) AS (
-        SELECT task.id
-        FROM agent_tasks task
-        LEFT JOIN agent_tasks parent ON parent.id = task.parentTaskId
-        WHERE COALESCE(task.taskType, '') IN ('heartbeat', 'heartbeat-checklist', 'boot-triage')
-          OR COALESCE(task.tags, '[]') LIKE '%"heartbeat"%'
-          OR (COALESCE(task.source, '') = 'schedule' AND task.requestedByUserId IS NULL)
-          OR (
-            task.parentTaskId IS NULL
-            AND COALESCE(task.source, '') = 'workflow'
-            AND task.requestedByUserId IS NULL
-            AND EXISTS (
-              SELECT 1
-              FROM workflow_runs run
-              WHERE run.id = task.workflowRunId
-                AND run.triggerType = 'schedule'
-                AND run.created_by IS NULL
-            )
-          )
-          OR (
-            COALESCE(task.source, '') = 'system'
-            AND parent.id IS NOT NULL
-            AND parent.requestedByUserId IS NULL
-          )
-
-        UNION
-
-        SELECT child.id
-        FROM agent_tasks child
-        JOIN human_free_tasks parent ON child.parentTaskId = parent.id
-        WHERE child.requestedByUserId IS NULL
-          OR child.requestedByUserIdInherited = 1
-      )`;
-/** True on rows joined through `human_free_tasks hf` (see `getSessionCostSummary`). */
-const HUMAN_FREE_SQL = "hf.id IS NOT NULL";
-const ROOT_HUMAN_FREE_SQL = `(
-        COALESCE(t.taskType, '') IN ('heartbeat', 'heartbeat-checklist', 'boot-triage')
-        OR COALESCE(t.tags, '[]') LIKE '%"heartbeat"%'
-        OR (COALESCE(t.source, '') = 'schedule' AND t.requestedByUserId IS NULL)
-        OR (
-          COALESCE(t.source, '') = 'workflow'
-          AND t.requestedByUserId IS NULL
-          AND EXISTS (
-            SELECT 1
-            FROM workflow_runs run
-            WHERE run.id = t.workflowRunId
-              AND run.triggerType = 'schedule'
-              AND run.created_by IS NULL
-          )
-        )
-      )`;
+/**
+ * True for a session whose task is structurally human-free (`isHumanFree`, set
+ * when the task is created; rule in `src/be/db/tasks/human-free.ts`). A session
+ * with no task row is not human-free.
+ */
+const HUMAN_FREE_SQL = "(COALESCE(t.isHumanFree, 0) = 1)";
 
 export async function getSessionCostSummary(opts: {
   startDate?: string;
@@ -4966,19 +4988,15 @@ export async function getSessionCostSummary(opts: {
   // re-attributed after the fact, so the human requester is resolved by joining
   // through the task (same shape as `getDailySpendForUser`). Every column is
   // `sc.`-qualified because `createdAt`/`agentId` exist on both sides.
-  const from = "FROM session_costs sc LEFT JOIN agent_tasks t ON t.id = sc.taskId";
-  // One join marks the sessions of structurally-human-free tasks (`hf.id` set).
-  // The CTE holds each task id once, so the join never duplicates a session.
-  const fromHf = `${from} LEFT JOIN human_free_tasks hf ON hf.id = t.id`;
-
-  // Structurally-human-free: the swarm maintaining itself, with no human
-  // requester by construction — heartbeat/boot-triage tasks, scheduled runs
-  // without a human creator (including workflow roots launched by such a
-  // schedule), and `source='system'` follow-ups whose parent itself has no
-  // human requester. That classification propagates through descendants while
-  // they remain unattributed, so autonomous fan-out cannot leak back into the
-  // denominator. An explicitly attributed child is an independent handoff and
-  // stops propagation down that branch.
+  // INDEXED BY: without ANALYZE statistics (production has none) the planner
+  // probes tasks through the primary key and reads each full row, which follows
+  // the task's overflow pages. The covering index answers the same probe from
+  // the index alone (migration 182).
+  const from =
+    "FROM session_costs sc LEFT JOIN agent_tasks t INDEXED BY idx_agent_tasks_usage_cover ON t.id = sc.taskId";
+  // Only a user filter reads the task, and the join adds no rows (task id is unique).
+  const agentFrom = opts.userId === undefined ? "FROM session_costs sc" : from;
+
   const conditions: string[] = [];
   const params: string[] = [];
 
@@ -5002,11 +5020,6 @@ export async function getSessionCostSummary(opts: {
   }
 
   const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
-  // Only a user filter makes the daily and per-agent breakdowns classify rows.
-  // Without one they skip the CTE, which scans every task.
-  const classify = opts.userId !== undefined;
-  const withHf = classify ? `WITH RECURSIVE ${HUMAN_FREE_TASKS_CTE}` : "";
-  const scopedFrom = classify ? fromHf : from;
   const subscriptionSql = `t.credentialKeyType IN (${SUBSCRIPTION_KEY_TYPES.map((k) => `'${k}'`).join(", ")})`;
 
   // Totals
@@ -5026,8 +5039,7 @@ export async function getSessionCostSummary(opts: {
   };
 
   const totalsRow = await getDbClient().get<TotalsRow>(
-    `WITH RECURSIVE ${HUMAN_FREE_TASKS_CTE}
-      SELECT
+    `SELECT
         COALESCE(SUM(sc.totalCostUsd), 0) as totalCostUsd,
         COALESCE(SUM(sc.inputTokens), 0) as totalInputTokens,
         COALESCE(SUM(sc.outputTokens), 0) as totalOutputTokens,
@@ -5044,7 +5056,7 @@ export async function getSessionCostSummary(opts: {
         COUNT(DISTINCT CASE WHEN ${HUMAN_FREE_SQL} THEN t.id END) as excludedTaskCount,
         COALESCE(SUM(CASE WHEN ${subscriptionSql}
           THEN sc.totalCostUsd ELSE 0 END), 0) as subscriptionCostUsd
-      ${fromHf} ${where}`,
+      ${from} ${where}`,
     params,
   );
 
@@ -5082,8 +5094,7 @@ export async function getSessionCostSummary(opts: {
       sessions: number;
       subscriptionCostUsd: number;
     }>(
-      `${withHf}
-        SELECT
+      `SELECT
           DATE(sc.createdAt) as date,
           COALESCE(SUM(sc.totalCostUsd), 0) as costUsd,
           COALESCE(SUM(sc.inputTokens), 0) as inputTokens,
@@ -5091,7 +5102,7 @@ export async function getSessionCostSummary(opts: {
           COUNT(*) as sessions,
           COALESCE(SUM(CASE WHEN ${subscriptionSql} THEN sc.totalCostUsd ELSE 0 END), 0)
             as subscriptionCostUsd
-        ${scopedFrom} ${where}
+        ${from} ${where}
         GROUP BY DATE(sc.createdAt)
         ORDER BY date ASC`,
       params,
@@ -5109,15 +5120,14 @@ export async function getSessionCostSummary(opts: {
       sessions: number;
       durationMs: number;
     }>(
-      `${withHf}
-        SELECT
+      `SELECT
           sc.agentId as agentId,
           COALESCE(SUM(sc.totalCostUsd), 0) as costUsd,
           COALESCE(SUM(sc.inputTokens), 0) as inputTokens,
           COALESCE(SUM(sc.outputTokens), 0) as outputTokens,
           COUNT(*) as sessions,
           COALESCE(SUM(sc.durationMs), 0) as durationMs
-        ${scopedFrom} ${where}
+        ${agentFrom} ${where}
         GROUP BY sc.agentId
         ORDER BY costUsd DESC`,
       params,
@@ -5129,15 +5139,14 @@ export async function getSessionCostSummary(opts: {
   let byUser: SessionCostByUserRow[] = [];
   if (groupBy === "user" || groupBy === "both") {
     byUser = await getDbClient().query<SessionCostByUserRow>(
-      `WITH RECURSIVE ${HUMAN_FREE_TASKS_CTE}
-        SELECT
+      `SELECT
           CASE WHEN ${HUMAN_FREE_SQL} THEN NULL ELSE t.requestedByUserId END as userId,
           COALESCE(SUM(sc.totalCostUsd), 0) as costUsd,
           COALESCE(SUM(sc.inputTokens), 0) as inputTokens,
           COALESCE(SUM(sc.outputTokens), 0) as outputTokens,
           COUNT(DISTINCT sc.taskId) as tasks,
           COALESCE(SUM(sc.durationMs), 0) as durationMs
-        ${fromHf} ${where}
+        ${from} ${where}
         GROUP BY CASE WHEN ${HUMAN_FREE_SQL} THEN NULL ELSE t.requestedByUserId END
         ORDER BY costUsd DESC`,
       params,
@@ -5151,8 +5160,7 @@ export async function getSessionCostSummary(opts: {
       "name" | "subscription" | "plan" | "planSource"
     >;
     const rows = await getDbClient().query<CredentialRow>(
-      `${withHf}
-        SELECT
+      `SELECT
           t.credentialKeyType as keyType,
           t.credentialKeySuffix as keySuffix,
           COALESCE(SUM(sc.totalCostUsd), 0) as costUsd,
@@ -5161,7 +5169,7 @@ export async function getSessionCostSummary(opts: {
           COUNT(*) as sessions,
           MIN(sc.createdAt) as firstSessionAt,
           MAX(sc.createdAt) as lastSessionAt
-        ${scopedFrom} ${where}
+        ${from} ${where}
         GROUP BY t.credentialKeyType, t.credentialKeySuffix
         ORDER BY costUsd DESC`,
       params,
@@ -5223,18 +5231,6 @@ async function getCredentialLabels(): Promise<
   return labels;
 }
 
-/**
- * Changes when a session cost or a task is inserted. Part of the usage report
- * cache key (`src/http/usage-cache.ts`). Reads two rowid maxima, so it is cheap.
- */
-export async function getUsageDataVersion(): Promise<string> {
-  const row = await getDbClient().get<{ version: string }>(
-    `SELECT COALESCE((SELECT MAX(rowid) FROM session_costs), 0) || ':' ||
-            COALESCE((SELECT MAX(rowid) FROM agent_tasks), 0) as version`,
-  );
-  return row?.version ?? "0:0";
-}
-
 // --- Per-person attribution (four-metric view) ---
 
 export interface AttributionByPersonRow {
@@ -5265,7 +5261,7 @@ export interface AttributionByPersonRow {
  * a composite score. Scope is root tasks (`problemsInitiated`/`problemsShipped`)
  * or the person's entire task tree (`*Reached`); `requestedByUserId IS NULL`
  * (autonomous work) and structurally-human-free rows with a stale/inherited
- * requester (see `HUMAN_FREE_SQL` in `getSessionCostSummary`) are excluded —
+ * requester (`agent_tasks.isHumanFree`, see `src/be/db/tasks/human-free.ts`) are excluded —
  * neither belongs to a person.
  *
  * "Problems shipped" detection walks each root's full task tree, preferring a
@@ -5309,18 +5305,20 @@ export async function getAttributionByPerson(opts: {
 
   type RootRow = { userId: string; initiated: number; shipped: number };
   const rootRows = await getDbClient().query<RootRow>(
-    `WITH RECURSIVE selected_roots(id, requestedByUserId, status, output) AS (
-        SELECT t.id, t.requestedByUserId, t.status, t.output
+    `WITH RECURSIVE selected_roots(id, requestedByUserId, status) AS (
+        SELECT t.id, t.requestedByUserId, t.status
         FROM agent_tasks t
-        ${where} AND t.parentTaskId IS NULL AND NOT ${ROOT_HUMAN_FREE_SQL}
+        ${where} AND t.parentTaskId IS NULL AND t.isHumanFree = 0
       ),
-      task_tree(rootId, taskId, output) AS (
-        SELECT id, id, output
+      -- Ids only: carrying each descendant's \`output\` through the recursion copied
+      -- every task's text into the queue. The output fallback below reads it by id.
+      task_tree(rootId, taskId) AS (
+        SELECT id, id
         FROM selected_roots
 
         UNION ALL
 
-        SELECT tree.rootId, child.id, child.output
+        SELECT tree.rootId, child.id
         FROM agent_tasks child
         JOIN task_tree tree ON child.parentTaskId = tree.taskId
       ),
@@ -5353,8 +5351,11 @@ export async function getAttributionByPerson(opts: {
 
         SELECT tree.rootId
         FROM task_tree tree
-        WHERE tree.output LIKE '%github.com/%/pull/%'
-           OR tree.output LIKE '%/-/merge_requests/%'
+        -- CROSS JOIN pins the order (tree outer, task by id). Left to itself the
+        -- planner scans every task's output and probes the tree instead.
+        CROSS JOIN agent_tasks described ON described.id = tree.taskId
+        WHERE described.output LIKE '%github.com/%/pull/%'
+           OR described.output LIKE '%/-/merge_requests/%'
       )
       SELECT
         t.requestedByUserId as userId,
@@ -5373,65 +5374,13 @@ export async function getAttributionByPerson(opts: {
     surfacesReached: number;
   };
   const reachRows = await getDbClient().query<ReachRow>(
-    `WITH RECURSIVE report_tasks AS (
-        -- Only the columns read below: a task row carries its full prompt and output.
-        SELECT t.id, t.agentId, t.vcsRepo, t.source, t.parentTaskId, t.requestedByUserId,
-          t.requestedByUserIdInherited, t.taskType, t.tags, t.workflowRunId
-        FROM agent_tasks t
-        ${where}
-      ),
-      task_ancestry(
-        taskId, id, parentTaskId, requestedByUserId, requestedByUserIdInherited,
-        taskType, tags, source, workflowRunId
-      ) AS (
-        SELECT
-          id, id, parentTaskId, requestedByUserId, requestedByUserIdInherited,
-          taskType, tags, source, workflowRunId
-        FROM report_tasks
-
-        UNION ALL
-
-        SELECT
-          child.taskId, parent.id, parent.parentTaskId, parent.requestedByUserId,
-          parent.requestedByUserIdInherited, parent.taskType, parent.tags,
-          parent.source, parent.workflowRunId
-        FROM agent_tasks parent
-        JOIN task_ancestry child ON parent.id = child.parentTaskId
-        WHERE child.requestedByUserId IS NULL
-          OR child.requestedByUserIdInherited = 1
-      ),
-      human_free_report_tasks(id) AS (
-        SELECT DISTINCT ancestor.taskId
-        FROM task_ancestry ancestor
-        LEFT JOIN agent_tasks parent ON parent.id = ancestor.parentTaskId
-        WHERE COALESCE(ancestor.taskType, '') IN ('heartbeat', 'heartbeat-checklist', 'boot-triage')
-          OR COALESCE(ancestor.tags, '[]') LIKE '%"heartbeat"%'
-          OR (COALESCE(ancestor.source, '') = 'schedule' AND ancestor.requestedByUserId IS NULL)
-          OR (
-            ancestor.parentTaskId IS NULL
-            AND COALESCE(ancestor.source, '') = 'workflow'
-            AND ancestor.requestedByUserId IS NULL
-            AND EXISTS (
-              SELECT 1
-              FROM workflow_runs run
-              WHERE run.id = ancestor.workflowRunId
-                AND run.triggerType = 'schedule'
-                AND run.created_by IS NULL
-            )
-          )
-          OR (
-            COALESCE(ancestor.source, '') = 'system'
-            AND parent.id IS NOT NULL
-            AND parent.requestedByUserId IS NULL
-          )
-      )
-      SELECT
+    `SELECT
         t.requestedByUserId as userId,
         COUNT(DISTINCT t.agentId) as agentsReached,
         COUNT(DISTINCT t.vcsRepo) as reposReached,
         COUNT(DISTINCT t.source) as surfacesReached
-      FROM report_tasks t
-      WHERE NOT EXISTS (SELECT 1 FROM human_free_report_tasks WHERE id = t.id)
+      FROM agent_tasks t
+      ${where} AND t.isHumanFree = 0
       GROUP BY t.requestedByUserId`,
     params,
   );
@@ -7596,7 +7545,19 @@ export async function updateWorkflow(
 }
 
 export async function deleteWorkflow(id: string, source?: "api" | "mcp"): Promise<boolean> {
+  // One transaction: the stored human-free flags must never lag the deleted runs
+  // (a scheduled workflow root is human-free only while its run row exists).
+  return await getDbClient().transaction(() => deleteWorkflowRows(id, source));
+}
+
+async function deleteWorkflowRows(id: string, source?: "api" | "mcp"): Promise<boolean> {
   const client = getDbClient();
+  const linkedTaskIds = (
+    await client.query<{ id: string }>(
+      `SELECT id FROM agent_tasks WHERE workflowRunId IN (SELECT id FROM workflow_runs WHERE workflowId = ?)`,
+      [id],
+    )
+  ).map((row) => row.id);
   // Cascade delete in FK-safe order:
   // 1. Unlink agent_tasks (they reference steps and runs)
   await client.run(
@@ -7610,6 +7571,7 @@ export async function deleteWorkflow(id: string, source?: "api" | "mcp"): Promis
   );
   // 3. Delete runs (they reference workflow)
   await client.run("DELETE FROM workflow_runs WHERE workflowId = ?", [id]);
+  await reclassifyTaskHumanFree(linkedTaskIds);
   // 4. Delete workflow
   const result = await client.run("DELETE FROM workflows WHERE id = ?", [id]);
   const deleted = result.changes > 0;
@@ -11430,6 +11392,13 @@ export interface ApiKeyStatus {
   /** Subscription plan id (`SUBSCRIPTION_PLANS`), when known. */
   plan: string | null;
   planSource: PlanSource | null;
+  /** When the CLI last rejected a model with `credits_required` on this key. */
+  lastSeatMismatchAt: string | null;
+  /** Model family of that rejection (`fable`, `opus`, ...). */
+  lastSeatMismatchModel: string | null;
+  /** Auth failures in a row since the last success or clear. */
+  consecutiveAuthFailures: number;
+  lastAuthFailureAt: string | null;
   createdAt: string;
   updatedAt: string;
 }
@@ -11459,6 +11428,9 @@ export interface AvailableKeyIndicesResult {
   modelBlockedIndices: number[];
   /** ISO of the earliest resetsAt among modelBlockedIndices, or null when none. */
   earliestModelResetAt: string | null;
+  /** Indices excluded because the key's subscription plan cannot run the model family. */
+  seatBlockedIndices: number[];
+  authFailureFence: number; // highest `authFailureSeq` among these rows
 }
 
 /**
@@ -11468,7 +11440,10 @@ export interface AvailableKeyIndicesResult {
  * When `modelFamily` has a weekly window (fable/opus/sonnet), a key whose
  * `rateLimitWindows` carries an active rejected window for that family is
  * excluded from `availableIndices` and reported in `modelBlockedIndices`
- * instead — the key itself stays `available` for every other model.
+ * instead — the key itself stays `available` for every other model. A key
+ * whose `plan` cannot run the family (`planAllowsModelFamily`) is excluded
+ * and reported in `seatBlockedIndices`, also when it is key-wide or
+ * model-window blocked; a seat block has no reset time.
  */
 export async function getAvailableKeyIndices(
   keyType: string,
@@ -11482,21 +11457,23 @@ export async function getAvailableKeyIndices(
   const client = getDbClient();
   const effectiveScopeId = scopeId ?? "";
 
-  // Auto-clear expired rate limits
+  // Auto-clear expired rate limits. Never an auth bench: only `clearAuthBench` lifts it.
   await client.run(
     `UPDATE api_key_status
      SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
      WHERE keyType = ? AND scope = ? AND scopeId = ?
-       AND status = 'rate_limited' AND rateLimitedUntil IS NOT NULL AND rateLimitedUntil <= ?`,
-    [now, keyType, scope, effectiveScopeId, now],
+       AND status = 'rate_limited' AND rateLimitedUntil <= ? AND consecutiveAuthFailures < ?`,
+    [now, keyType, scope, effectiveScopeId, now, CODEX_AUTH_FAILURE_BENCH_THRESHOLD],
   );
 
   const rows = await client.query<{
     keyIndex: number;
     status: string;
     rateLimitWindows: string | null;
+    plan: string | null;
+    authFailureSeq: number;
   }>(
-    `SELECT keyIndex, status, rateLimitWindows FROM api_key_status
+    `SELECT keyIndex, status, rateLimitWindows, plan, authFailureSeq FROM api_key_status
        WHERE keyType = ? AND scope = ? AND scopeId = ?`,
     [keyType, scope, effectiveScopeId],
   );
@@ -11527,19 +11504,32 @@ export async function getAvailableKeyIndices(
   }
   const modelBlockedSet = new Set(modelBlockedIndices);
 
+  // A seat block is a plan fact: report it even when the key is also key-wide
+  // or model-window blocked, or the temporary block hides it from admission
+  // and the fallback pick can select a key that cannot run the model.
+  const seatBlockedIndices: number[] = [];
+  if (modelFamily) {
+    for (const row of rows) {
+      if (!planAllowsModelFamily(row.plan, modelFamily)) seatBlockedIndices.push(row.keyIndex);
+    }
+  }
+  const seatBlockedSet = new Set(seatBlockedIndices);
+
   const availableIndices: number[] = [];
   for (let i = 0; i < totalKeys; i++) {
-    if (blockedIndices.has(i) || modelBlockedSet.has(i)) continue;
+    if (blockedIndices.has(i) || modelBlockedSet.has(i) || seatBlockedSet.has(i)) continue;
     availableIndices.push(i);
   }
 
   return {
     availableIndices,
     modelBlockedIndices,
+    seatBlockedIndices,
     earliestModelResetAt:
       earliestModelResetsAtSec !== undefined
         ? new Date(earliestModelResetsAtSec * 1000).toISOString()
         : null,
+    authFailureFence: Math.max(0, ...rows.map((r) => r.authFailureSeq)),
   };
 }
 
@@ -11627,9 +11617,7 @@ export async function setApiKeyPlan(
   return result.changes > 0;
 }
 
-/**
- * Mark a key as rate-limited with a retry-after timestamp.
- */
+/** Mark a key as rate-limited until a retry-after timestamp; never shortens an auth-failure bench. */
 export async function markKeyRateLimited(
   keyType: string,
   keySuffix: string,
@@ -11647,7 +11635,7 @@ export async function markKeyRateLimited(
        ON CONFLICT(keyType, keySuffix, scope, scopeId)
        DO UPDATE SET
          status = 'rate_limited',
-         rateLimitedUntil = excluded.rateLimitedUntil,
+         rateLimitedUntil = CASE WHEN status = 'rate_limited' AND consecutiveAuthFailures >= ${CODEX_AUTH_FAILURE_BENCH_THRESHOLD} THEN MAX(COALESCE(rateLimitedUntil, ''), excluded.rateLimitedUntil) ELSE excluded.rateLimitedUntil END,
          lastRateLimitAt = excluded.lastRateLimitAt,
          rateLimitCount = rateLimitCount + 1,
          keyIndex = excluded.keyIndex,
@@ -11656,6 +11644,63 @@ export async function markKeyRateLimited(
   );
 }
 
+export interface KeyAuthFailureResult {
+  consecutiveAuthFailures: number;
+  benched: boolean;
+  rateLimitedUntil: string | null;
+}
+
+/** Count an auth failure, ordered by `authFailureSeq`; at the threshold, bench (extend only). */
+export async function recordKeyAuthFailure(
+  keyType: string,
+  keySuffix: string,
+  keyIndex: number,
+  scope = "global",
+  scopeId: string | null = null,
+): Promise<KeyAuthFailureResult> {
+  const effectiveScopeId = scopeId ?? "";
+  const provider = deriveProviderFromKeyType(keyType);
+  return await getDbClient().transaction(async (tx) => {
+    const now = new Date().toISOString();
+    const source = "report-auth-failure";
+    const row = await tx.get<{ consecutiveAuthFailures: number; rateLimitedUntil: string | null }>(
+      `INSERT INTO api_key_status (keyType, keySuffix, keyIndex, scope, scopeId, status, consecutiveAuthFailures, lastAuthFailureAt, authFailureSeq, provider, updatedAt)
+         VALUES (?, ?, ?, ?, ?, 'available', 1, ?, (SELECT COALESCE(MAX(authFailureSeq), 0) + 1 FROM api_key_status), ?, ?)
+         ON CONFLICT(keyType, keySuffix, scope, scopeId)
+         DO UPDATE SET
+           consecutiveAuthFailures = consecutiveAuthFailures + 1,
+           lastAuthFailureAt = excluded.lastAuthFailureAt, authFailureSeq = excluded.authFailureSeq,
+           keyIndex = excluded.keyIndex, updatedAt = excluded.updatedAt
+         RETURNING consecutiveAuthFailures, rateLimitedUntil`,
+      [keyType, keySuffix, keyIndex, scope, effectiveScopeId, now, provider, now],
+    );
+    if (!row) throw new Error("Failed to record auth failure");
+    const { consecutiveAuthFailures } = row;
+    if (consecutiveAuthFailures < CODEX_AUTH_FAILURE_BENCH_THRESHOLD) {
+      return { consecutiveAuthFailures, benched: false, rateLimitedUntil: row.rateLimitedUntil };
+    }
+    let benchedUntil = new Date(Date.parse(now) + CODEX_AUTH_FAILURE_BENCH_MS).toISOString();
+    if (row.rateLimitedUntil && Date.parse(row.rateLimitedUntil) > Date.parse(benchedUntil)) {
+      benchedUntil = row.rateLimitedUntil;
+    }
+    await tx.run(
+      `UPDATE api_key_status
+         SET status = 'rate_limited', rateLimitedUntil = ?, lastRateLimitAt = ?,
+             rateLimitCount = rateLimitCount + 1, updatedAt = ?
+         WHERE keyType = ? AND keySuffix = ? AND scope = ? AND scopeId = ?`,
+      [benchedUntil, now, now, keyType, keySuffix, scope, effectiveScopeId],
+    );
+    await upsertKv({
+      namespace: CODEX_AUTH_WATCH_NAMESPACE,
+      key: codexAuthBenchMarkerKey(keySuffix),
+      valueType: "json",
+      value: { keyIndex, keyType, benchedAt: now, benchedUntil, lastFailAt: now, source },
+      expiresAt: Date.parse(benchedUntil) + 86_400_000,
+    });
+    return { consecutiveAuthFailures, benched: true, rateLimitedUntil: benchedUntil };
+  });
+}
+
 /**
  * Merges reported window snapshots into the stored ones, one window type at a
  * time. A reported entry replaces the stored entry unless the stored entry was
@@ -11797,24 +11842,48 @@ export async function setApiKeyName(
 }
 
 /**
- * Clear a stale rate-limit record after a successful use proves the key is healthy.
+ * Clear a stale rate-limit record after a successful use proves the key is healthy. With
+ * `clearAuthBench` + `authFence` (the `authFailureFence` read before the task or credential write),
+ * also lift an auth bench, reset the count and delete the Codex bench marker, but never for a
+ * failure recorded after the fence. `keyIndex` (slot re-login) also retires the slot's other logins.
  */
 export async function clearKeyRateLimit(
   keyType: string,
   keySuffix: string,
   scope = "global",
   scopeId: string | null = null,
+  opts: { clearAuthBench?: boolean; keyIndex?: number; authFence?: number } = {},
 ): Promise<boolean> {
   const now = new Date().toISOString();
   const effectiveScopeId = scopeId ?? "";
-  const result = await getDbClient().run(
-    `UPDATE api_key_status
-       SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
-       WHERE keyType = ? AND keySuffix = ? AND scope = ? AND scopeId = ?
-         AND status = 'rate_limited'`,
-    [now, keyType, keySuffix, scope, effectiveScopeId],
-  );
-  return result.changes > 0;
+  const { keyIndex = -1, authFence = null } = opts;
+  const clearAuthBench = opts.clearAuthBench === true && authFence !== null;
+  const key = [keyType, keySuffix, scope, effectiveScopeId];
+  return await getDbClient().transaction(async (tx) => {
+    const result = await tx.run(
+      `UPDATE api_key_status
+         SET status = 'available', rateLimitedUntil = NULL, updatedAt = ?
+         WHERE keyType = ? AND keySuffix = ? AND scope = ? AND scopeId = ?
+           AND status = 'rate_limited' AND ((? AND authFailureSeq <= ?) OR consecutiveAuthFailures < ?)`,
+      [now, ...key, clearAuthBench ? 1 : 0, authFence, CODEX_AUTH_FAILURE_BENCH_THRESHOLD],
+    );
+    const cleared = result.changes > 0;
+    if (!clearAuthBench) return cleared;
+    const lifted = await tx.query<{ keySuffix: string }>(
+      `UPDATE api_key_status SET consecutiveAuthFailures = 0, updatedAt = ?1,
+           status = CASE WHEN keySuffix = ?2 THEN status ELSE 'available' END, rateLimitedUntil = CASE WHEN keySuffix = ?2 THEN rateLimitedUntil END
+         WHERE keyType = ?3 AND scope = ?4 AND scopeId = ?5 AND (consecutiveAuthFailures > 0 OR status = 'rate_limited')
+           AND ((keySuffix = ?2 AND authFailureSeq <= ?7) OR (keySuffix != ?2 AND keyIndex = ?6)) RETURNING keySuffix`,
+      [now, keySuffix, keyType, scope, effectiveScopeId, keyIndex, authFence],
+    );
+    const retired = lifted.filter((row) => row.keySuffix !== keySuffix);
+    if (keyType === "CODEX_OAUTH") {
+      for (const suffix of [...(cleared ? [keySuffix] : []), ...retired.map((r) => r.keySuffix)]) {
+        await deleteKv(CODEX_AUTH_WATCH_NAMESPACE, codexAuthBenchMarkerKey(suffix));
+      }
+    }
+    return cleared || retired.length > 0;
+  });
 }
 
 /**
@@ -12112,6 +12181,21 @@ export async function deleteUser(id: string, replacementUserId?: string): Promis
          )`,
     );
     const replacement = replacementUserId ?? null;
+    // The human-free rule only asks whether a task's requester or its workflow
+    // run's creator is NULL, so only clearing them (no replacement) can change a
+    // stored flag. Gather the tasks now: after the rewrite nothing identifies
+    // them. The workflow branch covers roots whose requester was never set.
+    const reclassifySeedIds: string[] = replacementUserId
+      ? []
+      : (
+          await tx.query<{ id: string }>(
+            `SELECT id FROM agent_tasks WHERE requestedByUserId = ?
+             UNION
+             SELECT id FROM agent_tasks
+             WHERE workflowRunId IN (SELECT id FROM workflow_runs WHERE created_by = ?)`,
+            [id, id],
+          )
+        ).map((row) => row.id);
     for (const reference of references) {
       const table = quoteSqlIdentifier(reference.tableName);
       const column = quoteSqlIdentifier(reference.columnName);
@@ -12139,6 +12223,8 @@ export async function deleteUser(id: string, replacementUserId?: string): Promis
       );
     }
 
+    await reclassifyTaskHumanFree(reclassifySeedIds);
+
     const result = await tx.run("DELETE FROM users WHERE id = ?", [id]);
     return result.changes > 0;
   });