@desplega.ai/agent-swarm 1.158.0 → 1.159.0

package/src/be/harness-model-support.ts

@@ -0,0 +1,182 @@
+/**
+ * Harness CLI model support (model-catalog phase 4).
+ *
+ * A catalog model can exist before the pinned `claude` / `codex` CLI accepts
+ * it. Workers report their CLI version on register and record each model's
+ * first-run outcome here; claim-time resolution reads it:
+ *
+ *   - `unknown` (no row) and `ok` are allowed at claim.
+ *   - `unsupported` + an alias/tier resolution → fall back to the newest
+ *     `ok`/`unknown` model in the same family (modelSource
+ *     `fallback:cli-unsupported`), with one notice per (harness, CLI, model).
+ *   - `unsupported` + an explicit task `model` → fail fast at claim.
+ *
+ * See runbooks/model-catalog.md.
+ */
+import {
+  claudeCatalogModelId,
+  harnessCatalogSection,
+  harnessModelIds,
+  modelFamilyKey,
+} from "@desplega/model-catalog";
+import { scrubSecrets } from "../utils/secret-scrubber";
+import { getDbClient } from "./db";
+
+export type HarnessModelSupportStatus = "ok" | "unsupported" | "unknown";
+
+export interface HarnessModelSupportRow {
+  harness: string;
+  cliVersion: string;
+  modelId: string;
+  status: HarnessModelSupportStatus;
+  checkedAt: number;
+  error: string | null;
+}
+
+/** Harnesses whose CLI pins the set of accepted model ids. */
+export const CLI_PINNED_HARNESSES = new Set(["claude", "codex"]);
+
+export async function getHarnessModelSupport(
+  harness: string,
+  cliVersion: string,
+  modelId: string,
+): Promise<HarnessModelSupportStatus> {
+  const row = await getDbClient().get<{ status: HarnessModelSupportStatus }>(
+    "SELECT status FROM harness_model_support WHERE harness = ? AND cliVersion = ? AND modelId = ?",
+    [harness, cliVersion, modelId],
+  );
+  return row?.status ?? "unknown";
+}
+
+export async function listHarnessModelSupport(filter?: {
+  harness?: string;
+  cliVersion?: string;
+}): Promise<HarnessModelSupportRow[]> {
+  const where: string[] = [];
+  const params: string[] = [];
+  if (filter?.harness) {
+    where.push("harness = ?");
+    params.push(filter.harness);
+  }
+  if (filter?.cliVersion) {
+    where.push("cliVersion = ?");
+    params.push(filter.cliVersion);
+  }
+  return await getDbClient().query<HarnessModelSupportRow>(
+    `SELECT harness, cliVersion, modelId, status, checkedAt, error FROM harness_model_support${
+      where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""
+    } ORDER BY checkedAt DESC`,
+    params,
+  );
+}
+
+/** Upsert one outcome. An `ok` never downgrades to `unknown`. */
+export async function recordHarnessModelSupport(input: {
+  harness: string;
+  cliVersion: string;
+  modelId: string;
+  status: HarnessModelSupportStatus;
+  error?: string | null;
+  now?: number;
+}): Promise<HarnessModelSupportRow> {
+  const checkedAt = input.now ?? Date.now();
+  const error = input.error ? scrubSecrets(input.error).slice(0, 2000) : null;
+  await getDbClient().run(
+    `INSERT INTO harness_model_support (harness, cliVersion, modelId, status, checkedAt, error)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT (harness, cliVersion, modelId) DO UPDATE SET
+       status = CASE WHEN excluded.status = 'unknown' THEN harness_model_support.status ELSE excluded.status END,
+       checkedAt = excluded.checkedAt,
+       error = excluded.error`,
+    [input.harness, input.cliVersion, input.modelId, input.status, checkedAt, error],
+  );
+  const row = await getDbClient().get<HarnessModelSupportRow>(
+    "SELECT harness, cliVersion, modelId, status, checkedAt, error FROM harness_model_support WHERE harness = ? AND cliVersion = ? AND modelId = ?",
+    [input.harness, input.cliVersion, input.modelId],
+  );
+  if (!row) throw new Error("harness_model_support upsert returned no row");
+  return row;
+}
+
+/**
+ * True when `agentId` was assigned a task that ran `modelId`: the server's claim-time resolution,
+ * or the task's own model when nothing was resolved. Binds an `unsupported` report to a model the
+ * worker actually ran, so it cannot mark an arbitrary model unsupported for its peers.
+ */
+export async function agentRanModel(agentId: string, modelId: string): Promise<boolean> {
+  const row = await getDbClient().get<{ id: string }>(
+    `SELECT id FROM agent_tasks
+     WHERE agentId = ? AND (resolvedModel = ? OR (resolvedModel IS NULL AND model = ?))
+     LIMIT 1`,
+    [agentId, modelId, modelId],
+  );
+  return row !== null && row !== undefined;
+}
+
+export async function getAgentHarnessCliVersion(agentId: string): Promise<string | null> {
+  const row = await getDbClient().get<{ harnessCliVersion: string | null }>(
+    "SELECT harnessCliVersion FROM agents WHERE id = ?",
+    [agentId],
+  );
+  return row?.harnessCliVersion ?? null;
+}
+
+/** Store the worker's CLI version; no write when unchanged. */
+export async function setAgentHarnessCliVersion(
+  agentId: string,
+  cliVersion: string | null,
+): Promise<void> {
+  await getDbClient().run(
+    "UPDATE agents SET harnessCliVersion = ? WHERE id = ? AND harnessCliVersion IS NOT ?",
+    [cliVersion, agentId, cliVersion],
+  );
+}
+
+export function unsupportedModelMessage(harness: string, cliVersion: string, model: string) {
+  return `Model "${model}" is not supported by the ${harness} CLI ${cliVersion} on this worker. A CLI bump is needed before tasks can pin it.`;
+}
+
+/**
+ * Newest model in the same family as `model` that this harness CLI has not
+ * rejected. Null when the family has nothing else. A Claude CLI shortname (the
+ * tier default `opus`) counts as the family of the model it stands for, so a
+ * rejected `opus` falls back to the newest usable `claude-opus-*`.
+ */
+export async function fallbackForUnsupportedModel(
+  harness: string,
+  cliVersion: string,
+  model: string,
+  catalogSection: Record<string, { release_date?: string; status?: string; reasoning?: boolean }>,
+): Promise<string | null> {
+  if (!harnessCatalogSection(harness)) return null;
+  const family = modelFamilyKey(
+    harness === "claude" ? claudeCatalogModelId(model, catalogSection) : model,
+  );
+  const rejected = new Set(
+    (await listHarnessModelSupport({ harness, cliVersion }))
+      .filter((r) => r.status === "unsupported")
+      .map((r) => r.modelId),
+  );
+  for (const id of harnessModelIds(harness, catalogSection)) {
+    if (id === model || rejected.has(id)) continue;
+    if (modelFamilyKey(id) === family) return id;
+  }
+  return null;
+}
+
+const noticed = new Set<string>();
+
+/** One console notice per (harness, CLI, model) per process. */
+export function noticeCliUnsupported(
+  harness: string,
+  cliVersion: string,
+  model: string,
+  fallback: string,
+): void {
+  const key = `${harness}|${cliVersion}|${model}`;
+  if (noticed.has(key)) return;
+  noticed.add(key);
+  console.warn(
+    `[model-tiers] ${model} is unsupported on ${harness} CLI ${cliVersion}; tier/alias tasks fall back to ${fallback} until the CLI is bumped`,
+  );
+}

package/src/be/human-free-drain.ts

@@ -0,0 +1,67 @@
+import { drainHumanFreeReclassifyQueue, setHumanFreeQueueListener } from "./db/tasks/human-free";
+
+/**
+ * Background drain for `human_free_reclassify_queue`: the part of a large
+ * subtree's human-free reclassification that did not fit in the request that
+ * changed it. It runs one batch per pass and yields between passes, so no
+ * pass holds the write lock or the event loop for longer than one batch.
+ *
+ * It wakes when a mutation queues work, and once at boot and on a slow timer to
+ * pick up rows that survived a restart or a failed pass.
+ */
+
+const YIELD_MS = 25;
+const IDLE_INTERVAL_MS = 60_000;
+
+let running = false;
+let idleTimer: ReturnType<typeof setTimeout> | null = null;
+let passTimer: ReturnType<typeof setTimeout> | null = null;
+
+function scheduleIdle(): void {
+  if (!running) return;
+  if (idleTimer) clearTimeout(idleTimer);
+  idleTimer = setTimeout(() => schedulePass(0), IDLE_INTERVAL_MS);
+  idleTimer.unref?.();
+}
+
+function schedulePass(delayMs: number): void {
+  if (!running || passTimer) return;
+  passTimer = setTimeout(() => {
+    passTimer = null;
+    void runPass();
+  }, delayMs);
+  passTimer.unref?.();
+}
+
+async function runPass(): Promise<void> {
+  if (!running) return;
+  try {
+    const { batches, remaining } = await drainHumanFreeReclassifyQueue({ maxBatches: 1 });
+    if (batches > 0 && remaining > 0) {
+      schedulePass(YIELD_MS);
+      return;
+    }
+  } catch (err) {
+    console.error(
+      "[human-free-drain] batch failed, retrying later:",
+      err instanceof Error ? err.message : String(err),
+    );
+  }
+  scheduleIdle();
+}
+
+export function startHumanFreeDrain(): void {
+  if (running) return;
+  running = true;
+  setHumanFreeQueueListener(() => schedulePass(0));
+  schedulePass(0);
+}
+
+export function stopHumanFreeDrain(): void {
+  running = false;
+  setHumanFreeQueueListener(undefined);
+  if (idleTimer) clearTimeout(idleTimer);
+  if (passTimer) clearTimeout(passTimer);
+  idleTimer = null;
+  passTimer = null;
+}

package/src/be/migrations/182_task_human_free_flag.sql

@@ -0,0 +1,63 @@
+-- Persist the structurally-human-free classification on agent_tasks.
+--
+-- The usage reports (GET /api/session-costs/summary, /api/attribution/by-person)
+-- rebuilt a recursive CTE over EVERY task on each call to decide which sessions
+-- belong to the swarm maintaining itself. That CTE is seeded from all tasks and
+-- date filters do not prune it, so a 30-day summary paid for the whole history.
+-- The classification is fixed when a task is created, so it is stored once
+-- (src/be/db/tasks/human-free.ts owns the rule) and the reports read a column.
+--
+-- Rule (mirrors HUMAN_FREE_BASE_SQL): heartbeat / boot-triage tasks, scheduled
+-- runs with no human creator (including workflow roots launched by one), and
+-- `source='system'` follow-ups whose parent has no requester. It propagates to
+-- children that stay unattributed (no requester, or one copied from the parent).
+ALTER TABLE agent_tasks
+ADD COLUMN isHumanFree INTEGER NOT NULL DEFAULT 0
+CHECK (isHumanFree IN (0, 1));
+
+WITH RECURSIVE human_free_tasks(id) AS (
+  SELECT task.id
+  FROM agent_tasks task
+  LEFT JOIN agent_tasks parent ON parent.id = task.parentTaskId
+  WHERE COALESCE(task.taskType, '') IN ('heartbeat', 'heartbeat-checklist', 'boot-triage')
+    OR COALESCE(task.tags, '[]') LIKE '%"heartbeat"%'
+    OR (COALESCE(task.source, '') = 'schedule' AND task.requestedByUserId IS NULL)
+    OR (
+      task.parentTaskId IS NULL
+      AND COALESCE(task.source, '') = 'workflow'
+      AND task.requestedByUserId IS NULL
+      AND EXISTS (
+        SELECT 1
+        FROM workflow_runs run
+        WHERE run.id = task.workflowRunId
+          AND run.triggerType = 'schedule'
+          AND run.created_by IS NULL
+      )
+    )
+    OR (
+      COALESCE(task.source, '') = 'system'
+      AND parent.id IS NOT NULL
+      AND parent.requestedByUserId IS NULL
+    )
+
+  UNION
+
+  SELECT child.id
+  FROM agent_tasks child
+  JOIN human_free_tasks parent ON child.parentTaskId = parent.id
+  WHERE child.requestedByUserId IS NULL
+    OR child.requestedByUserIdInherited = 1
+)
+UPDATE agent_tasks
+SET isHumanFree = 1
+WHERE id IN (SELECT id FROM human_free_tasks);
+
+-- Migration 164's covering index carried the columns the recursive CTE read
+-- (parentTaskId, taskType, source, tags, workflowRunId). Nothing reads them
+-- through it now. The session join needs only the requester, the flag and the
+-- credential, so keep the index to those and let it stay index-only.
+DROP INDEX IF EXISTS idx_agent_tasks_usage_cover;
+CREATE INDEX idx_agent_tasks_usage_cover
+  ON agent_tasks(
+    id, requestedByUserId, isHumanFree, credentialKeyType, credentialKeySuffix
+  );

package/src/be/migrations/183_human_free_reclassify_queue.sql

@@ -0,0 +1,25 @@
+-- Continuation queue for the human-free reclassification (migration 182).
+--
+-- Reclassifying a subtree used to walk every descendant inside the request that
+-- changed a classifying input, so a very large task tree could hold the write
+-- lock and the event loop for the whole walk. The walk now handles a bounded
+-- batch inline and parks the rest here; a background drain
+-- (src/be/human-free-drain.ts) works the queue off in bounded batches, each in
+-- its own transaction. Rows are written in the same transaction as the mutation
+-- that needs them, so a crash can neither lose nor half-apply the remainder.
+--
+--   scope = 'self'      re-evaluate `taskId` and everything below it
+--   scope = 'children'  re-evaluate the children of `taskId` whose rowid is
+--                       above `afterRowid`, and everything below them
+--
+-- One row per (taskId, scope): a repeat enqueue restarts that row from the top.
+-- `hops` counts how many batches deep a continuation is, so a parent cycle that
+-- the product never writes cannot re-queue itself forever.
+CREATE TABLE human_free_reclassify_queue (
+  seq INTEGER PRIMARY KEY AUTOINCREMENT,
+  taskId TEXT NOT NULL,
+  scope TEXT NOT NULL CHECK (scope IN ('self', 'children')),
+  afterRowid INTEGER NOT NULL DEFAULT 0,
+  hops INTEGER NOT NULL DEFAULT 0,
+  UNIQUE (taskId, scope)
+);

package/src/be/migrations/184_model_catalog.sql

@@ -0,0 +1,120 @@
+-- Model catalog, claim-time model resolution and harness CLI model support.
+--
+-- One migration for the whole model-catalog stack (persistent catalog, tier
+-- resolution, harness support). Three parts, in dependency order.
+
+-- ---------------------------------------------------------------------------
+-- Part 1: persistent model catalog (models.dev projection) + operator overlay.
+--
+-- model_catalog: one row per (provider, modelId) for the picker-reachable
+--   providers (CATALOG_PROVIDER_IDS). Rewritten by the catalog refresh; system
+--   written, so it carries no user-attribution columns (.non-audit-tables).
+-- model_catalog_overlay: hand-verified facts for models models.dev does not
+--   list yet (or lists wrong). Overlay wins per non-null field. Rows with
+--   expiresWhenUpstreamMatches=1 are deleted once upstream agrees.
+-- model_catalog_meta: single-row fetch state (ETag / Last-Modified / times).
+--
+-- JSON columns: reasoningOptions = [{type, values?}], pricing =
+-- {input?, output?, cache_read?, cache_write?} (USD per million tokens).
+-- ---------------------------------------------------------------------------
+
+CREATE TABLE IF NOT EXISTS model_catalog (
+  provider TEXT NOT NULL,
+  modelId TEXT NOT NULL,
+  name TEXT,
+  family TEXT,
+  releaseDate TEXT,
+  contextWindow INTEGER,
+  maxOutput INTEGER,
+  reasoning INTEGER,
+  reasoningOptions TEXT,
+  pricing TEXT,
+  status TEXT,
+  providerName TEXT,
+  checkedAt INTEGER NOT NULL,
+  PRIMARY KEY (provider, modelId)
+);
+
+CREATE TABLE IF NOT EXISTS model_catalog_overlay (
+  provider TEXT NOT NULL,
+  modelId TEXT NOT NULL,
+  name TEXT,
+  family TEXT,
+  releaseDate TEXT,
+  contextWindow INTEGER,
+  maxOutput INTEGER,
+  reasoning INTEGER,
+  reasoningOptions TEXT,
+  pricing TEXT,
+  status TEXT,
+  reason TEXT NOT NULL,
+  verifiedBy TEXT,
+  expiresWhenUpstreamMatches INTEGER NOT NULL DEFAULT 1,
+  createdAt INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER)),
+  updatedAt INTEGER NOT NULL DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER)),
+  created_by TEXT REFERENCES users(id),
+  updated_by TEXT REFERENCES users(id),
+  PRIMARY KEY (provider, modelId)
+);
+
+CREATE TABLE IF NOT EXISTS model_catalog_meta (
+  id INTEGER PRIMARY KEY CHECK (id = 1),
+  etag TEXT,
+  lastModified TEXT,
+  lastFetchAt INTEGER,
+  lastCheckedAt INTEGER
+);
+
+-- ---------------------------------------------------------------------------
+-- Part 2: claim-time model resolution.
+--
+-- agent_tasks.resolvedModel / modelSource / modelAlias: what the server
+--   resolved when the task was claimed. modelSource is one of
+--   model | worker-env | tier-config | tier-default | fallback:cli-unsupported.
+--   modelAlias is the `latest:` alias, when any.
+-- agents.modelTierOverrides: the worker's parsed MODEL_TIER_* env overrides,
+--   JSON {provider: {tier: value}}. Values only, never secrets. Sent on
+--   register and on every poll.
+-- model_alias_resolutions: one row each time a `latest:` alias resolves to a
+--   different concrete model than its previous resolution. System written.
+-- ---------------------------------------------------------------------------
+
+ALTER TABLE agent_tasks ADD COLUMN resolvedModel TEXT;
+ALTER TABLE agent_tasks ADD COLUMN modelSource TEXT;
+ALTER TABLE agent_tasks ADD COLUMN modelAlias TEXT;
+
+ALTER TABLE agents ADD COLUMN modelTierOverrides TEXT;
+
+CREATE TABLE IF NOT EXISTS model_alias_resolutions (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  alias TEXT NOT NULL,
+  previousModel TEXT,
+  newModel TEXT NOT NULL,
+  changedAt INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_model_alias_resolutions_alias
+  ON model_alias_resolutions (alias, id);
+
+-- ---------------------------------------------------------------------------
+-- Part 3: harness CLI model support.
+--
+-- harness_model_support: whether a catalog model runs on a given harness CLI
+--   version. Workers write it after a model's first run on their CLI: `ok` on
+--   success, `unsupported` when the CLI rejects the model id. No row means
+--   `unknown`, which is allowed at claim. System written.
+-- agents.harnessCliVersion: the worker's `claude --version` / `codex --version`,
+--   reported on register. Keys claim-time support lookups.
+-- ---------------------------------------------------------------------------
+
+CREATE TABLE IF NOT EXISTS harness_model_support (
+  harness TEXT NOT NULL,
+  cliVersion TEXT NOT NULL,
+  modelId TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('ok', 'unsupported', 'unknown')),
+  checkedAt INTEGER NOT NULL,
+  error TEXT,
+  PRIMARY KEY (harness, cliVersion, modelId)
+);
+
+ALTER TABLE agents ADD COLUMN harnessCliVersion TEXT;