@desplega.ai/agent-swarm 1.173.0 → 2.0.0

package/src/be/db.ts

@@ -136,6 +136,12 @@ import {
   CODEX_AUTH_WATCH_NAMESPACE,
   codexAuthBenchMarkerKey,
 } from "../utils/codex-auth-failure";
+import {
+  defaultExportToEnv,
+  effectiveExportToEnv,
+  envExportRestrictedError,
+  isEnvExportRestrictedKey,
+} from "../utils/config-env-export";
 import { MASKED_SECRET_VALUE } from "../utils/config-secrets";
 import { deriveProviderFromKeyType } from "../utils/credentials";
 import type { RateLimitWindowTelemetry } from "../utils/error-tracker";
@@ -205,7 +211,12 @@ import { scrubJsonValue } from "./scrub-json";
 import { openSealedJson, sealedJsonForDisplay, sealJson } from "./sealed-json";
 import { configSecretName, registerStoredSecret } from "./secret-registry";
 import { promotePendingSteeringForTask } from "./steering";
-import { isInternalConfigKey, isReservedConfigKey, reservedKeyError } from "./swarm-config-guard";
+import {
+  isInternalConfigKey,
+  isReservedConfigKey,
+  reservedKeyError,
+  teamsSdkEnvKeysSql,
+} from "./swarm-config-guard";
 import { emitTaskStarted } from "./task-lifecycle-events";
 import { publishWorkerConfigNudge, publishWorkerNudge } from "./worker-nudges";
 import { defineWorkflowPayload, type WorkflowPayloadView } from "./workflow-replay";
@@ -292,17 +303,21 @@ export {
   findTaskByGitHub,
   findTaskByVcs,
   getAgentCurrentTask,
+  getAgentWorkingOnContextKey,
   getAgentWorkingOnThread,
   getAllTasks,
   getChildTasks,
   getCompletedSlackTasks,
   getInProgressSlackTasks,
   getInProgressTasksByContextKey,
+  getLatestActiveTaskByContextKey,
   getLatestActiveTaskInThread,
+  getLatestLeadTaskByContextKey,
   getLatestLeadTaskInThread,
   getLatestScriptRunStepTaskByContextKey,
   getLatestTaskByContextKey,
   getMostRecentTaskInThread,
+  getPendingChatRelayTasks,
   getPendingSlackRelayTasks,
   getRecentlyFinishedWorkerTasks,
   getSlackTasksMissingTree,
@@ -313,13 +328,19 @@ export {
   getTasksByAgentId,
   getTasksByStatus,
   getTasksCount,
+  giveUpStaleChatRelays,
   hasNonTerminalRerouteDecisionChild,
   hasNonTerminalResumeChild,
   isLinearTrackerContextKey,
+  markChatRelayAttempted,
+  markChatRelayDelivered,
+  markChatRelayGivenUp,
   markFinalizedSlackRelaysDelivered,
   markSlackRelayAttempted,
   markSlackRelayDelivered,
   markTasksNotified,
+  type PendingChatRelay,
+  recordChatRelayError,
   resetTasksNotified,
   type TaskFilters,
 } from "./db/tasks/read";
@@ -4375,7 +4396,8 @@ export async function releaseStaleMentionProcessing(timeoutMinutes: number = 30)
 }
 
 // ============================================================================
-// Service Operations (PM2/background services)
+// Service Operations (removed PM2 service registry; the table and these
+// accessors stay until a drop migration lands)
 // ============================================================================
 
 type ServiceRow = {
@@ -6342,6 +6364,7 @@ type SwarmConfigRow = {
   createdAt: string;
   lastUpdatedAt: string;
   encrypted: number; // SQLite boolean: 0 = plaintext, 1 = AES-256-GCM ciphertext
+  exportToEnv?: number; // SQLite boolean; absent only on pre-215 test schemas
 };
 
 type SwarmConfigLookupRow = {
@@ -6370,6 +6393,7 @@ function rowToSwarmConfig(row: SwarmConfigRow): SwarmConfig {
       createdAt: row.createdAt,
       lastUpdatedAt: row.lastUpdatedAt,
       encrypted: isEncrypted,
+      exportToEnv: effectiveExportToEnv(row.key, row.exportToEnv !== 0),
     };
   }
 
@@ -6396,6 +6420,7 @@ function rowToSwarmConfig(row: SwarmConfigRow): SwarmConfig {
     createdAt: row.createdAt,
     lastUpdatedAt: row.lastUpdatedAt,
     encrypted: isEncrypted,
+    exportToEnv: effectiveExportToEnv(row.key, row.exportToEnv !== 0),
   };
 }
 
@@ -6550,7 +6575,7 @@ export async function getInjectableGlobalConfigs(): Promise<SwarmConfig[]> {
   const rows = await getDbClient().query<SwarmConfigRow>(
     `SELECT * FROM swarm_config
        WHERE scope = 'global'
-         AND UPPER(key) NOT IN ('API_KEY', 'SECRETS_ENCRYPTION_KEY', 'CORS_ALLOW_ANY_ORIGIN', 'EXTENSION_ALLOW_LEAD_ACTIVATION')
+         AND UPPER(key) NOT IN ('API_KEY', 'SECRETS_ENCRYPTION_KEY', 'CORS_ALLOW_ANY_ORIGIN', 'EXTENSION_ALLOW_LEAD_ACTIVATION', 'CHAT_SDK_API_URL_OVERRIDES', ${teamsSdkEnvKeysSql()})
        ORDER BY key ASC`,
   );
   return rows.filter((row) => !isInternalConfigKey(row.key)).map(rowToSwarmConfig);
@@ -6604,10 +6629,16 @@ export async function upsertSwarmConfig(data: {
   isSecret?: boolean;
   envPath?: string | null;
   description?: string | null;
+  /** Omitted: an update keeps the stored flag; an insert uses `defaultExportToEnv`. */
+  exportToEnv?: boolean;
 }): Promise<SwarmConfig> {
   if (isReservedConfigKey(data.key)) {
     throw reservedKeyError(data.key);
   }
+  if (data.exportToEnv === true && isEnvExportRestrictedKey(data.key)) {
+    throw new Error(envExportRestrictedError(data.key));
+  }
+  const restricted = isEnvExportRestrictedKey(data.key);
 
   const now = new Date().toISOString();
   const scopeId = data.scope === "global" ? null : (data.scopeId ?? null);
@@ -6641,17 +6672,30 @@ export async function upsertSwarmConfig(data: {
           );
 
     if (existing) {
+      // COALESCE keeps the stored flag when the caller did not send one.
       return await getDbClient().get<SwarmConfigRow>(
-        `UPDATE swarm_config SET value = ?, isSecret = ?, envPath = ?, description = ?, encrypted = ?, lastUpdatedAt = ?
+        `UPDATE swarm_config SET value = ?, isSecret = ?, envPath = ?, description = ?, encrypted = ?, lastUpdatedAt = ?,
+           exportToEnv = CASE WHEN ? THEN 0 ELSE COALESCE(?, exportToEnv) END
          WHERE id = ? RETURNING *`,
-        [storedValue, isSecret, envPath, description, encryptedFlag, now, existing.id],
+        [
+          storedValue,
+          isSecret,
+          envPath,
+          description,
+          encryptedFlag,
+          now,
+          restricted ? 1 : 0,
+          data.exportToEnv === undefined ? null : data.exportToEnv ? 1 : 0,
+          existing.id,
+        ],
       );
     }
 
     const id = crypto.randomUUID();
+    const exportToEnv = restricted ? 0 : (data.exportToEnv ?? defaultExportToEnv(data.key)) ? 1 : 0;
     return await getDbClient().get<SwarmConfigRow>(
-      `INSERT INTO swarm_config (id, scope, scopeId, key, value, isSecret, envPath, description, createdAt, lastUpdatedAt, encrypted)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
+      `INSERT INTO swarm_config (id, scope, scopeId, key, value, isSecret, envPath, description, createdAt, lastUpdatedAt, encrypted, exportToEnv)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING *`,
       [
         id,
         data.scope,
@@ -6664,6 +6708,7 @@ export async function upsertSwarmConfig(data: {
         now,
         now,
         encryptedFlag,
+        exportToEnv,
       ],
     );
   });

package/src/be/migrations/215_swarm_config_export_to_env.sql

@@ -0,0 +1,34 @@
+-- Per-row flag: does this swarm_config row reach an agent's process env?
+-- Workers export a row into the harness env (and so into every agent shell)
+-- only when exportToEnv = 1. Rows with 0 stay server-side and are used through
+-- connections or script credential bindings.
+--
+-- Existing rows keep today's behaviour (exported), so nothing breaks at deploy.
+ALTER TABLE swarm_config ADD COLUMN exportToEnv INTEGER NOT NULL DEFAULT 1;
+
+-- Deny list: rows only the API server reads. The first block matches
+-- ENV_EXPORT_RESTRICTED_KEYS in src/utils/config-env-export.ts (never
+-- exportable); the second block (ENV_EXPORT_DEFAULT_OFF_KEYS, also the insert
+-- default for new rows) is off by default but operators may turn it
+-- back on per row.
+UPDATE swarm_config SET exportToEnv = 0
+WHERE upper(key) IN (
+    'SLACK_APP_TOKEN',
+    'SLACK_SIGNING_SECRET',
+    'GITHUB_APP_PRIVATE_KEY',
+    'GITHUB_WEBHOOK_SECRET',
+    'GITLAB_WEBHOOK_SECRET',
+    'LINEAR_CLIENT_SECRET',
+    'LINEAR_SIGNING_SECRET',
+    'JIRA_CLIENT_SECRET',
+    'JIRA_WEBHOOK_TOKEN',
+    'AZURE_DEVOPS_WEBHOOK_SECRET',
+    'AGENTMAIL_WEBHOOK_SECRET',
+    'API_AGENT_FS_API_KEY',
+    'KAPSO_WEBHOOK_HMAC_SECRET',
+    'SENTRY_TEST_APP_WEBHOOK_SECRET',
+    'TELEMETRY_CH_AUTH',
+    'HACK_AGENT_FS_ADMIN_KEY',
+    'X_CLIENT_SECRET'
+  )
+  OR key GLOB 'connection.*.secret';

package/src/be/migrations/216_task_resolved_effort.sql

@@ -0,0 +1,5 @@
+-- The reasoning effort a task's session actually ran at. `effort` stays the
+-- requested value (a task-level pin); the effective one usually comes from the
+-- agent's REASONING_EFFORT_OVERRIDE and was never recorded. NULL means the
+-- harness default applied, or the row predates this column.
+ALTER TABLE agent_tasks ADD COLUMN resolvedEffort TEXT;

package/src/be/migrations/217_chat_sdk_state.sql

@@ -0,0 +1,104 @@
+-- Chat SDK (chat-sdk.dev) channel: StateAdapter tables + durable outbound relay.
+--
+-- The five chat_* state tables back `SqliteChatState` (src/integrations/chat-sdk/state.ts).
+-- Keys and thread ids written by the SDK already carry the adapter prefix
+-- (`telegram:...`, `slack:...`), so no adapter column is needed.
+-- Expiry columns are epoch milliseconds; NULL means no expiry.
+
+CREATE TABLE chat_state_kv (
+  key        TEXT PRIMARY KEY,
+  value      TEXT NOT NULL,
+  expires_at INTEGER
+);
+
+CREATE TABLE chat_subscriptions (
+  thread_id  TEXT PRIMARY KEY,
+  created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+);
+
+CREATE TABLE chat_locks (
+  thread_id  TEXT PRIMARY KEY,
+  token      TEXT NOT NULL,
+  expires_at INTEGER NOT NULL
+);
+
+CREATE TABLE chat_lists (
+  key        TEXT NOT NULL,
+  seq        INTEGER NOT NULL,
+  value      TEXT NOT NULL,
+  expires_at INTEGER,
+  PRIMARY KEY (key, seq)
+);
+
+CREATE TABLE chat_queue (
+  id        INTEGER PRIMARY KEY AUTOINCREMENT,
+  thread_id TEXT NOT NULL,
+  entry     TEXT NOT NULL
+);
+
+CREATE INDEX idx_chat_queue_thread ON chat_queue(thread_id, id);
+
+-- Persist the obligation to publish a terminal result for every Chat SDK-rooted
+-- task. Mirrors 144_slack_relay_obligations.sql, plus an attempt cap: a poison
+-- row is marked given up (with the reason) instead of retrying forever.
+CREATE TABLE chat_relay_obligations (
+  task_id         TEXT PRIMARY KEY REFERENCES agent_tasks(id) ON DELETE CASCADE,
+  delivered_at    TEXT,
+  attempt_count   INTEGER NOT NULL DEFAULT 0,
+  last_attempt_at TEXT,
+  given_up_at     TEXT,
+  last_error      TEXT,
+  created_at      TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  updated_at      TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  created_by      TEXT,
+  updated_by      TEXT
+);
+
+CREATE INDEX idx_chat_relay_obligations_pending
+  ON chat_relay_obligations(last_attempt_at, created_at)
+  WHERE delivered_at IS NULL AND given_up_at IS NULL;
+
+-- Lead follow-ups (`source='system'`, `taskType='follow-up'`) inherit the chat
+-- contextKey and carry the lead's final answer, so they relay too. A resume
+-- (`source='system'`, `taskType='resume'`, from `createResumeFollowUp` after a
+-- graceful shutdown or crash recovery) inherits the contextKey too. It relays
+-- only when a lead owns it: a worker task (resumed or not) reports back through
+-- a lead follow-up, and the lead never gets one for its own tasks. Other system
+-- tasks that inherit the key stay silent. GLOB, not LIKE: LIKE is
+-- case-insensitive in SQLite.
+CREATE TRIGGER enqueue_chat_relay_after_terminal_update
+AFTER UPDATE OF status ON agent_tasks
+WHEN OLD.status NOT IN ('completed', 'failed', 'cancelled', 'superseded')
+  AND NEW.status IN ('completed', 'failed', 'cancelled')
+  AND NEW.contextKey GLOB 'task:chat-sdk:*'
+  AND (
+    NEW.source = 'chat'
+    OR (NEW.source = 'system' AND NEW.taskType = 'follow-up')
+    OR (
+      NEW.source = 'system'
+      AND NEW.taskType = 'resume'
+      AND EXISTS (SELECT 1 FROM agents WHERE agents.id = NEW.agentId AND agents.isLead = 1)
+    )
+  )
+BEGIN
+  INSERT OR IGNORE INTO chat_relay_obligations (task_id)
+  VALUES (NEW.id);
+END;
+
+CREATE TRIGGER enqueue_chat_relay_after_terminal_insert
+AFTER INSERT ON agent_tasks
+WHEN NEW.status IN ('completed', 'failed', 'cancelled')
+  AND NEW.contextKey GLOB 'task:chat-sdk:*'
+  AND (
+    NEW.source = 'chat'
+    OR (NEW.source = 'system' AND NEW.taskType = 'follow-up')
+    OR (
+      NEW.source = 'system'
+      AND NEW.taskType = 'resume'
+      AND EXISTS (SELECT 1 FROM agents WHERE agents.id = NEW.agentId AND agents.isLead = 1)
+    )
+  )
+BEGIN
+  INSERT OR IGNORE INTO chat_relay_obligations (task_id)
+  VALUES (NEW.id);
+END;

package/src/be/model-catalog-store.ts

@@ -13,6 +13,7 @@
  * See runbooks/model-catalog.md.
  */
 
+import { withAgentChatRegistrySections } from "@desplega/model-catalog";
 import { type RuntimeCatalog, setRuntimeModelCatalog } from "../utils/runtime-model-catalog";
 import {
   getActivePricingRow,
@@ -180,7 +181,10 @@ function pickNumber(value: unknown): number | undefined {
 }
 
 /** Project a models.dev payload into `model_catalog` rows for CATALOG_PROVIDER_IDS. */
-export function buildCatalogEntries(cache: ModelsDevCache, checkedAt: number): ModelCatalogEntry[] {
+export function buildCatalogEntries(raw: ModelsDevCache, checkedAt: number): ModelCatalogEntry[] {
+  // A live models.dev payload still carries the embeddings and image models of
+  // the registry gateways (`vercel`); pi and opencode cannot run them.
+  const cache = withAgentChatRegistrySections(raw);
   const entries: ModelCatalogEntry[] = [];
   for (const providerId of CATALOG_PROVIDER_IDS) {
     const provider = cache[providerId];

package/src/be/model-tier-resolution.ts

@@ -509,6 +509,8 @@ export type ClaimModelFields = Partial<TaskModelResolution> & {
 /**
  * Resolve and record the model for a task the given agent is claiming. Runs
  * inside the claim transaction. Returns the fields to merge into the trigger.
+ * `task.model` must be the task's own pin (`taskModelPin`), never the model a
+ * previous session reported into the column.
  */
 export async function recordClaimModelResolution(
   task: { id: string; model?: string | null; modelTier?: string | null },
@@ -521,7 +523,16 @@ export async function recordClaimModelResolution(
     harnessProvider,
     workerOverrides: await getAgentModelTierOverrides(agent.id),
   });
-  if (!resolved) return {};
+  if (!resolved) {
+    // A re-claimed row keeps the claim record of its previous run; drop it so
+    // the trigger and the row do not replay that run's model.
+    await getDbClient().run(
+      `UPDATE agent_tasks SET resolvedModel = NULL, modelSource = NULL, modelAlias = NULL
+         WHERE id = ? AND (resolvedModel IS NOT NULL OR modelSource IS NOT NULL OR modelAlias IS NOT NULL)`,
+      [task.id],
+    );
+    return {};
+  }
   if ("harnessMismatch" in resolved) return { modelUnsupported: resolved.harnessMismatch };
   const checked = await applyCliSupport(
     resolved,

package/src/be/model-validation.ts

@@ -29,6 +29,7 @@ import {
   type ModelsDevCatalog,
   parseAlias,
 } from "@desplega/model-catalog";
+import { type TaskModelPinFields, taskModelPin } from "../tasks/model-pin";
 import { ampModelError, ampPinCatalogError, ampPinProviderError } from "../utils/amp-models";
 import { getAgentById, getAllAgents } from "./db";
 import { loadModelsCatalog } from "./model-catalog-store";
@@ -99,10 +100,12 @@ export async function harnessModelErrorFor(
  * stays sync and catalog-free; this runs next to it.
  */
 export async function poolTaskRunsOnHarness(
-  task: { model?: string | null } | null | undefined,
+  task: TaskModelPinFields | null | undefined,
   harness: string | null | undefined,
 ): Promise<boolean> {
-  const model = task?.model?.trim();
+  // Only the task's own pin restricts harnesses; a model a previous run
+  // reported into `model` does not.
+  const model = taskModelPin(task ?? {})?.trim();
   if (!model) return true;
   return (await harnessModelErrorFor(model, harness)) === null;
 }

package/src/be/models-catalog.ts

@@ -1,3 +1,4 @@
+import { openAICompatibleProviderIds, registryCatalogSections } from "@desplega/model-catalog";
 import { loadModelsDevCache, type ModelsDevCache, type ModelsDevModel } from "./modelsdev-cache";
 
 /**
@@ -15,13 +16,18 @@ import { loadModelsDevCache, type ModelsDevCache, type ModelsDevModel } from "./
  * same snapshot as its own fallback while the request is in flight.
  */
 
-/** Mirrors the provider groups used by the direct harness and ACP model pickers. */
+/**
+ * Mirrors the provider groups used by the direct harness and ACP model
+ * pickers, plus every OpenAI-compatible registry provider
+ * (`@desplega/model-catalog`).
+ */
 export const CATALOG_PROVIDER_IDS = [
   "openrouter",
   "anthropic",
   "openai",
   "amazon-bedrock",
   "opencode",
+  ...openAICompatibleProviderIds(),
 ] as const;
 
 export type CatalogProviderId = (typeof CATALOG_PROVIDER_IDS)[number];
@@ -163,6 +169,24 @@ function mergePinnedEntries(catalog: ModelsCatalog): void {
   }
 }
 
+/**
+ * Registry-synthesized sections (`catalog: "registry"` entries such as
+ * `helmcode`) are never on models.dev, so the live catalog built from it
+ * lacks them. Fill them from the bundled snapshot, which carries them
+ * (`loadModelsDevCache`); models already present (an overlay row) win.
+ */
+function mergeRegistrySections(catalog: ModelsCatalog): void {
+  const snapshot = bundledSnapshotCatalog();
+  for (const providerId of Object.keys(registryCatalogSections()) as CatalogProviderId[]) {
+    const section = snapshot[providerId];
+    if (!section) continue;
+    catalog[providerId] ??= { ...section, models: {} };
+    for (const [modelId, model] of Object.entries(section.models)) {
+      catalog[providerId].models[modelId] ??= model;
+    }
+  }
+}
+
 /**
  * Install a freshly built live catalog (already merged with the persistent
  * table + overlay by `src/be/model-catalog-store.ts`). Pinned snapshot entries
@@ -170,6 +194,7 @@ function mergePinnedEntries(catalog: ModelsCatalog): void {
  */
 export function setLiveModelsCatalog(catalog: ModelsCatalog, updatedAt: number | null): void {
   mergePinnedEntries(catalog);
+  mergeRegistrySections(catalog);
   liveCatalog = catalog;
   liveUpdatedAt = updatedAt;
 }