nodal-agents 0.4.0 → 0.4.2

package/runner.js

@@ -1102,6 +1102,143 @@ var init_connector_catalog = __esm({
   }
 });
 
+// ../../packages/shared/src/model-catalog.ts
+function findModelCatalogEntry(provider, modelId) {
+  return MODEL_CATALOG[provider]?.find((e) => e.modelId === modelId);
+}
+var MODEL_CATALOG;
+var init_model_catalog = __esm({
+  "../../packages/shared/src/model-catalog.ts"() {
+    "use strict";
+    MODEL_CATALOG = {
+      anthropic: [
+        {
+          modelId: "claude-opus-4-8",
+          label: "Claude Opus 4.8",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "claude-sonnet-4-6",
+          label: "Claude Sonnet 4.6",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "claude-haiku-4-5-20251001",
+          label: "Claude Haiku 4.5",
+          capabilities: { tools: true, forcedToolChoice: true }
+        }
+      ],
+      openai: [
+        { modelId: "gpt-5", label: "GPT-5", capabilities: { tools: true, forcedToolChoice: true } },
+        {
+          modelId: "gpt-5-mini",
+          label: "GPT-5 mini",
+          capabilities: { tools: true, forcedToolChoice: true }
+        }
+      ],
+      google: [
+        {
+          modelId: "gemini-2.0-flash",
+          label: "Gemini 2.0 Flash",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "gemini-2.5-pro",
+          label: "Gemini 2.5 Pro",
+          capabilities: { tools: true, forcedToolChoice: true }
+        }
+      ],
+      groq: [
+        {
+          modelId: "llama-3.3-70b-versatile",
+          label: "Llama 3.3 70B",
+          capabilities: { tools: true, forcedToolChoice: true }
+        }
+      ],
+      mistral: [
+        {
+          modelId: "mistral-large-latest",
+          label: "Mistral Large",
+          capabilities: { tools: true, forcedToolChoice: true }
+        }
+      ],
+      // OpenRouter models are namespaced by sub-vendor (anthropic/, deepseek/, …).
+      // The UI groups them by that vendor (see modelGroupLabel). Tested + working
+      // routes; all accept a forced tool_choice.
+      openrouter: [
+        // Anthropic
+        {
+          modelId: "anthropic/claude-haiku-4.5",
+          label: "Claude Haiku 4.5",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "anthropic/claude-opus-4.7",
+          label: "Claude Opus 4.7",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "anthropic/claude-opus-4.7-fast",
+          label: "Claude Opus 4.7 (fast)",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "anthropic/claude-opus-4.8",
+          label: "Claude Opus 4.8",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "anthropic/claude-opus-4.8-fast",
+          label: "Claude Opus 4.8 (fast)",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "anthropic/claude-sonnet-4.6",
+          label: "Claude Sonnet 4.6",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        // DeepSeek
+        {
+          modelId: "deepseek/deepseek-v3.2",
+          label: "DeepSeek V3.2",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "deepseek/deepseek-v4-flash",
+          label: "DeepSeek V4 Flash",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "deepseek/deepseek-v4-pro",
+          label: "DeepSeek V4 Pro",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        // Google
+        {
+          modelId: "google/gemini-3.1-flash-lite-preview",
+          label: "Gemini 3.1 Flash Lite (preview)",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "google/gemini-3.1-pro-preview",
+          label: "Gemini 3.1 Pro (preview)",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "google/gemini-3.5-flash",
+          label: "Gemini 3.5 Flash",
+          capabilities: { tools: true, forcedToolChoice: true }
+        },
+        {
+          modelId: "google/gemma-4-31b-it",
+          label: "Gemma 4 31B-IT",
+          capabilities: { tools: true, forcedToolChoice: true }
+        }
+      ]
+    };
+  }
+});
+
 // ../../packages/shared/src/index.ts
 var init_src = __esm({
   "../../packages/shared/src/index.ts"() {
@@ -1127,6 +1264,7 @@ var init_src = __esm({
     init_providers();
     init_root_agent();
     init_connector_catalog();
+    init_model_catalog();
   }
 });
 
@@ -1262,7 +1400,6 @@ var init_llm_keys = __esm({
         apiKeyLast4: text3("api_key_last4").notNull().default(""),
         baseUrl: text3("base_url"),
         nickname: text3("nickname"),
-        defaultModel: text3("default_model"),
         isActive: boolean2("is_active").notNull().default(true),
         createdAt: timestamp3("created_at", { withTimezone: true }).notNull().defaultNow(),
         updatedAt: timestamp3("updated_at", { withTimezone: true }).notNull().defaultNow()
@@ -1281,6 +1418,7 @@ import {
   integer,
   bigint,
   timestamp as timestamp4,
+  jsonb as jsonb2,
   index as index3,
   check as check2
 } from "drizzle-orm/pg-core";
@@ -1301,6 +1439,14 @@ var init_agents = __esm({
         personality: text4("personality").notNull(),
         model: text4("model").default("claude-sonnet-4-6-20260217"),
         llmKeyId: uuid4("llm_key_id").references(() => entityLlmKeys.id, { onDelete: "set null" }),
+        // Ordered LLM-key fallback chain (Guard 2). When the primary key
+        // (llmKeyId) exhausts retries / times out / hits quota mid-job, the runner
+        // fails over to these in order; all-down fails loud (`all_providers_failed`).
+        // Empty = no failover (default). Each link is a (keyId, model) pair so a
+        // fallback runs on a CHOSEN model (empty model ⇒ that provider's catalog
+        // default). FK integrity is enforced in the app layer; a deleted key is
+        // skipped at resolution time.
+        fallbackChain: jsonb2("fallback_chain").$type().default(sql2`'[]'::jsonb`),
         active: boolean3("active").default(true),
         isDefault: boolean3("is_default").default(false),
         role: text4("role").default("agent"),
@@ -1380,7 +1526,7 @@ var init_agents = __esm({
 });
 
 // ../../packages/db/src/schema/jobs.ts
-import { pgTable as pgTable5, text as text5, uuid as uuid5, integer as integer2, jsonb as jsonb2, timestamp as timestamp5, index as index4, check as check3 } from "drizzle-orm/pg-core";
+import { pgTable as pgTable5, text as text5, uuid as uuid5, integer as integer2, jsonb as jsonb3, timestamp as timestamp5, index as index4, check as check3 } from "drizzle-orm/pg-core";
 import { sql as sql3 } from "drizzle-orm";
 var agentJobs;
 var init_jobs = __esm({
@@ -1400,7 +1546,7 @@ var init_jobs = __esm({
         originalTask: text5("original_task"),
         chatId: text5("chat_id"),
         systemPrompt: text5("system_prompt"),
-        messages: jsonb2("messages").default(sql3`'[]'::jsonb`),
+        messages: jsonb3("messages").default(sql3`'[]'::jsonb`),
         toolsUsed: text5("tools_used").array().default(sql3`'{}'::text[]`),
         turn: integer2("turn").default(0),
         result: text5("result"),
@@ -1428,7 +1574,7 @@ var init_jobs = __esm({
          * different specialist (live regression: job `7767a3c1`, 2026-05-19).
          */
         lastFailedDelegationSlug: text5("last_failed_delegation_slug"),
-        pendingDelegation: jsonb2("pending_delegation"),
+        pendingDelegation: jsonb3("pending_delegation"),
         completedAt: timestamp5("completed_at", { withTimezone: true }),
         createdAt: timestamp5("created_at", { withTimezone: true }).defaultNow(),
         updatedAt: timestamp5("updated_at", { withTimezone: true }).defaultNow()
@@ -1463,7 +1609,7 @@ import {
   text as text6,
   uuid as uuid6,
   integer as integer3,
-  jsonb as jsonb3,
+  jsonb as jsonb4,
   timestamp as timestamp6,
   numeric,
   index as index5,
@@ -1499,7 +1645,7 @@ var init_tasks = __esm({
         outputTokens: integer3("output_tokens").default(0),
         costUsd: numeric("cost_usd", { precision: 10, scale: 6 }).default("0"),
         dependsOn: uuid6("depends_on").array().default(sql4`'{}'::uuid[]`),
-        context: jsonb3("context").default(sql4`'{}'::jsonb`),
+        context: jsonb4("context").default(sql4`'{}'::jsonb`),
         rootJobId: uuid6("root_job_id"),
         lockedAt: timestamp6("locked_at", { withTimezone: true }),
         lockedBy: text6("locked_by"),
@@ -1607,7 +1753,7 @@ var init_connectors = __esm({
 });
 
 // ../../packages/db/src/schema/tool_calls.ts
-import { pgTable as pgTable9, text as text9, uuid as uuid9, integer as integer4, jsonb as jsonb4, timestamp as timestamp9, index as index8 } from "drizzle-orm/pg-core";
+import { pgTable as pgTable9, text as text9, uuid as uuid9, integer as integer4, jsonb as jsonb5, timestamp as timestamp9, index as index8 } from "drizzle-orm/pg-core";
 import { sql as sql7 } from "drizzle-orm";
 var toolCalls;
 var init_tool_calls = __esm({
@@ -1622,7 +1768,7 @@ var init_tool_calls = __esm({
         entityId: uuid9("entity_id").references(() => entities.id, { onDelete: "cascade" }),
         jobId: uuid9("job_id").references(() => agentJobs.id, { onDelete: "cascade" }),
         toolName: text9("tool_name").notNull(),
-        toolInput: jsonb4("tool_input"),
+        toolInput: jsonb5("tool_input"),
         toolOutput: text9("tool_output"),
         durationMs: integer4("duration_ms"),
         turn: integer4("turn"),
@@ -1639,7 +1785,7 @@ var init_tool_calls = __esm({
 });
 
 // ../../packages/db/src/schema/approvals.ts
-import { pgTable as pgTable10, text as text10, uuid as uuid10, jsonb as jsonb5, timestamp as timestamp10, index as index9, check as check7 } from "drizzle-orm/pg-core";
+import { pgTable as pgTable10, text as text10, uuid as uuid10, jsonb as jsonb6, timestamp as timestamp10, index as index9, check as check7 } from "drizzle-orm/pg-core";
 import { sql as sql8 } from "drizzle-orm";
 var approvalRequests, approvalRules;
 var init_approvals = __esm({
@@ -1656,7 +1802,7 @@ var init_approvals = __esm({
         jobId: uuid10("job_id").notNull().references(() => agentJobs.id, { onDelete: "cascade" }),
         agentId: uuid10("agent_id").references(() => agents.id, { onDelete: "cascade" }),
         toolName: text10("tool_name").notNull(),
-        toolInput: jsonb5("tool_input").notNull(),
+        toolInput: jsonb6("tool_input").notNull(),
         status: text10("status").default("pending"),
         requestedAt: timestamp10("requested_at", { withTimezone: true }).defaultNow(),
         resolvedAt: timestamp10("resolved_at", { withTimezone: true }),
@@ -1691,7 +1837,7 @@ var init_approvals = __esm({
         agentId: uuid10("agent_id").references(() => agents.id, { onDelete: "cascade" }),
         toolName: text10("tool_name").notNull(),
         action: text10("action").notNull(),
-        conditionJson: jsonb5("condition_json").default(sql8`'{}'::jsonb`),
+        conditionJson: jsonb6("condition_json").default(sql8`'{}'::jsonb`),
         createdAt: timestamp10("created_at", { withTimezone: true }).defaultNow(),
         updatedAt: timestamp10("updated_at", { withTimezone: true }).defaultNow()
       },
@@ -1819,7 +1965,7 @@ import {
   uuid as uuid13,
   boolean as boolean7,
   integer as integer7,
-  jsonb as jsonb6,
+  jsonb as jsonb7,
   timestamp as timestamp13,
   index as index12
 } from "drizzle-orm/pg-core";
@@ -1843,8 +1989,8 @@ var init_skills = __esm({
         description: text13("description"),
         defaultContent: text13("default_content"),
         contentOverridden: boolean7("content_overridden").default(false),
-        requiredConfig: jsonb6("required_config").default(sql10`'[]'::jsonb`),
-        operations: jsonb6("operations").default(sql10`'[]'::jsonb`),
+        requiredConfig: jsonb7("required_config").default(sql10`'[]'::jsonb`),
+        operations: jsonb7("operations").default(sql10`'[]'::jsonb`),
         requiredBuiltins: text13("required_builtins").array().notNull().default(sql10`'{}'::text[]`),
         createdAt: timestamp13("created_at", { withTimezone: true }).defaultNow(),
         updatedAt: timestamp13("updated_at", { withTimezone: true }).defaultNow()
@@ -1889,7 +2035,7 @@ var init_skills = __esm({
         entityId: uuid13("entity_id").notNull().references(() => entities.id, { onDelete: "cascade" }),
         agentId: uuid13("agent_id").notNull().references(() => agents.id, { onDelete: "cascade" }),
         skillId: uuid13("skill_id").notNull().references(() => agentSkills.id, { onDelete: "cascade" }),
-        approvalOverrides: jsonb6("approval_overrides").default(sql10`'{}'::jsonb`),
+        approvalOverrides: jsonb7("approval_overrides").default(sql10`'{}'::jsonb`),
         useCustomInstructions: boolean7("use_custom_instructions").notNull().default(false),
         enabledOperations: text13("enabled_operations").array(),
         createdAt: timestamp13("created_at", { withTimezone: true }).notNull().defaultNow()
@@ -1999,7 +2145,7 @@ import {
   text as text16,
   uuid as uuid16,
   boolean as boolean10,
-  jsonb as jsonb7,
+  jsonb as jsonb8,
   timestamp as timestamp16,
   index as index15,
   uniqueIndex as uniqueIndex2,
@@ -2023,7 +2169,7 @@ var init_mcp = __esm({
         url: text16("url"),
         command: text16("command"),
         args: text16("args").array().default(sql13`'{}'::text[]`),
-        envVars: jsonb7("env_vars").default(sql13`'{}'::jsonb`),
+        envVars: jsonb8("env_vars").default(sql13`'{}'::jsonb`),
         // Encrypted (enc:v1: blob) credential for HTTP MCP servers — same pattern
         // as connectors.api_key. NULL for servers that need no auth.
         apiKey: text16("api_key"),
@@ -2036,7 +2182,7 @@ var init_mcp = __esm({
         // The literal header name or query param name (e.g. 'x-api-key', 'api_key').
         authParamName: text16("auth_param_name"),
         active: boolean10("active").default(true),
-        availableTools: jsonb7("available_tools"),
+        availableTools: jsonb8("available_tools"),
         createdAt: timestamp16("created_at", { withTimezone: true }).defaultNow(),
         updatedAt: timestamp16("updated_at", { withTimezone: true }).defaultNow()
       },
@@ -2058,7 +2204,7 @@ var init_mcp = __esm({
         entityId: uuid16("entity_id").notNull().references(() => entities.id, { onDelete: "cascade" }),
         agentId: uuid16("agent_id").notNull().references(() => agents.id, { onDelete: "cascade" }),
         mcpServerId: uuid16("mcp_server_id").notNull().references(() => mcpServers.id, { onDelete: "cascade" }),
-        enabledTools: jsonb7("enabled_tools"),
+        enabledTools: jsonb8("enabled_tools"),
         createdAt: timestamp16("created_at", { withTimezone: true }).notNull().defaultNow(),
         updatedAt: timestamp16("updated_at", { withTimezone: true }).notNull().defaultNow()
       },
@@ -2077,7 +2223,7 @@ var init_mcp = __esm({
         entityId: uuid16("entity_id").notNull().references(() => entities.id, { onDelete: "cascade" }),
         slug: text16("slug").notNull(),
         active: boolean10("active").notNull().default(true),
-        toolConfig: jsonb7("tool_config").notNull().default(sql13`'{}'::jsonb`),
+        toolConfig: jsonb8("tool_config").notNull().default(sql13`'{}'::jsonb`),
         createdAt: timestamp16("created_at", { withTimezone: true }).notNull().defaultNow(),
         updatedAt: timestamp16("updated_at", { withTimezone: true }).notNull().defaultNow()
       },
@@ -2093,7 +2239,7 @@ import {
   uuid as uuid17,
   boolean as boolean11,
   integer as integer9,
-  jsonb as jsonb8,
+  jsonb as jsonb9,
   timestamp as timestamp17,
   index as index16,
   check as check12
@@ -2111,7 +2257,7 @@ var init_misc = __esm({
         id: uuid17("id").primaryKey().defaultRandom(),
         entityId: uuid17("entity_id").notNull().references(() => entities.id, { onDelete: "cascade" }),
         agentId: uuid17("agent_id").references(() => agents.id, { onDelete: "cascade" }),
-        messages: jsonb8("messages").notNull().default(sql14`'[]'::jsonb`),
+        messages: jsonb9("messages").notNull().default(sql14`'[]'::jsonb`),
         status: text17("status").notNull().default("active"),
         turnCount: integer9("turn_count").notNull().default(0),
         createdAt: timestamp17("created_at", { withTimezone: true }).notNull().defaultNow(),
@@ -2131,7 +2277,7 @@ var init_misc = __esm({
         slug: text17("slug").notNull(),
         description: text17("description"),
         pluginType: text17("plugin_type").notNull(),
-        config: jsonb8("config").default(sql14`'{}'::jsonb`),
+        config: jsonb9("config").default(sql14`'{}'::jsonb`),
         active: boolean11("active").default(true),
         hook: text17("hook").notNull(),
         webhookUrl: text17("webhook_url"),
@@ -10307,6 +10453,21 @@ Caused by: ${underlyingCause.stack}`;
     }
   }
 };
+var AllProvidersFailedError = class extends Error {
+  code = "all_providers_failed";
+  underlyingCause;
+  constructor(providerCount, underlyingCause) {
+    super(
+      `All ${providerCount} LLM providers failed; last: ${formatCauseSummary(underlyingCause)}`
+    );
+    this.name = "AllProvidersFailedError";
+    this.underlyingCause = underlyingCause;
+    if (underlyingCause instanceof Error) {
+      this.stack = `${this.stack}
+Caused by: ${underlyingCause.stack}`;
+    }
+  }
+};
 function formatCauseSummary(cause) {
   if (cause instanceof Error) {
     const name = cause.name || "Error";
@@ -10587,6 +10748,48 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+// ../../packages/llm/src/tool-choice-floor.ts
+function isUnsupportedToolChoiceError(err) {
+  let cur = err;
+  for (let depth = 0; depth < 5 && cur; depth++) {
+    if (!(cur instanceof Error)) return false;
+    const parts = [cur.message ?? ""];
+    const body = cur.responseBody;
+    if (typeof body === "string") parts.push(body);
+    const data = cur.data;
+    if (data !== void 0 && data !== null) {
+      try {
+        parts.push(JSON.stringify(data));
+      } catch {
+      }
+    }
+    const text22 = parts.join(" ").toLowerCase();
+    if (text22.includes("tool_choice") && (text22.includes("no endpoints") || text22.includes("not supported") || text22.includes("does not support") || text22.includes("unsupported") || text22.includes("invalid value"))) {
+      return true;
+    }
+    const inner = cur.cause;
+    if (inner === cur) return false;
+    cur = inner;
+  }
+  return false;
+}
+async function generateWithToolChoiceFloor(run, toolChoice, label) {
+  try {
+    return await run();
+  } catch (err) {
+    const wasForced = toolChoice !== void 0 && toolChoice !== "auto";
+    if (wasForced && isUnsupportedToolChoiceError(err)) {
+      console.warn(
+        `[tool_choice_relaxed] ${label}: provider rejected tool_choice=${JSON.stringify(
+          toolChoice
+        )} \u2014 retrying with 'auto'`
+      );
+      return run("auto");
+    }
+    throw err;
+  }
+}
+
 // ../../packages/llm/src/providers/anthropic.ts
 import { createAnthropic } from "@ai-sdk/anthropic";
 function buildAnthropicModel(config) {
@@ -11068,21 +11271,27 @@ function createLlmClient(config) {
   };
   const clientGenerateText = async (args) => {
     validateIfMessages(args);
-    return withRetry(
-      () => callWithTimeout(
-        () => generateText({
-          ...args,
-          model,
-          // AI SDK native timeout via AbortSignal.timeout(). Survives middleware
-          // wrapping unlike a passed-in abortSignal which their internal retry
-          // can swallow.
-          timeout: LLM_TIMEOUT_MS,
-          // Disable AI SDK internal retry — we own retries via withRetry to
-          // preserve typed error handling (Quota/MessageStructure/LLMTimeout).
-          maxRetries: 0
-        })
+    const toolChoice = args.toolChoice;
+    return generateWithToolChoiceFloor(
+      (override) => withRetry(
+        () => callWithTimeout(
+          () => generateText({
+            ...args,
+            model,
+            ...override ? { toolChoice: override } : {},
+            // AI SDK native timeout via AbortSignal.timeout(). Survives
+            // middleware wrapping unlike a passed-in abortSignal which their
+            // internal retry can swallow.
+            timeout: LLM_TIMEOUT_MS,
+            // Disable AI SDK internal retry — we own retries via withRetry to
+            // preserve typed error handling (Quota/MessageStructure/LLMTimeout).
+            maxRetries: 0
+          })
+        ),
+        retryOpts
       ),
-      retryOpts
+      toolChoice,
+      `${config.provider}/${config.model}`
     );
   };
   const clientStreamText = (args) => {
@@ -11112,6 +11321,66 @@ function createLlmClient(config) {
   };
 }
 
+// ../../packages/llm/src/failover.ts
+function isFailoverWorthy(err) {
+  return err instanceof RetryExhaustedError || err instanceof LLMTimeoutError || err instanceof QuotaExhaustedError;
+}
+function errLabel(err) {
+  return err instanceof Error ? err.name : String(err);
+}
+function createFailoverFromClients(clients) {
+  if (clients.length === 0) {
+    throw new ProviderConfigError("failover: at least one client is required");
+  }
+  if (clients.length === 1) return clients[0];
+  let activeIndex = 0;
+  async function runWithFailover(op, label) {
+    let lastErr;
+    for (let i = activeIndex; i < clients.length; i++) {
+      try {
+        const result = await op(clients[i]);
+        activeIndex = i;
+        return result;
+      } catch (err) {
+        lastErr = err;
+        if (!isFailoverWorthy(err)) throw err;
+        const next = i + 1;
+        if (next < clients.length) {
+          console.warn(
+            `[llm-failover] ${label}: ${clients[i].config.provider}/${clients[i].config.model} failed (${errLabel(err)}) \u2014 failing over to ${clients[next].config.provider}/${clients[next].config.model}`
+          );
+        }
+      }
+    }
+    throw new AllProvidersFailedError(clients.length, lastErr);
+  }
+  const primary = clients[0];
+  return {
+    // Surface the primary's identity/capabilities; the chain is homogeneous in
+    // the capability that matters here (tool use). Failover is for outages, not
+    // capability switching.
+    config: primary.config,
+    capabilities: primary.capabilities,
+    generateText: ((args) => runWithFailover(
+      (c) => c.generateText(args),
+      "generateText"
+    )),
+    // Streaming keeps single-provider semantics (the runner loop uses
+    // generateText). Delegate to the currently-active provider.
+    streamText: ((args) => clients[activeIndex].streamText(args)),
+    generateObject: ((args) => runWithFailover(
+      (c) => c.generateObject(args),
+      "generateObject"
+    ))
+  };
+}
+function createFailoverLlmClient(configs) {
+  if (configs.length === 0) {
+    throw new ProviderConfigError("failover: at least one provider config is required");
+  }
+  return createFailoverFromClients(configs.map((c) => createLlmClient(c)));
+}
+
 // ../../packages/llm/src/embeddings.ts
 import { embed } from "ai";
 import { createOllama as createOllama2 } from "ollama-ai-provider-v2";
@@ -11344,7 +11613,10 @@ async function _writeToolCall(ctx, toolName, input, output, durationMs) {
 
 // ../../packages/tools/src/tool-choice.ts
 function computeToolChoice(cfg) {
-  const { isOrchestrator, turn, hasAdapterTools } = cfg;
+  const { isOrchestrator, turn, hasAdapterTools, modelSupportsForcedToolChoice = true } = cfg;
+  if (!modelSupportsForcedToolChoice) {
+    return "auto";
+  }
   if (hasAdapterTools && !isOrchestrator) {
     return "required";
   }
@@ -26212,6 +26484,67 @@ async function createMcpTools(opts) {
   return { tools, close: conn.close };
 }
 
+// src/job/resolve-llm.ts
+init_src();
+async function resolveAgentLlmClient(db, agent, onSkip) {
+  if (!agent.llmKeyId) return { ok: false, reason: "agent_no_llm_configured" };
+  const seen = /* @__PURE__ */ new Set();
+  const requested = [];
+  for (const link of [
+    { keyId: agent.llmKeyId, model: agent.model },
+    ...agent.fallbackChain ?? []
+  ]) {
+    if (typeof link.keyId === "string" && link.keyId.length > 0 && !seen.has(link.keyId)) {
+      seen.add(link.keyId);
+      requested.push({ keyId: link.keyId, model: link.model ?? "" });
+    }
+  }
+  const ids = requested.map((r) => r.keyId);
+  const rows = await db.select().from(entityLlmKeys).where(inArray3(entityLlmKeys.id, ids));
+  const byId = new Map(rows.map((r) => [r.id, r]));
+  try {
+    const configs = [];
+    for (const { keyId, model: requestedModel } of requested) {
+      const row = byId.get(keyId);
+      if (!row || !row.isActive) {
+        onSkip?.({ keyId, reason: "missing_or_inactive" });
+        continue;
+      }
+      const model = requestedModel.length > 0 ? requestedModel : MODEL_CATALOG[row.provider]?.[0]?.modelId ?? "";
+      if (!model) {
+        onSkip?.({ keyId, reason: "no_catalog_model" });
+        continue;
+      }
+      const plaintextKey = row.apiKey ? decrypt(row.apiKey) : "";
+      configs.push({
+        provider: row.provider,
+        model,
+        apiKey: plaintextKey || void 0,
+        baseURL: row.baseUrl ?? void 0
+      });
+    }
+    if (configs.length === 0) return { ok: false, reason: "agent_no_llm_configured" };
+    const effectivePrimary = configs[0];
+    const client = configs.length > 1 ? createFailoverLlmClient(configs) : createLlmClient(effectivePrimary);
+    return {
+      ok: true,
+      client,
+      primaryProvider: effectivePrimary.provider,
+      chainLength: configs.length,
+      // Capability comes from the model CATALOG (provider, model of the
+      // effective primary), not a stored column. Unknown/custom models default
+      // to true; the runtime tool_choice floor backstops a wrong guess.
+      primarySupportsForcedToolChoice: findModelCatalogEntry(effectivePrimary.provider, effectivePrimary.model)?.capabilities.forcedToolChoice ?? true
+    };
+  } catch (err) {
+    return {
+      ok: false,
+      reason: "llm_key_invalid",
+      detail: err instanceof Error ? err.message.slice(0, 200) : "llm_key_invalid"
+    };
+  }
+}
+
 // ../../packages/orchestration/src/errors.ts
 var DelegationPendingError = class extends Error {
   constructor(childJobId, childSlug) {
@@ -26273,7 +26606,16 @@ var DEFAULT_LIMITS = {
   maxDelegationDepth: 3,
   maxTurns: 50,
   // matches Hermes Agent's per-subagent iteration budget; cumulative cap across resumes
-  maxConsecutiveDeliveryTurns: 3
+  maxConsecutiveDeliveryTurns: 3,
+  // 1.5M total tokens: a loud backstop well above any legitimate single job
+  // (typical jobs sit in the tens of thousands) yet below the ~2.4M-token
+  // runaway that motivated it. Override per-deployment via MAX_TOTAL_TOKENS_PER_JOB.
+  maxTotalTokensPerJob: 15e5,
+  // 12 identical (toolName+input+output) turns in a row before declaring the job
+  // stuck. Deliberately conservative: a real poll completes (output changes) long
+  // before 12 identical reads, so this only catches genuinely degenerate loops —
+  // and maxTurns (50) is the ultimate backstop above it.
+  maxNoProgressRepeats: 12
 };
 var ChainCounters = class _ChainCounters {
   constructor(limits = DEFAULT_LIMITS) {
@@ -26971,7 +27313,7 @@ function buildJobContextBlock(ctx) {
   if (ctx.telegramChatId) lines.push(`- telegram_chat_id: ${ctx.telegramChatId}`);
   if (ctx.surface === "chat") {
     lines.push(
-      '- surface: in-app dashboard chat \u2014 you are talking directly with the user; reply in plain text. For conversation or recalling facts, just reply (your durable facts are loaded below). For ANY action \u2014 using a connector or skill, delegating to your team, sending/fetching/creating/publishing, or (as the workspace ROOT) creating agents, skills, MCP servers, connectors or automations \u2014 call `run_task` with a clear, self-contained instruction: it runs as a tracked job with your FULL toolset. `run_task` is your gateway to everything you can do \u2014 NEVER tell the user you cannot do something that an action could accomplish; escalate it via `run_task` instead. When you call `run_task`, ALSO write a brief one-line acknowledgment in your own voice (e.g. "Je m\'en occupe\u2026") so the user sees you started. Do not call any other named tool on this surface.'
+      '- surface: in-app dashboard chat \u2014 you are talking directly with the user; reply in plain text. For conversation or recalling facts, just reply (your durable facts are loaded below). For ANY action \u2014 using a connector or skill, delegating to your team, sending/fetching/creating/publishing, or (as the workspace ROOT) creating agents, skills, MCP servers, connectors or automations \u2014 you MUST call the `run_task` tool with a clear, self-contained instruction. CRITICAL: writing in text that you will do something (e.g. "Je lance X\u2026") does NOT start anything \u2014 ONLY an actual `run_task` tool call performs the action. If you intend to act, the `run_task` tool call is mandatory; a text-only reply about an action accomplishes nothing. It runs as a tracked job with your FULL toolset. `run_task` is your gateway to everything you can do \u2014 NEVER tell the user you cannot do something that an action could accomplish; escalate it via `run_task` instead. You may add a one-line acknowledgment in your own voice alongside the call, but the `run_task` call is what actually does the work. Do not call any other named tool on this surface.'
     );
   }
   if (ctx.notifyOnSuccess) {
@@ -27312,9 +27654,21 @@ function truncateForContext(value) {
 
 [... truncated: ${dropped} chars dropped (total ${value.length}) ...]`;
 }
+function stableStringify(value) {
+  return JSON.stringify(value, (_key, val) => {
+    if (val && typeof val === "object" && !Array.isArray(val)) {
+      return Object.keys(val).sort().reduce((acc, k) => {
+        acc[k] = val[k];
+        return acc;
+      }, {});
+    }
+    return val;
+  });
+}
 async function executeJob(jobId, deps, _runnerEnv) {
   const { db, registry } = deps;
   let llmClient;
+  let modelSupportsForcedToolChoice = true;
   const startedAt = Date.now();
   const trace = (event, data) => {
     console.error(`[exec ${jobId}] ${event}`, data ? JSON.stringify(data) : "");
@@ -27378,28 +27732,27 @@ async function executeJob(jobId, deps, _runnerEnv) {
     return { status: "failed", error: "agent_no_llm_configured" };
   }
   {
-    const [keyRow] = await db.select().from(entityLlmKeys).where(eq4(entityLlmKeys.id, agentRow.llmKeyId)).limit(1);
-    if (!keyRow || !keyRow.isActive) {
-      await failJob(db, jobId, "agent_no_llm_configured", runStats());
-      return { status: "failed", error: "agent_no_llm_configured" };
-    }
-    try {
-      const plaintextKey = keyRow.apiKey ? decrypt(keyRow.apiKey) : "";
-      llmClient = createLlmClient({
-        provider: keyRow.provider,
-        model: agent.model,
-        apiKey: plaintextKey || void 0,
-        baseURL: keyRow.baseUrl ?? void 0
-      });
-      trace("llm_client_from_key", {
-        keyId: keyRow.id,
-        provider: keyRow.provider
-      });
-    } catch (err) {
-      const errorCode = err instanceof Error ? err.message.slice(0, 200) : "llm_key_invalid";
-      await failJob(db, jobId, `llm_key_invalid:${errorCode}`, runStats());
-      return { status: "failed", error: `llm_key_invalid:${errorCode}` };
-    }
+    const resolved = await resolveAgentLlmClient(
+      db,
+      {
+        llmKeyId: agentRow.llmKeyId,
+        fallbackChain: agentRow.fallbackChain ?? null,
+        model: agent.model
+      },
+      (info) => trace("fallback_key_skipped", info)
+    );
+    if (!resolved.ok) {
+      const code = resolved.reason === "agent_no_llm_configured" ? "agent_no_llm_configured" : `llm_key_invalid:${resolved.detail}`;
+      await failJob(db, jobId, code, runStats());
+      return { status: "failed", error: code };
+    }
+    llmClient = resolved.client;
+    modelSupportsForcedToolChoice = resolved.primarySupportsForcedToolChoice;
+    trace("llm_client_from_key", {
+      provider: resolved.primaryProvider,
+      chainLength: resolved.chainLength,
+      forcedToolChoice: modelSupportsForcedToolChoice
+    });
   }
   const childRows = await db.select({ id: agents.id, slug: agents.slug, role: agents.role }).from(agentAssignments).innerJoin(agents, eq4(agentAssignments.subAgentId, agents.id)).where(and3(eq4(agentAssignments.orchestratorId, agentRow.id), eq4(agents.active, true)));
   const children = childRows.map((r) => ({
@@ -27734,6 +28087,20 @@ async function executeJob(jobId, deps, _runnerEnv) {
     await setJobStatus(db, jobId, "awaiting_approval");
     return { status: "awaiting_approval" };
   };
+  const maxTotalTokensPerJob = (() => {
+    const raw = process.env["MAX_TOTAL_TOKENS_PER_JOB"];
+    const n = raw ? Number(raw) : NaN;
+    return Number.isFinite(n) && n > 0 ? n : DEFAULT_LIMITS.maxTotalTokensPerJob;
+  })();
+  const recentTurnSignatures = [];
+  const maxNoProgressRepeats = (() => {
+    const raw = process.env["MAX_NO_PROGRESS_REPEATS"];
+    const n = raw ? Number(raw) : NaN;
+    return Number.isFinite(n) && n >= 2 ? Math.floor(n) : DEFAULT_LIMITS.maxNoProgressRepeats;
+  })();
+  const unresolvedToolFailures = /* @__PURE__ */ new Set();
+  const MAX_UNRESOLVED_FAILURE_NUDGES = 2;
+  let unresolvedFailureNudges = 0;
   try {
     while (true) {
       turn += 1;
@@ -27749,7 +28116,12 @@ async function executeJob(jobId, deps, _runnerEnv) {
         return { status: "failed", error: "turn_limit_exceeded" };
       }
       validateMessageStructure(messages);
-      const toolChoice = computeToolChoice({ isOrchestrator, turn, hasAdapterTools });
+      const toolChoice = computeToolChoice({
+        isOrchestrator,
+        turn,
+        hasAdapterTools,
+        modelSupportsForcedToolChoice
+      });
       const aiSdkTools = {};
       for (const [name, toolDef] of toolMap) {
         const description = authoringToolsSuffix && (name === "create_skill" || name === "update_skill") ? toolDef.description + authoringToolsSuffix : toolDef.description;
@@ -27767,6 +28139,11 @@ async function executeJob(jobId, deps, _runnerEnv) {
       const completionT = Number(usage?.outputTokens ?? 0);
       inputTokens += Number.isFinite(promptT) ? promptT : 0;
       outputTokens += Number.isFinite(completionT) ? completionT : 0;
+      if (inputTokens + outputTokens > maxTotalTokensPerJob) {
+        trace("token_budget_exceeded", { turn, inputTokens, outputTokens, maxTotalTokensPerJob });
+        await failJob(db, jobId, "token_budget_exceeded", runStats());
+        return { status: "failed", error: "token_budget_exceeded" };
+      }
       const rawToolCalls = response.toolCalls ?? [];
       trace("llm_call_done", {
         turn,
@@ -28011,6 +28388,11 @@ async function executeJob(jobId, deps, _runnerEnv) {
         if (toolResult.outcome === "success" && DELIVERY_TOOL_NAMES.has(call.name)) {
           telegramDelivered = true;
         }
+        if (toolResult.outcome === "success") {
+          unresolvedToolFailures.delete(call.name);
+        } else {
+          unresolvedToolFailures.add(call.name);
+        }
         toolResultBlocks.push({
           type: "tool-result",
           toolCallId: call.id,
@@ -28059,6 +28441,38 @@ async function executeJob(jobId, deps, _runnerEnv) {
       }
       if (returnResultCall) {
         trace("return_result_branch", { turn });
+        const rrStatus = returnResultCall.input?.status;
+        if (rrStatus === "success" && unresolvedToolFailures.size > 0) {
+          const stuck = [...unresolvedToolFailures];
+          if (unresolvedFailureNudges < MAX_UNRESOLVED_FAILURE_NUDGES) {
+            unresolvedFailureNudges += 1;
+            trace("unresolved_tool_failure_nudge", {
+              turn,
+              attempt: unresolvedFailureNudges,
+              stuck
+            });
+            toolResultBlocks.push({
+              type: "tool-result",
+              toolCallId: returnResultCall.toolCallId,
+              toolName: "return_result",
+              output: toResultOutput({
+                error: "deferred: tu signales success mais ces actions ont \xE9chou\xE9 sans \xEAtre corrig\xE9es (" + stuck.join(", ") + "). R\xE9essaie l'action jusqu'\xE0 r\xE9ussite, ou appelle return_result avec status='blocked'."
+              })
+            });
+            messages = [...messages, { role: "tool", content: toolResultBlocks }];
+            messages = [
+              ...messages,
+              {
+                role: "user",
+                content: "[syst\xE8me] Ne d\xE9clare pas un succ\xE8s qui n'a pas eu lieu. Une action a \xE9chou\xE9 et n'a pas \xE9t\xE9 corrig\xE9e. Corrige-la, ou termine honn\xEAtement avec status='blocked'."
+              }
+            ];
+            continue;
+          }
+          trace("unresolved_tool_failure", { turn, stuck });
+          await failJob(db, jobId, "unresolved_tool_failure", runStats());
+          return { status: "failed", error: "unresolved_tool_failure" };
+        }
         const taskRows = await db.select({ id: agentTasks.id }).from(agentTasks).where(eq4(agentTasks.rootJobId, jobId));
         if (requiresToolDelivery && !telegramDelivered && taskRows.length === 0) {
           if (telegramRedeliveryNudges < MAX_TELEGRAM_REDELIVERY_NUDGES) {
@@ -28118,6 +28532,23 @@ async function executeJob(jobId, deps, _runnerEnv) {
       if (toolResultBlocks.length > 0) {
         messages = [...messages, { role: "tool", content: toolResultBlocks }];
       }
+      const turnSignature = toolResultBlocks.map((b) => {
+        const call = callsToProcess.find((c) => c.id === b.toolCallId);
+        const input = call ? stableStringify(call.input) : "";
+        const output = b.output.type === "text" ? b.output.value : stableStringify(b.output.value ?? null);
+        return `${b.toolName}\0${input}\0${output}`;
+      }).sort().join("\n");
+      if (turnSignature !== "") {
+        recentTurnSignatures.push(turnSignature);
+        if (recentTurnSignatures.length > maxNoProgressRepeats) {
+          recentTurnSignatures.shift();
+        }
+        if (recentTurnSignatures.length === maxNoProgressRepeats && recentTurnSignatures.every((s) => s === turnSignature)) {
+          trace("no_progress_detected", { turn, repeats: recentTurnSignatures.length });
+          await failJob(db, jobId, "no_progress_detected", runStats());
+          return { status: "failed", error: "no_progress_detected" };
+        }
+      }
       await saveCheckpoint(db, jobId, {
         messages,
         turn,
@@ -28148,6 +28579,10 @@ async function executeJob(jobId, deps, _runnerEnv) {
       await failJob(db, jobId, "quota_exhausted", runStats());
       return { status: "failed", error: "quota_exhausted" };
     }
+    if (err instanceof AllProvidersFailedError) {
+      await failJob(db, jobId, err.code, runStats());
+      return { status: "failed", error: err.code };
+    }
     if (err instanceof MessageStructureError) {
       await failJob(db, jobId, `message_structure_invalid:${err.code}`, runStats());
       return { status: "failed", error: `message_structure_invalid:${err.code}` };
@@ -28786,20 +29221,18 @@ async function runChatTurn(opts) {
     const title = message.trim().slice(0, TITLE_MAX) + (message.trim().length > TITLE_MAX ? "\u2026" : "");
     await db.update(conversations).set({ title }).where(eq4(conversations.id, conversationId));
   }
-  const [keyRow] = await db.select().from(entityLlmKeys).where(eq4(entityLlmKeys.id, agentRow.llmKeyId)).limit(1);
-  if (!keyRow || !keyRow.isActive) return { ok: false, error: "agent_no_llm_configured" };
-  let llmClient;
-  try {
-    const plaintextKey = keyRow.apiKey ? decrypt(keyRow.apiKey) : "";
-    llmClient = createLlmClient({
-      provider: keyRow.provider,
-      model: agentRow.model ?? DEFAULT_MODEL,
-      apiKey: plaintextKey || void 0,
-      baseURL: keyRow.baseUrl ?? void 0
-    });
-  } catch {
-    return { ok: false, error: "llm_key_invalid" };
+  const resolved = await resolveAgentLlmClient(db, {
+    llmKeyId: agentRow.llmKeyId,
+    fallbackChain: agentRow.fallbackChain ?? null,
+    model: agentRow.model ?? DEFAULT_MODEL
+  });
+  if (!resolved.ok) {
+    return {
+      ok: false,
+      error: resolved.reason === "agent_no_llm_configured" ? "agent_no_llm_configured" : "llm_key_invalid"
+    };
   }
+  const llmClient = resolved.client;
   const agent = {
     id: agentRow.id,
     name: agentRow.name,
@@ -28915,7 +29348,6 @@ async function seedDefaultLlmKey(db, env2) {
     apiKeyLast4: last4(plaintextKey),
     baseUrl: env2.LLM_BASE_URL ?? null,
     nickname: "Default (env)",
-    defaultModel: env2.LLM_MODEL,
     isActive: true
   }).returning({ id: entityLlmKeys.id });
   if (!newKey) return;
@@ -30710,7 +31142,7 @@ function createApp(deps, runnerEnv) {
       throw err;
     }
   });
-  app.use("/api/approve", async (c, next) => {
+  const bearerOrSession = async (c, next) => {
     const auth2 = c.req.header("authorization") ?? "";
     const bearer = auth2.startsWith("Bearer ") ? auth2.slice(7) : null;
     if (bearer && runnerEnv.WORKER_SECRET && bearer === runnerEnv.WORKER_SECRET) {
@@ -30729,7 +31161,8 @@ function createApp(deps, runnerEnv) {
       }
       throw err;
     }
-  });
+  };
+  app.use("/api/approve", bearerOrSession);
   app.get("/api/health", (c) => healthRoute(c, deps));
   app.post("/api/agent", (c) => agentRoute(c, deps, runnerEnv));
   app.post("/api/worker", (c) => workerRoute(c, deps, runnerEnv));