@exulu/backend 3.1.0 → 3.3.0

package/ee/agentic-retrieval/pipeline/index.test.ts

@@ -4,13 +4,13 @@ import { createAgenticRetrievalTool, parsePreselectedItems } from "./index";
 jest.mock("@EE/entitlements", () => ({ checkLicense: () => ({ "agentic-retrieval": true }) }));
 jest.mock("@SRC/exulu/resolve-reranker", () => ({ resolveReranker: jest.fn(async () => ({ model: "m", rerank: async (_q: any, c: any) => c })) }));
 jest.mock("@SRC/exulu/resolve-model", () => ({ resolveModel: jest.fn() }));
-jest.mock("@SRC/exulu/app/singleton", () => ({ exuluApp: { get: () => ({ providers: [] }) } }));
+jest.mock("@SRC/exulu/app/singleton", () => ({ exuluApp: { get: () => ({}) } }));
 jest.mock("./routing", () => ({ runRoutingPhase: jest.fn(async () => ({
   mainContexts: ["docs"], fallbackContexts: [], userPinnedItemIdsByContext: new Map(),
   userRequestedPage: null, hasExplicitDocAndPage: false, steps: [{ text: "routed" }] })) }));
 jest.mock("./memory", () => ({ runMemoryPhase: jest.fn(async () => ({
   memoryChunksForAnswer: [], memoryOverride: { active: false, chunks: [], reason: "" },
-  memoryPinnedItemIds: new Set(), updatedQuestion: "q", updatedKeywords: ["k"],
+  memoryPinnedItemIdsByContext: new Map(), updatedQuestion: "q", updatedKeywords: ["k"],
   updatedImportantKeyword: "k", steps: [] })) }));
 jest.mock("./prefilter", () => ({ resolveIdentifierPins: jest.fn(async () => ({
   pinsByContext: new Map(), exactPinsByContext: new Map(), steps: [] })) }));
@@ -75,7 +75,7 @@ describe("createAgenticRetrievalTool", () => {
     runMemoryPhase.mockResolvedValueOnce({
       memoryChunksForAnswer: [{ chunk_id: "m1" }],
       memoryOverride: { active: false, chunks: [], reason: "" },
-      memoryPinnedItemIds: new Set(),
+      memoryPinnedItemIdsByContext: new Map(),
       updatedQuestion: "q",
       updatedKeywords: ["k"],
       updatedImportantKeyword: "k",
@@ -114,7 +114,7 @@ describe("payload deduplication", () => {
     runMemoryPhase.mockResolvedValueOnce({
       memoryChunksForAnswer: [memChunk],
       memoryOverride: { active: false, chunks: [], reason: "" },
-      memoryPinnedItemIds: new Set(), updatedQuestion: "q", updatedKeywords: ["k"],
+      memoryPinnedItemIdsByContext: new Map(), updatedQuestion: "q", updatedKeywords: ["k"],
       updatedImportantKeyword: "k",
       steps: [{ text: "memory step", chunks: [memChunk] }],
     });

package/ee/agentic-retrieval/pipeline/index.ts

@@ -274,7 +274,6 @@ export function createAgenticRetrievalTool(opts: {
           const resolved = await resolveModel({
             modelId: cfg.utilityModel,
             user,
-            providers: exuluApp.get().providers,
             rbacBypass: true,
           });
           utilityModel = resolved.languageModel ?? model;
@@ -426,7 +425,7 @@ export function createAgenticRetrievalTool(opts: {
         updatedQuestion,
         updatedKeywords,
         updatedImportantKeyword,
-        memoryPinnedItemIds,
+        memoryPinnedItemIdsByContext,
         memoryOverride,
       } = memResult;
 
@@ -465,7 +464,7 @@ export function createAgenticRetrievalTool(opts: {
           model: utilityModel,
           preselectedItems,
           identifierPinsByContext,
-          memoryPinnedItemIds,
+          memoryPinnedItemIdsByContext,
           userPinnedItemIdsByContext,
           scopedItemsByContext: resolvedProject?.scopedItemsByContext,
           rewrites: cfg.vocabulary.rewrites,
@@ -486,7 +485,7 @@ export function createAgenticRetrievalTool(opts: {
               model: utilityModel,
               preselectedItems,
               identifierPinsByContext,
-              memoryPinnedItemIds,
+              memoryPinnedItemIdsByContext,
               userPinnedItemIdsByContext,
               scopedItemsByContext: resolvedProject?.scopedItemsByContext,
               rewrites: cfg.vocabulary.rewrites,
@@ -500,7 +499,9 @@ export function createAgenticRetrievalTool(opts: {
       // ── Build rerank state ────────────────────────────────────────────────
       // pinnedItemIds = memory ∪ exact identifier pins ∪ user pins ∪ project pins
       const pinnedItemIds = new Set<string>([
-        ...memoryPinnedItemIds,
+        ...(function* () {
+          for (const s of memoryPinnedItemIdsByContext.values()) yield* s;
+        })(),
         ...(function* () {
           for (const s of exactPinsByContext.values()) yield* s;
         })(),

package/ee/agentic-retrieval/pipeline/memory.test.ts

@@ -74,7 +74,7 @@ describe("runMemoryPhase", () => {
     expect(r.updatedImportantKeyword).toBe("FST-2XT");
   });
 
-  it("resolves file-prioritization pins across all document contexts", async () => {
+  it("resolves file-prioritization pins keyed by their document context", async () => {
     (generateText as jest.Mock)
       .mockResolvedValueOnce({ output: { relevantChunkIds: ["1"] } })
       .mockResolvedValueOnce({ output: { shouldPrioritizeFiles: true, fileNameHints: ["PROJECT_NOTES"] } });
@@ -82,7 +82,9 @@ describe("runMemoryPhase", () => {
     const r = await runMemoryPhase({ ...baseOpts, memoryChunks: [memChunk("1", "always check PROJECT_NOTES")],
       memoryContext: undefined, documentContexts: [{ id: "docs" }],
       memoryConfig: { enabled: true, override: false, filePrioritization: true, queryAugmentation: false } });
-    expect([...r.memoryPinnedItemIds]).toEqual(["d1"]);
+    // Pins are keyed by the context they were resolved in, so a consumer can apply them
+    // only to that context (no cross-context leak). See search.ts rule 2b.
+    expect([...(r.memoryPinnedItemIdsByContext.get("docs") ?? [])]).toEqual(["d1"]);
   });
 
   it("never throws even when post-Promise.all processing encounters runtime errors", async () => {

package/ee/agentic-retrieval/pipeline/memory.ts

@@ -129,7 +129,7 @@ function neutralResult(
   return {
     memoryChunksForAnswer: [],
     memoryOverride: { active: false, chunks: [], reason: "" },
-    memoryPinnedItemIds: new Set(),
+    memoryPinnedItemIdsByContext: new Map(),
     updatedQuestion: question,
     updatedKeywords: keywords,
     updatedImportantKeyword: importantKeyword,
@@ -272,7 +272,7 @@ export async function runMemoryPhase({
       chunks: [],
       reason: "",
     };
-    let memoryPinnedItemIds = new Set<string>();
+    const memoryPinnedItemIdsByContext = new Map<string, Set<string>>();
     let updatedQuestion = question;
     let updatedKeywords = keywords;
     let updatedImportantKeyword = importantKeyword;
@@ -466,12 +466,15 @@ export async function runMemoryPhase({
         };
       }
 
-      // File prioritization: resolve hints via fuzzyPrefilter against EVERY documentContexts entry
+      // File prioritization: resolve hints via fuzzyPrefilter against EVERY documentContexts entry.
+      // Pins are kept keyed by the context they were resolved in, so the search phase can apply a
+      // pin only to its home context — a tech_doc file must never filter a software-docs search to 0.
       if (fileResult.output?.shouldPrioritizeFiles && fileResult.output?.fileNameHints?.length) {
         const hints = fileResult.output.fileNameHints;
         const pinResults = await Promise.all(
-          documentContexts.map((ctx) =>
-            fuzzyPrefilter({
+          documentContexts.map(async (ctx) => ({
+            ctxId: ctx.id as string,
+            matches: await fuzzyPrefilter({
               cacheKey: `memory-pin:${ctx.id}`,
               relevantKeywords: hints,
               context: ctx,
@@ -479,17 +482,21 @@ export async function runMemoryPhase({
               normalize: (item: any) =>
                 item.external_id ? normalizeFileName(item.external_id) : item.name,
             }).catch(() => []),
-          ),
+          })),
         );
-        for (const results of pinResults) {
-          for (const r of results) {
-            memoryPinnedItemIds.add(r.id);
+        const pinnedNames: string[] = [];
+        for (const { ctxId, matches } of pinResults) {
+          if (!matches.length) continue;
+          const set = memoryPinnedItemIdsByContext.get(ctxId) ?? new Set<string>();
+          for (const m of matches) {
+            set.add(m.id);
+            pinnedNames.push(m.name);
           }
+          memoryPinnedItemIdsByContext.set(ctxId, set);
         }
-        if (memoryPinnedItemIds.size > 0) {
-          const names = pinResults.flat().map((i) => i.name).join(", ");
+        if (pinnedNames.length > 0) {
           steps.push({
-            text: `Memory prioritizes specific document(s); pinning ${memoryPinnedItemIds.size} file(s) into the search: ${names}`,
+            text: `Memory prioritizes specific document(s); pinning ${pinnedNames.length} file(s) into the search: ${pinnedNames.join(", ")}`,
           });
         }
       }
@@ -520,7 +527,7 @@ export async function runMemoryPhase({
     return {
       memoryChunksForAnswer,
       memoryOverride,
-      memoryPinnedItemIds,
+      memoryPinnedItemIdsByContext,
       updatedQuestion,
       updatedKeywords,
       updatedImportantKeyword,

package/ee/agentic-retrieval/pipeline/search.test.ts

@@ -20,7 +20,7 @@ const base = {
   } as any,
   question: "how to fix door error E42", keywords: ["door", "E42"], importantKeyword: "E42",
   user: {}, role: "r", model: {},
-  preselectedItems: new Map(), identifierPinsByContext: new Map(), memoryPinnedItemIds: new Set<string>(),
+  preselectedItems: new Map(), identifierPinsByContext: new Map(), memoryPinnedItemIdsByContext: new Map<string, Set<string>>(),
   userPinnedItemIdsByContext: new Map(), rewrites: [{ find: "fix", replace: "repair" }],
   styleHint: "", maxQueries: 5, skipPrefilter: false,
 };
@@ -55,7 +55,7 @@ describe("searchContexts", () => {
     await searchContexts({
       ...base, contextIds: ["docs"],
       identifierPinsByContext: new Map([["docs", new Set(["i1"])]]),
-      memoryPinnedItemIds: new Set(["m1"]),
+      memoryPinnedItemIdsByContext: new Map([["docs", new Set(["m1"])]]),
     });
     expect(new Set((multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds)).toEqual(new Set(["i1", "m1"]));
 
@@ -63,12 +63,27 @@ describe("searchContexts", () => {
     await searchContexts({
       ...base, contextIds: ["docs"],
       identifierPinsByContext: new Map([["docs", new Set(["i1"])]]),
-      memoryPinnedItemIds: new Set(["m1"]),
+      memoryPinnedItemIdsByContext: new Map([["docs", new Set(["m1"])]]),
       userPinnedItemIdsByContext: new Map([["docs", new Set(["u1"])]]),
     });
     expect((multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds).toEqual(["u1"]);
   });
 
+  it("memory pins apply ONLY to their own context (no cross-context leak)", async () => {
+    // Regression: a memory pin resolved in another context (e.g. tech_doc's FST-Miscel-Secrets)
+    // must not become a hard id-whitelist on THIS context, which would filter it down to 0 chunks.
+    await searchContexts({
+      ...base, contextIds: ["docs"],
+      memoryPinnedItemIdsByContext: new Map([
+        ["docs", new Set(["m_docs"])],
+        ["other", new Set(["m_other"])],
+      ]),
+    });
+    const pins = (multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds;
+    expect(pins).toEqual(["m_docs"]);        // own-context pin applied
+    expect(pins).not.toContain("m_other");   // foreign-context pin did NOT leak in
+  });
+
   it("preselected items win over everything and skip prefilters", async () => {
     await searchContexts({
       ...base, contextIds: ["docs"],
@@ -95,7 +110,7 @@ describe("searchContexts", () => {
     await searchContexts({
       ...base, contextIds: ["docs"], skipPrefilter: true,
       identifierPinsByContext: new Map([["docs", new Set(["i1"])]]),
-      memoryPinnedItemIds: new Set(["m1"]),
+      memoryPinnedItemIdsByContext: new Map([["docs", new Set(["m1"])]]),
     });
     expect((multiQuerySearch as jest.Mock).mock.calls[0][0].pinnedItemIds).toEqual([]);
   });

package/ee/agentic-retrieval/pipeline/search.ts

@@ -25,7 +25,7 @@ export async function searchContexts(opts: {
   preselectedItems: Map<string, string[] | null>;
   scopedItemsByContext?: Map<string, string[] | null>;  // Project-added sources: hard item filter per context (null = whole context).
   identifierPinsByContext: Map<string, Set<string>>;   // from resolveIdentifierPins
-  memoryPinnedItemIds: Set<string>;                     // from memory phase (documents kind only)
+  memoryPinnedItemIdsByContext: Map<string, Set<string>>; // from memory phase (documents kind only), keyed by home context
   userPinnedItemIdsByContext: Map<string, Set<string>>; // from routing phase
   rewrites: { find: string; replace: string }[];
   styleHint: string;
@@ -45,7 +45,7 @@ export async function searchContexts(opts: {
     preselectedItems,
     scopedItemsByContext,
     identifierPinsByContext,
-    memoryPinnedItemIds,
+    memoryPinnedItemIdsByContext,
     userPinnedItemIdsByContext,
     rewrites,
     styleHint,
@@ -94,9 +94,13 @@ export async function searchContexts(opts: {
           const identifierPins = identifierPinsByContext.get(ctxId) ?? new Set<string>();
           let pins = new Set<string>(identifierPins);
 
-          // 2b: documents kind only — UNION with memoryPinnedItemIds
+          // 2b: documents kind only — UNION with memory pins that belong to THIS context.
+          // Memory pins are keyed by their home context so a pin resolved elsewhere (e.g. a
+          // tech_doc file) never becomes a hard id-whitelist on a context that lacks it, which
+          // would prefilter that context down to 0 chunks.
           if (kind === "documents") {
-            for (const id of memoryPinnedItemIds) pins.add(id);
+            const memPins = memoryPinnedItemIdsByContext.get(ctxId);
+            if (memPins) for (const id of memPins) pins.add(id);
           }
 
           // 2c: user pins REPLACE everything (authoritative)

package/ee/agentic-retrieval/pipeline/types.ts

@@ -21,7 +21,7 @@ export type RoutingPhaseResult = {
 export type MemoryPhaseResult = {
   memoryChunksForAnswer: ChunkWithScore[];
   memoryOverride: { active: boolean; chunks: ChunkWithScore[]; reason: string };
-  memoryPinnedItemIds: Set<string>;
+  memoryPinnedItemIdsByContext: Map<string, Set<string>>;
   updatedQuestion: string;
   updatedKeywords: string[];
   updatedImportantKeyword: string;

package/ee/python/documents/processing/doc_processor.ts

@@ -146,7 +146,6 @@ async function resolveVlmModel(
 
   const { languageModel } = await resolveModel({
     modelId,
-    providers: [], // unused in LiteLLM mode; resolveModel ignores it there
     user: config?.attribution?.user,
     project: config?.attribution?.project,
     agent: config?.attribution?.agent,

package/ee/queues/queues.ts

@@ -5,6 +5,19 @@ import { BullMQOtel } from "bullmq-otel";
 import type { ExuluQueueConfig } from "@EXULU_TYPES/queue-config";
 import { checkLicense } from "@EE/entitlements";
 
+/**
+ * Built-in/system queues that ExuluApp registers automatically. They back
+ * internal processing (eval runs, inbound email intake) and must never be
+ * offered to users as a routine's run queue — the `queues` GraphQL query
+ * excludes them (see resolveAvailableQueues). Underscore, not hyphen:
+ * registered queue names are interpolated verbatim into the GraphQL QueueEnum,
+ * where "-" is illegal.
+ */
+export const global_queues = {
+  eval_runs: "eval_runs",
+  email_intake: "email_intake",
+};
+
 // Used for workflows and embedders
 class ExuluQueues {
   queues: {

package/ee/schemas.ts

@@ -419,6 +419,10 @@ export const workflowTemplatesSchema: ExuluTableDefinition = {
         type: "boolean",
         default: false,
       },
+      {
+        name: "queue",
+        type: "text"
+      }
     ],
   };
 
@@ -454,16 +458,13 @@ export const workflowTemplatesSchema: ExuluTableDefinition = {
           type: "boolean",
           default: false,
         },
-        {
-          // Generated server-side: {routine-slug}-{8 hex}@{inbound_domain}.
-          // Real UNIQUE column (not JSON) because the webhook resolves
-          // triggers by recipient address.
-          name: "address",
-          type: "text",
-          required: true,
-          unique: true,
-          index: true,
-        },
+        // Secret capability URL key: base64url randomBytes(32). Both routes
+        // and authorizes the webhook (POST /webhooks/routine/:secret).
+        { name: "secret", type: "text", required: true, unique: true, index: true },
+        // Optional per-trigger HMAC shared secret, AES-encrypted at rest.
+        { name: "signing_secret", type: "text" },
+        // Stamped on every verified webhook hit; per-trigger setup aid.
+        { name: "last_fired_at", type: "date" },
         {
           // allowed_senders / filters / filtered_run_retention /
           // rate_limit_per_hour / sender_rate_limit_per_hour (spec §3.1).

package/ee/workers.ts

@@ -16,7 +16,7 @@ import type { BullMqJobData } from "@EE/queues/decorator.ts";
 import { maybePruneJobResults } from "@EE/queues/prune-job-results.ts";
 import { type Tracer } from "@opentelemetry/api";
 import { v4 as uuidv4 } from "uuid";
-import { type UIMessage } from "ai";
+import { createIdGenerator, type UIMessage } from "ai";
 import CryptoJS from "crypto-js";
 import { STATISTICS_TYPE_ENUM, type STATISTICS_TYPE } from "@EXULU_TYPES/enums/statistics";
 import type { User } from "@EXULU_TYPES/models/user";
@@ -29,14 +29,15 @@ import type { STATISTICS_LABELS } from "@EXULU_TYPES/statistics.ts";
 import { sanitizeToolName } from "@SRC/utils/sanitize-tool-name.ts";
 import type { ExuluConfig } from "@SRC/exulu/app/index.ts";
 import { updateStatistic } from "@SRC/exulu/statistics";
-import type { ExuluProvider } from "@SRC/exulu/provider.ts";
-import { saveChat, getAgentMessages } from "@SRC/exulu/provider.ts";
+import { saveChat, getAgentMessages, generateStream } from "@SRC/exulu/generate-stream";
 import { exuluApp } from "@SRC/exulu/app/singleton";
 import { handleEmailIntake } from "@SRC/exulu/email-inbound/intake";
 import { markStreamActive, clearStreamActive } from "@SRC/exulu/active-streams.ts";
 import { messageHasPendingApproval, substituteVariablesInMessage } from "@SRC/exulu/routines/flow-steps.ts";
 import { createRunSession } from "@SRC/exulu/routines/run-session.ts";
 import { casJobResultState, parseRunMetadata, upsertWorkflowRunStart } from "@SRC/exulu/routines/run-state.ts";
+import { findLiteLLMModel } from "@SRC/exulu/litellm/catalog.ts";
+import { computeRunCostUsd } from "@SRC/exulu/routines/run-cost.ts";
 
 /**
  * Session-backed runs persist messages at each step boundary, so retries must
@@ -132,7 +133,6 @@ const installGlobalErrorHandlers = () => {
 let isShuttingDown = false;
 
 export const createWorkers = async (
-  providers: ExuluProvider[],
   queues: ExuluQueueConfig[],
   config: ExuluConfig,
   contexts: ExuluContext[],
@@ -525,11 +525,10 @@ export const createWorkers = async (
 
               const {
                 agent,
-                provider,
                 user,
                 workflow,
                 messages: inputMessages,
-              } = await validateWorkflowPayload(data, providers);
+              } = await validateWorkflowPayload(data);
 
               // Session-backed runs (spec §3.4): reuse the session provided by
               // the enqueuer (email intake / continuation / retry / previous
@@ -576,9 +575,7 @@ export const createWorkers = async (
                     // + substituted text) — pass a fresh deep copy each attempt
                     // so a retry/resume never reuses the mutated array.
                     const messages = await processUiMessagesFlow({
-                      providers,
                       agent,
-                      provider,
                       inputMessages: structuredClone(inputMessages),
                       contexts,
                       user,
@@ -658,6 +655,22 @@ export const createWorkers = async (
                   (priorTokens?.cachedInputTokens ?? 0) + metadata.tokens.cachedInputTokens,
               };
 
+              // Approximate per-run $ cost (spec 2026-07-29): recompute from the
+              // cumulative token totals × the run model's catalog list price on every
+              // persist, so it stays correct across pause/resume. Null when the model
+              // has no catalog price — the UI shows "—", not a fabricated $0.
+              const modelPrice = await findLiteLLMModel(agent.model ?? "");
+              (tokens as Record<string, number | null>).costUsd = computeRunCostUsd(
+                tokens.inputTokens,
+                tokens.outputTokens,
+                modelPrice
+                  ? {
+                      input_cost_per_million_tokens: modelPrice.input_cost_per_million_tokens,
+                      output_cost_per_million_tokens: modelPrice.output_cost_per_million_tokens,
+                    }
+                  : null,
+              );
+
               if (result.pausedAtStepIndex !== undefined) {
                 // Pause is success (spec §5.3): persist progress and flip to
                 // waiting_approval synchronously BEFORE returning — the
@@ -741,17 +754,15 @@ export const createWorkers = async (
 
               const {
                 agent,
-                provider,
                 user,
                 evalRun,
                 testCase,
                 messages: inputMessages,
-              } = await validateEvalPayload(data, providers);
+              } = await validateEvalPayload(data);
 
               const retries = 3;
               let attempts = 0;
 
-              // todo allow setting queue on agent Provider and then create a job with type "agent"
               const promise = new Promise<{
                 messages: UIMessage[];
                 metadata: {
@@ -768,9 +779,7 @@ export const createWorkers = async (
                 while (attempts < retries) {
                   try {
                     const messages = await processUiMessagesFlow({
-                      providers,
                       agent,
-                      provider,
                       inputMessages,
                       contexts,
                       user,
@@ -877,7 +886,6 @@ export const createWorkers = async (
                 } else {
                   result = await evalMethod.run(
                     agent,
-                    provider,
                     testCase,
                     messages,
                     evalFunction.config || {},
@@ -971,10 +979,9 @@ export const createWorkers = async (
               const {
                 evalRun,
                 agent,
-                provider,
                 testCase,
                 messages: inputMessages,
-              } = await validateEvalPayload(data, providers);
+              } = await validateEvalPayload(data);
 
               const evalFunctions: {
                 id: string;
@@ -995,7 +1002,6 @@ export const createWorkers = async (
 
                 result = await evalMethod.run(
                   agent,
-                  provider,
                   testCase,
                   inputMessages,
                   evalFunction.config || {},
@@ -1085,24 +1091,19 @@ export const createWorkers = async (
             }
 
             if (data.type === "email_intake") {
-              console.log("[EXULU] running an email intake job.", bullmqJob.name);
-
-              if (!data.inputs?.s3Key) {
-                throw new Error(`No s3Key set for email intake job.`);
+              console.log("[EXULU] running a routine webhook intake job.", bullmqJob.name);
+              if (!data.inputs?.s3Key || !data.inputs?.triggerId) {
+                throw new Error(`Missing s3Key/triggerId for email intake job.`);
               }
-
               const result = await handleEmailIntake(
                 {
                   s3Key: data.inputs.s3Key,
-                  recipient: data.inputs.recipient,
+                  triggerId: data.inputs.triggerId,
+                  format: data.inputs.format === "json" ? "json" : "eml",
                 },
-                { config, providers },
+                { config },
               );
-
-              return {
-                result,
-                metadata: {},
-              };
+              return { result, metadata: {} };
             }
 
             throw new Error(`Invalid job type: ${data.type} for job ${bullmqJob.name}.`);
@@ -1313,10 +1314,8 @@ export const createWorkers = async (
 
 export const validateWorkflowPayload = async (
   data: BullMqJobData,
-  providers: ExuluProvider[],
 ): Promise<{
   agent: ExuluAgent;
-  provider: ExuluProvider;
   user: User;
   workflow: ExuluWorkflow;
   variables: Record<string, any>;
@@ -1348,12 +1347,6 @@ export const validateWorkflowPayload = async (
     throw new Error(`Agent ${workflow.agent} not found in the database.`);
   }
 
-  const provider = providers.find((a) => a.id === agent.provider);
-
-  if (!provider) {
-    throw new Error(`Provider ${agent.provider} not found in the database.`);
-  }
-
   const user = await db.from("users").where({ id: data.user }).first();
 
   if (!user) {
@@ -1362,7 +1355,6 @@ export const validateWorkflowPayload = async (
 
   return {
     agent,
-    provider,
     user,
     workflow,
     variables: data.inputs,
@@ -1372,10 +1364,8 @@ export const validateWorkflowPayload = async (
 
 const validateEvalPayload = async (
   data: BullMqJobData,
-  providers: ExuluProvider[],
 ): Promise<{
   agent: ExuluAgent;
-  provider: ExuluProvider;
   user: User;
   testCase: TestCase;
   evalRun: EvalRun;
@@ -1419,12 +1409,6 @@ const validateEvalPayload = async (
     throw new Error(`Agent ${evalRun.agent_id} not found in the database.`);
   }
 
-  const provider = providers.find((a) => a.id === agent.provider);
-
-  if (!provider) {
-    throw new Error(`Provider ${agent.provider} not found in the database.`);
-  }
-
   const user = await db.from("users").where({ id: data.user }).first();
 
   if (!user) {
@@ -1439,7 +1423,6 @@ const validateEvalPayload = async (
 
   return {
     agent,
-    provider,
     user,
     testCase,
     evalRun,
@@ -1503,9 +1486,7 @@ const pollJobResult = async ({
 };
 
 export const processUiMessagesFlow = async ({
-  providers,
   agent,
-  provider,
   inputMessages,
   contexts,
   user,
@@ -1517,9 +1498,7 @@ export const processUiMessagesFlow = async ({
   resumeFromIndex,
   respectToolApprovals,
 }: {
-  providers: ExuluProvider[];
   agent: ExuluAgent;
-  provider: ExuluProvider;
   inputMessages: UIMessage[];
   contexts: ExuluContext[];
   user: User;
@@ -1579,7 +1558,6 @@ export const processUiMessagesFlow = async ({
     tools,
     contexts,
     disabledTools,
-    providers,
     user,
   );
 
@@ -1597,11 +1575,10 @@ export const processUiMessagesFlow = async ({
   const resolved = await resolveModel({
     modelId: agent.model,
     user,
-    providers,
     agent: agent,
     routine,
   });
-  const providerapikey = resolved.apiKey;
+
   const resolvedLanguageModel = resolved.languageModel;
 
   // Remove placeholder agent response before sending
@@ -1698,7 +1675,7 @@ export const processUiMessagesFlow = async ({
           const startTime = Date.now();
 
           try {
-            const result = await provider.generateStream({
+            const result = await generateStream({
               contexts,
               agent: agent,
               user,
@@ -1714,7 +1691,6 @@ export const processUiMessagesFlow = async ({
               currentTools: enabledTools,
               allExuluTools: tools,
               languageModel: resolvedLanguageModel,
-              providerapikey,
               toolConfigs: agent.tools,
               exuluConfig: config,
             });
@@ -1737,6 +1713,16 @@ export const processUiMessagesFlow = async ({
               originalMessages: result.originalMessages,
               sendReasoning: true,
               sendSources: true,
+              // Give each assistant message a real unique id (matches the live
+              // chat path in routes.ts). Without this the SDK assigns id "",
+              // and saveChat's global message_id upsert collapses every empty-id
+              // message onto one frozen-createdAt row — which sorts the tool
+              // approval to the top of the transcript and leaves it out of the
+              // last-message slot the approval handler acts on (buttons inert).
+              generateMessageId: createIdGenerator({
+                prefix: "msg_",
+                size: 16,
+              }),
               onError: (error) => {
                 console.error("[EXULU] Ui message stream error.", error);
                 reject(new Error(error instanceof Error ? error.message : String(error)));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {