@exulu/backend 1.66.0 → 1.68.0

package/ee/workers.ts

@@ -7,12 +7,12 @@ import { ExuluStorage } from "@SRC/exulu/storage.ts";
 import type { ExuluAgent } from "@EXULU_TYPES/models/agent.ts";
 import type { ExuluQueueConfig } from "@EXULU_TYPES/queue-config.ts";
 import { getTableName, type ExuluContext } from "@SRC/exulu/context.ts";
-import type { ExuluReranker } from "@SRC/exulu/reranker.ts";
 import type { ExuluEval } from "@SRC/exulu/evals.ts";
 import type { ExuluTool } from "@SRC/exulu/tool.ts";
 import { resolveModel } from "@SRC/exulu/resolve-model.ts";
 import { postgresClient } from "@SRC/postgres/client";
 import type { BullMqJobData } from "@EE/queues/decorator.ts";
+import { maybePruneJobResults } from "@EE/queues/prune-job-results.ts";
 import { type Tracer } from "@opentelemetry/api";
 import { v4 as uuidv4 } from "uuid";
 import { type UIMessage } from "ai";
@@ -114,7 +114,6 @@ export const createWorkers = async (
   queues: ExuluQueueConfig[],
   config: ExuluConfig,
   contexts: ExuluContext[],
-  rerankers: ExuluReranker[],
   evals: ExuluEval[],
   tools: ExuluTool[],
   tracer?: Tracer,
@@ -275,13 +274,7 @@ export const createWorkers = async (
 
               const label = `embedder-${bullmqJob.name}`;
 
-              await db.from("job_results").insert({
-                job_id: bullmqJob.id,
-                label: label,
-                state: await bullmqJob.getState(),
-                result: null,
-                metadata: {},
-              });
+              await upsertJobStart(db, bullmqJob, label, "embedder");
 
               const context = contexts.find((context) => context.id === data.context);
 
@@ -289,14 +282,8 @@ export const createWorkers = async (
                 throw new Error(`Context ${data.context} not found in the registry.`);
               }
 
-              if (!data.embedder) {
-                throw new Error(`No embedder set for embedder job.`);
-              }
-
-              const embedder = contexts.find((context) => context.embedder?.id === data.embedder);
-
-              if (!embedder) {
-                throw new Error(`Embedder ${data.embedder} not found in the registry.`);
+              if (!context.embedder) {
+                throw new Error(`No embedder configured for context ${data.context}.`);
               }
 
               const result = await context.createAndUpsertEmbeddings(
@@ -304,7 +291,7 @@ export const createWorkers = async (
                 config,
                 data.user,
                 {
-                  label: embedder.name,
+                  label: context.embedder.model,
                   trigger: data.trigger,
                 },
                 data.role,
@@ -331,13 +318,7 @@ export const createWorkers = async (
 
               const label = `processor-${bullmqJob.name}`;
 
-              await db.from("job_results").insert({
-                job_id: bullmqJob.id,
-                label: label,
-                state: await bullmqJob.getState(),
-                result: null,
-                metadata: {},
-              });
+              await upsertJobStart(db, bullmqJob, label, "processor");
 
               const context = contexts.find((context) => context.id === data.context);
 
@@ -502,6 +483,7 @@ export const createWorkers = async (
                 agent,
                 provider,
                 user,
+                workflow,
                 messages: inputMessages,
               } = await validateWorkflowPayload(data, providers);
 
@@ -530,11 +512,12 @@ export const createWorkers = async (
                       provider,
                       inputMessages,
                       contexts,
-                      rerankers,
                       user,
                       tools,
                       config,
                       variables: data.inputs,
+                      // Tag LLM spend to this routine (cron + ad-hoc share this path).
+                      routine: { id: workflow.id, name: workflow.name },
                     });
                     resolve(messages);
                     break;
@@ -631,7 +614,6 @@ export const createWorkers = async (
                       provider,
                       inputMessages,
                       contexts,
-                      rerankers,
                       user,
                       tools,
                       config,
@@ -1021,6 +1003,9 @@ export const createWorkers = async (
             result: returnvalue.result != null ? JSON.stringify(returnvalue.result) : null,
             metadata: returnvalue.metadata != null ? JSON.stringify(returnvalue.metadata) : null,
           });
+
+        // Cap the table as rows become terminal (every Nth, idempotent).
+        void maybePruneJobResults(db);
       },
     );
 
@@ -1034,6 +1019,9 @@ export const createWorkers = async (
           state: JOB_STATUS_ENUM.failed,
           error,
         });
+
+        // Cap the table as rows become terminal (every Nth, idempotent).
+        void maybePruneJobResults(db);
         return;
       }
       console.error(
@@ -1326,22 +1314,29 @@ export const processUiMessagesFlow = async ({
   provider,
   inputMessages,
   contexts,
-  rerankers,
   user,
   tools,
   config,
   variables,
+  routine,
 }: {
   providers: ExuluProvider[];
   agent: ExuluAgent;
   provider: ExuluProvider;
   inputMessages: UIMessage[];
   contexts: ExuluContext[];
-  rerankers: ExuluReranker[];
   user: User;
   tools: ExuluTool[];
   config: ExuluConfig;
   variables?: Record<string, any>;
+  /**
+   * Set when this flow is invoked from a workflow_template run (one-shot via
+   * runWorkflow or cron via upsertWorkflowSchedule). Forwarded to resolveModel
+   * so buildTags() emits routine_id_/routine_name_ alongside user/agent tags
+   * for /analytics + /admin/budgets attribution. /chat and /openai-gateway
+   * callers leave this undefined — they have no routine context.
+   */
+  routine?: { id: string; name: string };
 }): Promise<{
   messages: UIMessage[];
   metadata: {
@@ -1369,7 +1364,6 @@ export const processUiMessagesFlow = async ({
     agent,
     tools,
     contexts,
-    rerankers,
     disabledTools,
     providers,
     user,
@@ -1390,7 +1384,8 @@ export const processUiMessagesFlow = async ({
     modelId: agent.model,
     user,
     providers,
-    agent: agent
+    agent: agent,
+    routine,
   });
   const providerapikey = resolved.apiKey;
   const resolvedLanguageModel = resolved.languageModel;
@@ -1487,7 +1482,6 @@ export const processUiMessagesFlow = async ({
       try {
         const result = await provider.generateStream({
           contexts,
-          rerankers,
           agent: agent,
           user,
           approvedTools: tools.map((tool) => "tool-" + sanitizeToolName(tool.name)),
@@ -1632,3 +1626,36 @@ function getAverage(arr: number[]): number {
   if (arr.length === 0) return 0; // Handle empty array
   return arr.reduce((a, b) => a + b, 0) / arr.length;
 }
+
+// KB-7: at worker pickup, advance the enqueue-time job_results row (state
+// "waiting", written by the queue decorator) to the live state instead of
+// inserting a duplicate. Falls back to an insert for jobs enqueued before
+// this change. Used by the processor + embedder handlers; the completed/
+// failed worker events then drive the same row to its terminal state.
+async function upsertJobStart(
+  db: any,
+  bullmqJob: { id?: string; data?: any; getState: () => Promise<string> },
+  label: string,
+  fallbackType: string,
+): Promise<void> {
+  const state = await bullmqJob.getState();
+  const rawItem = bullmqJob.data?.item;
+  const itemId =
+    rawItem == null ? null : typeof rawItem === "object" ? (rawItem.id ?? null) : rawItem;
+  const updated = await db
+    .from("job_results")
+    .where({ job_id: bullmqJob.id })
+    .update({ label, state });
+  if (!updated) {
+    await db.from("job_results").insert({
+      job_id: bullmqJob.id,
+      label,
+      state,
+      result: null,
+      metadata: {},
+      type: bullmqJob.data?.type ?? fallbackType,
+      item: itemId == null ? null : String(itemId),
+      context: bullmqJob.data?.context ? String(bullmqJob.data.context) : null,
+    });
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.66.0",
+  "version": "1.68.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {