@exulu/backend 1.66.0 → 1.67.0

package/ee/python/requirements.txt

@@ -1,5 +1,9 @@
 docling
-transformers
+# transformers <5: the 5.x line requires huggingface_hub>=1.0, which removed the
+# `use_auth_token` kwarg that pyannote.audio 3.x still passes to hf_hub_download()
+# (→ "unexpected keyword argument 'use_auth_token'", diarization silently
+# disabled). whisperx only needs transformers>=4.48, so the 4.x line is fine.
+transformers>=4.48,<5
 pyinstaller
 docling-hierarchical-pdf
 defusedxml
@@ -17,6 +21,9 @@ torchaudio==2.5.1
 torchvision==0.20.1
 whisperx>=3.4.0
 pyannote.audio>=3.3.0
+# Belt-and-suspenders: keep huggingface_hub on the 0.x line so pyannote 3.x's
+# `use_auth_token=` calls keep working (1.x removed that kwarg → diarization off).
+huggingface_hub<1.0
 fastapi
 uvicorn
 python-multipart

package/ee/python/setup.sh

@@ -253,46 +253,6 @@ if [ -n "$LITELLM_PROXY_DIR" ] && [ -f "$LITELLM_PROXY_DIR/schema.prisma" ]; the
         || print_warning "Prisma generate failed; LiteLLM database mode (database_url in config.litellm.yaml) may not work until you run 'cd $LITELLM_PROXY_DIR && PATH=$VENV_DIR/bin:\$PATH $VENV_DIR/bin/prisma generate'"
 fi
 
-# Step 6.6: Install the Hermes Agent harness (advanced agent mode).
-# Opt-in via ENABLE_HERMES_AGENT=true. Hermes is NOT a pip package — it ships
-# as a standalone binary via Nous Research's official installer (lands in
-# ~/.local/bin/hermes). We only install if it's not already present so re-runs
-# are fast, and we never fail the whole setup if the install fails (advanced
-# mode is optional; the operator can install it manually and retry).
-if [ "${ENABLE_HERMES_AGENT}" = "true" ]; then
-    echo ""
-    echo "Step 6.6: Installing Hermes Agent harness (ENABLE_HERMES_AGENT=true)..."
-    if command -v hermes &> /dev/null || [ -x "$HOME/.local/bin/hermes" ]; then
-        HERMES_VERSION=$( (command -v hermes &> /dev/null && hermes --version 2>/dev/null) || "$HOME/.local/bin/hermes" --version 2>/dev/null || echo "unknown")
-        print_success "Hermes already installed ($HERMES_VERSION) — skipping installer"
-    else
-        print_info "Running Hermes official installer..."
-        if curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh | bash; then
-            print_success "Hermes Agent installed (binary at ~/.local/bin/hermes)"
-        else
-            print_warning "Hermes installer failed. Advanced agent mode will be unavailable until 'hermes' is on PATH. Install manually: https://hermes-agent.nousresearch.com/docs/getting-started/installation"
-        fi
-    fi
-
-    # Pre-pull the docker terminal-backend image so the first agent request
-    # isn't blocked on a cold image pull (~minute). Only when the backend is
-    # docker (the default) and docker is available; non-fatal otherwise.
-    HERMES_BACKEND="${HERMES_TERMINAL_BACKEND:-docker}"
-    if [ "${HERMES_BACKEND}" = "docker" ]; then
-        HERMES_IMG="${HERMES_DOCKER_IMAGE:-nikolaik/python-nodejs:python3.11-nodejs20}"
-        if command -v docker &> /dev/null; then
-            print_info "Pre-pulling Hermes docker backend image: ${HERMES_IMG}..."
-            if docker pull "${HERMES_IMG}" > /dev/null 2>&1; then
-                print_success "Docker backend image ready (${HERMES_IMG})"
-            else
-                print_warning "Could not pre-pull ${HERMES_IMG}; the first advanced-mode request will pull it (slower)."
-            fi
-        else
-            print_warning "Docker not found, but HERMES_TERMINAL_BACKEND=docker. Install Docker, or set HERMES_TERMINAL_BACKEND=local (unsandboxed)."
-        fi
-    fi
-fi
-
 # Step 7: Validate installation
 echo ""
 echo "Step 7: Validating installation..."
@@ -309,15 +269,6 @@ $PYTHON_CMD -c "import whisperx" 2>/dev/null && print_success "whisperx imported
 $PYTHON_CMD -c "import pyannote.audio" 2>/dev/null && print_success "pyannote.audio imported successfully" || print_warning "pyannote.audio not importable (diarization will be disabled even with HF_AUTH_TOKEN)"
 $PYTHON_CMD -c "import fastapi, uvicorn" 2>/dev/null && print_success "fastapi/uvicorn imported successfully" || print_warning "fastapi/uvicorn not importable (transcription server will not start)"
 
-# Hermes Agent binary check (advanced agent mode) — only when opted in.
-if [ "${ENABLE_HERMES_AGENT}" = "true" ]; then
-    if command -v hermes &> /dev/null || [ -x "$HOME/.local/bin/hermes" ]; then
-        print_success "hermes binary available (advanced agent mode ready)"
-    else
-        print_warning "hermes binary not found (advanced agent mode will be unavailable)"
-    fi
-fi
-
 # Step 8: Display summary
 echo ""
 echo -e "${GREEN}========================================${NC}"

package/ee/queues/decorator.ts

@@ -2,6 +2,8 @@ import { Queue } from "bullmq";
 import { v4 as uuidv4 } from "uuid";
 import type { UIMessage } from "ai";
 import type { STATISTICS_LABELS } from "@EXULU_TYPES/statistics";
+import { postgresClient } from "@SRC/postgres/client";
+import { maybePruneJobResults } from "./prune-job-results";
 
 type ExuluJobType = "embedder" | "workflow" | "eval" | "processor";
 
@@ -120,6 +122,40 @@ export const bullmqDecorator = async ({
   };
 
   const redisId = uuidv4();
+
+  // Knowledge V2 (KB-7): record the job in job_results at ENQUEUE time (state
+  // "waiting") for processor/embedder jobs, so the item detail page can detect
+  // jobs that are queued-but-not-yet-started (which it couldn't if the row was
+  // only written at worker pickup). Inserted BEFORE queue.add so the row is
+  // guaranteed present before any worker can grab the job (no insert/update
+  // race). The worker-start update + completed/failed handlers drive the row
+  // through active → completed/failed, all keyed by this job_id.
+  if ((type === "processor" || type === "embedder") && context) {
+    try {
+      const { db } = await postgresClient();
+      const itemId =
+        item == null
+          ? null
+          : typeof item === "object"
+            ? ((item as { id?: unknown }).id ?? null)
+            : item;
+      await db.from("job_results").insert({
+        job_id: redisId,
+        label,
+        state: "waiting",
+        type,
+        item: itemId == null ? null : String(itemId),
+        context: String(context),
+        result: null,
+        metadata: {},
+      });
+      // Bound the table: every Nth added row, prune the oldest terminal rows.
+      void maybePruneJobResults(db);
+    } catch (err) {
+      console.error("[EXULU] enqueue job_results insert failed", err);
+    }
+  }
+
   const job = await queue.add(`${embedder || workflow || processor || evaluation}`, jobData, {
     jobId: redisId,
     // Setting it to 3 as a sensible default, as

package/ee/queues/prune-job-results.ts

@@ -0,0 +1,55 @@
+/**
+ * Periodic job_results cap (knowledge V2 KB-7 follow-up).
+ *
+ * We now write a job_results row at enqueue time, so the table grows faster.
+ * To bound it, every PRUNE_EVERY-th call we delete the oldest terminal rows
+ * (state failed/completed) beyond the newest MAX_TERMINAL — keeping a rolling
+ * window of recent finished jobs. Waiting/active/delayed rows are never
+ * pruned (they're still live).
+ *
+ * The counter is per-process (the API process counts enqueues; the worker
+ * process counts completions) — that's fine: the prune is idempotent, so it
+ * doesn't matter which process triggers it. A `pruning` guard avoids
+ * overlapping runs.
+ */
+
+const MAX_TERMINAL = 10_000;
+const PRUNE_EVERY = 100;
+const TERMINAL_STATES = ["failed", "completed"];
+
+let sinceLastPrune = 0;
+let pruning = false;
+
+export async function maybePruneJobResults(db: any): Promise<void> {
+  sinceLastPrune += 1;
+  if (sinceLastPrune < PRUNE_EVERY || pruning) return;
+  sinceLastPrune = 0;
+  pruning = true;
+  try {
+    // The (MAX_TERMINAL+1)-th newest terminal row marks the boundary; delete it
+    // and everything older. Dialect-agnostic (knex offset/limit) so it works on
+    // both Postgres and MySQL.
+    const boundary = await db("job_results")
+      .whereIn("state", TERMINAL_STATES)
+      .orderBy("createdAt", "desc")
+      .offset(MAX_TERMINAL)
+      .limit(1)
+      .first();
+
+    if (boundary?.createdAt) {
+      const deleted = await db("job_results")
+        .whereIn("state", TERMINAL_STATES)
+        .where("createdAt", "<=", boundary.createdAt)
+        .del();
+      if (deleted) {
+        console.log(
+          `[EXULU] pruned ${deleted} terminal job_results rows (cap ${MAX_TERMINAL}).`,
+        );
+      }
+    }
+  } catch (err) {
+    console.error("[EXULU] job_results prune failed", err);
+  } finally {
+    pruning = false;
+  }
+}

package/ee/schemas.ts

@@ -241,6 +241,25 @@ export const jobResultsSchema: ExuluTableDefinition = {
             name: "metadata",
             type: "json",
         },
+        // Knowledge V2 (KB-7): per-item pipeline tracking. Written at ENQUEUE
+        // time (state "waiting") by the queue decorator so the item page can
+        // detect waiting jobs — not only worker-started ones. `type` is the
+        // job kind (processor/embedder/...); item + context indexed for the
+        // item-page query.
+        {
+            name: "item",
+            type: "text",
+            index: true,
+        },
+        {
+            name: "context",
+            type: "text",
+            index: true,
+        },
+        {
+            name: "type",
+            type: "text",
+        },
     ],
 };
 

package/ee/workers.ts

@@ -13,6 +13,7 @@ import type { ExuluTool } from "@SRC/exulu/tool.ts";
 import { resolveModel } from "@SRC/exulu/resolve-model.ts";
 import { postgresClient } from "@SRC/postgres/client";
 import type { BullMqJobData } from "@EE/queues/decorator.ts";
+import { maybePruneJobResults } from "@EE/queues/prune-job-results.ts";
 import { type Tracer } from "@opentelemetry/api";
 import { v4 as uuidv4 } from "uuid";
 import { type UIMessage } from "ai";
@@ -275,13 +276,7 @@ export const createWorkers = async (
 
               const label = `embedder-${bullmqJob.name}`;
 
-              await db.from("job_results").insert({
-                job_id: bullmqJob.id,
-                label: label,
-                state: await bullmqJob.getState(),
-                result: null,
-                metadata: {},
-              });
+              await upsertJobStart(db, bullmqJob, label, "embedder");
 
               const context = contexts.find((context) => context.id === data.context);
 
@@ -331,13 +326,7 @@ export const createWorkers = async (
 
               const label = `processor-${bullmqJob.name}`;
 
-              await db.from("job_results").insert({
-                job_id: bullmqJob.id,
-                label: label,
-                state: await bullmqJob.getState(),
-                result: null,
-                metadata: {},
-              });
+              await upsertJobStart(db, bullmqJob, label, "processor");
 
               const context = contexts.find((context) => context.id === data.context);
 
@@ -502,6 +491,7 @@ export const createWorkers = async (
                 agent,
                 provider,
                 user,
+                workflow,
                 messages: inputMessages,
               } = await validateWorkflowPayload(data, providers);
 
@@ -535,6 +525,8 @@ export const createWorkers = async (
                       tools,
                       config,
                       variables: data.inputs,
+                      // Tag LLM spend to this routine (cron + ad-hoc share this path).
+                      routine: { id: workflow.id, name: workflow.name },
                     });
                     resolve(messages);
                     break;
@@ -1021,6 +1013,9 @@ export const createWorkers = async (
             result: returnvalue.result != null ? JSON.stringify(returnvalue.result) : null,
             metadata: returnvalue.metadata != null ? JSON.stringify(returnvalue.metadata) : null,
           });
+
+        // Cap the table as rows become terminal (every Nth, idempotent).
+        void maybePruneJobResults(db);
       },
     );
 
@@ -1034,6 +1029,9 @@ export const createWorkers = async (
           state: JOB_STATUS_ENUM.failed,
           error,
         });
+
+        // Cap the table as rows become terminal (every Nth, idempotent).
+        void maybePruneJobResults(db);
         return;
       }
       console.error(
@@ -1331,6 +1329,7 @@ export const processUiMessagesFlow = async ({
   tools,
   config,
   variables,
+  routine,
 }: {
   providers: ExuluProvider[];
   agent: ExuluAgent;
@@ -1342,6 +1341,14 @@ export const processUiMessagesFlow = async ({
   tools: ExuluTool[];
   config: ExuluConfig;
   variables?: Record<string, any>;
+  /**
+   * Set when this flow is invoked from a workflow_template run (one-shot via
+   * runWorkflow or cron via upsertWorkflowSchedule). Forwarded to resolveModel
+   * so buildTags() emits routine_id_/routine_name_ alongside user/agent tags
+   * for /analytics + /admin/budgets attribution. /chat and /openai-gateway
+   * callers leave this undefined — they have no routine context.
+   */
+  routine?: { id: string; name: string };
 }): Promise<{
   messages: UIMessage[];
   metadata: {
@@ -1390,7 +1397,8 @@ export const processUiMessagesFlow = async ({
     modelId: agent.model,
     user,
     providers,
-    agent: agent
+    agent: agent,
+    routine,
   });
   const providerapikey = resolved.apiKey;
   const resolvedLanguageModel = resolved.languageModel;
@@ -1632,3 +1640,36 @@ function getAverage(arr: number[]): number {
   if (arr.length === 0) return 0; // Handle empty array
   return arr.reduce((a, b) => a + b, 0) / arr.length;
 }
+
+// KB-7: at worker pickup, advance the enqueue-time job_results row (state
+// "waiting", written by the queue decorator) to the live state instead of
+// inserting a duplicate. Falls back to an insert for jobs enqueued before
+// this change. Used by the processor + embedder handlers; the completed/
+// failed worker events then drive the same row to its terminal state.
+async function upsertJobStart(
+  db: any,
+  bullmqJob: { id?: string; data?: any; getState: () => Promise<string> },
+  label: string,
+  fallbackType: string,
+): Promise<void> {
+  const state = await bullmqJob.getState();
+  const rawItem = bullmqJob.data?.item;
+  const itemId =
+    rawItem == null ? null : typeof rawItem === "object" ? (rawItem.id ?? null) : rawItem;
+  const updated = await db
+    .from("job_results")
+    .where({ job_id: bullmqJob.id })
+    .update({ label, state });
+  if (!updated) {
+    await db.from("job_results").insert({
+      job_id: bullmqJob.id,
+      label,
+      state,
+      result: null,
+      metadata: {},
+      type: bullmqJob.data?.type ?? fallbackType,
+      item: itemId == null ? null : String(itemId),
+      context: bullmqJob.data?.context ? String(bullmqJob.data.context) : null,
+    });
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.66.0",
+  "version": "1.67.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {