@exulu/backend 1.15.0 → 1.17.0

package/dist/index.js

@@ -175,87 +175,48 @@ import pgvector2 from "pgvector/knex";
 import "bullmq";
 import { v4 as uuidv4 } from "uuid";
 var bullmqDecorator = async ({
+  queue,
   label,
-  type,
-  workflow,
   embedder,
   inputs,
-  queue,
   user,
-  agent,
-  session,
-  configuration,
-  updater,
-  context,
-  steps,
-  source,
-  documents,
+  role,
   trigger,
-  item
+  workflow,
+  item,
+  context
 }) => {
+  if (embedder && workflow) {
+    throw new Error("Cannot have both embedder and workflow in the same job.");
+  }
+  if (workflow && item) {
+    throw new Error("Cannot have both workflow and item in the same job.");
+  }
+  let type = "embedder";
+  if (workflow) {
+    type = "workflow";
+  }
   const redisId = uuidv4();
   const job = await queue.add(
     `${embedder || workflow}`,
     {
-      type: `${type}`,
+      label,
       ...embedder && { embedder },
-      ...workflow && { workflow },
-      ...configuration && { configuration },
-      ...updater && { updater },
-      ...context && { context },
-      ...source && { source },
-      ...documents && { documents },
-      ...steps && { steps },
+      type: `${type}`,
+      inputs,
+      ...user && { user },
+      ...role && { role },
       ...trigger && { trigger },
+      ...workflow && { workflow },
       ...item && { item },
-      agent,
-      user,
-      inputs,
-      label,
-      session
+      ...context && { context }
     },
     {
       jobId: redisId
     }
   );
-  const { db: db3 } = await postgresClient();
-  const now = /* @__PURE__ */ new Date();
-  console.log("[EXULU] scheduling new job", inputs);
-  const insertData = {
-    name: `${label}`,
-    redis: job.id,
-    status: "waiting",
-    type,
-    inputs,
-    agent,
-    item,
-    createdAt: now,
-    updatedAt: now,
-    user,
-    session,
-    ...embedder && { embedder },
-    ...workflow && { workflow },
-    ...configuration && { configuration },
-    ...steps && { steps },
-    ...updater && { updater },
-    ...context && { context },
-    ...source && { source },
-    ...documents && { documents: documents.map((doc2) => doc2.id) },
-    ...trigger && { trigger }
-  };
-  await db3("jobs").insert(insertData).onConflict("redis").merge({
-    ...insertData,
-    updatedAt: now
-    // Only updatedAt changes on updates
-  });
-  const doc = await db3.from("jobs").where({ redis: job.id }).first();
-  if (!doc?.id) {
-    throw new Error("Failed to get job ID after insert/update");
-  }
-  console.log("[EXULU] created job", doc?.id);
   return {
     ...job,
-    id: doc?.id,
     redis: job.id
   };
 };
@@ -427,6 +388,7 @@ var ExuluEvalUtils = {
 // src/registry/classes.ts
 import CryptoJS from "crypto-js";
 import "express";
+import "@opentelemetry/api";
 function sanitizeToolName(name) {
   if (typeof name !== "string") return "";
   let sanitized = name.replace(/[^a-zA-Z0-9_-]+/g, "_");
@@ -436,7 +398,7 @@ function sanitizeToolName(name) {
   }
   return sanitized;
 }
-var convertToolsArrayToObject = (tools, configs, providerApiKey) => {
+var convertToolsArrayToObject = (tools, configs, providerApiKey, user, role) => {
   if (!tools) return {};
   const sanitizedTools = tools ? tools.map((tool2) => ({
     ...tool2,
@@ -471,6 +433,8 @@ var convertToolsArrayToObject = (tools, configs, providerApiKey) => {
               // is available, after we added the .value property
               // by hydrating it from the variables table.
               providerApiKey,
+              user,
+              role,
               config: config ? config.config.reduce((acc, curr) => {
                 acc[curr.name] = curr.value;
                 return acc;
@@ -576,10 +540,12 @@ var ExuluAgent = class {
       }),
       description: `A function that calls an AI agent named: ${this.name}. The agent does the following: ${this.description}.`,
       config: [],
-      execute: async ({ prompt, config, providerApiKey }) => {
+      execute: async ({ prompt, config, providerApiKey, user, role }) => {
         return await this.generateSync({
           prompt,
           providerApiKey,
+          user,
+          role,
           statistics: {
             label: "",
             trigger: "tool"
@@ -588,7 +554,7 @@ var ExuluAgent = class {
       }
     });
   };
-  generateSync = async ({ prompt, user, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateSync = async ({ prompt, user, role, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
@@ -598,6 +564,9 @@ var ExuluAgent = class {
     if (prompt && message) {
       throw new Error("Message and prompt cannot be provided at the same time.");
     }
+    if (!prompt && !message) {
+      throw new Error("Prompt or message is required for generating.");
+    }
     const model = this.model.create({
       apiKey: providerApiKey
     });
@@ -617,28 +586,54 @@ var ExuluAgent = class {
     }
     console.log("[EXULU] Model provider key", providerApiKey);
     console.log("[EXULU] Tool configs", toolConfigs);
-    const { text } = await generateText({
-      model,
-      // Should be a LanguageModelV1
-      system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
-      messages: messages ? convertToModelMessages(messages) : void 0,
-      prompt,
-      maxRetries: 2,
-      tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey),
-      stopWhen: [stepCountIs(5)]
-    });
-    if (statistics) {
-      await updateStatistic({
-        name: "count",
-        label: statistics.label,
-        type: STATISTICS_TYPE_ENUM.AGENT_RUN,
-        trigger: statistics.trigger,
-        count: 1
+    if (prompt) {
+      const { text } = await generateText({
+        model,
+        // Should be a LanguageModelV1
+        system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
+        prompt,
+        maxRetries: 2,
+        tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
+        stopWhen: [stepCountIs(5)]
+      });
+      if (statistics) {
+        await updateStatistic({
+          name: "count",
+          label: statistics.label,
+          type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+          trigger: statistics.trigger,
+          count: 1,
+          user,
+          role
+        });
+      }
+      return text;
+    }
+    if (messages) {
+      const { text } = await generateText({
+        model,
+        // Should be a LanguageModelV1
+        system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
+        messages: convertToModelMessages(messages),
+        maxRetries: 2,
+        tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
+        stopWhen: [stepCountIs(5)]
       });
+      if (statistics) {
+        await updateStatistic({
+          name: "count",
+          label: statistics.label,
+          type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+          trigger: statistics.trigger,
+          count: 1,
+          user,
+          role
+        });
+      }
+      return text;
     }
-    return text;
   };
-  generateStream = async ({ express: express3, user, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
+  generateStream = async ({ express: express3, user, role, session, message, tools, statistics, toolConfigs, providerApiKey }) => {
     if (!this.model) {
       throw new Error("Model is required for streaming.");
     }
@@ -668,10 +663,10 @@ var ExuluAgent = class {
     const result = streamText({
       model,
       // Should be a LanguageModelV1
-      messages: messages ? convertToModelMessages(messages) : void 0,
+      messages: convertToModelMessages(messages),
       system: "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.",
       maxRetries: 2,
-      tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey),
+      tools: convertToolsArrayToObject(tools, toolConfigs, providerApiKey, user, role),
       onError: (error) => console.error("[EXULU] chat stream error.", error),
       stopWhen: [stepCountIs(5)]
     });
@@ -699,7 +694,9 @@ var ExuluAgent = class {
             label: statistics.label,
             type: STATISTICS_TYPE_ENUM.AGENT_RUN,
             trigger: statistics.trigger,
-            count: 1
+            count: 1,
+            user,
+            role
           });
         }
       }
@@ -743,14 +740,16 @@ var ExuluEmbedder = class {
     this.queue = queue;
     this.generateEmbeddings = generateEmbeddings;
   }
-  async generateFromQuery(query, statistics) {
+  async generateFromQuery(query, statistics, user, role) {
     if (statistics) {
       await updateStatistic({
         name: "count",
         label: statistics.label,
         type: STATISTICS_TYPE_ENUM.EMBEDDER_GENERATE,
         trigger: statistics.trigger,
-        count: 1
+        count: 1,
+        user,
+        role
       });
     }
     return await this.generateEmbeddings({
@@ -763,14 +762,16 @@ var ExuluEmbedder = class {
       }]
     });
   }
-  async generateFromDocument(input, statistics) {
+  async generateFromDocument(input, statistics, user, role) {
     if (statistics) {
       await updateStatistic({
         name: "count",
         label: statistics.label,
         type: STATISTICS_TYPE_ENUM.EMBEDDER_GENERATE,
         trigger: statistics.trigger,
-        count: 1
+        count: 1,
+        user,
+        role
       });
     }
     if (!this.chunker) {
@@ -1024,8 +1025,36 @@ var ExuluContext = class {
     const tableExists = await db3.schema.hasTable(this.getTableName());
     return tableExists;
   };
-  async updateItem(user, id, item) {
-    if (!id) {
+  createAndUpsertEmbeddings = async (item, user, statistics, role) => {
+    const { db: db3 } = await postgresClient();
+    const { id: source, chunks } = await this.embedder.generateFromDocument({
+      ...item,
+      id: item.id
+    }, {
+      label: statistics.label || this.name,
+      trigger: statistics.trigger || "agent"
+    }, user, role);
+    const exists = await db3.schema.hasTable(this.getChunksTableName());
+    if (!exists) {
+      await this.createChunksTable();
+    }
+    await db3.from(this.getChunksTableName()).where({ source }).delete();
+    await db3.from(this.getChunksTableName()).insert(chunks.map((chunk) => ({
+      source,
+      content: chunk.content,
+      chunk_index: chunk.index,
+      embedding: pgvector2.toSql(chunk.vector)
+    })));
+    await db3.from(this.getTableName()).where({ id: item.id }).update({
+      embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
+    }).returning("id");
+    return {
+      id: item.id,
+      chunks: chunks?.length || 0
+    };
+  };
+  async updateItem(user, item, role, trigger) {
+    if (!item.id) {
       throw new Error("Id is required for updating an item.");
     }
     const { db: db3 } = await postgresClient();
@@ -1042,51 +1071,37 @@ var ExuluContext = class {
     delete item.created_at;
     delete item.upsert;
     item.updated_at = db3.fn.now();
-    const result = await db3.from(this.getTableName()).where({ id }).update(item).returning("id");
+    const result = await db3.from(this.getTableName()).where({ id: item.id }).update(item).returning("id");
     if (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always") {
       if (this.embedder.queue?.name) {
         console.log("[EXULU] embedder is in queue mode, scheduling job.");
         const job = await bullmqDecorator({
-          label: `Job running '${this.embedder.name}' for '${item.name} (${item.id}).'`,
+          label: `${this.embedder.name}`,
           embedder: this.embedder.id,
-          type: "embedder",
+          context: this.id,
           inputs: item,
+          item: item.id,
           queue: this.embedder.queue,
-          user
+          user,
+          role,
+          trigger: trigger || "agent"
         });
         return {
           id: result[0].id,
           job: job.id
         };
       }
-      const { id: source, chunks } = await this.embedder.generateFromDocument({
-        ...item,
-        id
-      }, {
-        label: this.name,
-        trigger: "agent"
-      });
-      const exists = await db3.schema.hasTable(this.getChunksTableName());
-      if (!exists) {
-        await this.createChunksTable();
-      }
-      await db3.from(this.getChunksTableName()).where({ source }).delete();
-      await db3.from(this.getChunksTableName()).insert(chunks.map((chunk) => ({
-        source,
-        content: chunk.content,
-        chunk_index: chunk.index,
-        embedding: pgvector2.toSql(chunk.vector)
-      })));
-      await db3.from(this.getTableName()).where({ id }).update({
-        embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
-      }).returning("id");
+      await this.createAndUpsertEmbeddings(item, user, {
+        label: this.embedder.name,
+        trigger: trigger || "agent"
+      }, role);
     }
     return {
       id: result[0].id,
       job: void 0
     };
   }
-  async insertItem(user, item, upsert = false) {
+  async insertItem(user, item, upsert = false, role, trigger) {
     if (!item.name) {
       throw new Error("Name field is required.");
     }
@@ -1097,14 +1112,20 @@ var ExuluContext = class {
         throw new Error("Item with external id " + item.external_id + " already exists.");
       }
       if (existingItem && upsert) {
-        await this.updateItem(user, existingItem.id, item);
+        await this.updateItem(user, {
+          ...item,
+          id: existingItem.id
+        }, role, trigger);
         return existingItem.id;
       }
     }
     if (upsert && item.id) {
       const existingItem = await db3.from(this.getTableName()).where({ id: item.id }).first();
       if (existingItem && upsert) {
-        await this.updateItem(user, existingItem.id, item);
+        await this.updateItem(user, {
+          ...item,
+          id: existingItem.id
+        }, role, trigger);
         return existingItem.id;
       }
     }
@@ -1129,12 +1150,15 @@ var ExuluContext = class {
       if (this.embedder.queue?.name) {
         console.log("[EXULU] embedder is in queue mode, scheduling job.");
         const job = await bullmqDecorator({
-          label: `Job running '${this.embedder.name}' for '${item.name} (${item.id}).'`,
+          label: `${this.embedder.name}`,
           embedder: this.embedder.id,
-          type: "embedder",
+          context: this.id,
           inputs: item,
+          item: item.id,
           queue: this.embedder.queue,
-          user
+          user,
+          role,
+          trigger: trigger || "agent"
         });
         return {
           id: result[0].id,
@@ -1142,27 +1166,13 @@ var ExuluContext = class {
         };
       }
       console.log("[EXULU] embedder is not in queue mode, calculating vectors directly.");
-      const { id: source, chunks } = await this.embedder.generateFromDocument({
+      await this.createAndUpsertEmbeddings({
         ...item,
         id: result[0].id
-      }, {
-        label: this.name,
-        trigger: "agent"
-      });
-      const exists = await db3.schema.hasTable(this.getChunksTableName());
-      if (!exists) {
-        await this.createChunksTable();
-      }
-      console.log("[EXULU] Inserting chunks.");
-      await db3.from(this.getChunksTableName()).insert(chunks.map((chunk) => ({
-        source,
-        content: chunk.content,
-        chunk_index: chunk.index,
-        embedding: pgvector2.toSql(chunk.vector)
-      })));
-      await db3.from(this.getTableName()).where({ id: result[0].id }).update({
-        embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
-      }).returning("id");
+      }, user, {
+        label: this.embedder.name,
+        trigger: trigger || "agent"
+      }, role);
     }
     return {
       id: result[0].id,
@@ -1176,6 +1186,8 @@ var ExuluContext = class {
     order,
     page,
     name,
+    user,
+    role,
     archived,
     query,
     method
@@ -1244,7 +1256,9 @@ var ExuluContext = class {
           name: "count",
           label: statistics.label,
           type: STATISTICS_TYPE_ENUM.CONTEXT_RETRIEVE,
-          trigger: statistics.trigger
+          trigger: statistics.trigger,
+          user,
+          role
         });
       }
       if (this.queryRewriter) {
@@ -1260,7 +1274,10 @@ var ExuluContext = class {
       itemsQuery.select(chunksTable + ".chunk_index");
       itemsQuery.select(chunksTable + ".created_at as chunk_created_at");
       itemsQuery.select(chunksTable + ".updated_at as chunk_updated_at");
-      const { chunks } = await this.embedder.generateFromQuery(query);
+      const { chunks } = await this.embedder.generateFromQuery(query, {
+        label: this.name,
+        trigger: "agent"
+      }, user, role);
       if (!chunks?.[0]?.vector) {
         throw new Error("No vector generated for query.");
       }
@@ -1408,11 +1425,13 @@ var ExuluContext = class {
       }),
       config: [],
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
-      execute: async ({ query }) => {
+      execute: async ({ query, user, role }) => {
         return await this.getItems({
           page: 1,
           limit: 10,
           query,
+          user,
+          role,
           statistics: {
             label: this.name,
             trigger: "agent"
@@ -1854,6 +1873,7 @@ import { zerialize } from "zodex";
 
 // src/bullmq/queues.ts
 import { Queue as Queue3 } from "bullmq";
+import { BullMQOtel } from "bullmq-otel";
 var ExuluQueues = class {
   queues;
   constructor() {
@@ -1871,7 +1891,13 @@ var ExuluQueues = class {
       console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.use() ).`);
       throw new Error(`[EXULU] no redis server configured.`);
     }
-    const newQueue = new Queue3(`${name}`, { connection: redisServer });
+    const newQueue = new Queue3(
+      `${name}`,
+      {
+        connection: redisServer,
+        telemetry: new BullMQOtel("simple-guide")
+      }
+    );
     this.queues.push(newQueue);
     return newQueue;
   }
@@ -3658,6 +3684,7 @@ Intelligence Management Platform
 import OpenAI from "openai";
 import fs2 from "fs";
 import { randomUUID } from "crypto";
+import "@opentelemetry/api";
 var REQUEST_SIZE_LIMIT = "50mb";
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -3696,7 +3723,7 @@ var createRecurringJobs = async () => {
   console.table(recurringJobSchedulersLogs);
   return queue;
 };
-var createExpressRoutes = async (app, agents, tools, contexts) => {
+var createExpressRoutes = async (app, logger, agents, tools, contexts, config, tracer) => {
   const routeLogs = [];
   var corsOptions = {
     origin: "*",
@@ -3790,6 +3817,16 @@ var createExpressRoutes = async (app, agents, tools, contexts) => {
     express.json({ limit: REQUEST_SIZE_LIMIT }),
     expressMiddleware(server, {
       context: async ({ req }) => {
+        logger.info("================");
+        logger.info({
+          message: "Incoming Request",
+          method: req.method,
+          path: req.path,
+          requestId: "req-" + Date.now(),
+          ipAddress: req.ip,
+          userAgent: req.get("User-Agent")
+        });
+        logger.info("================");
         const authenticationResult = await requestValidators.authenticate(req);
         if (!authenticationResult.user?.id) {
           throw new Error(authenticationResult.message);
@@ -4176,7 +4213,15 @@ Mood: friendly and intelligent.
     if (!exists) {
       await context.createItemsTable();
     }
-    const result = await context.updateItem(authenticationResult.user.id, req.params.id, req.body);
+    const result = await context.updateItem(
+      authenticationResult.user.id,
+      {
+        ...req.body,
+        id: req.params.id
+      },
+      authenticationResult.user.role?.id,
+      authenticationResult.user.type === "api" ? "api" : "user"
+    );
     res.status(200).json({
       message: "Item updated successfully.",
       id: result
@@ -4223,7 +4268,13 @@ Mood: friendly and intelligent.
         await context.createItemsTable();
       }
       console.log("[EXULU] inserting item", req.body);
-      const result = await context.insertItem(authenticationResult.user.id, req.body, !!req.body.upsert);
+      const result = await context.insertItem(
+        authenticationResult.user.id,
+        req.body,
+        !!req.body.upsert,
+        authenticationResult.user.role?.id,
+        authenticationResult.user.type === "api" ? "api" : "user"
+      );
       console.log("[EXULU] result", result);
       res.status(200).json({
         message: "Item created successfully.",
@@ -4284,6 +4335,8 @@ Mood: friendly and intelligent.
     const result = await context.getItems({
       sort,
       order,
+      user: authenticationResult.user?.id,
+      role: authenticationResult.user?.role?.id,
       page,
       limit,
       archived: req.query.archived === "true",
@@ -4445,6 +4498,8 @@ Mood: friendly and intelligent.
     const items = await context.getItems({
       page: 1,
       // todo add pagination
+      user: authenticationResult.user?.id,
+      role: authenticationResult.user?.role?.id,
       limit: 500
     });
     const csv = Papa.unparse(items);
@@ -4568,7 +4623,7 @@ Mood: friendly and intelligent.
         return;
       }
       console.log("[EXULU] agent tools", agentInstance.tools);
-      const enabledTools = agentInstance.tools.map(({ config, toolId }) => tools.find(({ id }) => id === toolId)).filter(Boolean);
+      const enabledTools = agentInstance.tools.map(({ config: config2, toolId }) => tools.find(({ id }) => id === toolId)).filter(Boolean);
       console.log("[EXULU] enabled tools", enabledTools);
       const variableName = agentInstance.providerApiKey;
       const variable = await db3.from("variables").where({ name: variableName }).first();
@@ -4595,7 +4650,8 @@ Mood: friendly and intelligent.
             res,
             req
           },
-          user: headers.user,
+          user: user?.id,
+          role: user?.role?.id,
           session: headers.session,
           message: req.body.message,
           tools: enabledTools,
@@ -4609,8 +4665,9 @@ Mood: friendly and intelligent.
         return;
       } else {
         const response = await agent.generateSync({
-          user: headers.user,
+          user: user?.id,
           session: headers.session,
+          role: user?.role?.id,
           message: req.body.message,
           tools: enabledTools.map((tool2) => tool2.tool),
           providerApiKey,
@@ -4709,7 +4766,9 @@ Mood: friendly and intelligent.
         label: "Claude Code",
         type: STATISTICS_TYPE_ENUM.AGENT_RUN,
         trigger: "claude-code",
-        count: 1
+        count: 1,
+        user: authenticationResult.user?.id,
+        role: authenticationResult.user.role?.id
       });
       response.headers.forEach((value, key) => {
         res.setHeader(key, value);
@@ -4770,43 +4829,22 @@ import IORedis from "ioredis";
 import { Worker } from "bullmq";
 
 // src/registry/utils.ts
-import "bullmq";
 var bullmq = {
-  validate: (bullmqJob) => {
-    if (!bullmqJob.data) {
-      throw new Error(`Missing job data for job ${bullmqJob.id}.`);
-    }
-    if (!bullmqJob.data.type) {
-      throw new Error(`Missing property "type" in data for job ${bullmqJob.id}.`);
+  validate: (id, data) => {
+    if (!data) {
+      throw new Error(`Missing job data for job ${id}.`);
     }
-    if (!bullmqJob.data.inputs) {
-      throw new Error(`Missing property "inputs" in data for job ${bullmqJob.id}.`);
+    if (!data.type) {
+      throw new Error(`Missing property "type" in data for job ${id}.`);
     }
-    if (bullmqJob.data.type !== "embedder" && bullmqJob.data.type !== "workflow") {
-      throw new Error(`Property "type" in data for job ${bullmqJob.id} must be of value "embedder" or "workflow".`);
+    if (!data.inputs) {
+      throw new Error(`Missing property "inputs" in data for job ${id}.`);
     }
-    if (!bullmqJob.data.workflow && !bullmqJob.data.embedder) {
-      throw new Error(`Property "backend" in data for job ${bullmqJob.id} missing. Job  data: ${JSON.stringify(bullmqJob)}`);
+    if (data.type !== "embedder" && data.type !== "workflow") {
+      throw new Error(`Property "type" in data for job ${id} must be of value "embedder" or "workflow".`);
     }
-  },
-  process: {
-    workflow: async (bullmqJob, exuluJob, workflow, logsDir) => {
-      if (!workflow) {
-        throw new Error(`Workflow function with id: ${bullmqJob.data.backend} not found in registry.`);
-      }
-      console.log("[EXULU] starting workflow with job inputs.", bullmqJob.data.inputs);
-      const logger = new ExuluLogger(exuluJob, logsDir);
-      const output = await workflow.start({
-        job: exuluJob,
-        inputs: bullmqJob.data.inputs,
-        user: bullmqJob.data.user,
-        logger,
-        session: bullmqJob.data.session,
-        agent: bullmqJob.data.agent,
-        label: bullmqJob.data.label
-      });
-      await logger.write(`Workflow completed. ${JSON.stringify(output)}`, "INFO");
-      return output;
+    if (!data.workflow && !data.embedder) {
+      throw new Error(`Either a workflow or embedder must be set for job ${id}.`);
     }
   }
 };
@@ -4814,9 +4852,10 @@ var bullmq = {
 // src/registry/workers.ts
 import * as fs3 from "fs";
 import path2 from "path";
+import "@opentelemetry/api";
 var defaultLogsDir = path2.join(process.cwd(), "logs");
 var redisConnection;
-var createWorkers = async (queues2, contexts, _logsDir) => {
+var createWorkers = async (queues2, logger, contexts, _logsDir, tracer) => {
   if (!redisServer.host || !redisServer.port) {
     console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
     throw new Error("No redis server configured in the environment, so cannot start worker.");
@@ -4833,84 +4872,30 @@ var createWorkers = async (queues2, contexts, _logsDir) => {
     const worker = new Worker(
       `${queue}`,
       async (bullmqJob) => {
+        const logger2 = new ExuluLogger(bullmqJob, logsDir);
         const { db: db3 } = await postgresClient();
         try {
-          bullmq.validate(bullmqJob);
-          if (bullmqJob.data.type === "embedder") {
-            if (!bullmqJob.data.updater) {
-              throw new Error("No updater set for embedder job.");
-            }
-            const context = contexts.find((context2) => context2.id === bullmqJob.data.context);
+          const data = bullmqJob.data;
+          bullmq.validate(bullmqJob.id, data);
+          if (data.type === "embedder") {
+            const context = contexts.find((context2) => context2.id === data.context);
             if (!context) {
-              throw new Error(`Context ${bullmqJob.data.context} not found in the registry.`);
+              throw new Error(`Context ${data.context} not found in the registry.`);
             }
-            if (!bullmqJob.data.embedder) {
+            if (!data.embedder) {
               throw new Error(`No embedder set for embedder job.`);
             }
-            const embedder = contexts.find((context2) => context2.embedder?.id === bullmqJob.data.embedder);
+            const embedder = contexts.find((context2) => context2.embedder?.id === data.embedder);
             if (!embedder) {
-              throw new Error(`Embedder ${bullmqJob.data.embedder} not found in the registry.`);
-            }
-            if (!bullmqJob.data.source) {
-              throw new Error("No source set for embedder job.");
-            }
-            const source = context.sources.get(bullmqJob.data.source);
-            if (!source) {
-              throw new Error(`Source ${bullmqJob.data.source} not found in the registry.`);
-            }
-            if (!bullmqJob.data.updater) {
-              throw new Error("No updater set for embedder job.");
+              throw new Error(`Embedder ${data.embedder} not found in the registry.`);
             }
-            const updater = source.updaters.find((updater2) => updater2.id === bullmqJob.data.updater);
-            if (!updater) {
-              throw new Error(`Updater ${bullmqJob.data.updater} not found in the registry.`);
-            }
-            if (!bullmqJob.data.documents) {
-              throw new Error("No input documents set for embedder job.");
-            }
-            if (!Array.isArray(bullmqJob.data.documents)) {
-              throw new Error("Input documents must be an array.");
-            }
-            const result = await embedder.upsert(bullmqJob.data.context, bullmqJob.data.documents, {
-              label: context.name,
-              trigger: bullmqJob.data.trigger || "unknown"
+            const result = await context.createAndUpsertEmbeddings(data.inputs, data.user, {
+              label: embedder.name,
+              trigger: data.trigger
             });
-            const mongoRecord = await db3.from("jobs").where({ redis: bullmqJob.id }).first();
-            if (!mongoRecord) {
-              throw new Error("Job not found in the database.");
-            }
-            const finishedAt = /* @__PURE__ */ new Date();
-            const duration = (finishedAt.getTime() - new Date(mongoRecord.createdAt).getTime()) / 1e3;
-            await db3.from("jobs").where({ redis: bullmqJob.id }).update({
-              status: "completed",
-              finishedAt,
-              duration,
-              result: JSON.stringify(result)
-            });
-            await db3.from((void 0).getTableName()).where({ id: result[0].id }).update({
-              embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
-            }).returning("id");
             return result;
           }
           if (bullmqJob.data.type === "workflow") {
-            const workflow = workflows.find((workflow2) => workflow2.id === bullmqJob.data.workflow);
-            if (!workflow) {
-              throw new Error(`Workflow ${bullmqJob.data.workflow} not found in the registry.`);
-            }
-            const exuluJob = await db3.from("jobs").where({ redis: bullmqJob.id }).first();
-            if (!exuluJob) {
-              throw new Error("Job not found in the database.");
-            }
-            const result = await bullmq.process.workflow(bullmqJob, exuluJob, workflow, logsDir);
-            const finishedAt = /* @__PURE__ */ new Date();
-            const duration = (finishedAt.getTime() - new Date(exuluJob.createdAt).getTime()) / 1e3;
-            await db3.from("jobs").where({ redis: bullmqJob.id }).update({
-              status: "completed",
-              finishedAt,
-              duration,
-              result: JSON.stringify(result)
-            });
-            return result;
           }
         } catch (error) {
           await db3.from("jobs").where({ redis: bullmqJob.id }).update({
@@ -4980,6 +4965,7 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/
 import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
 import { z as z2 } from "zod";
 import "express";
+import "@opentelemetry/api";
 var SESSION_ID_HEADER = "mcp-session-id";
 var ExuluMCP = class {
   server;
@@ -4987,7 +4973,7 @@ var ExuluMCP = class {
   express;
   constructor() {
   }
-  create = async ({ express: express3, contexts, agents, config, tools }) => {
+  create = async ({ express: express3, contexts, agents, config, tools, tracer, logger }) => {
     this.express = express3;
     if (!this.server) {
       console.log("[EXULU] Creating MCP server.");
@@ -5165,6 +5151,38 @@ var defaultAgent = new ExuluAgent({
   }
 });
 
+// src/registry/index.ts
+import { trace as trace2 } from "@opentelemetry/api";
+
+// src/registry/logger.ts
+import { OpenTelemetryTransportV3 } from "@opentelemetry/winston-transport";
+import winston from "winston";
+var createLogger = ({
+  enableOtel
+}) => {
+  const logger = winston.createLogger({
+    level: "debug",
+    format: winston.format.combine(
+      winston.format.timestamp(),
+      winston.format.errors({
+        stack: true
+      }),
+      winston.format.metadata(),
+      winston.format.json()
+    ),
+    defaultMeta: {
+      service: "Test-Exulu",
+      environment: process.env.NODE_ENV || "development"
+    },
+    transports: [
+      new winston.transports.Console(),
+      ...enableOtel ? [new OpenTelemetryTransportV3()] : []
+    ]
+  });
+  return logger;
+};
+var logger_default = createLogger;
+
 // src/registry/index.ts
 var ExuluApp = class {
   _agents = [];
@@ -5231,10 +5249,20 @@ var ExuluApp = class {
   bullmq = {
     workers: {
       create: async () => {
+        let tracer;
+        if (this._config?.telemetry?.enabled) {
+          console.log("[EXULU] telemetry enabled");
+          tracer = trace2.getTracer("exulu", "1.0.0");
+        }
+        const logger = logger_default({
+          enableOtel: this._config?.workers?.telemetry?.enabled ?? false
+        });
         return await createWorkers(
           this._queues,
+          logger,
           Object.values(this._contexts ?? {}),
-          this._config?.workers?.logsDir
+          this._config?.workers?.logsDir,
+          tracer
         );
       }
     }
@@ -5246,11 +5274,22 @@ var ExuluApp = class {
           throw new Error("Express app not initialized");
         }
         const app = this._expressApp;
+        let tracer;
+        if (this._config?.telemetry?.enabled) {
+          console.log("[EXULU] telemetry enabled");
+          tracer = trace2.getTracer("exulu", "1.0.0");
+        }
+        const logger = logger_default({
+          enableOtel: this._config?.telemetry?.enabled ?? false
+        });
         await createExpressRoutes(
           app,
+          logger,
           this._agents,
           this._tools,
-          Object.values(this._contexts ?? {})
+          Object.values(this._contexts ?? {}),
+          this._config,
+          tracer
         );
         if (this._config?.MCP.enabled) {
           const mcp = new ExuluMCP();
@@ -5259,7 +5298,9 @@ var ExuluApp = class {
             contexts: this._contexts,
             agents: this._agents,
             config: this._config,
-            tools: this._tools
+            tools: this._tools,
+            tracer,
+            logger
           });
           await mcp.connect();
         }
@@ -6672,6 +6713,50 @@ var execute = async () => {
   return;
 };
 
+// src/registry/otel.ts
+import process2 from "process";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import { BatchLogRecordProcessor } from "@opentelemetry/sdk-logs";
+var create = ({
+  SIGNOZ_ACCESS_TOKEN,
+  SIGNOZ_TRACES_URL,
+  SIGNOZ_LOGS_URL
+}) => {
+  console.log("SIGNOZ_ACCESS_TOKEN", SIGNOZ_ACCESS_TOKEN);
+  console.log("SIGNOZ_TRACES_URL", SIGNOZ_TRACES_URL);
+  console.log("SIGNOZ_LOGS_URL", SIGNOZ_LOGS_URL);
+  const exporterOptions = {
+    url: SIGNOZ_TRACES_URL,
+    headers: {
+      "signoz-access-token": SIGNOZ_ACCESS_TOKEN
+    }
+  };
+  const traceExporter = new OTLPTraceExporter(exporterOptions);
+  const logExporter = new OTLPLogExporter({
+    url: SIGNOZ_LOGS_URL,
+    headers: {
+      "signoz-access-token": SIGNOZ_ACCESS_TOKEN
+    }
+  });
+  const sdk = new NodeSDK({
+    traceExporter,
+    logRecordProcessors: [new BatchLogRecordProcessor(logExporter)],
+    instrumentations: [getNodeAutoInstrumentations()],
+    resource: resourceFromAttributes({
+      [ATTR_SERVICE_NAME]: "Test-Exulu"
+    })
+  });
+  process2.on("SIGTERM", () => {
+    sdk.shutdown().then(() => console.log("Tracing terminated")).catch((error) => console.log("Error terminating tracing", error)).finally(() => process2.exit(0));
+  });
+  return sdk;
+};
+
 // types/enums/jobs.ts
 var JOB_STATUS_ENUM = {
   completed: "completed",
@@ -6690,6 +6775,19 @@ var ExuluJobs = {
     validate: validateJob
   }
 };
+var ExuluOtel = {
+  create: ({
+    SIGNOZ_ACCESS_TOKEN,
+    SIGNOZ_TRACES_URL,
+    SIGNOZ_LOGS_URL
+  }) => {
+    return create({
+      SIGNOZ_ACCESS_TOKEN,
+      SIGNOZ_TRACES_URL,
+      SIGNOZ_LOGS_URL
+    });
+  }
+};
 var db2 = {
   init: async () => {
     await execute();
@@ -6721,6 +6819,7 @@ export {
   ExuluEval,
   ExuluJobs,
   ExuluLogger,
+  ExuluOtel,
   queues as ExuluQueues,
   ExuluSource,
   ExuluTool,