@exulu/backend 0.2.2 → 0.2.4

package/dist/index.cjs

@@ -30,14 +30,17 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  EXULU_JOB_STATUS_ENUM: () => JOB_STATUS_ENUM,
   EXULU_STATISTICS_TYPE_ENUM: () => STATISTICS_TYPE_ENUM,
   ExuluAgent: () => ExuluAgent,
   ExuluApp: () => ExuluApp,
   ExuluAuthentication: () => authentication,
+  ExuluChunkers: () => ExuluChunkers,
   ExuluContext: () => ExuluContext,
   ExuluDatabase: () => ExuluDatabase,
   ExuluEmbedder: () => ExuluEmbedder,
   ExuluJobs: () => ExuluJobs,
+  ExuluLogger: () => ExuluLogger,
   ExuluQueues: () => queues,
   ExuluSource: () => ExuluSource,
   ExuluTool: () => ExuluTool,
@@ -45,6 +48,7 @@ __export(index_exports, {
   ExuluZodFileType: () => ExuluZodFileType
 });
 module.exports = __toCommonJS(index_exports);
+var import_config = require("dotenv/config");
 
 // src/redis/client.ts
 var import_redis = require("redis");
@@ -155,6 +159,10 @@ var usersSchema = {
       name: "firstname",
       type: "text"
     },
+    {
+      name: "name",
+      type: "text"
+    },
     {
       name: "lastname",
       type: "text"
@@ -258,6 +266,70 @@ var statisticsSchema = {
     }
   ]
 };
+var workflowSchema = {
+  name: {
+    plural: "workflows",
+    singular: "workflow"
+  },
+  fields: [
+    {
+      name: "workflow_name",
+      type: "text"
+    },
+    {
+      name: "run_id",
+      type: "text"
+    },
+    {
+      name: "snapshot",
+      type: "text"
+    }
+  ]
+};
+var threadsSchema = {
+  name: {
+    plural: "threads",
+    singular: "thread"
+  },
+  fields: [
+    {
+      name: "resourceId",
+      type: "text"
+    },
+    {
+      name: "title",
+      type: "text"
+    },
+    {
+      name: "metadata",
+      type: "text"
+    }
+  ]
+};
+var messagesSchema = {
+  name: {
+    plural: "messages",
+    singular: "message"
+  },
+  fields: [
+    {
+      name: "thread_id",
+      type: "text"
+    },
+    {
+      name: "content",
+      type: "text"
+    },
+    {
+      name: "role",
+      type: "text"
+    },
+    {
+      name: "type",
+      type: "text"
+    }
+  ]
+};
 var jobsSchema = {
   name: {
     plural: "jobs",
@@ -282,7 +354,7 @@ var jobsSchema = {
     },
     {
       name: "result",
-      type: "text"
+      type: "longText"
     },
     {
       name: "name",
@@ -292,6 +364,10 @@ var jobsSchema = {
       name: "agent",
       type: "text"
     },
+    {
+      name: "workflow",
+      type: "text"
+    },
     {
       name: "user",
       type: "text"
@@ -300,6 +376,10 @@ var jobsSchema = {
       name: "item",
       type: "text"
     },
+    {
+      name: "steps",
+      type: "number"
+    },
     {
       name: "inputs",
       type: "json"
@@ -403,14 +483,11 @@ var sanitizeName = (name) => {
   return name.toLowerCase().replace(/ /g, "_");
 };
 
-// src/postgres/init-db.ts
-var import_bcryptjs2 = __toESM(require("bcryptjs"), 1);
-
 // src/auth/generate-key.ts
 var import_bcryptjs = __toESM(require("bcryptjs"), 1);
 var SALT_ROUNDS = 12;
-async function encryptApiKey(apikey) {
-  const hash = await import_bcryptjs.default.hash(apikey, SALT_ROUNDS);
+async function encryptString(string) {
+  const hash = await import_bcryptjs.default.hash(string, SALT_ROUNDS);
   return hash;
 }
 var generateApiKey = async (name, email) => {
@@ -432,7 +509,7 @@ var generateApiKey = async (name, email) => {
   const newKeyName = name;
   const plainKey = `sk_${Math.random().toString(36).substring(2, 15)}_${Math.random().toString(36).substring(2, 15)}`;
   const postFix = `/${newKeyName.toLowerCase().trim().replaceAll(" ", "_")}`;
-  const encryptedKey = await encryptApiKey(plainKey);
+  const encryptedKey = await encryptString(plainKey);
   const existingApiUser = await db2.from("users").where({ email }).first();
   if (!existingApiUser) {
     console.log("[EXULU] Creating default api user.");
@@ -443,6 +520,7 @@ var generateApiKey = async (name, email) => {
       createdAt: /* @__PURE__ */ new Date(),
       updatedAt: /* @__PURE__ */ new Date(),
       type: "api",
+      emailVerified: /* @__PURE__ */ new Date(),
       apikey: `${encryptedKey}${postFix}`,
       // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
       role: roleId
@@ -607,11 +685,6 @@ var up = async function(knex) {
     });
   }
 };
-var SALT_ROUNDS2 = 12;
-async function encryptApiKey2(apikey) {
-  const hash = await import_bcryptjs2.default.hash(apikey, SALT_ROUNDS2);
-  return hash;
-}
 var execute = async () => {
   console.log("[EXULU] Initializing database.");
   const { db: db2 } = await postgresClient();
@@ -624,33 +697,33 @@ var execute = async () => {
     const role = await db2.from("roles").insert({
       name: "admin",
       is_admin: true,
-      agents: []
+      agents: JSON.stringify([])
     }).returning("id");
     roleId = role[0].id;
   } else {
     roleId = existingRole.id;
   }
-  const newKeyName = "exulu_default_key";
-  const plainKey = `sk_${Math.random().toString(36).substring(2, 15)}_${Math.random().toString(36).substring(2, 15)}`;
-  const postFix = `/${newKeyName.toLowerCase().trim().replaceAll(" ", "_")}`;
-  const encryptedKey = await encryptApiKey2(plainKey);
   const existingUser = await db2.from("users").where({ email: "admin@exulu.com" }).first();
   if (!existingUser) {
+    const password = await encryptString("admin");
     console.log("[EXULU] Creating default admin user.");
     await db2.from("users").insert({
       name: "exulu",
       email: "admin@exulu.com",
       super_admin: true,
       createdAt: /* @__PURE__ */ new Date(),
+      emailVerified: /* @__PURE__ */ new Date(),
       updatedAt: /* @__PURE__ */ new Date(),
+      password,
       type: "user",
-      // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
       role: roleId
     });
   }
   const { key } = await generateApiKey("exulu", "api@exulu.com");
   console.log("[EXULU] Database initialized.");
   console.log("[EXULU] Default api key: ", `${key}`);
+  console.log("[EXULU] Default password if using password auth: ", `admin`);
+  console.log("[EXULU] Default email if using password auth: ", `admin@exulu.com`);
   return;
 };
 
@@ -661,7 +734,6 @@ var import_core = require("@mastra/core");
 var import_zod2 = require("zod");
 var fs = __toESM(require("fs"), 1);
 var path = __toESM(require("path"), 1);
-var import_bullmq3 = require("bullmq");
 var import_memory = require("@mastra/memory");
 var import_pg = require("@mastra/pg");
 
@@ -697,6 +769,7 @@ var bullmqDecorator = async ({
   configuration,
   updater,
   context,
+  steps,
   source,
   documents,
   trigger,
@@ -714,11 +787,13 @@ var bullmqDecorator = async ({
       ...context && { context },
       ...source && { source },
       ...documents && { documents },
+      ...steps && { steps },
       ...trigger && { trigger },
       ...item && { item },
       agent,
       user,
       inputs,
+      label,
       session
     },
     {
@@ -743,6 +818,7 @@ var bullmqDecorator = async ({
     ...embedder && { embedder },
     ...workflow && { workflow },
     ...configuration && { configuration },
+    ...steps && { steps },
     ...updater && { updater },
     ...context && { context },
     ...source && { source },
@@ -766,6 +842,17 @@ var bullmqDecorator = async ({
   };
 };
 
+// types/enums/jobs.ts
+var JOB_STATUS_ENUM = {
+  completed: "completed",
+  failed: "failed",
+  delayed: "delayed",
+  active: "active",
+  waiting: "waiting",
+  paused: "paused",
+  stuck: "stuck"
+};
+
 // src/registry/classes.ts
 function generateSlug(name) {
   const normalized = name.normalize("NFKD").replace(/[\u0300-\u036f]/g, "");
@@ -800,6 +887,7 @@ var ExuluAgent = class {
   config;
   memory;
   tools;
+  agent;
   capabilities;
   constructor({ id, name, description, outputSchema, config, rateLimit, type, capabilities, tools }) {
     this.id = id;
@@ -812,6 +900,14 @@ var ExuluAgent = class {
     this.config = config;
     this.capabilities = capabilities;
     this.slug = `/agents/${generateSlug(this.name)}/run`;
+    if (this.type === "agent") {
+      this.agent = new import_core.Agent({
+        name: this.config.name,
+        instructions: this.config.instructions,
+        model: this.config.model,
+        memory: this.memory ? this.memory : void 0
+      });
+    }
     if (config?.memory) {
       console.log("[EXULU] Initializing memory for agent " + this.name);
       const connectionString = `postgresql://${process.env.POSTGRES_DB_USER}:${process.env.POSTGRES_DB_PASSWORD}@${process.env.POSTGRES_DB_HOST}:${process.env.POSTGRES_DB_PORT}/exulu`;
@@ -825,7 +921,11 @@ var ExuluAgent = class {
           password: process.env.POSTGRES_DB_PASSWORD || "",
           ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
         }),
-        ...config?.memory.vector ? { vector: new import_pg.PgVector(connectionString) } : {},
+        ...config?.memory.vector ? {
+          vector: new import_pg.PgVector({
+            connectionString
+          })
+        } : {},
         options: {
           lastMessages: config?.memory.lastMessages || 10,
           semanticRecall: {
@@ -836,33 +936,17 @@ var ExuluAgent = class {
       });
     }
   }
-  chat = async (id) => {
-    const { db: db2 } = await postgresClient();
-    const agent = await db2.from("agents").select("*").where("id", "=", id).first();
-    if (!agent) {
+  chat = () => {
+    if (!this.agent) {
       throw new Error("Agent not found");
     }
-    let tools = {};
-    agent.tools?.forEach(({ name }) => {
-      const tool = this.tools?.find((t) => t.name === name);
-      if (!tool) {
-        return;
-      }
-      return tool;
-    });
     updateStatistic({
       name: "count",
       label: this.name,
       type: STATISTICS_TYPE_ENUM.AGENT_RUN,
       trigger: "agent"
     });
-    return new import_core.Agent({
-      name: this.config.name,
-      instructions: this.config.instructions,
-      model: this.config.model,
-      tools,
-      memory: this.memory ? this.memory : void 0
-    });
+    return this.agent;
   };
 };
 var ExuluEmbedder = class {
@@ -919,35 +1003,141 @@ var ExuluWorkflow = class {
   enable_batch = false;
   slug = "";
   queue;
-  workflow;
-  inputSchema;
-  constructor({ id, name, description, workflow, queue, enable_batch, inputSchema }) {
+  steps;
+  constructor({ id, name, description, steps, queue, enable_batch }) {
     this.id = id;
     this.name = name;
     this.description = description;
     this.enable_batch = enable_batch;
     this.slug = `/workflows/${generateSlug(this.name)}/run`;
     this.queue = queue;
-    this.inputSchema = inputSchema;
-    this.workflow = workflow;
+    this.steps = steps;
   }
+  start = async ({
+    inputs: initialInputs,
+    user,
+    logger,
+    job,
+    session,
+    agent,
+    label
+  }) => {
+    let inputs;
+    const { db: db2 } = await postgresClient();
+    if (!job?.id) {
+      logger.write(`Creating new job for workflow ${this.name} with inputs: ${JSON.stringify(initialInputs)}`, "INFO");
+      const result = await db2("jobs").insert({
+        status: JOB_STATUS_ENUM.active,
+        name: `Job running '${this.name}' for '${label}'`,
+        agent,
+        workflow: this.id,
+        type: "workflow",
+        steps: this.steps?.length || 0,
+        inputs: initialInputs,
+        session,
+        user
+      }).returning(["id", "status"]);
+      job = result[0];
+      logger.write(`Created new job for workflow ${this.name}, job id: ${job?.id}`, "INFO");
+    }
+    if (!job) {
+      throw new Error("Job not found, or failed to be created.");
+    }
+    if (job.status !== JOB_STATUS_ENUM.active) {
+      await db2("jobs").update({
+        status: JOB_STATUS_ENUM.active,
+        inputs: initialInputs
+      }).where({ id: job?.id }).returning("id");
+    }
+    const runStep = async (step, inputs2) => {
+      let result;
+      try {
+        result = await step.fn({
+          inputs: inputs2,
+          logger,
+          job,
+          user
+        });
+        return result;
+      } catch (error) {
+        logger.write(`Step ${step.name} failed with error: ${error.message}`, "ERROR");
+        if (step.retries && step.retries > 0) {
+          logger.write(`Retrying step ${step.name} with ${step.retries} retries left`, "INFO");
+          step.retries--;
+          let result2 = await runStep(step, inputs2);
+          return result2;
+        }
+        logger.write(`Step ${step.name} failed with error: ${error.message}`, "ERROR");
+        throw error;
+      }
+    };
+    let final;
+    try {
+      for (let i = 0; i < this.steps.length; i++) {
+        const step = this.steps[i];
+        if (!step) {
+          throw new Error("Step not found.");
+        }
+        if (i === 0) {
+          inputs = initialInputs;
+        }
+        logger.write(`Running step ${step.name} with inputs: ${JSON.stringify(inputs)}`, "INFO");
+        let result = await runStep(step, inputs);
+        inputs = result;
+        logger.write(`Step ${step.name} output: ${JSON.stringify(result)}`, "INFO");
+        final = result;
+      }
+      await db2("jobs").update({
+        status: JOB_STATUS_ENUM.completed,
+        result: JSON.stringify(final),
+        finished_at: db2.fn.now()
+      }).where({ id: job?.id }).returning("id");
+      return final;
+    } catch (error) {
+      logger.write(`Workflow ${this.name} failed with error: ${error.message} for job ${job?.id}`, "ERROR");
+      await db2("jobs").update({
+        status: JOB_STATUS_ENUM.failed,
+        result: JSON.stringify({
+          error: error.message || error,
+          stack: error.stack || "No stack trace available"
+        })
+      }).where({ id: job?.id }).returning("id");
+      throw error;
+    }
+  };
 };
 var ExuluLogger = class {
   logPath;
   job;
   constructor(job, logsDir) {
     this.job = job;
-    if (!fs.existsSync(logsDir)) {
-      fs.mkdirSync(logsDir, { recursive: true });
+    if (logsDir && job) {
+      if (!fs.existsSync(logsDir)) {
+        fs.mkdirSync(logsDir, { recursive: true });
+      }
+      this.logPath = path.join(logsDir, `${job.id}_${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.txt`);
     }
-    this.logPath = path.join(logsDir, `${job.id}_${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.txt`);
   }
   async write(message, level) {
     const logMessage = message.endsWith("\n") ? message : message + "\n";
+    if (!this.logPath) {
+      switch (level) {
+        case "INFO":
+          console.log(message);
+          break;
+        case "WARNING":
+          console.warn(message);
+          break;
+        case "ERROR":
+          console.error(message);
+          break;
+      }
+      return;
+    }
     try {
       await fs.promises.appendFile(this.logPath, `[EXULU][${level}] - ${(/* @__PURE__ */ new Date()).toISOString()}: ${logMessage}`);
     } catch (error) {
-      console.error(`Error writing to log file ${this.job.id}:`, error);
+      console.error(`Error writing to log file ${this.job ? this.job.id : "unknown job"}:`, error);
       throw error;
     }
   }
@@ -1510,7 +1700,7 @@ var import_express = require("express");
 var import_jwt = require("next-auth/jwt");
 
 // src/auth/auth.ts
-var import_bcryptjs3 = __toESM(require("bcryptjs"), 1);
+var import_bcryptjs2 = __toESM(require("bcryptjs"), 1);
 var authentication = async ({
   apikey,
   authtoken,
@@ -1603,16 +1793,14 @@ var authentication = async ({
         code: 401
       };
     }
-    console.log("[EXULU] users", users);
     console.log("[EXULU] request_key_name", request_key_name);
     console.log("[EXULU] request_key_compare_value", request_key_compare_value);
     const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(request_key_name));
-    console.log("[EXULU] filtered", filtered);
     for (const user of filtered) {
       const user_key_last_slash_index = user.apikey.lastIndexOf("/");
       const user_key_compare_value = user.apikey.substring(0, user_key_last_slash_index);
       console.log("[EXULU] user_key_compare_value", user_key_compare_value);
-      const isMatch = await import_bcryptjs3.default.compare(request_key_compare_value, user_key_compare_value);
+      const isMatch = await import_bcryptjs2.default.compare(request_key_compare_value, user_key_compare_value);
       console.log("[EXULU] isMatch", isMatch);
       if (isMatch) {
         await db2.from("users").where({ id: user.id }).update({
@@ -1796,7 +1984,7 @@ var requestValidators = {
 var import_zodex = require("zodex");
 
 // src/bullmq/queues.ts
-var import_bullmq5 = require("bullmq");
+var import_bullmq4 = require("bullmq");
 var ExuluQueues = class {
   queues;
   constructor() {
@@ -1814,7 +2002,7 @@ var ExuluQueues = class {
       console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or workflow (look for ExuluQueues.use() ).`);
       throw new Error(`[EXULU] no redis server configured.`);
     }
-    const newQueue = new import_bullmq5.Queue(`${name}`, { connection: redisServer });
+    const newQueue = new import_bullmq4.Queue(`${name}`, { connection: redisServer });
     this.queues.push(newQueue);
     return newQueue;
   }
@@ -1834,6 +2022,7 @@ var VectorMethodEnum = {
 // src/registry/routes.ts
 var import_express4 = __toESM(require("express"), 1);
 var import_server3 = require("@apollo/server");
+var Papa = __toESM(require("papaparse"), 1);
 var import_cors = __toESM(require("cors"), 1);
 var import_reflect_metadata = require("reflect-metadata");
 
@@ -2170,8 +2359,8 @@ var import_express5 = require("@as-integrations/express5");
 
 // src/registry/uppy.ts
 var import_express2 = require("express");
+var import_body_parser = __toESM(require("body-parser"), 1);
 var import_jwt2 = require("next-auth/jwt");
-var bodyParser = require("body-parser");
 var createUppyRoutes = async (app) => {
   const {
     S3Client,
@@ -2232,7 +2421,7 @@ var createUppyRoutes = async (app) => {
     });
     return stsClient;
   }
-  app.use(bodyParser.urlencoded({ extended: true }), bodyParser.json());
+  app.use(import_body_parser.default.urlencoded({ extended: true }), import_body_parser.default.json());
   app.get("/s3/list", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
     let authtoken = null;
@@ -2567,7 +2756,6 @@ var createUppyRoutes = async (app) => {
 
 // src/registry/routes.ts
 var import_utils = require("@apollo/utils.keyvaluecache");
-var Papa = require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
 };
@@ -2665,7 +2853,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
   } else {
     console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
   }
-  const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
+  const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema, workflowSchema, threadsSchema, messagesSchema]);
   console.log("[EXULU] graphql server");
   const server = new import_server3.ApolloServer({
     cache: new import_utils.InMemoryLRUCache(),
@@ -2721,6 +2909,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         description: agent.description,
         active: agent.active,
         public: agent.public,
+        type: agent.type,
         slug: backend?.slug,
         rateLimit: backend?.rateLimit,
         streaming: backend?.streaming,
@@ -2778,13 +2967,22 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     if (!exists) {
       throw new Error("Table with name " + context.getTableName() + " does not exist.");
     }
-    const mutation = db2.from(context.getTableName()).delete().returning("id");
+    const query = db2.from(context.getTableName()).select("id");
     if (id) {
-      mutation.where({ id });
+      query.where({ id });
     }
     if (external_id) {
-      mutation.where({ external_id });
+      query.where({ external_id });
     }
+    const item = await query.first();
+    if (!item) {
+      throw new Error("Item not found.");
+    }
+    const chunks = await db2.from(context.getChunksTableName()).where({ source: item.id }).select("id");
+    if (chunks.length > 0) {
+      await db2.from(context.getChunksTableName()).where({ source: item.id }).delete();
+    }
+    const mutation = db2.from(context.getTableName()).where({ id: item.id }).delete().returning("id");
     const result = await mutation;
     return result;
   };
@@ -2921,21 +3119,27 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
       const context = contexts.find((context2) => context2.id === req.params.context);
       if (!context) {
+        console.error("[EXULU] context not found in registry.", req.params.context);
         res.status(400).json({
           message: "Context not found in registry."
         });
         return;
       }
+      console.log("[EXULU] context", context);
       const exists = await context.tableExists();
       if (!exists) {
+        console.log("[EXULU] context table does not exist, creating it.");
         await context.createItemsTable();
       }
+      console.log("[EXULU] inserting item", req.body);
       const result = await context.insertItem(authenticationResult.user.id, req.body, !!req.body.upsert);
+      console.log("[EXULU] result", result);
       res.status(200).json({
         message: "Item created successfully.",
         id: result
       });
     } catch (error) {
+      console.error("[EXULU] error upserting item", error);
       res.status(500).json({
         message: error?.message || "An error occurred while creating the item."
       });
@@ -3280,7 +3484,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       slug: workflow.slug,
       enable_batch: workflow.enable_batch,
       queue: workflow.queue?.name,
-      inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null
+      inputSchema: workflow.steps[0]?.inputSchema ? (0, import_zodex.zerialize)(workflow.steps[0].inputSchema) : null
     })));
   });
   console.log("[EXULU] workflow by id");
@@ -3307,7 +3511,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     res.status(200).json({
       ...workflow,
       queue: workflow.queue?.name,
-      inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null,
+      inputSchema: workflow.steps[0]?.inputSchema ? (0, import_zodex.zerialize)(workflow.steps[0].inputSchema) : null,
       workflow: void 0
     });
   });
@@ -3443,10 +3647,6 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       note: `Execute workflow ${workflow.name}`
     });
     app.post(`${workflow.slug}`, async (req, res) => {
-      if (!workflow.queue) {
-        res.status(500).json({ detail: "No queue set for workflow." });
-        return;
-      }
       const authenticationResult = await requestValidators.authenticate(req);
       if (!authenticationResult.user?.id) {
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
@@ -3463,6 +3663,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
           label: `Job running '${workflow.name}' for '${req.body.label}'`,
           agent: req.body.agent,
           workflow: workflow.id,
+          steps: workflow.steps?.length || 0,
           type: "workflow",
           inputs,
           session: req.body.session,
@@ -3481,22 +3682,19 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         });
         return;
       }
-      const { runId, start, watch } = workflow.workflow.createRun();
       console.log("[EXULU] running workflow with inputs.", inputs);
-      const output = await start({
-        triggerData: {
-          ...inputs,
-          user: authenticationResult.user.id
-        }
+      const logger = new ExuluLogger();
+      const result = await workflow.start({
+        inputs,
+        user: authenticationResult.user.id,
+        logger,
+        session: req.body.session,
+        agent: req.body.agent,
+        label: req.body.label
       });
-      const failedSteps = Object.entries(output.results).filter(([_, step]) => step.status === "failed").map(([id, step]) => `${id}: ${step.error}`);
-      if (failedSteps.length > 0) {
-        const message = `Workflow has failed steps: ${failedSteps.join("\n - ")}`;
-        throw new Error(message);
-      }
       res.status(200).json({
         "job": {},
-        "output": output
+        "output": result
       });
       return;
     });
@@ -3554,48 +3752,45 @@ var getPresignedFileUrl = async (key) => {
 
 // src/registry/workers.ts
 var import_ioredis = __toESM(require("ioredis"), 1);
-var import_bullmq8 = require("bullmq");
+var import_bullmq7 = require("bullmq");
 
 // src/registry/utils.ts
-var import_bullmq7 = require("bullmq");
+var import_bullmq6 = require("bullmq");
 var bullmq = {
-  validate: (job) => {
-    if (!job.data) {
-      throw new Error(`Missing job data for job ${job.id}.`);
+  validate: (bullmqJob) => {
+    if (!bullmqJob.data) {
+      throw new Error(`Missing job data for job ${bullmqJob.id}.`);
     }
-    if (!job.data.type) {
-      throw new Error(`Missing property "type" in data for job ${job.id}.`);
+    if (!bullmqJob.data.type) {
+      throw new Error(`Missing property "type" in data for job ${bullmqJob.id}.`);
     }
-    if (!job.data.inputs) {
-      throw new Error(`Missing property "inputs" in data for job ${job.id}.`);
+    if (!bullmqJob.data.inputs) {
+      throw new Error(`Missing property "inputs" in data for job ${bullmqJob.id}.`);
     }
-    if (job.data.type !== "embedder" && job.data.type !== "workflow") {
-      throw new Error(`Property "type" in data for job ${job.id} must be of value "embedder" or "workflow".`);
+    if (bullmqJob.data.type !== "embedder" && bullmqJob.data.type !== "workflow") {
+      throw new Error(`Property "type" in data for job ${bullmqJob.id} must be of value "embedder" or "workflow".`);
     }
-    if (!job.data.workflow && !job.data.embedder) {
-      throw new Error(`Property "backend" in data for job ${job.id} missing. Job  data: ${JSON.stringify(job)}`);
+    if (!bullmqJob.data.workflow && !bullmqJob.data.embedder) {
+      throw new Error(`Property "backend" in data for job ${bullmqJob.id} missing. Job  data: ${JSON.stringify(bullmqJob)}`);
     }
   },
   process: {
-    workflow: async (job, workflow, logsDir) => {
+    workflow: async (bullmqJob, exuluJob, workflow, logsDir) => {
       if (!workflow) {
-        throw new Error(`Workflow function with id: ${job.data.backend} not found in registry.`);
-      }
-      const { runId, start, watch } = workflow.workflow.createRun();
-      console.log("[EXULU] starting workflow with job inputs.", job.data.inputs);
-      const logger = new ExuluLogger(job, logsDir);
-      const output = await start({ triggerData: {
-        ...job.data.inputs,
-        redis: job.id,
-        logger
-      } });
-      const failedSteps = Object.entries(output.results).filter(([_, step]) => step.status === "failed").map(([id, step]) => `${id}: ${step.error}`);
-      if (failedSteps.length > 0) {
-        const message = `Workflow has failed steps: ${failedSteps.join("\n - ")}`;
-        logger.write(message, "ERROR");
-        throw new Error(message);
-      }
-      await logger.write(`Workflow completed. ${JSON.stringify(output.results)}`, "INFO");
+        throw new Error(`Workflow function with id: ${bullmqJob.data.backend} not found in registry.`);
+      }
+      console.log("[EXULU] starting workflow with job inputs.", bullmqJob.data.inputs);
+      const logger = new ExuluLogger(exuluJob, logsDir);
+      const output = await workflow.start({
+        job: exuluJob,
+        inputs: bullmqJob.data.inputs,
+        user: bullmqJob.data.user,
+        logger,
+        session: bullmqJob.data.session,
+        agent: bullmqJob.data.agent,
+        label: bullmqJob.data.label
+      });
+      await logger.write(`Workflow completed. ${JSON.stringify(output)}`, "INFO");
       return output;
     }
   }
@@ -3620,58 +3815,58 @@ var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) =>
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);
-    const worker = new import_bullmq8.Worker(
+    const worker = new import_bullmq7.Worker(
       `${queue}`,
-      async (job) => {
+      async (bullmqJob) => {
         const { db: db2 } = await postgresClient();
         try {
-          bullmq.validate(job);
-          if (job.data.type === "embedder") {
-            if (!job.data.updater) {
+          bullmq.validate(bullmqJob);
+          if (bullmqJob.data.type === "embedder") {
+            if (!bullmqJob.data.updater) {
               throw new Error("No updater set for embedder job.");
             }
-            const context = contexts.find((context2) => context2.id === job.data.context);
+            const context = contexts.find((context2) => context2.id === bullmqJob.data.context);
             if (!context) {
-              throw new Error(`Context ${job.data.context} not found in the registry.`);
+              throw new Error(`Context ${bullmqJob.data.context} not found in the registry.`);
             }
-            if (!job.data.embedder) {
+            if (!bullmqJob.data.embedder) {
               throw new Error(`No embedder set for embedder job.`);
             }
-            const embedder = embedders.find((embedder2) => embedder2.id === job.data.embedder);
+            const embedder = embedders.find((embedder2) => embedder2.id === bullmqJob.data.embedder);
             if (!embedder) {
-              throw new Error(`Embedder ${job.data.embedder} not found in the registry.`);
+              throw new Error(`Embedder ${bullmqJob.data.embedder} not found in the registry.`);
             }
-            if (!job.data.source) {
+            if (!bullmqJob.data.source) {
               throw new Error("No source set for embedder job.");
             }
-            const source = context.sources.get(job.data.source);
+            const source = context.sources.get(bullmqJob.data.source);
             if (!source) {
-              throw new Error(`Source ${job.data.source} not found in the registry.`);
+              throw new Error(`Source ${bullmqJob.data.source} not found in the registry.`);
             }
-            if (!job.data.updater) {
+            if (!bullmqJob.data.updater) {
               throw new Error("No updater set for embedder job.");
             }
-            const updater = source.updaters.find((updater2) => updater2.id === job.data.updater);
+            const updater = source.updaters.find((updater2) => updater2.id === bullmqJob.data.updater);
             if (!updater) {
-              throw new Error(`Updater ${job.data.updater} not found in the registry.`);
+              throw new Error(`Updater ${bullmqJob.data.updater} not found in the registry.`);
             }
-            if (!job.data.documents) {
+            if (!bullmqJob.data.documents) {
               throw new Error("No input documents set for embedder job.");
             }
-            if (!Array.isArray(job.data.documents)) {
+            if (!Array.isArray(bullmqJob.data.documents)) {
               throw new Error("Input documents must be an array.");
             }
-            const result = await embedder.upsert(job.data.context, job.data.documents, {
+            const result = await embedder.upsert(bullmqJob.data.context, bullmqJob.data.documents, {
               label: context.name,
-              trigger: job.data.trigger || "unknown"
+              trigger: bullmqJob.data.trigger || "unknown"
             });
-            const mongoRecord = await db2.from("jobs").where({ redis: job.id }).first();
+            const mongoRecord = await db2.from("jobs").where({ redis: bullmqJob.id }).first();
             if (!mongoRecord) {
               throw new Error("Job not found in the database.");
             }
             const finishedAt = /* @__PURE__ */ new Date();
             const duration = (finishedAt.getTime() - new Date(mongoRecord.createdAt).getTime()) / 1e3;
-            await db2.from("jobs").where({ redis: job.id }).update({
+            await db2.from("jobs").where({ redis: bullmqJob.id }).update({
               status: "completed",
               finishedAt,
               duration,
@@ -3679,19 +3874,19 @@ var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) =>
             });
             return result;
           }
-          if (job.data.type === "workflow") {
-            const workflow = workflows.find((workflow2) => workflow2.id === job.data.workflow);
+          if (bullmqJob.data.type === "workflow") {
+            const workflow = workflows.find((workflow2) => workflow2.id === bullmqJob.data.workflow);
             if (!workflow) {
-              throw new Error(`Workflow ${job.data.workflow} not found in the registry.`);
+              throw new Error(`Workflow ${bullmqJob.data.workflow} not found in the registry.`);
             }
-            const result = await bullmq.process.workflow(job, workflow, logsDir);
-            const mongoRecord = await db2.from("jobs").where({ redis: job.id }).first();
-            if (!mongoRecord) {
+            const exuluJob = await db2.from("jobs").where({ redis: bullmqJob.id }).first();
+            if (!exuluJob) {
               throw new Error("Job not found in the database.");
             }
+            const result = await bullmq.process.workflow(bullmqJob, exuluJob, workflow, logsDir);
             const finishedAt = /* @__PURE__ */ new Date();
-            const duration = (finishedAt.getTime() - new Date(mongoRecord.createdAt).getTime()) / 1e3;
-            await db2.from("jobs").where({ redis: job.id }).update({
+            const duration = (finishedAt.getTime() - new Date(exuluJob.createdAt).getTime()) / 1e3;
+            await db2.from("jobs").where({ redis: bullmqJob.id }).update({
               status: "completed",
               finishedAt,
               duration,
@@ -3700,7 +3895,7 @@ var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) =>
             return result;
           }
         } catch (error) {
-          await db2.from("jobs").where({ redis: job.id }).update({
+          await db2.from("jobs").where({ redis: bullmqJob.id }).update({
             status: "failed",
             finishedAt: /* @__PURE__ */ new Date(),
             error: error instanceof Error ? error.message : String(error)
@@ -3729,7 +3924,7 @@ var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) =>
   return workers;
 };
 var createLogsCleanerWorker = (logsDir) => {
-  const logsCleaner = new import_bullmq8.Worker(
+  const logsCleaner = new import_bullmq7.Worker(
     global_queues.logs_cleaner,
     async (job) => {
       console.log(`[EXULU] recurring job ${job.id}.`);
@@ -3842,12 +4037,18 @@ var ExuluApp = class {
 };
 
 // src/index.ts
+var import_chonkie = require("chonkie");
 var ExuluJobs = {
   redis: redisClient,
   jobs: {
     validate: validateJob
   }
 };
+var ExuluChunkers = {
+  chonkie: {
+    sentence: import_chonkie.SentenceChunker
+  }
+};
 var ExuluDatabase = {
   init: async () => {
     await execute();
@@ -3858,14 +4059,17 @@ var ExuluDatabase = {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  EXULU_JOB_STATUS_ENUM,
   EXULU_STATISTICS_TYPE_ENUM,
   ExuluAgent,
   ExuluApp,
   ExuluAuthentication,
+  ExuluChunkers,
   ExuluContext,
   ExuluDatabase,
   ExuluEmbedder,
   ExuluJobs,
+  ExuluLogger,
   ExuluQueues,
   ExuluSource,
   ExuluTool,