@exulu/backend 1.39.3 → 1.40.0

package/dist/index.js

@@ -15,7 +15,6 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
-  console.log("[EXULU] redisServer:", redisServer);
   if (!redisServer.host || !redisServer.port) {
     return { client: null };
   }
@@ -96,7 +95,6 @@ var db = {};
 var databaseExistsChecked = false;
 var dbName = process.env.POSTGRES_DB_NAME || "exulu";
 async function ensureDatabaseExists() {
-  console.log(`[EXULU] Ensuring ${dbName} database exists...`);
   const defaultKnex = Knex({
     client: "pg",
     connection: {
@@ -140,16 +138,7 @@ async function ensureDatabaseExists() {
 async function postgresClient() {
   if (!db["exulu"]) {
     try {
-      console.log(`[EXULU] Connecting to ${dbName} database.`);
-      console.log("[EXULU] POSTGRES_DB_HOST:", process.env.POSTGRES_DB_HOST);
-      console.log("[EXULU] POSTGRES_DB_PORT:", process.env.POSTGRES_DB_PORT);
-      console.log("[EXULU] POSTGRES_DB_USER:", process.env.POSTGRES_DB_USER);
-      console.log("[EXULU] POSTGRES_DB_PASSWORD:", process.env.POSTGRES_DB_PASSWORD);
-      console.log("[EXULU] POSTGRES_DB_NAME:", dbName);
-      console.log("[EXULU] POSTGRES_DB_SSL:", process.env.POSTGRES_DB_SSL);
-      console.log("[EXULU] Database exists checked:", databaseExistsChecked);
       if (!databaseExistsChecked) {
-        console.log(`[EXULU] Ensuring ${dbName} database exists...`);
         await ensureDatabaseExists();
         databaseExistsChecked = true;
       }
@@ -474,7 +463,6 @@ var authentication = async ({
   }
   if (authtoken) {
     try {
-      console.log("[EXULU] authtoken", authtoken);
       if (!authtoken?.email) {
         return {
           error: true,
@@ -3678,7 +3666,10 @@ type PageInfo {
     const config2 = await queue.use();
     return {
       name: config2.queue.name,
-      concurrency: config2.concurrency,
+      concurrency: {
+        worker: config2.concurrency?.worker || void 0,
+        queue: config2.concurrency?.queue || void 0
+      },
       ratelimit: config2.ratelimit,
       isMaxed: await config2.queue.isMaxed(),
       isPaused: await config2.queue.isPaused(),
@@ -3728,7 +3719,10 @@ type PageInfo {
     if (!agentInstance) {
       throw new Error("Agent instance not found for eval run.");
     }
-    const evalQueue = await queues.register("eval_runs", 1, 1).use();
+    const evalQueue = await queues.register("eval_runs", {
+      worker: 1,
+      queue: 1
+    }, 1).use();
     const jobIds = [];
     for (const testCase of testCases) {
       const jobData = {
@@ -3852,7 +3846,6 @@ type PageInfo {
     if (!client2) {
       throw new Error("Redis client not created properly");
     }
-    console.log("[EXULU] Jobs pagination args", args);
     const {
       jobs,
       count
@@ -3862,7 +3855,6 @@ type PageInfo {
       args.page || 1,
       args.limit || 100
     );
-    console.log("[EXULU] jobs", jobs.map((job) => job.name));
     const requestedFields = getRequestedFields(info);
     return {
       items: await Promise.all(jobs.map(async (job) => {
@@ -3891,6 +3883,25 @@ type PageInfo {
   };
   resolvers.Query["contexts"] = async (_, args, context, info) => {
     const data = await Promise.all(contexts.map(async (context2) => {
+      let processors = await Promise.all(context2.fields.map(async (field) => {
+        if (field.processor) {
+          let queueName = void 0;
+          if (field.processor?.config?.queue) {
+            const config2 = await field.processor?.config?.queue;
+            queueName = config2?.queue?.name || void 0;
+          }
+          return {
+            field: field.name,
+            description: field.processor?.description,
+            queue: queueName,
+            trigger: field.processor?.config?.trigger,
+            timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
+            generateEmbeddings: field.processor?.config?.generateEmbeddings || false
+          };
+        }
+        return null;
+      }));
+      processors = processors.filter((processor) => processor !== null);
       const sources = await Promise.all(context2.sources.map(async (source) => {
         let queueName = void 0;
         if (source.config) {
@@ -3922,6 +3933,7 @@ type PageInfo {
         slug: "/contexts/" + context2.id,
         active: context2.active,
         sources,
+        processors,
         fields: context2.fields.map((field) => {
           return {
             ...field,
@@ -3947,6 +3959,25 @@ type PageInfo {
     if (!data) {
       return null;
     }
+    let processors = await Promise.all(data.fields.map(async (field) => {
+      if (field.processor) {
+        let queueName = void 0;
+        if (field.processor?.config?.queue) {
+          const config2 = await field.processor?.config?.queue;
+          queueName = config2?.queue?.name || void 0;
+        }
+        return {
+          field: field.name,
+          description: field.processor?.description,
+          queue: queueName,
+          trigger: field.processor?.config?.trigger,
+          timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
+          generateEmbeddings: field.processor?.config?.generateEmbeddings || false
+        };
+      }
+      return null;
+    }));
+    processors = processors.filter((processor) => processor !== null);
     const sources = await Promise.all(data.sources.map(async (source) => {
       let queueName = void 0;
       if (source.config) {
@@ -3983,6 +4014,7 @@ type PageInfo {
       slug: "/contexts/" + data.id,
       active: data.active,
       sources,
+      processors,
       fields: await Promise.all(data.fields.map(async (field) => {
         const label = field.name?.replace("_s3key", "");
         if (field.type === "file" && !field.name.endsWith("_s3key")) {
@@ -4006,7 +4038,10 @@ type PageInfo {
                 queue: {
                   name: queue?.queue.name || void 0,
                   ratelimit: queue?.ratelimit || void 0,
-                  concurrency: queue?.concurrency || void 0
+                  concurrency: {
+                    worker: queue?.concurrency?.worker || void 0,
+                    queue: queue?.concurrency?.queue || void 0
+                  }
                 }
               },
               execute: "function"
@@ -4079,13 +4114,19 @@ type PageInfo {
   modelDefs += `
     type QueueResult {
         name: String!
-        concurrency: Int!
+        concurrency: QueueConcurrency!
         ratelimit: Int!
         isMaxed: Boolean!
         isPaused: Boolean!
         jobs: QueueJobsCounts
     }
     `;
+  modelDefs += `
+    type QueueConcurrency {
+        worker: Int
+        queue: Int
+    }
+    `;
   modelDefs += `
     type QueueJobsCounts {
         paused: Int!
@@ -4200,7 +4241,8 @@ type Context {
     active: Boolean
     fields: JSON
     configuration: JSON
-    sources: [ContextSource!]
+    sources: [ContextSource]
+    processors: [ContextProcessor]
 }
 type Embedder {
     name: String!
@@ -4213,6 +4255,14 @@ type EmbedderConfig {
     description: String
     default: String
 }
+type ContextProcessor {
+    field: String!
+    description: String
+    queue: String
+    trigger: String
+    timeoutInSeconds: Int
+    generateEmbeddings: Boolean
+}
 
 type ContextSource {
     id: String!
@@ -4331,10 +4381,7 @@ async function getJobsByQueueName(queueName, statusses, page, limit) {
   const config = await queue.use();
   const startIndex = (page || 1) - 1;
   const endIndex = startIndex - 1 + (limit || 100);
-  console.log("[EXULU] Jobs pagination startIndex", startIndex);
-  console.log("[EXULU] Jobs pagination endIndex", endIndex);
   const jobs = await config.queue.getJobs(statusses || [], startIndex, endIndex, false);
-  console.log("[EXULU] Jobs pagination jobs", jobs?.length);
   const counts = await config.queue.getJobCounts(...statusses || []);
   let total = 0;
   if (counts) {
@@ -4394,21 +4441,10 @@ function getS3Client(config) {
   });
   return s3Client;
 }
-var getPresignedUrl = async (key, config) => {
+var getPresignedUrl = async (bucket, key, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
-  let bucket = config.fileUploads.s3Bucket;
-  if (key.includes("[bucket:")) {
-    console.log("[EXULU] key includes [bucket:name]", key);
-    bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-    if (!bucket?.length) {
-      throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-    }
-    key = key.split("]")[1] || "";
-    console.log("[EXULU] bucket", bucket);
-    console.log("[EXULU] key", key);
-  }
   const url = await getSignedUrl(
     getS3Client(config),
     new GetObjectCommand({
@@ -4419,7 +4455,7 @@ var getPresignedUrl = async (key, config) => {
   );
   return url;
 };
-var addPrefixToKey = (keyPath, config) => {
+var addGeneralPrefixToKey = (keyPath, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
@@ -4432,58 +4468,50 @@ var addPrefixToKey = (keyPath, config) => {
   }
   return `${prefix}/${keyPath}`;
 };
-var uploadFile = async (file, key, config, options = {}, user) => {
+var addUserPrefixToKey = (key, user) => {
+  if (!user) {
+    return key;
+  }
+  if (key.includes(`/user_${user}/`)) {
+    return key;
+  }
+  return `user_${user}/${key}`;
+};
+var addBucketPrefixToKey = (key, bucket) => {
+  if (key.includes(`/${bucket}/`)) {
+    return key;
+  }
+  return `${bucket}/${key}`;
+};
+var uploadFile = async (file, fileName, config, options = {}, user, customBucket) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured (in the exported uploadFile function)");
   }
   const client2 = getS3Client(config);
   let defaultBucket = config.fileUploads.s3Bucket;
-  let customBucket = false;
-  if (key.includes("[bucket:")) {
-    console.log("[EXULU] key includes [bucket:name]", key);
-    customBucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-    if (!customBucket?.length) {
-      throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-    }
-    key = key.split("]")[1] || "";
-    console.log("[EXULU] custom bucket", customBucket);
-  }
-  let folder = user ? `${user}/` : "";
-  const fullKey = addPrefixToKey(!key.includes(folder) ? folder + key : key, config);
-  console.log("[EXULU] uploading file to s3 into bucket", customBucket || defaultBucket, "with key", fullKey);
+  let key = fileName;
+  key = addGeneralPrefixToKey(key, config);
+  key = addUserPrefixToKey(key, user || "api");
+  console.log("[EXULU] uploading file to s3 into bucket", defaultBucket, "with key", key);
   const command = new PutObjectCommand({
     Bucket: customBucket || defaultBucket,
-    Key: fullKey,
+    Key: key,
     Body: file,
     ContentType: options.contentType,
     Metadata: options.metadata,
     ContentLength: file.byteLength
   });
   await client2.send(command);
-  console.log("[EXULU] file uploaded to s3 into bucket", customBucket || defaultBucket, "with key", fullKey);
-  if (customBucket) {
-    return "[bucket:" + customBucket + "]" + fullKey;
-  }
-  return fullKey;
+  console.log("[EXULU] file uploaded to s3 into bucket", customBucket || defaultBucket, "with key", key);
+  return addBucketPrefixToKey(
+    key,
+    customBucket || defaultBucket
+  );
 };
-var createUppyRoutes = async (app, config) => {
+var createUppyRoutes = async (app, contexts, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
-  const extractUserPrefix = (key) => {
-    if (!config.fileUploads) {
-      throw new Error("File uploads are not configured");
-    }
-    if (!config.fileUploads.s3prefix) {
-      return key.split("/")[0];
-    }
-    const prefix = config.fileUploads.s3prefix.replace(/\/$/, "");
-    if (key.startsWith(prefix + "/")) {
-      const keyWithoutPrefix = key.slice(prefix.length + 1);
-      return keyWithoutPrefix.split("/")[0];
-    }
-    return key.split("/")[0];
-  };
   const policy = {
     Version: "2012-10-17",
     Statement: [
@@ -4535,20 +4563,16 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
+    const user = authenticationResult.user;
     let { key } = req.query;
     if (typeof key !== "string" || key.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    let bucket = config.fileUploads.s3Bucket;
-    if (key.includes("[bucket:")) {
-      console.log("[EXULU] key includes [bucket:name]", key);
-      bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-      if (!bucket?.length) {
-        throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-      }
-      key = key.split("]")[1] || "";
-      console.log("[EXULU] bucket", bucket);
+    let bucket = key.split("/")[0];
+    if (user.type !== "api" && !key.includes(`/user_${user.id}/`) && !user.super_admin) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
     }
     const client2 = getS3Client(config);
     const command = new DeleteObjectCommand({
@@ -4576,13 +4600,32 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
-    const { key } = req.query;
+    const user = authenticationResult.user;
+    let { key } = req.query;
+    if (!key || typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
+      return;
+    }
+    let bucket = key.split("/")[0];
+    if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `bucket` (should be the first part of the key before the first slash)." });
+      return;
+    }
+    key = key.split("/").slice(1).join("/");
     if (typeof key !== "string" || key.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
+    let allowed = false;
+    if (user.type === "api" || user.super_admin || key.includes(`user_${user.id}/`)) {
+      allowed = true;
+    }
+    if (!allowed) {
+      res.status(405).json({ error: "Not allowed to access the file based on authenticated user." });
+      return;
+    }
     try {
-      const url = await getPresignedUrl(key, config);
+      const url = await getPresignedUrl(bucket, key, config);
       res.setHeader("Access-Control-Allow-Origin", "*");
       res.json({ url, method: "GET", expiresIn });
     } catch (err) {
@@ -4611,16 +4654,15 @@ var createUppyRoutes = async (app, config) => {
       return;
     }
     let { key } = req.body;
-    let bucket = config.fileUploads.s3Bucket;
-    if (key.includes("[bucket:")) {
-      console.log("[EXULU] key includes [bucket:name]", key);
-      bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-      console.log("[EXULU] bucket", bucket);
-      if (!bucket?.length) {
-        throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-      }
-      key = key.split("]")[1] || "";
-      console.log("[EXULU] key", key);
+    let bucket = key.split("/")[0];
+    if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `bucket` (should be the first part of the key before the first slash)." });
+      return;
+    }
+    key = key.split("/").slice(1).join("/");
+    if (!key || typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
+      return;
     }
     const client2 = getS3Client(config);
     const command = new HeadObjectCommand({
@@ -4724,11 +4766,12 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
+    const user = authenticationResult.user;
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
     const key = generateS3Key2(filename);
-    let folder = `${authenticationResult.user.id}/`;
-    const fullKey = addPrefixToKey(folder + key, config);
+    let fullKey = addGeneralPrefixToKey(key, config);
+    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
     getSignedUrl(
       getS3Client(config),
       new PutObjectCommand({
@@ -4772,6 +4815,7 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
+    const user = authenticationResult.user;
     const client2 = getS3Client(config);
     const { type, metadata, filename } = req.body;
     if (typeof filename !== "string") {
@@ -4781,13 +4825,8 @@ var createUppyRoutes = async (app, config) => {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
     const key = `${randomUUID()}-_EXULU_${filename}`;
-    let folder = "";
-    if (authenticationResult.user.type === "api") {
-      folder = `api/`;
-    } else {
-      folder = `${authenticationResult.user.id}/`;
-    }
-    const fullKey = addPrefixToKey(folder + key, config);
+    let fullKey = addGeneralPrefixToKey(key, config);
+    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
     const params = {
       Bucket: config.fileUploads.s3Bucket,
       Key: fullKey,
@@ -4990,7 +5029,7 @@ var createProjectRetrievalTool = async ({
         if (!context) {
           throw new Error("The item added to the project does not have a valid gid with the context id as the prefix before the first slash.");
         }
-        const id = item.split("/")[1];
+        const id = item.split("/").slice(1).join("/");
         if (set[context]) {
           set[context].push(id);
         } else {
@@ -5936,12 +5975,20 @@ var ExuluStorage = class {
     this.config = config;
   }
   getPresignedUrl = async (key) => {
-    return await getPresignedUrl(key, this.config);
+    const bucket = key.split("/")[0];
+    if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
+      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
+    }
+    key = key.split("/").slice(1).join("/");
+    if (!key || typeof key !== "string" || key.trim() === "") {
+      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
+    }
+    return await getPresignedUrl(bucket, key, this.config);
   };
-  uploadFile = async (file, key, type, user, metadata) => {
+  uploadFile = async (file, fileName, type, user, metadata, customBucket) => {
     return await uploadFile(
       file,
-      key,
+      fileName,
       this.config,
       {
         contentType: type,
@@ -5950,7 +5997,8 @@ var ExuluStorage = class {
           type
         }
       },
-      user
+      user,
+      customBucket
     );
   };
   // todo add upload and delete methods
@@ -6001,6 +6049,9 @@ var ExuluContext = class {
     this.resultReranker = resultReranker;
   }
   processField = async (trigger, item, exuluConfig, user, role) => {
+    if (!item.field) {
+      throw new Error("Field property on item is required for running a specific processor.");
+    }
     console.log("[EXULU] processing field", item.field, " in context", this.id);
     console.log("[EXULU] fields", this.fields.map((field2) => field2.name));
     const field = this.fields.find((field2) => {
@@ -6032,27 +6083,32 @@ var ExuluContext = class {
         trigger
       });
       return {
-        result: "",
+        result: {},
         job: job.id
       };
     }
     console.log("[EXULU] POS 1 -- EXULU CONTEXT PROCESS FIELD");
-    const result = await field.processor.execute({
+    const processorResult = await field.processor.execute({
       item,
       user,
       role,
       utils: {
-        storage: exuluStorage,
-        items: {
-          update: this.updateItem,
-          create: this.createItem,
-          delete: this.deleteItem
-        }
+        storage: exuluStorage
       },
       exuluConfig
     });
+    if (!processorResult) {
+      throw new Error("Processor result is required for updating the item in the db.");
+    }
+    const { db: db3 } = await postgresClient();
+    delete processorResult.field;
+    await db3.from(getTableName(this.id)).where({
+      id: processorResult.id
+    }).update({
+      ...processorResult
+    });
     return {
-      result,
+      result: processorResult,
       job: void 0
     };
   };
@@ -6137,6 +6193,8 @@ var ExuluContext = class {
     };
   };
   createItem = async (item, config, user, role, upsert, generateEmbeddingsOverwrite) => {
+    console.log("[EXULU] creating item", item);
+    console.log("[EXULU] upsert", upsert);
     if (upsert && (!item.id && !item.external_id)) {
       throw new Error("Item id or external id is required for upsert.");
     }
@@ -6164,7 +6222,7 @@ var ExuluContext = class {
     }
     console.log("[EXULU] context configuration", this.configuration);
     let jobs = [];
-    let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always");
+    let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always");
     for (const [key, value] of Object.entries(item)) {
       console.log("[EXULU] Checking for processors for field", key);
       const processor = this.fields.find((field) => field.name === key.replace("_s3key", ""))?.processor;
@@ -6187,8 +6245,13 @@ var ExuluContext = class {
         if (processorJob) {
           jobs.push(processorJob);
         }
-        if (!processorJob && processor.config?.generateEmbeddings) {
-          shouldGenerateEmbeddings = true;
+        if (!processorJob) {
+          await db3.from(getTableName(this.id)).where({ id: results[0].id }).update({
+            ...processorResult
+          });
+          if (processor.config?.generateEmbeddings) {
+            shouldGenerateEmbeddings = true;
+          }
         }
       }
     }
@@ -6214,6 +6277,7 @@ var ExuluContext = class {
     };
   };
   updateItem = async (item, config, user, role, generateEmbeddingsOverwrite) => {
+    console.log("[EXULU] updating item", item);
     const { db: db3 } = await postgresClient();
     if (item.field) {
       delete item.field;
@@ -6259,8 +6323,13 @@ var ExuluContext = class {
         if (processorJob) {
           jobs.push(processorJob);
         }
-        if (!processorJob && processor.config?.generateEmbeddings) {
-          shouldGenerateEmbeddings = true;
+        if (!processorJob) {
+          await db3.from(getTableName(this.id)).where({ id: record.id }).update({
+            ...processorResult
+          });
+          if (processor.config?.generateEmbeddings) {
+            shouldGenerateEmbeddings = true;
+          }
         }
       }
     }
@@ -6284,23 +6353,23 @@ var ExuluContext = class {
   };
   deleteItem = async (item, user, role) => {
     const { db: db3 } = await postgresClient();
+    if (!item.id && !item.external_id) {
+      throw new Error("Item id or external id is required for deleting an item.");
+    }
     if (!item.id?.length && item?.external_id) {
       item = await db3.from(getTableName(this.id)).where({ external_id: item.external_id }).first();
       if (!item || !item.id) {
         throw new Error(`Item not found for external id ${item?.external_id || "undefined"}.`);
       }
     }
-    await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
-    if (!this.embedder) {
-      return {
-        id: item.id,
-        job: void 0
-      };
-    }
-    const chunks = await db3.from(getChunksTableName(this.id)).where({ source: item.id }).select("id");
-    if (chunks.length > 0) {
-      await db3.from(getChunksTableName(this.id)).where({ source: item.id }).delete();
+    const chunkTableExists = await this.chunksTableExists();
+    if (chunkTableExists) {
+      const chunks = await db3.from(getChunksTableName(this.id)).where({ source: item.id }).select("id");
+      if (chunks.length > 0) {
+        await db3.from(getChunksTableName(this.id)).where({ source: item.id }).delete();
+      }
     }
+    await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
     return {
       id: item.id,
       job: void 0
@@ -7111,7 +7180,7 @@ Mood: friendly and intelligent.
     });
   });
   if (config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
-    await createUppyRoutes(app, config);
+    await createUppyRoutes(app, contexts ?? [], config);
   } else {
     console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in ExuluApp instance.");
   }
@@ -7424,6 +7493,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
       async (bullmqJob) => {
         console.log("[EXULU] starting execution for job", logMetadata2(bullmqJob.name, {
           name: bullmqJob.name,
+          jobId: bullmqJob.id,
           status: await bullmqJob.getState(),
           type: bullmqJob.data.type
         }));
@@ -7438,6 +7508,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         });
         const workPromise = (async () => {
           try {
+            console.log(`[EXULU] Job ${bullmqJob.id} - Log file: logs/jobs/job-${bullmqJob.id}.log`);
             bullmq.validate(bullmqJob.id, data);
             if (data.type === "embedder") {
               console.log("[EXULU] running an embedder job.", logMetadata2(bullmqJob.name));
@@ -7460,17 +7531,17 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!embedder) {
                 throw new Error(`Embedder ${data.embedder} not found in the registry.`);
               }
-              const result = await context.createAndUpsertEmbeddings(data.inputs, config, data.user, {
+              const result2 = await context.createAndUpsertEmbeddings(data.inputs, config, data.user, {
                 label: embedder.name,
                 trigger: data.trigger
               }, data.role, bullmqJob.id);
               return {
-                result,
+                result: result2,
                 metadata: {}
               };
             }
             if (data.type === "processor") {
-              console.log("[EXULU] running a processor job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running a processor job, job name: ", bullmqJob.name, " job id: ", bullmqJob.id, " job data: ", data, " job queue: ", bullmqJob.queueName);
               const label = `processor-${bullmqJob.name}`;
               await db3.from("job_results").insert({
                 job_id: bullmqJob.id,
@@ -7492,35 +7563,42 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!field.processor) {
                 throw new Error(`Processor not set for field ${data.inputs.field} in the context ${data.context}.`);
               }
-              const exuluStorage = new ExuluStorage({ config });
-              if (!data.user) {
-                throw new Error(`User not set for processor job.`);
-              }
-              if (!data.role) {
-                throw new Error(`Role not set for processor job.`);
+              if (!data.inputs.id) {
+                throw new Error(`[EXULU] Item not set for processor ${field.name} in context ${context.id}, running in job ${bullmqJob.id}.`);
               }
+              const exuluStorage = new ExuluStorage({ config });
               console.log("[EXULU] POS 2 -- EXULU CONTEXT PROCESS FIELD");
-              const result = await field.processor.execute({
+              const processorResult = await field.processor.execute({
                 item: data.inputs,
                 user: data.user,
                 role: data.role,
                 utils: {
-                  storage: exuluStorage,
-                  items: {
-                    update: context.updateItem,
-                    create: context.createItem,
-                    delete: context.deleteItem
-                  }
+                  storage: exuluStorage
                 },
-                config
+                exuluConfig: config
+              });
+              if (!processorResult) {
+                throw new Error(`[EXULU] Processor ${field.name} in context ${context.id}, running in job ${bullmqJob.id} did not return an item.`);
+              }
+              delete processorResult.field;
+              await db3.from(getTableName(context.id)).where({
+                id: processorResult.id
+              }).update({
+                ...processorResult
               });
               let jobs = [];
               if (field.processor.config?.generateEmbeddings) {
+                const fullItem = await db3.from(getTableName(context.id)).where({
+                  id: processorResult.id
+                }).first();
+                if (!fullItem) {
+                  throw new Error(`[EXULU] Item ${processorResult.id} not found after processor update in context ${context.id}`);
+                }
                 const { job: embeddingsJob } = await context.embeddings.generate.one({
-                  item: data.inputs,
+                  item: fullItem,
                   user: data.user,
                   role: data.role,
-                  trigger: "api",
+                  trigger: "processor",
                   config
                 });
                 if (embeddingsJob) {
@@ -7528,7 +7606,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 }
               }
               return {
-                result,
+                result: processorResult,
                 metadata: {
                   jobs: jobs.length > 0 ? jobs.join(",") : void 0
                 }
@@ -7595,9 +7673,9 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   }
                 }
               });
-              const result = await promise;
-              const messages = result.messages;
-              const metadata = result.metadata;
+              const result2 = await promise;
+              const messages = result2.messages;
+              const metadata = result2.metadata;
               const evalFunctions = evalRun.eval_functions;
               let evalFunctionResults = [];
               for (const evalFunction of evalFunctions) {
@@ -7605,7 +7683,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 if (!evalMethod) {
                   throw new Error(`Eval function ${evalFunction.id} not found in the registry, check your code and make sure the eval function is registered correctly.`);
                 }
-                let result2;
+                let result3;
                 if (evalMethod.queue) {
                   const queue2 = await evalMethod.queue;
                   const jobData = {
@@ -7636,21 +7714,21 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   if (!job.id) {
                     throw new Error(`Tried to add job to queue ${queue2.queue.name} but failed to get the job ID.`);
                   }
-                  result2 = await pollJobResult({ queue: queue2, jobId: job.id });
+                  result3 = await pollJobResult({ queue: queue2, jobId: job.id });
                   const evalFunctionResult = {
                     test_case_id: testCase.id,
                     eval_run_id: evalRun.id,
                     eval_function_id: evalFunction.id,
                     eval_function_name: evalFunction.name,
                     eval_function_config: evalFunction.config || {},
-                    result: result2 || 0
+                    result: result3 || 0
                   };
-                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
-                    result: result2 || 0
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result3}`, logMetadata2(bullmqJob.name, {
+                    result: result3 || 0
                   }));
                   evalFunctionResults.push(evalFunctionResult);
                 } else {
-                  result2 = await evalMethod.run(
+                  result3 = await evalMethod.run(
                     agentInstance,
                     agentBackend,
                     testCase,
@@ -7661,15 +7739,15 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     test_case_id: testCase.id,
                     eval_run_id: evalRun.id,
                     eval_function_id: evalFunction.id,
-                    result: result2 || 0
+                    result: result3 || 0
                   };
                   evalFunctionResults.push(evalFunctionResult);
-                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
-                    result: result2 || 0
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result3}`, logMetadata2(bullmqJob.name, {
+                    result: result3 || 0
                   }));
                 }
               }
-              const scores = evalFunctionResults.map((result2) => result2.result);
+              const scores = evalFunctionResults.map((result3) => result3.result);
               console.log("[EXULU] Exulu eval run scores for test case: " + testCase.id, scores);
               let score = 0;
               switch (data.scoring_method?.toLowerCase()) {
@@ -7734,25 +7812,25 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 messages: inputMessages
               } = await validateEvalPayload(data, agents);
               const evalFunctions = evalRun.eval_functions;
-              let result;
+              let result2;
               for (const evalFunction of evalFunctions) {
                 const evalMethod = evals.find((e) => e.id === evalFunction.id);
                 if (!evalMethod) {
                   throw new Error(`Eval function ${evalFunction.id} not found in the registry, check your code and make sure the eval function is registered correctly.`);
                 }
-                result = await evalMethod.run(
+                result2 = await evalMethod.run(
                   agentInstance,
                   backend,
                   testCase,
                   inputMessages,
                   evalFunction.config || {}
                 );
-                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result}`, logMetadata2(bullmqJob.name, {
-                  result: result || 0
+                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
+                  result: result2 || 0
                 }));
               }
               return {
-                result,
+                result: result2,
                 metadata: {}
               };
             }
@@ -7772,10 +7850,10 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!source) {
                 throw new Error(`Source ${data.source} not found in the context ${context.id}.`);
               }
-              const result = await source.execute(data.inputs);
+              const result2 = await source.execute(data.inputs);
               let jobs = [];
               let items = [];
-              for (const item of result) {
+              for (const item of result2) {
                 const { item: createdItem, job } = await context.createItem(
                   item,
                   config,
@@ -7807,7 +7885,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 role: data?.role
               });
               return {
-                result,
+                result: result2,
                 metadata: {
                   jobs,
                   items
@@ -7820,11 +7898,13 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
             throw error;
           }
         })();
-        return Promise.race([workPromise, timeoutPromise]);
+        const result = await Promise.race([workPromise, timeoutPromise]);
+        return result;
       },
       {
         autorun: true,
         connection: redisConnection,
+        concurrency: queue.concurrency?.worker || 1,
         removeOnComplete: { count: 1e3 },
         removeOnFail: { count: 5e3 },
         ...queue.ratelimit && {
@@ -9028,18 +9108,23 @@ var ExuluQueues = class {
   // method of ExuluQueues we need to store the desired rate limit on the queue
   // here so we can use the value when creating workers for the queue instance
   // as there is no way to store a rate limit value natively on a bullm queue.
-  register = (name, concurrency = 1, ratelimit = 1) => {
+  register = (name, concurrency, ratelimit = 1) => {
+    const queueConcurrency = concurrency.queue || 1;
+    const workerConcurrency = concurrency.worker || 1;
     const use = async () => {
       const existing = this.queues.find((x) => x.queue?.name === name);
       if (existing) {
         const globalConcurrency = await existing.queue.getGlobalConcurrency();
-        if (globalConcurrency !== concurrency) {
-          await existing.queue.setGlobalConcurrency(concurrency);
+        if (globalConcurrency !== queueConcurrency) {
+          await existing.queue.setGlobalConcurrency(queueConcurrency);
         }
         return {
           queue: existing.queue,
           ratelimit,
-          concurrency
+          concurrency: {
+            worker: workerConcurrency,
+            queue: queueConcurrency
+          }
         };
       }
       if (!redisServer.host?.length || !redisServer.port?.length) {
@@ -9058,21 +9143,30 @@ var ExuluQueues = class {
           telemetry: new BullMQOtel("simple-guide")
         }
       );
-      await newQueue.setGlobalConcurrency(concurrency);
+      await newQueue.setGlobalConcurrency(queueConcurrency);
       this.queues.push({
         queue: newQueue,
         ratelimit,
-        concurrency
+        concurrency: {
+          worker: workerConcurrency,
+          queue: queueConcurrency
+        }
       });
       return {
         queue: newQueue,
         ratelimit,
-        concurrency
+        concurrency: {
+          worker: workerConcurrency,
+          queue: queueConcurrency
+        }
       };
     };
     this.list.set(name, {
       name,
-      concurrency,
+      concurrency: {
+        worker: workerConcurrency,
+        queue: queueConcurrency
+      },
       ratelimit,
       use
     });
@@ -9128,7 +9222,10 @@ var llmAsJudgeEval = () => {
         name: "prompt",
         description: "The prompt to send to the LLM as a judge, make sure to instruct the LLM to output a numerical score between 0 and 100. Add {actual_output} to the prompt to replace with the last message content, and {expected_output} to replace with the expected output."
       }],
-      queue: queues.register("llm_as_judge", 1, 1).use(),
+      queue: queues.register("llm_as_judge", {
+        worker: 1,
+        queue: 1
+      }, 1).use(),
       llm: true
     });
   }
@@ -9763,11 +9860,16 @@ var previewPdfTool = new ExuluTool2({
   type: "function",
   config: [],
   inputSchema: z5.object({
-    s3key: z5.string().describe("The S3 key of the PDF file to preview, can also optionally include a [bucket:name] to specify the bucket."),
+    s3key: z5.string().describe("The S3 key of the PDF file to preview."),
     page: z5.number().describe("The page number to preview, defaults to 1.").optional()
   }),
   execute: async ({ s3key, page, exuluConfig }) => {
-    const url = await getPresignedUrl(s3key, exuluConfig);
+    const bucket = s3key.split("/")[0];
+    const key = s3key.split("/").slice(1).join("/");
+    if (!bucket || !key) {
+      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
+    }
+    const url = await getPresignedUrl(bucket, key, exuluConfig);
     if (!url) {
       throw new Error("No URL provided for PDF preview");
     }
@@ -10161,7 +10263,10 @@ var ExuluApp = class {
     }
     const queueSet = /* @__PURE__ */ new Set();
     if (redisServer.host?.length && redisServer.port?.length) {
-      queues.register(global_queues.eval_runs, 1, 1);
+      queues.register(global_queues.eval_runs, {
+        worker: 1,
+        queue: 1
+      }, 1);
       for (const queue of queues.list.values()) {
         const config2 = await queue.use();
         queueSet.add(config2);