@exulu/backend 1.39.3 → 1.41.0

package/dist/index.cjs

@@ -48,7 +48,7 @@ __export(index_exports, {
   ExuluUtils: () => ExuluUtils,
   ExuluVariables: () => ExuluVariables,
   db: () => db2,
-  logMetadata: () => logMetadata2
+  logMetadata: () => logMetadata
 });
 module.exports = __toCommonJS(index_exports);
 var import_config = require("dotenv/config");
@@ -67,7 +67,6 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
-  console.log("[EXULU] redisServer:", redisServer);
   if (!redisServer.host || !redisServer.port) {
     return { client: null };
   }
@@ -148,7 +147,6 @@ var db = {};
 var databaseExistsChecked = false;
 var dbName = process.env.POSTGRES_DB_NAME || "exulu";
 async function ensureDatabaseExists() {
-  console.log(`[EXULU] Ensuring ${dbName} database exists...`);
   const defaultKnex = (0, import_knex.default)({
     client: "pg",
     connection: {
@@ -192,16 +190,7 @@ async function ensureDatabaseExists() {
 async function postgresClient() {
   if (!db["exulu"]) {
     try {
-      console.log(`[EXULU] Connecting to ${dbName} database.`);
-      console.log("[EXULU] POSTGRES_DB_HOST:", process.env.POSTGRES_DB_HOST);
-      console.log("[EXULU] POSTGRES_DB_PORT:", process.env.POSTGRES_DB_PORT);
-      console.log("[EXULU] POSTGRES_DB_USER:", process.env.POSTGRES_DB_USER);
-      console.log("[EXULU] POSTGRES_DB_PASSWORD:", process.env.POSTGRES_DB_PASSWORD);
-      console.log("[EXULU] POSTGRES_DB_NAME:", dbName);
-      console.log("[EXULU] POSTGRES_DB_SSL:", process.env.POSTGRES_DB_SSL);
-      console.log("[EXULU] Database exists checked:", databaseExistsChecked);
       if (!databaseExistsChecked) {
-        console.log(`[EXULU] Ensuring ${dbName} database exists...`);
         await ensureDatabaseExists();
         databaseExistsChecked = true;
       }
@@ -214,21 +203,37 @@ async function postgresClient() {
           database: dbName,
           password: process.env.POSTGRES_DB_PASSWORD,
           ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
-          connectionTimeoutMillis: 1e4
+          connectionTimeoutMillis: 3e4,
+          // Increased from 10s to 30s to handle connection spikes
+          // PostgreSQL statement timeout (in milliseconds) - kills queries that run too long
+          // This prevents runaway queries from blocking connections
+          statement_timeout: 18e5,
+          // 30 minutes - should be longer than max job timeout (1200s = 20m)
+          // Connection idle timeout - how long pg client waits before timing out
+          query_timeout: 18e5
+          // 30 minutes
         },
         pool: {
-          min: 2,
-          max: 20,
-          // Increased from 20 to handle more concurrent operations
-          acquireTimeoutMillis: 3e4,
+          min: 5,
+          // Increased from 2 to ensure enough connections available
+          max: 50,
+          // Increased from 20 to handle more concurrent operations with processor jobs
+          acquireTimeoutMillis: 6e4,
+          // Increased from 30s to 60s to handle pool contention
           createTimeoutMillis: 3e4,
-          idleTimeoutMillis: 3e4,
+          idleTimeoutMillis: 6e4,
+          // Increased to keep connections alive longer
           reapIntervalMillis: 1e3,
           createRetryIntervalMillis: 200,
           // Log pool events to help debug connection issues
           afterCreate: (conn, done) => {
             console.log("[EXULU] New database connection created");
-            done(null, conn);
+            conn.query("SET statement_timeout = 1800000", (err) => {
+              if (err) {
+                console.error("[EXULU] Error setting statement_timeout:", err);
+              }
+              done(err, conn);
+            });
           }
         }
       });
@@ -526,7 +531,6 @@ var authentication = async ({
   }
   if (authtoken) {
     try {
-      console.log("[EXULU] authtoken", authtoken);
       if (!authtoken?.email) {
         return {
           error: true,
@@ -1479,6 +1483,10 @@ var addCoreFields = (schema) => {
       field.name = field.name + "_s3key";
     }
   });
+  schema.fields.push({
+    name: "last_processed_at",
+    type: "date"
+  });
   if (schema.RBAC) {
     if (!schema.fields.some((field) => field.name === "rights_mode")) {
       schema.fields.push({
@@ -2361,33 +2369,38 @@ function createMutations(table, agents, contexts, tools, config) {
     }
   };
   if (table.type === "items") {
-    if (table.fields.some((field) => field.processor?.execute)) {
-      mutations[`${tableNameSingular}ProcessItemField`] = async (_, args, context, info) => {
+    if (table.processor) {
+      const contextItemProcessorMutation = async (context, items, user, role) => {
+        let jobs = [];
+        let results = [];
+        await Promise.all(items.map(async (item) => {
+          const result = await context.processField(
+            "api",
+            item,
+            config,
+            user,
+            role
+          );
+          if (result.job) {
+            jobs.push(result.job);
+          }
+          if (result.result) {
+            results.push(result.result);
+          }
+        }));
+        return {
+          message: jobs.length > 0 ? "Processing job scheduled." : "Items processed successfully.",
+          results: results.map((result) => JSON.stringify(result)),
+          jobs
+        };
+      };
+      mutations[`${tableNameSingular}ProcessItem`] = async (_, args, context, info) => {
         if (!context.user?.super_admin) {
           throw new Error("You are not authorized to process fields via API, user must be super admin.");
         }
-        const exists = contexts.find((context2) => context2.id === table.id);
-        if (!exists) {
-          throw new Error(`Context ${table.id} not found.`);
-        }
-        if (!args.field) {
-          throw new Error("Field argument missing, the field argument is required.");
-        }
         if (!args.item) {
           throw new Error("Item argument missing, the item argument is required.");
         }
-        const name = args.field?.replace("_s3key", "");
-        console.log("[EXULU] name", name);
-        console.log("[EXULU] fields", exists.fields.map((field2) => field2.name));
-        const field = exists.fields.find((field2) => {
-          return field2.name.replace("_s3key", "") === name;
-        });
-        if (!field) {
-          throw new Error(`Field ${name} not found in context ${exists.id}].`);
-        }
-        if (!field.processor) {
-          throw new Error(`Processor not set for field ${args.field} in context ${exists.id}.`);
-        }
         const { db: db3 } = context;
         let query = db3.from(tableNamePlural).select("*").where({ id: args.item });
         query = applyAccessControl(table, query, context.user);
@@ -2395,21 +2408,38 @@ function createMutations(table, agents, contexts, tools, config) {
         if (!item) {
           throw new Error("Item not found, or your user does not have access to it.");
         }
-        const { job, result } = await exists.processField(
-          "api",
-          {
-            ...item,
-            field: args.field
-          },
-          config,
+        const exists = contexts.find((context2) => context2.id === table.id);
+        if (!exists) {
+          throw new Error(`Context ${table.id} not found.`);
+        }
+        return contextItemProcessorMutation(exists, [item], context.user.id, context.user.role?.id);
+      };
+      mutations[`${tableNameSingular}ProcessItems`] = async (_, args, context, info) => {
+        if (!context.user?.super_admin) {
+          throw new Error("You are not authorized to process fields via API, user must be super admin.");
+        }
+        const { limit = 10, filters = [], sort } = args;
+        const { db: db3 } = context;
+        const { items } = await paginationRequest({
+          db: db3,
+          limit,
+          page: 0,
+          filters,
+          sort,
+          table,
+          user: context.user,
+          fields: "*"
+        });
+        const exists = contexts.find((context2) => context2.id === table.id);
+        if (!exists) {
+          throw new Error(`Context ${table.id} not found.`);
+        }
+        return contextItemProcessorMutation(
+          exists,
+          items,
           context.user.id,
           context.user.role?.id
         );
-        return {
-          message: job ? "Processing job scheduled." : "Item processed successfully.",
-          result,
-          job
-        };
       };
     }
     mutations[`${tableNameSingular}ExecuteSource`] = async (_, args, context, info) => {
@@ -2577,7 +2607,7 @@ function createMutations(table, agents, contexts, tools, config) {
   }
   return mutations;
 }
-var applyAccessControl = (table, query, user) => {
+var applyAccessControl = (table, query, user, field_prefix) => {
   const tableNamePlural = table.name.plural.toLowerCase();
   if (table.name.plural !== "agent_sessions" && user?.super_admin === true) {
     return query;
@@ -2593,18 +2623,19 @@ var applyAccessControl = (table, query, user) => {
   if (!hasRBAC) {
     return query;
   }
+  const prefix = field_prefix ? field_prefix + "." : "";
   try {
     query = query.where(function() {
-      this.where("rights_mode", "public");
-      this.orWhere("created_by", user.id);
+      this.where(`${prefix}rights_mode`, "public");
+      this.orWhere(`${prefix}created_by`, user.id);
       this.orWhere(function() {
-        this.where("rights_mode", "users").whereExists(function() {
+        this.where(`${prefix}rights_mode`, "users").whereExists(function() {
           this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
         });
       });
       if (user.role) {
         this.orWhere(function() {
-          this.where("rights_mode", "roles").whereExists(function() {
+          this.where(`${prefix}rights_mode`, "roles").whereExists(function() {
             this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
           });
         });
@@ -2616,9 +2647,11 @@ var applyAccessControl = (table, query, user) => {
   }
   return query;
 };
-var converOperatorToQuery = (query, fieldName, operators, table) => {
+var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) => {
   const field = table?.fields.find((f) => f.name === fieldName);
   const isJsonField = field?.type === "json";
+  const prefix = field_prefix ? field_prefix + "." : "";
+  fieldName = prefix + fieldName;
   if (operators.eq !== void 0) {
     if (isJsonField) {
       query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
@@ -2928,53 +2961,95 @@ var finalizeRequestedFields = async ({
         }
         const { db: db3 } = await postgresClient();
         const query = db3.from(getChunksTableName(context.id)).where({ source: result.id }).select("id", "content", "source", "chunk_index", "createdAt", "updatedAt");
-        query.select(
-          db3.raw("vector_dims(??) as embedding_size", [`embedding`])
-        );
         const chunks = await query;
         result.chunks = chunks.map((chunk) => ({
-          cosine_distance: 0,
-          fts_rank: 0,
-          hybrid_score: 0,
-          content: chunk.content,
-          source: chunk.source,
+          chunk_content: chunk.content,
+          chunk_source: chunk.source,
           chunk_index: chunk.chunk_index,
           chunk_id: chunk.id,
           chunk_created_at: chunk.createdAt,
           chunk_updated_at: chunk.updatedAt,
-          embedding_size: chunk.embedding_size
+          item_updated_at: chunk.item_updated_at,
+          item_created_at: chunk.item_created_at,
+          item_id: chunk.item_id,
+          item_external_id: chunk.item_external_id,
+          item_name: chunk.item_name
         }));
       }
     }
   }
   return result;
 };
-var applyFilters = (query, filters, table) => {
+var applyFilters = (query, filters, table, field_prefix) => {
   filters.forEach((filter) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
         if (operators.and !== void 0) {
           operators.and.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator, table);
+            query = converOperatorToQuery(query, fieldName, operator, table, field_prefix);
           });
         }
         if (operators.or !== void 0) {
           operators.or.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator, table);
+            query = converOperatorToQuery(query, fieldName, operator, table, field_prefix);
           });
         }
-        query = converOperatorToQuery(query, fieldName, operators, table);
+        query = converOperatorToQuery(query, fieldName, operators, table, field_prefix);
       }
     });
   });
   return query;
 };
-var applySorting = (query, sort) => {
+var applySorting = (query, sort, field_prefix) => {
+  const prefix = field_prefix ? field_prefix + "." : "";
   if (sort) {
+    sort.field = prefix + sort.field;
     query = query.orderBy(sort.field, sort.direction.toLowerCase());
   }
   return query;
 };
+var paginationRequest = async ({
+  db: db3,
+  limit,
+  page,
+  filters,
+  sort,
+  table,
+  user,
+  fields
+}) => {
+  if (limit > 1e4) {
+    throw new Error("Limit cannot be greater than 10.000.");
+  }
+  const tableName = table.name.plural.toLowerCase();
+  let countQuery = db3(tableName);
+  countQuery = applyFilters(countQuery, filters, table);
+  countQuery = applyAccessControl(table, countQuery, user);
+  const countResult = await countQuery.count("* as count");
+  const itemCount = Number(countResult[0]?.count || 0);
+  const pageCount = Math.ceil(itemCount / limit);
+  const currentPage = page;
+  const hasPreviousPage = currentPage > 1;
+  const hasNextPage = currentPage < pageCount - 1;
+  let dataQuery = db3(tableName);
+  dataQuery = applyFilters(dataQuery, filters, table);
+  dataQuery = applyAccessControl(table, dataQuery, user);
+  dataQuery = applySorting(dataQuery, sort);
+  if (page > 1) {
+    dataQuery = dataQuery.offset((page - 1) * limit);
+  }
+  let items = await dataQuery.select(fields ? fields : "*").limit(limit);
+  return {
+    items,
+    pageInfo: {
+      pageCount,
+      itemCount,
+      currentPage,
+      hasPreviousPage,
+      hasNextPage
+    }
+  };
+};
 function createQueries(table, agents, tools, contexts) {
   const tableNamePlural = table.name.plural.toLowerCase();
   const tableNameSingular = table.name.singular.toLowerCase();
@@ -3010,38 +3085,22 @@ function createQueries(table, agents, tools, contexts) {
       return finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNamePlural}Pagination`]: async (_, args, context, info) => {
-      const { limit = 10, page = 0, filters = [], sort } = args;
       const { db: db3 } = context;
-      if (limit > 500) {
-        throw new Error("Limit cannot be greater than 500.");
-      }
-      let countQuery = db3(tableNamePlural);
-      countQuery = applyFilters(countQuery, filters, table);
-      countQuery = applyAccessControl(table, countQuery, context.user);
-      const countResult = await countQuery.count("* as count");
-      const itemCount = Number(countResult[0]?.count || 0);
-      const pageCount = Math.ceil(itemCount / limit);
-      const currentPage = page;
-      const hasPreviousPage = currentPage > 1;
-      const hasNextPage = currentPage < pageCount - 1;
-      let dataQuery = db3(tableNamePlural);
-      dataQuery = applyFilters(dataQuery, filters, table);
-      dataQuery = applyAccessControl(table, dataQuery, context.user);
+      const { limit = 10, page = 0, filters = [], sort } = args;
       const requestedFields = getRequestedFields(info);
-      dataQuery = applySorting(dataQuery, sort);
-      if (page > 1) {
-        dataQuery = dataQuery.offset((page - 1) * limit);
-      }
       const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
-      let items = await dataQuery.select(sanitizedFields).limit(limit);
+      const { items, pageInfo } = await paginationRequest({
+        db: db3,
+        limit,
+        page,
+        filters,
+        sort,
+        table,
+        user: context.user,
+        fields: sanitizedFields
+      });
       return {
-        pageInfo: {
-          pageCount,
-          itemCount,
-          currentPage,
-          hasPreviousPage,
-          hasNextPage
-        },
+        pageInfo,
         items: finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result: items, user: context.user })
       };
     },
@@ -3090,7 +3149,7 @@ function createQueries(table, agents, tools, contexts) {
       }
       const { limit = 10, page = 0, filters = [], sort } = args;
       return await vectorSearch({
-        limit,
+        limit: limit || exists.configuration.maxRetrievalResults || 10,
         page,
         filters,
         sort,
@@ -3149,106 +3208,111 @@ var vectorSearch = async ({
   }
   const mainTable = getTableName(id);
   const chunksTable = getChunksTableName(id);
-  let countQuery = db3(mainTable);
-  countQuery = applyFilters(countQuery, filters, table);
-  countQuery = applyAccessControl(table, countQuery, user);
-  const columns = await db3(mainTable).columnInfo();
-  let itemsQuery = db3(mainTable).select(Object.keys(columns).map((column) => mainTable + "." + column));
-  itemsQuery = applyFilters(itemsQuery, filters, table);
-  itemsQuery = applyAccessControl(table, itemsQuery, user);
-  itemsQuery = applySorting(itemsQuery, sort);
+  let chunksQuery = db3(chunksTable + " as chunks").select([
+    "chunks.id as chunk_id",
+    "chunks.source",
+    "chunks.content",
+    "chunks.chunk_index",
+    db3.raw('chunks."createdAt" as chunk_created_at'),
+    db3.raw('chunks."updatedAt" as chunk_updated_at'),
+    "chunks.metadata",
+    "items.id as item_id",
+    "items.name as item_name",
+    "items.external_id as item_external_id",
+    db3.raw('items."updatedAt" as item_updated_at'),
+    db3.raw('items."createdAt" as item_created_at')
+  ]);
+  chunksQuery.leftJoin(mainTable + " as items", function() {
+    this.on("chunks.source", "=", "items.id");
+  });
+  chunksQuery = applyFilters(chunksQuery, filters, table, "items");
+  chunksQuery = applyAccessControl(table, chunksQuery, user, "items");
+  chunksQuery = applySorting(chunksQuery, sort, "items");
   if (queryRewriter) {
     query = await queryRewriter(query);
   }
-  itemsQuery.limit(limit * 3);
-  itemsQuery.leftJoin(chunksTable, function() {
-    this.on(chunksTable + ".source", "=", mainTable + ".id");
-  });
-  itemsQuery.select(chunksTable + ".id as chunk_id");
-  itemsQuery.select(chunksTable + ".source");
-  itemsQuery.select(chunksTable + ".content");
-  itemsQuery.select(chunksTable + ".chunk_index");
-  itemsQuery.select(chunksTable + ".createdAt as chunk_created_at");
-  itemsQuery.select(chunksTable + ".updatedAt as chunk_updated_at");
-  itemsQuery.select(db3.raw("vector_dims(??) as embedding_size", [`${chunksTable}.embedding`]));
-  const { chunks } = await embedder.generateFromQuery(context.id, query, {
+  const { chunks: queryChunks } = await embedder.generateFromQuery(context.id, query, {
     label: table.name.singular,
     trigger
   }, user?.id, role);
-  if (!chunks?.[0]?.vector) {
+  if (!queryChunks?.[0]?.vector) {
     throw new Error("No vector generated for query.");
   }
-  const vector = chunks[0].vector;
+  const vector = queryChunks[0].vector;
   const vectorStr = `ARRAY[${vector.join(",")}]`;
   const vectorExpr = `${vectorStr}::vector`;
   const language = configuration.language || "english";
-  let items = [];
+  let resultChunks = [];
   switch (method) {
     case "tsvector":
-      itemsQuery.select(db3.raw(
-        `ts_rank(${chunksTable}.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
+      chunksQuery.limit(limit * 2);
+      chunksQuery.select(db3.raw(
+        `ts_rank(chunks.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
         [language, query]
       )).whereRaw(
-        `${chunksTable}.fts @@ websearch_to_tsquery(?, ?)`,
+        `chunks.fts @@ websearch_to_tsquery(?, ?)`,
         [language, query]
       ).orderByRaw(`fts_rank DESC`);
-      items = await itemsQuery;
+      resultChunks = await chunksQuery;
       break;
     case "cosineDistance":
-    default:
-      itemsQuery.whereNotNull(`${chunksTable}.embedding`);
-      itemsQuery.select(
-        db3.raw(`1 - (${chunksTable}.embedding <=> ${vectorExpr}) AS cosine_distance`)
+      chunksQuery.limit(limit * 2);
+      chunksQuery.whereNotNull(`chunks.embedding`);
+      console.log("[EXULU] Chunks query:", chunksQuery.toQuery());
+      chunksQuery.select(
+        db3.raw(`1 - (chunks.embedding <=> ${vectorExpr}) AS cosine_distance`)
       );
-      itemsQuery.orderByRaw(
-        `${chunksTable}.embedding <=> ${vectorExpr} ASC NULLS LAST`
+      chunksQuery.orderByRaw(
+        `chunks.embedding <=> ${vectorExpr} ASC NULLS LAST`
       );
-      items = await itemsQuery;
+      resultChunks = await chunksQuery;
       break;
     case "hybridSearch":
-      const matchCount = Math.min(limit * 5, 100);
+      const matchCount = Math.min(limit * 2, 100);
       const fullTextWeight = 1;
       const semanticWeight = 1;
       const rrfK = 50;
       const hybridSQL = `
             WITH full_text AS (
               SELECT
-                c.id,
-                c.source,
+                chunks.id,
+                chunks.source,
                 row_number() OVER (
-                  ORDER BY ts_rank_cd(c.fts, websearch_to_tsquery(?, ?)) DESC
+                  ORDER BY ts_rank(chunks.fts, websearch_to_tsquery(?, ?)) DESC
                 ) AS rank_ix
-              FROM ${chunksTable} c
-              WHERE c.fts @@ websearch_to_tsquery(?, ?)
+              FROM ${chunksTable} as chunks
+              WHERE chunks.fts @@ websearch_to_tsquery(?, ?)
               ORDER BY rank_ix
               LIMIT LEAST(?, 15) * 2
             ),
             semantic AS (
               SELECT
-                c.id,
-                c.source,
+                chunks.id,
+                chunks.source,
                 row_number() OVER (
-                  ORDER BY c.embedding <=> ${vectorExpr} ASC
+                  ORDER BY chunks.embedding <=> ${vectorExpr} ASC
                 ) AS rank_ix
-              FROM ${chunksTable} c
-              WHERE c.embedding IS NOT NULL
+              FROM ${chunksTable} as chunks
+              WHERE chunks.embedding IS NOT NULL
               ORDER BY rank_ix
               LIMIT LEAST(?, 50) * 2
             )
             SELECT
-              m.*,
-              c.id AS chunk_id,
-              c.source,
-              c.content,
-              c.chunk_index,
-              c.metadata,
-              c."createdAt" AS chunk_created_at,
-              c."updatedAt" AS chunk_updated_at,
-              vector_dims(c.embedding) as embedding_size,
-        
+              items.id as item_id,
+              items.name as item_name,
+              items.external_id as item_external_id,
+              chunks.id AS chunk_id,
+              chunks.source,
+              chunks.content,
+              chunks.chunk_index,
+              chunks.metadata,
+              chunks."createdAt" as chunk_created_at,
+              chunks."updatedAt" as chunk_updated_at,
+              items."updatedAt" as item_updated_at,
+              items."createdAt" as item_created_at,
               /* Per-signal scores for introspection */
-              ts_rank(c.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
-              (1 - (c.embedding <=> ${vectorExpr})) AS cosine_distance,
+              ts_rank(chunks.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
+              (1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
         
               /* Hybrid RRF score */
               (
@@ -3260,10 +3324,10 @@ var vectorSearch = async ({
             FROM full_text ft
             FULL OUTER JOIN semantic se
               ON ft.id = se.id
-            JOIN ${chunksTable} c
-              ON COALESCE(ft.id, se.id) = c.id
-            JOIN ${mainTable} m
-              ON m.id = c.source
+            JOIN ${chunksTable} as chunks
+              ON COALESCE(ft.id, se.id) = chunks.id
+            JOIN ${mainTable} as items
+              ON items.id = chunks.source
             ORDER BY hybrid_score DESC
             LIMIT LEAST(?, 50)
             OFFSET 0
@@ -3289,86 +3353,57 @@ var vectorSearch = async ({
         matchCount
         // final limit
       ];
-      items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
-  }
-  console.log("[EXULU] Vector search results:", items?.length);
-  const seenSources = /* @__PURE__ */ new Map();
-  items = items.reduce((acc, item) => {
-    if (!seenSources.has(item.source)) {
-      seenSources.set(item.source, {
-        ...Object.fromEntries(
-          Object.keys(item).filter(
-            (key) => key !== "cosine_distance" && // kept per chunk below
-            key !== "fts_rank" && // kept per chunk below
-            key !== "hybrid_score" && // we will compute per item below
-            key !== "content" && key !== "source" && key !== "chunk_index" && key !== "chunk_id" && key !== "chunk_created_at" && key !== "chunk_updated_at" && key !== "embedding_size"
-          ).map((key) => [key, item[key]])
-        ),
-        chunks: [{
-          content: item.content,
-          chunk_index: item.chunk_index,
-          chunk_id: item.chunk_id,
-          source: item.source,
-          metadata: item.metadata,
-          chunk_created_at: item.chunk_created_at,
-          chunk_updated_at: item.chunk_updated_at,
-          embedding_size: item.embedding_size,
-          ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-          ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-          ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-        }]
-      });
-      acc.push(seenSources.get(item.source));
-    } else {
-      seenSources.get(item.source).chunks.push({
-        content: item.content,
-        chunk_index: item.chunk_index,
-        chunk_id: item.chunk_id,
-        chunk_created_at: item.chunk_created_at,
-        embedding_size: item.embedding_size,
-        metadata: item.metadata,
-        source: item.source,
-        chunk_updated_at: item.chunk_updated_at,
-        ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-        ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-        ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-      });
-    }
-    return acc;
-  }, []);
-  console.log("[EXULU] Vector search results after deduplication:", items?.length);
-  items.forEach((item) => {
-    if (!item.chunks?.length) {
-      return;
-    }
-    if (method === "tsvector") {
-      const ranks = item.chunks.map((c) => typeof c.fts_rank === "number" ? c.fts_rank : 0);
-      const total = ranks.reduce((a, b) => a + b, 0);
-      const average = ranks.length ? total / ranks.length : 0;
-      item.averageRelevance = average;
-      item.totalRelevance = total;
-    } else if (method === "cosineDistance") {
-      let methodProperty = "cosine_distance";
-      const average = item.chunks.reduce((acc, item2) => {
-        return acc + item2[methodProperty];
-      }, 0) / item.chunks.length;
-      const total = item.chunks.reduce((acc, item2) => {
-        return acc + item2[methodProperty];
-      }, 0);
-      item.averageRelevance = average;
-      item.totalRelevance = total;
-    } else if (method === "hybridSearch") {
-      const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score * 10 + 1 : 0);
-      const total = scores.reduce((a, b) => a + b, 0);
-      const average = scores.length ? total / scores.length : 0;
-      item.averageRelevance = average;
-      item.totalRelevance = total;
+      resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
+  }
+  console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
+  resultChunks = resultChunks.map((chunk) => ({
+    chunk_content: chunk.content,
+    chunk_index: chunk.chunk_index,
+    chunk_id: chunk.chunk_id,
+    chunk_source: chunk.source,
+    chunk_metadata: chunk.metadata,
+    chunk_created_at: chunk.chunk_created_at,
+    chunk_updated_at: chunk.chunk_updated_at,
+    item_updated_at: chunk.item_updated_at,
+    item_created_at: chunk.item_created_at,
+    item_id: chunk.item_id,
+    item_external_id: chunk.item_external_id,
+    item_name: chunk.item_name,
+    context: {
+      name: table.name.singular,
+      id: table.id || ""
+    },
+    ...method === "cosineDistance" && { chunk_cosine_distance: chunk.cosine_distance },
+    ...(method === "tsvector" || method === "hybridSearch") && { chunk_fts_rank: chunk.fts_rank },
+    ...method === "hybridSearch" && { chunk_hybrid_score: chunk.hybrid_score }
+  }));
+  if (resultChunks.length > 0 && (method === "cosineDistance" || method === "hybridSearch")) {
+    const scoreKey = method === "cosineDistance" ? "chunk_cosine_distance" : "chunk_hybrid_score";
+    const topScore = resultChunks[0][scoreKey];
+    const bottomScore = resultChunks[resultChunks.length - 1][scoreKey];
+    const medianScore = resultChunks[Math.floor(resultChunks.length / 2)][scoreKey];
+    console.log("[EXULU] Score distribution:", {
+      method,
+      count: resultChunks.length,
+      topScore: topScore?.toFixed(4),
+      bottomScore: bottomScore?.toFixed(4),
+      medianScore: medianScore?.toFixed(4)
+    });
+    const adaptiveThreshold = topScore * 0.7;
+    const beforeFilterCount = resultChunks.length;
+    resultChunks = resultChunks.filter((chunk) => {
+      const score = chunk[scoreKey];
+      return score !== void 0 && score >= adaptiveThreshold;
+    });
+    const filteredCount = beforeFilterCount - resultChunks.length;
+    if (filteredCount > 0) {
+      console.log(`[EXULU] Filtered ${filteredCount} low-quality results (threshold: ${adaptiveThreshold.toFixed(4)})`);
     }
-  });
+  }
   if (resultReranker && query) {
-    items = await resultReranker(items);
+    resultChunks = await resultReranker(resultChunks);
   }
-  console.log("[EXULU] Vector search results after slicing:", items?.length);
+  resultChunks = resultChunks.slice(0, limit);
   await updateStatistic({
     name: "count",
     label: table.name.singular,
@@ -3386,7 +3421,7 @@ var vectorSearch = async ({
       id: table.id || "",
       embedder: embedder.name
     },
-    items
+    chunks: resultChunks
   };
 };
 var RBACResolver = async (db3, entityName, resourceId, rights_mode) => {
@@ -3416,10 +3451,10 @@ var contextToTableDefinition = (context) => {
       plural: tableName?.endsWith("s") ? tableName : tableName + "s"
     },
     RBAC: true,
+    processor: context.processor,
     fields: context.fields.map((field) => ({
       name: sanitizeName(field.name),
       type: field.type,
-      processor: field.processor,
       required: field.required,
       default: field.default,
       index: field.index,
@@ -3549,7 +3584,6 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     const tableNamePlural = table.name.plural.toLowerCase();
     const tableNameSingular = table.name.singular.toLowerCase();
     const tableNameSingularUpperCaseFirst = table.name.singular.charAt(0).toUpperCase() + table.name.singular.slice(1);
-    const processorFields = table.fields.filter((field) => field.processor?.execute);
     typeDefs += `
         ${tableNameSingular === "agent" ? `${tableNameSingular}ById(id: ID!, project: ID): ${tableNameSingular}` : `${tableNameSingular}ById(id: ID!): ${tableNameSingular}`}
       
@@ -3576,9 +3610,10 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     ${tableNameSingular}ExecuteSource(source: ID!, inputs: JSON!): ${tableNameSingular}ExecuteSourceReturnPayload
     ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}DeleteChunksReturnPayload
     `;
-      if (processorFields?.length > 0) {
+      if (table.processor) {
         mutationDefs += `
-    ${tableNameSingular}ProcessItemField(item: ID!, field: ${tableNameSingular}ProcessorFieldEnum!): ${tableNameSingular}ProcessItemFieldReturnPayload
+    ${tableNameSingular}ProcessItem(item: ID!): ${tableNameSingular}ProcessItemFieldReturnPayload
+    ${tableNameSingular}ProcessItems(limit: Int, filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingular}ProcessItemFieldReturnPayload
     `;
       }
       modelDefs += `
@@ -3596,8 +3631,8 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
 
     type ${tableNameSingular}ProcessItemFieldReturnPayload {
         message: String!
-        result: String!
-        job: String
+        results: [String]
+        jobs: [String]
     }
 
     type ${tableNameSingular}DeleteChunksReturnPayload {
@@ -3612,20 +3647,31 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
         tsvector
     }
 
-    ${processorFields.length > 0 ? `
-    enum ${tableNameSingular}ProcessorFieldEnum {
-        ${processorFields.map((field) => field.name).join("\n")}
-    }
-    ` : ""}
- 
-
-  type ${tableNameSingular}VectorSearchResult {
-        items: [${tableNameSingular}]!
+    type ${tableNameSingular}VectorSearchResult {
+        chunks: [${tableNameSingular}VectorSearchChunk!]!
         context: VectoSearchResultContext!
         filters: JSON!
         query: String!
         method: VectorMethodEnum!
     }
+        
+    type ${tableNameSingular}VectorSearchChunk {
+        chunk_content: String
+        chunk_index: Int
+        chunk_id: String
+        chunk_source: String
+        chunk_metadata: JSON
+        chunk_created_at: Date
+        chunk_updated_at: Date
+        item_updated_at: Date
+        item_created_at: Date
+        item_id: String!
+        item_external_id: String
+        item_name: String!
+        chunk_cosine_distance: Float
+        chunk_fts_rank: Float
+        chunk_hybrid_score: Float
+    }
 
     type VectoSearchResultContext {
         name: String!
@@ -3730,7 +3776,11 @@ type PageInfo {
     const config2 = await queue.use();
     return {
       name: config2.queue.name,
-      concurrency: config2.concurrency,
+      concurrency: {
+        worker: config2.concurrency?.worker || void 0,
+        queue: config2.concurrency?.queue || void 0
+      },
+      timeoutInSeconds: config2.timeoutInSeconds,
       ratelimit: config2.ratelimit,
       isMaxed: await config2.queue.isMaxed(),
       isPaused: await config2.queue.isPaused(),
@@ -3780,7 +3830,10 @@ type PageInfo {
     if (!agentInstance) {
       throw new Error("Agent instance not found for eval run.");
     }
-    const evalQueue = await queues.register("eval_runs", 1, 1).use();
+    const evalQueue = await queues.register("eval_runs", {
+      worker: 1,
+      queue: 1
+    }, 1).use();
     const jobIds = [];
     for (const testCase of testCases) {
       const jobData = {
@@ -3904,7 +3957,6 @@ type PageInfo {
     if (!client2) {
       throw new Error("Redis client not created properly");
     }
-    console.log("[EXULU] Jobs pagination args", args);
     const {
       jobs,
       count
@@ -3914,7 +3966,6 @@ type PageInfo {
       args.page || 1,
       args.limit || 100
     );
-    console.log("[EXULU] jobs", jobs.map((job) => job.name));
     const requestedFields = getRequestedFields(info);
     return {
       items: await Promise.all(jobs.map(async (job) => {
@@ -3943,6 +3994,21 @@ type PageInfo {
   };
   resolvers.Query["contexts"] = async (_, args, context, info) => {
     const data = await Promise.all(contexts.map(async (context2) => {
+      let processor = null;
+      if (context2.processor) {
+        processor = await new Promise(async (resolve, reject) => {
+          const config2 = await context2.processor?.config;
+          const queue = await config2?.queue;
+          resolve({
+            name: context2.processor.name,
+            description: context2.processor.description,
+            queue: queue?.queue?.name || void 0,
+            trigger: context2.processor?.config?.trigger || "manual",
+            timeoutInSeconds: queue?.timeoutInSeconds || 600,
+            generateEmbeddings: context2.processor?.config?.generateEmbeddings || false
+          });
+        });
+      }
       const sources = await Promise.all(context2.sources.map(async (source) => {
         let queueName = void 0;
         if (source.config) {
@@ -3974,6 +4040,7 @@ type PageInfo {
         slug: "/contexts/" + context2.id,
         active: context2.active,
         sources,
+        processor,
         fields: context2.fields.map((field) => {
           return {
             ...field,
@@ -3999,6 +4066,21 @@ type PageInfo {
     if (!data) {
       return null;
     }
+    let processor = null;
+    if (data.processor) {
+      processor = await new Promise(async (resolve, reject) => {
+        const config2 = await data.processor?.config;
+        const queue = await config2?.queue;
+        resolve({
+          name: data.processor.name,
+          description: data.processor.description,
+          queue: queue?.queue?.name || void 0,
+          trigger: data.processor?.config?.trigger || "manual",
+          timeoutInSeconds: queue?.timeoutInSeconds || 600,
+          generateEmbeddings: data.processor?.config?.generateEmbeddings || false
+        });
+      });
+    }
     const sources = await Promise.all(data.sources.map(async (source) => {
       let queueName = void 0;
       if (source.config) {
@@ -4035,35 +4117,18 @@ type PageInfo {
       slug: "/contexts/" + data.id,
       active: data.active,
       sources,
+      processor,
       fields: await Promise.all(data.fields.map(async (field) => {
         const label = field.name?.replace("_s3key", "");
         if (field.type === "file" && !field.name.endsWith("_s3key")) {
           field.name = field.name + "_s3key";
         }
-        let queue = null;
-        if (field.processor?.config?.queue) {
-          queue = await field.processor.config.queue;
-        }
         return {
           ...field,
           name: sanitizeName(field.name),
           ...field.type === "file" ? {
             allowedFileTypes: field.allowedFileTypes
           } : {},
-          ...field.processor ? {
-            processor: {
-              description: field.processor?.description,
-              config: {
-                trigger: field.processor?.config?.trigger,
-                queue: {
-                  name: queue?.queue.name || void 0,
-                  ratelimit: queue?.ratelimit || void 0,
-                  concurrency: queue?.concurrency || void 0
-                }
-              },
-              execute: "function"
-            }
-          } : {},
           label
         };
       })),
@@ -4131,13 +4196,20 @@ type PageInfo {
   modelDefs += `
     type QueueResult {
         name: String!
-        concurrency: Int!
+        concurrency: QueueConcurrency!
+        timeoutInSeconds: Int!
         ratelimit: Int!
         isMaxed: Boolean!
         isPaused: Boolean!
         jobs: QueueJobsCounts
     }
     `;
+  modelDefs += `
+    type QueueConcurrency {
+        worker: Int
+        queue: Int
+    }
+    `;
   modelDefs += `
     type QueueJobsCounts {
         paused: Int!
@@ -4207,17 +4279,12 @@ type AgentEvalFunctionConfig {
 }
 
 type ItemChunks {
-    cosine_distance: Float
-    fts_rank: Float
-    hybrid_score: Float
-    content: String
-    source: ID
-    chunk_index: Int
-    chunk_id: ID
-    chunk_created_at: Date
-    chunk_updated_at: Date
-    embedding_size: Float
-    metadata: JSON
+    chunk_id: String!
+    chunk_index: Int!
+    chunk_content: String!
+    chunk_source: String!
+    chunk_created_at: Date!
+    chunk_updated_at: Date!
 }
 
 type Provider {
@@ -4252,7 +4319,8 @@ type Context {
     active: Boolean
     fields: JSON
     configuration: JSON
-    sources: [ContextSource!]
+    sources: [ContextSource]
+    processor: ContextProcessor
 }
 type Embedder {
     name: String!
@@ -4265,6 +4333,14 @@ type EmbedderConfig {
     description: String
     default: String
 }
+type ContextProcessor {
+    name: String!
+    description: String
+    queue: String
+    trigger: String
+    timeoutInSeconds: Int
+    generateEmbeddings: Boolean
+}
 
 type ContextSource {
     id: String!
@@ -4322,6 +4398,9 @@ type Job {
   name: String!
   returnvalue: JSON
   stacktrace: [String]
+  finishedOn: Date
+  processedOn: Date
+  attemptsMade: Int
   failedReason: String
   state: String!
   data: JSON
@@ -4383,10 +4462,7 @@ async function getJobsByQueueName(queueName, statusses, page, limit) {
   const config = await queue.use();
   const startIndex = (page || 1) - 1;
   const endIndex = startIndex - 1 + (limit || 100);
-  console.log("[EXULU] Jobs pagination startIndex", startIndex);
-  console.log("[EXULU] Jobs pagination endIndex", endIndex);
   const jobs = await config.queue.getJobs(statusses || [], startIndex, endIndex, false);
-  console.log("[EXULU] Jobs pagination jobs", jobs?.length);
   const counts = await config.queue.getJobCounts(...statusses || []);
   let total = 0;
   if (counts) {
@@ -4427,21 +4503,12 @@ function getS3Client(config) {
   });
   return s3Client;
 }
-var getPresignedUrl = async (key, config) => {
+var getPresignedUrl = async (bucket, key, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
-  let bucket = config.fileUploads.s3Bucket;
-  if (key.includes("[bucket:")) {
-    console.log("[EXULU] key includes [bucket:name]", key);
-    bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-    if (!bucket?.length) {
-      throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-    }
-    key = key.split("]")[1] || "";
-    console.log("[EXULU] bucket", bucket);
-    console.log("[EXULU] key", key);
-  }
+  console.log("[EXULU] getting presigned url for bucket", bucket);
+  console.log("[EXULU] getting presigned url for key", key);
   const url = await (0, import_s3_request_presigner.getSignedUrl)(
     getS3Client(config),
     new import_client_s3.GetObjectCommand({
@@ -4452,7 +4519,7 @@ var getPresignedUrl = async (key, config) => {
   );
   return url;
 };
-var addPrefixToKey = (keyPath, config) => {
+var addGeneralPrefixToKey = (keyPath, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
@@ -4465,58 +4532,50 @@ var addPrefixToKey = (keyPath, config) => {
   }
   return `${prefix}/${keyPath}`;
 };
-var uploadFile = async (file, key, config, options = {}, user) => {
+var addUserPrefixToKey = (key, user) => {
+  if (!user) {
+    return key;
+  }
+  if (key.includes(`/user_${user}/`)) {
+    return key;
+  }
+  return `user_${user}/${key}`;
+};
+var addBucketPrefixToKey = (key, bucket) => {
+  if (key.includes(`/${bucket}/`)) {
+    return key;
+  }
+  return `${bucket}/${key}`;
+};
+var uploadFile = async (file, fileName, config, options = {}, user, customBucket) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured (in the exported uploadFile function)");
   }
   const client2 = getS3Client(config);
   let defaultBucket = config.fileUploads.s3Bucket;
-  let customBucket = false;
-  if (key.includes("[bucket:")) {
-    console.log("[EXULU] key includes [bucket:name]", key);
-    customBucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-    if (!customBucket?.length) {
-      throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-    }
-    key = key.split("]")[1] || "";
-    console.log("[EXULU] custom bucket", customBucket);
-  }
-  let folder = user ? `${user}/` : "";
-  const fullKey = addPrefixToKey(!key.includes(folder) ? folder + key : key, config);
-  console.log("[EXULU] uploading file to s3 into bucket", customBucket || defaultBucket, "with key", fullKey);
+  let key = fileName;
+  key = addGeneralPrefixToKey(key, config);
+  key = addUserPrefixToKey(key, user || "api");
+  console.log("[EXULU] uploading file to s3 into bucket", defaultBucket, "with key", key);
   const command = new import_client_s3.PutObjectCommand({
     Bucket: customBucket || defaultBucket,
-    Key: fullKey,
+    Key: key,
     Body: file,
     ContentType: options.contentType,
     Metadata: options.metadata,
     ContentLength: file.byteLength
   });
   await client2.send(command);
-  console.log("[EXULU] file uploaded to s3 into bucket", customBucket || defaultBucket, "with key", fullKey);
-  if (customBucket) {
-    return "[bucket:" + customBucket + "]" + fullKey;
-  }
-  return fullKey;
+  console.log("[EXULU] file uploaded to s3 into bucket", customBucket || defaultBucket, "with key", key);
+  return addBucketPrefixToKey(
+    key,
+    customBucket || defaultBucket
+  );
 };
-var createUppyRoutes = async (app, config) => {
+var createUppyRoutes = async (app, contexts, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
-  const extractUserPrefix = (key) => {
-    if (!config.fileUploads) {
-      throw new Error("File uploads are not configured");
-    }
-    if (!config.fileUploads.s3prefix) {
-      return key.split("/")[0];
-    }
-    const prefix = config.fileUploads.s3prefix.replace(/\/$/, "");
-    if (key.startsWith(prefix + "/")) {
-      const keyWithoutPrefix = key.slice(prefix.length + 1);
-      return keyWithoutPrefix.split("/")[0];
-    }
-    return key.split("/")[0];
-  };
   const policy = {
     Version: "2012-10-17",
     Statement: [
@@ -4568,20 +4627,16 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
+    const user = authenticationResult.user;
     let { key } = req.query;
     if (typeof key !== "string" || key.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    let bucket = config.fileUploads.s3Bucket;
-    if (key.includes("[bucket:")) {
-      console.log("[EXULU] key includes [bucket:name]", key);
-      bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-      if (!bucket?.length) {
-        throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-      }
-      key = key.split("]")[1] || "";
-      console.log("[EXULU] bucket", bucket);
+    let bucket = key.split("/")[0];
+    if (user.type !== "api" && !key.includes(`/user_${user.id}/`) && !user.super_admin) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
     }
     const client2 = getS3Client(config);
     const command = new import_client_s3.DeleteObjectCommand({
@@ -4609,13 +4664,32 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
-    const { key } = req.query;
+    const user = authenticationResult.user;
+    let { key } = req.query;
+    if (!key || typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
+      return;
+    }
+    let bucket = key.split("/")[0];
+    if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `bucket` (should be the first part of the key before the first slash)." });
+      return;
+    }
+    key = key.split("/").slice(1).join("/");
     if (typeof key !== "string" || key.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
+    let allowed = false;
+    if (user.type === "api" || user.super_admin || key.includes(`user_${user.id}/`)) {
+      allowed = true;
+    }
+    if (!allowed) {
+      res.status(405).json({ error: "Not allowed to access the file based on authenticated user." });
+      return;
+    }
     try {
-      const url = await getPresignedUrl(key, config);
+      const url = await getPresignedUrl(bucket, key, config);
       res.setHeader("Access-Control-Allow-Origin", "*");
       res.json({ url, method: "GET", expiresIn });
     } catch (err) {
@@ -4644,16 +4718,15 @@ var createUppyRoutes = async (app, config) => {
       return;
     }
     let { key } = req.body;
-    let bucket = config.fileUploads.s3Bucket;
-    if (key.includes("[bucket:")) {
-      console.log("[EXULU] key includes [bucket:name]", key);
-      bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
-      console.log("[EXULU] bucket", bucket);
-      if (!bucket?.length) {
-        throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
-      }
-      key = key.split("]")[1] || "";
-      console.log("[EXULU] key", key);
+    let bucket = key.split("/")[0];
+    if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `bucket` (should be the first part of the key before the first slash)." });
+      return;
+    }
+    key = key.split("/").slice(1).join("/");
+    if (!key || typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
+      return;
     }
     const client2 = getS3Client(config);
     const command = new import_client_s3.HeadObjectCommand({
@@ -4757,11 +4830,12 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
+    const user = authenticationResult.user;
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
     const key = generateS3Key2(filename);
-    let folder = `${authenticationResult.user.id}/`;
-    const fullKey = addPrefixToKey(folder + key, config);
+    let fullKey = addGeneralPrefixToKey(key, config);
+    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
     (0, import_s3_request_presigner.getSignedUrl)(
       getS3Client(config),
       new import_client_s3.PutObjectCommand({
@@ -4805,6 +4879,7 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
+    const user = authenticationResult.user;
     const client2 = getS3Client(config);
     const { type, metadata, filename } = req.body;
     if (typeof filename !== "string") {
@@ -4814,13 +4889,8 @@ var createUppyRoutes = async (app, config) => {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
     const key = `${(0, import_node_crypto.randomUUID)()}-_EXULU_${filename}`;
-    let folder = "";
-    if (authenticationResult.user.type === "api") {
-      folder = `api/`;
-    } else {
-      folder = `${authenticationResult.user.id}/`;
-    }
-    const fullKey = addPrefixToKey(folder + key, config);
+    let fullKey = addGeneralPrefixToKey(key, config);
+    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
     const params = {
       Bucket: config.fileUploads.s3Bucket,
       Key: fullKey,
@@ -5023,7 +5093,7 @@ var createProjectRetrievalTool = async ({
         if (!context) {
           throw new Error("The item added to the project does not have a valid gid with the context id as the prefix before the first slash.");
         }
-        const id = item.split("/")[1];
+        const id = item.split("/").slice(1).join("/");
         if (set[context]) {
           set[context].push(id);
         } else {
@@ -5969,12 +6039,20 @@ var ExuluStorage = class {
     this.config = config;
   }
   getPresignedUrl = async (key) => {
-    return await getPresignedUrl(key, this.config);
+    const bucket = key.split("/")[0];
+    if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
+      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
+    }
+    key = key.split("/").slice(1).join("/");
+    if (!key || typeof key !== "string" || key.trim() === "") {
+      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
+    }
+    return await getPresignedUrl(bucket, key, this.config);
   };
-  uploadFile = async (file, key, type, user, metadata) => {
+  uploadFile = async (file, fileName, type, user, metadata, customBucket) => {
     return await uploadFile(
       file,
-      key,
+      fileName,
       this.config,
       {
         contentType: type,
@@ -5983,7 +6061,8 @@ var ExuluStorage = class {
           type
         }
       },
-      user
+      user,
+      customBucket
     );
   };
   // todo add upload and delete methods
@@ -5996,12 +6075,12 @@ var ExuluContext = class {
   name;
   active;
   fields;
+  processor;
   rateLimit;
   description;
   embedder;
   queryRewriter;
   resultReranker;
-  // todo typings
   configuration;
   sources = [];
   constructor({
@@ -6009,6 +6088,7 @@ var ExuluContext = class {
     name,
     description,
     embedder,
+    processor,
     active,
     rateLimit,
     fields,
@@ -6021,10 +6101,12 @@ var ExuluContext = class {
     this.name = name;
     this.fields = fields || [];
     this.sources = sources || [];
+    this.processor = processor;
     this.configuration = configuration || {
       calculateVectors: "manual",
       language: "english",
-      defaultRightsMode: "private"
+      defaultRightsMode: "private",
+      maxRetrievalResults: 10
     };
     this.description = description;
     this.embedder = embedder;
@@ -6034,23 +6116,18 @@ var ExuluContext = class {
     this.resultReranker = resultReranker;
   }
   processField = async (trigger, item, exuluConfig, user, role) => {
-    console.log("[EXULU] processing field", item.field, " in context", this.id);
-    console.log("[EXULU] fields", this.fields.map((field2) => field2.name));
-    const field = this.fields.find((field2) => {
-      return field2.name.replace("_s3key", "") === item.field.replace("_s3key", "");
-    });
-    if (!field || !field.processor) {
-      console.error("[EXULU] field not found or processor not set for field", item.field, " in context", this.id);
-      throw new Error("Field not found or processor not set for field " + item.field + " in context " + this.id);
-    }
+    console.log("[EXULU] processing item, ", item, " in context", this.id);
     const exuluStorage = new ExuluStorage({ config: exuluConfig });
-    const queue = await field.processor.config?.queue;
+    if (!this.processor) {
+      throw new Error(`Processor is not set for this context: ${this.id}.`);
+    }
+    const queue = await this.processor.config?.queue;
     if (queue?.queue.name) {
       console.log("[EXULU] processor is in queue mode, scheduling job.");
       const job = await bullmqDecorator({
-        timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
-        label: `${this.name} ${field.name} data processor`,
-        processor: `${this.id}-${field.name}`,
+        timeoutInSeconds: this.processor.config?.timeoutInSeconds || 600,
+        label: `${this.name} ${this.processor.name} data processor`,
+        processor: `${this.id}-${this.processor.name}`,
         context: this.id,
         inputs: item,
         item: item.id,
@@ -6065,27 +6142,33 @@ var ExuluContext = class {
         trigger
       });
       return {
-        result: "",
+        result: void 0,
         job: job.id
       };
     }
     console.log("[EXULU] POS 1 -- EXULU CONTEXT PROCESS FIELD");
-    const result = await field.processor.execute({
+    const processorResult = await this.processor.execute({
       item,
       user,
       role,
       utils: {
-        storage: exuluStorage,
-        items: {
-          update: this.updateItem,
-          create: this.createItem,
-          delete: this.deleteItem
-        }
+        storage: exuluStorage
       },
       exuluConfig
     });
+    if (!processorResult) {
+      throw new Error("Processor result is required for updating the item in the db.");
+    }
+    const { db: db3 } = await postgresClient();
+    delete processorResult.field;
+    await db3.from(getTableName(this.id)).where({
+      id: processorResult.id
+    }).update({
+      ...processorResult,
+      last_processed_at: (/* @__PURE__ */ new Date()).toISOString()
+    });
     return {
-      result,
+      result: processorResult,
       job: void 0
     };
   };
@@ -6096,7 +6179,8 @@ var ExuluContext = class {
       user: options.user,
       role: options.role,
       context: this,
-      db: db3
+      db: db3,
+      limit: options?.limit || this.configuration.maxRetrievalResults || 10
     });
     return result;
   };
@@ -6170,6 +6254,8 @@ var ExuluContext = class {
     };
   };
   createItem = async (item, config, user, role, upsert, generateEmbeddingsOverwrite) => {
+    console.log("[EXULU] creating item", item);
+    console.log("[EXULU] upsert", upsert);
     if (upsert && (!item.id && !item.external_id)) {
       throw new Error("Item id or external id is required for upsert.");
     }
@@ -6197,10 +6283,9 @@ var ExuluContext = class {
     }
     console.log("[EXULU] context configuration", this.configuration);
     let jobs = [];
-    let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always");
-    for (const [key, value] of Object.entries(item)) {
-      console.log("[EXULU] Checking for processors for field", key);
-      const processor = this.fields.find((field) => field.name === key.replace("_s3key", ""))?.processor;
+    let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always");
+    if (this.processor) {
+      const processor = this.processor;
       console.log("[EXULU] Processor found", processor);
       if (processor && (processor?.config?.trigger === "onInsert" || processor?.config?.trigger === "onUpdate" || processor?.config?.trigger === "always")) {
         const {
@@ -6210,8 +6295,7 @@ var ExuluContext = class {
           "api",
           {
             ...item,
-            id: results[0].id,
-            field: key
+            id: results[0].id
           },
           config,
           user,
@@ -6220,8 +6304,13 @@ var ExuluContext = class {
         if (processorJob) {
           jobs.push(processorJob);
         }
-        if (!processorJob && processor.config?.generateEmbeddings) {
-          shouldGenerateEmbeddings = true;
+        if (!processorJob) {
+          await db3.from(getTableName(this.id)).where({ id: results[0].id }).update({
+            ...processorResult
+          });
+          if (processor.config?.generateEmbeddings) {
+            shouldGenerateEmbeddings = true;
+          }
         }
       }
     }
@@ -6247,6 +6336,7 @@ var ExuluContext = class {
     };
   };
   updateItem = async (item, config, user, role, generateEmbeddingsOverwrite) => {
+    console.log("[EXULU] updating item", item);
     const { db: db3 } = await postgresClient();
     if (item.field) {
       delete item.field;
@@ -6272,8 +6362,8 @@ var ExuluContext = class {
     await mutation;
     let jobs = [];
     let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always");
-    for (const [key, value] of Object.entries(item)) {
-      const processor = this.fields.find((field) => field.name === key.replace("_s3key", ""))?.processor;
+    if (this.processor) {
+      const processor = this.processor;
       if (processor && (processor?.config?.trigger === "onInsert" || processor?.config?.trigger === "onUpdate" || processor?.config?.trigger === "always")) {
         const {
           job: processorJob,
@@ -6282,8 +6372,7 @@ var ExuluContext = class {
           "api",
           {
             ...item,
-            id: record.id,
-            field: key
+            id: record.id
           },
           config,
           user,
@@ -6292,8 +6381,13 @@ var ExuluContext = class {
         if (processorJob) {
           jobs.push(processorJob);
         }
-        if (!processorJob && processor.config?.generateEmbeddings) {
-          shouldGenerateEmbeddings = true;
+        if (!processorJob) {
+          await db3.from(getTableName(this.id)).where({ id: record.id }).update({
+            ...processorResult
+          });
+          if (processor.config?.generateEmbeddings) {
+            shouldGenerateEmbeddings = true;
+          }
         }
       }
     }
@@ -6317,23 +6411,23 @@ var ExuluContext = class {
   };
   deleteItem = async (item, user, role) => {
     const { db: db3 } = await postgresClient();
+    if (!item.id && !item.external_id) {
+      throw new Error("Item id or external id is required for deleting an item.");
+    }
     if (!item.id?.length && item?.external_id) {
       item = await db3.from(getTableName(this.id)).where({ external_id: item.external_id }).first();
       if (!item || !item.id) {
         throw new Error(`Item not found for external id ${item?.external_id || "undefined"}.`);
       }
     }
-    await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
-    if (!this.embedder) {
-      return {
-        id: item.id,
-        job: void 0
-      };
-    }
-    const chunks = await db3.from(getChunksTableName(this.id)).where({ source: item.id }).select("id");
-    if (chunks.length > 0) {
-      await db3.from(getChunksTableName(this.id)).where({ source: item.id }).delete();
+    const chunkTableExists = await this.chunksTableExists();
+    if (chunkTableExists) {
+      const chunks = await db3.from(getChunksTableName(this.id)).where({ source: item.id }).select("id");
+      if (chunks.length > 0) {
+        await db3.from(getChunksTableName(this.id)).where({ source: item.id }).delete();
+      }
     }
+    await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
     return {
       id: item.id,
       job: void 0
@@ -6516,22 +6610,26 @@ var ExuluContext = class {
   };
   // Exports the context as a tool that can be used by an agent
   tool = () => {
+    if (this.configuration.enableAsTool === false) {
+      return null;
+    }
     return new ExuluTool2({
       id: this.id,
-      name: `${this.name}`,
+      name: `${this.name}_context_search`,
       type: "context",
       category: "contexts",
       inputSchema: import_zod.z.object({
-        query: import_zod.z.string()
+        originalQuestion: import_zod.z.string().describe("The original question that the user asked"),
+        relevantKeywords: import_zod.z.array(import_zod.z.string()).describe("The keywords that are relevant to the user's question, for example names of specific products, systems or parts, IDs, etc.")
       }),
       config: [],
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
-      execute: async ({ query, user, role }) => {
+      execute: async ({ originalQuestion, relevantKeywords, user, role }) => {
         const { db: db3 } = await postgresClient();
         const result = await vectorSearch({
           page: 1,
-          limit: 50,
-          query,
+          limit: this.configuration.maxRetrievalResults ?? 10,
+          query: originalQuestion,
           filters: [],
           user,
           role,
@@ -6551,7 +6649,13 @@ var ExuluContext = class {
           role: user?.role?.id
         });
         return {
-          items: result.items
+          result: JSON.stringify(result.chunks.map((chunk) => ({
+            ...chunk,
+            context: {
+              name: this.name,
+              id: this.id
+            }
+          })))
         };
       }
     });
@@ -7144,7 +7248,7 @@ Mood: friendly and intelligent.
     });
   });
   if (config?.fileUploads && config?.fileUploads?.s3region && config?.fileUploads?.s3key && config?.fileUploads?.s3secret && config?.fileUploads?.s3Bucket) {
-    await createUppyRoutes(app, config);
+    await createUppyRoutes(app, contexts ?? [], config);
   } else {
     console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in ExuluApp instance.");
   }
@@ -7418,7 +7522,7 @@ var import_ai3 = require("ai");
 var import_crypto_js4 = __toESM(require("crypto-js"), 1);
 
 // src/registry/log-metadata.ts
-function logMetadata2(id, additionalMetadata) {
+function logMetadata(id, additionalMetadata) {
   return {
     __logMetadata: true,
     id,
@@ -7428,9 +7532,32 @@ function logMetadata2(id, additionalMetadata) {
 
 // src/registry/workers.ts
 var redisConnection;
+var unhandledRejectionHandlerInstalled = false;
+var installGlobalErrorHandlers = () => {
+  if (unhandledRejectionHandlerInstalled) return;
+  process.on("unhandledRejection", (reason, promise) => {
+    console.error("[EXULU] Unhandled Promise Rejection detected! This would have crashed the worker.", {
+      reason: reason instanceof Error ? reason.message : String(reason),
+      stack: reason instanceof Error ? reason.stack : void 0
+    });
+  });
+  process.on("uncaughtException", (error) => {
+    console.error("[EXULU] Uncaught Exception detected! This would have crashed the worker.", {
+      error: error.message,
+      stack: error.stack
+    });
+    if (error.message.includes("FATAL") || error.message.includes("Cannot find module")) {
+      console.error("[EXULU] Fatal error detected, exiting process.");
+      process.exit(1);
+    }
+  });
+  unhandledRejectionHandlerInstalled = true;
+  console.log("[EXULU] Global error handlers installed to prevent worker crashes");
+};
 var createWorkers = async (agents, queues2, config, contexts, evals, tools, tracer) => {
   console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
   console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
+  installGlobalErrorHandlers();
   if (!redisServer.host || !redisServer.port) {
     console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
     throw new Error("No redis server configured in the environment, so cannot start worker.");
@@ -7455,8 +7582,9 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
     const worker = new import_bullmq4.Worker(
       `${queue.queue.name}`,
       async (bullmqJob) => {
-        console.log("[EXULU] starting execution for job", logMetadata2(bullmqJob.name, {
+        console.log("[EXULU] starting execution for job", logMetadata(bullmqJob.name, {
           name: bullmqJob.name,
+          jobId: bullmqJob.id,
           status: await bullmqJob.getState(),
           type: bullmqJob.data.type
         }));
@@ -7464,16 +7592,20 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         const data = bullmqJob.data;
         const timeoutInSeconds = data.timeoutInSeconds || 600;
         const timeoutMs = timeoutInSeconds * 1e3;
+        let timeoutHandle;
         const timeoutPromise = new Promise((_, reject) => {
-          setTimeout(() => {
-            reject(new Error(`Timeout for job ${bullmqJob.id} reached after ${timeoutInSeconds}s`));
+          timeoutHandle = setTimeout(() => {
+            const timeoutError = new Error(`Timeout for job ${bullmqJob.id} reached after ${timeoutInSeconds}s`);
+            console.error(`[EXULU] ${timeoutError.message}`);
+            reject(timeoutError);
           }, timeoutMs);
         });
         const workPromise = (async () => {
           try {
+            console.log(`[EXULU] Job ${bullmqJob.id} - Log file: logs/jobs/job-${bullmqJob.id}.log`);
             bullmq.validate(bullmqJob.id, data);
             if (data.type === "embedder") {
-              console.log("[EXULU] running an embedder job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running an embedder job.", logMetadata(bullmqJob.name));
               const label = `embedder-${bullmqJob.name}`;
               await db3.from("job_results").insert({
                 job_id: bullmqJob.id,
@@ -7503,7 +7635,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               };
             }
             if (data.type === "processor") {
-              console.log("[EXULU] running a processor job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running a processor job, job name: ", bullmqJob.name, " job id: ", bullmqJob.id, " job data: ", data, " job queue: ", bullmqJob.queueName);
               const label = `processor-${bullmqJob.name}`;
               await db3.from("job_results").insert({
                 job_id: bullmqJob.id,
@@ -7516,44 +7648,46 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!context) {
                 throw new Error(`Context ${data.context} not found in the registry.`);
               }
-              const field = context.fields.find((field2) => {
-                return field2.name.replace("_s3key", "") === data.inputs.field.replace("_s3key", "");
-              });
-              if (!field) {
-                throw new Error(`Field ${data.inputs.field} not found in the context ${data.context}.`);
+              if (!data.inputs.id) {
+                throw new Error(`[EXULU] Item not set for processor in context ${context.id}, running in job ${bullmqJob.id}.`);
               }
-              if (!field.processor) {
-                throw new Error(`Processor not set for field ${data.inputs.field} in the context ${data.context}.`);
+              if (!context.processor) {
+                throw new Error(`Tried to run a processor job for context ${context.id}, but no processor is set.`);
               }
               const exuluStorage = new ExuluStorage({ config });
-              if (!data.user) {
-                throw new Error(`User not set for processor job.`);
-              }
-              if (!data.role) {
-                throw new Error(`Role not set for processor job.`);
-              }
               console.log("[EXULU] POS 2 -- EXULU CONTEXT PROCESS FIELD");
-              const result = await field.processor.execute({
+              const processorResult = await context.processor.execute({
                 item: data.inputs,
                 user: data.user,
                 role: data.role,
                 utils: {
-                  storage: exuluStorage,
-                  items: {
-                    update: context.updateItem,
-                    create: context.createItem,
-                    delete: context.deleteItem
-                  }
+                  storage: exuluStorage
                 },
-                config
+                exuluConfig: config
+              });
+              if (!processorResult) {
+                throw new Error(`[EXULU] Processor in context ${context.id}, running in job ${bullmqJob.id} did not return an item.`);
+              }
+              delete processorResult.field;
+              await db3.from(getTableName(context.id)).where({
+                id: processorResult.id
+              }).update({
+                ...processorResult,
+                last_processed_at: (/* @__PURE__ */ new Date()).toISOString()
               });
               let jobs = [];
-              if (field.processor.config?.generateEmbeddings) {
+              if (context.processor?.config?.generateEmbeddings) {
+                const fullItem = await db3.from(getTableName(context.id)).where({
+                  id: processorResult.id
+                }).first();
+                if (!fullItem) {
+                  throw new Error(`[EXULU] Item ${processorResult.id} not found after processor update in context ${context.id}`);
+                }
                 const { job: embeddingsJob } = await context.embeddings.generate.one({
-                  item: data.inputs,
+                  item: fullItem,
                   user: data.user,
                   role: data.role,
-                  trigger: "api",
+                  trigger: "processor",
                   config
                 });
                 if (embeddingsJob) {
@@ -7561,14 +7695,14 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 }
               }
               return {
-                result,
+                result: processorResult,
                 metadata: {
                   jobs: jobs.length > 0 ? jobs.join(",") : void 0
                 }
               };
             }
             if (data.type === "eval_run") {
-              console.log("[EXULU] running an eval run job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running an eval run job.", logMetadata(bullmqJob.name));
               const label = `eval-run-${data.eval_run_id}-${data.test_case_id}`;
               const existingResult = await db3.from("job_results").where({ label }).first();
               if (existingResult) {
@@ -7617,7 +7751,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     resolve(messages2);
                     break;
                   } catch (error) {
-                    console.error(`[EXULU] error processing UI messages flow for agent ${agentInstance.name} (${agentInstance.id}).`, logMetadata2(bullmqJob.name, {
+                    console.error(`[EXULU] error processing UI messages flow for agent ${agentInstance.name} (${agentInstance.id}).`, logMetadata(bullmqJob.name, {
                       error: error instanceof Error ? error.message : String(error)
                     }));
                     attempts++;
@@ -7678,7 +7812,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     eval_function_config: evalFunction.config || {},
                     result: result2 || 0
                   };
-                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata(bullmqJob.name, {
                     result: result2 || 0
                   }));
                   evalFunctionResults.push(evalFunctionResult);
@@ -7697,7 +7831,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     result: result2 || 0
                   };
                   evalFunctionResults.push(evalFunctionResult);
-                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata(bullmqJob.name, {
                     result: result2 || 0
                   }));
                 }
@@ -7734,7 +7868,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               };
             }
             if (data.type === "eval_function") {
-              console.log("[EXULU] running an eval function job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running an eval function job.", logMetadata(bullmqJob.name));
               if (data.eval_functions?.length !== 1) {
                 throw new Error(`Expected 1 eval function for eval function job, got ${data.eval_functions?.length}.`);
               }
@@ -7780,7 +7914,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   inputMessages,
                   evalFunction.config || {}
                 );
-                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result}`, logMetadata2(bullmqJob.name, {
+                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result}`, logMetadata(bullmqJob.name, {
                   result: result || 0
                 }));
               }
@@ -7790,7 +7924,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               };
             }
             if (data.type === "source") {
-              console.log("[EXULU] running a source job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running a source job.", logMetadata(bullmqJob.name));
               if (!data.source) {
                 throw new Error(`No source id set for source job.`);
               }
@@ -7818,14 +7952,14 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 );
                 if (job) {
                   jobs.push(job);
-                  console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata2(bullmqJob.name, {
+                  console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata(bullmqJob.name, {
                     item: createdItem,
                     job
                   }));
                 }
                 if (createdItem.id) {
                   items.push(createdItem.id);
-                  console.log(`[EXULU] created item through source update job ${createdItem.id}`, logMetadata2(bullmqJob.name, {
+                  console.log(`[EXULU] created item through source update job ${createdItem.id}`, logMetadata(bullmqJob.name, {
                     item: createdItem
                   }));
                 }
@@ -7853,11 +7987,20 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
             throw error;
           }
         })();
-        return Promise.race([workPromise, timeoutPromise]);
+        try {
+          const result = await Promise.race([workPromise, timeoutPromise]);
+          clearTimeout(timeoutHandle);
+          return result;
+        } catch (error) {
+          clearTimeout(timeoutHandle);
+          console.error(`[EXULU] job ${bullmqJob.id} failed (error caught in race handler).`, error instanceof Error ? error.message : String(error));
+          throw error;
+        }
       },
       {
         autorun: true,
         connection: redisConnection,
+        concurrency: queue.concurrency?.worker || 1,
         removeOnComplete: { count: 1e3 },
         removeOnFail: { count: 5e3 },
         ...queue.ratelimit && {
@@ -7887,7 +8030,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         });
         return;
       }
-      console.error(`[EXULU] job failed.`, job?.name ? logMetadata2(job.name, {
+      console.error(`[EXULU] job failed.`, job?.name ? logMetadata(job.name, {
         error: error instanceof Error ? error.message : String(error)
       }) : error);
     });
@@ -7895,7 +8038,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
       console.error(`[EXULU] worker error.`, error);
     });
     worker.on("progress", (job, progress) => {
-      console.log(`[EXULU] job progress ${job.id}.`, logMetadata2(job.name, {
+      console.log(`[EXULU] job progress ${job.id}.`, logMetadata(job.name, {
         progress
       }));
     });
@@ -9061,18 +9204,24 @@ var ExuluQueues = class {
   // method of ExuluQueues we need to store the desired rate limit on the queue
   // here so we can use the value when creating workers for the queue instance
   // as there is no way to store a rate limit value natively on a bullm queue.
-  register = (name, concurrency = 1, ratelimit = 1) => {
+  register = (name, concurrency, ratelimit = 1, timeoutInSeconds = 180) => {
+    const queueConcurrency = concurrency.queue || 1;
+    const workerConcurrency = concurrency.worker || 1;
     const use = async () => {
       const existing = this.queues.find((x) => x.queue?.name === name);
       if (existing) {
         const globalConcurrency = await existing.queue.getGlobalConcurrency();
-        if (globalConcurrency !== concurrency) {
-          await existing.queue.setGlobalConcurrency(concurrency);
+        if (globalConcurrency !== queueConcurrency) {
+          await existing.queue.setGlobalConcurrency(queueConcurrency);
         }
         return {
           queue: existing.queue,
           ratelimit,
-          concurrency
+          concurrency: {
+            worker: workerConcurrency,
+            queue: queueConcurrency
+          },
+          timeoutInSeconds
         };
       }
       if (!redisServer.host?.length || !redisServer.port?.length) {
@@ -9091,22 +9240,34 @@ var ExuluQueues = class {
           telemetry: new import_bullmq_otel.BullMQOtel("simple-guide")
         }
       );
-      await newQueue.setGlobalConcurrency(concurrency);
+      await newQueue.setGlobalConcurrency(queueConcurrency);
       this.queues.push({
         queue: newQueue,
         ratelimit,
-        concurrency
+        concurrency: {
+          worker: workerConcurrency,
+          queue: queueConcurrency
+        },
+        timeoutInSeconds
       });
       return {
         queue: newQueue,
         ratelimit,
-        concurrency
+        concurrency: {
+          worker: workerConcurrency,
+          queue: queueConcurrency
+        },
+        timeoutInSeconds
       };
     };
     this.list.set(name, {
       name,
-      concurrency,
+      concurrency: {
+        worker: workerConcurrency,
+        queue: queueConcurrency
+      },
       ratelimit,
+      timeoutInSeconds,
       use
     });
     return {
@@ -9161,7 +9322,10 @@ var llmAsJudgeEval = () => {
         name: "prompt",
         description: "The prompt to send to the LLM as a judge, make sure to instruct the LLM to output a numerical score between 0 and 100. Add {actual_output} to the prompt to replace with the last message content, and {expected_output} to replace with the expected output."
       }],
-      queue: queues.register("llm_as_judge", 1, 1).use(),
+      queue: queues.register("llm_as_judge", {
+        worker: 1,
+        queue: 1
+      }, 1).use(),
       llm: true
     });
   }
@@ -9796,11 +9960,16 @@ var previewPdfTool = new ExuluTool2({
   type: "function",
   config: [],
   inputSchema: import_zod5.z.object({
-    s3key: import_zod5.z.string().describe("The S3 key of the PDF file to preview, can also optionally include a [bucket:name] to specify the bucket."),
+    s3key: import_zod5.z.string().describe("The S3 key of the PDF file to preview."),
     page: import_zod5.z.number().describe("The page number to preview, defaults to 1.").optional()
   }),
   execute: async ({ s3key, page, exuluConfig }) => {
-    const url = await getPresignedUrl(s3key, exuluConfig);
+    const bucket = s3key.split("/")[0];
+    const key = s3key.split("/").slice(1).join("/");
+    if (!bucket || !key) {
+      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
+    }
+    const url = await getPresignedUrl(bucket, key, exuluConfig);
     if (!url) {
       throw new Error("No URL provided for PDF preview");
     }
@@ -10164,7 +10333,7 @@ var ExuluApp = class {
       ...[previewPdfTool],
       ...todoTools,
       // Add contexts as tools
-      ...Object.values(contexts || {}).map((context) => context.tool())
+      ...Object.values(contexts || {}).map((context) => context.tool()).filter(Boolean)
       // Because agents are stored in the database,  we add those as tools
       // at request time, not during ExuluApp initialization. We add them
       // in the grahql tools resolver.
@@ -10194,7 +10363,10 @@ var ExuluApp = class {
     }
     const queueSet = /* @__PURE__ */ new Set();
     if (redisServer.host?.length && redisServer.port?.length) {
-      queues.register(global_queues.eval_runs, 1, 1);
+      queues.register(global_queues.eval_runs, {
+        worker: 1,
+        queue: 1
+      }, 1);
       for (const queue of queues.list.values()) {
         const config2 = await queue.use();
         queueSet.add(config2);
@@ -10323,10 +10495,7 @@ var ExuluApp = class {
               console.warn("[EXULU] No queue configured for source", source.name);
               continue;
             }
-            if (queue) {
-              if (!source.config?.schedule) {
-                throw new Error("Schedule is required for source when configuring a queue: " + source.name);
-              }
+            if (queue && source.config?.schedule) {
               console.log("[EXULU] Creating ContextSource scheduler for", source.name, "in queue", queue.queue?.name);
               await queue.queue?.upsertJobScheduler(source.id, {
                 pattern: source.config?.schedule