@exulu/backend 1.40.0 → 1.42.1

package/dist/index.cjs

@@ -48,7 +48,7 @@ __export(index_exports, {
   ExuluUtils: () => ExuluUtils,
   ExuluVariables: () => ExuluVariables,
   db: () => db2,
-  logMetadata: () => logMetadata2
+  logMetadata: () => logMetadata
 });
 module.exports = __toCommonJS(index_exports);
 var import_config = require("dotenv/config");
@@ -203,21 +203,37 @@ async function postgresClient() {
           database: dbName,
           password: process.env.POSTGRES_DB_PASSWORD,
           ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false,
-          connectionTimeoutMillis: 1e4
+          connectionTimeoutMillis: 3e4,
+          // Increased from 10s to 30s to handle connection spikes
+          // PostgreSQL statement timeout (in milliseconds) - kills queries that run too long
+          // This prevents runaway queries from blocking connections
+          statement_timeout: 18e5,
+          // 30 minutes - should be longer than max job timeout (1200s = 20m)
+          // Connection idle timeout - how long pg client waits before timing out
+          query_timeout: 18e5
+          // 30 minutes
         },
         pool: {
-          min: 2,
-          max: 20,
-          // Increased from 20 to handle more concurrent operations
-          acquireTimeoutMillis: 3e4,
+          min: 5,
+          // Increased from 2 to ensure enough connections available
+          max: 50,
+          // Increased from 20 to handle more concurrent operations with processor jobs
+          acquireTimeoutMillis: 6e4,
+          // Increased from 30s to 60s to handle pool contention
           createTimeoutMillis: 3e4,
-          idleTimeoutMillis: 3e4,
+          idleTimeoutMillis: 6e4,
+          // Increased to keep connections alive longer
           reapIntervalMillis: 1e3,
           createRetryIntervalMillis: 200,
           // Log pool events to help debug connection issues
           afterCreate: (conn, done) => {
             console.log("[EXULU] New database connection created");
-            done(null, conn);
+            conn.query("SET statement_timeout = 1800000", (err) => {
+              if (err) {
+                console.error("[EXULU] Error setting statement_timeout:", err);
+              }
+              done(err, conn);
+            });
           }
         }
       });
@@ -1467,6 +1483,10 @@ var addCoreFields = (schema) => {
       field.name = field.name + "_s3key";
     }
   });
+  schema.fields.push({
+    name: "last_processed_at",
+    type: "date"
+  });
   if (schema.RBAC) {
     if (!schema.fields.some((field) => field.name === "rights_mode")) {
       schema.fields.push({
@@ -2349,33 +2369,38 @@ function createMutations(table, agents, contexts, tools, config) {
     }
   };
   if (table.type === "items") {
-    if (table.fields.some((field) => field.processor?.execute)) {
-      mutations[`${tableNameSingular}ProcessItemField`] = async (_, args, context, info) => {
+    if (table.processor) {
+      const contextItemProcessorMutation = async (context, items, user, role) => {
+        let jobs = [];
+        let results = [];
+        await Promise.all(items.map(async (item) => {
+          const result = await context.processField(
+            "api",
+            item,
+            config,
+            user,
+            role
+          );
+          if (result.job) {
+            jobs.push(result.job);
+          }
+          if (result.result) {
+            results.push(result.result);
+          }
+        }));
+        return {
+          message: jobs.length > 0 ? "Processing job scheduled." : "Items processed successfully.",
+          results: results.map((result) => JSON.stringify(result)),
+          jobs
+        };
+      };
+      mutations[`${tableNameSingular}ProcessItem`] = async (_, args, context, info) => {
         if (!context.user?.super_admin) {
           throw new Error("You are not authorized to process fields via API, user must be super admin.");
         }
-        const exists = contexts.find((context2) => context2.id === table.id);
-        if (!exists) {
-          throw new Error(`Context ${table.id} not found.`);
-        }
-        if (!args.field) {
-          throw new Error("Field argument missing, the field argument is required.");
-        }
         if (!args.item) {
           throw new Error("Item argument missing, the item argument is required.");
         }
-        const name = args.field?.replace("_s3key", "");
-        console.log("[EXULU] name", name);
-        console.log("[EXULU] fields", exists.fields.map((field2) => field2.name));
-        const field = exists.fields.find((field2) => {
-          return field2.name.replace("_s3key", "") === name;
-        });
-        if (!field) {
-          throw new Error(`Field ${name} not found in context ${exists.id}].`);
-        }
-        if (!field.processor) {
-          throw new Error(`Processor not set for field ${args.field} in context ${exists.id}.`);
-        }
         const { db: db3 } = context;
         let query = db3.from(tableNamePlural).select("*").where({ id: args.item });
         query = applyAccessControl(table, query, context.user);
@@ -2383,21 +2408,38 @@ function createMutations(table, agents, contexts, tools, config) {
         if (!item) {
           throw new Error("Item not found, or your user does not have access to it.");
         }
-        const { job, result } = await exists.processField(
-          "api",
-          {
-            ...item,
-            field: args.field
-          },
-          config,
+        const exists = contexts.find((context2) => context2.id === table.id);
+        if (!exists) {
+          throw new Error(`Context ${table.id} not found.`);
+        }
+        return contextItemProcessorMutation(exists, [item], context.user.id, context.user.role?.id);
+      };
+      mutations[`${tableNameSingular}ProcessItems`] = async (_, args, context, info) => {
+        if (!context.user?.super_admin) {
+          throw new Error("You are not authorized to process fields via API, user must be super admin.");
+        }
+        const { limit = 10, filters = [], sort } = args;
+        const { db: db3 } = context;
+        const { items } = await paginationRequest({
+          db: db3,
+          limit,
+          page: 0,
+          filters,
+          sort,
+          table,
+          user: context.user,
+          fields: "*"
+        });
+        const exists = contexts.find((context2) => context2.id === table.id);
+        if (!exists) {
+          throw new Error(`Context ${table.id} not found.`);
+        }
+        return contextItemProcessorMutation(
+          exists,
+          items,
           context.user.id,
           context.user.role?.id
         );
-        return {
-          message: job ? "Processing job scheduled." : "Item processed successfully.",
-          result,
-          job
-        };
       };
     }
     mutations[`${tableNameSingular}ExecuteSource`] = async (_, args, context, info) => {
@@ -2565,14 +2607,14 @@ function createMutations(table, agents, contexts, tools, config) {
   }
   return mutations;
 }
-var applyAccessControl = (table, query, user) => {
+var applyAccessControl = (table, query, user, field_prefix) => {
   const tableNamePlural = table.name.plural.toLowerCase();
   if (table.name.plural !== "agent_sessions" && user?.super_admin === true) {
     return query;
   }
   console.log("[EXULU] user.role", user?.role);
   console.log("[EXULU] table.name.plural", table.name.plural);
-  if (!user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
+  if (user && !user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
     console.error("==== Access control error: no role found or no access to entity type. ====");
     throw new Error("Access control error: no role found or no access to entity type.");
   }
@@ -2581,18 +2623,25 @@ var applyAccessControl = (table, query, user) => {
   if (!hasRBAC) {
     return query;
   }
+  if (user?.super_admin) {
+    return query;
+  }
+  const prefix = field_prefix ? field_prefix + "." : "";
+  console.log("[EXULU] applying access control with this prefix", prefix);
   try {
     query = query.where(function() {
-      this.where("rights_mode", "public");
-      this.orWhere("created_by", user.id);
-      this.orWhere(function() {
-        this.where("rights_mode", "users").whereExists(function() {
-          this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
+      this.where(`${prefix}rights_mode`, "public");
+      if (user) {
+        this.orWhere(`${prefix}created_by`, user.id);
+        this.orWhere(function() {
+          this.where(`${prefix}rights_mode`, "users").whereExists(function() {
+            this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
+          });
         });
-      });
-      if (user.role) {
+      }
+      if (user?.role) {
         this.orWhere(function() {
-          this.where("rights_mode", "roles").whereExists(function() {
+          this.where(`${prefix}rights_mode`, "roles").whereExists(function() {
             this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
           });
         });
@@ -2604,9 +2653,11 @@ var applyAccessControl = (table, query, user) => {
   }
   return query;
 };
-var converOperatorToQuery = (query, fieldName, operators, table) => {
+var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) => {
   const field = table?.fields.find((f) => f.name === fieldName);
   const isJsonField = field?.type === "json";
+  const prefix = field_prefix ? field_prefix + "." : "";
+  fieldName = prefix + fieldName;
   if (operators.eq !== void 0) {
     if (isJsonField) {
       query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
@@ -2916,53 +2967,95 @@ var finalizeRequestedFields = async ({
         }
         const { db: db3 } = await postgresClient();
         const query = db3.from(getChunksTableName(context.id)).where({ source: result.id }).select("id", "content", "source", "chunk_index", "createdAt", "updatedAt");
-        query.select(
-          db3.raw("vector_dims(??) as embedding_size", [`embedding`])
-        );
         const chunks = await query;
         result.chunks = chunks.map((chunk) => ({
-          cosine_distance: 0,
-          fts_rank: 0,
-          hybrid_score: 0,
-          content: chunk.content,
-          source: chunk.source,
+          chunk_content: chunk.content,
+          chunk_source: chunk.source,
           chunk_index: chunk.chunk_index,
           chunk_id: chunk.id,
           chunk_created_at: chunk.createdAt,
           chunk_updated_at: chunk.updatedAt,
-          embedding_size: chunk.embedding_size
+          item_updated_at: chunk.item_updated_at,
+          item_created_at: chunk.item_created_at,
+          item_id: chunk.item_id,
+          item_external_id: chunk.item_external_id,
+          item_name: chunk.item_name
         }));
       }
     }
   }
   return result;
 };
-var applyFilters = (query, filters, table) => {
+var applyFilters = (query, filters, table, field_prefix) => {
   filters.forEach((filter) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
         if (operators.and !== void 0) {
           operators.and.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator, table);
+            query = converOperatorToQuery(query, fieldName, operator, table, field_prefix);
           });
         }
         if (operators.or !== void 0) {
           operators.or.forEach((operator) => {
-            query = converOperatorToQuery(query, fieldName, operator, table);
+            query = converOperatorToQuery(query, fieldName, operator, table, field_prefix);
           });
         }
-        query = converOperatorToQuery(query, fieldName, operators, table);
+        query = converOperatorToQuery(query, fieldName, operators, table, field_prefix);
       }
     });
   });
   return query;
 };
-var applySorting = (query, sort) => {
+var applySorting = (query, sort, field_prefix) => {
+  const prefix = field_prefix ? field_prefix + "." : "";
   if (sort) {
+    sort.field = prefix + sort.field;
     query = query.orderBy(sort.field, sort.direction.toLowerCase());
   }
   return query;
 };
+var paginationRequest = async ({
+  db: db3,
+  limit,
+  page,
+  filters,
+  sort,
+  table,
+  user,
+  fields
+}) => {
+  if (limit > 1e4) {
+    throw new Error("Limit cannot be greater than 10.000.");
+  }
+  const tableName = table.name.plural.toLowerCase();
+  let countQuery = db3(tableName);
+  countQuery = applyFilters(countQuery, filters, table);
+  countQuery = applyAccessControl(table, countQuery, user);
+  const countResult = await countQuery.count("* as count");
+  const itemCount = Number(countResult[0]?.count || 0);
+  const pageCount = Math.ceil(itemCount / limit);
+  const currentPage = page;
+  const hasPreviousPage = currentPage > 1;
+  const hasNextPage = currentPage < pageCount - 1;
+  let dataQuery = db3(tableName);
+  dataQuery = applyFilters(dataQuery, filters, table);
+  dataQuery = applyAccessControl(table, dataQuery, user);
+  dataQuery = applySorting(dataQuery, sort);
+  if (page > 1) {
+    dataQuery = dataQuery.offset((page - 1) * limit);
+  }
+  let items = await dataQuery.select(fields ? fields : "*").limit(limit);
+  return {
+    items,
+    pageInfo: {
+      pageCount,
+      itemCount,
+      currentPage,
+      hasPreviousPage,
+      hasNextPage
+    }
+  };
+};
 function createQueries(table, agents, tools, contexts) {
   const tableNamePlural = table.name.plural.toLowerCase();
   const tableNameSingular = table.name.singular.toLowerCase();
@@ -2998,38 +3091,22 @@ function createQueries(table, agents, tools, contexts) {
       return finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result, user: context.user });
     },
     [`${tableNamePlural}Pagination`]: async (_, args, context, info) => {
-      const { limit = 10, page = 0, filters = [], sort } = args;
       const { db: db3 } = context;
-      if (limit > 500) {
-        throw new Error("Limit cannot be greater than 500.");
-      }
-      let countQuery = db3(tableNamePlural);
-      countQuery = applyFilters(countQuery, filters, table);
-      countQuery = applyAccessControl(table, countQuery, context.user);
-      const countResult = await countQuery.count("* as count");
-      const itemCount = Number(countResult[0]?.count || 0);
-      const pageCount = Math.ceil(itemCount / limit);
-      const currentPage = page;
-      const hasPreviousPage = currentPage > 1;
-      const hasNextPage = currentPage < pageCount - 1;
-      let dataQuery = db3(tableNamePlural);
-      dataQuery = applyFilters(dataQuery, filters, table);
-      dataQuery = applyAccessControl(table, dataQuery, context.user);
+      const { limit = 10, page = 0, filters = [], sort } = args;
       const requestedFields = getRequestedFields(info);
-      dataQuery = applySorting(dataQuery, sort);
-      if (page > 1) {
-        dataQuery = dataQuery.offset((page - 1) * limit);
-      }
       const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
-      let items = await dataQuery.select(sanitizedFields).limit(limit);
+      const { items, pageInfo } = await paginationRequest({
+        db: db3,
+        limit,
+        page,
+        filters,
+        sort,
+        table,
+        user: context.user,
+        fields: sanitizedFields
+      });
       return {
-        pageInfo: {
-          pageCount,
-          itemCount,
-          currentPage,
-          hasPreviousPage,
-          hasNextPage
-        },
+        pageInfo,
         items: finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result: items, user: context.user })
       };
     },
@@ -3078,7 +3155,7 @@ function createQueries(table, agents, tools, contexts) {
       }
       const { limit = 10, page = 0, filters = [], sort } = args;
       return await vectorSearch({
-        limit,
+        limit: limit || exists.configuration.maxRetrievalResults || 10,
         page,
         filters,
         sort,
@@ -3088,7 +3165,9 @@ function createQueries(table, agents, tools, contexts) {
         method: args.method,
         user: context.user,
         role: context.user?.role?.id,
-        trigger: "api"
+        trigger: "api",
+        cutoffs: args.cutoffs,
+        expand: args.expand
       });
     };
   }
@@ -3105,7 +3184,9 @@ var vectorSearch = async ({
   method,
   user,
   role,
-  trigger
+  trigger,
+  cutoffs,
+  expand
 }) => {
   const table = contextToTableDefinition(context);
   console.log("[EXULU] Called vector search.", {
@@ -3117,10 +3198,12 @@ var vectorSearch = async ({
     query,
     method,
     user,
-    role
+    role,
+    cutoffs,
+    expand
   });
-  if (limit > 50) {
-    throw new Error("Limit cannot be greater than 50.");
+  if (limit > 250) {
+    throw new Error("Limit cannot be greater than 1000.");
   }
   if (!query) {
     throw new Error("Query is required.");
@@ -3137,106 +3220,135 @@ var vectorSearch = async ({
   }
   const mainTable = getTableName(id);
   const chunksTable = getChunksTableName(id);
-  let countQuery = db3(mainTable);
-  countQuery = applyFilters(countQuery, filters, table);
-  countQuery = applyAccessControl(table, countQuery, user);
-  const columns = await db3(mainTable).columnInfo();
-  let itemsQuery = db3(mainTable).select(Object.keys(columns).map((column) => mainTable + "." + column));
-  itemsQuery = applyFilters(itemsQuery, filters, table);
-  itemsQuery = applyAccessControl(table, itemsQuery, user);
-  itemsQuery = applySorting(itemsQuery, sort);
+  cutoffs = {
+    cosineDistance: cutoffs?.cosineDistance || context.configuration?.cutoffs?.cosineDistance || 0,
+    tsvector: cutoffs?.tsvector || context.configuration?.cutoffs?.tsvector || 0,
+    hybrid: cutoffs?.hybrid ? (cutoffs?.hybrid ?? 0) / 100 : context.configuration?.cutoffs ? (context.configuration?.cutoffs?.hybrid ?? 0) / 100 : 0
+  };
+  expand = {
+    before: expand?.before || context.configuration?.expand?.before || 0,
+    after: expand?.after || context.configuration?.expand?.after || 0
+  };
+  let chunksQuery = db3(chunksTable + " as chunks").select([
+    "chunks.id as chunk_id",
+    "chunks.source",
+    "chunks.content",
+    "chunks.chunk_index",
+    db3.raw('chunks."createdAt" as chunk_created_at'),
+    db3.raw('chunks."updatedAt" as chunk_updated_at'),
+    "chunks.metadata",
+    "items.id as item_id",
+    "items.name as item_name",
+    "items.external_id as item_external_id",
+    db3.raw('items."updatedAt" as item_updated_at'),
+    db3.raw('items."createdAt" as item_created_at')
+  ]);
+  chunksQuery.leftJoin(mainTable + " as items", function() {
+    this.on("chunks.source", "=", "items.id");
+  });
+  chunksQuery = applyFilters(chunksQuery, filters, table, "items");
+  chunksQuery = applyAccessControl(table, chunksQuery, user, "items");
+  chunksQuery = applySorting(chunksQuery, sort, "items");
   if (queryRewriter) {
     query = await queryRewriter(query);
   }
-  itemsQuery.limit(limit * 3);
-  itemsQuery.leftJoin(chunksTable, function() {
-    this.on(chunksTable + ".source", "=", mainTable + ".id");
-  });
-  itemsQuery.select(chunksTable + ".id as chunk_id");
-  itemsQuery.select(chunksTable + ".source");
-  itemsQuery.select(chunksTable + ".content");
-  itemsQuery.select(chunksTable + ".chunk_index");
-  itemsQuery.select(chunksTable + ".createdAt as chunk_created_at");
-  itemsQuery.select(chunksTable + ".updatedAt as chunk_updated_at");
-  itemsQuery.select(db3.raw("vector_dims(??) as embedding_size", [`${chunksTable}.embedding`]));
-  const { chunks } = await embedder.generateFromQuery(context.id, query, {
+  const { chunks: queryChunks } = await embedder.generateFromQuery(context.id, query, {
     label: table.name.singular,
     trigger
   }, user?.id, role);
-  if (!chunks?.[0]?.vector) {
+  if (!queryChunks?.[0]?.vector) {
     throw new Error("No vector generated for query.");
   }
-  const vector = chunks[0].vector;
+  const vector = queryChunks[0].vector;
   const vectorStr = `ARRAY[${vector.join(",")}]`;
   const vectorExpr = `${vectorStr}::vector`;
   const language = configuration.language || "english";
-  let items = [];
+  console.log("[EXULU] Vector search params:", { method, query, cutoffs });
+  let resultChunks = [];
   switch (method) {
     case "tsvector":
-      itemsQuery.select(db3.raw(
-        `ts_rank(${chunksTable}.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
-        [language, query]
+      chunksQuery.limit(limit * 2);
+      const tokens = query.trim().split(/\s+/).filter((t) => t.length > 0);
+      const sanitizedTokens = tokens.flatMap((t) => {
+        return t.split(/[^\w]+/).filter((part) => part.length > 0);
+      });
+      const orQuery = sanitizedTokens.join(" | ");
+      console.log("[EXULU] FTS query transformation:", { original: query, tokens, sanitizedTokens, orQuery, cutoff: cutoffs?.tsvector });
+      chunksQuery.select(db3.raw(
+        `ts_rank(chunks.fts, to_tsquery(?, ?)) as fts_rank`,
+        [language, orQuery]
       )).whereRaw(
-        `${chunksTable}.fts @@ websearch_to_tsquery(?, ?)`,
-        [language, query]
+        `(chunks.fts @@ to_tsquery(?, ?)) AND (items.archived IS FALSE OR items.archived IS NULL)`,
+        [language, orQuery]
       ).orderByRaw(`fts_rank DESC`);
-      items = await itemsQuery;
+      console.log("[EXULU] FTS query SQL:", chunksQuery.toQuery());
+      resultChunks = await chunksQuery;
       break;
     case "cosineDistance":
-    default:
-      itemsQuery.whereNotNull(`${chunksTable}.embedding`);
-      itemsQuery.select(
-        db3.raw(`1 - (${chunksTable}.embedding <=> ${vectorExpr}) AS cosine_distance`)
+      chunksQuery.limit(limit * 2);
+      chunksQuery.whereNotNull(`chunks.embedding`).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`);
+      console.log("[EXULU] Chunks query:", chunksQuery.toQuery());
+      chunksQuery.select(
+        db3.raw(`1 - (chunks.embedding <=> ${vectorExpr}) AS cosine_distance`)
       );
-      itemsQuery.orderByRaw(
-        `${chunksTable}.embedding <=> ${vectorExpr} ASC NULLS LAST`
+      chunksQuery.orderByRaw(
+        `chunks.embedding <=> ${vectorExpr} ASC NULLS LAST`
       );
-      items = await itemsQuery;
+      chunksQuery.whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr}) >= ?)`, [cutoffs?.cosineDistance || 0]);
+      resultChunks = await chunksQuery;
       break;
     case "hybridSearch":
-      const matchCount = Math.min(limit * 5, 100);
-      const fullTextWeight = 1;
+      const matchCount = Math.min(limit * 2);
+      const fullTextWeight = 2;
       const semanticWeight = 1;
       const rrfK = 50;
       const hybridSQL = `
             WITH full_text AS (
               SELECT
-                c.id,
-                c.source,
+                chunks.id,
+                chunks.source,
                 row_number() OVER (
-                  ORDER BY ts_rank_cd(c.fts, websearch_to_tsquery(?, ?)) DESC
+                  ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC
                 ) AS rank_ix
-              FROM ${chunksTable} c
-              WHERE c.fts @@ websearch_to_tsquery(?, ?)
+              FROM ${chunksTable} as chunks
+              LEFT JOIN ${mainTable} as items ON items.id = chunks.source
+              WHERE chunks.fts @@ plainto_tsquery(?, ?)
+                AND ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?
+                AND (items.archived IS FALSE OR items.archived IS NULL)
               ORDER BY rank_ix
-              LIMIT LEAST(?, 15) * 2
+              LIMIT LEAST(?, 250) * 2
             ),
             semantic AS (
               SELECT
-                c.id,
-                c.source,
+                chunks.id,
+                chunks.source,
                 row_number() OVER (
-                  ORDER BY c.embedding <=> ${vectorExpr} ASC
+                  ORDER BY chunks.embedding <=> ${vectorExpr} ASC
                 ) AS rank_ix
-              FROM ${chunksTable} c
-              WHERE c.embedding IS NOT NULL
+              FROM ${chunksTable} as chunks
+              LEFT JOIN ${mainTable} as items ON items.id = chunks.source
+              WHERE chunks.embedding IS NOT NULL
+                AND (1 - (chunks.embedding <=> ${vectorExpr})) >= ?
+                AND (items.archived IS FALSE OR items.archived IS NULL)
               ORDER BY rank_ix
-              LIMIT LEAST(?, 50) * 2
+              LIMIT LEAST(?, 250) * 2
             )
             SELECT
-              m.*,
-              c.id AS chunk_id,
-              c.source,
-              c.content,
-              c.chunk_index,
-              c.metadata,
-              c."createdAt" AS chunk_created_at,
-              c."updatedAt" AS chunk_updated_at,
-              vector_dims(c.embedding) as embedding_size,
-        
+              items.id as item_id,
+              items.name as item_name,
+              items.external_id as item_external_id,
+              chunks.id AS chunk_id,
+              chunks.source,
+              chunks.content,
+              chunks.chunk_index,
+              chunks.metadata,
+              chunks."createdAt" as chunk_created_at,
+              chunks."updatedAt" as chunk_updated_at,
+              items."updatedAt" as item_updated_at,
+              items."createdAt" as item_created_at,
               /* Per-signal scores for introspection */
-              ts_rank(c.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
-              (1 - (c.embedding <=> ${vectorExpr})) AS cosine_distance,
+              ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank,
+              (1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
         
               /* Hybrid RRF score */
               (
@@ -3248,22 +3360,35 @@ var vectorSearch = async ({
             FROM full_text ft
             FULL OUTER JOIN semantic se
               ON ft.id = se.id
-            JOIN ${chunksTable} c
-              ON COALESCE(ft.id, se.id) = c.id
-            JOIN ${mainTable} m
-              ON m.id = c.source
+            JOIN ${chunksTable} as chunks
+              ON COALESCE(ft.id, se.id) = chunks.id
+            JOIN ${mainTable} as items
+              ON items.id = chunks.source
+            WHERE (
+              COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
+              +
+              COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
+            ) >= ?
+            AND (chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)
+            AND (chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)
             ORDER BY hybrid_score DESC
-            LIMIT LEAST(?, 50)
+            LIMIT LEAST(?, 250)
             OFFSET 0
           `;
       const bindings = [
-        // full_text: websearch_to_tsquery(lang, query) in rank and where
+        // full_text: plainto_tsquery(lang, query) in rank and where
+        language,
+        query,
         language,
         query,
         language,
         query,
+        cutoffs?.tsvector || 0,
+        // full_text tsvector cutoff
         matchCount,
         // full_text limit
+        cutoffs?.cosineDistance || 0,
+        // semantic cosine distance cutoff
         matchCount,
         // semantic limit
         // fts_rank (ts_rank) call
@@ -3274,89 +3399,171 @@ var vectorSearch = async ({
         fullTextWeight,
         rrfK,
         semanticWeight,
+        // WHERE clause hybrid_score filter
+        rrfK,
+        fullTextWeight,
+        rrfK,
+        semanticWeight,
+        cutoffs?.hybrid || 0,
+        // Additional cutoff filters in main WHERE clause
+        language,
+        query,
+        cutoffs?.tsvector || 0,
+        // tsvector cutoff for results from semantic CTE
+        cutoffs?.cosineDistance || 0,
+        // cosine distance cutoff for results from full_text CTE
         matchCount
         // final limit
       ];
-      items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
-  }
-  console.log("[EXULU] Vector search results:", items?.length);
-  const seenSources = /* @__PURE__ */ new Map();
-  items = items.reduce((acc, item) => {
-    if (!seenSources.has(item.source)) {
-      seenSources.set(item.source, {
-        ...Object.fromEntries(
-          Object.keys(item).filter(
-            (key) => key !== "cosine_distance" && // kept per chunk below
-            key !== "fts_rank" && // kept per chunk below
-            key !== "hybrid_score" && // we will compute per item below
-            key !== "content" && key !== "source" && key !== "chunk_index" && key !== "chunk_id" && key !== "chunk_created_at" && key !== "chunk_updated_at" && key !== "embedding_size"
-          ).map((key) => [key, item[key]])
-        ),
-        chunks: [{
-          content: item.content,
-          chunk_index: item.chunk_index,
-          chunk_id: item.chunk_id,
-          source: item.source,
-          metadata: item.metadata,
-          chunk_created_at: item.chunk_created_at,
-          chunk_updated_at: item.chunk_updated_at,
-          embedding_size: item.embedding_size,
-          ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-          ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-          ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-        }]
-      });
-      acc.push(seenSources.get(item.source));
-    } else {
-      seenSources.get(item.source).chunks.push({
-        content: item.content,
-        chunk_index: item.chunk_index,
-        chunk_id: item.chunk_id,
-        chunk_created_at: item.chunk_created_at,
-        embedding_size: item.embedding_size,
-        metadata: item.metadata,
-        source: item.source,
-        chunk_updated_at: item.chunk_updated_at,
-        ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
-        ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
-        ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
-      });
+      resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
+  }
+  console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
+  let results = resultChunks.map((chunk) => ({
+    chunk_content: chunk.content,
+    chunk_index: chunk.chunk_index,
+    chunk_id: chunk.chunk_id,
+    chunk_source: chunk.source,
+    chunk_metadata: chunk.metadata,
+    chunk_created_at: chunk.chunk_created_at,
+    chunk_updated_at: chunk.chunk_updated_at,
+    item_updated_at: chunk.item_updated_at,
+    item_created_at: chunk.item_created_at,
+    item_id: chunk.item_id,
+    item_external_id: chunk.item_external_id,
+    item_name: chunk.item_name,
+    context: {
+      name: table.name.singular,
+      id: table.id || ""
+    },
+    ...(method === "cosineDistance" || method === "hybridSearch") && { chunk_cosine_distance: chunk.cosine_distance },
+    ...(method === "tsvector" || method === "hybridSearch") && { chunk_fts_rank: chunk.fts_rank },
+    ...method === "hybridSearch" && { chunk_hybrid_score: chunk.hybrid_score * 1e4 / 100 }
+  }));
+  if (results.length > 0 && (method === "cosineDistance" || method === "hybridSearch")) {
+    const scoreKey = method === "cosineDistance" ? "chunk_cosine_distance" : "chunk_hybrid_score";
+    const topScore = results[0]?.[scoreKey];
+    const bottomScore = results[results.length - 1]?.[scoreKey];
+    const medianScore = results[Math.floor(results.length / 2)]?.[scoreKey];
+    console.log("[EXULU] Score distribution:", {
+      method,
+      count: results.length,
+      topScore: topScore?.toFixed(4),
+      bottomScore: bottomScore?.toFixed(4),
+      medianScore: medianScore?.toFixed(4)
+    });
+    const adaptiveThreshold = topScore ? topScore * 0.6 : 0;
+    const beforeFilterCount = results.length;
+    results = results.filter((chunk) => {
+      const score = chunk[scoreKey];
+      return score !== void 0 && score >= adaptiveThreshold;
+    });
+    const filteredCount = beforeFilterCount - results.length;
+    if (filteredCount > 0) {
+      console.log(`[EXULU] Filtered ${filteredCount} low-quality results (threshold: ${adaptiveThreshold.toFixed(4)})`);
     }
-    return acc;
-  }, []);
-  console.log("[EXULU] Vector search results after deduplication:", items?.length);
-  items.forEach((item) => {
-    if (!item.chunks?.length) {
-      return;
+  }
+  if (resultReranker && query) {
+  }
+  results = results.slice(0, limit);
+  if (expand?.before || expand?.after) {
+    const expandedMap = /* @__PURE__ */ new Map();
+    for (const chunk of results) {
+      expandedMap.set(`${chunk.item_id}-${chunk.chunk_index}`, chunk);
+    }
+    if (expand?.before) {
+      for (const chunk of results) {
+        const indicesToFetch = Array.from(
+          { length: expand.before },
+          (_, i) => chunk.chunk_index - expand.before + i
+        ).filter((index) => index >= 0);
+        console.log("[EXULU] Indices to fetch:", indicesToFetch);
+        await Promise.all(indicesToFetch.map(async (index) => {
+          if (expandedMap.has(`${chunk.item_id}-${index}`)) {
+            return;
+          }
+          const expandedChunk = await db3(chunksTable).where({
+            source: chunk.item_id,
+            chunk_index: index
+          }).first();
+          if (expandedChunk) {
+            if (expandedChunk) {
+              expandedMap.set(`${chunk.item_id}-${index}`, {
+                chunk_content: expandedChunk.content,
+                chunk_index: expandedChunk.chunk_index,
+                chunk_id: expandedChunk.id,
+                chunk_source: expandedChunk.source,
+                chunk_metadata: expandedChunk.metadata,
+                chunk_created_at: expandedChunk.createdAt,
+                chunk_updated_at: expandedChunk.updatedAt,
+                item_updated_at: chunk.item_updated_at,
+                item_created_at: chunk.item_created_at,
+                item_id: chunk.item_id,
+                item_external_id: chunk.item_external_id,
+                item_name: chunk.item_name,
+                chunk_cosine_distance: 0,
+                chunk_fts_rank: 0,
+                chunk_hybrid_score: 0,
+                context: {
+                  name: table.name.singular,
+                  id: table.id || ""
+                }
+              });
+            }
+          }
+        }));
+      }
     }
-    if (method === "tsvector") {
-      const ranks = item.chunks.map((c) => typeof c.fts_rank === "number" ? c.fts_rank : 0);
-      const total = ranks.reduce((a, b) => a + b, 0);
-      const average = ranks.length ? total / ranks.length : 0;
-      item.averageRelevance = average;
-      item.totalRelevance = total;
-    } else if (method === "cosineDistance") {
-      let methodProperty = "cosine_distance";
-      const average = item.chunks.reduce((acc, item2) => {
-        return acc + item2[methodProperty];
-      }, 0) / item.chunks.length;
-      const total = item.chunks.reduce((acc, item2) => {
-        return acc + item2[methodProperty];
-      }, 0);
-      item.averageRelevance = average;
-      item.totalRelevance = total;
-    } else if (method === "hybridSearch") {
-      const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score * 10 + 1 : 0);
-      const total = scores.reduce((a, b) => a + b, 0);
-      const average = scores.length ? total / scores.length : 0;
-      item.averageRelevance = average;
-      item.totalRelevance = total;
+    if (expand?.after) {
+      for (const chunk of results) {
+        const indicesToFetch = Array.from(
+          { length: expand.after },
+          (_, i) => chunk.chunk_index + i + 1
+        );
+        console.log("[EXULU] Indices to fetch:", indicesToFetch);
+        await Promise.all(indicesToFetch.map(async (index) => {
+          if (expandedMap.has(`${chunk.item_id}-${index}`)) {
+            return;
+          }
+          const expandedChunk = await db3(chunksTable).where({
+            source: chunk.item_id,
+            chunk_index: index
+          }).first();
+          if (expandedChunk) {
+            expandedMap.set(`${chunk.item_id}-${index}`, {
+              chunk_content: expandedChunk.content,
+              chunk_index: expandedChunk.chunk_index,
+              chunk_id: expandedChunk.id,
+              chunk_source: expandedChunk.source,
+              chunk_metadata: expandedChunk.metadata,
+              chunk_created_at: expandedChunk.createdAt,
+              chunk_updated_at: expandedChunk.updatedAt,
+              item_updated_at: chunk.item_updated_at,
+              item_created_at: chunk.item_created_at,
+              item_id: chunk.item_id,
+              item_external_id: chunk.item_external_id,
+              item_name: chunk.item_name,
+              chunk_cosine_distance: 0,
+              chunk_fts_rank: 0,
+              chunk_hybrid_score: 0,
+              context: {
+                name: table.name.singular,
+                id: table.id || ""
+              }
+            });
+          }
+        }));
+      }
     }
-  });
-  if (resultReranker && query) {
-    items = await resultReranker(items);
+    results = Array.from(expandedMap.values());
+    results = results.sort((a, b) => {
+      if (a.item_id !== b.item_id) {
+        return a.item_id.localeCompare(b.item_id);
+      }
+      const aIndex = Number(a.chunk_index);
+      const bIndex = Number(b.chunk_index);
+      return aIndex - bIndex;
+    });
   }
-  console.log("[EXULU] Vector search results after slicing:", items?.length);
   await updateStatistic({
     name: "count",
     label: table.name.singular,
@@ -3374,7 +3581,7 @@ var vectorSearch = async ({
       id: table.id || "",
       embedder: embedder.name
     },
-    items
+    chunks: results
   };
 };
 var RBACResolver = async (db3, entityName, resourceId, rights_mode) => {
@@ -3404,10 +3611,10 @@ var contextToTableDefinition = (context) => {
       plural: tableName?.endsWith("s") ? tableName : tableName + "s"
     },
     RBAC: true,
+    processor: context.processor,
     fields: context.fields.map((field) => ({
       name: sanitizeName(field.name),
       type: field.type,
-      processor: field.processor,
       required: field.required,
       default: field.default,
       index: field.index,
@@ -3537,7 +3744,6 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     const tableNamePlural = table.name.plural.toLowerCase();
     const tableNameSingular = table.name.singular.toLowerCase();
     const tableNameSingularUpperCaseFirst = table.name.singular.charAt(0).toUpperCase() + table.name.singular.slice(1);
-    const processorFields = table.fields.filter((field) => field.processor?.execute);
     typeDefs += `
         ${tableNameSingular === "agent" ? `${tableNameSingular}ById(id: ID!, project: ID): ${tableNameSingular}` : `${tableNameSingular}ById(id: ID!): ${tableNameSingular}`}
       
@@ -3548,7 +3754,7 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     `;
     if (table.type === "items") {
       typeDefs += `
-      ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}VectorSearchResult
+      ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
     `;
     }
     mutationDefs += `
@@ -3564,9 +3770,10 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     ${tableNameSingular}ExecuteSource(source: ID!, inputs: JSON!): ${tableNameSingular}ExecuteSourceReturnPayload
     ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}DeleteChunksReturnPayload
     `;
-      if (processorFields?.length > 0) {
+      if (table.processor) {
         mutationDefs += `
-    ${tableNameSingular}ProcessItemField(item: ID!, field: ${tableNameSingular}ProcessorFieldEnum!): ${tableNameSingular}ProcessItemFieldReturnPayload
+    ${tableNameSingular}ProcessItem(item: ID!): ${tableNameSingular}ProcessItemFieldReturnPayload
+    ${tableNameSingular}ProcessItems(limit: Int, filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingular}ProcessItemFieldReturnPayload
     `;
       }
       modelDefs += `
@@ -3584,8 +3791,8 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
 
     type ${tableNameSingular}ProcessItemFieldReturnPayload {
         message: String!
-        result: String!
-        job: String
+        results: [String]
+        jobs: [String]
     }
 
     type ${tableNameSingular}DeleteChunksReturnPayload {
@@ -3600,20 +3807,42 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
         tsvector
     }
 
-    ${processorFields.length > 0 ? `
-    enum ${tableNameSingular}ProcessorFieldEnum {
-        ${processorFields.map((field) => field.name).join("\n")}
+    input SearchCutoffs {
+        cosineDistance: Float
+        hybrid: Float
+        tsvector: Float
+    }
+
+    input SearchExpand {
+        before: Int
+        after: Int
     }
-    ` : ""}
- 
 
-  type ${tableNameSingular}VectorSearchResult {
-        items: [${tableNameSingular}]!
+    type ${tableNameSingular}VectorSearchResult {
+        chunks: [${tableNameSingular}VectorSearchChunk!]!
         context: VectoSearchResultContext!
         filters: JSON!
         query: String!
         method: VectorMethodEnum!
     }
+        
+    type ${tableNameSingular}VectorSearchChunk {
+        chunk_content: String
+        chunk_index: Int
+        chunk_id: String
+        chunk_source: String
+        chunk_metadata: JSON
+        chunk_created_at: Date
+        chunk_updated_at: Date
+        item_updated_at: Date
+        item_created_at: Date
+        item_id: String!
+        item_external_id: String
+        item_name: String!
+        chunk_cosine_distance: Float
+        chunk_fts_rank: Float
+        chunk_hybrid_score: Float
+    }
 
     type VectoSearchResultContext {
         name: String!
@@ -3722,6 +3951,7 @@ type PageInfo {
         worker: config2.concurrency?.worker || void 0,
         queue: config2.concurrency?.queue || void 0
       },
+      timeoutInSeconds: config2.timeoutInSeconds,
       ratelimit: config2.ratelimit,
       isMaxed: await config2.queue.isMaxed(),
       isPaused: await config2.queue.isPaused(),
@@ -3935,25 +4165,21 @@ type PageInfo {
   };
   resolvers.Query["contexts"] = async (_, args, context, info) => {
     const data = await Promise.all(contexts.map(async (context2) => {
-      let processors = await Promise.all(context2.fields.map(async (field) => {
-        if (field.processor) {
-          let queueName = void 0;
-          if (field.processor?.config?.queue) {
-            const config2 = await field.processor?.config?.queue;
-            queueName = config2?.queue?.name || void 0;
-          }
-          return {
-            field: field.name,
-            description: field.processor?.description,
-            queue: queueName,
-            trigger: field.processor?.config?.trigger,
-            timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
-            generateEmbeddings: field.processor?.config?.generateEmbeddings || false
-          };
-        }
-        return null;
-      }));
-      processors = processors.filter((processor) => processor !== null);
+      let processor = null;
+      if (context2.processor) {
+        processor = await new Promise(async (resolve, reject) => {
+          const config2 = await context2.processor?.config;
+          const queue = await config2?.queue;
+          resolve({
+            name: context2.processor.name,
+            description: context2.processor.description,
+            queue: queue?.queue?.name || void 0,
+            trigger: context2.processor?.config?.trigger || "manual",
+            timeoutInSeconds: queue?.timeoutInSeconds || 600,
+            generateEmbeddings: context2.processor?.config?.generateEmbeddings || false
+          });
+        });
+      }
       const sources = await Promise.all(context2.sources.map(async (source) => {
         let queueName = void 0;
         if (source.config) {
@@ -3985,7 +4211,7 @@ type PageInfo {
         slug: "/contexts/" + context2.id,
         active: context2.active,
         sources,
-        processors,
+        processor,
         fields: context2.fields.map((field) => {
           return {
             ...field,
@@ -4011,25 +4237,21 @@ type PageInfo {
     if (!data) {
       return null;
     }
-    let processors = await Promise.all(data.fields.map(async (field) => {
-      if (field.processor) {
-        let queueName = void 0;
-        if (field.processor?.config?.queue) {
-          const config2 = await field.processor?.config?.queue;
-          queueName = config2?.queue?.name || void 0;
-        }
-        return {
-          field: field.name,
-          description: field.processor?.description,
-          queue: queueName,
-          trigger: field.processor?.config?.trigger,
-          timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
-          generateEmbeddings: field.processor?.config?.generateEmbeddings || false
-        };
-      }
-      return null;
-    }));
-    processors = processors.filter((processor) => processor !== null);
+    let processor = null;
+    if (data.processor) {
+      processor = await new Promise(async (resolve, reject) => {
+        const config2 = await data.processor?.config;
+        const queue = await config2?.queue;
+        resolve({
+          name: data.processor.name,
+          description: data.processor.description,
+          queue: queue?.queue?.name || void 0,
+          trigger: data.processor?.config?.trigger || "manual",
+          timeoutInSeconds: queue?.timeoutInSeconds || 600,
+          generateEmbeddings: data.processor?.config?.generateEmbeddings || false
+        });
+      });
+    }
     const sources = await Promise.all(data.sources.map(async (source) => {
       let queueName = void 0;
       if (source.config) {
@@ -4066,39 +4288,18 @@ type PageInfo {
       slug: "/contexts/" + data.id,
       active: data.active,
       sources,
-      processors,
+      processor,
       fields: await Promise.all(data.fields.map(async (field) => {
         const label = field.name?.replace("_s3key", "");
         if (field.type === "file" && !field.name.endsWith("_s3key")) {
           field.name = field.name + "_s3key";
         }
-        let queue = null;
-        if (field.processor?.config?.queue) {
-          queue = await field.processor.config.queue;
-        }
         return {
           ...field,
           name: sanitizeName(field.name),
           ...field.type === "file" ? {
             allowedFileTypes: field.allowedFileTypes
           } : {},
-          ...field.processor ? {
-            processor: {
-              description: field.processor?.description,
-              config: {
-                trigger: field.processor?.config?.trigger,
-                queue: {
-                  name: queue?.queue.name || void 0,
-                  ratelimit: queue?.ratelimit || void 0,
-                  concurrency: {
-                    worker: queue?.concurrency?.worker || void 0,
-                    queue: queue?.concurrency?.queue || void 0
-                  }
-                }
-              },
-              execute: "function"
-            }
-          } : {},
           label
         };
       })),
@@ -4167,6 +4368,7 @@ type PageInfo {
     type QueueResult {
         name: String!
         concurrency: QueueConcurrency!
+        timeoutInSeconds: Int!
         ratelimit: Int!
         isMaxed: Boolean!
         isPaused: Boolean!
@@ -4248,17 +4450,12 @@ type AgentEvalFunctionConfig {
 }
 
 type ItemChunks {
-    cosine_distance: Float
-    fts_rank: Float
-    hybrid_score: Float
-    content: String
-    source: ID
-    chunk_index: Int
-    chunk_id: ID
-    chunk_created_at: Date
-    chunk_updated_at: Date
-    embedding_size: Float
-    metadata: JSON
+    chunk_id: String!
+    chunk_index: Int!
+    chunk_content: String!
+    chunk_source: String!
+    chunk_created_at: Date!
+    chunk_updated_at: Date!
 }
 
 type Provider {
@@ -4294,7 +4491,7 @@ type Context {
     fields: JSON
     configuration: JSON
     sources: [ContextSource]
-    processors: [ContextProcessor]
+    processor: ContextProcessor
 }
 type Embedder {
     name: String!
@@ -4308,7 +4505,7 @@ type EmbedderConfig {
     default: String
 }
 type ContextProcessor {
-    field: String!
+    name: String!
     description: String
     queue: String
     trigger: String
@@ -4372,6 +4569,9 @@ type Job {
   name: String!
   returnvalue: JSON
   stacktrace: [String]
+  finishedOn: Date
+  processedOn: Date
+  attemptsMade: Int
   failedReason: String
   state: String!
   data: JSON
@@ -4478,6 +4678,8 @@ var getPresignedUrl = async (bucket, key, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
+  console.log("[EXULU] getting presigned url for bucket", bucket);
+  console.log("[EXULU] getting presigned url for key", key);
   const url = await (0, import_s3_request_presigner.getSignedUrl)(
     getS3Client(config),
     new import_client_s3.GetObjectCommand({
@@ -4488,6 +4690,18 @@ var getPresignedUrl = async (bucket, key, config) => {
   );
   return url;
 };
+function sanitizeMetadata(metadata) {
+  if (!metadata) return void 0;
+  const sanitized = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    if (typeof value === "string") {
+      sanitized[key] = encodeURIComponent(value);
+    } else {
+      sanitized[key] = String(value);
+    }
+  }
+  return sanitized;
+}
 var addGeneralPrefixToKey = (keyPath, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
@@ -4523,19 +4737,41 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
   const client2 = getS3Client(config);
   let defaultBucket = config.fileUploads.s3Bucket;
   let key = fileName;
-  key = addGeneralPrefixToKey(key, config);
   key = addUserPrefixToKey(key, user || "api");
-  console.log("[EXULU] uploading file to s3 into bucket", defaultBucket, "with key", key);
+  key = addGeneralPrefixToKey(key, config);
+  const sanitizedMetadata = sanitizeMetadata(options.metadata);
   const command = new import_client_s3.PutObjectCommand({
     Bucket: customBucket || defaultBucket,
     Key: key,
     Body: file,
     ContentType: options.contentType,
-    Metadata: options.metadata,
+    Metadata: sanitizedMetadata,
     ContentLength: file.byteLength
   });
-  await client2.send(command);
-  console.log("[EXULU] file uploaded to s3 into bucket", customBucket || defaultBucket, "with key", key);
+  const maxRetries = 3;
+  let lastError = null;
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      await client2.send(command);
+      break;
+    } catch (error) {
+      lastError = error;
+      if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
+        if (attempt < maxRetries) {
+          const backoffMs = Math.pow(2, attempt) * 1e3;
+          await new Promise((resolve) => setTimeout(resolve, backoffMs));
+          s3Client = void 0;
+          getS3Client(config);
+          continue;
+        }
+      } else {
+        throw error;
+      }
+    }
+  }
+  if (lastError) {
+    throw lastError;
+  }
   return addBucketPrefixToKey(
     key,
     customBucket || defaultBucket
@@ -4607,6 +4843,8 @@ var createUppyRoutes = async (app, contexts, config) => {
       res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
       return;
     }
+    key = key.replace(`${bucket}/`, "");
+    console.log("[EXULU] deleting file from s3 into bucket", bucket, "with key", key);
     const client2 = getS3Client(config);
     const command = new import_client_s3.DeleteObjectCommand({
       Bucket: bucket,
@@ -4730,7 +4968,7 @@ var createUppyRoutes = async (app, contexts, config) => {
     const client2 = getS3Client(config);
     const command = new import_client_s3.ListObjectsV2Command({
       Bucket: config.fileUploads.s3Bucket,
-      Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}${authenticationResult.user.id}`,
+      Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}user_${authenticationResult.user.id}`,
       MaxKeys: 9,
       ...req.query.continuationToken && { ContinuationToken: req.query.continuationToken }
     });
@@ -4742,7 +4980,17 @@ var createUppyRoutes = async (app, contexts, config) => {
         search.toLowerCase()
       ));
     }
-    res.json(response);
+    res.json({
+      ...response,
+      Contents: response.Contents?.map((content) => {
+        return {
+          ...content,
+          // For consistency and to support multi-bucket environments
+          // we prepend the bucket name to the key here.
+          Key: `${config.fileUploads?.s3Bucket}/${content.Key}`
+        };
+      })
+    });
     res.end();
   });
   app.get("/s3/sts", (req, res, next) => {
@@ -4803,8 +5051,9 @@ var createUppyRoutes = async (app, contexts, config) => {
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
     const key = generateS3Key2(filename);
-    let fullKey = addGeneralPrefixToKey(key, config);
-    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
+    let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    fullKey = addGeneralPrefixToKey(fullKey, config);
+    console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
     (0, import_s3_request_presigner.getSignedUrl)(
       getS3Client(config),
       new import_client_s3.PutObjectCommand({
@@ -4858,8 +5107,9 @@ var createUppyRoutes = async (app, contexts, config) => {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
     const key = `${(0, import_node_crypto.randomUUID)()}-_EXULU_${filename}`;
-    let fullKey = addGeneralPrefixToKey(key, config);
-    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
+    let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    fullKey = addGeneralPrefixToKey(fullKey, config);
+    console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
     const params = {
       Bucket: config.fileUploads.s3Bucket,
       Key: fullKey,
@@ -5110,7 +5360,10 @@ var createProjectRetrievalTool = async ({
 };
 var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID, req, project) => {
   if (!currentTools) return {};
-  if (!allExuluTools) return {};
+  if (!allExuluTools) {
+    allExuluTools = [];
+  }
+  ;
   if (!contexts) {
     contexts = [];
   }
@@ -5144,6 +5397,7 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
             ...cur.tool,
             description,
             async *execute(inputs, options) {
+              console.log("[EXULU] Executing tool", cur.name, "with inputs", inputs, "and options", options);
               if (!cur.tool?.execute) {
                 console.error("[EXULU] Tool execute function is undefined.", cur.tool);
                 throw new Error("Tool execute function is undefined.");
@@ -5995,6 +6249,68 @@ var ExuluTool2 = class {
       execute: execute2
     });
   }
+  execute = async ({
+    agent,
+    config,
+    user,
+    inputs,
+    project
+  }) => {
+    const agentInstance = await loadAgent(agent);
+    if (!agentInstance) {
+      throw new Error("Agent not found.");
+    }
+    const { db: db3 } = await postgresClient();
+    let providerapikey;
+    const variableName = agentInstance.providerapikey;
+    if (variableName) {
+      console.log("[EXULU] provider api key variable name", variableName);
+      const variable = await db3.from("variables").where({ name: variableName }).first();
+      if (!variable) {
+        throw new Error("Provider API key variable not found for " + agentInstance.name + " (" + agentInstance.id + ").");
+      }
+      providerapikey = variable.value;
+      if (!variable.encrypted) {
+        throw new Error("Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys.");
+      }
+      if (variable.encrypted) {
+        const bytes = import_crypto_js2.default.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+        providerapikey = bytes.toString(import_crypto_js2.default.enc.Utf8);
+      }
+    }
+    const tools = await convertToolsArrayToObject(
+      [this],
+      [],
+      agentInstance.tools,
+      providerapikey,
+      void 0,
+      user,
+      config,
+      void 0,
+      void 0,
+      project
+    );
+    const tool2 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
+    if (!tool2?.execute) {
+      throw new Error("Tool " + sanitizeName(this.name) + " not found in " + JSON.stringify(tools));
+    }
+    console.log("[EXULU] Tool found", this.name);
+    const generator = tool2.execute(inputs, {
+      toolCallId: this.id + "_" + (0, import_node_crypto2.randomUUID)(),
+      messages: []
+    });
+    let lastValue;
+    for await (const chunk of generator) {
+      lastValue = chunk;
+    }
+    if (typeof lastValue === "string") {
+      lastValue = JSON.parse(lastValue);
+    }
+    if (lastValue?.result && typeof lastValue.result === "string") {
+      lastValue.result = JSON.parse(lastValue.result);
+    }
+    return lastValue;
+  };
 };
 var getTableName = (id) => {
   return sanitizeName(id) + "_items";
@@ -6044,12 +6360,12 @@ var ExuluContext = class {
   name;
   active;
   fields;
+  processor;
   rateLimit;
   description;
   embedder;
   queryRewriter;
   resultReranker;
-  // todo typings
   configuration;
   sources = [];
   constructor({
@@ -6057,6 +6373,7 @@ var ExuluContext = class {
     name,
     description,
     embedder,
+    processor,
     active,
     rateLimit,
     fields,
@@ -6069,10 +6386,21 @@ var ExuluContext = class {
     this.name = name;
     this.fields = fields || [];
     this.sources = sources || [];
+    this.processor = processor;
     this.configuration = configuration || {
       calculateVectors: "manual",
       language: "english",
-      defaultRightsMode: "private"
+      defaultRightsMode: "private",
+      maxRetrievalResults: 10,
+      expand: {
+        before: 0,
+        after: 0
+      },
+      cutoffs: {
+        cosineDistance: 0.5,
+        tsvector: 0.5,
+        hybrid: 0.5
+      }
     };
     this.description = description;
     this.embedder = embedder;
@@ -6082,26 +6410,35 @@ var ExuluContext = class {
     this.resultReranker = resultReranker;
   }
   processField = async (trigger, item, exuluConfig, user, role) => {
-    if (!item.field) {
-      throw new Error("Field property on item is required for running a specific processor.");
+    console.log("[EXULU] processing item, ", item, " in context", this.id);
+    const exuluStorage = new ExuluStorage({ config: exuluConfig });
+    if (!this.processor) {
+      throw new Error(`Processor is not set for this context: ${this.id}.`);
     }
-    console.log("[EXULU] processing field", item.field, " in context", this.id);
-    console.log("[EXULU] fields", this.fields.map((field2) => field2.name));
-    const field = this.fields.find((field2) => {
-      return field2.name.replace("_s3key", "") === item.field.replace("_s3key", "");
-    });
-    if (!field || !field.processor) {
-      console.error("[EXULU] field not found or processor not set for field", item.field, " in context", this.id);
-      throw new Error("Field not found or processor not set for field " + item.field + " in context " + this.id);
+    if (this.processor.filter) {
+      const result = await this.processor.filter({
+        item,
+        user,
+        role,
+        utils: {
+          storage: exuluStorage
+        },
+        exuluConfig
+      });
+      if (!result) {
+        return {
+          result: void 0,
+          job: void 0
+        };
+      }
     }
-    const exuluStorage = new ExuluStorage({ config: exuluConfig });
-    const queue = await field.processor.config?.queue;
+    const queue = await this.processor.config?.queue;
     if (queue?.queue.name) {
       console.log("[EXULU] processor is in queue mode, scheduling job.");
       const job = await bullmqDecorator({
-        timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
-        label: `${this.name} ${field.name} data processor`,
-        processor: `${this.id}-${field.name}`,
+        timeoutInSeconds: this.processor.config?.timeoutInSeconds || 600,
+        label: `${this.name} ${this.processor.name} data processor`,
+        processor: `${this.id}-${this.processor.name}`,
         context: this.id,
         inputs: item,
         item: item.id,
@@ -6116,12 +6453,12 @@ var ExuluContext = class {
         trigger
       });
       return {
-        result: {},
+        result: void 0,
         job: job.id
       };
     }
     console.log("[EXULU] POS 1 -- EXULU CONTEXT PROCESS FIELD");
-    const processorResult = await field.processor.execute({
+    const processorResult = await this.processor.execute({
       item,
       user,
       role,
@@ -6138,7 +6475,8 @@ var ExuluContext = class {
     await db3.from(getTableName(this.id)).where({
       id: processorResult.id
     }).update({
-      ...processorResult
+      ...processorResult,
+      last_processed_at: (/* @__PURE__ */ new Date()).toISOString()
     });
     return {
       result: processorResult,
@@ -6152,7 +6490,10 @@ var ExuluContext = class {
       user: options.user,
       role: options.role,
       context: this,
-      db: db3
+      db: db3,
+      limit: options?.limit || this.configuration.maxRetrievalResults || 10,
+      cutoffs: options.cutoffs,
+      expand: options.expand
     });
     return result;
   };
@@ -6256,9 +6597,8 @@ var ExuluContext = class {
     console.log("[EXULU] context configuration", this.configuration);
     let jobs = [];
     let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always");
-    for (const [key, value] of Object.entries(item)) {
-      console.log("[EXULU] Checking for processors for field", key);
-      const processor = this.fields.find((field) => field.name === key.replace("_s3key", ""))?.processor;
+    if (this.processor) {
+      const processor = this.processor;
       console.log("[EXULU] Processor found", processor);
       if (processor && (processor?.config?.trigger === "onInsert" || processor?.config?.trigger === "onUpdate" || processor?.config?.trigger === "always")) {
         const {
@@ -6268,8 +6608,7 @@ var ExuluContext = class {
           "api",
           {
             ...item,
-            id: results[0].id,
-            field: key
+            id: results[0].id
           },
           config,
           user,
@@ -6336,8 +6675,8 @@ var ExuluContext = class {
     await mutation;
     let jobs = [];
     let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always");
-    for (const [key, value] of Object.entries(item)) {
-      const processor = this.fields.find((field) => field.name === key.replace("_s3key", ""))?.processor;
+    if (this.processor) {
+      const processor = this.processor;
       if (processor && (processor?.config?.trigger === "onInsert" || processor?.config?.trigger === "onUpdate" || processor?.config?.trigger === "always")) {
         const {
           job: processorJob,
@@ -6346,8 +6685,7 @@ var ExuluContext = class {
           "api",
           {
             ...item,
-            id: record.id,
-            field: key
+            id: record.id
           },
           config,
           user,
@@ -6585,22 +6923,26 @@ var ExuluContext = class {
   };
   // Exports the context as a tool that can be used by an agent
   tool = () => {
+    if (this.configuration.enableAsTool === false) {
+      return null;
+    }
     return new ExuluTool2({
       id: this.id,
-      name: `${this.name}`,
+      name: `${this.name}_context_search`,
       type: "context",
       category: "contexts",
       inputSchema: import_zod.z.object({
-        query: import_zod.z.string()
+        originalQuestion: import_zod.z.string().describe("The original question that the user asked"),
+        relevantKeywords: import_zod.z.array(import_zod.z.string()).describe("The keywords that are relevant to the user's question, for example names of specific products, systems or parts, IDs, etc.")
       }),
       config: [],
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
-      execute: async ({ query, user, role }) => {
+      execute: async ({ originalQuestion, relevantKeywords, user, role }) => {
         const { db: db3 } = await postgresClient();
         const result = await vectorSearch({
           page: 1,
-          limit: 50,
-          query,
+          limit: this.configuration.maxRetrievalResults ?? 10,
+          query: originalQuestion,
           filters: [],
           user,
           role,
@@ -6620,7 +6962,13 @@ var ExuluContext = class {
           role: user?.role?.id
         });
         return {
-          items: result.items
+          result: JSON.stringify(result.chunks.map((chunk) => ({
+            ...chunk,
+            context: {
+              name: this.name,
+              id: this.id
+            }
+          })))
         };
       }
     });
@@ -7487,7 +7835,7 @@ var import_ai3 = require("ai");
 var import_crypto_js4 = __toESM(require("crypto-js"), 1);
 
 // src/registry/log-metadata.ts
-function logMetadata2(id, additionalMetadata) {
+function logMetadata(id, additionalMetadata) {
   return {
     __logMetadata: true,
     id,
@@ -7497,9 +7845,32 @@ function logMetadata2(id, additionalMetadata) {
 
 // src/registry/workers.ts
 var redisConnection;
+var unhandledRejectionHandlerInstalled = false;
+var installGlobalErrorHandlers = () => {
+  if (unhandledRejectionHandlerInstalled) return;
+  process.on("unhandledRejection", (reason, promise) => {
+    console.error("[EXULU] Unhandled Promise Rejection detected! This would have crashed the worker.", {
+      reason: reason instanceof Error ? reason.message : String(reason),
+      stack: reason instanceof Error ? reason.stack : void 0
+    });
+  });
+  process.on("uncaughtException", (error) => {
+    console.error("[EXULU] Uncaught Exception detected! This would have crashed the worker.", {
+      error: error.message,
+      stack: error.stack
+    });
+    if (error.message.includes("FATAL") || error.message.includes("Cannot find module")) {
+      console.error("[EXULU] Fatal error detected, exiting process.");
+      process.exit(1);
+    }
+  });
+  unhandledRejectionHandlerInstalled = true;
+  console.log("[EXULU] Global error handlers installed to prevent worker crashes");
+};
 var createWorkers = async (agents, queues2, config, contexts, evals, tools, tracer) => {
   console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
   console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
+  installGlobalErrorHandlers();
   if (!redisServer.host || !redisServer.port) {
     console.error("[EXULU] you are trying to start worker, but no redis server is configured in the environment.");
     throw new Error("No redis server configured in the environment, so cannot start worker.");
@@ -7524,7 +7895,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
     const worker = new import_bullmq4.Worker(
       `${queue.queue.name}`,
       async (bullmqJob) => {
-        console.log("[EXULU] starting execution for job", logMetadata2(bullmqJob.name, {
+        console.log("[EXULU] starting execution for job", logMetadata(bullmqJob.name, {
           name: bullmqJob.name,
           jobId: bullmqJob.id,
           status: await bullmqJob.getState(),
@@ -7534,9 +7905,12 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         const data = bullmqJob.data;
         const timeoutInSeconds = data.timeoutInSeconds || 600;
         const timeoutMs = timeoutInSeconds * 1e3;
+        let timeoutHandle;
         const timeoutPromise = new Promise((_, reject) => {
-          setTimeout(() => {
-            reject(new Error(`Timeout for job ${bullmqJob.id} reached after ${timeoutInSeconds}s`));
+          timeoutHandle = setTimeout(() => {
+            const timeoutError = new Error(`Timeout for job ${bullmqJob.id} reached after ${timeoutInSeconds}s`);
+            console.error(`[EXULU] ${timeoutError.message}`);
+            reject(timeoutError);
           }, timeoutMs);
         });
         const workPromise = (async () => {
@@ -7544,7 +7918,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
             console.log(`[EXULU] Job ${bullmqJob.id} - Log file: logs/jobs/job-${bullmqJob.id}.log`);
             bullmq.validate(bullmqJob.id, data);
             if (data.type === "embedder") {
-              console.log("[EXULU] running an embedder job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running an embedder job.", logMetadata(bullmqJob.name));
               const label = `embedder-${bullmqJob.name}`;
               await db3.from("job_results").insert({
                 job_id: bullmqJob.id,
@@ -7564,12 +7938,12 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!embedder) {
                 throw new Error(`Embedder ${data.embedder} not found in the registry.`);
               }
-              const result2 = await context.createAndUpsertEmbeddings(data.inputs, config, data.user, {
+              const result = await context.createAndUpsertEmbeddings(data.inputs, config, data.user, {
                 label: embedder.name,
                 trigger: data.trigger
               }, data.role, bullmqJob.id);
               return {
-                result: result2,
+                result,
                 metadata: {}
               };
             }
@@ -7587,21 +7961,15 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!context) {
                 throw new Error(`Context ${data.context} not found in the registry.`);
               }
-              const field = context.fields.find((field2) => {
-                return field2.name.replace("_s3key", "") === data.inputs.field.replace("_s3key", "");
-              });
-              if (!field) {
-                throw new Error(`Field ${data.inputs.field} not found in the context ${data.context}.`);
-              }
-              if (!field.processor) {
-                throw new Error(`Processor not set for field ${data.inputs.field} in the context ${data.context}.`);
-              }
               if (!data.inputs.id) {
-                throw new Error(`[EXULU] Item not set for processor ${field.name} in context ${context.id}, running in job ${bullmqJob.id}.`);
+                throw new Error(`[EXULU] Item not set for processor in context ${context.id}, running in job ${bullmqJob.id}.`);
+              }
+              if (!context.processor) {
+                throw new Error(`Tried to run a processor job for context ${context.id}, but no processor is set.`);
               }
               const exuluStorage = new ExuluStorage({ config });
               console.log("[EXULU] POS 2 -- EXULU CONTEXT PROCESS FIELD");
-              const processorResult = await field.processor.execute({
+              const processorResult = await context.processor.execute({
                 item: data.inputs,
                 user: data.user,
                 role: data.role,
@@ -7611,16 +7979,17 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 exuluConfig: config
               });
               if (!processorResult) {
-                throw new Error(`[EXULU] Processor ${field.name} in context ${context.id}, running in job ${bullmqJob.id} did not return an item.`);
+                throw new Error(`[EXULU] Processor in context ${context.id}, running in job ${bullmqJob.id} did not return an item.`);
               }
               delete processorResult.field;
               await db3.from(getTableName(context.id)).where({
                 id: processorResult.id
               }).update({
-                ...processorResult
+                ...processorResult,
+                last_processed_at: (/* @__PURE__ */ new Date()).toISOString()
               });
               let jobs = [];
-              if (field.processor.config?.generateEmbeddings) {
+              if (context.processor?.config?.generateEmbeddings) {
                 const fullItem = await db3.from(getTableName(context.id)).where({
                   id: processorResult.id
                 }).first();
@@ -7646,7 +8015,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               };
             }
             if (data.type === "eval_run") {
-              console.log("[EXULU] running an eval run job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running an eval run job.", logMetadata(bullmqJob.name));
               const label = `eval-run-${data.eval_run_id}-${data.test_case_id}`;
               const existingResult = await db3.from("job_results").where({ label }).first();
               if (existingResult) {
@@ -7695,7 +8064,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     resolve(messages2);
                     break;
                   } catch (error) {
-                    console.error(`[EXULU] error processing UI messages flow for agent ${agentInstance.name} (${agentInstance.id}).`, logMetadata2(bullmqJob.name, {
+                    console.error(`[EXULU] error processing UI messages flow for agent ${agentInstance.name} (${agentInstance.id}).`, logMetadata(bullmqJob.name, {
                       error: error instanceof Error ? error.message : String(error)
                     }));
                     attempts++;
@@ -7706,9 +8075,9 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   }
                 }
               });
-              const result2 = await promise;
-              const messages = result2.messages;
-              const metadata = result2.metadata;
+              const result = await promise;
+              const messages = result.messages;
+              const metadata = result.metadata;
               const evalFunctions = evalRun.eval_functions;
               let evalFunctionResults = [];
               for (const evalFunction of evalFunctions) {
@@ -7716,7 +8085,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 if (!evalMethod) {
                   throw new Error(`Eval function ${evalFunction.id} not found in the registry, check your code and make sure the eval function is registered correctly.`);
                 }
-                let result3;
+                let result2;
                 if (evalMethod.queue) {
                   const queue2 = await evalMethod.queue;
                   const jobData = {
@@ -7747,21 +8116,21 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                   if (!job.id) {
                     throw new Error(`Tried to add job to queue ${queue2.queue.name} but failed to get the job ID.`);
                   }
-                  result3 = await pollJobResult({ queue: queue2, jobId: job.id });
+                  result2 = await pollJobResult({ queue: queue2, jobId: job.id });
                   const evalFunctionResult = {
                     test_case_id: testCase.id,
                     eval_run_id: evalRun.id,
                     eval_function_id: evalFunction.id,
                     eval_function_name: evalFunction.name,
                     eval_function_config: evalFunction.config || {},
-                    result: result3 || 0
+                    result: result2 || 0
                   };
-                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result3}`, logMetadata2(bullmqJob.name, {
-                    result: result3 || 0
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata(bullmqJob.name, {
+                    result: result2 || 0
                   }));
                   evalFunctionResults.push(evalFunctionResult);
                 } else {
-                  result3 = await evalMethod.run(
+                  result2 = await evalMethod.run(
                     agentInstance,
                     agentBackend,
                     testCase,
@@ -7772,15 +8141,15 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                     test_case_id: testCase.id,
                     eval_run_id: evalRun.id,
                     eval_function_id: evalFunction.id,
-                    result: result3 || 0
+                    result: result2 || 0
                   };
                   evalFunctionResults.push(evalFunctionResult);
-                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result3}`, logMetadata2(bullmqJob.name, {
-                    result: result3 || 0
+                  console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata(bullmqJob.name, {
+                    result: result2 || 0
                   }));
                 }
               }
-              const scores = evalFunctionResults.map((result3) => result3.result);
+              const scores = evalFunctionResults.map((result2) => result2.result);
               console.log("[EXULU] Exulu eval run scores for test case: " + testCase.id, scores);
               let score = 0;
               switch (data.scoring_method?.toLowerCase()) {
@@ -7812,7 +8181,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               };
             }
             if (data.type === "eval_function") {
-              console.log("[EXULU] running an eval function job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running an eval function job.", logMetadata(bullmqJob.name));
               if (data.eval_functions?.length !== 1) {
                 throw new Error(`Expected 1 eval function for eval function job, got ${data.eval_functions?.length}.`);
               }
@@ -7845,30 +8214,30 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 messages: inputMessages
               } = await validateEvalPayload(data, agents);
               const evalFunctions = evalRun.eval_functions;
-              let result2;
+              let result;
               for (const evalFunction of evalFunctions) {
                 const evalMethod = evals.find((e) => e.id === evalFunction.id);
                 if (!evalMethod) {
                   throw new Error(`Eval function ${evalFunction.id} not found in the registry, check your code and make sure the eval function is registered correctly.`);
                 }
-                result2 = await evalMethod.run(
+                result = await evalMethod.run(
                   agentInstance,
                   backend,
                   testCase,
                   inputMessages,
                   evalFunction.config || {}
                 );
-                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result2}`, logMetadata2(bullmqJob.name, {
-                  result: result2 || 0
+                console.log(`[EXULU] eval function ${evalFunction.id} result: ${result}`, logMetadata(bullmqJob.name, {
+                  result: result || 0
                 }));
               }
               return {
-                result: result2,
+                result,
                 metadata: {}
               };
             }
             if (data.type === "source") {
-              console.log("[EXULU] running a source job.", logMetadata2(bullmqJob.name));
+              console.log("[EXULU] running a source job.", logMetadata(bullmqJob.name));
               if (!data.source) {
                 throw new Error(`No source id set for source job.`);
               }
@@ -7883,10 +8252,10 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!source) {
                 throw new Error(`Source ${data.source} not found in the context ${context.id}.`);
               }
-              const result2 = await source.execute(data.inputs);
+              const result = await source.execute(data.inputs);
               let jobs = [];
               let items = [];
-              for (const item of result2) {
+              for (const item of result) {
                 const { item: createdItem, job } = await context.createItem(
                   item,
                   config,
@@ -7896,14 +8265,14 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 );
                 if (job) {
                   jobs.push(job);
-                  console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata2(bullmqJob.name, {
+                  console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata(bullmqJob.name, {
                     item: createdItem,
                     job
                   }));
                 }
                 if (createdItem.id) {
                   items.push(createdItem.id);
-                  console.log(`[EXULU] created item through source update job ${createdItem.id}`, logMetadata2(bullmqJob.name, {
+                  console.log(`[EXULU] created item through source update job ${createdItem.id}`, logMetadata(bullmqJob.name, {
                     item: createdItem
                   }));
                 }
@@ -7918,7 +8287,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 role: data?.role
               });
               return {
-                result: result2,
+                result,
                 metadata: {
                   jobs,
                   items
@@ -7931,8 +8300,15 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
             throw error;
           }
         })();
-        const result = await Promise.race([workPromise, timeoutPromise]);
-        return result;
+        try {
+          const result = await Promise.race([workPromise, timeoutPromise]);
+          clearTimeout(timeoutHandle);
+          return result;
+        } catch (error) {
+          clearTimeout(timeoutHandle);
+          console.error(`[EXULU] job ${bullmqJob.id} failed (error caught in race handler).`, error instanceof Error ? error.message : String(error));
+          throw error;
+        }
       },
       {
         autorun: true,
@@ -7967,7 +8343,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         });
         return;
       }
-      console.error(`[EXULU] job failed.`, job?.name ? logMetadata2(job.name, {
+      console.error(`[EXULU] job failed.`, job?.name ? logMetadata(job.name, {
         error: error instanceof Error ? error.message : String(error)
       }) : error);
     });
@@ -7975,7 +8351,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
       console.error(`[EXULU] worker error.`, error);
     });
     worker.on("progress", (job, progress) => {
-      console.log(`[EXULU] job progress ${job.id}.`, logMetadata2(job.name, {
+      console.log(`[EXULU] job progress ${job.id}.`, logMetadata(job.name, {
         progress
       }));
     });
@@ -9141,7 +9517,7 @@ var ExuluQueues = class {
   // method of ExuluQueues we need to store the desired rate limit on the queue
   // here so we can use the value when creating workers for the queue instance
   // as there is no way to store a rate limit value natively on a bullm queue.
-  register = (name, concurrency, ratelimit = 1) => {
+  register = (name, concurrency, ratelimit = 1, timeoutInSeconds = 180) => {
     const queueConcurrency = concurrency.queue || 1;
     const workerConcurrency = concurrency.worker || 1;
     const use = async () => {
@@ -9157,7 +9533,8 @@ var ExuluQueues = class {
           concurrency: {
             worker: workerConcurrency,
             queue: queueConcurrency
-          }
+          },
+          timeoutInSeconds
         };
       }
       if (!redisServer.host?.length || !redisServer.port?.length) {
@@ -9183,7 +9560,8 @@ var ExuluQueues = class {
         concurrency: {
           worker: workerConcurrency,
           queue: queueConcurrency
-        }
+        },
+        timeoutInSeconds
       });
       return {
         queue: newQueue,
@@ -9191,7 +9569,8 @@ var ExuluQueues = class {
         concurrency: {
           worker: workerConcurrency,
           queue: queueConcurrency
-        }
+        },
+        timeoutInSeconds
       };
     };
     this.list.set(name, {
@@ -9201,6 +9580,7 @@ var ExuluQueues = class {
         queue: queueConcurrency
       },
       ratelimit,
+      timeoutInSeconds,
       use
     });
     return {
@@ -10266,7 +10646,7 @@ var ExuluApp = class {
       ...[previewPdfTool],
       ...todoTools,
       // Add contexts as tools
-      ...Object.values(contexts || {}).map((context) => context.tool())
+      ...Object.values(contexts || {}).map((context) => context.tool()).filter(Boolean)
       // Because agents are stored in the database,  we add those as tools
       // at request time, not during ExuluApp initialization. We add them
       // in the grahql tools resolver.
@@ -10428,10 +10808,7 @@ var ExuluApp = class {
               console.warn("[EXULU] No queue configured for source", source.name);
               continue;
             }
-            if (queue) {
-              if (!source.config?.schedule) {
-                throw new Error("Schedule is required for source when configuring a queue: " + source.name);
-              }
+            if (queue && source.config?.schedule) {
               console.log("[EXULU] Creating ContextSource scheduler for", source.name, "in queue", queue.queue?.name);
               await queue.queue?.upsertJobScheduler(source.id, {
                 pattern: source.config?.schedule