@exulu/backend 1.41.0 → 1.42.1

package/dist/index.js

@@ -2562,7 +2562,7 @@ var applyAccessControl = (table, query, user, field_prefix) => {
   }
   console.log("[EXULU] user.role", user?.role);
   console.log("[EXULU] table.name.plural", table.name.plural);
-  if (!user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
+  if (user && !user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
     console.error("==== Access control error: no role found or no access to entity type. ====");
     throw new Error("Access control error: no role found or no access to entity type.");
   }
@@ -2571,17 +2571,23 @@ var applyAccessControl = (table, query, user, field_prefix) => {
   if (!hasRBAC) {
     return query;
   }
+  if (user?.super_admin) {
+    return query;
+  }
   const prefix = field_prefix ? field_prefix + "." : "";
+  console.log("[EXULU] applying access control with this prefix", prefix);
   try {
     query = query.where(function() {
       this.where(`${prefix}rights_mode`, "public");
-      this.orWhere(`${prefix}created_by`, user.id);
-      this.orWhere(function() {
-        this.where(`${prefix}rights_mode`, "users").whereExists(function() {
-          this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
+      if (user) {
+        this.orWhere(`${prefix}created_by`, user.id);
+        this.orWhere(function() {
+          this.where(`${prefix}rights_mode`, "users").whereExists(function() {
+            this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
+          });
         });
-      });
-      if (user.role) {
+      }
+      if (user?.role) {
         this.orWhere(function() {
           this.where(`${prefix}rights_mode`, "roles").whereExists(function() {
             this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
@@ -3107,7 +3113,9 @@ function createQueries(table, agents, tools, contexts) {
         method: args.method,
         user: context.user,
         role: context.user?.role?.id,
-        trigger: "api"
+        trigger: "api",
+        cutoffs: args.cutoffs,
+        expand: args.expand
       });
     };
   }
@@ -3124,7 +3132,9 @@ var vectorSearch = async ({
   method,
   user,
   role,
-  trigger
+  trigger,
+  cutoffs,
+  expand
 }) => {
   const table = contextToTableDefinition(context);
   console.log("[EXULU] Called vector search.", {
@@ -3136,10 +3146,12 @@ var vectorSearch = async ({
     query,
     method,
     user,
-    role
+    role,
+    cutoffs,
+    expand
   });
-  if (limit > 50) {
-    throw new Error("Limit cannot be greater than 50.");
+  if (limit > 250) {
+    throw new Error("Limit cannot be greater than 1000.");
   }
   if (!query) {
     throw new Error("Query is required.");
@@ -3156,6 +3168,15 @@ var vectorSearch = async ({
   }
   const mainTable = getTableName(id);
   const chunksTable = getChunksTableName(id);
+  cutoffs = {
+    cosineDistance: cutoffs?.cosineDistance || context.configuration?.cutoffs?.cosineDistance || 0,
+    tsvector: cutoffs?.tsvector || context.configuration?.cutoffs?.tsvector || 0,
+    hybrid: cutoffs?.hybrid ? (cutoffs?.hybrid ?? 0) / 100 : context.configuration?.cutoffs ? (context.configuration?.cutoffs?.hybrid ?? 0) / 100 : 0
+  };
+  expand = {
+    before: expand?.before || context.configuration?.expand?.before || 0,
+    after: expand?.after || context.configuration?.expand?.after || 0
+  };
   let chunksQuery = db3(chunksTable + " as chunks").select([
     "chunks.id as chunk_id",
     "chunks.source",
@@ -3190,22 +3211,30 @@ var vectorSearch = async ({
   const vectorStr = `ARRAY[${vector.join(",")}]`;
   const vectorExpr = `${vectorStr}::vector`;
   const language = configuration.language || "english";
+  console.log("[EXULU] Vector search params:", { method, query, cutoffs });
   let resultChunks = [];
   switch (method) {
     case "tsvector":
       chunksQuery.limit(limit * 2);
+      const tokens = query.trim().split(/\s+/).filter((t) => t.length > 0);
+      const sanitizedTokens = tokens.flatMap((t) => {
+        return t.split(/[^\w]+/).filter((part) => part.length > 0);
+      });
+      const orQuery = sanitizedTokens.join(" | ");
+      console.log("[EXULU] FTS query transformation:", { original: query, tokens, sanitizedTokens, orQuery, cutoff: cutoffs?.tsvector });
       chunksQuery.select(db3.raw(
-        `ts_rank(chunks.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
-        [language, query]
+        `ts_rank(chunks.fts, to_tsquery(?, ?)) as fts_rank`,
+        [language, orQuery]
       )).whereRaw(
-        `chunks.fts @@ websearch_to_tsquery(?, ?)`,
-        [language, query]
+        `(chunks.fts @@ to_tsquery(?, ?)) AND (items.archived IS FALSE OR items.archived IS NULL)`,
+        [language, orQuery]
       ).orderByRaw(`fts_rank DESC`);
+      console.log("[EXULU] FTS query SQL:", chunksQuery.toQuery());
       resultChunks = await chunksQuery;
       break;
     case "cosineDistance":
       chunksQuery.limit(limit * 2);
-      chunksQuery.whereNotNull(`chunks.embedding`);
+      chunksQuery.whereNotNull(`chunks.embedding`).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`);
       console.log("[EXULU] Chunks query:", chunksQuery.toQuery());
       chunksQuery.select(
         db3.raw(`1 - (chunks.embedding <=> ${vectorExpr}) AS cosine_distance`)
@@ -3213,11 +3242,12 @@ var vectorSearch = async ({
       chunksQuery.orderByRaw(
         `chunks.embedding <=> ${vectorExpr} ASC NULLS LAST`
       );
+      chunksQuery.whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr}) >= ?)`, [cutoffs?.cosineDistance || 0]);
       resultChunks = await chunksQuery;
       break;
     case "hybridSearch":
-      const matchCount = Math.min(limit * 2, 100);
-      const fullTextWeight = 1;
+      const matchCount = Math.min(limit * 2);
+      const fullTextWeight = 2;
       const semanticWeight = 1;
       const rrfK = 50;
       const hybridSQL = `
@@ -3226,12 +3256,15 @@ var vectorSearch = async ({
                 chunks.id,
                 chunks.source,
                 row_number() OVER (
-                  ORDER BY ts_rank(chunks.fts, websearch_to_tsquery(?, ?)) DESC
+                  ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC
                 ) AS rank_ix
               FROM ${chunksTable} as chunks
-              WHERE chunks.fts @@ websearch_to_tsquery(?, ?)
+              LEFT JOIN ${mainTable} as items ON items.id = chunks.source
+              WHERE chunks.fts @@ plainto_tsquery(?, ?)
+                AND ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?
+                AND (items.archived IS FALSE OR items.archived IS NULL)
               ORDER BY rank_ix
-              LIMIT LEAST(?, 15) * 2
+              LIMIT LEAST(?, 250) * 2
             ),
             semantic AS (
               SELECT
@@ -3241,9 +3274,12 @@ var vectorSearch = async ({
                   ORDER BY chunks.embedding <=> ${vectorExpr} ASC
                 ) AS rank_ix
               FROM ${chunksTable} as chunks
+              LEFT JOIN ${mainTable} as items ON items.id = chunks.source
               WHERE chunks.embedding IS NOT NULL
+                AND (1 - (chunks.embedding <=> ${vectorExpr})) >= ?
+                AND (items.archived IS FALSE OR items.archived IS NULL)
               ORDER BY rank_ix
-              LIMIT LEAST(?, 50) * 2
+              LIMIT LEAST(?, 250) * 2
             )
             SELECT
               items.id as item_id,
@@ -3259,7 +3295,7 @@ var vectorSearch = async ({
               items."updatedAt" as item_updated_at,
               items."createdAt" as item_created_at,
               /* Per-signal scores for introspection */
-              ts_rank(chunks.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
+              ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank,
               (1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
         
               /* Hybrid RRF score */
@@ -3276,18 +3312,31 @@ var vectorSearch = async ({
               ON COALESCE(ft.id, se.id) = chunks.id
             JOIN ${mainTable} as items
               ON items.id = chunks.source
+            WHERE (
+              COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
+              +
+              COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
+            ) >= ?
+            AND (chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)
+            AND (chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)
             ORDER BY hybrid_score DESC
-            LIMIT LEAST(?, 50)
+            LIMIT LEAST(?, 250)
             OFFSET 0
           `;
       const bindings = [
-        // full_text: websearch_to_tsquery(lang, query) in rank and where
+        // full_text: plainto_tsquery(lang, query) in rank and where
         language,
         query,
         language,
         query,
+        language,
+        query,
+        cutoffs?.tsvector || 0,
+        // full_text tsvector cutoff
         matchCount,
         // full_text limit
+        cutoffs?.cosineDistance || 0,
+        // semantic cosine distance cutoff
         matchCount,
         // semantic limit
         // fts_rank (ts_rank) call
@@ -3298,13 +3347,26 @@ var vectorSearch = async ({
         fullTextWeight,
         rrfK,
         semanticWeight,
+        // WHERE clause hybrid_score filter
+        rrfK,
+        fullTextWeight,
+        rrfK,
+        semanticWeight,
+        cutoffs?.hybrid || 0,
+        // Additional cutoff filters in main WHERE clause
+        language,
+        query,
+        cutoffs?.tsvector || 0,
+        // tsvector cutoff for results from semantic CTE
+        cutoffs?.cosineDistance || 0,
+        // cosine distance cutoff for results from full_text CTE
         matchCount
         // final limit
       ];
       resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
   }
   console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
-  resultChunks = resultChunks.map((chunk) => ({
+  let results = resultChunks.map((chunk) => ({
     chunk_content: chunk.content,
     chunk_index: chunk.chunk_index,
     chunk_id: chunk.chunk_id,
@@ -3321,37 +3383,135 @@ var vectorSearch = async ({
       name: table.name.singular,
       id: table.id || ""
     },
-    ...method === "cosineDistance" && { chunk_cosine_distance: chunk.cosine_distance },
+    ...(method === "cosineDistance" || method === "hybridSearch") && { chunk_cosine_distance: chunk.cosine_distance },
     ...(method === "tsvector" || method === "hybridSearch") && { chunk_fts_rank: chunk.fts_rank },
-    ...method === "hybridSearch" && { chunk_hybrid_score: chunk.hybrid_score }
+    ...method === "hybridSearch" && { chunk_hybrid_score: chunk.hybrid_score * 1e4 / 100 }
   }));
-  if (resultChunks.length > 0 && (method === "cosineDistance" || method === "hybridSearch")) {
+  if (results.length > 0 && (method === "cosineDistance" || method === "hybridSearch")) {
     const scoreKey = method === "cosineDistance" ? "chunk_cosine_distance" : "chunk_hybrid_score";
-    const topScore = resultChunks[0][scoreKey];
-    const bottomScore = resultChunks[resultChunks.length - 1][scoreKey];
-    const medianScore = resultChunks[Math.floor(resultChunks.length / 2)][scoreKey];
+    const topScore = results[0]?.[scoreKey];
+    const bottomScore = results[results.length - 1]?.[scoreKey];
+    const medianScore = results[Math.floor(results.length / 2)]?.[scoreKey];
     console.log("[EXULU] Score distribution:", {
       method,
-      count: resultChunks.length,
+      count: results.length,
       topScore: topScore?.toFixed(4),
       bottomScore: bottomScore?.toFixed(4),
       medianScore: medianScore?.toFixed(4)
     });
-    const adaptiveThreshold = topScore * 0.7;
-    const beforeFilterCount = resultChunks.length;
-    resultChunks = resultChunks.filter((chunk) => {
+    const adaptiveThreshold = topScore ? topScore * 0.6 : 0;
+    const beforeFilterCount = results.length;
+    results = results.filter((chunk) => {
       const score = chunk[scoreKey];
       return score !== void 0 && score >= adaptiveThreshold;
     });
-    const filteredCount = beforeFilterCount - resultChunks.length;
+    const filteredCount = beforeFilterCount - results.length;
     if (filteredCount > 0) {
       console.log(`[EXULU] Filtered ${filteredCount} low-quality results (threshold: ${adaptiveThreshold.toFixed(4)})`);
     }
   }
   if (resultReranker && query) {
-    resultChunks = await resultReranker(resultChunks);
   }
-  resultChunks = resultChunks.slice(0, limit);
+  results = results.slice(0, limit);
+  if (expand?.before || expand?.after) {
+    const expandedMap = /* @__PURE__ */ new Map();
+    for (const chunk of results) {
+      expandedMap.set(`${chunk.item_id}-${chunk.chunk_index}`, chunk);
+    }
+    if (expand?.before) {
+      for (const chunk of results) {
+        const indicesToFetch = Array.from(
+          { length: expand.before },
+          (_, i) => chunk.chunk_index - expand.before + i
+        ).filter((index) => index >= 0);
+        console.log("[EXULU] Indices to fetch:", indicesToFetch);
+        await Promise.all(indicesToFetch.map(async (index) => {
+          if (expandedMap.has(`${chunk.item_id}-${index}`)) {
+            return;
+          }
+          const expandedChunk = await db3(chunksTable).where({
+            source: chunk.item_id,
+            chunk_index: index
+          }).first();
+          if (expandedChunk) {
+            if (expandedChunk) {
+              expandedMap.set(`${chunk.item_id}-${index}`, {
+                chunk_content: expandedChunk.content,
+                chunk_index: expandedChunk.chunk_index,
+                chunk_id: expandedChunk.id,
+                chunk_source: expandedChunk.source,
+                chunk_metadata: expandedChunk.metadata,
+                chunk_created_at: expandedChunk.createdAt,
+                chunk_updated_at: expandedChunk.updatedAt,
+                item_updated_at: chunk.item_updated_at,
+                item_created_at: chunk.item_created_at,
+                item_id: chunk.item_id,
+                item_external_id: chunk.item_external_id,
+                item_name: chunk.item_name,
+                chunk_cosine_distance: 0,
+                chunk_fts_rank: 0,
+                chunk_hybrid_score: 0,
+                context: {
+                  name: table.name.singular,
+                  id: table.id || ""
+                }
+              });
+            }
+          }
+        }));
+      }
+    }
+    if (expand?.after) {
+      for (const chunk of results) {
+        const indicesToFetch = Array.from(
+          { length: expand.after },
+          (_, i) => chunk.chunk_index + i + 1
+        );
+        console.log("[EXULU] Indices to fetch:", indicesToFetch);
+        await Promise.all(indicesToFetch.map(async (index) => {
+          if (expandedMap.has(`${chunk.item_id}-${index}`)) {
+            return;
+          }
+          const expandedChunk = await db3(chunksTable).where({
+            source: chunk.item_id,
+            chunk_index: index
+          }).first();
+          if (expandedChunk) {
+            expandedMap.set(`${chunk.item_id}-${index}`, {
+              chunk_content: expandedChunk.content,
+              chunk_index: expandedChunk.chunk_index,
+              chunk_id: expandedChunk.id,
+              chunk_source: expandedChunk.source,
+              chunk_metadata: expandedChunk.metadata,
+              chunk_created_at: expandedChunk.createdAt,
+              chunk_updated_at: expandedChunk.updatedAt,
+              item_updated_at: chunk.item_updated_at,
+              item_created_at: chunk.item_created_at,
+              item_id: chunk.item_id,
+              item_external_id: chunk.item_external_id,
+              item_name: chunk.item_name,
+              chunk_cosine_distance: 0,
+              chunk_fts_rank: 0,
+              chunk_hybrid_score: 0,
+              context: {
+                name: table.name.singular,
+                id: table.id || ""
+              }
+            });
+          }
+        }));
+      }
+    }
+    results = Array.from(expandedMap.values());
+    results = results.sort((a, b) => {
+      if (a.item_id !== b.item_id) {
+        return a.item_id.localeCompare(b.item_id);
+      }
+      const aIndex = Number(a.chunk_index);
+      const bIndex = Number(b.chunk_index);
+      return aIndex - bIndex;
+    });
+  }
   await updateStatistic({
     name: "count",
     label: table.name.singular,
@@ -3369,7 +3529,7 @@ var vectorSearch = async ({
       id: table.id || "",
       embedder: embedder.name
     },
-    chunks: resultChunks
+    chunks: results
   };
 };
 var RBACResolver = async (db3, entityName, resourceId, rights_mode) => {
@@ -3542,7 +3702,7 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     `;
     if (table.type === "items") {
       typeDefs += `
-      ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}VectorSearchResult
+      ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
     `;
     }
     mutationDefs += `
@@ -3595,6 +3755,17 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
         tsvector
     }
 
+    input SearchCutoffs {
+        cosineDistance: Float
+        hybrid: Float
+        tsvector: Float
+    }
+
+    input SearchExpand {
+        before: Int
+        after: Int
+    }
+
     type ${tableNameSingular}VectorSearchResult {
         chunks: [${tableNameSingular}VectorSearchChunk!]!
         context: VectoSearchResultContext!
@@ -4486,6 +4657,18 @@ var getPresignedUrl = async (bucket, key, config) => {
   );
   return url;
 };
+function sanitizeMetadata(metadata) {
+  if (!metadata) return void 0;
+  const sanitized = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    if (typeof value === "string") {
+      sanitized[key] = encodeURIComponent(value);
+    } else {
+      sanitized[key] = String(value);
+    }
+  }
+  return sanitized;
+}
 var addGeneralPrefixToKey = (keyPath, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
@@ -4521,19 +4704,41 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
   const client2 = getS3Client(config);
   let defaultBucket = config.fileUploads.s3Bucket;
   let key = fileName;
-  key = addGeneralPrefixToKey(key, config);
   key = addUserPrefixToKey(key, user || "api");
-  console.log("[EXULU] uploading file to s3 into bucket", defaultBucket, "with key", key);
+  key = addGeneralPrefixToKey(key, config);
+  const sanitizedMetadata = sanitizeMetadata(options.metadata);
   const command = new PutObjectCommand({
     Bucket: customBucket || defaultBucket,
     Key: key,
     Body: file,
     ContentType: options.contentType,
-    Metadata: options.metadata,
+    Metadata: sanitizedMetadata,
     ContentLength: file.byteLength
   });
-  await client2.send(command);
-  console.log("[EXULU] file uploaded to s3 into bucket", customBucket || defaultBucket, "with key", key);
+  const maxRetries = 3;
+  let lastError = null;
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      await client2.send(command);
+      break;
+    } catch (error) {
+      lastError = error;
+      if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
+        if (attempt < maxRetries) {
+          const backoffMs = Math.pow(2, attempt) * 1e3;
+          await new Promise((resolve) => setTimeout(resolve, backoffMs));
+          s3Client = void 0;
+          getS3Client(config);
+          continue;
+        }
+      } else {
+        throw error;
+      }
+    }
+  }
+  if (lastError) {
+    throw lastError;
+  }
   return addBucketPrefixToKey(
     key,
     customBucket || defaultBucket
@@ -4605,6 +4810,8 @@ var createUppyRoutes = async (app, contexts, config) => {
       res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
       return;
     }
+    key = key.replace(`${bucket}/`, "");
+    console.log("[EXULU] deleting file from s3 into bucket", bucket, "with key", key);
     const client2 = getS3Client(config);
     const command = new DeleteObjectCommand({
       Bucket: bucket,
@@ -4728,7 +4935,7 @@ var createUppyRoutes = async (app, contexts, config) => {
     const client2 = getS3Client(config);
     const command = new ListObjectsV2Command({
       Bucket: config.fileUploads.s3Bucket,
-      Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}${authenticationResult.user.id}`,
+      Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}user_${authenticationResult.user.id}`,
       MaxKeys: 9,
       ...req.query.continuationToken && { ContinuationToken: req.query.continuationToken }
     });
@@ -4740,7 +4947,17 @@ var createUppyRoutes = async (app, contexts, config) => {
         search.toLowerCase()
       ));
     }
-    res.json(response);
+    res.json({
+      ...response,
+      Contents: response.Contents?.map((content) => {
+        return {
+          ...content,
+          // For consistency and to support multi-bucket environments
+          // we prepend the bucket name to the key here.
+          Key: `${config.fileUploads?.s3Bucket}/${content.Key}`
+        };
+      })
+    });
     res.end();
   });
   app.get("/s3/sts", (req, res, next) => {
@@ -4801,8 +5018,9 @@ var createUppyRoutes = async (app, contexts, config) => {
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
     const key = generateS3Key2(filename);
-    let fullKey = addGeneralPrefixToKey(key, config);
-    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
+    let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    fullKey = addGeneralPrefixToKey(fullKey, config);
+    console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
     getSignedUrl(
       getS3Client(config),
       new PutObjectCommand({
@@ -4856,8 +5074,9 @@ var createUppyRoutes = async (app, contexts, config) => {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
     const key = `${randomUUID()}-_EXULU_${filename}`;
-    let fullKey = addGeneralPrefixToKey(key, config);
-    fullKey = addUserPrefixToKey(fullKey, user.type === "api" ? "api" : user.id);
+    let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    fullKey = addGeneralPrefixToKey(fullKey, config);
+    console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
     const params = {
       Bucket: config.fileUploads.s3Bucket,
       Key: fullKey,
@@ -5108,7 +5327,10 @@ var createProjectRetrievalTool = async ({
 };
 var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID, req, project) => {
   if (!currentTools) return {};
-  if (!allExuluTools) return {};
+  if (!allExuluTools) {
+    allExuluTools = [];
+  }
+  ;
   if (!contexts) {
     contexts = [];
   }
@@ -5142,6 +5364,7 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
             ...cur.tool,
             description,
             async *execute(inputs, options) {
+              console.log("[EXULU] Executing tool", cur.name, "with inputs", inputs, "and options", options);
               if (!cur.tool?.execute) {
                 console.error("[EXULU] Tool execute function is undefined.", cur.tool);
                 throw new Error("Tool execute function is undefined.");
@@ -5993,6 +6216,68 @@ var ExuluTool2 = class {
       execute: execute2
     });
   }
+  execute = async ({
+    agent,
+    config,
+    user,
+    inputs,
+    project
+  }) => {
+    const agentInstance = await loadAgent(agent);
+    if (!agentInstance) {
+      throw new Error("Agent not found.");
+    }
+    const { db: db3 } = await postgresClient();
+    let providerapikey;
+    const variableName = agentInstance.providerapikey;
+    if (variableName) {
+      console.log("[EXULU] provider api key variable name", variableName);
+      const variable = await db3.from("variables").where({ name: variableName }).first();
+      if (!variable) {
+        throw new Error("Provider API key variable not found for " + agentInstance.name + " (" + agentInstance.id + ").");
+      }
+      providerapikey = variable.value;
+      if (!variable.encrypted) {
+        throw new Error("Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys.");
+      }
+      if (variable.encrypted) {
+        const bytes = CryptoJS2.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+        providerapikey = bytes.toString(CryptoJS2.enc.Utf8);
+      }
+    }
+    const tools = await convertToolsArrayToObject(
+      [this],
+      [],
+      agentInstance.tools,
+      providerapikey,
+      void 0,
+      user,
+      config,
+      void 0,
+      void 0,
+      project
+    );
+    const tool2 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
+    if (!tool2?.execute) {
+      throw new Error("Tool " + sanitizeName(this.name) + " not found in " + JSON.stringify(tools));
+    }
+    console.log("[EXULU] Tool found", this.name);
+    const generator = tool2.execute(inputs, {
+      toolCallId: this.id + "_" + randomUUID2(),
+      messages: []
+    });
+    let lastValue;
+    for await (const chunk of generator) {
+      lastValue = chunk;
+    }
+    if (typeof lastValue === "string") {
+      lastValue = JSON.parse(lastValue);
+    }
+    if (lastValue?.result && typeof lastValue.result === "string") {
+      lastValue.result = JSON.parse(lastValue.result);
+    }
+    return lastValue;
+  };
 };
 var getTableName = (id) => {
   return sanitizeName(id) + "_items";
@@ -6073,7 +6358,16 @@ var ExuluContext = class {
       calculateVectors: "manual",
       language: "english",
       defaultRightsMode: "private",
-      maxRetrievalResults: 10
+      maxRetrievalResults: 10,
+      expand: {
+        before: 0,
+        after: 0
+      },
+      cutoffs: {
+        cosineDistance: 0.5,
+        tsvector: 0.5,
+        hybrid: 0.5
+      }
     };
     this.description = description;
     this.embedder = embedder;
@@ -6088,6 +6382,23 @@ var ExuluContext = class {
     if (!this.processor) {
       throw new Error(`Processor is not set for this context: ${this.id}.`);
     }
+    if (this.processor.filter) {
+      const result = await this.processor.filter({
+        item,
+        user,
+        role,
+        utils: {
+          storage: exuluStorage
+        },
+        exuluConfig
+      });
+      if (!result) {
+        return {
+          result: void 0,
+          job: void 0
+        };
+      }
+    }
     const queue = await this.processor.config?.queue;
     if (queue?.queue.name) {
       console.log("[EXULU] processor is in queue mode, scheduling job.");
@@ -6147,7 +6458,9 @@ var ExuluContext = class {
       role: options.role,
       context: this,
       db: db3,
-      limit: options?.limit || this.configuration.maxRetrievalResults || 10
+      limit: options?.limit || this.configuration.maxRetrievalResults || 10,
+      cutoffs: options.cutoffs,
+      expand: options.expand
     });
     return result;
   };