@exulu/backend 1.36.1 → 1.38.0

package/dist/index.js

@@ -411,7 +411,9 @@ var getToken = async (authHeader) => {
   }
   try {
     const secret = process.env.NEXTAUTH_SECRET;
-    const jwk = await importJWK({ k: secret, alg: "HS256", kty: "oct" });
+    const secretBuffer = Buffer.from(secret, "utf-8");
+    const base64Secret = secretBuffer.toString("base64url");
+    const jwk = await importJWK({ k: base64Secret, alg: "HS256", kty: "oct" });
     const { payload } = await jwtVerify(token, jwk);
     return payload;
   } catch (error) {
@@ -687,25 +689,11 @@ var requestValidators = {
         message: "Missing body."
       };
     }
-    if (!req.headers["user"]) {
+    if (!req.body.message && !req.body.messages) {
       return {
         error: true,
         code: 400,
-        message: 'Missing "user" property in headers.'
-      };
-    }
-    if (!req.headers["session"]) {
-      return {
-        error: true,
-        code: 400,
-        message: 'Missing "session" property in headers.'
-      };
-    }
-    if (!req.body.message) {
-      return {
-        error: true,
-        code: 400,
-        message: 'Missing "message" property in body.'
+        message: 'Missing "message" or "messages" property in body.'
       };
     }
     return {
@@ -737,6 +725,12 @@ var agentMessagesSchema = {
       name: "user",
       type: "number"
     },
+    {
+      name: "message_id",
+      type: "text",
+      index: true,
+      unique: true
+    },
     {
       name: "session",
       type: "text"
@@ -2371,9 +2365,11 @@ function createMutations(table, agents, contexts, tools, config) {
         const name = args.field?.replace("_s3key", "");
         console.log("[EXULU] name", name);
         console.log("[EXULU] fields", exists.fields.map((field2) => field2.name));
-        const field = exists.fields.find((field2) => field2.name === name);
+        const field = exists.fields.find((field2) => {
+          return field2.name.replace("_s3key", "") === name;
+        });
         if (!field) {
-          throw new Error(`Field ${name} not found in context ${exists.id}.`);
+          throw new Error(`Field ${name} not found in context ${exists.id}].`);
         }
         if (!field.processor) {
           throw new Error(`Processor not set for field ${args.field} in context ${exists.id}.`);
@@ -2448,7 +2444,8 @@ function createMutations(table, agents, contexts, tools, config) {
           item,
           config,
           context.user.id,
-          context.user.role?.id
+          context.user.role?.id,
+          item.external_id || item.id ? true : false
         );
         if (job) {
           jobs.push(job);
@@ -2535,22 +2532,30 @@ function createMutations(table, agents, contexts, tools, config) {
       if (!id) {
         throw new Error(`Context ${table.id} not found.`);
       }
-      let query = db3.from(getTableName(id)).select("id");
       if (args.where) {
+        let query = db3.from(getTableName(id)).select("id");
         query = applyFilters(query, args.where, table);
+        const items = await query;
+        if (items.length === 0) {
+          throw new Error("No items found to delete chunks for.");
+        }
+        for (const item of items) {
+          await db3.from(getChunksTableName(id)).where({ source: item.id }).delete();
+        }
+        return {
+          message: "Chunks deleted successfully.",
+          items: items.length,
+          jobs: []
+        };
+      } else {
+        const count = await db3.from(getChunksTableName(id)).count();
+        await db3.from(getChunksTableName(id)).truncate();
+        return {
+          message: "Chunks deleted successfully.",
+          items: parseInt(count[0].count),
+          jobs: []
+        };
       }
-      const items = await query;
-      if (items.length === 0) {
-        throw new Error("No items found to delete chunks for.");
-      }
-      for (const item of items) {
-        await db3.from(getChunksTableName(id)).where({ source: item.id }).delete();
-      }
-      return {
-        message: "Chunks deleted successfully.",
-        items: items.length,
-        jobs: []
-      };
     };
   }
   return mutations;
@@ -2825,6 +2830,13 @@ var postprocessDeletion = async ({
       }
       return result;
     }
+    if (table.type === "agent_sessions") {
+      if (!result.id) {
+        return result;
+      }
+      const { db: db3 } = await postgresClient();
+      await db3.from("agent_messages").where({ session: result.id }).where({ session: result.id }).delete();
+    }
   }
   return result;
 };
@@ -3148,7 +3160,7 @@ var vectorSearch = async ({
       items = await itemsQuery;
       break;
     case "hybridSearch":
-      const matchCount = Math.min(limit * 5, 30);
+      const matchCount = Math.min(limit * 5, 100);
       const fullTextWeight = 1;
       const semanticWeight = 1;
       const rrfK = 50;
@@ -3175,7 +3187,7 @@ var vectorSearch = async ({
               FROM ${chunksTable} c
               WHERE c.embedding IS NOT NULL
               ORDER BY rank_ix
-              LIMIT LEAST(?, 15) * 2
+              LIMIT LEAST(?, 50) * 2
             )
             SELECT
               m.*,
@@ -3183,6 +3195,7 @@ var vectorSearch = async ({
               c.source,
               c.content,
               c.chunk_index,
+              c.metadata,
               c."createdAt" AS chunk_created_at,
               c."updatedAt" AS chunk_updated_at,
               vector_dims(c.embedding) as embedding_size,
@@ -3206,7 +3219,7 @@ var vectorSearch = async ({
             JOIN ${mainTable} m
               ON m.id = c.source
             ORDER BY hybrid_score DESC
-            LIMIT LEAST(?, 10)
+            LIMIT LEAST(?, 50)
             OFFSET 0
           `;
       const bindings = [
@@ -3232,6 +3245,7 @@ var vectorSearch = async ({
       ];
       items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
   }
+  console.log("[EXULU] Vector search results:", items?.length);
   const seenSources = /* @__PURE__ */ new Map();
   items = items.reduce((acc, item) => {
     if (!seenSources.has(item.source)) {
@@ -3249,6 +3263,7 @@ var vectorSearch = async ({
           chunk_index: item.chunk_index,
           chunk_id: item.chunk_id,
           source: item.source,
+          metadata: item.metadata,
           chunk_created_at: item.chunk_created_at,
           chunk_updated_at: item.chunk_updated_at,
           embedding_size: item.embedding_size,
@@ -3265,6 +3280,7 @@ var vectorSearch = async ({
         chunk_id: item.chunk_id,
         chunk_created_at: item.chunk_created_at,
         embedding_size: item.embedding_size,
+        metadata: item.metadata,
         source: item.source,
         chunk_updated_at: item.chunk_updated_at,
         ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
@@ -3274,6 +3290,7 @@ var vectorSearch = async ({
     }
     return acc;
   }, []);
+  console.log("[EXULU] Vector search results after deduplication:", items?.length);
   items.forEach((item) => {
     if (!item.chunks?.length) {
       return;
@@ -3305,7 +3322,7 @@ var vectorSearch = async ({
   if (resultReranker && query) {
     items = await resultReranker(items);
   }
-  items = items.slice(0, limit);
+  console.log("[EXULU] Vector search results after slicing:", items?.length);
   await updateStatistic({
     name: "count",
     label: table.name.singular,
@@ -3967,6 +3984,10 @@ type PageInfo {
         }
       };
     }));
+    let embedderQueue = void 0;
+    if (data.embedder?.queue) {
+      embedderQueue = await data.embedder.queue;
+    }
     const clean = {
       id: data.id,
       name: data.name,
@@ -3974,7 +3995,8 @@ type PageInfo {
       embedder: data.embedder ? {
         name: data.embedder.name,
         id: data.embedder.id,
-        config: data.embedder?.config || void 0
+        config: data.embedder?.config || void 0,
+        queue: embedderQueue?.queue.name || void 0
       } : void 0,
       slug: "/contexts/" + data.id,
       active: data.active,
@@ -4161,6 +4183,7 @@ type ItemChunks {
     chunk_created_at: Date
     chunk_updated_at: Date
     embedding_size: Float
+    metadata: JSON
 }
 
 type Provider {
@@ -4201,6 +4224,7 @@ type Embedder {
     name: String!
     id: ID!
     config: [EmbedderConfig!]
+    queue: String
 }
 type EmbedderConfig {
     name: String!
@@ -4392,10 +4416,21 @@ var getPresignedUrl = async (key, config) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured");
   }
+  let bucket = config.fileUploads.s3Bucket;
+  if (key.includes("[bucket:")) {
+    console.log("[EXULU] key includes [bucket:name]", key);
+    bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
+    if (!bucket?.length) {
+      throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
+    }
+    key = key.split("]")[1] || "";
+    console.log("[EXULU] bucket", bucket);
+    console.log("[EXULU] key", key);
+  }
   const url = await getSignedUrl(
     getS3Client(config),
     new GetObjectCommand({
-      Bucket: config.fileUploads.s3Bucket,
+      Bucket: bucket,
       Key: key
     }),
     { expiresIn }
@@ -4432,7 +4467,7 @@ var uploadFile = async (user, file, key, config, options = {}) => {
     ContentLength: file.byteLength
   });
   await client2.send(command);
-  return key;
+  return fullKey;
 };
 var createUppyRoutes = async (app, config) => {
   if (!config.fileUploads) {
@@ -4503,29 +4538,24 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
-    const { key } = req.query;
+    let { key } = req.query;
     if (typeof key !== "string" || key.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    const userPrefix = extractUserPrefix(key);
-    console.log("userPrefix", userPrefix);
-    console.log("authenticationResult.user.id", authenticationResult.user.id);
-    if (!userPrefix) {
-      res.status(405).json({ error: 'Invalid key, does not contain a user prefix like "<user_id>/<key>.' });
-      return;
-    }
-    if (userPrefix !== authenticationResult.user.id.toString()) {
-      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
-      return;
-    }
-    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id.toString())) {
-      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
-      return;
+    let bucket = config.fileUploads.s3Bucket;
+    if (key.includes("[bucket:")) {
+      console.log("[EXULU] key includes [bucket:name]", key);
+      bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
+      if (!bucket?.length) {
+        throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
+      }
+      key = key.split("]")[1] || "";
+      console.log("[EXULU] bucket", bucket);
     }
     const client2 = getS3Client(config);
     const command = new DeleteObjectCommand({
-      Bucket: config.fileUploads.s3Bucket,
+      Bucket: bucket,
       Key: key
     });
     await client2.send(command);
@@ -4554,19 +4584,6 @@ var createUppyRoutes = async (app, config) => {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
     }
-    const userPrefix = extractUserPrefix(key);
-    if (!userPrefix) {
-      res.status(405).json({ error: 'Invalid key, does not contain a user prefix like "<user_id>/<key>.' });
-      return;
-    }
-    if (userPrefix !== authenticationResult.user.id.toString()) {
-      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
-      return;
-    }
-    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id.toString())) {
-      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
-      return;
-    }
     try {
       const url = await getPresignedUrl(key, config);
       res.setHeader("Access-Control-Allow-Origin", "*");
@@ -4596,11 +4613,24 @@ var createUppyRoutes = async (app, config) => {
       res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
-    const { key } = req.body;
+    let { key } = req.body;
+    let bucket = config.fileUploads.s3Bucket;
+    if (key.includes("[bucket:")) {
+      console.log("[EXULU] key includes [bucket:name]", key);
+      bucket = key.split("[bucket:")[1]?.split("]")[0] || "";
+      console.log("[EXULU] bucket", bucket);
+      if (!bucket?.length) {
+        throw new Error("Invalid key, does not contain a bucket name like '[bucket:name]'.");
+      }
+      key = key.split("]")[1] || "";
+      console.log("[EXULU] key", key);
+    }
     console.log("[EXULU] Getting object metadata from s3", key);
+    console.log("[EXULU] bucket", bucket);
+    console.log("[EXULU] key", key);
     const client2 = getS3Client(config);
     const command = new HeadObjectCommand({
-      Bucket: config.fileUploads.s3Bucket,
+      Bucket: bucket,
       Key: key
     });
     const response = await client2.send(command);
@@ -4908,6 +4938,7 @@ var createUppyRoutes = async (app, config) => {
 };
 
 // src/registry/classes.ts
+import { parseOfficeAsync } from "officeparser";
 var s3Client2;
 function sanitizeToolName(name) {
   if (typeof name !== "string") return "";
@@ -4918,7 +4949,7 @@ function sanitizeToolName(name) {
   }
   return sanitized;
 }
-var convertToolsArrayToObject = (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID) => {
+var convertToolsArrayToObject = (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID, req) => {
   if (!currentTools) return {};
   if (!allExuluTools) return {};
   if (!contexts) {
@@ -5005,6 +5036,7 @@ var convertToolsArrayToObject = (currentTools, allExuluTools, configs, providera
               const response = await cur.tool.execute({
                 ...inputs,
                 sessionID,
+                req,
                 // Convert config to object format if a config object 
                 // is available, after we added the .value property
                 // by hydrating it from the variables table.
@@ -5014,6 +5046,7 @@ var convertToolsArrayToObject = (currentTools, allExuluTools, configs, providera
                 user,
                 contexts: contextsMap,
                 upload,
+                exuluConfig,
                 config: config ? config.config.reduce((acc, curr) => {
                   acc[curr.name] = curr.value;
                   return acc;
@@ -5235,6 +5268,7 @@ var ExuluAgent2 = class {
   };
   generateSync = async ({
     prompt,
+    req,
     user,
     session,
     inputMessages,
@@ -5316,7 +5350,8 @@ var ExuluAgent2 = class {
             contexts,
             user,
             exuluConfig,
-            session
+            session,
+            req
           ),
           stopWhen: [stepCountIs(2)]
         });
@@ -5375,7 +5410,8 @@ var ExuluAgent2 = class {
           contexts,
           user,
           exuluConfig,
-          session
+          session,
+          req
         ),
         stopWhen: [stepCountIs(2)]
       });
@@ -5416,6 +5452,71 @@ var ExuluAgent2 = class {
     }
     return "";
   };
+  /**
+   * Convert file parts in messages to OpenAI Responses API compatible format.
+   * The OpenAI Responses API doesn't support inline file parts with type 'file'.
+   * This function converts:
+   * - Document files (PDF, DOCX, etc.) -> text parts with extracted content using officeparser
+   * - Image files -> image parts (which ARE supported by Responses API)
+   */
+  async processFilePartsInMessages(messages) {
+    const processedMessages = await Promise.all(messages.map(async (message) => {
+      console.log("[EXULU] Processing file parts in messages: " + JSON.stringify(message, null, 2));
+      if (message.role !== "user" || !Array.isArray(message.parts)) {
+        return message;
+      }
+      const processedParts = await Promise.all(message.parts.map(async (part) => {
+        if (part.type !== "file") {
+          return part;
+        }
+        const { mediaType, url, filename } = part;
+        const imageTypes = ["image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp"];
+        if (imageTypes.includes(mediaType)) {
+          console.log(`[EXULU] Converting file part to image part: ${filename}`);
+          return {
+            type: "image",
+            image: url,
+            mimeType: mediaType
+          };
+        }
+        console.log(`[EXULU] Converting file part to text using officeparser: ${filename}`);
+        try {
+          const response = await fetch(url);
+          if (!response.ok) {
+            console.error(`[EXULU] Failed to fetch file: ${filename}, status: ${response.status}`);
+            return {
+              type: "text",
+              text: `[Error: Could not load file ${filename}]`
+            };
+          }
+          const arrayBuffer = await response.arrayBuffer();
+          const extractedText = await parseOfficeAsync(arrayBuffer, {
+            outputErrorToConsole: false,
+            newlineDelimiter: "\n"
+          });
+          return {
+            type: "text",
+            text: `<file name="${filename}">
+${extractedText}
+</file>`
+          };
+        } catch (error) {
+          console.error(`[EXULU] Error processing file ${filename}:`, error);
+          return {
+            type: "text",
+            text: `[Error extracting text from file ${filename}: ${error instanceof Error ? error.message : "Unknown error"}]`
+          };
+        }
+      }));
+      const result = {
+        ...message,
+        parts: processedParts
+      };
+      console.log("[EXULU] Result: " + JSON.stringify(result, null, 2));
+      return result;
+    }));
+    return processedMessages;
+  }
   generateStream = async ({
     user,
     session,
@@ -5427,7 +5528,8 @@ var ExuluAgent2 = class {
     providerapikey,
     contexts,
     exuluConfig,
-    instructions
+    instructions,
+    req
   }) => {
     if (!this.model) {
       console.error("[EXULU] Model is required for streaming.");
@@ -5450,7 +5552,7 @@ var ExuluAgent2 = class {
       console.log("[EXULU] loading previous messages from session: " + session);
       const previousMessages2 = await getAgentMessages({
         session,
-        user: user.id,
+        user: user?.id,
         limit: 50,
         page: 1
       });
@@ -5462,6 +5564,11 @@ var ExuluAgent2 = class {
       // append the new message to the previous messages:
       messages: [...previousMessagesContent, message]
     });
+    messages = messages.filter(
+      (message2, index, self) => index === self.findIndex((t) => t.id === message2.id)
+    );
+    console.log("[EXULU] Processing file parts in messages for OpenAI Responses API compatibility");
+    messages = await this.processFilePartsInMessages(messages);
     const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
     let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
     system += "\n\n" + genericContext;
@@ -5489,7 +5596,8 @@ var ExuluAgent2 = class {
         contexts,
         user,
         exuluConfig,
-        session
+        session,
+        req
       ),
       onError: (error) => {
         console.error("[EXULU] chat stream error.", error);
@@ -5507,7 +5615,7 @@ var ExuluAgent2 = class {
 var getAgentMessages = async ({ session, user, limit, page }) => {
   const { db: db3 } = await postgresClient();
   console.log("[EXULU] getting agent messages for session: " + session + " and user: " + user + " and page: " + page);
-  const query = db3.from("agent_messages").where({ session, user }).limit(limit);
+  const query = db3.from("agent_messages").where({ session, user: user || null }).limit(limit);
   if (page > 0) {
     query.offset((page - 1) * limit);
   }
@@ -5525,15 +5633,17 @@ var getSession = async ({ sessionID }) => {
 };
 var saveChat = async ({ session, user, messages }) => {
   const { db: db3 } = await postgresClient();
-  const promises = messages.map((message) => {
-    return db3.from("agent_messages").insert({
+  for (const message of messages) {
+    const mutation = db3.from("agent_messages").insert({
       session,
       user,
       content: JSON.stringify(message),
+      message_id: message.id,
       title: message.role === "user" ? "User" : "Assistant"
-    });
-  });
-  await Promise.all(promises);
+    }).returning("id");
+    mutation.onConflict("message_id").merge();
+    await mutation;
+  }
 };
 var ExuluEmbedder = class {
   id;
@@ -5574,19 +5684,36 @@ var ExuluEmbedder = class {
     });
     for (const config of this.config || []) {
       const name = config.name;
+      const setting = variables.find((v) => v.name === name);
+      if (!setting) {
+        throw new Error("Setting value not found for embedder setting: " + name + ", for context: " + context + " and embedder: " + this.id + ". Make sure to set the value for this setting in the embedder settings.");
+      }
       const {
         value: variableName,
         id
-      } = variables.find((v) => v.name === name);
+      } = setting;
       let value = "";
       console.log("[EXULU] variable name", variableName);
       const variable = await db3.from("variables").where({ name: variableName }).first();
       if (!variable) {
         throw new Error("Variable not found for embedder setting: " + name + " in context: " + context + " and embedder: " + this.id);
       }
+      console.log("[EXULU] variable", variable);
       if (variable.encrypted) {
-        const bytes = CryptoJS2.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
-        value = bytes.toString(CryptoJS2.enc.Utf8);
+        if (!process.env.NEXTAUTH_SECRET) {
+          throw new Error("NEXTAUTH_SECRET environment variable is not set, cannot decrypt variable: " + name);
+        }
+        try {
+          const bytes = CryptoJS2.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
+          const decrypted = bytes.toString(CryptoJS2.enc.Utf8);
+          if (!decrypted) {
+            throw new Error("Decryption returned empty string - invalid key or corrupted data");
+          }
+          value = decrypted;
+          console.log("[EXULU] successfully decrypted value for", name);
+        } catch (error) {
+          throw new Error(`Failed to decrypt variable "${name}" for embedder setting in context "${context}": ${error instanceof Error ? error.message : "Unknown error"}. Verify that NEXTAUTH_SECRET matches the key used during encryption.`);
+        }
       } else {
         value = variable.value;
       }
@@ -5745,7 +5872,9 @@ var ExuluContext = class {
   processField = async (trigger, user, role, item, config) => {
     console.log("[EXULU] processing field", item.field, " in context", this.id);
     console.log("[EXULU] fields", this.fields.map((field2) => field2.name));
-    const field = this.fields.find((field2) => field2.name === item.field?.replace("_s3key", ""));
+    const field = this.fields.find((field2) => {
+      return field2.name.replace("_s3key", "") === item.field.replace("_s3key", "");
+    });
     if (!field || !field.processor) {
       console.error("[EXULU] field not found or processor not set for field", item.field, " in context", this.id);
       throw new Error("Field not found or processor not set for field " + item.field + " in context " + this.id);
@@ -5755,7 +5884,7 @@ var ExuluContext = class {
     if (queue?.queue.name) {
       console.log("[EXULU] processor is in queue mode, scheduling job.");
       const job = await bullmqDecorator({
-        timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 180,
+        timeoutInSeconds: field.processor?.config?.timeoutInSeconds || 600,
         label: `${this.name} ${field.name} data processor`,
         processor: `${this.id}-${field.name}`,
         context: this.id,
@@ -5846,6 +5975,7 @@ var ExuluContext = class {
     if (chunks?.length) {
       await db3.from(getChunksTableName(this.id)).insert(chunks.map((chunk) => ({
         source,
+        metadata: chunk.metadata,
         content: chunk.content,
         chunk_index: chunk.index,
         embedding: pgvector2.toSql(chunk.vector)
@@ -5886,9 +6016,15 @@ var ExuluContext = class {
     if (!results[0]) {
       throw new Error("Failed to create item.");
     }
+    console.log("[EXULU] context configuration", this.configuration);
     if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
+      console.log("[EXULU] generating embeddings for item", results[0].id);
       const { job } = await this.embeddings.generate.one({
-        item: results[0],
+        item: {
+          ...item,
+          // important we need to full record here with all fields for the embedder
+          id: results[0].id
+        },
         user,
         role,
         trigger: "api",
@@ -5931,7 +6067,7 @@ var ExuluContext = class {
     if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
       const { job } = await this.embeddings.generate.one({
         item: record,
-        // important we need to full record here with all fields
+        // important we need to full record here with all fields for the embedder
         user,
         role,
         trigger: "api",
@@ -5948,10 +6084,13 @@ var ExuluContext = class {
     };
   };
   deleteItem = async (item, user, role) => {
-    if (!item.id) {
-      throw new Error("Item id is required for deleting item.");
-    }
     const { db: db3 } = await postgresClient();
+    if (!item.id?.length && item?.external_id) {
+      item = await db3.from(getTableName(this.id)).where({ external_id: item.external_id }).first();
+      if (!item || !item.id) {
+        throw new Error(`Item not found for external id ${item?.external_id || "undefined"}.`);
+      }
+    }
     await db3.from(getTableName(this.id)).where({ id: item.id }).delete();
     if (!this.embedder) {
       return {
@@ -6125,7 +6264,7 @@ var ExuluContext = class {
         const { db: db3 } = await postgresClient();
         const result = await vectorSearch({
           page: 1,
-          limit: 10,
+          limit: 50,
           query,
           filters: [],
           user,
@@ -6153,7 +6292,6 @@ var ExuluContext = class {
   };
 };
 var updateStatistic = async (statistic) => {
-  console.log("[EXULU] updating statistic", statistic);
   const currentDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
   const { db: db3 } = await postgresClient();
   const existing = await db3.from("tracking").where({
@@ -6278,6 +6416,7 @@ var CLAUDE_MESSAGES = {
 
 // src/registry/routes.ts
 import { createIdGenerator } from "ai";
+import cookieParser from "cookie-parser";
 var REQUEST_SIZE_LIMIT = "50mb";
 var global_queues = {
   eval_runs: "eval_runs"
@@ -6314,6 +6453,7 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
   app.use(cors(corsOptions));
   app.use(bodyParser.urlencoded({ extended: true, limit: REQUEST_SIZE_LIMIT }));
   app.use(bodyParser.json({ limit: REQUEST_SIZE_LIMIT }));
+  app.use(cookieParser());
   console.log(`
     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2557      \u2588\u2588\u2557   \u2588\u2588\u2557
     \u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u255A\u2588\u2588\u2557\u2588\u2588\u2554\u255D\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
@@ -6535,8 +6675,9 @@ Mood: friendly and intelligent.
         res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
         return;
       }
+      console.log("[EXULU] agentInstance.rights_mode", agentInstance.rights_mode);
       const authenticationResult = await requestValidators.authenticate(req);
-      if (!authenticationResult.user?.id) {
+      if (!authenticationResult.user?.id && agentInstance.rights_mode !== "public") {
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
         return;
       }
@@ -6560,7 +6701,7 @@ Mood: friendly and intelligent.
           return;
         }
       }
-      if (user.type !== "api" && !user.super_admin && req.body.resourceId !== user.id) {
+      if (user?.type !== "api" && !user?.super_admin && req.body.resourceId !== user?.id) {
         res.status(400).json({
           message: "The provided user id in the resourceId field is not the same as the authenticated user. Only super admins and API users can impersonate other users."
         });
@@ -6581,7 +6722,6 @@ Mood: friendly and intelligent.
           return;
         }
         providerapikey = variable.value;
-        console.log("[EXULU] encrypted value", providerapikey);
         if (!variable.encrypted) {
           res.status(400).json({
             message: "Provider API key variable not encrypted, for security reasons you are only allowed to use encrypted variables for provider API keys."
@@ -6591,7 +6731,6 @@ Mood: friendly and intelligent.
         if (variable.encrypted) {
           const bytes = CryptoJS3.AES.decrypt(variable.value, process.env.NEXTAUTH_SECRET);
           providerapikey = bytes.toString(CryptoJS3.enc.Utf8);
-          console.log("[EXULU] decrypted value", providerapikey);
         }
       }
       if (!!headers.stream) {
@@ -6599,17 +6738,27 @@ Mood: friendly and intelligent.
           label: agent.name,
           trigger: "agent"
         };
+        let previousMessages = [];
+        let message;
+        if (!req.body.message && !headers.session && req.body.messages) {
+          message = req.body.messages[req.body.messages.length - 1];
+          previousMessages = req.body.messages.slice(0, -1);
+        } else {
+          message = req.body.message;
+        }
         const result = await agent.generateStream({
           contexts,
           user,
           instructions: agentInstance.instructions,
           session: headers.session,
-          message: req.body.message,
+          message,
+          previousMessages,
           currentTools: enabledTools,
           allExuluTools: tools,
           providerapikey,
           toolConfigs: agentInstance.tools,
-          exuluConfig: config
+          exuluConfig: config,
+          req
         });
         result.stream.consumeStream();
         result.stream.pipeUIMessageStreamToResponse(res, {
@@ -6636,11 +6785,12 @@ Mood: friendly and intelligent.
             size: 16
           }),
           onFinish: async ({ messages, isContinuation, isAborted, responseMessage }) => {
-            if (headers.session) {
+            console.log("[EXULU] onFinish", messages?.map((msg) => msg.parts?.map((part) => part.type === "text" ? part.text : null)).join("\n"));
+            if (headers.session && user?.id) {
               await saveChat({
                 session: headers.session,
                 user: user.id,
-                messages: messages.filter((x) => !result.previousMessages.find((y) => y.id === x.id))
+                messages
               });
             }
             const metadata = messages[messages.length - 1]?.metadata;
@@ -6657,7 +6807,7 @@ Mood: friendly and intelligent.
                   type: STATISTICS_TYPE_ENUM.AGENT_RUN,
                   trigger: statistics.trigger,
                   count: 1,
-                  user: user.id,
+                  user: user?.id,
                   role: user?.role?.id
                 }),
                 ...metadata?.inputTokens ? [
@@ -6667,7 +6817,7 @@ Mood: friendly and intelligent.
                     type: STATISTICS_TYPE_ENUM.AGENT_RUN,
                     trigger: statistics.trigger,
                     count: metadata?.inputTokens,
-                    user: user.id,
+                    user: user?.id,
                     role: user?.role?.id
                   })
                 ] : [],
@@ -6688,6 +6838,7 @@ Mood: friendly and intelligent.
       } else {
         const response = await agent.generateSync({
           user,
+          req,
           instructions: agentInstance.instructions,
           session: headers.session,
           inputMessages: [req.body.message],
@@ -7026,7 +7177,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
         }));
         const { db: db3 } = await postgresClient();
         const data = bullmqJob.data;
-        const timeoutInSeconds = data.timeoutInSeconds || 300;
+        const timeoutInSeconds = data.timeoutInSeconds || 600;
         const timeoutMs = timeoutInSeconds * 1e3;
         const timeoutPromise = new Promise((_, reject) => {
           setTimeout(() => {
@@ -7080,7 +7231,9 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               if (!context) {
                 throw new Error(`Context ${data.context} not found in the registry.`);
               }
-              const field = context.fields.find((field2) => field2.name === data.inputs.field);
+              const field = context.fields.find((field2) => {
+                return field2.name.replace("_s3key", "") === data.inputs.field.replace("_s3key", "");
+              });
               if (!field) {
                 throw new Error(`Field ${data.inputs.field} not found in the context ${data.context}.`);
               }
@@ -7355,7 +7508,13 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
               let jobs = [];
               let items = [];
               for (const item of result) {
-                const { item: createdItem, job } = await context.createItem(item, config, data.user, data.role);
+                const { item: createdItem, job } = await context.createItem(
+                  item,
+                  config,
+                  data.user,
+                  data.role,
+                  item.external_id || item.id ? true : false
+                );
                 if (job) {
                   jobs.push(job);
                   console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata2(bullmqJob.name, {
@@ -7409,14 +7568,12 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
       }
     );
     worker.on("completed", async (job, returnvalue) => {
-      console.log(`[EXULU] completed job ${job.id}.`, logMetadata2(job.name, {
-        result: returnvalue
-      }));
+      console.log(`[EXULU] completed job ${job.id}.`, returnvalue);
       const { db: db3 } = await postgresClient();
       await db3.from("job_results").where({ job_id: job.id }).update({
         state: JOB_STATUS_ENUM.completed,
-        result: returnvalue.result,
-        metadata: returnvalue.metadata
+        result: returnvalue.result ? JSON.stringify(returnvalue.result) : null,
+        metadata: returnvalue.metadata ? JSON.stringify(returnvalue.metadata) : null
       });
     });
     worker.on("failed", async (job, error, prev) => {
@@ -8100,7 +8257,6 @@ var claudeSonnet45Agent = new ExuluAgent2({
 
 // src/templates/agents/google/vertex/index.ts
 import { createVertex } from "@ai-sdk/google-vertex";
-import "@ai-sdk/google-vertex";
 var vertexAuthenticationInformation = `
 ### Vertex Authentication Setup (Google Auth)
 
@@ -8155,7 +8311,6 @@ var vertexGemini25FlashAgent = new ExuluAgent2({
     instructions: "",
     model: {
       create: ({ apiKey }) => {
-        console.log("[EXULU] apiKey", apiKey);
         if (!apiKey) {
           throw new Error("Auth credentials not found for Google Vertex agent, make sure you have set the provider api key to a valid google authentication json.");
         }
@@ -8166,8 +8321,45 @@ var vertexGemini25FlashAgent = new ExuluAgent2({
         if (!googleAuthPayload.location) {
           throw new Error("Location not set in authentication json for Google Vertex Gemini 2.5 Flash agent, should be for example 'europe-west1'");
         }
-        const vertex2 = createVertex(googleAuthPayload);
-        const model = vertex2("gemini-2.5-flash");
+        const vertex = createVertex(googleAuthPayload);
+        const model = vertex("gemini-2.5-flash");
+        return model;
+      }
+    }
+  }
+});
+var vertexGemini20FlashAgent = new ExuluAgent2({
+  id: `default_vertex_gemini_2_0_flash_agent`,
+  name: `GEMINI-2.0-FLASH`,
+  provider: "vertex",
+  description: `Google Vertex Gemini 2.0 Flash model. High intelligence and capability. Moderately Fast.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [".png", ".jpg", ".jpeg", ".webp"],
+    files: [".pdf", ".txt"],
+    audio: [".mpeg", ".mp3", ".m4a", ".wav", ".mp4"],
+    video: [".mp4", ".mpeg"]
+  },
+  authenticationInformation: vertexAuthenticationInformation,
+  maxContextLength: 1048576,
+  config: {
+    name: `GEMINI-2.0-FLASH`,
+    instructions: "",
+    model: {
+      create: ({ apiKey }) => {
+        if (!apiKey) {
+          throw new Error("Auth credentials not found for Google Vertex agent, make sure you have set the provider api key to a valid google authentication json.");
+        }
+        const googleAuthPayload = JSON.parse(apiKey || "{}");
+        if (!googleAuthPayload) {
+          throw new Error("API key not found for Google Vertex Gemini 2.0 Flash agent.");
+        }
+        if (!googleAuthPayload.location) {
+          throw new Error("Location not set in authentication json for Google Vertex Gemini 2.0 Flash agent, should be for example 'europe-west1'");
+        }
+        const vertex = createVertex(googleAuthPayload);
+        const model = vertex("gemini-2.0-flash");
         return model;
       }
     }
@@ -8193,7 +8385,6 @@ var vertexGemini3ProAgent = new ExuluAgent2({
     instructions: "",
     model: {
       create: ({ apiKey }) => {
-        console.log("[EXULU] apiKey", apiKey);
         if (!apiKey) {
           throw new Error("Auth credentials not found for Google Vertex agent, make sure you have set the provider api key to a valid google authentication json.");
         }
@@ -8204,8 +8395,8 @@ var vertexGemini3ProAgent = new ExuluAgent2({
         if (!googleAuthPayload.location) {
           throw new Error("Location not set in authentication json for Google Vertex Gemini 3 Pro agent, should be for example 'europe-west1'");
         }
-        const vertex2 = createVertex(googleAuthPayload);
-        const model = vertex2("gemini-3-pro-preview");
+        const vertex = createVertex(googleAuthPayload);
+        const model = vertex("gemini-3-pro-preview");
         return model;
       }
     }
@@ -9295,6 +9486,32 @@ var mathTools = [
   degreesToRadiansTool
 ];
 
+// src/templates/tools/preview-pdf.ts
+import { z as z5 } from "zod";
+var previewPdfTool = new ExuluTool2({
+  id: "preview_pdf",
+  name: "Preview PDF",
+  description: "Used to display a PDF file in an iframe web view",
+  type: "function",
+  config: [],
+  inputSchema: z5.object({
+    s3key: z5.string().describe("The S3 key of the PDF file to preview, can also optionally include a [bucket:name] to specify the bucket."),
+    page: z5.number().describe("The page number to preview, defaults to 1.").optional()
+  }),
+  execute: async ({ s3key, page, exuluConfig }) => {
+    const url = await getPresignedUrl(s3key, exuluConfig);
+    if (!url) {
+      throw new Error("No URL provided for PDF preview");
+    }
+    return {
+      result: JSON.stringify({
+        url,
+        page: page ?? 1
+      })
+    };
+  }
+});
+
 // src/templates/tools/todo/todowrite.txt
 var todowrite_default = `Use this tool to create and manage a structured task list for your current coding session. This helps you track progress, organize complex tasks, and demonstrate thoroughness to the user.
 It also helps the user understand the progress of the task and overall progress of their requests.
@@ -9481,12 +9698,12 @@ Usage:
 - If no todos exist yet, an empty list will be returned`;
 
 // src/templates/tools/todo/todo.ts
-import z5 from "zod";
-var TodoSchema = z5.object({
-  content: z5.string().describe("Brief description of the task"),
-  status: z5.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
-  priority: z5.string().describe("Priority level of the task: high, medium, low"),
-  id: z5.string().describe("Unique identifier for the todo item")
+import z6 from "zod";
+var TodoSchema = z6.object({
+  content: z6.string().describe("Brief description of the task"),
+  status: z6.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
+  priority: z6.string().describe("Priority level of the task: high, medium, low"),
+  id: z6.string().describe("Unique identifier for the todo item")
 });
 var TodoWriteTool = new ExuluTool2({
   id: "todo_write",
@@ -9499,8 +9716,8 @@ var TodoWriteTool = new ExuluTool2({
     description: "The description of the todo list, if set overwrites the default description.",
     default: todowrite_default
   }],
-  inputSchema: z5.object({
-    todos: z5.array(TodoSchema).describe("The updated todo list")
+  inputSchema: z6.object({
+    todos: z6.array(TodoSchema).describe("The updated todo list")
   }),
   execute: async (inputs) => {
     const { sessionID, todos, user } = inputs;
@@ -9531,7 +9748,7 @@ var TodoReadTool = new ExuluTool2({
   id: "todo_read",
   name: "Todo Read",
   description: "Use this tool to read your todo list",
-  inputSchema: z5.object({}),
+  inputSchema: z6.object({}),
   type: "function",
   category: "todo",
   config: [{
@@ -9643,6 +9860,7 @@ var ExuluApp = class {
     this._tools = [
       ...tools ?? [],
       ...mathTools,
+      ...[previewPdfTool],
       ...todoTools,
       // Add contexts as tools
       ...Object.values(contexts || {}).map((context) => context.tool())