@exulu/backend 1.45.0 → 1.46.0

package/dist/index.cjs

@@ -43,7 +43,8 @@ __export(index_exports, {
   ExuluJobs: () => ExuluJobs,
   ExuluOtel: () => ExuluOtel,
   ExuluQueues: () => queues,
-  ExuluTool: () => ExuluTool3,
+  ExuluReranker: () => ExuluReranker4,
+  ExuluTool: () => ExuluTool,
   ExuluUtils: () => ExuluUtils,
   ExuluVariables: () => ExuluVariables,
   db: () => db2,
@@ -122,8 +123,8 @@ var validateJob = (job) => {
 
 // src/registry/classes.ts
 var import_bullmq3 = require("bullmq");
-var import_zod = require("zod");
-var import_ai2 = require("ai");
+var import_zod2 = require("zod");
+var import_ai3 = require("ai");
 
 // types/enums/statistics.ts
 var STATISTICS_TYPE_ENUM = {
@@ -251,16 +252,6 @@ async function postgresClient() {
     db: db["exulu"]
   };
 }
-var refreshPostgresClient = async () => {
-  if (db["exulu"]) {
-    await db["exulu"].destroy();
-    db["exulu"] = void 0;
-  }
-  const { db: refreshed } = await postgresClient();
-  return {
-    db: refreshed
-  };
-};
 
 // src/registry/classes.ts
 var import_knex5 = __toESM(require("pgvector/knex"), 1);
@@ -458,7 +449,7 @@ var import_crypto_js3 = __toESM(require("crypto-js"), 1);
 // src/registry/utils/graphql.ts
 var import_schema = require("@graphql-tools/schema");
 var import_graphql_type_json = __toESM(require("graphql-type-json"), 1);
-var import_graphql2 = require("graphql");
+var import_graphql3 = require("graphql");
 var import_crypto_js2 = __toESM(require("crypto-js"), 1);
 
 // src/auth/get-token.ts
@@ -822,6 +813,11 @@ var agentSessionsSchema = {
       name: "role",
       type: "uuid"
     },
+    {
+      name: "session_items",
+      // array of items as global ids ('<context_id>/<item_id>')
+      type: "json"
+    },
     {
       name: "title",
       type: "text"
@@ -945,10 +941,19 @@ var agentsSchema = {
       name: "description",
       type: "text"
     },
+    {
+      name: "welcomemessage",
+      type: "text"
+    },
     {
       name: "instructions",
       type: "text"
     },
+    {
+      name: "memory",
+      type: "text"
+      // allows selecting a exulu context as native memory for the agent
+    },
     {
       name: "providerapikey",
       type: "text"
@@ -1560,6 +1565,795 @@ var rateLimiter = async (key, windowSeconds, limit, points) => {
   }
 };
 
+// src/registry/agentic-retrieval.ts
+var import_zod = require("zod");
+var import_ai = require("ai");
+var import_fs = __toESM(require("fs"), 1);
+
+// src/registry/query-preprocessing.ts
+var import_franc = require("franc");
+var import_natural = __toESM(require("natural"), 1);
+var STEMMER_MAP = {
+  "eng": import_natural.default.PorterStemmer,
+  // English
+  "deu": import_natural.default.PorterStemmerDe,
+  // German
+  "fra": import_natural.default.PorterStemmerFr,
+  // French
+  "rus": import_natural.default.PorterStemmerRu,
+  // Russian
+  "ita": import_natural.default.PorterStemmerIt,
+  // Italian
+  "nld": import_natural.default.PorterStemmerNl,
+  // Dutch
+  "por": import_natural.default.PorterStemmerPt,
+  // Portuguese
+  "spa": import_natural.default.PorterStemmerEs,
+  // Spanish
+  "swe": import_natural.default.PorterStemmerSv,
+  // Swedish
+  "nor": import_natural.default.PorterStemmerNo,
+  // Norwegian
+  "dan": import_natural.default.PorterStemmer
+  // Danish (fallback to English)
+};
+var COMMON_LANGUAGES = ["eng", "deu", "fra", "spa", "ita", "por", "rus", "nld"];
+function detectQueryLanguage(query, minLength = 10) {
+  const cleaned = query.trim();
+  if (cleaned.length < minLength) {
+    if (/[äöüßÄÖÜ]/.test(cleaned)) {
+      return "deu";
+    }
+    if (/[àâæçéèêëîïôùûüÿœÀÂÆÇÉÈÊËÎÏÔÙÛÜŸŒ]/.test(cleaned)) {
+      return "fra";
+    }
+    if (/[áéíóúñüÁÉÍÓÚÑÜ¿¡]/.test(cleaned)) {
+      return "spa";
+    }
+    return "eng";
+  }
+  const detected = (0, import_franc.franc)(cleaned, { only: COMMON_LANGUAGES, minLength: 3 });
+  if (detected === "und") {
+    return "eng";
+  }
+  return detected;
+}
+function stemWord(word, languageCode) {
+  const stemmer = STEMMER_MAP[languageCode] || import_natural.default.PorterStemmer;
+  const cleaned = word.replace(/[^\p{L}\p{N}]/gu, "").toLowerCase();
+  if (!cleaned) {
+    return word;
+  }
+  try {
+    return stemmer.stem(cleaned);
+  } catch (error) {
+    console.warn(`[EXULU] Error stemming word "${word}":`, error);
+    return cleaned;
+  }
+}
+function preprocessQuery(query, options = {}) {
+  const {
+    enableStemming = true,
+    detectLanguage = true,
+    preserveCase = false,
+    minDetectionLength = 10
+  } = options;
+  const language = detectLanguage ? detectQueryLanguage(query, minDetectionLength) : "eng";
+  console.log(`[EXULU] Query preprocessing - Detected language: ${language} for query: "${query}"`);
+  if (!enableStemming) {
+    return {
+      original: query,
+      processed: query,
+      language,
+      stemmed: false
+    };
+  }
+  const words = query.split(/\s+/);
+  const stemmedWords = words.map((word) => {
+    const stemmed = stemWord(word, language);
+    if (preserveCase && word[0] === word[0].toUpperCase()) {
+      return stemmed.charAt(0).toUpperCase() + stemmed.slice(1);
+    }
+    return stemmed;
+  });
+  const processed = stemmedWords.join(" ");
+  console.log(`[EXULU] Query preprocessing - Original: "${query}" \u2192 Stemmed: "${processed}"`);
+  return {
+    original: query,
+    processed,
+    language,
+    stemmed: true
+  };
+}
+
+// src/registry/agentic-retrieval.ts
+var import_zod_to_json_schema = require("zod-to-json-schema");
+async function withRetry(generateFn, maxRetries = 3) {
+  let lastError;
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      return await generateFn();
+    } catch (error) {
+      lastError = error;
+      console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
+      if (attempt === maxRetries) {
+        throw error;
+      }
+      await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 1e3));
+    }
+  }
+  throw lastError;
+}
+var baseInstructions = `
+You are an intelligent information retrieval assistant with access to multiple knowledge bases. You MUST do all your reasoning and
+    outputs in the same language as the user query.
+
+Your goal is to efficiently retrieve the most relevant information to answer user queries. You don't answer the question yourself, you only
+retrieve the information and return it, another tool will answer the question based on the information you retrieve.
+
+CRITICAL: STRUCTURED REASONING PROCESS
+You MUST follow this structured thinking process for EVERY step. Keep outputs VERY SHORT and in the same language as the user query - ideally one line each:
+
+1. BEFORE EACH TOOL CALL - Output your search strategy in ONE CONCISE LINE:
+   Format: \u{1F50D} [What you'll search]: [Tool(s)] [Method/Filters] \u2192 [Expected outcome]
+
+   Examples:
+   - "\u{1F50D} Searching for WPA-2 config in DPO-1 docs: search_items_products (name:DPO-1) \u2192 search_chunks_products (hybrid) \u2192 Expecting setup instructions"
+   - "\u{1F50D} Listing tourism items: search_items_hamburg (name contains 'tourism') \u2192 Document list"
+   - "\u{1F50D} Broad search for elevator data: search_chunks_* (hybrid, all contexts) \u2192 Statistical data"
+
+2. AFTER RECEIVING RESULTS - Reflect in ONE CONCISE LINE:
+   Format: \u{1F4AD} [Count] results | [Relevance] | [Next action]
+
+   Examples:
+   - "\u{1F4AD} 5 chunks found | Highly relevant WPA-2 instructions | Sufficient - returning results"
+   - "\u{1F4AD} 12 items with 'tourism' | Complete list | Done"
+   - "\u{1F4AD} 0 results in context_1 | Not found | Trying remaining contexts"
+
+IMPORTANT:
+- ONE LINE per reasoning block - be extremely concise
+- Use the same language as the user query
+- Focus only on: what, tool/method, outcome
+
+Choose your strategy based on the query type:
+
+FOR LISTING QUERIES (e.g., "list all documents about X" or "what items mention Y"):
+
+You have two options, you can use the search_items_by_name_[context_id] tool to find the items by name, if it is likely that
+the name includes a specific keyword, for example when looking for documentation regarding a specific product. Or you can use
+the search_[context_id] tool with the parameter "includeContent: false" to search inside the actual content of all or specific
+items, if you are looking for any items that might be relevant or contain information relevant to the query, but do not need 
+the actual content in your response.
+
+IMPORTANT: For listing queries, NEVER set includeContent: true unless you need the actual text content to answer the question
+
+FOR TARGETED QUERIES ABOUT SPECIFIC ITEMS (e.g., "how do I configure WPA-2 on my DPO-1? router" or "search in Document ABC"):
+TWO-STEP PATTERN:
+1. STEP 1 - Find the items: 
+As described above, you have two options, you can use the search_items_by_name_[context_id] tool to find the items by name, if it is likely that
+the name includes a specific keyword, for example when looking for documentation regarding a specific product. Or you can use
+the search_[context_id] tool with the parameter "includeContent: false" to search inside the actual content of all or specific
+items, if you are looking for any items that might be relevant or contain information relevant to the query, but do not need 
+the actual content in your response.
+2. STEP 2 - If step 1 returned any items, search for relevant information within those items: Use search_[context_id] with 
+hybrid search method. Example: search_[context_id] with hybrid search method and parameters item_name: 'DPO-1'
+   This searches only within the specific items you found. If no items were found in step 1 you should still
+   do the search_[context_id] but without pre-filtering them.
+
+Note that the query input for the search_[context_id] tools should not be used to search for things like "Page 2",
+ "Section 3", "Chapter 4", etc. but rather be used to search for specific information or answers within the
+content of the items.
+
+ONLY say "no information found" if you have:
+\u2713 Searched ALL available contexts (not just likely ones)
+\u2713 Tried hybrid, keyword, AND semantic search in each context
+\u2713 Tried variations of the search terms
+\u2713 Confirmed zero results across all attempts
+
+IMPORTANT OPTIMIZATION RULES:
+
+\u26A0\uFE0F CRITICAL: Always set includeContent: false when:
+- User asks for a list, overview, or count of documents/items
+- User wants to know "which documents" or "what items" without needing their content
+- You only need item names/metadata to answer the query
+- You can ALWAYS fetch the actual content later if needed with includeContent: true
+
+\u2713 Only set includeContent: true (or use default) when:
+- User needs specific information, details, or answers from the content
+- User asks "how to", "what does it say about", "explain", etc.
+- You need the actual text to answer the question
+
+Search Method Selection (for search_* tools):
+- Use 'hybrid' method by default for best relevance (combines semantic understanding + keyword matching)
+- Use 'keyword' method for exact term matching (technical terms, product names, IDs, specific phrases)
+- Use 'semantic' method for conceptual queries where synonyms and paraphrasing matter most
+
+Filtering and Limits:
+- Limit results appropriately (don't retrieve more than needed)
+`;
+function createCustomAgenticRetrievalToolLoopAgent({
+  language,
+  tools,
+  model,
+  customInstructions
+}) {
+  return {
+    generate: async function* ({
+      reranker,
+      query,
+      onFinish
+    }) {
+      import_fs.default.writeFileSync("reranker.json", JSON.stringify(reranker, null, 2));
+      let finished = false;
+      let maxSteps = 2;
+      let currentStep = 0;
+      const output = {
+        reasoning: [],
+        chunks: [],
+        usage: [],
+        totalTokens: 0
+      };
+      let dynamicTools = {};
+      while (!finished && currentStep < maxSteps) {
+        currentStep++;
+        console.log("[EXULU] Agentic retrieval step", currentStep);
+        import_fs.default.writeFileSync(currentStep - 1 + "_step_output.json", JSON.stringify(output, null, 2));
+        const systemPrompt = `
+                            ${baseInstructions}
+
+                            AVAILABLE TOOLS:
+                            ${Object.keys({ ...tools, ...dynamicTools }).map((tool3) => `
+                                <tool_${tool3}>
+                                    <name>${tool3}</name>
+                                    <description>${tools[tool3]?.description}</description>
+                                    ${tools[tool3]?.inputSchema ? `
+                                        <inputSchema>${(0, import_zod_to_json_schema.zodToJsonSchema)(tools[tool3]?.inputSchema)}</inputSchema>
+                                    ` : ""}
+                                </tool_${tool3}>
+                            `).join("\n\n")}
+
+                            ${customInstructions ? `
+                            CUSTOM INSTRUCTIONS: ${customInstructions}` : ""}
+                            `;
+        import_fs.default.writeFileSync("system_prompt_" + currentStep + ".txt", systemPrompt);
+        let reasoningOutput;
+        try {
+          reasoningOutput = await withRetry(async () => {
+            console.log("[EXULU] Generating reasoning for step", currentStep);
+            return await (0, import_ai.generateText)({
+              model,
+              output: import_ai.Output.object({
+                schema: import_zod.z.object({
+                  reasoning: import_zod.z.string().describe("The reasoning for the next step and why the agent needs to take this step. It MUST start with 'I must call tool XYZ', and MUST include the inputs for that tool."),
+                  finished: import_zod.z.boolean().describe("Whether the agent has finished meaning no further steps are needed, this should only be true if the agent believes no further tool calls are needed to get the relevant information for the query.")
+                })
+              }),
+              toolChoice: "none",
+              system: systemPrompt,
+              prompt: `
+                    Original query: ${query}
+                    
+                    Previous step reasoning and output:
+
+                    ${output.reasoning.map((reasoning, index) => `
+                    <step_${index + 1}>
+                        <reasoning>
+                        ${reasoning.text}
+                        </reasoning>
+
+                        ${reasoning.chunks ? `<retrieved_chunks>
+                            ${reasoning.chunks.map((chunk) => `
+                            ${chunk.item_name} - ${chunk.item_id} - ${chunk.context} - ${chunk.chunk_id} - ${chunk.chunk_index}
+                            `).join("\n")}
+                        </retrieved_chunks>` : ""}
+                        
+                        ${reasoning.tools ? `
+                        <used_tools>
+                            ${reasoning.tools.map((tool3) => `
+                            ${tool3.name} - ${tool3.id} - ${tool3.input}
+                            `).join("\n")}
+                        </used_tools>
+                        ` : ""}
+
+                    </step_${index + 1}>
+                    `).join("\n")}
+                    `,
+              stopWhen: [(0, import_ai.stepCountIs)(1)]
+            });
+          });
+          console.log("[EXULU] Reasoning generated for step", currentStep);
+        } catch (error) {
+          console.error("[EXULU] Failed to generate reasoning after 3 retries:", error);
+          throw error;
+        }
+        const {
+          reasoning: briefing,
+          finished: finished2
+        } = reasoningOutput?.output || {};
+        const { usage: reasoningUsage } = reasoningOutput || {};
+        output.usage.push(reasoningUsage);
+        if (finished2) {
+          console.log("[EXULU] Agentic retrieval finished for step", currentStep);
+          break;
+        }
+        let toolOutput;
+        try {
+          toolOutput = await withRetry(async () => {
+            console.log("[EXULU] Generating tool output for step", currentStep);
+            return await (0, import_ai.generateText)({
+              model,
+              tools: { ...tools, ...dynamicTools },
+              toolChoice: "required",
+              prompt: `${briefing}`,
+              stopWhen: [(0, import_ai.stepCountIs)(1)]
+            });
+          });
+          console.log("[EXULU] Tool output generated for step", currentStep);
+        } catch (error) {
+          console.error("[EXULU] Failed to generate tool output after 3 retries:", error);
+          throw error;
+        }
+        const toolResults = toolOutput.toolResults;
+        const toolCalls = toolOutput.toolCalls;
+        let chunks = [];
+        console.log("[EXULU] Processing tool results for step", currentStep);
+        if (Array.isArray(toolResults)) {
+          chunks = toolResults.map((result) => {
+            import_fs.default.writeFileSync(result.toolCallId + "_tool_call_result.json", JSON.stringify(result, null, 2));
+            let chunks2 = [];
+            if (typeof result.output === "string") {
+              chunks2 = JSON.parse(result.output);
+            } else {
+              chunks2 = result.output;
+            }
+            console.log("[EXULU] Chunks", chunks2);
+            return chunks2.map((chunk) => ({
+              ...chunk,
+              context: {
+                name: chunk.context ? chunk.context.replaceAll("_", " ") : "",
+                id: chunk.context
+              }
+            }));
+          }).flat();
+        }
+        if (chunks) {
+          if (reranker) {
+            console.log("[EXULU] Reranking chunks for step, using reranker", reranker.name + "(" + reranker.id + ")", "for step", currentStep, " for " + chunks?.length + " chunks");
+            import_fs.default.writeFileSync("chunks-before-reranking.json", JSON.stringify(chunks, null, 2));
+            chunks = await reranker.run(query, chunks);
+            import_fs.default.writeFileSync("chunks-after-reranking.json", JSON.stringify(chunks, null, 2));
+            console.log("[EXULU] Reranked chunks for step", currentStep, "using reranker", reranker.name + "(" + reranker.id + ")", " resulting in ", chunks?.length + " chunks");
+          }
+          output.chunks.push(...chunks);
+        }
+        console.log("[EXULU] Pushing reasoning for step", currentStep);
+        const exludedContent = toolCalls?.some(
+          (toolCall) => toolCall.input?.includeContent === false || toolCall.toolName.startsWith("search_items_by_name")
+        );
+        for (const chunk of chunks) {
+          const getMoreToolName = sanitizeToolName("get_more_content_from_" + chunk.item_name);
+          const { db: db3 } = await postgresClient();
+          if (!dynamicTools[getMoreToolName]) {
+            const chunksTable = getChunksTableName(chunk.context.id);
+            const countChunksQuery = await db3.from(chunksTable).where(
+              { source: chunk.item_id }
+            ).count("id");
+            const chunksCount = Number(countChunksQuery[0].count) || 0;
+            if (chunksCount > 1) {
+              dynamicTools[getMoreToolName] = (0, import_ai.tool)({
+                description: `The item ${chunk.item_name} has a total of${chunksCount} chunks, this tool allows you to get more content from this item across all its pages / chunks.`,
+                inputSchema: import_zod.z.object({
+                  from_index: import_zod.z.number().default(1).describe("The index of the chunk to start from."),
+                  to_index: import_zod.z.number().max(chunksCount).describe("The index of the chunk to end at, max is " + chunksCount)
+                }),
+                execute: async ({ from_index, to_index }) => {
+                  const chunks2 = await db3(chunksTable).select("*").where("source", chunk.item_id).whereBetween("chunk_index", [from_index, to_index]).orderBy("chunk_index", "asc");
+                  return JSON.stringify(chunks2.map((resultChunk) => ({
+                    chunk_content: resultChunk.content,
+                    chunk_index: resultChunk.chunk_index,
+                    chunk_id: resultChunk.id,
+                    chunk_source: resultChunk.source,
+                    chunk_metadata: resultChunk.metadata,
+                    item_id: chunk.item_id,
+                    item_name: chunk.item_name,
+                    context: chunk.context?.id
+                  })), null, 2);
+                }
+              });
+            }
+          }
+          if (exludedContent) {
+            const getContentToolName = sanitizeToolName("get_" + chunk.item_name + "_page_" + chunk.chunk_index + "_content");
+            dynamicTools[getContentToolName] = (0, import_ai.tool)({
+              description: `Get the content of the page ${chunk.chunk_index} for the item ${chunk.item_name}`,
+              inputSchema: import_zod.z.object({
+                reasoning: import_zod.z.string().describe("The reasoning for why you need to get the content of the page.")
+              }),
+              execute: async ({ reasoning }) => {
+                const { db: db4 } = await postgresClient();
+                const chunksTable = getChunksTableName(chunk.context.id);
+                const resultChunks = await db4(chunksTable).select("*").where("id", chunk.chunk_id).limit(1);
+                if (!resultChunks || !resultChunks[0]) {
+                  return null;
+                }
+                return JSON.stringify([{
+                  reasoning,
+                  chunk_content: resultChunks[0].content,
+                  chunk_index: resultChunks[0].chunk_index,
+                  chunk_id: resultChunks[0].id,
+                  chunk_source: resultChunks[0].source,
+                  chunk_metadata: resultChunks[0].metadata,
+                  chunk_created_at: resultChunks[0].chunk_created_at,
+                  item_id: chunk.item_id,
+                  item_name: chunk.item_name,
+                  context: chunk.context?.id
+                }], null, 2);
+              }
+            });
+          }
+        }
+        output.reasoning.push({
+          text: briefing,
+          chunks: chunks || [],
+          tools: toolCalls?.length > 0 ? toolCalls.map((toolCall) => ({
+            name: toolCall.toolName,
+            id: toolCall.toolCallId,
+            input: toolCall.input,
+            output: chunks
+          })) : []
+        });
+        const { usage: toolUsage } = toolOutput || {};
+        console.log("[EXULU] Pushing tool usage for step", currentStep);
+        output.usage.push(toolUsage);
+        console.log(`[EXULU] Agentic retrieval step ${currentStep} completed`);
+        console.log("[EXULU] Agentic retrieval step output", output);
+        yield output;
+      }
+      const totalTokens = output.usage.reduce((acc, usage) => acc + (usage.totalTokens || 0), 0);
+      output.totalTokens = totalTokens;
+      console.log("[EXULU] Agentic retrieval finished", output);
+      onFinish(output);
+    }
+  };
+}
+var createAgenticRetrievalAgent = ({
+  contexts,
+  user,
+  role,
+  model,
+  instructions: custom,
+  projectRetrievalTool,
+  language = "eng"
+}) => {
+  const searchItemsByNameTool = {
+    "search_items_by_name": (0, import_ai.tool)({
+      description: `
+        Search for relevant items by name across the available knowledge bases.`,
+      inputSchema: import_zod.z.object({
+        knowledge_base_ids: import_zod.z.array(import_zod.z.enum(contexts.map((ctx) => ctx.id))).describe(`
+                    The available knowledge bases are:
+                    ${contexts.map((ctx) => `
+                        <knowledge_base>
+                            <id>${ctx.id}</id>
+                            <name>${ctx.name}</name>
+                            <description>${ctx.description}</description>
+                        </knowledge_base>
+                    `).join("\n")}
+                `),
+        item_name: import_zod.z.string().describe("The name of the item to search for."),
+        limit: import_zod.z.number().default(100).describe("Maximum number of items to return (max 400), if searching through multiple knowledge bases, the limit is applied for each knowledge base individually.")
+      }),
+      execute: async ({ item_name, limit, knowledge_base_ids }) => {
+        if (!knowledge_base_ids?.length) {
+          knowledge_base_ids = contexts.map((ctx) => ctx.id);
+        }
+        let itemFilters = [];
+        if (item_name) {
+          itemFilters.push({ name: { contains: item_name } });
+        }
+        const { db: db3 } = await postgresClient();
+        const results = await Promise.all(knowledge_base_ids.map(async (knowledge_base_id) => {
+          const ctx = contexts.find((ctx2) => ctx2.id === knowledge_base_id || ctx2.id.toLowerCase().includes(knowledge_base_id.toLowerCase()));
+          if (!ctx) {
+            console.error("[EXULU] Knowledge base ID that was provided to search items by name not found.", knowledge_base_id);
+            throw new Error("Knowledge base ID that was provided to search items by name not found.");
+          }
+          let itemsQuery = db3(getTableName(ctx.id) + " as items").select([
+            "items.id as item_id",
+            "items.name as item_name",
+            "items.external_id as item_external_id",
+            db3.raw('items."updatedAt" as item_updated_at'),
+            db3.raw('items."createdAt" as item_created_at'),
+            ...ctx.fields.map((field) => `items.${field.name} as ${field.name}`)
+          ]);
+          if (!limit) {
+            limit = 100;
+          }
+          limit = Math.min(limit, 400);
+          itemsQuery = itemsQuery.limit(limit);
+          const tableDefinition = contextToTableDefinition(ctx);
+          itemsQuery = applyFilters(itemsQuery, itemFilters || [], tableDefinition, "items");
+          itemsQuery = applyAccessControl(tableDefinition, itemsQuery, user, "items");
+          const items2 = await itemsQuery;
+          return items2?.map((item) => ({
+            ...item,
+            context: ctx.id
+          }));
+        }));
+        const items = results.flat();
+        const formattedResults = await Promise.all(items.map(async (item, index) => {
+          if (!item.item_id || !item.context) {
+            console.error("[EXULU] Item id and context are required to get chunks.", item);
+            throw new Error("Item id is required to get chunks.");
+          }
+          const chunksTable = getChunksTableName(item.context);
+          const chunks = await db3.from(chunksTable).select(["id", "source", "metadata"]).where("source", item.item_id).limit(1);
+          if (!chunks || !chunks[0]) {
+            return null;
+          }
+          return {
+            item_name: item.item_name,
+            item_id: item.item_id,
+            context: item.context || "",
+            chunk_id: chunks[0].id,
+            chunk_index: 1,
+            chunk_content: void 0,
+            metadata: chunks[0].metadata
+          };
+        }));
+        return JSON.stringify(formattedResults.filter((result) => result !== null), null, 2);
+      }
+    })
+  };
+  const searchTools = {
+    "search_content": (0, import_ai.tool)({
+      description: `
+            Search for relevant information within the actual content of the items across available knowledge bases.
+
+            This tool provides a number of strategies:
+            - Keyword search: search for exact terms, technical names, IDs, or specific phrases
+            - Semantic search: search for conceptual queries where synonyms and paraphrasing matter
+            - Hybrid search: best for most queries - combines semantic understanding with exact term matching
+
+            You can use the includeContent parameter to control whether to return
+            the full chunk content or just metadata.
+
+            Use with includeContent: true (default) when you need to:
+            - Find specific information or answers within documents
+            - Get actual text content that answers a query
+            - Extract details, explanations, or instructions from content
+
+            Use with includeContent: false when you need to:
+            - List which documents/items contain certain topics
+            - Count or overview items that match a content query
+            - Find item names/metadata without loading full content
+            - You can always fetch content later if needed
+            
+            `,
+      inputSchema: import_zod.z.object({
+        query: import_zod.z.string().describe("The search query to find relevant chunks, this must always be related to the content you are looking for, not something like 'Page 2'."),
+        knowledge_base_ids: import_zod.z.array(import_zod.z.enum(contexts.map((ctx) => ctx.id))).describe(`
+                    The available knowledge bases are:
+                    ${contexts.map((ctx) => `
+                        <knowledge_base>
+                            <id>${ctx.id}</id>
+                            <name>${ctx.name}</name>
+                            <description>${ctx.description}</description>
+                        </knowledge_base>
+                    `).join("\n")}
+                `),
+        keywords: import_zod.z.array(import_zod.z.string()).optional().describe("Keywords to search for. Usually extracted from the query, allowing for more precise search results."),
+        searchMethod: import_zod.z.enum(["keyword", "semantic", "hybrid"]).default("hybrid").describe(
+          "Search method: 'hybrid' (best for most queries - combines semantic understanding with exact term matching), 'keyword' (best for exact terms, technical names, IDs, or specific phrases), 'semantic' (best for conceptual queries where synonyms and paraphrasing matter)"
+        ),
+        includeContent: import_zod.z.boolean().default(true).describe(
+          "Whether to include the full chunk content in results. Set to FALSE when you only need to know WHICH documents/items are relevant (lists, overviews, counts). Set to TRUE when you need the ACTUAL content to answer the question (information, details, explanations). You can always fetch content later, so prefer FALSE for efficiency when listing documents."
+        ),
+        item_ids: import_zod.z.array(import_zod.z.string()).optional().describe("Use if you wish to retrieve content from specific items (documents) based on the item ID."),
+        item_names: import_zod.z.array(import_zod.z.string()).optional().describe("Use if you wish to retrieve content from specific items (documents) based on the item name. Can be a partial match."),
+        item_external_ids: import_zod.z.array(import_zod.z.string()).optional().describe("Use if you wish to retrieve content from specific items (documents) based on the item external ID. Can be a partial match."),
+        limit: import_zod.z.number().default(10).describe("Maximum number of chunks to return (max 10)")
+      }),
+      execute: async ({ query, searchMethod, limit, includeContent, item_ids, item_names, item_external_ids, keywords, knowledge_base_ids }) => {
+        if (!knowledge_base_ids?.length) {
+          knowledge_base_ids = contexts.map((ctx) => ctx.id);
+        }
+        const results = await Promise.all(knowledge_base_ids.map(async (knowledge_base_id) => {
+          const ctx = contexts.find((ctx2) => ctx2.id === knowledge_base_id || ctx2.id.toLowerCase().includes(knowledge_base_id.toLowerCase()));
+          if (!ctx) {
+            console.error("[EXULU] Knowledge base ID that was provided to search content not found.", knowledge_base_id);
+            throw new Error("Knowledge base ID that was provided to search content not found.");
+          }
+          let itemFilters = [];
+          if (item_ids) {
+            itemFilters.push({ id: { in: item_ids } });
+          }
+          if (item_names) {
+            itemFilters.push({ name: { or: item_names.map((name) => ({ contains: name })) } });
+          }
+          if (item_external_ids) {
+            itemFilters.push({ external_id: { in: item_external_ids } });
+          }
+          if (!query && keywords) {
+            query = keywords.join(" ");
+          }
+          const results2 = await ctx.search({
+            query,
+            keywords,
+            method: searchMethod === "hybrid" ? "hybridSearch" : searchMethod === "keyword" ? "tsvector" : "cosineDistance",
+            limit: includeContent ? Math.min(limit, 10) : Math.min(limit * 20, 400),
+            page: 1,
+            itemFilters: itemFilters || [],
+            chunkFilters: [],
+            sort: { field: "updatedAt", direction: "desc" },
+            user,
+            role,
+            trigger: "tool"
+          });
+          return results2.chunks.map((chunk) => chunk);
+        }));
+        const resultsFlat = results.flat();
+        const formattedResults = resultsFlat.map((chunk, index) => ({
+          item_name: chunk.item_name,
+          item_id: chunk.item_id,
+          context: chunk.context?.id || "",
+          chunk_id: chunk.chunk_id,
+          chunk_index: chunk.chunk_index,
+          chunk_content: includeContent ? chunk.chunk_content : void 0,
+          metadata: {
+            ...chunk.chunk_metadata,
+            cosine_distance: chunk.chunk_cosine_distance,
+            fts_rank: chunk.chunk_fts_rank,
+            hybrid_score: chunk.chunk_hybrid_score
+          }
+        }));
+        return JSON.stringify(formattedResults, null, 2);
+      }
+    })
+  };
+  console.log("[EXULU] Search tools:", Object.keys(searchTools));
+  const agent = createCustomAgenticRetrievalToolLoopAgent({
+    language,
+    model,
+    customInstructions: custom,
+    tools: {
+      ...searchTools,
+      ...searchItemsByNameTool,
+      ...projectRetrievalTool ? { [projectRetrievalTool.id]: projectRetrievalTool.tool } : {}
+    }
+  });
+  return agent;
+};
+async function* executeAgenticRetrieval({
+  contexts,
+  reranker,
+  query,
+  user,
+  role,
+  model,
+  instructions,
+  projectRetrievalTool
+}) {
+  const { language } = preprocessQuery(query, {
+    detectLanguage: true
+  });
+  console.log("[EXULU] Language detected:", language);
+  console.log("[EXULU] Creating agentic retrieval agent", "available contexts:", contexts.map((ctx) => ctx.id));
+  const agent = createAgenticRetrievalAgent({ contexts, user, role, model, instructions, projectRetrievalTool, language });
+  console.log("[EXULU] Starting agentic retrieval");
+  try {
+    let finishResolver;
+    let finishRejector;
+    const finishPromise = new Promise((resolve, reject) => {
+      finishResolver = resolve;
+      finishRejector = reject;
+    });
+    const timeoutId = setTimeout(() => {
+      finishRejector(new Error("Agentic retrieval timed out after 240 seconds"));
+    }, 24e4);
+    const result = agent.generate({
+      reranker,
+      query,
+      onFinish: (output) => {
+        clearTimeout(timeoutId);
+        finishResolver(output);
+      }
+    });
+    for await (const output of result) {
+      yield output;
+    }
+    const finalOutput = await finishPromise;
+    console.log("[EXULU] Agentic retrieval output", finalOutput);
+    return finalOutput;
+  } catch (error) {
+    console.error("[EXULU] Agentic retrieval error:", error);
+    yield JSON.stringify({
+      status: "error",
+      message: error instanceof Error ? error.message : "Unknown error occurred"
+    });
+  }
+}
+var createAgenticRetrievalTool = ({
+  contexts,
+  rerankers,
+  user,
+  role,
+  model,
+  projectRetrievalTool
+}) => {
+  const contextNames = contexts.map((ctx) => ctx.id).join(", ");
+  return new ExuluTool({
+    id: "agentic_context_search",
+    name: "Agentic Context Search",
+    description: `Intelligent context search tool that uses AI to reason through and retrieve relevant information from available knowledge bases (${contextNames}). This tool can understand complex queries, search across multiple contexts, filter items by name, id, external id, and expand context as needed.`,
+    category: "contexts",
+    type: "context",
+    // Config to enable / disable individual contexts
+    config: [
+      {
+        name: "instructions",
+        description: `Custom instructions to use when searching the knowledge bases. This is appended to the default system instructions.`,
+        type: "string",
+        default: ""
+      },
+      {
+        name: "reranker",
+        description: "The reranker to use for the retrieval process.",
+        type: "string",
+        default: "none"
+      },
+      ...contexts.map((ctx) => ({
+        name: ctx.id,
+        description: `Enable search in the ${ctx.name} context. ${ctx.description}`,
+        type: "boolean",
+        default: true
+      }))
+    ],
+    inputSchema: import_zod.z.object({
+      query: import_zod.z.string().describe("The question or query to answer using the knowledge bases"),
+      userInstructions: import_zod.z.string().optional().describe("Instructions provided by the user to customize the retrieval process.")
+    }),
+    execute: async function* ({ query, userInstructions, toolVariablesConfig }) {
+      let configInstructions = "";
+      let configuredReranker;
+      if (toolVariablesConfig) {
+        configInstructions = toolVariablesConfig.instructions;
+        contexts = contexts.filter(
+          (ctx) => toolVariablesConfig[ctx.id] === true || toolVariablesConfig[ctx.id] === "true" || toolVariablesConfig[ctx.id] === 1
+        );
+        if (toolVariablesConfig.reranker) {
+          configuredReranker = rerankers.find((reranker) => reranker.id === toolVariablesConfig.reranker);
+          if (!configuredReranker) {
+            throw new Error("Reranker not found: " + toolVariablesConfig.reranker + ", check with a developer if the reranker was removed from the system.");
+          }
+        }
+      }
+      console.log("[EXULU] Executing agentic retrieval tool with data", {
+        // Log only first level properties
+        // Keys of all toolVariablesConfig vars
+        configs: Object.keys(toolVariablesConfig),
+        query,
+        instructions: configInstructions,
+        reranker: configuredReranker?.id || void 0,
+        contexts: contexts.map((ctx) => ctx.id)
+      });
+      console.log("[EXULU] Executing agentic retrieval tool");
+      for await (const chunk of executeAgenticRetrieval({
+        contexts,
+        reranker: configuredReranker,
+        query,
+        user,
+        role,
+        model,
+        instructions: `${configInstructions ? `CUSTOM INSTRUCTIONS PROVIDED BY THE ADMIN: ${configInstructions}` : ""} ${userInstructions ? `INSTRUCTIONS PROVIDED BY THE USER: ${userInstructions}` : ""}`,
+        projectRetrievalTool
+      })) {
+        yield { result: JSON.stringify(chunk, null, 2) };
+      }
+    }
+  });
+};
+
 // src/registry/utils.ts
 var bullmq = {
   validate: (id, data) => {
@@ -1580,12 +2374,24 @@ var bullmq = {
     }
   }
 };
-var getEnabledTools = async (agentInstance, allExuluTools, disabledTools = [], agents, user) => {
+var getEnabledTools = async (agentInstance, allExuluTools, allContexts, allRerankers, disabledTools = [], agents, user) => {
   let enabledTools = [];
   if (agentInstance.tools) {
     const results = await Promise.all(agentInstance.tools.map(
       async ({ config, id, type }) => {
         let hydrated;
+        if (id === "agentic_context_search") {
+          return createAgenticRetrievalTool({
+            // This tool is reinstantiated in the convertToolsArrayToObject function, where 
+            // we can access the activated contexts and model that is calling it but we also 
+            // return it here so we know it was generally enabled as a tool.
+            contexts: allContexts,
+            rerankers: allRerankers || [],
+            user,
+            role: user?.role?.id,
+            model: void 0
+          });
+        }
         if (type === "agent") {
           if (id === agentInstance.id) {
             return null;
@@ -1602,7 +2408,7 @@ var getEnabledTools = async (agentInstance, allExuluTools, disabledTools = [], a
           if (!hasAccessToAgent) {
             return null;
           }
-          hydrated = await backend.tool(instance.id, agents);
+          hydrated = await backend.tool(instance.id, agents, allContexts, allRerankers || []);
         } else {
           hydrated = allExuluTools.find((t) => t.id === id);
         }
@@ -1613,7 +2419,7 @@ var getEnabledTools = async (agentInstance, allExuluTools, disabledTools = [], a
   }
   console.log("[EXULU] available tools", enabledTools?.length);
   console.log("[EXULU] disabled tools", disabledTools?.length);
-  enabledTools = enabledTools.filter((tool2) => !disabledTools.includes(tool2.id));
+  enabledTools = enabledTools.filter((tool3) => !disabledTools.includes(tool3.id));
   return enabledTools;
 };
 var loadAgentCache = /* @__PURE__ */ new Map();
@@ -1799,7 +2605,7 @@ var import_ioredis = __toESM(require("ioredis"), 1);
 var import_bullmq2 = require("bullmq");
 var import_api = require("@opentelemetry/api");
 var import_uuid2 = require("uuid");
-var import_ai = require("ai");
+var import_ai2 = require("ai");
 var import_crypto_js = __toESM(require("crypto-js"), 1);
 
 // src/registry/log-metadata.ts
@@ -1835,7 +2641,7 @@ var installGlobalErrorHandlers = () => {
   unhandledRejectionHandlerInstalled = true;
   console.log("[EXULU] Global error handlers installed to prevent worker crashes");
 };
-var createWorkers = async (agents, queues2, config, contexts, evals, tools, tracer) => {
+var createWorkers = async (agents, queues2, config, contexts, rerankers, evals, tools, tracer) => {
   console.log("[EXULU] creating workers for " + queues2?.length + " queues.");
   console.log("[EXULU] queues", queues2.map((q) => q.queue.name));
   installGlobalErrorHandlers();
@@ -2011,6 +2817,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                       agentBackend,
                       inputMessages,
                       contexts,
+                      rerankers,
                       user,
                       tools,
                       config,
@@ -2085,6 +2892,7 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                       agentBackend,
                       inputMessages,
                       contexts,
+                      rerankers,
                       user,
                       tools,
                       config
@@ -2524,6 +3332,7 @@ var processUiMessagesFlow = async ({
   agentBackend,
   inputMessages,
   contexts,
+  rerankers,
   user,
   tools,
   config,
@@ -2533,7 +3342,7 @@ var processUiMessagesFlow = async ({
   console.log("[EXULU] input messages", inputMessages);
   console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
   const disabledTools = [];
-  let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
+  let enabledTools = await getEnabledTools(agentInstance, tools, contexts, rerankers, disabledTools, agents, user);
   console.log("[EXULU] enabled tools", enabledTools?.map((x) => x.name + " (" + x.id + ")"));
   const variableName = agentInstance.providerapikey;
   const { db: db3 } = await postgresClient();
@@ -2606,7 +3415,10 @@ var processUiMessagesFlow = async ({
       try {
         const result = await agentBackend.generateStream({
           contexts,
+          rerankers,
+          agentInstance,
           user,
+          approvedTools: tools.map((tool3) => "tool-" + sanitizeToolName(tool3.name)),
           instructions: agentInstance.instructions,
           session: void 0,
           previousMessages: messageHistory.messages,
@@ -2722,7 +3534,7 @@ function getAverage(arr) {
 }
 
 // src/registry/utils/graphql.ts
-var GraphQLDate = new import_graphql2.GraphQLScalarType({
+var GraphQLDate = new import_graphql3.GraphQLScalarType({
   name: "Date",
   description: "Date custom scalar type",
   serialize(value) {
@@ -2747,10 +3559,10 @@ var GraphQLDate = new import_graphql2.GraphQLScalarType({
     return value;
   },
   parseLiteral(ast) {
-    if (ast.kind === import_graphql2.Kind.STRING) {
+    if (ast.kind === import_graphql3.Kind.STRING) {
       return new Date(ast.value);
     }
-    if (ast.kind === import_graphql2.Kind.INT) {
+    if (ast.kind === import_graphql3.Kind.INT) {
       return new Date(parseInt(ast.value, 10));
     }
     return null;
@@ -2962,6 +3774,32 @@ var getRequestedFields = (info) => {
     (field) => field !== "pageInfo" && field !== "items" && field !== "RBAC"
   );
 };
+var contextItemProcessorMutation = async (context, config, items, user, role) => {
+  let jobs = [];
+  let results = [];
+  await Promise.all(
+    items.map(async (item) => {
+      const result = await context.processField(
+        "api",
+        item,
+        config,
+        user,
+        role
+      );
+      if (result.job) {
+        jobs.push(result.job);
+      }
+      if (result.result) {
+        results.push(result.result);
+      }
+    })
+  );
+  return {
+    message: jobs.length > 0 ? "Processing job scheduled." : "Items processed successfully.",
+    results: results.map((result) => JSON.stringify(result)),
+    jobs
+  };
+};
 var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRbacRecords) => {
   const {
     users = [],
@@ -3036,7 +3874,7 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
     await db3.from("rbac").insert(recordsToInsert);
   }
 };
-function createMutations(table, agents, contexts, tools, config) {
+function createMutations(table, agents, contexts, rerankers, tools, config) {
   const tableNamePlural = table.name.plural.toLowerCase();
   const tableNameSingular = table.name.singular.toLowerCase();
   const validateWriteAccess = async (id, context) => {
@@ -3161,6 +3999,7 @@ function createMutations(table, agents, contexts, tools, config) {
           requestedFields,
           agents,
           contexts,
+          rerankers,
           tools,
           result: result[0],
           user: context.user
@@ -3236,6 +4075,7 @@ function createMutations(table, agents, contexts, tools, config) {
           requestedFields,
           agents,
           contexts,
+          rerankers,
           tools,
           result: results[0],
           user: context.user
@@ -3315,6 +4155,7 @@ function createMutations(table, agents, contexts, tools, config) {
           requestedFields,
           agents,
           contexts,
+          rerankers,
           tools,
           result,
           user: context.user.id
@@ -3385,6 +4226,7 @@ function createMutations(table, agents, contexts, tools, config) {
           requestedFields,
           agents,
           contexts,
+          rerankers,
           tools,
           result,
           user: context.user.id
@@ -3433,6 +4275,7 @@ function createMutations(table, agents, contexts, tools, config) {
         requestedFields,
         agents,
         contexts,
+        rerankers,
         tools,
         result,
         user: context.user.id
@@ -3473,6 +4316,7 @@ function createMutations(table, agents, contexts, tools, config) {
         requestedFields,
         agents,
         contexts,
+        rerankers,
         tools,
         result,
         user: context.user.id
@@ -3481,32 +4325,6 @@ function createMutations(table, agents, contexts, tools, config) {
   };
   if (table.type === "items") {
     if (table.processor) {
-      const contextItemProcessorMutation = async (context, items, user, role) => {
-        let jobs = [];
-        let results = [];
-        await Promise.all(
-          items.map(async (item) => {
-            const result = await context.processField(
-              "api",
-              item,
-              config,
-              user,
-              role
-            );
-            if (result.job) {
-              jobs.push(result.job);
-            }
-            if (result.result) {
-              results.push(result.result);
-            }
-          })
-        );
-        return {
-          message: jobs.length > 0 ? "Processing job scheduled." : "Items processed successfully.",
-          results: results.map((result) => JSON.stringify(result)),
-          jobs
-        };
-      };
       mutations[`${tableNameSingular}ProcessItem`] = async (_, args, context, info) => {
         if (!context.user?.super_admin) {
           throw new Error(
@@ -3533,6 +4351,7 @@ function createMutations(table, agents, contexts, tools, config) {
         }
         return contextItemProcessorMutation(
           exists,
+          config,
           [item],
           context.user.id,
           context.user.role?.id
@@ -3562,6 +4381,7 @@ function createMutations(table, agents, contexts, tools, config) {
         }
         return contextItemProcessorMutation(
           exists,
+          config,
           items,
           context.user.id,
           context.user.role?.id
@@ -3792,14 +4612,14 @@ var applyAccessControl = (table, query, user, field_prefix) => {
         this.orWhere(`${prefix}created_by`, user.id);
         this.orWhere(function() {
           this.where(`${prefix}rights_mode`, "users").whereExists(function() {
-            this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
+            this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + (prefix ? prefix.slice(0, -1) : tableNamePlural) + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "User").where("rbac.user_id", user.id);
           });
         });
       }
       if (user?.role) {
         this.orWhere(function() {
           this.where(`${prefix}rights_mode`, "roles").whereExists(function() {
-            this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + tableNamePlural + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
+            this.select("*").from("rbac").whereRaw("rbac.target_resource_id = " + (prefix ? prefix.slice(0, -1) : tableNamePlural) + ".id").where("rbac.entity", table.name.singular).where("rbac.access_type", "Role").where("rbac.role_id", user.role.id);
           });
         });
       }
@@ -3895,7 +4715,7 @@ var removeAgentFields = (requestedFields) => {
   filtered.push("backend");
   return filtered;
 };
-var addAgentFields = async (args, requestedFields, agents, result, tools, user, contexts) => {
+var addAgentFields = async (args, requestedFields, agents, result, tools, user, contexts, rerankers) => {
   let backend = agents.find((a) => a.id === result?.backend);
   if (requestedFields.includes("providerName")) {
     result.providerName = backend?.providerName || "";
@@ -3913,22 +4733,38 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
     if (result.tools) {
       result.tools = await Promise.all(
         result.tools.map(
-          async (tool2) => {
+          async (tool3) => {
             let hydrated;
-            if (tool2.type === "agent") {
-              if (tool2.id === result.id) {
+            if (tool3.id === "agentic_context_search") {
+              const instance = createAgenticRetrievalTool({
+                contexts: [],
+                rerankers: [],
+                user,
+                role: user.role?.id,
+                model: void 0
+              });
+              return {
+                ...instance,
+                name: instance.name,
+                description: instance.description,
+                category: instance.category,
+                config: tool3.config
+              };
+            }
+            if (tool3.type === "agent") {
+              if (tool3.id === result.id) {
                 return null;
               }
-              const instance = await loadAgent(tool2.id);
+              const instance = await loadAgent(tool3.id);
               if (!instance) {
                 throw new Error(
-                  "Trying to load a tool of type 'agent', but the associated agent with id " + tool2.id + " was not found in the database."
+                  "Trying to load a tool of type 'agent', but the associated agent with id " + tool3.id + " was not found in the database."
                 );
               }
               const backend2 = agents.find((a) => a.id === instance.backend);
               if (!backend2) {
                 throw new Error(
-                  "Trying to load a tool of type 'agent', but the associated agent with id " + tool2.id + " does not have a backend set for it."
+                  "Trying to load a tool of type 'agent', but the associated agent with id " + tool3.id + " does not have a backend set for it."
                 );
               }
               const hasAccessToAgent = await checkRecordAccess(
@@ -3939,15 +4775,15 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
               if (!hasAccessToAgent) {
                 return null;
               }
-              hydrated = await backend2.tool(instance.id, agents);
+              hydrated = await backend2.tool(instance.id, agents, contexts, rerankers);
             } else {
-              hydrated = tools.find((t) => t.id === tool2.id);
+              hydrated = tools.find((t) => t.id === tool3.id);
             }
             const hydratedTool = {
-              ...tool2,
+              ...tool3,
               name: hydrated?.name || "",
               description: hydrated?.description || "",
-              category: tool2?.category || "default"
+              category: tool3?.category || "default"
             };
             console.log("[EXULU] hydratedTool", hydratedTool);
             return hydratedTool;
@@ -3955,7 +4791,7 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
         )
       );
       if (args.project) {
-        const projectTool = await createProjectRetrievalTool({
+        const projectTool = await createProjectItemsRetrievalTool({
           projectId: args.project,
           user,
           role: user.role?.id,
@@ -3965,7 +4801,7 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
           result.tools.unshift(projectTool);
         }
       }
-      result.tools = result.tools.filter((tool2) => tool2 !== null);
+      result.tools = result.tools.filter((tool3) => tool3 !== null);
     } else {
       result.tools = [];
     }
@@ -4085,6 +4921,21 @@ var postprocessUpdate = async ({
           job
         };
       }
+      if (context.processor && (context.processor.config?.trigger === "onUpdate" || context.processor.config?.trigger === "always")) {
+        const {
+          jobs
+        } = await contextItemProcessorMutation(
+          context,
+          config,
+          [result],
+          user,
+          role
+        );
+        return {
+          result,
+          job: jobs[0]
+        };
+      }
       return result;
     }
   }
@@ -4156,6 +5007,7 @@ var finalizeRequestedFields = async ({
   requestedFields,
   agents,
   contexts,
+  rerankers,
   tools,
   result,
   user
@@ -4174,6 +5026,7 @@ var finalizeRequestedFields = async ({
         requestedFields,
         agents,
         contexts,
+        rerankers,
         tools,
         result: item,
         user
@@ -4215,7 +5068,8 @@ var finalizeRequestedFields = async ({
         result,
         tools,
         user,
-        contexts
+        contexts,
+        rerankers
       );
       if (!requestedFields.includes("backend")) {
         delete result.backend;
@@ -4264,6 +5118,9 @@ var finalizeRequestedFields = async ({
   return result;
 };
 var applyFilters = (query, filters, table, field_prefix) => {
+  if (!filters) {
+    return query;
+  }
   filters.forEach((filter) => {
     Object.entries(filter).forEach(([fieldName, operators]) => {
       if (operators) {
@@ -4279,14 +5136,18 @@ var applyFilters = (query, filters, table, field_prefix) => {
           });
         }
         if (operators.or !== void 0) {
-          operators.or.forEach((operator) => {
-            query = converOperatorToQuery(
-              query,
-              fieldName,
-              operator,
-              table,
-              field_prefix
-            );
+          query = query.where((builder) => {
+            operators.or.forEach((operator) => {
+              builder.orWhere((subBuilder) => {
+                converOperatorToQuery(
+                  subBuilder,
+                  fieldName,
+                  operator,
+                  table,
+                  field_prefix
+                );
+              });
+            });
           });
         }
         query = converOperatorToQuery(
@@ -4301,7 +5162,7 @@ var applyFilters = (query, filters, table, field_prefix) => {
   });
   return query;
 };
-var applySorting = (query, sort, field_prefix) => {
+var applySorting2 = (query, sort, field_prefix) => {
   const prefix = field_prefix ? field_prefix + "." : "";
   if (sort) {
     sort.field = prefix + sort.field;
@@ -4335,7 +5196,7 @@ var itemsPaginationRequest = async ({
   let dataQuery = db3(tableName);
   dataQuery = applyFilters(dataQuery, filters, table);
   dataQuery = applyAccessControl(table, dataQuery, user);
-  dataQuery = applySorting(dataQuery, sort);
+  dataQuery = applySorting2(dataQuery, sort);
   if (page > 1) {
     dataQuery = dataQuery.offset((page - 1) * limit);
   }
@@ -4352,7 +5213,7 @@ var itemsPaginationRequest = async ({
     }
   };
 };
-function createQueries(table, agents, tools, contexts) {
+function createQueries(table, agents, tools, contexts, rerankers) {
   const tableNamePlural = table.name.plural.toLowerCase();
   const tableNameSingular = table.name.singular.toLowerCase();
   const queries = {
@@ -4369,6 +5230,7 @@ function createQueries(table, agents, tools, contexts) {
         requestedFields,
         agents,
         contexts,
+        rerankers,
         tools,
         result,
         user: context.user
@@ -4387,6 +5249,7 @@ function createQueries(table, agents, tools, contexts) {
         requestedFields,
         agents,
         contexts,
+        rerankers,
         tools,
         result,
         user: context.user
@@ -4400,7 +5263,7 @@ function createQueries(table, agents, tools, contexts) {
       let query = db3.from(tableNamePlural).select(sanitizedFields);
       query = applyFilters(query, filters, table);
       query = applyAccessControl(table, query, context.user);
-      query = applySorting(query, sort);
+      query = applySorting2(query, sort);
       let result = await query.first();
       return finalizeRequestedFields({
         args,
@@ -4408,6 +5271,7 @@ function createQueries(table, agents, tools, contexts) {
         requestedFields,
         agents,
         contexts,
+        rerankers,
         tools,
         result,
         user: context.user
@@ -4436,6 +5300,7 @@ function createQueries(table, agents, tools, contexts) {
           requestedFields,
           agents,
           contexts,
+          rerankers,
           tools,
           result: items,
           user: context.user
@@ -4444,11 +5309,12 @@ function createQueries(table, agents, tools, contexts) {
     },
     // Add generic statistics query for all tables
     [`${tableNamePlural}Statistics`]: async (_, args, context, info) => {
-      const { filters = [], groupBy } = args;
+      const { filters = [], groupBy, limit = 10 } = args;
       const { db: db3 } = context;
       let query = db3(tableNamePlural);
       query = applyFilters(query, filters, table);
       query = applyAccessControl(table, query, context.user);
+      query = query.limit(limit);
       if (groupBy) {
         query = query.select(groupBy).groupBy(groupBy);
         if (tableNamePlural === "tracking") {
@@ -4489,11 +5355,12 @@ function createQueries(table, agents, tools, contexts) {
       if (!exists) {
         throw new Error("Context " + table.id + " not found in registry.");
       }
-      const { limit = 10, page = 0, filters = [], sort } = args;
+      const { limit = 10, page = 0, itemFilters = [], chunkFilters = [], sort } = args;
       return await vectorSearch({
         limit: limit || exists.configuration.maxRetrievalResults || 10,
         page,
-        filters,
+        itemFilters,
+        chunkFilters,
         sort,
         context: exists,
         db: context.db,
@@ -4552,11 +5419,13 @@ function createQueries(table, agents, tools, contexts) {
 var vectorSearch = async ({
   limit,
   page,
-  filters,
+  itemFilters,
+  chunkFilters,
   sort,
   context,
   db: db3,
   query,
+  keywords,
   method,
   user,
   role,
@@ -4568,7 +5437,8 @@ var vectorSearch = async ({
   console.log("[EXULU] Called vector search.", {
     limit,
     page,
-    filters,
+    itemFilters,
+    chunkFilters,
     sort,
     context: context.id,
     query,
@@ -4581,7 +5451,7 @@ var vectorSearch = async ({
   if (limit > 250) {
     throw new Error("Limit cannot be greater than 1000.");
   }
-  if (!query) {
+  if (!query && !keywords) {
     throw new Error("Query is required.");
   }
   if (!method) {
@@ -4624,43 +5494,68 @@ var vectorSearch = async ({
   chunksQuery.leftJoin(mainTable + " as items", function() {
     this.on("chunks.source", "=", "items.id");
   });
-  chunksQuery = applyFilters(chunksQuery, filters, table, "items");
+  chunksQuery = applyFilters(chunksQuery, itemFilters, table, "items");
+  chunksQuery = applyFilters(chunksQuery, chunkFilters, table, "chunks");
   chunksQuery = applyAccessControl(table, chunksQuery, user, "items");
-  chunksQuery = applySorting(chunksQuery, sort, "items");
-  if (queryRewriter) {
+  chunksQuery = applySorting2(chunksQuery, sort, "items");
+  if (queryRewriter && query) {
     query = await queryRewriter(query);
   }
-  const { chunks: queryChunks } = await embedder.generateFromQuery(
-    context.id,
-    query,
-    {
-      label: table.name.singular,
-      trigger
-    },
-    user?.id,
-    role
-  );
-  if (!queryChunks?.[0]?.vector) {
-    throw new Error("No vector generated for query.");
-  }
-  const vector = queryChunks[0].vector;
-  const vectorStr = `ARRAY[${vector.join(",")}]`;
-  const vectorExpr = `${vectorStr}::vector`;
-  const language = configuration.language || "english";
-  console.log("[EXULU] Vector search params:", { method, query, cutoffs });
-  let resultChunks = [];
-  switch (method) {
+  let vector = [];
+  let vectorStr = "";
+  let vectorExpr = "";
+  if (query) {
+    const { processed: stemmedQuery, language: language2 } = preprocessQuery(query, {
+      enableStemming: true,
+      detectLanguage: true
+    });
+    console.log("[EXULU] Stemmed query:", stemmedQuery);
+    if (stemmedQuery) {
+      query = stemmedQuery;
+    }
+    const result = await embedder.generateFromQuery(
+      context.id,
+      query,
+      {
+        label: table.name.singular,
+        trigger
+      },
+      user?.id,
+      role
+    );
+    if (!result?.chunks?.[0]?.vector) {
+      throw new Error("No vector generated for query.");
+    }
+    vector = result.chunks[0].vector;
+    vectorStr = `ARRAY[${vector.join(",")}]`;
+    vectorExpr = `${vectorStr}::vector`;
+  }
+  let keywordsQuery = [];
+  if (keywords) {
+    console.log("[EXULU] Using keywords:", keywords);
+    let tokens = keywords?.map((keyword) => keyword.trim()).filter((token) => token.length > 0);
+    const sanitized = tokens?.map((token) => {
+      return token.replace(/[^a-zA-Z0-9]/g, "");
+    }).filter((token) => token.length > 0);
+    keywordsQuery = [.../* @__PURE__ */ new Set([...sanitized, ...tokens])];
+  } else if (query) {
+    console.log("[EXULU] Extracting keywords from query:", query);
+    const tokens = query.trim().split(/\s+/).filter((t) => t.length > 0);
+    console.log("[EXULU] Query tokens:", tokens);
+    keywordsQuery = Array.from(new Set(tokens.flatMap((t) => {
+      return t.split(/[^\w]+/).filter((part) => part.length > 0);
+    })));
+  }
+  const language = configuration.language || "english";
+  console.log("[EXULU] Vector search params:", { method, query, cutoffs });
+  let resultChunks = [];
+  switch (method) {
     case "tsvector":
       chunksQuery.limit(limit * 2);
-      const tokens = query.trim().split(/\s+/).filter((t) => t.length > 0);
-      const sanitizedTokens = tokens.flatMap((t) => {
-        return t.split(/[^\w]+/).filter((part) => part.length > 0);
-      });
-      const orQuery = sanitizedTokens.join(" | ");
+      const orQuery = keywordsQuery.join(" | ");
       console.log("[EXULU] FTS query transformation:", {
         original: query,
-        tokens,
-        sanitizedTokens,
+        keywordsQuery,
         orQuery,
         cutoff: cutoffs?.tsvector
       });
@@ -4708,7 +5603,8 @@ var vectorSearch = async ({
         query,
         cutoffs?.tsvector || 0
       ]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
-      fullTextQuery = applyFilters(fullTextQuery, filters, table, "items");
+      fullTextQuery = applyFilters(fullTextQuery, itemFilters, table, "items");
+      fullTextQuery = applyFilters(fullTextQuery, chunkFilters, table, "chunks");
       fullTextQuery = applyAccessControl(table, fullTextQuery, user, "items");
       let semanticQuery = db3(chunksTable + " as chunks").select([
         "chunks.id",
@@ -4719,7 +5615,8 @@ var vectorSearch = async ({
       ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereNotNull("chunks.embedding").whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr})) >= ?`, [
         cutoffs?.cosineDistance || 0
       ]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
-      semanticQuery = applyFilters(semanticQuery, filters, table, "items");
+      semanticQuery = applyFilters(semanticQuery, itemFilters, table, "items");
+      semanticQuery = applyFilters(semanticQuery, chunkFilters, table, "chunks");
       semanticQuery = applyAccessControl(table, semanticQuery, user, "items");
       let hybridQuery = db3.with("full_text", fullTextQuery).with("semantic", semanticQuery).select([
         "items.id as item_id",
@@ -4939,8 +5836,10 @@ var vectorSearch = async ({
     role
   });
   return {
-    filters,
+    itemFilters,
+    chunkFilters,
     query,
+    keywords,
     method,
     context: {
       name: table.name.singular,
@@ -5042,7 +5941,7 @@ var contextToTableDefinition = (context) => {
   });
   return addCoreFields(definition);
 };
-function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
+function createSDL(tables, contexts, agents, tools, config, evals, queues2, rerankers) {
   const contextSchemas = contexts.map(
     (context) => contextToTableDefinition(context)
   );
@@ -5123,11 +6022,11 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
       ${tableNameSingular}ByIds(ids: [ID!]!): [${tableNameSingular}]!
       ${tableNamePlural}Pagination(limit: Int, page: Int, filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingularUpperCaseFirst}PaginationResult
       ${tableNameSingular}One(filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingular}
-      ${tableNamePlural}Statistics(filters: [Filter${tableNameSingularUpperCaseFirst}], groupBy: String): [StatisticsResult]!
+      ${tableNamePlural}Statistics(filters: [Filter${tableNameSingularUpperCaseFirst}], groupBy: String, limit: Int): [StatisticsResult]!
     `;
     if (table.type === "items") {
       typeDefs += `
-      ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
+      ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, itemFilters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
       ${tableNameSingular}ChunkById(id: ID!): ${tableNameSingular}VectorSearchChunk
     `;
     }
@@ -5197,7 +6096,8 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     type ${tableNameSingular}VectorSearchResult {
         chunks: [${tableNameSingular}VectorSearchChunk!]!
         context: VectoSearchResultContext!
-        filters: JSON!
+        itemFilters: JSON!
+        chunkFilters: JSON!
         query: String!
         method: VectorMethodEnum!
     }
@@ -5249,11 +6149,11 @@ type PageInfo {
 `;
     Object.assign(
       resolvers.Query,
-      createQueries(table, agents, tools, contexts)
+      createQueries(table, agents, tools, contexts, rerankers)
     );
     Object.assign(
       resolvers.Mutation,
-      createMutations(table, agents, contexts, tools, config)
+      createMutations(table, agents, contexts, rerankers, tools, config)
     );
     if (table.RBAC) {
       const rbacResolverName = table.name.singular;
@@ -5284,6 +6184,9 @@ type PageInfo {
   typeDefs += `
     contexts: ContextPaginationResult
     `;
+  typeDefs += `
+    rerankers: RerankerPaginationResult
+    `;
   typeDefs += `
     contextById(id: ID!): Context
     `;
@@ -5291,7 +6194,7 @@ type PageInfo {
     getUniquePromptTags: [String!]!
     `;
   mutationDefs += `
-    runEval(id: ID!, cases: [ID!]): RunEvalReturnPayload
+    runEval(id: ID!, test_case_ids: [ID!]): RunEvalReturnPayload
     `;
   mutationDefs += `
     runWorkflow(id: ID!, variables: JSON): RunWorkflowReturnPayload
@@ -5628,6 +6531,7 @@ type PageInfo {
                 agentBackend: agentBackend2,
                 inputMessages,
                 contexts,
+                rerankers,
                 user: user2,
                 tools,
                 config,
@@ -5889,6 +6793,18 @@ type PageInfo {
       }
     };
   };
+  resolvers.Query["rerankers"] = async (_, args, context, info) => {
+    const requestedFields = getRequestedFields(info);
+    return {
+      items: rerankers.map((reranker) => {
+        const object = {};
+        requestedFields.forEach((field) => {
+          object[field] = reranker[field];
+        });
+        return object;
+      })
+    };
+  };
   resolvers.Query["contexts"] = async (_, args, context, info) => {
     const data = await Promise.all(
       contexts.map(async (context2) => {
@@ -5941,13 +6857,23 @@ type PageInfo {
           active: context2.active,
           sources,
           processor,
-          fields: context2.fields.map((field) => {
-            return {
-              ...field,
-              name: sanitizeName(field.name),
-              label: field.name?.replace("_s3key", "")
-            };
-          })
+          fields: await Promise.all(
+            context2.fields.map(async (field) => {
+              const label = field.name?.replace("_s3key", "");
+              if (field.type === "file" && !field.name.endsWith("_s3key")) {
+                field.name = field.name + "_s3key";
+              }
+              return {
+                ...field,
+                name: sanitizeName(field.name),
+                editable: field.editable,
+                ...field.type === "file" ? {
+                  allowedFileTypes: field.allowedFileTypes
+                } : {},
+                label: field.name?.replace("_s3key", "")
+              };
+            })
+          )
         };
       })
     );
@@ -6032,6 +6958,7 @@ type PageInfo {
           return {
             ...field,
             name: sanitizeName(field.name),
+            editable: field.editable,
             ...field.type === "file" ? {
               allowedFileTypes: field.allowedFileTypes
             } : {},
@@ -6060,31 +6987,45 @@ type PageInfo {
         if (!backend) {
           return null;
         }
-        return await backend.tool(instance.id, agents);
+        return await backend.tool(instance.id, agents, contexts, rerankers);
       })
     );
+    let agenticRetrievalTool = void 0;
     const filtered = agentTools.filter(
-      (tool2) => tool2 !== null
+      (tool3) => tool3 !== null
     );
     let allTools = [...filtered, ...tools];
+    if (contexts?.length) {
+      agenticRetrievalTool = createAgenticRetrievalTool({
+        contexts,
+        rerankers,
+        user: context.user,
+        role: context.user?.role?.id,
+        model: void 0
+        // irrelevant at this point as we only retrieve the tool information here, not execute it
+      });
+      if (agenticRetrievalTool) {
+        allTools.push(agenticRetrievalTool);
+      }
+    }
     if (search && search.trim()) {
       const searchTerm = search.toLowerCase().trim();
       allTools = allTools.filter(
-        (tool2) => tool2.name?.toLowerCase().includes(searchTerm) || tool2.description?.toLowerCase().includes(searchTerm)
+        (tool3) => tool3.name?.toLowerCase().includes(searchTerm) || tool3.description?.toLowerCase().includes(searchTerm)
       );
     }
     if (category && category.trim()) {
-      allTools = allTools.filter((tool2) => tool2.category === category);
+      allTools = allTools.filter((tool3) => tool3.category === category);
     }
     const total = allTools.length;
     const start = page * limit;
     const end = start + limit;
     const paginatedTools = allTools.slice(start, end);
     return {
-      items: paginatedTools.map((tool2) => {
+      items: paginatedTools.map((tool3) => {
         const object = {};
         requestedFields.forEach((field) => {
-          object[field] = tool2[field];
+          object[field] = tool3[field];
         });
         return object;
       }),
@@ -6094,7 +7035,7 @@ type PageInfo {
     };
   };
   resolvers.Query["toolCategories"] = async () => {
-    const array = tools.map((tool2) => tool2.category).filter((category) => category && typeof category === "string");
+    const array = tools.map((tool3) => tool3.category).filter((category) => category && typeof category === "string");
     array.push("contexts");
     array.push("agents");
     return [...new Set(array)].sort();
@@ -6181,6 +7122,11 @@ type PageInfo {
     items: [Context]!
     }
     `;
+  modelDefs += `
+    type RerankerPaginationResult {
+    items: [Reranker]!
+    }
+    `;
   modelDefs += `
     type ToolPaginationResult {
     items: [Tool]!
@@ -6282,6 +7228,11 @@ type Context {
     sources: [ContextSource]
     processor: ContextProcessor
 }
+type Reranker {
+    id: ID!
+    name: String!
+    description: String
+}
 type Embedder {
     name: String!
     id: ID!
@@ -6693,11 +7644,14 @@ var createUppyRoutes = async (app, contexts, config) => {
       return;
     }
     let bucket = key.split("/")[0];
+    console.log("[EXULU] bucket", bucket);
     if (!bucket || typeof bucket !== "string" || bucket.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `bucket` (should be the first part of the key before the first slash)." });
       return;
     }
+    console.log("[EXULU] key for download before split", key);
     key = key.split("/").slice(1).join("/");
+    console.log("[EXULU] key for download after split", key);
     if (typeof key !== "string" || key.trim() === "") {
       res.status(400).json({ error: "Missing or invalid `key` query parameter." });
       return;
@@ -7088,53 +8042,135 @@ var createUppyRoutes = async (app, contexts, config) => {
 
 // src/registry/classes.ts
 var import_officeparser = require("officeparser");
+var import_fs2 = require("fs");
 var s3Client2;
 function sanitizeToolName(name) {
   if (typeof name !== "string") return "";
-  let sanitized = name.replace(/[^a-zA-Z0-9_-]+/g, "_");
+  let sanitized = name.replace(/[^a-zA-Z0-9_.\:-]+/g, "_");
+  if (sanitized.length > 0 && !/^[a-zA-Z_]/.test(sanitized)) {
+    sanitized = "_" + sanitized;
+  }
   sanitized = sanitized.replace(/^_+|_+$/g, "");
-  if (sanitized.length > 128) {
-    sanitized = sanitized.substring(0, 128);
+  if (!sanitized) {
+    sanitized = "tool";
+  }
+  if (sanitized.length > 64) {
+    sanitized = sanitized.substring(0, 64);
+  }
+  if (!/^[a-zA-Z_]/.test(sanitized)) {
+    sanitized = "_" + sanitized.substring(1);
   }
   return sanitized;
 }
-var projectsCache = /* @__PURE__ */ new Map();
-var createProjectRetrievalTool = async ({
+var createNewMemoryItemTool = (agent, context) => {
+  const fields = {
+    name: import_zod2.z.string().describe("The name of the item to create"),
+    description: import_zod2.z.string().describe("The description of the item to create")
+  };
+  for (const field of context.fields) {
+    switch (field.type) {
+      case "text":
+      case "longText":
+      case "shortText":
+      case "code":
+      case "enum":
+        fields[field.name] = import_zod2.z.string().describe("The " + field.name + " of the item to create");
+        break;
+      case "json":
+        fields[field.name] = import_zod2.z.string({}).describe("The " + field.name + " of the item to create, it should be a valid JSON string.");
+        break;
+      case "markdown":
+        fields[field.name] = import_zod2.z.string().describe("The " + field.name + " of the item to create, it should be a valid Markdown string.");
+        break;
+      case "number":
+        fields[field.name] = import_zod2.z.number().describe("The " + field.name + " of the item to create");
+        break;
+      case "boolean":
+        fields[field.name] = import_zod2.z.boolean().describe("The " + field.name + " of the item to create");
+        break;
+      case "file":
+      case "uuid":
+      case "date":
+        break;
+      default:
+        fields[field.name] = import_zod2.z.string().describe("The " + field.name + " of the item to create");
+        break;
+    }
+  }
+  return new ExuluTool({
+    id: "create_" + agent.name + "_memory_item",
+    name: "Create " + agent.name + " Memory Item",
+    category: agent.name + "_memory",
+    description: "Create a new memory item in the " + agent.name + " memory context",
+    type: "function",
+    inputSchema: import_zod2.z.object(fields),
+    config: [],
+    execute: async ({ name, description, mode, information, exuluConfig, user }) => {
+      let result = { result: "" };
+      switch (mode) {
+        case "learnings":
+          break;
+        case "knowledge":
+          const newItem = {
+            name,
+            description,
+            information,
+            rights_mode: "public"
+          };
+          const { item: createdItem, job: createdJob } = await context.createItem(
+            newItem,
+            exuluConfig,
+            user?.id,
+            user?.role?.id,
+            false
+          );
+          if (createdJob) {
+            result = {
+              result: `Created a Job to create the memory item with the following ID: ${createdJob}`
+            };
+          } else if (createdItem) {
+            result = {
+              result: `Created memory item with the following ID: ${createdItem.id}`
+            };
+          } else {
+            result = {
+              result: `Failed to create memory item`
+            };
+          }
+          break;
+        default:
+          throw new Error(`Invalid mode: ${mode}`);
+      }
+      return result;
+    }
+  });
+};
+var createProjectItemsRetrievalTool = async ({
   user,
   role,
   contexts,
   projectId
 }) => {
   let project;
-  const cachedProject = projectsCache.get(projectId);
   const OneMinuteAgo = new Date(Date.now() - 1e3 * 60);
-  if (cachedProject && cachedProject.age > OneMinuteAgo) {
-    project = cachedProject.project;
-  } else {
-    const { db: db3 } = await postgresClient();
-    project = await db3.from("projects").where("id", projectId).first();
-    if (project) {
-      projectsCache.set(projectId, {
-        age: /* @__PURE__ */ new Date(),
-        project
-      });
-    } else {
-      return;
-    }
+  const { db: db3 } = await postgresClient();
+  project = await db3.from("projects").where("id", projectId).first();
+  if (!project) {
+    return;
   }
   console.log("[EXULU] Project search tool created for project", project);
   if (!project.project_items?.length) {
     return;
   }
-  const projectRetrievalTool = new ExuluTool3({
-    id: "project_information_retrieval_tool_" + projectId,
-    name: "Project information retrieval tool for project " + project.name,
+  const projectRetrievalTool = new ExuluTool({
+    id: "context_search_in_knowledge_items_added_to_project_" + projectId,
+    name: "context_search in knowledge items added to project " + project.name,
     description: "This tool retrieves information about a project from conversations and items that were added to the project " + project.name + ".",
-    inputSchema: import_zod.z.object({
-      query: import_zod.z.string().describe("The query to retrieve information about the project " + project.name + "."),
-      keywords: import_zod.z.array(import_zod.z.string()).describe("The most relevant keywords in the query, such as names of people, companies, products, etc. in the project " + project.name + ".")
+    inputSchema: import_zod2.z.object({
+      query: import_zod2.z.string().describe("The query to retrieve information about the project " + project.name + "."),
+      keywords: import_zod2.z.array(import_zod2.z.string()).describe("The most relevant keywords in the query, such as names of people, companies, products, etc. in the project " + project.name + ".")
     }),
-    type: "function",
+    type: "context",
     category: "project",
     config: [],
     execute: async ({ query, keywords }) => {
@@ -7162,16 +8198,17 @@ var createProjectRetrievalTool = async ({
         }
         const itemIds = set[contextName];
         console.log("[EXULU] Project search tool searching through items", itemIds);
-        return await context.search({
+        const result = await context.search({
           // todo check if it is more performant to use a concatenation of
           // the query and keywords, or just the keywords, instead of the 
           // query itself.
           query,
-          filters: [{
+          itemFilters: [{
             id: {
               in: itemIds
             }
           }],
+          chunkFilters: [],
           user,
           role,
           method: "hybridSearch",
@@ -7183,6 +8220,15 @@ var createProjectRetrievalTool = async ({
           limit: 10,
           page: 1
         });
+        return {
+          result: result.chunks.map((chunk) => ({
+            ...chunk,
+            context: {
+              name: context.name,
+              id: context.id
+            }
+          }))
+        };
       }));
       console.log("[EXULU] Project search tool results", results);
       return {
@@ -7192,7 +8238,79 @@ var createProjectRetrievalTool = async ({
   });
   return projectRetrievalTool;
 };
-var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, providerapikey, contexts, user, exuluConfig, sessionID, req, project) => {
+var createSessionItemsRetrievalTool = async ({
+  user,
+  role,
+  contexts,
+  items
+}) => {
+  console.log("[EXULU] Session search tool created for session", items);
+  const sessionItemsRetrievalTool = new ExuluTool({
+    id: "session_items_information_context_search",
+    name: "context_search in knowledge items added to session.",
+    description: "Context search in knowledge items added to session.",
+    inputSchema: import_zod2.z.object({
+      query: import_zod2.z.string().describe("The query to retrieve information from knowledge items added to the session.")
+    }),
+    type: "context",
+    category: "session",
+    config: [],
+    execute: async ({ query, keywords }) => {
+      console.log("[EXULU] Session search tool searching for session items", items);
+      const set = {};
+      for (const item of items) {
+        const context = item.split("/")[0];
+        if (!context) {
+          throw new Error("The item added to the project does not have a valid gid with the context id as the prefix before the first slash.");
+        }
+        const id = item.split("/").slice(1).join("/");
+        if (set[context]) {
+          set[context].push(id);
+        } else {
+          set[context] = [id];
+        }
+      }
+      console.log("[EXULU] Session search tool searching through contexts", Object.keys(set));
+      const results = await Promise.all(Object.keys(set).map(async (contextName, index) => {
+        const context = contexts.find((context2) => context2.id === contextName);
+        if (!context) {
+          console.error("[EXULU] Context not found for project information retrieval tool.", contextName);
+          return [];
+        }
+        const itemIds = set[contextName];
+        console.log("[EXULU] Session search tool searching through items", itemIds);
+        return await context.search({
+          // todo check if it is more performant to use a concatenation of
+          // the query and keywords, or just the keywords, instead of the 
+          // query itself.
+          query,
+          itemFilters: [{
+            id: {
+              in: itemIds
+            }
+          }],
+          chunkFilters: [],
+          user,
+          role,
+          method: "hybridSearch",
+          sort: {
+            field: "updatedAt",
+            direction: "desc"
+          },
+          trigger: "tool",
+          limit: 10,
+          page: 1
+        });
+      }));
+      console.log("[EXULU] Session search tool results", results);
+      return {
+        result: JSON.stringify(results.flat())
+      };
+    }
+  });
+  return sessionItemsRetrievalTool;
+};
+var convertToolsArrayToObject = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project, items, model, agentInstance) => {
   if (!currentTools) return {};
   if (!allExuluTools) {
     allExuluTools = [];
@@ -7201,8 +8319,9 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
   if (!contexts) {
     contexts = [];
   }
+  let projectRetrievalTool;
   if (project) {
-    const projectRetrievalTool = await createProjectRetrievalTool({
+    projectRetrievalTool = await createProjectItemsRetrievalTool({
       user,
       role: user?.role?.id,
       contexts,
@@ -7212,11 +8331,54 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
       currentTools.push(projectRetrievalTool);
     }
   }
-  const sanitizedTools = currentTools ? currentTools.map((tool2) => ({
-    ...tool2,
-    name: sanitizeToolName(tool2.name)
+  console.log("[EXULU] Convert tools array to object, session items", items);
+  if (items) {
+    const sessionItemsRetrievalTool = await createSessionItemsRetrievalTool({
+      user,
+      role: user?.role?.id,
+      contexts,
+      items
+    });
+    if (sessionItemsRetrievalTool) {
+      currentTools.push(sessionItemsRetrievalTool);
+    }
+  }
+  console.log("[EXULU] Creating agentic search tool", contexts?.length, model);
+  if (contexts?.length && model) {
+    const agenticSearchTool = createAgenticRetrievalTool({
+      contexts: contexts.filter((context) => context.id !== agentInstance?.memory),
+      // dont include the agents memory in the agentic search tool!
+      rerankers: rerankers || [],
+      user,
+      role: user?.role?.id,
+      model,
+      projectRetrievalTool
+    });
+    if (!agenticSearchTool) {
+      throw new Error("Agentic search tool could not be instantiated, this is an internal error in the Exulu framework, please contact support.");
+    }
+    if (agenticSearchTool) {
+      const index = currentTools.findIndex((tool3) => tool3.id === "agentic_context_search");
+      if (index !== -1) {
+        currentTools[index] = {
+          ...currentTools[index],
+          // important to keep the original tool config
+          ...agenticSearchTool
+        };
+      }
+    }
+  } else {
+    const agenticSearchTool = currentTools.find((tool3) => tool3.id === "agentic_context_search");
+    if (agenticSearchTool) {
+      currentTools.splice(currentTools.indexOf(agenticSearchTool), 1);
+    }
+  }
+  const sanitizedTools = currentTools ? currentTools.map((tool3) => ({
+    ...tool3,
+    name: sanitizeToolName(tool3.name)
   })) : [];
   console.log("[EXULU] Sanitized tools", sanitizedTools.map((x) => x.name + " (" + x.id + ")"));
+  console.log("[EXULU] Approved tools", approvedTools);
   return {
     ...sanitizedTools?.reduce(
       (prev, cur) => {
@@ -7225,11 +8387,16 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
         const defaultConfigDescription = toolVariableConfig?.config.find((config) => config.name === "description")?.default;
         const toolDescription = cur.description;
         const description = userDefinedConfigDescription || defaultConfigDescription || toolDescription;
+        console.log("[EXULU] Tool", cur.name, "needs approval", approvedTools?.includes(cur.name) ? false : true);
         return {
           ...prev,
           [cur.name]: {
             ...cur.tool,
             description,
+            // The approvedTools array uses the tool.name lookup as the frontend
+            // Vercel AI SDK uses the sanitized tool name as the key, so this matches.
+            needsApproval: approvedTools?.includes("tool-" + cur.name) ? false : true,
+            // todo make configurable
             async *execute(inputs, options) {
               console.log("[EXULU] Executing tool", cur.name, "with inputs", inputs, "and options", options);
               if (!cur.tool?.execute) {
@@ -7290,11 +8457,16 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
                 acc[curr.id] = curr;
                 return acc;
               }, {});
+              const toolVariablesConfigData = toolVariableConfig ? toolVariableConfig.config.reduce((acc, curr) => {
+                acc[curr.name] = curr.value;
+                return acc;
+              }, {}) : {};
               const response = await cur.tool.execute({
                 ...inputs,
+                model,
                 sessionID,
                 req,
-                // Convert config to object format if a config object 
+                // Convert config to object format if a config object
                 // is available, after we added the .value property
                 // by hydrating it from the variables table.
                 providerapikey,
@@ -7304,10 +8476,7 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
                 contexts: contextsMap,
                 upload,
                 exuluConfig,
-                toolVariablesConfig: toolVariableConfig ? toolVariableConfig.config.reduce((acc, curr) => {
-                  acc[curr.name] = curr.value;
-                  return acc;
-                }, {}) : {}
+                toolVariablesConfig: toolVariablesConfigData
               }, options);
               await updateStatistic({
                 name: "count",
@@ -7318,8 +8487,17 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
                 user: user?.id,
                 role: user?.role?.id
               });
-              yield response;
-              return response;
+              if (response && typeof response === "object" && Symbol.asyncIterator in response) {
+                let lastValue;
+                for await (const value of response) {
+                  yield value;
+                  lastValue = value;
+                }
+                return lastValue;
+              } else {
+                yield response;
+                return response;
+              }
             }
           }
         };
@@ -7329,16 +8507,27 @@ var convertToolsArrayToObject = async (currentTools, allExuluTools, configs, pro
     // askForConfirmation
   };
 };
-var hydrateVariables = async (tool2) => {
+var hydrateVariables = async (tool3) => {
   const { db: db3 } = await postgresClient();
-  const promises = tool2.config.map(async (toolConfig) => {
+  const promises = tool3.config.map(async (toolConfig) => {
     if (!toolConfig.variable) {
       return toolConfig;
     }
     const variableName = toolConfig.variable;
+    const type = toolConfig.type;
+    if (type === "boolean") {
+      toolConfig.value = toolConfig.variable === "true" || toolConfig.variable === true || toolConfig.variable === 1;
+      return toolConfig;
+    } else if (type === "number") {
+      toolConfig.value = parseInt(toolConfig.variable.toString());
+      return toolConfig;
+    } else if (type === "string") {
+      toolConfig.value = toolConfig.variable;
+      return toolConfig;
+    }
     const variable = await db3.from("variables").where({ name: variableName }).first();
     if (!variable) {
-      throw new Error("Variable " + variableName + " not found.");
+      throw new Error("Variable " + variableName + " not found in hydrateVariables method, with type " + type + ".");
     }
     let value = variable.value;
     if (variable.encrypted) {
@@ -7349,7 +8538,7 @@ var hydrateVariables = async (tool2) => {
     return toolConfig;
   });
   await Promise.all(promises);
-  return tool2;
+  return tool3;
 };
 function generateSlug(name) {
   const normalized = name.normalize("NFKD").replace(/[\u0300-\u036f]/g, "");
@@ -7357,6 +8546,21 @@ function generateSlug(name) {
   const slug = lowercase.replace(/[\W_]+/g, "-").replace(/^-+|-+$/g, "");
   return slug;
 }
+var ExuluReranker4 = class {
+  id;
+  name;
+  description;
+  execute;
+  constructor({ id, name, description, execute: execute2 }) {
+    this.id = id;
+    this.name = name;
+    this.description = description;
+    this.execute = execute2;
+  }
+  async run(query, chunks) {
+    return await this.execute({ query, chunks });
+  }
+};
 function errorHandler(error) {
   if (error == null) {
     return "unknown error";
@@ -7467,19 +8671,19 @@ var ExuluAgent3 = class {
     return this.config?.name || "";
   }
   // Exports the agent as a tool that can be used by another agent
-  tool = async (instance, agents) => {
+  tool = async (instance, agents, contexts, rerankers) => {
     const agentInstance = await loadAgent(instance);
     if (!agentInstance) {
       return null;
     }
-    return new ExuluTool3({
+    return new ExuluTool({
       id: agentInstance.id,
       name: `${agentInstance.name}`,
       type: "agent",
       category: "agents",
-      inputSchema: import_zod.z.object({
-        prompt: import_zod.z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
-        information: import_zod.z.string().describe("A summary of relevant context / information from the current session")
+      inputSchema: import_zod2.z.object({
+        prompt: import_zod2.z.string().describe("The prompt (usually a question for the agent) to send to the agent."),
+        information: import_zod2.z.string().describe("A summary of relevant context / information from the current session")
       }),
       description: `This tool calls an AI agent named: ${agentInstance.name}. The agent does the following: ${agentInstance.description}.`,
       config: [],
@@ -7488,7 +8692,7 @@ var ExuluAgent3 = class {
         if (!hasAccessToAgent) {
           throw new Error("You don't have access to this agent.");
         }
-        let enabledTools = await getEnabledTools(agentInstance, allExuluTools, [], agents, user);
+        let enabledTools = await getEnabledTools(agentInstance, allExuluTools, contexts, rerankers, [], agents, user);
         const variableName = agentInstance.providerapikey;
         let providerapikey;
         if (variableName) {
@@ -7510,6 +8714,9 @@ var ExuluAgent3 = class {
         console.log("[EXULU] Prompt for agent '" + agentInstance.name + "' that is being called as a tool", prompt.slice(0, 100) + "...");
         console.log("[EXULU] Instructions for agent '" + agentInstance.name + "' that is being called as a tool", agentInstance.instructions?.slice(0, 100) + "...");
         const response = await this.generateSync({
+          agentInstance,
+          contexts,
+          rerankers,
           instructions: agentInstance.instructions,
           prompt: "The user has asked the following question: " + prompt + " and the following information is available: " + information,
           providerapikey,
@@ -7548,8 +8755,10 @@ var ExuluAgent3 = class {
     toolConfigs,
     providerapikey,
     contexts,
+    rerankers,
     exuluConfig,
     outputSchema,
+    agentInstance,
     instructions
   }) => {
     console.log("[EXULU] Called generate sync for agent: " + this.name, "with prompt: " + prompt?.slice(0, 100) + "...");
@@ -7581,17 +8790,54 @@ var ExuluAgent3 = class {
         page: 1
       });
       const previousMessagesContent = previousMessages.map((message) => JSON.parse(message.content));
-      messages = await (0, import_ai2.validateUIMessages)({
+      messages = await (0, import_ai3.validateUIMessages)({
         // append the new message to the previous messages:
         messages: [...previousMessagesContent, ...messages]
       });
     }
     console.log("[EXULU] Message count for agent: " + this.name, "loaded for generating sync.", messages.length);
     let project;
+    let sessionItems;
     if (session) {
       const sessionData = await getSession({ sessionID: session });
+      sessionItems = sessionData.session_items;
       project = sessionData.project;
     }
+    const query = prompt;
+    let memoryContext = "";
+    if (agentInstance?.memory && contexts?.length && query) {
+      const context = contexts.find((context2) => context2.id === agentInstance?.memory);
+      if (!context) {
+        throw new Error("Context was set for agent memory but not found in the contexts: " + agentInstance?.memory + " please double check with a developer to see if the context was removed from code.");
+      }
+      const result = await context?.search({
+        query,
+        itemFilters: [],
+        chunkFilters: [],
+        method: "hybridSearch",
+        sort: {
+          field: "updatedAt",
+          direction: "desc"
+        },
+        trigger: "agent",
+        limit: 10,
+        // todo make this configurable?
+        page: 1
+      });
+      if (result?.chunks?.length) {
+        memoryContext = `
+                Pre-fetched relevant information for this query:
+
+                ${result.chunks.map((chunk) => chunk.chunk_content).join("\n\n")}`;
+      }
+      const createNewMemoryTool = createNewMemoryItemTool(agentInstance, context);
+      if (createNewMemoryTool) {
+        if (!currentTools) {
+          currentTools = [];
+        }
+        currentTools.push(createNewMemoryTool);
+      }
+    }
     const personalizationInformation = exuluConfig?.privacy?.systemPromptPersonalization !== false ? `
                 ${user?.firstname ? `The users first name is "${user.firstname}"` : ""}
                 ${user?.lastname ? `The users last name is "${user.lastname}"` : ""}
@@ -7604,8 +8850,16 @@ var ExuluAgent3 = class {
                 they are talking with the current date in mind as a reference.`;
     let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
     system += "\n\n" + genericContext;
-    const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
-    console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
+    if (memoryContext) {
+      system += "\n\n" + memoryContext;
+    }
+    const includesContextSearchTool = currentTools?.some(
+      (tool3) => tool3.name.toLowerCase().includes("context_search") || tool3.id.includes("context_search") || tool3.type === "context"
+    );
+    const includesWebSearchTool = currentTools?.some(
+      (tool3) => tool3.name.toLowerCase().includes("web_search") || tool3.id.includes("web_search") || tool3.type === "web_search"
+    );
+    console.log("[EXULU] Current tools: " + currentTools?.map((tool3) => tool3.name));
     console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
     if (includesContextSearchTool) {
       system += `
@@ -7613,14 +8867,14 @@ var ExuluAgent3 = class {
 
 
             When you use a context search tool, you will include references to the items
-            retrieved from the context search tool inline in the response using this exact JSON format
+            retrieved from the tool call result inline in the response using this exact JSON format
             (all on one line, no line breaks):
             {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
 
             IMPORTANT formatting rules:
             - Use the exact format shown above, all on ONE line
             - Do NOT use quotes around field names or values
-            - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
+            - Use the context ID from the tool result
             - Include the file/item name, not the full path
             - Separate multiple citations with spaces
 
@@ -7629,12 +8883,28 @@ var ExuluAgent3 = class {
             The citations will be rendered as interactive badges in the UI.
             `;
     }
+    if (includesWebSearchTool) {
+      system += `
+
+
+            When you use a web search tool, you will include references to the results of the tool call result inline in the response using this exact JSON format
+            (all on one line, no line breaks):
+            {url: <url>, title: <title>, snippet: <snippet>}
+
+            IMPORTANT formatting rules:
+            - Use the exact format shown above, all on ONE line
+            - Do NOT use quotes around field names or values
+            - Separate multiple results with spaces
+
+            Example: {url: https://www.google.com, title: Google, snippet: The result of the web search.}
+            `;
+    }
     if (prompt) {
       let result = { object: null, text: "" };
       let inputTokens = 0;
       let outputTokens = 0;
       if (outputSchema) {
-        const { object, usage } = await (0, import_ai2.generateObject)({
+        const { object, usage } = await (0, import_ai3.generateObject)({
           model,
           system,
           prompt,
@@ -7646,24 +8916,29 @@ var ExuluAgent3 = class {
         outputTokens = usage.outputTokens || 0;
       } else {
         console.log("[EXULU] Generating text for agent: " + this.name, "with prompt: " + prompt?.slice(0, 100) + "...");
-        const { text, totalUsage } = await (0, import_ai2.generateText)({
+        const { text, totalUsage } = await (0, import_ai3.generateText)({
           model,
           system,
           prompt,
           maxRetries: 2,
           tools: await convertToolsArrayToObject(
             currentTools,
+            [],
             allExuluTools,
             toolConfigs,
             providerapikey,
             contexts,
+            rerankers,
             user,
             exuluConfig,
             session,
             req,
-            project
+            project,
+            sessionItems,
+            model,
+            agentInstance
           ),
-          stopWhen: [(0, import_ai2.stepCountIs)(5)]
+          stopWhen: [(0, import_ai3.stepCountIs)(5)]
         });
         result.text = text;
         inputTokens = totalUsage?.inputTokens || 0;
@@ -7704,27 +8979,32 @@ var ExuluAgent3 = class {
     }
     if (messages) {
       console.log("[EXULU] Generating text for agent: " + this.name, "with messages: " + messages.length);
-      const { text, totalUsage } = await (0, import_ai2.generateText)({
+      const { text, totalUsage } = await (0, import_ai3.generateText)({
         model,
         // Should be a LanguageModelV1
         system,
-        messages: (0, import_ai2.convertToModelMessages)(messages, {
+        messages: await (0, import_ai3.convertToModelMessages)(messages, {
           ignoreIncompleteToolCalls: true
         }),
         maxRetries: 2,
         tools: await convertToolsArrayToObject(
           currentTools,
+          [],
           allExuluTools,
           toolConfigs,
           providerapikey,
           contexts,
+          rerankers,
           user,
           exuluConfig,
           session,
           req,
-          project
+          project,
+          sessionItems,
+          model,
+          agentInstance
         ),
-        stopWhen: [(0, import_ai2.stepCountIs)(5)]
+        stopWhen: [(0, import_ai3.stepCountIs)(5)]
       });
       if (statistics) {
         await Promise.all([
@@ -7835,13 +9115,16 @@ ${extractedText}
   generateStream = async ({
     user,
     session,
+    agentInstance,
     message,
     previousMessages,
     currentTools,
+    approvedTools,
     allExuluTools,
     toolConfigs,
     providerapikey,
     contexts,
+    rerankers,
     exuluConfig,
     instructions,
     req
@@ -7864,9 +9147,11 @@ ${extractedText}
     let messages = [];
     let previousMessagesContent = previousMessages || [];
     let project;
+    let sessionItems;
     if (session) {
       const sessionData = await getSession({ sessionID: session });
       project = sessionData.project;
+      sessionItems = sessionData.session_items;
       console.log("[EXULU] loading previous messages from session: " + session);
       const previousMessages2 = await getAgentMessages({
         session,
@@ -7878,34 +9163,78 @@ ${extractedText}
         (message2) => JSON.parse(message2.content)
       );
     }
-    messages = await (0, import_ai2.validateUIMessages)({
+    messages = await (0, import_ai3.validateUIMessages)({
       // append the new message to the previous messages:
       messages: [...previousMessagesContent, message]
     });
+    const query = message.parts?.[0]?.type === "text" ? message.parts[0].text : void 0;
+    let memoryContext = "";
+    if (agentInstance?.memory && contexts?.length && query) {
+      const context = contexts.find((context2) => context2.id === agentInstance?.memory);
+      if (!context) {
+        throw new Error("Context was set for agent memory but not found in the contexts: " + agentInstance?.memory + " please double check with a developer to see if the context was removed from code.");
+      }
+      const result2 = await context?.search({
+        query,
+        itemFilters: [],
+        chunkFilters: [],
+        method: "hybridSearch",
+        sort: {
+          field: "updatedAt",
+          direction: "desc"
+        },
+        trigger: "agent",
+        limit: 10,
+        // todo make this configurable?
+        page: 1
+      });
+      if (result2?.chunks?.length) {
+        memoryContext = `
+                Pre-fetched relevant information for this query:
+
+                ${result2.chunks.map((chunk) => chunk.chunk_content).join("\n\n")}`;
+      }
+      const createNewMemoryTool = createNewMemoryItemTool(agentInstance, context);
+      if (createNewMemoryTool) {
+        if (!currentTools) {
+          currentTools = [];
+        }
+        currentTools.push(createNewMemoryTool);
+      }
+    }
     messages = messages.filter(
-      (message2, index, self) => index === self.findIndex((t) => t.id === message2.id)
+      (message2, index, self) => index === self.findLastIndex((t) => t.id === message2.id)
     );
     messages = await this.processFilePartsInMessages(messages);
     const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
     let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
     system += "\n\n" + genericContext;
-    const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
-    console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
+    if (memoryContext) {
+      system += "\n\n" + memoryContext;
+    }
+    const includesContextSearchTool = currentTools?.some(
+      (tool3) => tool3.name.toLowerCase().includes("context_search") || tool3.id.includes("context_search") || tool3.type === "context"
+    );
+    const includesWebSearchTool = currentTools?.some(
+      (tool3) => tool3.name.toLowerCase().includes("web_search") || tool3.id.includes("web_search") || tool3.type === "web_search"
+    );
+    console.log("[EXULU] Current tools: " + currentTools?.map((tool3) => tool3.name));
     console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
+    console.log("[EXULU] Includes web search tool: " + includesWebSearchTool);
     if (includesContextSearchTool) {
       system += `
 
 
 
             When you use a context search tool, you will include references to the items
-            retrieved from the context search tool inline in the response using this exact JSON format
+            retrieved from the tool call result inline in the response using this exact JSON format
             (all on one line, no line breaks):
             {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
 
             IMPORTANT formatting rules:
             - Use the exact format shown above, all on ONE line
             - Do NOT use quotes around field names or values
-            - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
+            - Use the context ID from the tool result
             - Include the file/item name, not the full path
             - Separate multiple citations with spaces
 
@@ -7914,10 +9243,26 @@ ${extractedText}
             The citations will be rendered as interactive badges in the UI.
             `;
     }
-    const result = (0, import_ai2.streamText)({
+    if (includesWebSearchTool) {
+      system += `
+
+
+            When you use a web search tool, you will include references to the results of the tool call result inline in the response using this exact JSON format
+            (all on one line, no line breaks):
+            {url: <url>, title: <title>, snippet: <snippet>}
+
+            IMPORTANT formatting rules:
+            - Use the exact format shown above, all on ONE line
+            - Do NOT use quotes around field names or values
+            - Separate multiple results with spaces
+
+            Example: {url: https://www.google.com, title: Google, snippet: The result of the web search.}
+            `;
+    }
+    const result = (0, import_ai3.streamText)({
       model,
       // Should be a LanguageModelV1
-      messages: (0, import_ai2.convertToModelMessages)(messages, {
+      messages: await (0, import_ai3.convertToModelMessages)(messages, {
         ignoreIncompleteToolCalls: true
       }),
       // PrepareStep could be used here to set the model 
@@ -7931,21 +9276,26 @@ ${extractedText}
       },
       tools: await convertToolsArrayToObject(
         currentTools,
+        approvedTools,
         allExuluTools,
         toolConfigs,
         providerapikey,
         contexts,
+        rerankers,
         user,
         exuluConfig,
         session,
         req,
-        project
+        project,
+        sessionItems,
+        model,
+        agentInstance
       ),
       onError: (error) => {
         console.error("[EXULU] chat stream error.", error);
         throw new Error(`Chat stream error: ${error instanceof Error ? error.message : String(error)}`);
       },
-      stopWhen: [(0, import_ai2.stepCountIs)(5)]
+      stopWhen: [(0, import_ai3.stepCountIs)(5)]
     });
     return {
       stream: result,
@@ -8122,7 +9472,7 @@ var ExuluEmbedder = class {
     );
   }
 };
-var ExuluTool3 = class {
+var ExuluTool = class {
   // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or 
   // underscores and be a max length of 80 characters and at least 5 characters long.
   // The ID is used for storing references to tools so it is important it does not change.
@@ -8142,9 +9492,9 @@ var ExuluTool3 = class {
     this.description = description;
     this.inputSchema = inputSchema;
     this.type = type;
-    this.tool = (0, import_ai2.tool)({
+    this.tool = (0, import_ai3.tool)({
       description,
-      inputSchema: inputSchema || import_zod.z.object({}),
+      inputSchema: inputSchema || import_zod2.z.object({}),
       execute: execute2
     });
   }
@@ -8153,8 +9503,17 @@ var ExuluTool3 = class {
     config,
     user,
     inputs,
-    project
+    project,
+    items
   }) => {
+    console.log("[EXULU] Calling tool execute directly", {
+      agent,
+      config,
+      user,
+      inputs,
+      project,
+      items
+    });
     const agentInstance = await loadAgent(agent);
     if (!agentInstance) {
       throw new Error("Agent not found.");
@@ -8180,22 +9539,33 @@ var ExuluTool3 = class {
     const tools = await convertToolsArrayToObject(
       [this],
       [],
+      [],
       agentInstance.tools,
       providerapikey,
       void 0,
+      void 0,
       user,
       config,
       void 0,
       void 0,
-      project
+      project,
+      items,
+      void 0,
+      agentInstance
     );
-    const tool2 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
-    if (!tool2?.execute) {
+    const tool3 = tools[sanitizeName(this.name)] || tools[this.name] || tools[this.id];
+    if (!tool3?.execute) {
       throw new Error("Tool " + sanitizeName(this.name) + " not found in " + JSON.stringify(tools));
     }
     console.log("[EXULU] Tool found", this.name);
-    const generator = tool2.execute(inputs, {
-      toolCallId: this.id + "_" + (0, import_node_crypto2.randomUUID)(),
+    const toolCallId = this.id + "_" + (0, import_node_crypto2.randomUUID)();
+    console.log("[EXULU] Calling tool execute", {
+      inputs,
+      toolCallId,
+      messages: []
+    });
+    const generator = tool3.execute(inputs, {
+      toolCallId,
       messages: []
     });
     let lastValue;
@@ -8377,6 +9747,27 @@ var ExuluContext2 = class {
       ...processorResult,
       last_processed_at: (/* @__PURE__ */ new Date()).toISOString()
     });
+    if (this.processor?.config?.generateEmbeddings) {
+      const fullItem = await db3.from(getTableName(this.id)).where({
+        id: processorResult.id
+      }).first();
+      if (!fullItem) {
+        throw new Error(`[EXULU] Item ${processorResult.id} not found after processor update in context ${this.id}`);
+      }
+      const { job: embeddingsJob } = await this.embeddings.generate.one({
+        item: fullItem,
+        user,
+        role,
+        trigger: "processor",
+        config: exuluConfig
+      });
+      if (embeddingsJob) {
+        return {
+          result: processorResult,
+          job: embeddingsJob
+        };
+      }
+    }
     return {
       result: processorResult,
       job: void 0
@@ -8388,6 +9779,8 @@ var ExuluContext2 = class {
       ...options,
       user: options.user,
       role: options.role,
+      itemFilters: options.itemFilters,
+      chunkFilters: options.chunkFilters,
       context: this,
       db: db3,
       limit: options?.limit || this.configuration.maxRetrievalResults || 10,
@@ -8810,7 +10203,7 @@ var ExuluContext2 = class {
     });
   };
   createChunksTable = async () => {
-    const { db: db3 } = await refreshPostgresClient();
+    const { db: db3 } = await postgresClient();
     const tableName = getChunksTableName(this.id);
     console.log("[EXULU] Creating table: " + tableName);
     await db3.schema.createTable(tableName, (table) => {
@@ -8846,27 +10239,32 @@ var ExuluContext2 = class {
     if (this.configuration.enableAsTool === false) {
       return null;
     }
-    return new ExuluTool3({
+    return new ExuluTool({
       id: this.id,
       name: `${this.name}_context_search`,
       type: "context",
       category: "contexts",
-      inputSchema: import_zod.z.object({
-        originalQuestion: import_zod.z.string().describe("The original question that the user asked"),
-        relevantKeywords: import_zod.z.array(import_zod.z.string()).describe("The keywords that are relevant to the user's question, for example names of specific products, systems or parts, IDs, etc.")
+      inputSchema: import_zod2.z.object({
+        query: import_zod2.z.string().describe("The original question that the user asked"),
+        keywords: import_zod2.z.array(import_zod2.z.string()).describe("The keywords that are relevant to the user's question, for example names of specific products, systems or parts, IDs, etc."),
+        method: import_zod2.z.enum(["keyword", "semantic", "hybrid"]).default("hybrid").describe(
+          "Search method: 'hybrid' (best for most queries - combines semantic understanding with exact term matching), 'keyword' (best for exact terms, technical names, IDs, or specific phrases), 'semantic' (best for conceptual queries where synonyms and paraphrasing matter)"
+        )
       }),
       config: [],
       description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
-      execute: async ({ originalQuestion, relevantKeywords, user, role }) => {
+      execute: async ({ query, keywords, user, role, method }) => {
         const { db: db3 } = await postgresClient();
         const result = await vectorSearch({
           page: 1,
           limit: this.configuration.maxRetrievalResults ?? 10,
-          query: originalQuestion,
-          filters: [],
+          query,
+          keywords,
+          itemFilters: [],
+          chunkFilters: [],
           user,
           role,
-          method: "hybridSearch",
+          method: method === "hybrid" ? "hybridSearch" : method === "keyword" ? "tsvector" : "cosineDistance",
           context: this,
           db: db3,
           sort: void 0,
@@ -8987,7 +10385,7 @@ var import_utils4 = require("@apollo/utils.keyvaluecache");
 var import_body_parser = __toESM(require("body-parser"), 1);
 var import_crypto_js4 = __toESM(require("crypto-js"), 1);
 var import_openai = __toESM(require("openai"), 1);
-var import_fs = __toESM(require("fs"), 1);
+var import_fs3 = __toESM(require("fs"), 1);
 var import_node_crypto3 = require("crypto");
 var import_api2 = require("@opentelemetry/api");
 var import_sdk = __toESM(require("@anthropic-ai/sdk"), 1);
@@ -9018,13 +10416,13 @@ var CLAUDE_MESSAGES = {
 };
 
 // src/registry/routes.ts
-var import_ai3 = require("ai");
+var import_ai4 = require("ai");
 var import_cookie_parser = __toESM(require("cookie-parser"), 1);
 var REQUEST_SIZE_LIMIT = "50mb";
 var getExuluVersionNumber = async () => {
   try {
     const path = process.cwd();
-    const packageJson = import_fs.default.readFileSync(path + "/package.json", "utf8");
+    const packageJson = import_fs3.default.readFileSync(path + "/package.json", "utf8");
     const packageData = JSON.parse(packageJson);
     const exuluVersion = packageData.dependencies["@exulu/backend"];
     console.log(`[EXULU] Installed exulu-backend version: ${exuluVersion}`);
@@ -9056,7 +10454,7 @@ var {
   promptFavoritesSchema: promptFavoritesSchema2,
   statisticsSchema: statisticsSchema2
 } = coreSchemas.get();
-var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues2) => {
+var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tracer, queues2, rerankers) => {
   var corsOptions = {
     origin: "*",
     exposedHeaders: "*",
@@ -9105,7 +10503,7 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
     workflowTemplatesSchema2(),
     statisticsSchema2(),
     rbacSchema2()
-  ], contexts ?? [], agents, tools, config, evals, queues2 || []);
+  ], contexts ?? [], agents, tools, config, evals, queues2 || [], rerankers || []);
   const server = new import_server3.ApolloServer({
     cache: new import_utils4.InMemoryLRUCache(),
     schema,
@@ -9133,11 +10531,78 @@ var createExpressRoutes = async (app, agents, tools, contexts, config, evals, tr
       }
     })
   );
-  app.post("/generate/agent/image", async (req, res) => {
-    console.log("[EXULU] generate/agent/image", req.body);
-    const authenticationResult = await requestValidators.authenticate(req);
-    if (!authenticationResult.user?.id) {
-      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+  app.post("/test", async (req, res) => {
+    const { item_name, context_id } = req.body;
+    let itemFilters = [];
+    if (item_name) {
+      itemFilters.push({ name: { contains: item_name } });
+    }
+    const { db: db3 } = await postgresClient();
+    let itemsQuery = db3(getTableName(context_id) + " as items").select([
+      "items.id as item_id",
+      "items.name as item_name",
+      "items.external_id as item_external_id",
+      db3.raw('items."updatedAt" as item_updated_at'),
+      db3.raw('items."createdAt" as item_created_at')
+    ]);
+    const limit = 10;
+    itemsQuery = itemsQuery.limit(limit);
+    const ctx = contexts?.find((ctx2) => ctx2.id === context_id);
+    if (!ctx) {
+      res.status(400).json({
+        message: "Context not found."
+      });
+      return;
+    }
+    const tableDefinition = contextToTableDefinition(ctx);
+    itemsQuery = applyFilters(itemsQuery, itemFilters || [], tableDefinition, "items");
+    itemsQuery = applyAccessControl(tableDefinition, itemsQuery, {
+      email: "test@test.com",
+      id: 1,
+      role: {
+        id: "test",
+        name: "test",
+        agents: "read",
+        evals: "read",
+        workflows: "read",
+        variables: "read",
+        users: "read"
+      },
+      super_admin: true
+    }, "items");
+    const items = await itemsQuery;
+    const formattedResults = await Promise.all(items.map(async (item, index) => {
+      const chunksTable = getChunksTableName(ctx?.id || "");
+      console.log("[EXULU] chunksTable", chunksTable);
+      if (!item.item_id) {
+        console.error("[EXULU] Item id is required to get chunks.", item);
+        throw new Error("Item id is required to get chunks.");
+      }
+      const chunks = await db3.from(chunksTable).select(["id", "source", "metadata"]).where("source", item.item_id).limit(1);
+      if (!chunks || !chunks[0]) {
+        console.error("[EXULU] No chunks found for item.", item);
+        return null;
+      }
+      console.log("[EXULU] chunks found for item.", chunks);
+      return {
+        item_name: item.name,
+        item_id: item.id,
+        context: ctx?.id || "",
+        chunk_id: chunks[0].id,
+        chunk_index: 1,
+        chunk_content: void 0,
+        metadata: chunks[0].metadata
+      };
+    }));
+    res.json({
+      items: formattedResults.filter((result) => result !== null)
+    });
+  });
+  app.post("/generate/agent/image", async (req, res) => {
+    console.log("[EXULU] generate/agent/image", req.body);
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
       return;
     }
     const { name, description, style } = req.body;
@@ -9215,10 +10680,9 @@ Mood: friendly and intelligent.
     }
     const image_bytes = Buffer.from(image_base64, "base64");
     const uuid = (0, import_node_crypto3.randomUUID)();
-    if (!import_fs.default.existsSync("public")) {
-      import_fs.default.mkdirSync("public");
+    if (!import_fs3.default.existsSync("public")) {
+      import_fs3.default.mkdirSync("public");
     }
-    import_fs.default.writeFileSync(`public/${uuid}.png`, image_bytes);
     res.status(200).json({
       message: "Image generated successfully.",
       image: `${process.env.BACKEND}/${uuid}.png`
@@ -9277,6 +10741,7 @@ Mood: friendly and intelligent.
     const slug = agent.slug;
     if (!slug) return;
     app.post(slug + "/:instance", async (req, res) => {
+      console.log("[EXULU] POST " + slug + "/:instance", req.body);
       const headers = {
         stream: req.headers["stream"] === "true" || false,
         user: req.headers["user"] || null,
@@ -9325,7 +10790,7 @@ Mood: friendly and intelligent.
       }
       console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
       const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
-      let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
+      let enabledTools = await getEnabledTools(agentInstance, tools, contexts || [], rerankers || [], disabledTools, agents, user);
       let providerapikey;
       const variableName = agentInstance.providerapikey;
       if (variableName) {
@@ -9362,14 +10827,18 @@ Mood: friendly and intelligent.
         } else {
           message = req.body.message;
         }
+        const approvedTools = req.body.approvedTools ? typeof req.body.approvedTools === "string" ? JSON.parse(req.body.approvedTools) : req.body.approvedTools : [];
         const result = await agent.generateStream({
           contexts,
+          rerankers: rerankers || [],
+          agentInstance,
           user,
           instructions: agentInstance.instructions,
           session: headers.session,
           message,
           previousMessages,
           currentTools: enabledTools,
+          approvedTools,
           allExuluTools: tools,
           providerapikey,
           toolConfigs: agentInstance.tools,
@@ -9396,7 +10865,7 @@ Mood: friendly and intelligent.
             console.error("[EXULU] chat response error.", error);
             return errorHandler(error);
           },
-          generateMessageId: (0, import_ai3.createIdGenerator)({
+          generateMessageId: (0, import_ai4.createIdGenerator)({
             prefix: "msg_",
             size: 16
           }),
@@ -9453,12 +10922,14 @@ Mood: friendly and intelligent.
         return;
       } else {
         const response = await agent.generateSync({
+          contexts,
+          rerankers: rerankers || [],
+          agentInstance,
           user,
           req,
           instructions: agentInstance.instructions,
           session: headers.session,
           inputMessages: [req.body.message],
-          contexts,
           currentTools: enabledTools,
           allExuluTools: tools,
           providerapikey,
@@ -9575,7 +11046,7 @@ Mood: friendly and intelligent.
       });
       const tokens = {};
       const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
-      let enabledTools = await getEnabledTools(agent, tools, disabledTools, agents, user);
+      let enabledTools = await getEnabledTools(agent, tools, contexts || [], rerankers || [], disabledTools, agents, user);
       let system = req.body.system;
       if (Array.isArray(req.body.system)) {
         system = [
@@ -9628,12 +11099,12 @@ Mood: friendly and intelligent.
           console.log("[EXULU] Using tool", toolName);
           const inputs = event.message?.input;
           const id = event.message?.id;
-          const tool2 = enabledTools.find((tool3) => tool3.id === toolName.replace("exulu_", ""));
-          if (!tool2 || !tool2.tool.execute) {
+          const tool3 = enabledTools.find((tool4) => tool4.id === toolName.replace("exulu_", ""));
+          if (!tool3 || !tool3.tool.execute) {
             console.error("[EXULU] Tool not found or not enabled.", toolName);
             continue;
           }
-          const toolResult = await tool2.tool.execute(inputs, {
+          const toolResult = await tool3.tool.execute(inputs, {
             toolCallId: id,
             messages: [{
               ...event.message,
@@ -9748,14 +11219,14 @@ var import_types = require("@modelcontextprotocol/sdk/types.js");
 var import_express4 = require("express");
 var import_api3 = require("@opentelemetry/api");
 var import_crypto_js5 = __toESM(require("crypto-js"), 1);
-var import_zod2 = require("zod");
+var import_zod3 = require("zod");
 var SESSION_ID_HEADER = "mcp-session-id";
 var ExuluMCP = class {
   server = {};
   transports = {};
   constructor() {
   }
-  configure = async ({ user, agentInstance, allTools, allAgents, allContexts, config, tracer }) => {
+  configure = async ({ user, agentInstance, allTools, allAgents, allContexts, allRerankers, config, tracer }) => {
     let server = this.server[agentInstance.id];
     if (!server) {
       console.log("[EXULU] Creating MCP server.");
@@ -9769,12 +11240,12 @@ var ExuluMCP = class {
       this.server[agentInstance.id] = server;
     }
     const disabledTools = [];
-    let enabledTools = await getEnabledTools(agentInstance, allTools, disabledTools, allAgents, user);
+    let enabledTools = await getEnabledTools(agentInstance, allTools, allContexts, allRerankers, disabledTools, allAgents, user);
     const backend = allAgents.find((a) => a.id === agentInstance.backend);
     if (!backend) {
       throw new Error("Agent backend not found for agent " + agentInstance.name + " (" + agentInstance.id + ").");
     }
-    const agentTool = await backend.tool(agentInstance.id, allAgents);
+    const agentTool = await backend.tool(agentInstance.id, allAgents, allContexts, allRerankers);
     if (agentTool) {
       enabledTools = [
         ...enabledTools,
@@ -9799,37 +11270,42 @@ var ExuluMCP = class {
       }
     }
     console.log("[EXULU] Enabled tools", enabledTools?.map((x) => x.name + " (" + x.id + ")"));
-    for (const tool2 of enabledTools || []) {
-      if (server.tools[tool2.id]) {
+    for (const tool3 of enabledTools || []) {
+      if (server.tools[tool3.id]) {
         continue;
       }
-      server.mcp.registerTool(sanitizeToolName(tool2.name + "_agent_" + tool2.id), {
-        title: tool2.name + " agent",
-        description: tool2.description,
+      server.mcp.registerTool(sanitizeToolName(tool3.name + "_agent_" + tool3.id), {
+        title: tool3.name + " agent",
+        description: tool3.description,
         inputSchema: {
-          inputs: tool2.inputSchema || import_zod2.z.object({})
+          inputs: tool3.inputSchema || import_zod3.z.object({})
         }
       }, async ({ inputs }, args) => {
-        console.log("[EXULU] MCP tool name", tool2.name);
+        console.log("[EXULU] MCP tool name", tool3.name);
         console.log("[EXULU] MCP tool inputs", inputs);
         console.log("[EXULU] MCP tool args", args);
         const configValues = agentInstance.tools;
         const tools = await convertToolsArrayToObject(
-          [tool2],
+          [tool3],
+          [],
           allTools,
           configValues,
           providerapikey,
           allContexts,
+          allRerankers,
           user,
-          config
+          config,
+          void 0,
+          void 0,
+          void 0
         );
-        const convertedTool = tools[sanitizeToolName(tool2.name)];
+        const convertedTool = tools[sanitizeToolName(tool3.name)];
         if (!convertedTool?.execute) {
           console.error("[EXULU] Tool not found in converted tools array.", tools);
           throw new Error("Tool not found in converted tools array.");
         }
         const iterator = await convertedTool.execute(inputs, {
-          toolCallId: tool2.id + "_" + (0, import_node_crypto4.randomUUID)(),
+          toolCallId: tool3.id + "_" + (0, import_node_crypto4.randomUUID)(),
           messages: []
         });
         let result;
@@ -9842,7 +11318,7 @@ var ExuluMCP = class {
           structuredContent: result
         };
       });
-      server.tools[tool2.id] = tool2.name;
+      server.tools[tool3.id] = tool3.name;
     }
     const getListOfPromptTemplatesName = "getListOfPromptTemplates";
     if (!server.tools[getListOfPromptTemplatesName]) {
@@ -9850,7 +11326,7 @@ var ExuluMCP = class {
         title: "Get List of Prompt Templates",
         description: "Retrieves a list of prompt templates available for this agent. Returns the name, description, and ID of each template.",
         inputSchema: {
-          inputs: import_zod2.z.object({})
+          inputs: import_zod3.z.object({})
         }
       }, async ({ inputs }, args) => {
         console.log("[EXULU] Getting list of prompt templates for agent", agentInstance.id);
@@ -9886,8 +11362,8 @@ var ExuluMCP = class {
         title: "Get Prompt Template Details",
         description: "Retrieves the full details of a specific prompt template by ID, including the actual template content with variables.",
         inputSchema: {
-          inputs: import_zod2.z.object({
-            id: import_zod2.z.string().describe("The ID of the prompt template to retrieve")
+          inputs: import_zod3.z.object({
+            id: import_zod3.z.string().describe("The ID of the prompt template to retrieve")
           })
         }
       }, async ({ inputs }, args) => {
@@ -9932,7 +11408,7 @@ var ExuluMCP = class {
     server.tools[getPromptTemplateDetailsName] = getPromptTemplateDetailsName;
     return server.mcp;
   };
-  create = async ({ express: express3, allTools, allAgents, allContexts, config }) => {
+  create = async ({ express: express3, allTools, allAgents, allContexts, allRerankers, config }) => {
     if (!express3) {
       throw new Error("Express not initialized.");
     }
@@ -9974,6 +11450,7 @@ var ExuluMCP = class {
         allTools,
         allAgents,
         allContexts,
+        allRerankers,
         config
       });
       if (!server) {
@@ -10120,6 +11597,105 @@ var claudeSonnet45Agent = new ExuluAgent3({
   }
 });
 
+// src/templates/agents/cerebras/index.ts
+var import_cerebras = require("@ai-sdk/cerebras");
+var gptOss120bAgent = new ExuluAgent3({
+  id: `default_gpt_oss_120b_agent`,
+  name: `GPT-OSS-120B`,
+  provider: "custom",
+  description: `Custom GPT-OSS-120B model. High intelligence and capability. Moderately Fast.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [],
+    files: [".pdf", ".txt"],
+    audio: [],
+    video: []
+  },
+  authenticationInformation: "",
+  maxContextLength: 128e3,
+  config: {
+    name: `GPT-OSS-120B`,
+    instructions: "",
+    model: {
+      create: ({ apiKey }) => {
+        if (!apiKey) {
+          throw new Error("Auth credentials not found for GPT-OSS-120B agent, make sure you have set the provider api key to a valid custom API key.");
+        }
+        const vertex = (0, import_cerebras.createCerebras)({
+          apiKey
+        });
+        const model = vertex("gpt-oss-120b");
+        return model;
+      }
+    }
+  }
+});
+var llama38bAgent = new ExuluAgent3({
+  id: `default_llama_38b_agent`,
+  name: `LLAMA-38B`,
+  provider: "custom",
+  description: `Custom LLAMA-38B model. High intelligence and capability. Moderately Fast.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [],
+    files: [".pdf", ".txt"],
+    audio: [],
+    video: []
+  },
+  authenticationInformation: "",
+  maxContextLength: 32e3,
+  config: {
+    name: `LLAMA-38B`,
+    instructions: "",
+    model: {
+      create: ({ apiKey }) => {
+        if (!apiKey) {
+          throw new Error("Auth credentials not found for LLAMA-38B agent, make sure you have set the provider api key to a valid custom API key.");
+        }
+        const vertex = (0, import_cerebras.createCerebras)({
+          apiKey
+        });
+        const model = vertex("llama3.1-8b");
+        return model;
+      }
+    }
+  }
+});
+var llama3370bAgent = new ExuluAgent3({
+  id: `default_llama_3370b_agent`,
+  name: `LLAMA-3.3-70B`,
+  provider: "custom",
+  description: `Custom LLAMA-3.3-70B model. High intelligence and capability. Moderately Fast.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [],
+    files: [".pdf", ".txt"],
+    audio: [],
+    video: []
+  },
+  authenticationInformation: "",
+  maxContextLength: 32e3,
+  config: {
+    name: `LLAMA-3.3-70B`,
+    instructions: "",
+    model: {
+      create: ({ apiKey }) => {
+        if (!apiKey) {
+          throw new Error("Auth credentials not found for LLAMA-3.3-70B agent, make sure you have set the provider api key to a valid custom API key.");
+        }
+        const vertex = (0, import_cerebras.createCerebras)({
+          apiKey
+        });
+        const model = vertex("llama-3.3-70b");
+        return model;
+      }
+    }
+  }
+});
+
 // src/templates/agents/google/vertex/index.ts
 var import_google_vertex = require("@ai-sdk/google-vertex");
 var vertexAuthenticationInformation = `
@@ -10193,6 +11769,43 @@ var vertexGemini25FlashAgent = new ExuluAgent3({
     }
   }
 });
+var vertexGemini25ProAgent = new ExuluAgent3({
+  id: `default_vertex_gemini_2_5_pro_agent`,
+  name: `GEMINI-2.5-PRO`,
+  provider: "vertex",
+  description: `Google Vertex Gemini 2.5 Pro model. Very high intelligence and capability. Moderately Fast.`,
+  type: "agent",
+  capabilities: {
+    text: true,
+    images: [".png", ".jpg", ".jpeg", ".webp"],
+    files: [".pdf", ".txt"],
+    audio: [".mpeg", ".mp3", ".m4a", ".wav", ".mp4"],
+    video: [".mp4", ".mpeg"]
+  },
+  authenticationInformation: vertexAuthenticationInformation,
+  maxContextLength: 1048576,
+  config: {
+    name: `GEMINI-2.5-PRO`,
+    instructions: "",
+    model: {
+      create: ({ apiKey }) => {
+        if (!apiKey) {
+          throw new Error("Auth credentials not found for Google Vertex agent, make sure you have set the provider api key to a valid google authentication json.");
+        }
+        const googleAuthPayload = JSON.parse(apiKey || "{}");
+        if (!googleAuthPayload) {
+          throw new Error("API key not found for Google Vertex Gemini 2.5 Pro agent.");
+        }
+        if (!googleAuthPayload.location) {
+          throw new Error("Location not set in authentication json for Google Vertex Gemini 2.5 Pro agent, should be for example 'europe-west1'");
+        }
+        const vertex = (0, import_google_vertex.createVertex)(googleAuthPayload);
+        const model = vertex("gemini-2.5-pro");
+        return model;
+      }
+    }
+  }
+});
 var vertexGemini20FlashAgent = new ExuluAgent3({
   id: `default_vertex_gemini_2_0_flash_agent`,
   name: `GEMINI-2.0-FLASH`,
@@ -10672,7 +12285,7 @@ var ExuluQueues = class {
 var queues = new ExuluQueues();
 
 // src/templates/evals/index.ts
-var import_zod3 = require("zod");
+var import_zod4 = require("zod");
 var llmAsJudgeEval = () => {
   if (process.env.REDIS_HOST?.length && process.env.REDIS_PORT?.length) {
     return new ExuluEval3({
@@ -10705,9 +12318,12 @@ var llmAsJudgeEval = () => {
         const providerapikey = await ExuluVariables.get(agent.providerapikey);
         console.log("[EXULU] prompt", prompt);
         const response = await backend.generateSync({
+          agentInstance: agent,
+          contexts: [],
+          rerankers: [],
           prompt,
-          outputSchema: import_zod3.z.object({
-            score: import_zod3.z.number().min(0).max(100).describe("The score between 0 and 100.")
+          outputSchema: import_zod4.z.object({
+            score: import_zod4.z.number().min(0).max(100).describe("The score between 0 and 100.")
           }),
           providerapikey
         });
@@ -10741,647 +12357,6 @@ var getDefaultEvals = () => {
   return [];
 };
 
-// src/templates/tools/math.ts
-var import_zod4 = require("zod");
-
-// src/templates/tools/utils/arithmetic.ts
-var Arithmetic = class {
-  /**
-   * Add two numbers together
-   * @param firstNumber - The first number 
-   * @param secondNumber - The second number
-   * @returns sum
-   */
-  static add(firstNumber, secondNumber) {
-    const sum = firstNumber + secondNumber;
-    return sum;
-  }
-  /**
-   * Subtract one number from another
-   * @param minuend - The number to subtract from
-   * @param subtrahend - The number to subtract
-   * @returns difference
-   */
-  static subtract(minuend, subtrahend) {
-    const difference = minuend - subtrahend;
-    return difference;
-  }
-  /**
-   * Multiply two numbers together
-   * @param firstNumber - The first number
-   * @param secondNumber - The second number
-   * @returns product
-   */
-  static multiply(firstNumber, secondNumber) {
-    const product = firstNumber * secondNumber;
-    return product;
-  }
-  /**
-   * Divide one number by another
-   * @param numerator - The number to be divided
-   * @param denominator - The number to divide by
-   * @returns quotient
-   */
-  static division(numerator, denominator) {
-    const quotient = numerator / denominator;
-    return quotient;
-  }
-  /**
-   * Calculate the sum of an array of numbers
-   * @param numbers - Array of numbers to sum
-   * @returns sum of all numbers in the array
-   */
-  static sum(numbers) {
-    const sum = numbers.reduce((accumulator, currentValue) => accumulator + currentValue, 0);
-    return sum;
-  }
-  /**
-   * Calculate the floor of a number
-   * @param number - Number to find the floor of
-   * @returns floor of the number
-   */
-  static floor(number) {
-    const floor = Math.floor(number);
-    return floor;
-  }
-  /**
-   * Calculate the ceil of a number
-   * @param number - Number to find the ceil of
-   * @returns ceil of the number
-   */
-  static ceil(number) {
-    const ceil = Math.ceil(number);
-    return ceil;
-  }
-  /**
-   * Calculate the round of a number
-   * @param number - Number to find the round of
-   * @returns round of the number
-   */
-  static round(number) {
-    const round = Math.round(number);
-    return round;
-  }
-  /**
-   * Get the remainder of a division equation.
-   * Ex: modulo(5,2) = 1
-   * @param numerator - The number to be divided
-   * @param denominator - The number to divide by
-   * @returns remainder of division
-   */
-  static modulo(numerator, denominator) {
-    const remainder = numerator % denominator;
-    return remainder;
-  }
-};
-
-// src/templates/tools/utils/statistics.ts
-var Statistics = class {
-  /**
-   * Calculate the arithmetic mean (average) of an array of numbers
-   * @param numbers - Array of numbers to calculate the mean of
-   * @returns The arithmetic mean value
-   */
-  static mean(numbers) {
-    const sum = numbers.reduce((accumulator, currentValue) => accumulator + currentValue, 0);
-    const mean = sum / numbers.length;
-    return mean;
-  }
-  /**
-   * Calculate the median (middle value) of an array of numbers
-   * @param numbers - Array of numbers to calculate the median of
-   * @returns The median value
-   */
-  static median(numbers) {
-    numbers.sort();
-    const medianIndex = numbers.length / 2;
-    let medianValue;
-    if (numbers.length % 2 !== 0) {
-      medianValue = numbers[Math.floor(medianIndex)];
-    } else {
-      medianValue = (numbers[medianIndex] + numbers[medianIndex - 1]) / 2;
-    }
-    return medianValue;
-  }
-  /**
-   * Calculate the mode (most frequent value(s)) of an array of numbers
-   * @param numbers - Array of numbers to calculate the mode of
-   * @returns Object containing the mode value(s) and their frequency
-   */
-  static mode(numbers) {
-    const modeMap = /* @__PURE__ */ new Map();
-    numbers.forEach((value) => {
-      if (modeMap.has(value)) {
-        modeMap.set(value, modeMap.get(value) + 1);
-      } else {
-        modeMap.set(value, 1);
-      }
-    });
-    let maxFrequency = 0;
-    for (const numberFrequency of modeMap.values()) {
-      if (numberFrequency > maxFrequency) {
-        maxFrequency = numberFrequency;
-      }
-    }
-    const modeResult = [];
-    for (const [key, value] of modeMap.entries()) {
-      if (value === maxFrequency) {
-        modeResult.push(key);
-      }
-    }
-    return {
-      modeResult,
-      maxFrequency
-    };
-  }
-  /**
-   * Find the minimum value in an array of numbers
-   * @param numbers - Array of numbers to find the minimum of
-   * @returns The minimum value
-   */
-  static min(numbers) {
-    const minValue = Math.min(...numbers);
-    return minValue;
-  }
-  /**
-   * Find the maximum value in an array of numbers
-   * @param numbers - Array of numbers to find the maximum of
-   * @returns The maximum value
-   */
-  static max(numbers) {
-    const maxValue = Math.max(...numbers);
-    return maxValue;
-  }
-};
-
-// src/templates/tools/utils/trigonometric.ts
-var Trigonometric = class {
-  /**
-   * Calculate the sin of a number in radians
-   * @param number - The number to find the sin of
-   * @returns The sin of a number in radians
-   */
-  static sin(number) {
-    const sin = Math.sin(number);
-    return sin;
-  }
-  /**
-   * Calculate the arcsin of a number in radians
-   * @param number - The number to find the arcsin of
-   * @returns The arcsin of a number in radians
-   */
-  static arcsin(number) {
-    const arcsin = Math.asin(number);
-    return arcsin;
-  }
-  /**
-   * Calculate the cos of a number in radians
-   * @param number - The number to find the cos of
-   * @returns The cos of a number in radians
-   */
-  static cos(number) {
-    const cos = Math.cos(number);
-    return cos;
-  }
-  /**
-   * Calculate the arccos of a number in radians
-   * @param number - The number to find the arccos of
-   * @returns The arccos of a number in radians
-   */
-  static arccos(number) {
-    const arccos = Math.acos(number);
-    return arccos;
-  }
-  /**
-   * Calculate the tangent of a number in radians
-   * @param number - The number to find the tangent of
-   * @returns The tangent of a number in radians
-   */
-  static tan(number) {
-    const tangent = Math.tan(number);
-    return tangent;
-  }
-  /**
-   * Calculate the arc tangent of a number in radians
-   * @param number - The number to find the arc tangent of
-   * @returns The arc tangent of a number in radians
-   */
-  static arctan(number) {
-    const arctangent = Math.atan(number);
-    return arctangent;
-  }
-  /**
-   * Converts a radian into its equivalent value in degrees
-   * @param number - The number to get the degree of
-   * @returns The degree of the number
-   */
-  static radiansToDegrees(number) {
-    const degrees = number * (180 / Math.PI);
-    return degrees;
-  }
-  /**
-   * Converts a degree into its equivalent value in radians
-   * @param number - The number to get the radians of
-   * @returns The radians of the number
-   */
-  static degreesToRadians(number) {
-    const radians = number * (Math.PI / 180);
-    return radians;
-  }
-};
-
-// src/templates/tools/math.ts
-var additionTool = new ExuluTool3({
-  id: "addition",
-  name: "Addition",
-  description: "Adds two numbers together",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    firstNumber: import_zod4.z.number().describe("The first addend"),
-    secondNumber: import_zod4.z.number().describe("The second addend")
-  }),
-  execute: async ({ firstNumber, secondNumber }) => {
-    const value = Arithmetic.add(firstNumber, secondNumber);
-    return { result: `${value}` };
-  }
-});
-var subtractionTool = new ExuluTool3({
-  id: "subtraction",
-  name: "Subtraction",
-  description: "Subtracts the second number from the first number",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    minuend: import_zod4.z.number().describe("The number to subtract from (minuend)"),
-    subtrahend: import_zod4.z.number().describe("The number being subtracted (subtrahend)")
-  }),
-  execute: async ({ minuend, subtrahend }) => {
-    const value = Arithmetic.subtract(minuend, subtrahend);
-    return { result: `${value}` };
-  }
-});
-var multiplicationTool = new ExuluTool3({
-  id: "multiplication",
-  name: "Multiplication",
-  description: "Multiplies two numbers together",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    firstNumber: import_zod4.z.number().describe("The first number"),
-    SecondNumber: import_zod4.z.number().describe("The second number")
-  }),
-  execute: async ({ firstNumber, SecondNumber }) => {
-    const value = Arithmetic.multiply(firstNumber, SecondNumber);
-    return { result: `${value}` };
-  }
-});
-var divisionTool = new ExuluTool3({
-  id: "division",
-  name: "Division",
-  description: "Divides the first number by the second number",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numerator: import_zod4.z.number().describe("The number being divided (numerator)"),
-    denominator: import_zod4.z.number().describe("The number to divide by (denominator)")
-  }),
-  execute: async ({ numerator, denominator }) => {
-    const value = Arithmetic.division(numerator, denominator);
-    return { result: `${value}` };
-  }
-});
-var sumTool = new ExuluTool3({
-  id: "sum",
-  name: "Sum",
-  description: "Adds any number of numbers together",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numbers: import_zod4.z.array(import_zod4.z.number()).min(1).describe("Array of numbers to sum")
-  }),
-  execute: async ({ numbers }) => {
-    const value = Arithmetic.sum(numbers);
-    return { result: `${value}` };
-  }
-});
-var moduloTool = new ExuluTool3({
-  id: "modulo",
-  name: "Modulo",
-  description: "Divides two numbers and returns the remainder",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numerator: import_zod4.z.number().describe("The number being divided (numerator)"),
-    denominator: import_zod4.z.number().describe("The number to divide by (denominator)")
-  }),
-  execute: async ({ numerator, denominator }) => {
-    const value = Arithmetic.modulo(numerator, denominator);
-    return { result: `${value}` };
-  }
-});
-var meanTool = new ExuluTool3({
-  id: "mean",
-  name: "Mean",
-  description: "Calculates the arithmetic mean of a list of numbers",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numbers: import_zod4.z.array(import_zod4.z.number()).min(1).describe("Array of numbers to find the mean of")
-  }),
-  execute: async ({ numbers }) => {
-    const value = Statistics.mean(numbers);
-    return { result: `${value}` };
-  }
-});
-var medianTool = new ExuluTool3({
-  id: "median",
-  name: "Median",
-  description: "Calculates the median of a list of numbers",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numbers: import_zod4.z.array(import_zod4.z.number()).min(1).describe("Array of numbers to find the median of")
-  }),
-  execute: async ({ numbers }) => {
-    const value = Statistics.median(numbers);
-    return { result: `${value}` };
-  }
-});
-var modeTool = new ExuluTool3({
-  id: "mode",
-  name: "Mode",
-  description: "Finds the most common number in a list of numbers",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numbers: import_zod4.z.array(import_zod4.z.number()).describe("Array of numbers to find the mode of")
-  }),
-  execute: async ({ numbers }) => {
-    const value = Statistics.mode(numbers);
-    return { result: `Entries (${value.modeResult.join(", ")}) appeared ${value.maxFrequency} times` };
-  }
-});
-var minTool = new ExuluTool3({
-  id: "min",
-  name: "Minimum",
-  description: "Finds the minimum value from a list of numbers",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numbers: import_zod4.z.array(import_zod4.z.number()).describe("Array of numbers to find the minimum of")
-  }),
-  execute: async ({ numbers }) => {
-    const value = Statistics.min(numbers);
-    return { result: `${value}` };
-  }
-});
-var maxTool = new ExuluTool3({
-  id: "max",
-  name: "Maximum",
-  description: "Finds the maximum value from a list of numbers",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    numbers: import_zod4.z.array(import_zod4.z.number()).describe("Array of numbers to find the maximum of")
-  }),
-  execute: async ({ numbers }) => {
-    const value = Statistics.max(numbers);
-    return { result: `${value}` };
-  }
-});
-var floorTool = new ExuluTool3({
-  id: "floor",
-  name: "Floor",
-  description: "Rounds a number down to the nearest integer",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number to round down")
-  }),
-  execute: async ({ number }) => {
-    const value = Arithmetic.floor(number);
-    return { result: `${value}` };
-  }
-});
-var ceilingTool = new ExuluTool3({
-  id: "ceiling",
-  name: "Ceiling",
-  description: "Rounds a number up to the nearest integer",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number to round up")
-  }),
-  execute: async ({ number }) => {
-    const value = Arithmetic.ceil(number);
-    return { result: `${value}` };
-  }
-});
-var roundTool = new ExuluTool3({
-  id: "round",
-  name: "Round",
-  description: "Rounds a number to the nearest integer",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number to round")
-  }),
-  execute: async ({ number }) => {
-    const value = Arithmetic.round(number);
-    return { result: `${value}` };
-  }
-});
-var sinTool = new ExuluTool3({
-  id: "sin",
-  name: "Sine",
-  description: "Calculates the sine of a number in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number in radians to find the sine of")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.sin(number);
-    return { result: `${value}` };
-  }
-});
-var arcsinTool = new ExuluTool3({
-  id: "arcsin",
-  name: "Arcsine",
-  description: "Calculates the arcsine of a number in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number to find the arcsine of")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.arcsin(number);
-    return { result: `${value}` };
-  }
-});
-var cosTool = new ExuluTool3({
-  id: "cos",
-  name: "Cosine",
-  description: "Calculates the cosine of a number in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number in radians to find the cosine of")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.cos(number);
-    return { result: `${value}` };
-  }
-});
-var arccosTool = new ExuluTool3({
-  id: "arccos",
-  name: "Arccosine",
-  description: "Calculates the arccosine of a number in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number to find the arccosine of")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.arccos(number);
-    return { result: `${value}` };
-  }
-});
-var tanTool = new ExuluTool3({
-  id: "tan",
-  name: "Tangent",
-  description: "Calculates the tangent of a number in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number in radians to find the tangent of")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.tan(number);
-    return { result: `${value}` };
-  }
-});
-var arctanTool = new ExuluTool3({
-  id: "arctan",
-  name: "Arctangent",
-  description: "Calculates the arctangent of a number in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number to find the arctangent of")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.arctan(number);
-    return { result: `${value}` };
-  }
-});
-var radiansToDegreesTool = new ExuluTool3({
-  id: "radiansToDegrees",
-  name: "Radians to Degrees",
-  description: "Converts a radian value to its equivalent in degrees",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number in radians to convert to degrees")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.radiansToDegrees(number);
-    return { result: `${value}` };
-  }
-});
-var degreesToRadiansTool = new ExuluTool3({
-  id: "degreesToRadians",
-  name: "Degrees to Radians",
-  description: "Converts a degree value to its equivalent in radians",
-  type: "function",
-  category: "math",
-  config: [],
-  inputSchema: import_zod4.z.object({
-    number: import_zod4.z.number().describe("The number in degrees to convert to radians")
-  }),
-  execute: async ({ number }) => {
-    const value = Trigonometric.degreesToRadians(number);
-    return { result: `${value}` };
-  }
-});
-var mathTools = [
-  additionTool,
-  subtractionTool,
-  multiplicationTool,
-  divisionTool,
-  sumTool,
-  moduloTool,
-  meanTool,
-  medianTool,
-  modeTool,
-  minTool,
-  maxTool,
-  floorTool,
-  ceilingTool,
-  roundTool,
-  sinTool,
-  arcsinTool,
-  cosTool,
-  arccosTool,
-  tanTool,
-  arctanTool,
-  radiansToDegreesTool,
-  degreesToRadiansTool
-];
-
-// src/templates/tools/preview-pdf.ts
-var import_zod5 = require("zod");
-var previewPdfTool = new ExuluTool3({
-  id: "preview_pdf",
-  name: "Preview PDF",
-  description: "Used to display a PDF file in an iframe web view",
-  type: "function",
-  config: [],
-  inputSchema: import_zod5.z.object({
-    s3key: import_zod5.z.string().describe("The S3 key of the PDF file to preview."),
-    page: import_zod5.z.number().describe("The page number to preview, defaults to 1.").optional()
-  }),
-  execute: async ({ s3key, page, exuluConfig }) => {
-    const bucket = s3key.split("/")[0];
-    const key = s3key.split("/").slice(1).join("/");
-    if (!bucket || !key) {
-      throw new Error("Invalid S3 key, must be in the format of <bucket>/<key>.");
-    }
-    const url = await getPresignedUrl(bucket, key, exuluConfig);
-    if (!url) {
-      throw new Error("No URL provided for PDF preview");
-    }
-    return {
-      result: JSON.stringify({
-        url,
-        page: page ?? 1
-      })
-    };
-  }
-});
-
 // src/templates/tools/todo/todowrite.txt
 var todowrite_default = `Use this tool to create and manage a structured task list for your current coding session. This helps you track progress, organize complex tasks, and demonstrate thoroughness to the user.
 It also helps the user understand the progress of the task and overall progress of their requests.
@@ -11568,14 +12543,14 @@ Usage:
 - If no todos exist yet, an empty list will be returned`;
 
 // src/templates/tools/todo/todo.ts
-var import_zod6 = __toESM(require("zod"), 1);
-var TodoSchema = import_zod6.default.object({
-  content: import_zod6.default.string().describe("Brief description of the task"),
-  status: import_zod6.default.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
-  priority: import_zod6.default.string().describe("Priority level of the task: high, medium, low"),
-  id: import_zod6.default.string().describe("Unique identifier for the todo item")
+var import_zod5 = __toESM(require("zod"), 1);
+var TodoSchema = import_zod5.default.object({
+  content: import_zod5.default.string().describe("Brief description of the task"),
+  status: import_zod5.default.string().describe("Current status of the task: pending, in_progress, completed, cancelled"),
+  priority: import_zod5.default.string().describe("Priority level of the task: high, medium, low"),
+  id: import_zod5.default.string().describe("Unique identifier for the todo item")
 });
-var TodoWriteTool = new ExuluTool3({
+var TodoWriteTool = new ExuluTool({
   id: "todo_write",
   name: "Todo Write",
   description: "Use this tool to write your todo list",
@@ -11584,10 +12559,11 @@ var TodoWriteTool = new ExuluTool3({
   config: [{
     name: "description",
     description: "The description of the todo list, if set overwrites the default description.",
+    type: "string",
     default: todowrite_default
   }],
-  inputSchema: import_zod6.default.object({
-    todos: import_zod6.default.array(TodoSchema).describe("The updated todo list")
+  inputSchema: import_zod5.default.object({
+    todos: import_zod5.default.array(TodoSchema).describe("The updated todo list")
   }),
   execute: async (inputs) => {
     const { sessionID, todos, user } = inputs;
@@ -11614,16 +12590,17 @@ var TodoWriteTool = new ExuluTool3({
     };
   }
 });
-var TodoReadTool = new ExuluTool3({
+var TodoReadTool = new ExuluTool({
   id: "todo_read",
   name: "Todo Read",
   description: "Use this tool to read your todo list",
-  inputSchema: import_zod6.default.object({}),
+  inputSchema: import_zod5.default.object({}),
   type: "function",
   category: "todo",
   config: [{
     name: "description",
     description: "The description of the todo list, if set overwrites the default description.",
+    type: "string",
     default: todoread_default
   }],
   execute: async (inputs) => {
@@ -11656,6 +12633,109 @@ var todoTools = [
   TodoReadTool
 ];
 
+// src/templates/tools/perplexity.ts
+var import_zod6 = __toESM(require("zod"), 1);
+var import_perplexity_ai = __toESM(require("@perplexity-ai/perplexity_ai"), 1);
+var internetSearchTool = new ExuluTool({
+  id: "internet_search",
+  name: "Perplexity Live Internet Search",
+  description: "Search the internet for information.",
+  inputSchema: import_zod6.default.object({
+    query: import_zod6.default.string().describe("The query to the tool."),
+    search_recency_filter: import_zod6.default.enum(["day", "week", "month", "year"]).optional().describe("The recency filter for the search, can be day, week, month or year.")
+  }),
+  category: "internet_search",
+  type: "web_search",
+  config: [
+    {
+      name: "perplexity_api_key",
+      description: "The API key for the Perplexity API.",
+      type: "variable",
+      default: ""
+    },
+    {
+      name: "search_domain_filter",
+      description: "Comma seperated domain filter for the search.",
+      type: "string",
+      default: ""
+    }
+  ],
+  execute: async ({ query, search_recency_filter, toolVariablesConfig }) => {
+    const {
+      perplexity_api_key: apiKey,
+      search_domain_filter
+    } = toolVariablesConfig;
+    let domainFilterArray = [];
+    if (search_domain_filter) {
+      domainFilterArray = search_domain_filter.split(",").map((domain) => domain.trim());
+    }
+    const maxRetries = 3;
+    if (!apiKey) {
+      throw new Error("Perplexity API key is required.");
+    }
+    const client2 = new import_perplexity_ai.default({
+      apiKey
+    });
+    let recency_filter;
+    if (search_recency_filter && ["day", "week", "month", "year"].includes(search_recency_filter)) {
+      recency_filter = search_recency_filter;
+    }
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const result = await client2.chat.completions.create({
+          messages: [{
+            "role": "user",
+            "content": `${query}`
+          }],
+          model: "sonar-pro",
+          web_search_options: {
+            ...domainFilterArray.length > 0 ? { "search_domain_filter": domainFilterArray } : {},
+            ...recency_filter ? { "search_recency_filter": recency_filter } : {},
+            "user_location": {
+              "country": "CH"
+            }
+          }
+        });
+        if (!result || !result.choices[0]) {
+          throw new Error("No content returned from internet search.");
+        }
+        const content = result.choices[0].message.content;
+        const parsed = {
+          content: content.replace(/(\[([1-9][0-9]*)\])/g, (match, p1, p2) => {
+            const index = parseInt(p2) - 1;
+            if (!result.search_results || !result.search_results[index]) {
+              return match;
+            }
+            return JSON.stringify({
+              url: result.search_results[index].url,
+              title: result.search_results[index].title,
+              snippet: result.search_results[index].snippet
+            });
+          }),
+          citations: result.citations,
+          search_results: result.search_results
+        };
+        return {
+          result: JSON.stringify(parsed)
+        };
+      } catch (error) {
+        if (error instanceof import_perplexity_ai.default.RateLimitError && attempt < maxRetries - 1) {
+          const delay = Math.pow(2, attempt) * 1e3 + Math.random() * 1e3;
+          await new Promise((resolve) => setTimeout(resolve, delay));
+          continue;
+        }
+        throw error;
+      }
+    }
+    return {
+      result: "Max retries exceeded for perplexity research for query."
+    };
+  }
+});
+var perplexityTools = [
+  internetSearchTool
+];
+
 // src/registry/index.ts
 var isDev = process.env.NODE_ENV !== "production";
 var consoleTransport = new import_winston2.default.transports.Console({
@@ -11692,6 +12772,7 @@ var ExuluApp = class {
   _config;
   _evals = [];
   _queues = [];
+  _rerankers = [];
   _contexts = {};
   _tools = [];
   _expressApp = null;
@@ -11704,16 +12785,24 @@ var ExuluApp = class {
     agents,
     config,
     tools,
-    evals
+    evals,
+    rerankers
   }) => {
     this._evals = redisServer.host?.length && redisServer.port?.length ? [...getDefaultEvals(), ...evals ?? []] : [];
     this._contexts = {
       ...contexts
     };
+    this._rerankers = [
+      ...rerankers ?? []
+    ];
     this._agents = [
       claudeSonnet4Agent,
       claudeOpus4Agent,
+      gptOss120bAgent,
+      llama38bAgent,
+      llama3370bAgent,
       vertexGemini25FlashAgent,
+      vertexGemini25ProAgent,
       vertexGemini3ProAgent,
       claudeSonnet45Agent,
       gpt5MiniAgent,
@@ -11730,9 +12819,8 @@ var ExuluApp = class {
     this._config = config;
     this._tools = [
       ...tools ?? [],
-      ...mathTools,
-      ...[previewPdfTool],
       ...todoTools,
+      ...perplexityTools,
       // Add contexts as tools
       ...Object.values(contexts || {}).map((context) => context.tool()).filter(Boolean)
       // Because agents are stored in the database,  we add those as tools
@@ -11751,16 +12839,21 @@ var ExuluApp = class {
         id: agent.id ?? "",
         type: "agent"
       })),
-      ...this._tools.map((tool2) => ({
-        name: tool2.name ?? "",
-        id: tool2.id ?? "",
+      ...this._tools.map((tool3) => ({
+        name: tool3.name ?? "",
+        id: tool3.id ?? "",
         type: "tool"
+      })),
+      ...this._rerankers.map((reranker) => ({
+        name: reranker.name ?? "",
+        id: reranker.id ?? "",
+        type: "reranker"
       }))
     ];
     const invalid = checks.filter((x) => !isValidPostgresName(x?.id ?? ""));
     if (invalid.length > 0) {
       console.error(
-        `%c[EXULU] Invalid ID found for a context, tool or agent: ${invalid.map((x) => x.id).join(", ")}. An ID must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or underscores and be a max length of 80 characters and at least 5 characters long.`,
+        `%c[EXULU] Invalid ID found for a context, tool, reranker or agent: ${invalid.map((x) => x.id).join(", ")}. An ID must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or underscores and be a max length of 80 characters and at least 5 characters long.`,
         "color: orange; font-weight: bold; \n \n"
       );
       throw new Error(
@@ -11955,6 +13048,7 @@ var ExuluApp = class {
           filteredQueues,
           this._config,
           Object.values(this._contexts ?? {}),
+          this._rerankers,
           this._evals,
           this._tools,
           tracer
@@ -11998,7 +13092,8 @@ var ExuluApp = class {
           this._config,
           this._evals,
           tracer,
-          this._queues
+          this._queues,
+          this._rerankers
         );
         if (this._config?.MCP.enabled) {
           const mcp = new ExuluMCP();
@@ -12007,6 +13102,7 @@ var ExuluApp = class {
             allTools: this._tools,
             allAgents: this._agents,
             allContexts: Object.values(this._contexts ?? {}),
+            allRerankers: this._rerankers,
             config: this._config
           });
         }
@@ -13483,8 +14579,14 @@ var ExuluDefaultAgents = {
     sonnet4: claudeSonnet4Agent,
     sonnet45: claudeSonnet45Agent
   },
+  cerebras: {
+    gptOss120b: gptOss120bAgent,
+    llama38b: llama38bAgent,
+    llama3370b: llama3370bAgent
+  },
   google: {
     vertexGemini25Flash: vertexGemini25FlashAgent,
+    vertexGemini25Pro: vertexGemini25ProAgent,
     vertexGemini3Pro: vertexGemini3ProAgent
   },
   openai: {
@@ -13640,6 +14742,7 @@ var ExuluChunkers = {
   ExuluJobs,
   ExuluOtel,
   ExuluQueues,
+  ExuluReranker,
   ExuluTool,
   ExuluUtils,
   ExuluVariables,