@exulu/backend 1.55.0 → 1.57.0

package/dist/index.cjs

@@ -1253,7 +1253,7 @@ var hydrateVariables = async (tool5) => {
   await Promise.all(promises2);
   return tool5;
 };
-var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project, sessionItems, model, agent) => {
+var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExuluTools, configs, providerapikey, contexts, rerankers, user, exuluConfig, sessionID, req, project, sessionItems, model, agent, memoryItems) => {
   if (!currentTools) return {};
   if (!allExuluTools) {
     allExuluTools = [];
@@ -1309,7 +1309,8 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
       user,
       role: user?.role?.id,
       model,
-      preselectedItemIds: sessionItems
+      preselected: sessionItems,
+      memoryItems
     });
     if (agenticSearchTool) {
       const index = currentTools.findIndex((tool5) => tool5.id === "agentic_context_search");
@@ -1443,6 +1444,8 @@ var convertExuluToolsToAiSdkTools = async (currentTools, approvedTools, allExulu
                 ...inputs,
                 model,
                 sessionID,
+                sessionItems,
+                memory: memoryItems,
                 req,
                 // Convert config to object format if a config object
                 // is available, after we added the .value property
@@ -1543,7 +1546,7 @@ var ExuluTool = class {
     });
   }
   execute = async ({
-    agent: agentId2,
+    agent: agentId,
     config,
     user,
     inputs,
@@ -1551,14 +1554,14 @@ var ExuluTool = class {
     items
   }) => {
     console.log("[EXULU] Calling tool execute directly", {
-      agentId: agentId2,
+      agentId,
       config,
       user,
       inputs,
       project,
       items
     });
-    const agent = await exuluApp.get().agent(agentId2);
+    const agent = await exuluApp.get().agent(agentId);
     if (!agent) {
       throw new Error("Agent not found.");
     }
@@ -1630,6 +1633,95 @@ var ExuluTool = class {
   };
 };
 
+// ee/agentic-retrieval/v3/classifier.ts
+var import_ai2 = require("ai");
+var import_zod5 = require("zod");
+
+// src/utils/with-retry.ts
+async function withRetry(generateFn, maxRetries = 3) {
+  let lastError;
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      return await generateFn();
+    } catch (error) {
+      lastError = error;
+      console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
+      if (attempt === maxRetries) {
+        throw error;
+      }
+      await new Promise((resolve3) => setTimeout(resolve3, Math.pow(2, attempt) * 1e3));
+    }
+  }
+  throw lastError;
+}
+
+// ee/agentic-retrieval/v3/classifier.ts
+async function classifyQuery(query, contexts, samples, model) {
+  const contextDescriptions = contexts.map((ctx) => {
+    const sample = samples.find((s) => s.contextId === ctx.id);
+    const fieldList = sample?.fields.join(", ") ?? "name, external_id";
+    return `
+      <context>
+        <id>
+          ${ctx.id}
+        </id>
+        <name>
+          ${ctx.name}
+        </name>
+        <description>
+          ${ctx.description}
+        </description>
+        <fields>
+          ${fieldList}
+        </fields>
+        <example_items>
+          ${sample?.exampleItems.map((item) => JSON.stringify(item)).join("\n")}
+        </example_items>
+       </context>
+       `;
+  }).join("\n\n");
+  const result = await withRetry(async () => {
+    const result2 = await (0, import_ai2.generateText)({
+      model,
+      temperature: 0,
+      output: import_ai2.Output.object({
+        schema: import_zod5.z.object({
+          queryType: import_zod5.z.enum(["aggregate", "list", "targeted", "exploratory"]).describe(
+            "aggregate: ONLY use when the user explicitly asks to COUNT how many documents/items/tickets exist in the knowledge base (e.g. 'how many documents about X?', 'total number of tickets'). NEVER use for: real-world statistics stored in a document, intent statements, how-to questions, error/fault descriptions, configuration questions, or any query that does not explicitly ask for a count of knowledge base entries. When in doubt, choose targeted. list: user wants to enumerate matching items/documents (show me all, list documents about). targeted: use for almost everything \u2014 specific fact, answer, configuration, how-to, error/fault, feature/behavior question. Also use for intent statements and short commands describing a desired state (phrases that state what the user wants to do or achieve, even without an explicit question word). Real-world statistics stored in documents also go here. When in doubt, choose targeted over aggregate or exploratory. exploratory: only for broad conceptual questions needing multi-source synthesis (what is the process for Z, explain how X works, general overview of topic Y)."
+          ),
+          language: import_zod5.z.string().describe("ISO 639-3 language code of the query (e.g. eng, deu, fra)"),
+          suggestedContextIds: import_zod5.z.array(import_zod5.z.enum(contexts.map((c) => c.id))).describe(
+            "IDs of knowledge bases most likely to contain the answer. Return empty array to search all contexts."
+          )
+        })
+      }),
+      toolChoice: "none",
+      system: `You are a query classifier for a multi-knowledge-base retrieval system.
+    Classify the query and identify which knowledge bases are most relevant.
+    
+    Available knowledge bases:
+    ${contextDescriptions}
+    
+    Guidelines for queryType:
+    - Use "aggregate" ONLY when the query contains explicit counting language (e.g., "how many", "count", "total number", "wie viele"). Short statements, commands, or phrases without a question word are NEVER aggregate \u2014 classify them as targeted.
+    - When in doubt between aggregate and targeted: always choose targeted.
+    
+    Guidelines for suggestedContextIds:
+    - Be conservative: only suggest contexts that are genuinely likely to contain the answer.
+      Aim for 2\u20133 focused suggestions rather than listing everything.
+    - Use each knowledge base's name and description (shown above) to judge relevance.
+    - Return an empty array only if you truly cannot determine which contexts are relevant.`,
+      prompt: `Query: ${query}`
+    });
+    return result2.output;
+  }, 3);
+  return result;
+}
+
+// ee/agentic-retrieval/v3/tools.ts
+var import_zod6 = require("zod");
+var import_ai3 = require("ai");
+
 // src/uppy/index.ts
 var import_express = require("express");
 var import_client_s32 = require("@aws-sdk/client-s3");
@@ -3481,6 +3573,45 @@ var promptFavoritesSchema = {
     }
   ]
 };
+var contextPresetsSchema = {
+  type: "context_presets",
+  name: {
+    plural: "context_presets",
+    singular: "context_preset"
+  },
+  RBAC: true,
+  fields: [
+    {
+      name: "name",
+      type: "text",
+      required: true,
+      index: true
+    },
+    {
+      name: "description",
+      type: "text"
+    },
+    {
+      name: "preset_items",
+      type: "json",
+      required: true
+    },
+    {
+      name: "tags",
+      type: "json"
+    },
+    {
+      name: "usage_count",
+      type: "number",
+      default: 0
+    },
+    {
+      name: "favorite_count",
+      type: "number",
+      default: 0
+    }
+  ]
+};
 var addCoreFields = (schema) => {
   schema.fields.forEach((field) => {
     if (field.type === "file") {
@@ -3530,7 +3661,8 @@ var coreSchemas = {
       platformConfigurationsSchema: () => addCoreFields(platformConfigurationsSchema),
       promptLibrarySchema: () => addCoreFields(promptLibrarySchema),
       embedderSettingsSchema: () => addCoreFields(embedderSettingsSchema),
-      promptFavoritesSchema: () => addCoreFields(promptFavoritesSchema)
+      promptFavoritesSchema: () => addCoreFields(promptFavoritesSchema),
+      contextPresetsSchema: () => addCoreFields(contextPresetsSchema)
     };
     if (license["agent-feedback"]) {
       schemas.feedbackSchema = () => addCoreFields(feedbackSchema);
@@ -4572,7 +4704,7 @@ var ExuluContext2 = class {
       job: jobs.length > 0 ? jobs.join(",") : void 0
     };
   };
-  updateItem = async (item, config, user, role, generateEmbeddingsOverwrite) => {
+  updateItem = async (item, config, user, role, generateEmbeddingsOverwrite, runProcessorOverwrite) => {
     console.log("[EXULU] updating item", item);
     const { db: db2 } = await postgresClient();
     if (item.field) {
@@ -4598,7 +4730,7 @@ var ExuluContext2 = class {
     let shouldGenerateEmbeddings = this.embedder && generateEmbeddingsOverwrite !== false && (generateEmbeddingsOverwrite || this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always");
     if (this.processor) {
       const processor = this.processor;
-      if (processor && (processor?.config?.trigger === "onInsert" || processor?.config?.trigger === "onUpdate" || processor?.config?.trigger === "always")) {
+      if (processor && runProcessorOverwrite !== false && (runProcessorOverwrite || processor?.config?.trigger === "onInsert" || processor?.config?.trigger === "onUpdate" || processor?.config?.trigger === "always")) {
         const { job: processorJob, result: processorResult } = await this.processField(
           "api",
           {
@@ -4864,120 +4996,7 @@ var ExuluContext2 = class {
   };
 };
 
-// ee/agentic-retrieval/v3/context-sampler.ts
-var CACHE_TTL_MS = 60 * 60 * 1e3;
-var ContextSampler = class {
-  cache = /* @__PURE__ */ new Map();
-  async getSamples(contexts, user, role) {
-    return Promise.all(contexts.map((ctx) => this.getSample(ctx, user, role)));
-  }
-  async getSample(ctx, user, role) {
-    const cached = this.cache.get(ctx.id);
-    if (cached && Date.now() - cached.sampledAt < CACHE_TTL_MS) {
-      return cached;
-    }
-    const { db: db2 } = await postgresClient();
-    const tableName = getTableName(ctx.id);
-    const tableDefinition = convertContextToTableDefinition(ctx);
-    const customFieldNames = ctx.fields.map((f) => f.name);
-    const selectFields = ["id", "name", "external_id", ...customFieldNames];
-    let exampleItems = [];
-    try {
-      let query = db2(tableName).select(selectFields).whereNull("archived").limit(2);
-      query = applyAccessControl(tableDefinition, query, user, tableName);
-      exampleItems = await query;
-    } catch {
-    }
-    const sample = {
-      contextId: ctx.id,
-      contextName: ctx.name,
-      fields: ["name", "external_id", ...customFieldNames],
-      exampleItems,
-      sampledAt: Date.now()
-    };
-    this.cache.set(ctx.id, sample);
-    return sample;
-  }
-  /** Evict a context from cache so it's re-sampled on next use */
-  invalidate(contextId) {
-    this.cache.delete(contextId);
-  }
-};
-
-// ee/agentic-retrieval/v3/classifier.ts
-var import_ai2 = require("ai");
-var import_zod5 = require("zod");
-
-// src/utils/with-retry.ts
-async function withRetry(generateFn, maxRetries = 3) {
-  let lastError;
-  for (let attempt = 1; attempt <= maxRetries; attempt++) {
-    try {
-      return await generateFn();
-    } catch (error) {
-      lastError = error;
-      console.error(`[EXULU] generateText attempt ${attempt} failed:`, error);
-      if (attempt === maxRetries) {
-        throw error;
-      }
-      await new Promise((resolve3) => setTimeout(resolve3, Math.pow(2, attempt) * 1e3));
-    }
-  }
-  throw lastError;
-}
-
-// ee/agentic-retrieval/v3/classifier.ts
-async function classifyQuery(query, contexts, samples, model) {
-  const contextDescriptions = contexts.map((ctx) => {
-    const sample = samples.find((s) => s.contextId === ctx.id);
-    const fieldList = sample?.fields.join(", ") ?? "name, external_id";
-    const exampleStr = sample?.exampleItems.length ? `
-    Example records: ${JSON.stringify(sample.exampleItems.slice(0, 2))}` : "";
-    return `  - ${ctx.id}: ${ctx.name}
-    Description: ${ctx.description}
-    Fields: ${fieldList}${exampleStr}`;
-  }).join("\n\n");
-  const result = await withRetry(async () => {
-    const result2 = await (0, import_ai2.generateText)({
-      model,
-      temperature: 0,
-      output: import_ai2.Output.object({
-        schema: import_zod5.z.object({
-          queryType: import_zod5.z.enum(["aggregate", "list", "targeted", "exploratory"]).describe(
-            "aggregate: ONLY use when the user explicitly asks to COUNT how many documents/items/tickets exist in the knowledge base (e.g. 'how many documents about X?', 'total number of tickets'). NEVER use for: real-world statistics stored in a document, intent statements, how-to questions, error/fault descriptions, configuration questions, or any query that does not explicitly ask for a count of knowledge base entries. When in doubt, choose targeted. list: user wants to enumerate matching items/documents (show me all, list documents about). targeted: use for almost everything \u2014 specific fact, answer, configuration, how-to, error/fault, feature/behavior question. Also use for intent statements and short commands describing a desired state (phrases that state what the user wants to do or achieve, even without an explicit question word). Real-world statistics stored in documents also go here. When in doubt, choose targeted over aggregate or exploratory. exploratory: only for broad conceptual questions needing multi-source synthesis (what is the process for Z, explain how X works, general overview of topic Y)."
-          ),
-          language: import_zod5.z.string().describe("ISO 639-3 language code of the query (e.g. eng, deu, fra)"),
-          suggestedContextIds: import_zod5.z.array(import_zod5.z.enum(contexts.map((c) => c.id))).describe(
-            "IDs of knowledge bases most likely to contain the answer. Return empty array to search all contexts."
-          )
-        })
-      }),
-      toolChoice: "none",
-      system: `You are a query classifier for a multi-knowledge-base retrieval system.
-    Classify the query and identify which knowledge bases are most relevant.
-    
-    Available knowledge bases:
-    ${contextDescriptions}
-    
-    Guidelines for queryType:
-    - Use "aggregate" ONLY when the query contains explicit counting language (e.g., "how many", "count", "total number", "wie viele"). Short statements, commands, or phrases without a question word are NEVER aggregate \u2014 classify them as targeted.
-    - When in doubt between aggregate and targeted: always choose targeted.
-    
-    Guidelines for suggestedContextIds:
-    - Be conservative: only suggest contexts that are genuinely likely to contain the answer.
-      Aim for 2\u20133 focused suggestions rather than listing everything.
-    - Use each knowledge base's name and description (shown above) to judge relevance.
-    - Return an empty array only if you truly cannot determine which contexts are relevant.`,
-      prompt: `Query: ${query}`
-    });
-    return result2.output;
-  }, 3);
-  return result;
-}
-
 // ee/agentic-retrieval/v3/tools.ts
-var import_zod6 = require("zod");
-var import_ai3 = require("ai");
 function buildContextEnum(contexts) {
   return import_zod6.z.array(import_zod6.z.enum(contexts.map((c) => c.id))).describe(
     contexts.map(
@@ -5019,7 +5038,7 @@ function parseGlobalItemIds(globalIds) {
   return map;
 }
 function createRetrievalTools(params) {
-  const { contexts, user, role, updateVirtualFiles, preselectedItemsByContext } = params;
+  const { contexts, toolVariablesConfig, user, role, updateVirtualFiles, preselectedItemsByContext } = params;
   const ctxEnum = buildContextEnum(contexts);
   const count_items_or_chunks = (0, import_ai3.tool)({
     description: "Count items or chunks WITHOUT loading them into context. Use for 'how many', 'count', or 'total number of' queries.",
@@ -5152,7 +5171,7 @@ Use includeContent: true when you need the ACTUAL text to answer a question.
 
 For listing queries: always start with includeContent: false, then use dynamic tools to fetch specific pages.`,
     inputSchema: import_zod6.z.object({
-      query: import_zod6.z.string().describe("Search query about the content you're looking for"),
+      userQuery: import_zod6.z.string().describe("The original unaltered question from the user"),
       knowledge_base_id: import_zod6.z.enum(contexts.map((c) => c.id)).describe(
         contexts.map(
           (c) => `<knowledge_base id="${c.id}" name="${c.name}">${c.description}</knowledge_base>`
@@ -5171,7 +5190,7 @@ For listing queries: always start with includeContent: false, then use dynamic t
       limit: import_zod6.z.number().default(20).describe("Max chunks with content (max 20). Without content, up to 200 are returned.")
     }),
     execute: async ({
-      query,
+      userQuery,
       knowledge_base_id,
       keywords,
       searchMethod,
@@ -5182,7 +5201,8 @@ For listing queries: always start with includeContent: false, then use dynamic t
       limit
     }) => {
       const [ctx] = resolveContexts([knowledge_base_id], contexts);
-      const effectiveLimit = includeContent ? Math.min(limit ?? 20, 20) : Math.min((limit ?? 20) * 20, 400);
+      const maxResults = toolVariablesConfig?.[`${ctx.id}_|_max_results`] || 20;
+      const effectiveLimit = includeContent ? Math.min(limit ?? maxResults, maxResults) : Math.min((limit ?? maxResults) * maxResults, 400);
       const itemFilters = [];
       if (preselectedItemsByContext) {
         const contextItemIds = preselectedItemsByContext.get(knowledge_base_id);
@@ -5204,7 +5224,7 @@ For listing queries: always start with includeContent: false, then use dynamic t
       if (item_names)
         itemFilters.push({ name: { or: item_names.map((n) => ({ contains: n })) } });
       if (item_external_ids) itemFilters.push({ external_id: { in: item_external_ids } });
-      const effectiveQuery = query || keywords?.join(" ") || "";
+      const effectiveQuery = userQuery || keywords?.join(" ") || "";
       let method = mapSearchMethod(searchMethod ?? "hybrid");
       if (method === "hybridSearch" || method === "cosineDistance") {
         if (!ctx.embedder) {
@@ -5212,6 +5232,7 @@ For listing queries: always start with includeContent: false, then use dynamic t
           method = "tsvector";
         }
       }
+      const expandChunks = toolVariablesConfig?.[`${ctx.id}_|_expand_chunks`] || 0;
       try {
         const { chunks } = await ctx.search({
           query: effectiveQuery,
@@ -5224,7 +5245,11 @@ For listing queries: always start with includeContent: false, then use dynamic t
           sort: { field: "updatedAt", direction: "desc" },
           user,
           role,
-          trigger: "tool"
+          trigger: "tool",
+          expand: expandChunks > 0 ? {
+            before: expandChunks,
+            after: expandChunks
+          } : void 0
         });
         return JSON.stringify(
           chunks.map(
@@ -5996,6 +6021,46 @@ var TrajectoryLogger = class {
   }
 };
 
+// ee/agentic-retrieval/v3/context-sampler.ts
+var CACHE_TTL_MS = 60 * 60 * 1e3;
+var ContextSampler = class {
+  cache = /* @__PURE__ */ new Map();
+  async getSamples(contexts, user, role) {
+    return Promise.all(contexts.map((ctx) => this.getSample(ctx, user, role)));
+  }
+  async getSample(ctx, user, role) {
+    const cached = this.cache.get(ctx.id);
+    if (cached && Date.now() - cached.sampledAt < CACHE_TTL_MS) {
+      return cached;
+    }
+    const { db: db2 } = await postgresClient();
+    const tableName = getTableName(ctx.id);
+    const tableDefinition = convertContextToTableDefinition(ctx);
+    const customFieldNames = ctx.fields.map((f) => f.name);
+    const selectFields = ["id", "name", "external_id", ...customFieldNames];
+    let exampleItems = [];
+    try {
+      let query = db2(tableName).select(selectFields).whereNull("archived").limit(2);
+      query = applyAccessControl(tableDefinition, query, user, tableName);
+      exampleItems = await query;
+    } catch {
+    }
+    const sample = {
+      contextId: ctx.id,
+      contextName: ctx.name,
+      fields: ["name", "external_id", ...customFieldNames],
+      exampleItems,
+      sampledAt: Date.now()
+    };
+    this.cache.set(ctx.id, sample);
+    return sample;
+  }
+  /** Evict a context from cache so it's re-sampled on next use */
+  invalidate(contextId) {
+    this.cache.delete(contextId);
+  }
+};
+
 // ee/agentic-retrieval/v3/index.ts
 var sampler = new ContextSampler();
 async function* executeV3({
@@ -6003,6 +6068,7 @@ async function* executeV3({
   contexts,
   reranker,
   model,
+  toolVariablesConfig,
   user,
   role,
   customInstructions,
@@ -6028,15 +6094,39 @@ async function* executeV3({
   }
   console.log("[EXULU] v3 \u2014 classified as:", classification);
   const strategy = STRATEGIES[classification.queryType];
+  const contextSpecificInstructions = activeContexts.map((ctx) => {
+    const instructions = toolVariablesConfig?.[`${ctx.id}_|_instructions`] ?? "";
+    if (instructions) {
+      return `
+      <${ctx.id}>
+      ${instructions}
+      </${ctx.id}>
+    `;
+    } else {
+      return null;
+    }
+  }).filter(Boolean).join("\n");
   const suggestedIds = classification.suggestedContextIds;
   const fallbackIds = activeContexts.filter((c) => !suggestedIds.includes(c.id)).map((c) => c.id);
-  const contextBase = suggestedIds.length > 0 ? `Suggested priority contexts: [${suggestedIds.join(", ")}]. Also available: [${fallbackIds.join(", ")}]. Custom instructions may require searching additional or all contexts \u2014 follow them.` : `All contexts available: [${activeContexts.map((c) => c.id).join(", ")}].`;
+  let contextBase = suggestedIds.length > 0 ? `
+      Suggested priority contexts: [${suggestedIds.join(", ")}]. 
+      
+      Also available: [${fallbackIds.join(", ")}]. 
+      
+      Custom instructions may require searching additional or all contexts \u2014 follow them.` : `All contexts available: [${activeContexts.map((c) => c.id).join(", ")}].`;
   const preselectedNote = preselectedByContext?.size ? `
 SCOPE CONSTRAINT: Retrieval is scoped to preselected items/contexts. Per context: ${[...preselectedByContext.entries()].map(([ctx, ids]) => ids === null ? `${ctx} (full context)` : `${ctx} (${ids.length} item${ids.length === 1 ? "" : "s"})`).join(", ")}. All tools enforce this scope automatically. For full-context entries you may search freely; for item-restricted entries do NOT use search_items_by_name for discovery \u2014 go directly to search_content or save_search_results.` : "";
+  if (contextSpecificInstructions?.length) {
+    contextBase += `
+      Context specific instructions:
+      ${contextSpecificInstructions}
+      `;
+  }
   const contextGuidance = contextBase + preselectedNote;
   const bashToolkit = await (0, import_bash_tool.createBashTool)({ files: {} });
   const retrievalTools = createRetrievalTools({
     contexts: activeContexts,
+    toolVariablesConfig,
     user,
     role,
     updateVirtualFiles: (files) => bashToolkit.sandbox.writeFiles(files),
@@ -6091,7 +6181,8 @@ function createAgenticRetrievalToolV3({
   user,
   role,
   model,
-  preselectedItemIds
+  preselected,
+  memoryItems
 }) {
   const license = checkLicense();
   if (!license["agentic-retrieval"]) {
@@ -6144,27 +6235,57 @@ function createAgenticRetrievalToolV3({
         default: false
       },
       {
-        name: "log_trajectories",
+        name: "logging",
         description: "Save a detailed markdown + JSON log of every retrieval execution to disk. Useful for debugging and evaluation.",
         type: "boolean",
         default: false
       },
       ...contexts.map((ctx) => ({
-        name: ctx.id,
+        name: ctx.id + "_|_enabled",
         description: `Enable search in "${ctx.name}". ${ctx.description}`,
         type: "boolean",
         default: true
+      })),
+      ...contexts.map((ctx) => ({
+        name: `${ctx.id}_|_instructions`,
+        description: `Instructions for the retrieval agent about how to search in the ${ctx.name} context`,
+        type: "string",
+        default: ""
+      })),
+      ...contexts.map((ctx) => ({
+        name: `${ctx.id}_|_priority`,
+        description: `Defines in which order the context should be searched in, the higher the number the higher the priority, if contexts have the same priority they are searched in parallel`,
+        type: "number",
+        default: 0
+      })),
+      ...contexts.map((ctx) => ({
+        name: `${ctx.id}_|_max_results`,
+        description: `Defines the maximum number of results to return for the ${ctx.name} context`,
+        type: "number",
+        default: 0
+      })),
+      ...contexts.map((ctx) => ({
+        name: `${ctx.id}_|_max_steps`,
+        description: `Defines the maximum number of steps the agent is allowed to take when searching the ${ctx.name} context`,
+        type: "number",
+        default: 0
+      })),
+      ...contexts.map((ctx) => ({
+        name: `${ctx.id}_|_expand_chunks`,
+        description: `Defines if the agent automatically retrieves nearby chunks around the matched chunks, usefull if relevant content might be split up`,
+        type: "number",
+        default: 0
       }))
     ],
     inputSchema: import_zod9.z.object({
-      query: import_zod9.z.string().describe("The question or query to answer"),
+      userQuery: import_zod9.z.string().describe("The original unaltered question from the user"),
       userInstructions: import_zod9.z.string().optional().describe("Additional instructions from the user to guide retrieval"),
       confirmedContextIds: import_zod9.z.array(import_zod9.z.string()).optional().describe(
         "Knowledge base IDs explicitly confirmed by the user to be used in the retrieval. When presen only searches these contexts. "
       )
     }),
     execute: async function* ({
-      query,
+      userQuery,
       userInstructions,
       confirmedContextIds,
       toolVariablesConfig,
@@ -6182,10 +6303,10 @@ function createAgenticRetrievalToolV3({
       let managedContextEnabled = false;
       if (toolVariablesConfig) {
         configInstructions = toolVariablesConfig["instructions"] ?? "";
-        logTrajectory = toolVariablesConfig["log_trajectories"] === true || toolVariablesConfig["log_trajectories"] === "true";
+        logTrajectory = toolVariablesConfig["logging"] === true || toolVariablesConfig["logging"] === "true";
         managedContextEnabled = toolVariablesConfig["managed_context"] === true || toolVariablesConfig["managed_context"] === "true";
         activeContexts = contexts.filter(
-          (ctx) => toolVariablesConfig[ctx.id] === true || toolVariablesConfig[ctx.id] === "true" || toolVariablesConfig[ctx.id] === 1
+          (ctx) => toolVariablesConfig[ctx.id + "_|_enabled"] === true || toolVariablesConfig[ctx.id + "_|_enabled"] === "true" || toolVariablesConfig[ctx.id + "_|_enabled"] === 1
         );
         if (activeContexts.length === 0) activeContexts = contexts;
         requiresPreselectedContexts = toolVariablesConfig["require_preselected_contexts"] === true || toolVariablesConfig["require_preselected_contexts"] === "true";
@@ -6195,13 +6316,13 @@ function createAgenticRetrievalToolV3({
         }
       }
       console.log("[EXULU] Managed context enabled:", managedContextEnabled);
-      console.log("[EXULU] Preselected item IDs:", preselectedItemIds);
-      if (managedContextEnabled && !preselectedItemIds?.length) {
+      console.log("[EXULU] Preselected item IDs:", preselected);
+      if (managedContextEnabled && !preselected?.length) {
         console.log("[EXULU] Managed context was enabled for the agentic retrieval tool. This means that the user must preselect items that the agentic retrieval tool will search in, please notify the user to preselect items before executing the tool.");
         yield { result: "Managed context was enabled for the agentic retrieval tool. This means that the user must preselect items that the agentic retrieval tool will search in, please notify the user to preselect items before executing the tool." };
         return;
       }
-      if (requiresPreselectedContexts && !confirmedContextIds?.length && !preselectedItemIds?.length) {
+      if (requiresPreselectedContexts && !confirmedContextIds?.length && !preselected?.length) {
         console.log("[EXULU] The user must choose between the available contexts before executing the tool. The available contexts are: " + activeContexts.map((c) => c.id).join(", ") + ". If the question_ask tool is available use that to ask the user which contexts they want to search in, otherwise just ask them in plain text.");
         yield { result: "The user must choose between the available contexts before executing the tool, the available contexts are: " + activeContexts.map((c) => c.id).join(", ") + ". If the question_ask tool is available use that to ask the user which contexts they want to search in, otherwise just ask them in plain text." };
         return;
@@ -6212,21 +6333,43 @@ function createAgenticRetrievalToolV3({
         if (filtered.length > 0) activeContexts = filtered;
       }
       const combinedInstructions = [
-        configInstructions ? `Configuration instructions: ${configInstructions}` : "",
-        adminInstructions ? `Admin instructions: ${adminInstructions}` : "",
-        userInstructions ? `User instructions: ${userInstructions}` : ""
+        configInstructions ? `
+        Configuration instructions: 
+        <configuration_instructions>
+        ${configInstructions}
+        </configuration_instructions>
+        ` : "",
+        adminInstructions ? `
+        Admin instructions: 
+        <admin_instructions>
+        ${adminInstructions}
+        </admin_instructions>
+        ` : "",
+        userInstructions ? `
+        User instructions: 
+        <user_instructions>
+        ${userInstructions}
+        </user_instructions>
+        ` : "",
+        memoryItems ? `
+        Relevant memories (these are items that the agent has retrieved from the memory context and are relevant to the query):
+        <relevant_memories>
+        ${memoryItems?.map((item) => JSON.stringify(item)).join("\n")}
+        </relevant_memories>
+        ` : ""
       ].filter(Boolean).join("\n");
       for await (const output of executeV3({
-        query,
+        query: userQuery,
         contexts: activeContexts,
         reranker: configuredReranker,
+        toolVariablesConfig,
         model,
         user,
         role,
         customInstructions: combinedInstructions || void 0,
         logTrajectory,
         sessionId: sessionID,
-        preselectedItemIds
+        preselectedItemIds: preselected
       })) {
         yield { result: JSON.stringify(output) };
       }
@@ -11046,6 +11189,7 @@ var ExuluProvider = class {
       });
     }
     let memoryContext = "";
+    let memoryItems;
     if (agent?.memory && contexts?.length && query) {
       const context = contexts.find((context2) => context2.id === agent?.memory);
       if (!context) {
@@ -11068,6 +11212,7 @@ var ExuluProvider = class {
         page: 1
       });
       if (result?.chunks?.length) {
+        memoryItems = result.chunks;
         memoryContext = `
                   Pre-fetched relevant information for this query:
   
@@ -11188,7 +11333,8 @@ When a tool execution is not approved by the user, do not retry it unless explic
             project,
             sessionItems,
             model,
-            agent
+            agent,
+            memoryItems
           ),
           stopWhen: [(0, import_ai8.stepCountIs)(maxStepCount || 5)]
           // make configurable
@@ -11265,7 +11411,8 @@ When a tool execution is not approved by the user, do not retry it unless explic
           project,
           sessionItems,
           model,
-          agent
+          agent,
+          memoryItems
         ),
         stopWhen: [(0, import_ai8.stepCountIs)(maxStepCount || 5)]
       });
@@ -11447,6 +11594,7 @@ ${extractedText}
       });
     }
     let memoryContext = "";
+    let memoryItems;
     if (agent?.memory && contexts?.length && query) {
       const context = contexts.find((context2) => context2.id === agent?.memory);
       if (!context) {
@@ -11470,6 +11618,7 @@ ${extractedText}
       });
       import_fs.default.writeFileSync("pre-fetched-relevant-information.json", JSON.stringify(result2, null, 2));
       if (result2?.chunks?.length) {
+        memoryItems = result2.chunks;
         memoryContext = `
                   <pre-fetched relevant information for this query>:
   
@@ -11489,9 +11638,6 @@ ${extractedText}
     const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
     let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
     system += "\n\n" + genericContext;
-    if (memoryContext) {
-      system += "\n\n" + memoryContext;
-    }
     const includesContextSearchTool = currentTools?.some(
       (tool5) => tool5.name.toLowerCase().includes("context_search") || tool5.id.includes("context_search") || tool5.type === "context"
     );
@@ -11576,7 +11722,8 @@ When a tool execution is not approved by the user, do not retry it unless explic
         project,
         sessionItems,
         model,
-        agent
+        agent,
+        memoryItems
       ),
       onError: (error) => {
         console.error("[EXULU] chat stream error.", error);
@@ -11691,27 +11838,141 @@ var providerRateLimiter = async (key, windowSeconds, limit, points) => {
 // src/exulu/openai-gateway.ts
 var import_express2 = require("express");
 var import_ai9 = require("ai");
+
+// src/exulu/openai-transformer.ts
+function transformStreamChunk(chunk, ctx) {
+  const base = {
+    id: ctx.completionId,
+    object: "chat.completion.chunk",
+    created: ctx.created,
+    model: ctx.modelId
+  };
+  switch (chunk.type) {
+    case "text-delta":
+      return {
+        ...base,
+        choices: [{ index: 0, delta: { content: chunk.text }, finish_reason: null }]
+      };
+    case "tool-input-start":
+      return {
+        ...base,
+        choices: [
+          {
+            index: 0,
+            delta: {
+              tool_calls: [
+                {
+                  index: 0,
+                  id: chunk.id,
+                  type: "function",
+                  function: { name: chunk.toolName, arguments: "" }
+                }
+              ]
+            },
+            finish_reason: null
+          }
+        ]
+      };
+    case "tool-input-delta":
+      return {
+        ...base,
+        choices: [
+          {
+            index: 0,
+            delta: { tool_calls: [{ index: 0, function: { arguments: chunk.delta } }] },
+            finish_reason: null
+          }
+        ]
+      };
+    case "finish": {
+      const inputTokens = chunk.usage?.inputTokens ?? 0;
+      const outputTokens = chunk.usage?.outputTokens ?? 0;
+      const finishReason = chunk.finishReason === "tool-calls" ? "tool_calls" : "stop";
+      return {
+        ...base,
+        choices: [{ index: 0, delta: {}, finish_reason: finishReason }],
+        usage: {
+          prompt_tokens: inputTokens,
+          completion_tokens: outputTokens,
+          total_tokens: inputTokens + outputTokens
+        }
+      };
+    }
+    default:
+      return null;
+  }
+}
+function transformCompletion(text, inputTokens, outputTokens, ctx) {
+  return {
+    id: ctx.completionId,
+    object: "chat.completion",
+    created: ctx.created,
+    model: ctx.modelId,
+    choices: [
+      {
+        index: 0,
+        message: { role: "assistant", content: text },
+        finish_reason: "stop"
+      }
+    ],
+    usage: {
+      prompt_tokens: inputTokens,
+      completion_tokens: outputTokens,
+      total_tokens: inputTokens + outputTokens
+    }
+  };
+}
+
+// src/exulu/openai-gateway.ts
 var import_node_crypto4 = require("crypto");
 var import_crypto_js6 = __toESM(require("crypto-js"), 1);
 var import_express3 = __toESM(require("express"), 1);
-function convertOpenAIMessagesToCoreMessages(messages) {
+function convertOpenAIToolsToAiSdkTools(tools) {
+  return Object.fromEntries(
+    tools.map((t) => {
+      const params = t.function.parameters ?? {};
+      return [
+        t.function.name,
+        {
+          description: t.function.description ?? "",
+          inputSchema: (0, import_ai9.jsonSchema)({
+            type: "object",
+            properties: params.properties ?? {},
+            ...params.required ? { required: params.required } : {}
+          })
+        }
+      ];
+    })
+  );
+}
+function convertOpenAIMessagesToModelMessages(messages) {
   const systemParts = [];
   const coreMessages = [];
+  const toolCallIdToName = /* @__PURE__ */ new Map();
   for (const msg of messages) {
     if (msg.role === "system") {
       systemParts.push(typeof msg.content === "string" ? msg.content : "");
       continue;
     }
     if (msg.role === "user") {
+      const last = coreMessages[coreMessages.length - 1];
       if (typeof msg.content === "string") {
-        coreMessages.push({ role: "user", content: msg.content });
+        if (last?.role === "user" && typeof last.content === "string") {
+          last.content += "\n\n" + msg.content;
+        } else {
+          coreMessages.push({ role: "user", content: msg.content });
+        }
       } else if (Array.isArray(msg.content)) {
         const parts = msg.content.flatMap((part) => {
           if (part.type === "text") return [{ type: "text", text: part.text }];
           if (part.type === "image_url") return [{ type: "image", image: part.image_url.url }];
           return [];
         });
-        coreMessages.push({ role: "user", content: parts });
+        if (last?.role === "user" && Array.isArray(last.content)) {
+          last.content.push(...parts);
+        } else {
+          coreMessages.push({ role: "user", content: parts });
+        }
       }
       continue;
     }
@@ -11722,11 +11983,20 @@ function convertOpenAIMessagesToCoreMessages(messages) {
           parts.push({ type: "text", text: msg.content });
         }
         for (const tc of msg.tool_calls) {
+          toolCallIdToName.set(tc.id, tc.function.name);
+          const rawArgs = tc.function.arguments;
+          const input = rawArgs == null ? {} : typeof rawArgs === "object" ? rawArgs : (() => {
+            try {
+              return JSON.parse(rawArgs);
+            } catch {
+              return {};
+            }
+          })();
           parts.push({
             type: "tool-call",
             toolCallId: tc.id,
             toolName: tc.function.name,
-            args: JSON.parse(tc.function.arguments || "{}")
+            input
           });
         }
         coreMessages.push({ role: "assistant", content: parts });
@@ -11739,13 +12009,17 @@ function convertOpenAIMessagesToCoreMessages(messages) {
       continue;
     }
     if (msg.role === "tool") {
+      const toolCallId = msg.tool_call_id ?? "";
+      const toolName = toolCallIdToName.get(toolCallId) ?? "unknown";
+      const resultText = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content);
       coreMessages.push({
         role: "tool",
         content: [
           {
             type: "tool-result",
-            toolCallId: msg.tool_call_id ?? "",
-            result: msg.content
+            toolCallId,
+            toolName,
+            output: { type: "text", value: resultText }
           }
         ]
       });
@@ -11862,8 +12136,32 @@ var registerOpenAIGatewayRoutes = async (app, providers, tools, contexts, config
     }
   );
   app.post(
-    "/gateway/open-ai/v1/chat/completions",
+    ["/gateway/open-ai/v1/chat/completions", "/gateway/open-ai/v1/completions"],
     import_express3.default.json({ limit: REQUEST_SIZE_LIMIT }),
+    (req, _res, next) => {
+      console.log("[OPENAI GATEWAY] incoming request:", {
+        url: req.originalUrl,
+        method: req.method,
+        headers: {
+          authorization: req.headers["authorization"] ? "[present]" : "[missing]",
+          "x-api-key": req.headers["x-api-key"] ? "[present]" : "[missing]",
+          "exulu-api-key": req.headers["exulu-api-key"] ? "[present]" : "[missing]",
+          "content-type": req.headers["content-type"]
+        },
+        body: {
+          model: req.body?.model,
+          stream: req.body?.stream,
+          messagesCount: req.body?.messages?.length,
+          hasPrompt: typeof req.body?.prompt === "string",
+          tools: req.body?.tools
+        }
+      });
+      if (typeof req.body.prompt === "string") {
+        req.body.messages = [{ role: "user", content: req.body.prompt }];
+        delete req.body.prompt;
+      }
+      next();
+    },
     async (req, res) => {
       try {
         const { db: db2 } = await postgresClient();
@@ -11969,8 +12267,10 @@ var registerOpenAIGatewayRoutes = async (app, providers, tools, contexts, config
           languageModel,
           agent
         );
+        const clientTools = Array.isArray(req.body.tools) ? req.body.tools : [];
+        const activeTools = clientTools.length > 0 ? convertOpenAIToolsToAiSdkTools(clientTools) : convertedTools;
         const openaiMessages = req.body.messages ?? [];
-        const { systemPrompt: requestSystemPrompt, coreMessages } = convertOpenAIMessagesToCoreMessages(openaiMessages);
+        const { systemPrompt: requestSystemPrompt, coreMessages } = convertOpenAIMessagesToModelMessages(openaiMessages);
         const agentInstructions = agent.instructions ?? "";
         const systemParts = [
           agentInstructions ? `You are an agent named: ${agent.name}
@@ -11982,7 +12282,8 @@ ${project.description}` : ""}` : "",
         const systemPrompt = systemParts.join("\n\n");
         const completionId = `chatcmpl-${(0, import_node_crypto4.randomUUID)()}`;
         const created = Math.floor(Date.now() / 1e3);
-        const hasTools = Object.keys(convertedTools).length > 0;
+        const hasTools = Object.keys(activeTools).length > 0;
+        const ctx = { completionId, created, modelId };
         if (req.body.stream === true) {
           res.setHeader("Content-Type", "text/event-stream");
           res.setHeader("Cache-Control", "no-cache");
@@ -11991,9 +12292,9 @@ ${project.description}` : ""}` : "",
             model: languageModel,
             system: systemPrompt || void 0,
             messages: coreMessages,
-            tools: hasTools ? convertedTools : void 0,
+            tools: hasTools ? activeTools : void 0,
             maxRetries: 2,
-            stopWhen: [(0, import_ai9.stepCountIs)(5)],
+            stopWhen: clientTools.length > 0 ? void 0 : [(0, import_ai9.stepCountIs)(5)],
             onError: (error) => {
               console.error("[OPENAI GATEWAY] stream error:", error);
             }
@@ -12012,83 +12313,17 @@ ${project.description}` : ""}` : "",
           let inputTokens = 0;
           let outputTokens = 0;
           for await (const chunk of result.fullStream) {
-            if (chunk.type === "text-delta") {
-              res.write(
-                `data: ${JSON.stringify({
-                  id: completionId,
-                  object: "chat.completion.chunk",
-                  created,
-                  model: modelId,
-                  choices: [{ index: 0, delta: { content: chunk.text }, finish_reason: null }]
-                })}
-
-`
-              );
-            } else if (chunk.type === "tool-input-start") {
-              res.write(
-                `data: ${JSON.stringify({
-                  id: completionId,
-                  object: "chat.completion.chunk",
-                  created,
-                  model: modelId,
-                  choices: [
-                    {
-                      index: 0,
-                      delta: {
-                        tool_calls: [
-                          {
-                            index: 0,
-                            id: chunk.id,
-                            type: "function",
-                            function: { name: chunk.toolName, arguments: "" }
-                          }
-                        ]
-                      },
-                      finish_reason: null
-                    }
-                  ]
-                })}
-
-`
-              );
-            } else if (chunk.type === "tool-input-delta") {
-              res.write(
-                `data: ${JSON.stringify({
-                  id: completionId,
-                  object: "chat.completion.chunk",
-                  created,
-                  model: modelId,
-                  choices: [
-                    {
-                      index: 0,
-                      delta: { tool_calls: [{ index: 0, function: { arguments: chunk.delta } }] },
-                      finish_reason: null
-                    }
-                  ]
-                })}
-
-`
-              );
-            } else if (chunk.type === "finish") {
-              inputTokens = chunk.usage?.inputTokens ?? 0;
-              outputTokens = chunk.usage?.outputTokens ?? 0;
-              const finishReason = chunk.finishReason === "tool-calls" ? "tool_calls" : "stop";
-              res.write(
-                `data: ${JSON.stringify({
-                  id: completionId,
-                  object: "chat.completion.chunk",
-                  created,
-                  model: modelId,
-                  choices: [{ index: 0, delta: {}, finish_reason: finishReason }],
-                  usage: {
-                    prompt_tokens: inputTokens,
-                    completion_tokens: outputTokens,
-                    total_tokens: inputTokens + outputTokens
-                  }
-                })}
+            console.log("[OPENAI GATEWAY] chunk:", chunk.type);
+            const openAIChunk = transformStreamChunk(chunk, ctx);
+            if (openAIChunk) {
+              if (chunk.type === "finish") {
+                inputTokens = chunk.usage?.inputTokens ?? 0;
+                outputTokens = chunk.usage?.outputTokens ?? 0;
+                console.log("[OPENAI GATEWAY] finish_reason:", openAIChunk.choices[0]?.finish_reason);
+              }
+              res.write(`data: ${JSON.stringify(openAIChunk)}
 
-`
-              );
+`);
             }
           }
           res.write("data: [DONE]\n\n");
@@ -12099,29 +12334,12 @@ ${project.description}` : ""}` : "",
             model: languageModel,
             system: systemPrompt || void 0,
             messages: coreMessages,
-            tools: hasTools ? convertedTools : void 0,
+            tools: hasTools ? activeTools : void 0,
             maxRetries: 2,
-            stopWhen: [(0, import_ai9.stepCountIs)(5)]
-          });
-          res.json({
-            id: completionId,
-            object: "chat.completion",
-            created,
-            model: agentId,
-            choices: [
-              {
-                index: 0,
-                message: { role: "assistant", content: text },
-                finish_reason: "stop"
-              }
-            ],
-            usage: {
-              prompt_tokens: usage.promptTokens,
-              completion_tokens: usage.completionTokens,
-              total_tokens: usage.totalTokens
-            }
+            stopWhen: clientTools.length > 0 ? void 0 : [(0, import_ai9.stepCountIs)(5)]
           });
-          await writeStatistics(agent, project, user, usage.promptTokens, usage.completionTokens);
+          res.json(transformCompletion(text, usage.inputTokens ?? 0, usage.outputTokens ?? 0, ctx));
+          await writeStatistics(agent, project, user, usage.inputTokens ?? 0, usage.outputTokens ?? 0);
         }
       } catch (error) {
         console.error("[OPENAI GATEWAY] /v1/chat/completions error:", error);
@@ -12169,6 +12387,7 @@ var {
   workflowTemplatesSchema: workflowTemplatesSchema2,
   rbacSchema: rbacSchema2,
   promptLibrarySchema: promptLibrarySchema2,
+  contextPresetsSchema: contextPresetsSchema2,
   embedderSettingsSchema: embedderSettingsSchema2,
   promptFavoritesSchema: promptFavoritesSchema2,
   statisticsSchema: statisticsSchema2
@@ -12216,6 +12435,7 @@ var createExpressRoutes = async (app, providers, tools, contexts, config, evals,
       projectsSchema2(),
       jobResultsSchema2(),
       promptLibrarySchema2(),
+      contextPresetsSchema2(),
       embedderSettingsSchema2(),
       promptFavoritesSchema2(),
       evalRunsSchema2(),
@@ -17282,6 +17502,7 @@ var {
   projectsSchema: projectsSchema3,
   jobResultsSchema: jobResultsSchema3,
   promptLibrarySchema: promptLibrarySchema3,
+  contextPresetsSchema: contextPresetsSchema3,
   embedderSettingsSchema: embedderSettingsSchema3,
   promptFavoritesSchema: promptFavoritesSchema3
 } = coreSchemas.get();
@@ -17319,6 +17540,7 @@ var up = async function(knex) {
     projectsSchema3(),
     jobResultsSchema3(),
     promptLibrarySchema3(),
+    contextPresetsSchema3(),
     embedderSettingsSchema3(),
     promptFavoritesSchema3(),
     rbacSchema3(),