@exulu/backend 3.7.4 → 4.1.0

package/dist/index.cjs

@@ -4064,7 +4064,7 @@ async function validatePythonEnvironment(packageRoot, checkPackages = true) {
     };
   }
   if (checkPackages) {
-    const criticalPackages = ["docling", "transformers"];
+    const criticalPackages = ["pypdf", "transformers"];
     const missingPackages = [];
     for (const pkg of criticalPackages) {
       try {
@@ -7151,7 +7151,19 @@ var init_config2 = __esm({
       pinBoost: import_zod11.z.number().min(0).max(1).default(0.15),
       identifierBoost: import_zod11.z.number().min(0).max(1).default(0.15),
       pageWindow: import_zod11.z.number().int().min(0).default(1),
-      maxQueriesPerContext: import_zod11.z.number().int().positive().default(5)
+      maxQueriesPerContext: import_zod11.z.number().int().positive().default(5),
+      /**
+       * Orchestration engine. "v1" is the sequential flow every agent ran before 2026-09;
+       * "v2" merges the phase-1 LLM hops and runs identifier pins in parallel. Per agent,
+       * so a candidate agent can run v2 while the production agent stays on v1.
+       */
+      engine: import_zod11.z.enum(["v1", "v2"]).default("v1"),
+      /** v2 sub-features; each can be switched off on its own to bisect a regression. */
+      v2: import_zod11.z.object({
+        mergedMemoryCall: import_zod11.z.boolean().default(true),
+        mergedRoutingCall: import_zod11.z.boolean().default(true),
+        parallelPins: import_zod11.z.boolean().default(true)
+      }).default({ mergedMemoryCall: true, mergedRoutingCall: true, parallelPins: true })
     });
     boolVal = (v) => v === true || v === "true" || v === 1;
     strVal = (v, fallback) => typeof v === "string" && v.length > 0 ? v : fallback;
@@ -7246,6 +7258,23 @@ var init_micro_call = __esm({
   }
 });
 
+// ee/agentic-retrieval/pipeline/timing.ts
+async function withTiming(sink, key, work, now = Date.now) {
+  if (!sink) return work();
+  const started = now();
+  try {
+    return await work();
+  } finally {
+    sink[key] = Math.max(0, now() - started);
+  }
+}
+var init_timing = __esm({
+  "ee/agentic-retrieval/pipeline/timing.ts"() {
+    "use strict";
+    init_cjs_shims();
+  }
+});
+
 // ee/agentic-retrieval/pipeline/text-utils.ts
 function extractIdentifierTokens(parts) {
   const tokens = /* @__PURE__ */ new Set();
@@ -7553,6 +7582,49 @@ If the question references none, return hasMatches false and an empty array.`;
 });
 
 // ee/agentic-retrieval/pipeline/routing.ts
+async function runMergedRoutingCall({
+  model,
+  question,
+  knownIdentifiers,
+  enabledContexts,
+  routingRules,
+  extraInstructions
+}) {
+  const kbListing = enabledContexts.map((c) => `- ${c.id}: ${c.name}${c.description ? " \u2014 " + c.description : ""}`).join("\n");
+  const rulesLines = routingRules.map((r) => `- ${r.id} (${r.label}): ${r.description}`).join("\n");
+  const system = `You analyse the user's request and answer ${routingRules.length ? "three" : "two"} questions in one go.
+
+A. DOCUMENT / PAGE REFERENCE (docPage):
+${buildDocPagePrompt(knownIdentifiers)}
+
+B. EXPLICIT KNOWLEDGE BASE REQUEST (explicitlyRequestedKnowledgeBases): check if the user has EXPLICITLY asked you to search in one or multiple of the following knowledge bases:
+${kbListing}
+EXPLICIT means the user names a knowledge base or clearly commands searching a specific source (e.g. "search in the tickets", "look this up in the manuals KB"). A question that merely CONCERNS a topic related to a knowledge base's name or contents (e.g. asking about software changes, norms, or a product) is NOT an explicit request. When in doubt, return an empty array. If explicit, return the knowledge base ids.
+` + (routingRules.length ? `
+C. CLASSIFICATION (classification): classify the request into exactly one of these categories:
+${rulesLines}` + (extraInstructions ? `
+<instructions>
+${extraInstructions}
+</instructions>` : "") : "");
+  const ids = enabledContexts.map((c) => c.id);
+  const schema = import_zod13.z.object({
+    docPage: import_zod13.z.object({
+      hasFilenameHint: import_zod13.z.boolean(),
+      filenameHints: import_zod13.z.array(import_zod13.z.string()).optional(),
+      hasPageHint: import_zod13.z.boolean(),
+      pageNumber: import_zod13.z.number().int().nullable().optional()
+    }),
+    explicitlyRequestedKnowledgeBases: import_zod13.z.array(import_zod13.z.enum(ids)),
+    ...routingRules.length ? {
+      classification: import_zod13.z.object({
+        ruleId: import_zod13.z.enum(routingRules.map((r) => r.id)),
+        reason: import_zod13.z.string()
+      })
+    } : {}
+  });
+  const { output } = await microCall({ model, system, messages: [{ role: "user", content: question }], schema });
+  return output;
+}
 async function runRoutingPhase(opts) {
   const {
     question,
@@ -7562,6 +7634,7 @@ async function runRoutingPhase(opts) {
     preselectedItems,
     extraInstructions,
     knownIdentifiers = [],
+    mergedCall = false,
     model
   } = opts;
   try {
@@ -7581,7 +7654,23 @@ async function runRoutingPhase(opts) {
 ` + enabledContexts.map((c) => `- ${c.id}: ${c.name}${c.description ? " \u2014 " + c.description : ""}`).join("\n") + `
 EXPLICIT means the user names a knowledge base or clearly commands searching a specific source (e.g. "search in the tickets", "look this up in the manuals KB"). A question that merely CONCERNS a topic related to a knowledge base's name or contents (e.g. asking about software changes, norms, or a product) is NOT an explicit request \u2014 classification routing handles those. When in doubt, return an empty array.
 If explicit, return the knowledge base ids. If not, return an empty array.`;
-    const [docPageRaw, explicitKBRaw] = await Promise.all([
+    const wantsClassification = routingRules.length > 0 && preselectedItems.size === 0;
+    const merged = mergedCall ? await withTiming(
+      opts.timings,
+      "routing.mergedMs",
+      () => runMergedRoutingCall({
+        model,
+        question,
+        knownIdentifiers,
+        enabledContexts,
+        routingRules: wantsClassification ? routingRules : [],
+        extraInstructions
+      }).catch((err) => {
+        console.warn("[EXULU pipeline] merged routing call failed \u2014 falling back to the v1 hops.", err);
+        return null;
+      })
+    ) : null;
+    const [docPageRaw, explicitKBRaw] = merged ? [{ output: merged.docPage }, { output: { explicitlyRequestedKnowledgeBases: merged.explicitlyRequestedKnowledgeBases } }] : await withTiming(opts.timings, "routing.detectMs", () => Promise.all([
       (async () => {
         try {
           return await microCall({
@@ -7623,7 +7712,7 @@ If explicit, return the knowledge base ids. If not, return an empty array.`;
           return { output: { explicitlyRequestedKnowledgeBases: [] } };
         }
       })()
-    ]);
+    ]));
     const userPinnedItemIdsByContext = /* @__PURE__ */ new Map();
     let userRequestedPage = null;
     if (docPageRaw.output.hasFilenameHint && docPageRaw.output.filenameHints?.length) {
@@ -7701,7 +7790,7 @@ ${extraInstructions}
 </instructions>`;
       }
       try {
-        const { output: classified } = await microCall({
+        const { output: classified } = merged?.classification ? { output: merged.classification } : await withTiming(opts.timings, "routing.classifyMs", () => microCall({
           model,
           system: classifyPrompt,
           messages: [{ role: "user", content: question }],
@@ -7709,7 +7798,7 @@ ${extraInstructions}
             ruleId: import_zod13.z.enum(ruleIds),
             reason: import_zod13.z.string()
           })
-        });
+        }));
         const matchedRule = routingRules.find((r) => r.id === classified.ruleId);
         if (matchedRule) {
           const main = matchedRule.main.filter((id) => enabledIds.has(id));
@@ -7769,6 +7858,7 @@ var init_routing = __esm({
     init_cjs_shims();
     import_zod13 = require("zod");
     init_micro_call();
+    init_timing();
     init_prefilter();
     init_text_utils();
     MAX_USER_PIN_MATCHES = 8;
@@ -7910,7 +8000,8 @@ async function recallMemoryByKeywords({
   importantKeyword,
   user,
   role,
-  memoryContext
+  memoryContext,
+  timings
 }) {
   const allKeywords = [
     ...new Set(
@@ -7927,7 +8018,7 @@ async function recallMemoryByKeywords({
     ...new Set(allKeywords.flatMap(deriveKeywordVariants).map(stripSeparators))
   ].filter((v) => v.length >= 4);
   if (!allVariants.length) return [];
-  const items = await loadMemoryItems(memoryContext);
+  const items = await withTiming(timings, "memory.keywordRecall.itemsMs", () => loadMemoryItems(memoryContext));
   const scored = [];
   for (const item of items) {
     const haystack = stripSeparators(
@@ -7949,14 +8040,14 @@ async function recallMemoryByKeywords({
     "[EXULU pipeline] keyword-triggered memory matches:",
     topMatches.map((s) => `${s.name} (hits=${s.hits}, important=${s.importantHit})`)
   );
-  const chunks = await singleSearch({
+  const chunks = await withTiming(timings, "memory.keywordRecall.searchMs", () => singleSearch({
     query: allKeywords.join(", "),
-    config: { method: "hybridSearch", cutoffs: void 0, limit: 50 },
+    config: { method: "tsvector", cutoffs: void 0, limit: 50 },
     user,
     role,
     pinnedItemIds: topMatches.map((s) => s.id),
     context: memoryContext
-  });
+  }));
   return chunks;
 }
 function neutralResult(question, keywords, importantKeyword, steps = []) {
@@ -7970,7 +8061,103 @@ function neutralResult(question, keywords, importantKeyword, steps = []) {
     steps
   };
 }
+async function runMergedMemoryCall({
+  model,
+  retrievedMemory,
+  question,
+  keywords,
+  importantKeyword,
+  memoryConfig,
+  glossary
+}) {
+  const glossaryText = glossary.length > 0 ? `
+The organization's documents use the following abbreviations/terms:
+${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
+  const system = `
+    You review the shared company memory for the user's question and answer FOUR questions in one go.
+
+    1. RELEVANCE (relevantChunkIds): return the chunk_ids of chunks containing information relevant to the
+    question, or an empty array. Be generous: include chunks that are topically related, share key
+    terminology, describe the same symptom from a different angle, or could plausibly help diagnose the
+    issue \u2014 even if they don't answer the question directly. Memory entries are deliberately broad,
+    hand-curated hints written by domain experts; the user's wording will rarely match the memory verbatim.
+    When in doubt, include the chunk.
+
+    2. OVERRIDE (override): decide whether ONE of the relevant chunks should become the AUTHORITATIVE basis
+    of the answer, taking precedence over the official documentation even if the documents state something
+    different. This is a deliberately STRICT check. Set overrides=true ONLY if a single chunk, on its own,
+    contains a DIRECT and SUFFICIENT answer to exactly what the user asked. Being topically related, sharing
+    terminology, describing the same component, or only partially addressing the question is NOT sufficient:
+    then set overrides=false. When in doubt, set overrides=false. Memory entries may capture field experience
+    that the manuals get wrong, so a confident, direct match is meant to win over the documents.
+    ${memoryConfig.override ? "" : "(Override is disabled for this agent: return overrides=false.)"}
+
+    3. FILE PRIORITIZATION (filePrioritization): set shouldPrioritizeFiles=true only if a relevant memory
+    entry explicitly says to look in, prioritize, prefer, or always search a particular document, file, or
+    file family (for example "When asked about X, always search in Y-Dateien first"). General background
+    facts, glossaries, or synonyms are NOT a file prioritization instruction. When true, return
+    fileNameHints exactly as referenced in the memory, bare names without folder paths.
+    ${memoryConfig.filePrioritization ? "" : "(File prioritization is disabled for this agent: return false.)"}
+
+    4. QUERY AUGMENTATION (augmentation): if, and only if, the relevant memory (or the glossary below)
+    contains synonyms or similar terms for what the user asked, return the user question and keywords
+    updated to include those synonyms \u2014 always keeping the original wording as well. Otherwise return the
+    original question, keywords and important keyword unchanged.
+    ${memoryConfig.queryAugmentation ? "" : "(Query augmentation is disabled for this agent: return the originals.)"}
+
+    <memory_chunks>
+    ${retrievedMemory.map((chunk) => `- ${chunk.chunk_id}: ${chunk.item_name} - ${chunk.chunk_content}`).join("\n")}
+    </memory_chunks>
+    ${glossaryText}
+  `;
+  const { output } = await microCall({
+    model,
+    system,
+    messages: [
+      {
+        role: "user",
+        content: `
+        <user_question>${question}</user_question>
+        <relevant_keywords>${keywords.join(", ")}</relevant_keywords>
+        <important_keyword>${importantKeyword}</important_keyword>
+        `
+      }
+    ],
+    schema: import_zod14.z.object({
+      relevantChunkIds: import_zod14.z.array(import_zod14.z.string()).describe("chunk_ids (UUIDs at the start of each bullet) of relevant chunks; empty array if none."),
+      override: import_zod14.z.object({
+        overrides: import_zod14.z.boolean().describe("True ONLY if a chunk directly and sufficiently answers the question."),
+        confidence: import_zod14.z.enum(["high", "medium", "low"]),
+        authoritativeChunkIds: import_zod14.z.array(import_zod14.z.string()).describe("chunk_ids that directly answer the question; empty if overrides is false."),
+        reason: import_zod14.z.string().describe("One short sentence.")
+      }),
+      filePrioritization: import_zod14.z.object({
+        shouldPrioritizeFiles: import_zod14.z.boolean(),
+        fileNameHints: import_zod14.z.array(import_zod14.z.string()).optional()
+      }),
+      augmentation: import_zod14.z.object({
+        updatedUserQuestion: import_zod14.z.string(),
+        updatedRelevantKeywords: import_zod14.z.array(import_zod14.z.string()),
+        updatedImportantKeyword: import_zod14.z.string()
+      })
+    })
+  });
+  return output;
+}
+function mergedFollowups(merged, memoryConfig, hasAugmentationContent, question, importantKeyword) {
+  const overrideResult = {
+    output: memoryConfig.override ? merged.override : { overrides: false, confidence: "low", authoritativeChunkIds: [], reason: "" }
+  };
+  const fileResult = {
+    output: memoryConfig.filePrioritization ? merged.filePrioritization : { shouldPrioritizeFiles: false, fileNameHints: [] }
+  };
+  const queryResult = {
+    output: memoryConfig.queryAugmentation && hasAugmentationContent ? merged.augmentation : { updatedUserQuestion: question, updatedRelevantKeywords: [], updatedImportantKeyword: importantKeyword }
+  };
+  return [overrideResult, fileResult, queryResult];
+}
 async function runMemoryPhase({
+  timings,
   memoryChunks,
   memoryContext,
   question,
@@ -7981,7 +8168,8 @@ async function runMemoryPhase({
   model,
   memoryConfig,
   glossary,
-  documentContexts
+  documentContexts,
+  mergedCall = false
 }) {
   try {
     if (!memoryConfig.enabled || memoryChunks.length === 0 && !memoryContext) {
@@ -7991,13 +8179,14 @@ async function runMemoryPhase({
     let retrieved_memory = [...memoryChunks];
     if (memoryContext) {
       try {
-        const keywordMatched = await recallMemoryByKeywords({
+        const keywordMatched = await withTiming(timings, "memory.keywordRecallMs", () => recallMemoryByKeywords({
           keywords,
           importantKeyword,
           user,
           role,
-          memoryContext
-        });
+          memoryContext,
+          timings
+        }));
         if (keywordMatched.length > 0) {
           const seen = new Set(retrieved_memory.map((c) => c.chunk_id));
           const additions = keywordMatched.filter((c) => !seen.has(c.chunk_id));
@@ -8022,8 +8211,24 @@ async function runMemoryPhase({
     </memory_chunks>
     `;
     let relevantMemoryChunks = [];
+    let mergedOutput;
     try {
-      const { output: output_relevant_memory } = await microCall({
+      if (mergedCall) {
+        mergedOutput = await withTiming(
+          timings,
+          "memory.mergedMs",
+          () => runMergedMemoryCall({
+            model,
+            retrievedMemory: retrieved_memory,
+            question,
+            keywords,
+            importantKeyword,
+            memoryConfig,
+            glossary
+          })
+        );
+      }
+      const { output: output_relevant_memory } = mergedOutput ? { output: { relevantChunkIds: mergedOutput.relevantChunkIds } } : await withTiming(timings, "memory.relevanceMs", () => microCall({
         model,
         system: CHECK_MEMORIES_FOR_RELEVANT_INFORMATION,
         messages: [
@@ -8041,7 +8246,7 @@ async function runMemoryPhase({
             "The chunk_ids (UUIDs at the start of each bullet) of chunks containing information relevant to the user's question. Empty array if none are relevant."
           )
         })
-      });
+      }));
       const ids = new Set(output_relevant_memory?.relevantChunkIds ?? []);
       relevantMemoryChunks = ids.size === 0 ? [] : retrieved_memory.filter((c) => ids.has(c.chunk_id));
     } catch (e) {
@@ -8131,7 +8336,7 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
 
       Otherwise, return the original user question, relevant keywords and important keyword.
       `;
-      const [overrideResult, fileResult, queryResult] = await Promise.all([
+      const [overrideResult, fileResult, queryResult] = mergedOutput ? mergedFollowups(mergedOutput, memoryConfig, hasAugmentationContent, question, importantKeyword) : await withTiming(timings, "memory.followupsMs", () => Promise.all([
         // Override check: strict gate to decide if memory should be authoritative
         memoryConfig.override ? microCall({
           model,
@@ -8212,7 +8417,7 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
             updatedImportantKeyword: importantKeyword
           }
         })
-      ]);
+      ]));
       const overrideIds = new Set(overrideResult.output?.authoritativeChunkIds ?? []);
       const authoritativeChunks = memoryChunksForAnswer.filter(
         (c) => c.chunk_id && overrideIds.has(c.chunk_id)
@@ -8291,6 +8496,7 @@ var init_memory = __esm({
     init_cjs_shims();
     import_zod14 = require("zod");
     init_micro_call();
+    init_timing();
     init_multi_query();
     init_prefilter();
     init_text_utils();
@@ -8412,7 +8618,7 @@ async function searchContexts(opts) {
     skipPrefilter
   } = opts;
   const chunkArrays = await Promise.all(
-    contextIds.map(async (ctxId) => {
+    contextIds.map((ctxId) => withTiming(opts.timings, `${opts.timingPrefix ?? "search"}.${ctxId}Ms`, async () => {
       try {
         const ctx = contextsById.get(ctxId);
         if (!ctx) return [];
@@ -8450,13 +8656,13 @@ async function searchContexts(opts) {
           if (multiQuery) {
             let hydePassage = null;
             if (hyde) {
-              hydePassage = await generateHydePassage({
+              hydePassage = await withTiming(opts.timings, `${opts.timingPrefix ?? "search"}.${ctxId}.hydeMs`, () => generateHydePassage({
                 originalQuestion: question,
                 relevantKeywords: keywords,
                 importantKeyword,
                 styleHint,
                 model
-              });
+              }));
             }
             const candidates = [
               question,
@@ -8471,14 +8677,14 @@ async function searchContexts(opts) {
         }
         if (kind === "conversations") {
           if (keywordPrefilter && pinnedItemIds.length === 0) {
-            const prefiltered = await fuzzyPrefilter({
+            const prefiltered = await withTiming(opts.timings, `${opts.timingPrefix ?? "search"}.${ctxId}.prefilterMs`, () => fuzzyPrefilter({
               cacheKey: `conversations:${ctxId}`,
               relevantKeywords: keywords,
               importantKeyword,
               context: ctx,
               fields: ["name", "id", "external_id", "description"],
               normalize: (i) => [i.name, i.description].filter(Boolean).join(": ")
-            });
+            }));
             pinnedItemIds = prefiltered.map((r) => r.id);
           }
           const keywordQuery = keywords.length ? keywords.join(" ") + " " + importantKeyword : question;
@@ -8493,7 +8699,7 @@ async function searchContexts(opts) {
         console.warn(`[EXULU pipeline] searchContexts failed for context "${ctxId}":`, err);
         return [];
       }
-    })
+    }))
   );
   return { chunks: chunkArrays.flat() };
 }
@@ -8506,6 +8712,7 @@ var init_search = __esm({
     init_multi_query();
     init_hyde();
     init_prefilter();
+    init_timing();
     init_text_utils();
     tagContext = (chunks, ctx) => chunks.map((c) => ({ ...c, context: { id: ctx.id, name: ctx.name ?? ctx.id } }));
   }
@@ -8659,6 +8866,33 @@ var init_rerank = __esm({
   }
 });
 
+// ee/agentic-retrieval/pipeline/pin-rerun.ts
+function needsPinRerun(originalQuestion, updatedQuestion) {
+  if (originalQuestion === updatedQuestion) return false;
+  const before = designations(originalQuestion);
+  for (const token of designations(updatedQuestion)) {
+    if (!before.has(token)) return true;
+  }
+  return false;
+}
+function designations(text) {
+  const out = /* @__PURE__ */ new Set();
+  for (const raw of text.split(/[\s,;:()\[\]"'?!]+/)) {
+    const token = raw.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+    if (!token) continue;
+    const hasDigit = /\p{N}/u.test(token);
+    const capitals = (token.match(/\p{Lu}/gu) ?? []).length;
+    if (hasDigit || capitals >= 2) out.add(token.toLowerCase());
+  }
+  return out;
+}
+var init_pin_rerun = __esm({
+  "ee/agentic-retrieval/pipeline/pin-rerun.ts"() {
+    "use strict";
+    init_cjs_shims();
+  }
+});
+
 // ee/agentic-retrieval/pipeline/index.ts
 function addChunks(result, chunks) {
   const seen = new Set(result.chunks.map((c) => c.chunk_id));
@@ -8824,6 +9058,14 @@ function createAgenticRetrievalTool(opts) {
         usage: [],
         totalTokens: 0
       };
+      const t0 = Date.now();
+      const timings = {};
+      let tPhase = t0;
+      const lap = (name) => {
+        const now = Date.now();
+        timings[name] = (timings[name] ?? 0) + (now - tPhase);
+        tPhase = now;
+      };
       try {
         let enabledContexts = contexts.filter(
           (ctx) => cfg.knowledgeBases[ctx.id]?.enabled !== false
@@ -8896,8 +9138,19 @@ function createAgenticRetrievalTool(opts) {
           resolvedProject && projectScope?.customInstructions ? `Instructions for the attached project "${projectScope.name}":
 ${projectScope.customInstructions}` : ""
         ].filter(Boolean).join("\n");
-        const [memResult, routResult] = await Promise.all([
-          runMemoryPhase({
+        const engineV2 = cfg.tuning.engine === "v2";
+        const v2 = cfg.tuning.v2;
+        const pinsFor = (question) => resolveIdentifierPins({
+          question,
+          identifierSets: cfg.vocabulary.identifiers,
+          contextsById,
+          kbKindById,
+          model: utilityModel
+        });
+        const [memResult, routResult, parallelPins] = await Promise.all([
+          withTiming(timings, "memoryMs", () => runMemoryPhase({
+            timings,
+            mergedCall: engineV2 && v2.mergedMemoryCall,
             memoryChunks: memoryItems ?? [],
             memoryContext,
             question: userQuery,
@@ -8909,8 +9162,10 @@ ${projectScope.customInstructions}` : ""
             memoryConfig: cfg.memory,
             glossary: cfg.vocabulary.glossary,
             documentContexts
-          }),
-          runRoutingPhase({
+          })),
+          withTiming(timings, "routingMs", () => runRoutingPhase({
+            timings,
+            mergedCall: engineV2 && v2.mergedRoutingCall,
             question: userQuery,
             enabledContexts,
             documentContexts,
@@ -8921,8 +9176,12 @@ ${projectScope.customInstructions}` : ""
             // detector must never treat these as filename hints.
             knownIdentifiers: cfg.vocabulary.identifiers.flatMap((i) => i.examples),
             model: utilityModel
-          })
+          })),
+          // engine v2: identifier pins depend only on the question, so they run alongside
+          // memory and routing instead of after them (re-run below if memory rewrote the question).
+          engineV2 && v2.parallelPins ? withTiming(timings, "pinsParallelMs", () => pinsFor(userQuery)) : Promise.resolve(null)
         ]);
+        lap("memoryRoutingMs");
         for (const step of [...memResult.steps, ...routResult.steps]) {
           result.steps.push({
             stepNumber: 1,
@@ -8972,13 +9231,8 @@ ${projectScope.customInstructions}` : ""
           memoryPinnedItemIdsByContext,
           memoryOverride
         } = memResult;
-        const { pinsByContext: identifierPinsByContext, exactPinsByContext, steps: pinSteps } = await resolveIdentifierPins({
-          question: updatedQuestion,
-          identifierSets: cfg.vocabulary.identifiers,
-          contextsById,
-          kbKindById,
-          model: utilityModel
-        });
+        const { pinsByContext: identifierPinsByContext, exactPinsByContext, steps: pinSteps } = parallelPins && !needsPinRerun(userQuery, updatedQuestion) ? parallelPins : await pinsFor(updatedQuestion);
+        lap("pinsMs");
         for (const step of pinSteps) {
           result.steps.push({
             stepNumber: 1,
@@ -9008,6 +9262,8 @@ ${projectScope.customInstructions}` : ""
             rewrites: cfg.vocabulary.rewrites,
             styleHint: cfg.vocabulary.styleHint,
             maxQueries: cfg.tuning.maxQueriesPerContext,
+            timings,
+            timingPrefix: "search.main",
             skipPrefilter: false
           }),
           fallbackContextsToSearch.length > 0 && !hasExplicitDocAndPage ? searchContexts({
@@ -9028,9 +9284,12 @@ ${projectScope.customInstructions}` : ""
             rewrites: cfg.vocabulary.rewrites,
             styleHint: cfg.vocabulary.styleHint,
             maxQueries: cfg.tuning.maxQueriesPerContext,
+            timings,
+            timingPrefix: "search.fallback",
             skipPrefilter: true
           }) : Promise.resolve({ chunks: [] })
         ]);
+        lap("searchMs");
         const pinnedItemIds = /* @__PURE__ */ new Set([
           ...(function* () {
             for (const s of memoryPinnedItemIdsByContext.values()) yield* s;
@@ -9093,6 +9352,7 @@ ${projectScope.customInstructions}` : ""
           chunks: [],
           tokens: 0
         });
+        lap("rerankMs");
         addChunks(result, mainRerank.limited_results);
         yield { result: serializeOutput(result) };
         const literalLookupSatisfied = hasExplicitDocAndPage && mainRerank.limited_results.length > 0 && mainRerank.limited_results.some((r) => {
@@ -9141,6 +9401,7 @@ ${projectScope.customInstructions}` : ""
           });
           result.reasoning.push({ text: "Fallback results reranked", tools: [] });
           addChunks(result, fallbackRerank.limited_results);
+          lap("fallbackRerankMs");
           yield { result: serializeOutput(result) };
         }
         if (memoryOverride.active) {
@@ -9163,9 +9424,19 @@ Verified answer:
           addChunks(result, memoryOverride.chunks);
           yield { result: serializeOutput(result) };
         }
+        timings.totalMs = Date.now() - t0;
+        result.timings = timings;
+        result.steps.push({
+          stepNumber: 1,
+          text: `Timing: memory+routing ${timings.memoryRoutingMs ?? 0}ms (memory ${timings.memoryMs ?? 0}ms, routing ${timings.routingMs ?? 0}ms), pins ${timings.pinsMs ?? 0}ms, search ${timings.searchMs ?? 0}ms, rerank ${timings.rerankMs ?? 0}ms` + (timings.fallbackRerankMs !== void 0 ? `, fallback rerank ${timings.fallbackRerankMs}ms` : "") + `, total ${timings.totalMs}ms`,
+          toolCalls: [],
+          chunks: [],
+          tokens: 0
+        });
         if (cfg.logging) {
-          console.log("[EXULU pipeline] final result:", JSON.stringify({ steps: result.steps.length, chunks: result.chunks.length }));
+          console.log("[EXULU pipeline] final result:", JSON.stringify({ steps: result.steps.length, chunks: result.chunks.length, timings }));
         }
+        yield { result: serializeOutput(result) };
         return { result: serializeOutput(result) };
       } catch (err) {
         console.warn("[EXULU pipeline] retrieval pipeline failed:", err);
@@ -9200,6 +9471,8 @@ var init_pipeline = __esm({
     init_prefilter();
     init_search();
     init_rerank();
+    init_timing();
+    init_pin_rerun();
     init_global_ids();
     init_global_ids();
   }
@@ -10608,6 +10881,68 @@ function resolveSearchQueryTexts(query) {
     hybridOrQuery: buildFullTextOrQuery(query, ftsText)
   };
 }
+function chooseFullTextQuery(opts) {
+  if (opts.strictMatches || countOrTerms(opts.orText) > MAX_OR_TERMS) {
+    return { fn: "plainto_tsquery", text: opts.strictText };
+  }
+  return { fn: "websearch_to_tsquery", text: opts.orText };
+}
+var MAX_OR_TERMS = 12;
+function countOrTerms(orText) {
+  return orText.trim() ? orText.split(/\s+or\s+/i).length : 0;
+}
+
+// src/graphql/resolvers/expand-neighbours.ts
+init_cjs_shims();
+function planNeighbourFetch(results, expand) {
+  const before = Math.max(0, expand.before ?? 0);
+  const after = Math.max(0, expand.after ?? 0);
+  const plan = /* @__PURE__ */ new Map();
+  if (before === 0 && after === 0) return plan;
+  const present = new Set(results.map((r) => `${r.item_id}-${r.chunk_index}`));
+  for (const r of results) {
+    for (let i = r.chunk_index - before; i <= r.chunk_index + after; i++) {
+      if (i < 0 || i === r.chunk_index || present.has(`${r.item_id}-${i}`)) continue;
+      if (!plan.has(r.item_id)) plan.set(r.item_id, /* @__PURE__ */ new Set());
+      plan.get(r.item_id).add(i);
+    }
+  }
+  return plan;
+}
+function mergeNeighbours(results, rows, plan, context) {
+  const byItem = /* @__PURE__ */ new Map();
+  for (const r of results) if (!byItem.has(r.item_id)) byItem.set(r.item_id, r);
+  const merged = /* @__PURE__ */ new Map();
+  for (const r of results) merged.set(`${r.item_id}-${r.chunk_index}`, r);
+  for (const row of rows) {
+    if (!plan.get(row.source)?.has(row.chunk_index)) continue;
+    const key = `${row.source}-${row.chunk_index}`;
+    if (merged.has(key)) continue;
+    const parent = byItem.get(row.source);
+    if (!parent) continue;
+    merged.set(key, {
+      chunk_content: row.content,
+      chunk_index: row.chunk_index,
+      chunk_id: row.id,
+      chunk_source: row.source,
+      chunk_metadata: row.metadata,
+      chunk_created_at: row.createdAt,
+      chunk_updated_at: row.updatedAt,
+      item_updated_at: parent.item_updated_at,
+      item_created_at: parent.item_created_at,
+      item_id: parent.item_id,
+      item_external_id: parent.item_external_id,
+      item_name: parent.item_name,
+      chunk_cosine_distance: 0,
+      chunk_fts_rank: 0,
+      chunk_hybrid_score: 0,
+      context
+    });
+  }
+  return Array.from(merged.values()).sort(
+    (a, b) => a.item_id === b.item_id ? a.chunk_index - b.chunk_index : 0
+  );
+}
 
 // src/graphql/resolvers/apply-sorting.ts
 init_cjs_shims();
@@ -11598,6 +11933,14 @@ var agentsSchema = {
       name: "max_tool_steps",
       type: "number"
     },
+    {
+      // Thinking budget of the answer model, forwarded as LiteLLM's
+      // reasoning_effort ("none" | "disable" | "minimal" | "low" | "medium" |
+      // "high"). null = provider default. See resolve-reasoning-effort.ts.
+      // Auto-ALTERed on boot.
+      name: "reasoning_effort",
+      type: "text"
+    },
     {
       name: "guest_access",
       type: "boolean",
@@ -12315,6 +12658,15 @@ var convertContextToTableDefinition = (context) => {
 // src/graphql/resolvers/vector-search.ts
 init_statistics2();
 
+// src/graphql/resolvers/query-embedding-policy.ts
+init_cjs_shims();
+function needsQueryEmbedding(method) {
+  return method !== "tsvector";
+}
+function boostsWithQueryEntities(method) {
+  return method !== "tsvector";
+}
+
 // src/exulu/entities/index.ts
 init_cjs_shims();
 init_client();
@@ -13095,7 +13447,9 @@ var vectorSearch = async ({
     const embedText = texts.embedText;
     hybridOrQuery = texts.hybridOrQuery;
     query = texts.ftsText;
-    if (queryEmbedding && queryEmbedding.length) {
+    if (!needsQueryEmbedding(method)) {
+      _embedSource = "none";
+    } else if (queryEmbedding && queryEmbedding.length) {
       vector = queryEmbedding;
       _embedSource = "reused";
     } else {
@@ -13128,8 +13482,10 @@ var vectorSearch = async ({
       vector = queryVector;
       _embedSource = "computed";
     }
-    vectorStr = `ARRAY[${vector.join(",")}]`;
-    vectorExpr = `${vectorStr}::vector`;
+    if (vector.length) {
+      vectorStr = `ARRAY[${vector.join(",")}]`;
+      vectorExpr = `${vectorStr}::vector`;
+    }
   }
   let keywordsQuery = [];
   if (keywords) {
@@ -13187,14 +13543,22 @@ var vectorSearch = async ({
       ]);
       resultChunks = await chunksQuery;
       break;
-    case "hybridSearch":
+    case "hybridSearch": {
+      let strictMatches = false;
+      if (query && hybridOrQuery) {
+        const probe = await db2(chunksTable + " as chunks").select(db2.raw("1")).whereRaw(`(${languages.map((lang) => `chunks.fts @@ plainto_tsquery('${lang}', ?)`).join(" OR ")})`, languages.map(() => query)).first();
+        strictMatches = Boolean(probe);
+      }
+      const fullText = chooseFullTextQuery({ strictMatches, strictText: query ?? "", orText: hybridOrQuery });
+      const ftsFn = fullText.fn;
+      hybridOrQuery = fullText.text;
       const matchCount = Math.min(limit * 2);
       const fullTextWeight = 2;
       const semanticWeight = 1;
       const rrfK = 50;
-      const ftRankExpression = languages.map((lang) => `ts_rank(chunks.fts, websearch_to_tsquery('${lang}', ?))`).join(", ");
+      const ftRankExpression = languages.map((lang) => `ts_rank(chunks.fts, ${ftsFn}('${lang}', ?))`).join(", ");
       const ftRankParams = languages.map(() => hybridOrQuery);
-      const ftMatchExpression = languages.map((lang) => `chunks.fts @@ websearch_to_tsquery('${lang}', ?)`).join(" OR ");
+      const ftMatchExpression = languages.map((lang) => `chunks.fts @@ ${ftsFn}('${lang}', ?)`).join(" OR ");
       const ftMatchParams = languages.map(() => hybridOrQuery);
       let fullTextQuery = db2(chunksTable + " as chunks").select([
         "chunks.id",
@@ -13235,7 +13599,7 @@ var vectorSearch = async ({
         db2.raw('items."updatedAt" as item_updated_at'),
         db2.raw('items."createdAt" as item_created_at'),
         db2.raw(
-          `GREATEST(${languages.map((lang) => `ts_rank(chunks.fts, websearch_to_tsquery('${lang}', ?))`).join(", ")}) AS fts_rank`,
+          `GREATEST(${languages.map((lang) => `ts_rank(chunks.fts, ${ftsFn}('${lang}', ?))`).join(", ")}) AS fts_rank`,
           languages.map(() => hybridOrQuery)
         ),
         db2.raw(`(1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance`),
@@ -13261,12 +13625,14 @@ var vectorSearch = async ({
                   `,
         [rrfK, fullTextWeight, rrfK, semanticWeight, cutoffs?.hybrid || 0]
       ).whereRaw(
-        `(chunks.fts IS NULL OR GREATEST(${languages.map((lang) => `ts_rank(chunks.fts, websearch_to_tsquery('${lang}', ?))`).join(", ")}) > ?)`,
+        `(chunks.fts IS NULL OR GREATEST(${languages.map((lang) => `ts_rank(chunks.fts, ${ftsFn}('${lang}', ?))`).join(", ")}) > ?)`,
         [...languages.map(() => hybridOrQuery), cutoffs?.tsvector || 0]
       ).whereRaw(`(chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)`, [
         cutoffs?.cosineDistance || 0
       ]).orderByRaw("hybrid_score DESC").limit(Math.min(matchCount, 250));
       resultChunks = await hybridQuery;
+      break;
+    }
   }
   if (process.env.EXULU_VS_TIMING) {
     console.log(
@@ -13318,7 +13684,7 @@ var vectorSearch = async ({
   }
   let queryEntities = [];
   let entityInsights;
-  if (entitiesOn && rawQuery) {
+  if (entitiesOn && rawQuery && boostsWithQueryEntities(method)) {
     try {
       const types = await hydrateEntityTypes(context);
       const { mentions: queryMentions } = await extractEntitiesForItem({
@@ -13350,104 +13716,18 @@ var vectorSearch = async ({
   }
   results = results.slice(0, limit);
   if (expand?.before || expand?.after) {
-    const expandedMap = /* @__PURE__ */ new Map();
-    for (const chunk of results) {
-      expandedMap.set(`${chunk.item_id}-${chunk.chunk_index}`, chunk);
-    }
-    if (expand?.before) {
-      for (const chunk of results) {
-        const indicesToFetch = Array.from(
-          { length: expand.before },
-          (_, i) => chunk.chunk_index - expand.before + i
-        ).filter((index) => index >= 0);
-        await Promise.all(
-          indicesToFetch.map(async (index) => {
-            if (expandedMap.has(`${chunk.item_id}-${index}`)) {
-              return;
-            }
-            const expandedChunk = await db2(chunksTable).where({
-              source: chunk.item_id,
-              chunk_index: index
-            }).first();
-            if (expandedChunk) {
-              if (expandedChunk) {
-                expandedMap.set(`${chunk.item_id}-${index}`, {
-                  chunk_content: expandedChunk.content,
-                  chunk_index: expandedChunk.chunk_index,
-                  chunk_id: expandedChunk.id,
-                  chunk_source: expandedChunk.source,
-                  chunk_metadata: expandedChunk.metadata,
-                  chunk_created_at: expandedChunk.createdAt,
-                  chunk_updated_at: expandedChunk.updatedAt,
-                  item_updated_at: chunk.item_updated_at,
-                  item_created_at: chunk.item_created_at,
-                  item_id: chunk.item_id,
-                  item_external_id: chunk.item_external_id,
-                  item_name: chunk.item_name,
-                  chunk_cosine_distance: 0,
-                  chunk_fts_rank: 0,
-                  chunk_hybrid_score: 0,
-                  context: {
-                    name: table.name.singular,
-                    id: table.id || ""
-                  }
-                });
-              }
-            }
-          })
-        );
-      }
-    }
-    if (expand?.after) {
-      for (const chunk of results) {
-        const indicesToFetch = Array.from(
-          { length: expand.after },
-          (_, i) => chunk.chunk_index + i + 1
-        );
-        await Promise.all(
-          indicesToFetch.map(async (index) => {
-            if (expandedMap.has(`${chunk.item_id}-${index}`)) {
-              return;
-            }
-            const expandedChunk = await db2(chunksTable).where({
-              source: chunk.item_id,
-              chunk_index: index
-            }).first();
-            if (expandedChunk) {
-              expandedMap.set(`${chunk.item_id}-${index}`, {
-                chunk_content: expandedChunk.content,
-                chunk_index: expandedChunk.chunk_index,
-                chunk_id: expandedChunk.id,
-                chunk_source: expandedChunk.source,
-                chunk_metadata: expandedChunk.metadata,
-                chunk_created_at: expandedChunk.createdAt,
-                chunk_updated_at: expandedChunk.updatedAt,
-                item_updated_at: chunk.item_updated_at,
-                item_created_at: chunk.item_created_at,
-                item_id: chunk.item_id,
-                item_external_id: chunk.item_external_id,
-                item_name: chunk.item_name,
-                chunk_cosine_distance: 0,
-                chunk_fts_rank: 0,
-                chunk_hybrid_score: 0,
-                context: {
-                  name: table.name.singular,
-                  id: table.id || ""
-                }
-              });
-            }
-          })
-        );
-      }
+    const plan = planNeighbourFetch(results, expand);
+    if (plan.size > 0) {
+      const itemIds = Array.from(plan.keys());
+      const indices = Array.from(new Set(Array.from(plan.values()).flatMap((s) => Array.from(s))));
+      const rows = await db2(chunksTable).select(["id", "source", "chunk_index", "content", "metadata", "createdAt", "updatedAt"]).whereIn("source", itemIds).whereIn("chunk_index", indices);
+      results = mergeNeighbours(results, rows, plan, { name: table.name.singular, id: table.id || "" });
     }
-    results = Array.from(expandedMap.values());
     results = results.sort((a, b) => {
       if (a.item_id !== b.item_id) {
         return a.item_id.localeCompare(b.item_id);
       }
-      const aIndex = Number(a.chunk_index);
-      const bIndex = Number(b.chunk_index);
-      return aIndex - bIndex;
+      return Number(a.chunk_index) - Number(b.chunk_index);
     });
   }
   if (entitiesOn) {
@@ -17038,6 +17318,20 @@ function serializeError(err, depth = 0) {
   return { message: String(err) };
 }
 
+// src/exulu/turn-metadata.ts
+init_cjs_shims();
+function finishTurnMetadata(opts) {
+  const now = opts.now ?? Date.now();
+  return {
+    totalTokens: opts.totalUsage.totalTokens,
+    reasoningTokens: opts.totalUsage.reasoningTokens,
+    inputTokens: opts.totalUsage.inputTokens,
+    outputTokens: opts.totalUsage.outputTokens,
+    cachedInputTokens: opts.totalUsage.cachedInputTokens,
+    durationMs: Math.max(0, now - opts.startedAt)
+  };
+}
+
 // src/utils/enabled-tools.ts
 init_cjs_shims();
 init_pipeline();
@@ -17162,6 +17456,58 @@ async function resolveFreshFileUrl(url, opts) {
   }
 }
 
+// src/exulu/session-file-listing.ts
+init_cjs_shims();
+init_uppy();
+init_artifact_filter();
+init_session_files();
+var DEFAULT_MAX = 25;
+function formatSize(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+function formatAge(then, now) {
+  const minutes = Math.max(0, Math.round((now.getTime() - then.getTime()) / 6e4));
+  if (minutes < 1) return "just now";
+  if (minutes < 60) return `${minutes} minute${minutes === 1 ? "" : "s"} ago`;
+  const hours = Math.round(minutes / 60);
+  if (hours < 48) return `${hours} hour${hours === 1 ? "" : "s"} ago`;
+  const days = Math.round(hours / 24);
+  return `${days} day${days === 1 ? "" : "s"} ago`;
+}
+function describeSessionFiles(files, opts = {}) {
+  const now = opts.now ?? /* @__PURE__ */ new Date();
+  const max = opts.max ?? DEFAULT_MAX;
+  const usable = files.filter((f) => f.name && !f.name.endsWith("/") && !isIgnoredArtifactPath(f.name)).sort((a, b) => b.lastModified.getTime() - a.lastModified.getTime());
+  if (usable.length === 0) return "";
+  const shown = usable.slice(0, max);
+  const lines = shown.map((f) => {
+    const isNew = opts.lastTurnAt ? f.lastModified.getTime() > opts.lastTurnAt.getTime() : false;
+    return `- ${f.name} (${formatSize(f.size)}, ${formatAge(f.lastModified, now)})${isNew ? " [NEW since your last answer]" : ""}`;
+  });
+  const omitted = usable.length - shown.length;
+  if (omitted > 0) lines.push(`\u2026 and ${omitted} more file${omitted === 1 ? "" : "s"} (list them with \`ls\`).`);
+  return "Files currently in this session (newest first):\n" + lines.join("\n") + '\nThese files are available to you. When the user refers to one of these files, to "the document" or "the attachment", or asks something only such a file can answer, read it with parse_document, view_document_page or read_session_file. Otherwise proceed as usual, e.g. with the knowledge bases.';
+}
+async function loadSessionFileListing(opts) {
+  const uploads = opts.exuluConfig?.fileUploads;
+  if (!uploads?.s3Bucket) return "";
+  const prefix = sessionFilePrefix(opts.ownerId, opts.sessionID, uploads.s3prefix);
+  try {
+    const objects = await listS3ObjectsByPrefix(prefix, opts.exuluConfig);
+    const files = objects.map((o) => ({
+      name: o.key.slice(o.key.indexOf(prefix) + prefix.length),
+      size: o.size,
+      lastModified: new Date(o.lastModified)
+    }));
+    return describeSessionFiles(files, { lastTurnAt: opts.lastTurnAt });
+  } catch (err) {
+    console.warn(`[EXULU] could not list session files for prompt (session ${opts.sessionID}):`, err);
+    return "";
+  }
+}
+
 // src/exulu/generate-stream.ts
 init_uppy();
 var import_ai6 = require("ai");
@@ -17413,11 +17759,12 @@ function resolveTurnStepBudget(maxStepCount, agent) {
   }
   return DEFAULT_MAX_STEPS;
 }
+var TOOL_INPUT_FLATTEN_CHARS = 6e3;
 function flattenPart(part) {
   const p = part;
   if (p?.type === "text") return p.text ?? "";
   if (p?.type === "tool-call") {
-    return `Earlier, the assistant ran the "${p.toolName}" tool with input: ${JSON.stringify(p.input ?? {}).slice(0, 300)}`;
+    return `Earlier, the assistant ran the "${p.toolName}" tool with input: ${JSON.stringify(p.input ?? {}).slice(0, TOOL_INPUT_FLATTEN_CHARS)}`;
   }
   if (p?.type === "tool-result") {
     const out = p.output?.value ?? p.output;
@@ -17439,7 +17786,7 @@ function flattenToolHistory(messages) {
     return m;
   });
 }
-var FINAL_ANSWER_INSTRUCTION = `This is your last step for this turn. Answer the user's original question now, in plain text, using only the information gathered above. If you could not finish the task, tell the user you reached the maximum number of tool steps, summarize what you found and did so far, and say what remains \u2014 they can ask you to continue. Do not attempt any further tool calls. Write your answer as normal prose for the user: do not output tool-call syntax, JSON commands, or bracketed lines such as "[called tool ...]" \u2014 describe anything you did or still plan to do in plain language.`;
+var FINAL_ANSWER_INSTRUCTION = `This is your last step for this turn. Answer the user's original question now, in plain text, using only the information gathered above. Do not invent, estimate or "fill in" values that were not actually gathered: report only what the tools returned or what you wrote down, and name explicitly what is missing. If you could not finish the task, tell the user you reached the maximum number of tool steps, summarize what you found and did so far, and say what remains \u2014 they can ask you to continue. Do not attempt any further tool calls. Write your answer as normal prose for the user: do not output tool-call syntax, JSON commands, or bracketed lines such as "[called tool ...]" \u2014 describe anything you did or still plan to do in plain language.`;
 function finalAnswerGuard(maxSteps) {
   return ({ stepNumber, messages }) => stepNumber >= maxSteps - 1 ? {
     toolChoice: "none",
@@ -17464,6 +17811,37 @@ function retrievalBudgetGuard(limit, agenticToolKey, allToolKeys) {
   };
 }
 
+// src/exulu/resolve-reasoning-effort.ts
+init_cjs_shims();
+var REASONING_EFFORTS = ["none", "disable", "minimal", "low", "medium", "high"];
+function resolveReasoningEffort(agent) {
+  const raw = agent?.reasoning_effort;
+  if (typeof raw !== "string") return void 0;
+  const normalized = raw.trim().toLowerCase();
+  return REASONING_EFFORTS.includes(normalized) ? normalized : void 0;
+}
+function resolveProviderOptions(agent) {
+  const effort = resolveReasoningEffort(agent);
+  return {
+    openai: { reasoningSummary: "auto" },
+    ...effort ? { litellm: { reasoningEffort: effort } } : {}
+  };
+}
+
+// src/exulu/stream-error.ts
+init_cjs_shims();
+function onChatStreamError({ error }) {
+  const detail = error instanceof Error ? error.message : error === void 0 ? "unknown error" : safeStringify(error);
+  console.error("[EXULU] chat stream error.", detail);
+}
+function safeStringify(value) {
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}
+
 // src/exulu/generate-stream.ts
 init_sanitize_tool_name();
 init_tool_image_attachments();
@@ -17561,6 +17939,15 @@ var saveChat = async ({
     await mutation;
   }
 };
+var lastMessageTime = (rows) => {
+  let latest;
+  for (const row of rows) {
+    if (!row.createdAt) continue;
+    const d = new Date(row.createdAt);
+    if (!latest || d > latest) latest = d;
+  }
+  return latest;
+};
 var getAgentMessages = async ({
   session,
   user,
@@ -17618,6 +18005,7 @@ var generateSync = async ({
   let project;
   let sessionItems;
   let sessionOwnerId;
+  let lastTurnAt;
   if (session) {
     const sessionData = await getSession({ sessionID: session });
     sessionItems = sessionData.session_items;
@@ -17632,6 +18020,7 @@ var generateSync = async ({
       session,
       user: user.id
     });
+    lastTurnAt = lastMessageTime(previousMessages);
     const previousMessagesContent = previousMessages.map(
       (message) => JSON.parse(message.content)
     );
@@ -17794,6 +18183,15 @@ var generateSync = async ({
         commands like \`node create_doc.js\`) live in the same place. These files are scoped to
         this single session; they are NOT visible in other sessions, projects, or knowledge bases.
       `;
+  if (session) {
+    const listing = await loadSessionFileListing({
+      sessionID: session,
+      ownerId: sessionOwnerId ?? user?.id ?? "api",
+      exuluConfig,
+      lastTurnAt
+    });
+    if (listing) system += "\n\n" + listing;
+  }
   system += `
 
 When a tool execution is not approved by the user, do not retry it unless explicitly asked by the user. ' +
@@ -17963,6 +18361,7 @@ var generateStream = async ({
   let project;
   let sessionItems;
   let sessionOwnerId;
+  let lastTurnAt;
   if (session) {
     const sessionData = await getSession({ sessionID: session });
     project = sessionData.project;
@@ -17978,6 +18377,7 @@ var generateStream = async ({
       includeAllUsers: isRunSessionMetadata(sessionData.metadata)
     });
     previousMessagesContent = previousMessages2.map((message2) => JSON.parse(message2.content));
+    lastTurnAt = lastMessageTime(previousMessages2);
   }
   const model = languageModel;
   messages = await (0, import_ai6.validateUIMessages)({
@@ -18120,6 +18520,15 @@ ${skillsList}
         truncation notice, e.g. tool-output-*.txt). Use the read_session_file tool with offset/limit
         to page through it \u2014 do not ask the user to re-upload.
       `;
+  if (session) {
+    const listing = await loadSessionFileListing({
+      sessionID: session,
+      ownerId: sessionOwnerId ?? user?.id ?? "api",
+      exuluConfig,
+      lastTurnAt
+    });
+    if (listing) system += "\n\n" + listing;
+  }
   system += `
 
 When a tool execution is not approved by the user, do not retry it unless explicitly asked by the user. ' +
@@ -18222,18 +18631,12 @@ When a tool execution is not approved by the user, do not retry it unless explic
     // for the first step or change other parameters.
     system,
     maxRetries: 2,
-    providerOptions: {
-      openai: {
-        reasoningSummary: "auto"
-      }
-    },
+    // OpenAI reasoning summaries + the agent's optional thinking budget
+    // (agents.reasoning_effort → LiteLLM reasoning_effort).
+    providerOptions: resolveProviderOptions(agent),
     tools,
-    onError: (error) => {
-      console.error("[EXULU] chat stream error.", error);
-      throw new Error(
-        `Chat stream error: ${error instanceof Error ? error.message : JSON.stringify(error)}`
-      );
-    },
+    // Log only — throwing here crashed the process (see stream-error.ts).
+    onError: onChatStreamError,
     // todo allow configuring the step budget per skill
     prepareStep: composePrepareSteps(contextGuard(contextWindow), retrievalGuard, finalAnswerGuard(turnBudget), imageAttachmentGuard()),
     stopWhen: [(0, import_ai6.stepCountIs)(turnBudget), (0, import_ai6.hasToolCall)("image_generation")]
@@ -18245,6 +18648,45 @@ When a tool execution is not approved by the user, do not retry it unless explic
   };
 };
 
+// src/utils/agent-glossary.ts
+init_cjs_shims();
+function extractGlossaryFromAgentTools(tools) {
+  if (!Array.isArray(tools)) return [];
+  const searchTool = tools.find((t) => t?.id === "agentic_context_search");
+  const entry = searchTool?.config?.find((c) => c?.name === "vocabulary");
+  if (entry === void 0 || entry.variable === void 0 || entry.variable === null || entry.variable === "") {
+    return [];
+  }
+  try {
+    const parsed = typeof entry.variable === "string" ? JSON.parse(entry.variable) : entry.variable;
+    const glossary = parsed?.glossary;
+    if (!Array.isArray(glossary)) return [];
+    return glossary.filter(
+      (g) => !!g && typeof g.term === "string" && typeof g.meaning === "string" && g.term.trim().length > 0
+    );
+  } catch (err) {
+    console.warn(
+      "[EXULU] Failed to parse the agentic_context_search vocabulary config while building agent instructions.",
+      err
+    );
+    return [];
+  }
+}
+function formatGlossaryBlock(glossary) {
+  if (!glossary.length) return "";
+  const lines = glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n");
+  return `The organization's documents and internal terminology use the following abbreviations/terms:
+${lines}`;
+}
+function withGlossary(baseInstructions, tools) {
+  const instructions = baseInstructions ?? "";
+  const block = formatGlossaryBlock(extractGlossaryFromAgentTools(tools));
+  if (!block) return instructions;
+  return instructions ? `${block}
+
+${instructions}` : block;
+}
+
 // src/exulu/agent-as-tool.ts
 var createAgentTool = async (instance2, contexts) => {
   const agent = await exuluApp.get().agent(instance2);
@@ -18303,7 +18745,7 @@ var createAgentTool = async (instance2, contexts) => {
       const response = await generateSync({
         agent,
         contexts,
-        instructions: agent.instructions,
+        instructions: withGlossary(agent.instructions, agent.tools),
         prompt: "The user has asked the following question: " + prompt + " and the following information is available: " + information,
         languageModel: resolved.languageModel,
         user,
@@ -20097,13 +20539,7 @@ var processUiMessagesFlow = async ({
               messageMetadata: ({ part }) => {
                 console.log("[EXULU] part", part.type);
                 if (part.type === "finish") {
-                  return {
-                    totalTokens: part.totalUsage.totalTokens,
-                    reasoningTokens: part.totalUsage.reasoningTokens,
-                    inputTokens: part.totalUsage.inputTokens,
-                    outputTokens: part.totalUsage.outputTokens,
-                    cachedInputTokens: part.totalUsage.cachedInputTokens
-                  };
+                  return finishTurnMetadata({ totalUsage: part.totalUsage, startedAt: startTime });
                 }
                 return void 0;
               },
@@ -21337,7 +21773,7 @@ var recallService = {
       );
       const { text } = await (0, import_ai8.generateText)({
         model: resolved.languageModel,
-        system: agent.instructions || void 0,
+        system: withGlossary(agent.instructions, agent.tools) || void 0,
         prompt: `${prompt.content}
 
 ---
@@ -26130,10 +26566,14 @@ var createExpressRoutes = async (app, tools, contexts, config, evals, tracer) =>
         }
         const approvedTools = req.body.approvedTools ? typeof req.body.approvedTools === "string" ? JSON.parse(req.body.approvedTools) : req.body.approvedTools : [];
         const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
-        const instructions = customInstructions ? `${agent.instructions}
+        const instructions = withGlossary(
+          customInstructions ? `${agent.instructions}
 
-${customInstructions}` : agent.instructions;
+${customInstructions}` : agent.instructions,
+          agent.tools
+        );
         if (headers.session) markStreamActive(headers.session);
+        const turnStartedAt = Date.now();
         let result;
         try {
           result = await generateStream({
@@ -26175,13 +26615,7 @@ ${customInstructions}` : agent.instructions;
               };
             }
             if (part.type === "finish") {
-              return {
-                totalTokens: part.totalUsage.totalTokens,
-                reasoningTokens: part.totalUsage.reasoningTokens,
-                inputTokens: part.totalUsage.inputTokens,
-                outputTokens: part.totalUsage.outputTokens,
-                cachedInputTokens: part.totalUsage.cachedInputTokens
-              };
+              return finishTurnMetadata({ totalUsage: part.totalUsage, startedAt: turnStartedAt });
             }
             return void 0;
           },
@@ -26273,9 +26707,12 @@ ${customInstructions}` : agent.instructions;
         return;
       } else {
         const customInstructions = req.body.customInstructions ? typeof req.body.customInstructions === "string" ? req.body.customInstructions : JSON.stringify(req.body.customInstructions) : "";
-        const instructions = customInstructions ? `${agent.instructions}
+        const instructions = withGlossary(
+          customInstructions ? `${agent.instructions}
 
-${customInstructions}` : agent.instructions;
+${customInstructions}` : agent.instructions,
+          agent.tools
+        );
         let response;
         try {
           response = await generateSync({
@@ -33317,7 +33754,6 @@ ${command}`;
 }
 
 // ee/python/documents/processing/doc_processor.ts
-init_python_setup();
 var import_liteparse = require("@llamaindex/liteparse");
 
 // src/exulu/resolve-ocr.ts
@@ -33704,7 +34140,7 @@ function reconstructTableHeaders(document2, validationResults, verbose = false)
   }
 }
 async function validateWithVLM(document2, model, verbose = false, concurrency = 10) {
-  console.log(`[EXULU] Starting VLM validation for docling output, ${document2.length} pages...`);
+  console.log(`[EXULU] Starting VLM validation for processor output, ${document2.length} pages...`);
   console.log(`[EXULU] Concurrency limit: ${concurrency}`);
   const limit = (0, import_p_limit.default)(concurrency);
   const validationResults = /* @__PURE__ */ new Map();
@@ -33839,48 +34275,7 @@ async function processDocument(filePath, fileType, buffer, tempDir, config, verb
 async function processPdf(buffer, paths, config, verbose = false) {
   try {
     let json = [];
-    if (config?.processor.name === "docling") {
-      console.log(`[EXULU] Validating Python environment...`);
-      const validation = await validatePythonEnvironment(void 0, true);
-      if (!validation.valid) {
-        console.log(`[EXULU] Python environment not ready, setting up automatically...`);
-        console.log(`[EXULU] Reason: ${validation.message}`);
-        const setupResult = await setupPythonEnvironment({
-          verbose: true,
-          force: false
-          // Only setup if not already done
-        });
-        if (!setupResult.success) {
-          throw new Error(`Failed to setup Python environment: ${setupResult.message}
-
-${setupResult.output || ""}`);
-        }
-        console.log(`[EXULU] Python environment setup completed successfully`);
-      } else {
-        console.log(`[EXULU] Python environment is valid`);
-      }
-      console.log(`[EXULU] Processing document with document_to_markdown.py`);
-      const result = await executePythonScript({
-        scriptPath: "ee/python/documents/processing/document_to_markdown.py",
-        args: [
-          paths.source,
-          "-o",
-          paths.json,
-          "--images-dir",
-          paths.images
-        ],
-        timeout: 30 * 60 * 1e3
-        // 30 minutes for large documents
-      });
-      if (result.stderr) {
-        console.log("Processing info:", result.stderr.trim());
-      }
-      if (!result.success) {
-        throw new Error(`Document processing failed: ${result.stderr}`);
-      }
-      const jsonContent = await fs4.promises.readFile(paths.json, "utf-8");
-      json = JSON.parse(jsonContent);
-    } else if (config?.processor.name === "officeparser") {
+    if (config?.processor.name === "officeparser") {
       const text = await (0, import_officeparser3.parseOfficeAsync)(buffer, {
         outputErrorToConsole: false,
         newlineDelimiter: "\n"
@@ -33971,14 +34366,16 @@ stderr: ${splitResult.stderr.slice(-1e3)}`
         image: screenshots.find((s) => s.pageNum === page.pageNum)?.imagePath
       }));
       fs4.writeFileSync(paths.json, JSON.stringify(json, null, 2));
+    } else {
+      const configured = String(config?.processor?.name ?? "");
+      throw new Error(
+        configured === "" ? "[EXULU] No document processor configured. Set processor.name to one of: mistral, liteparse, officeparser." : `[EXULU] Unknown document processor "${configured}". Supported processors are: mistral, liteparse, officeparser.` + (configured === "docling" ? ' The "docling" processor was removed: it depended on PyMuPDF, which is AGPL-licensed. Use "mistral" for PDF OCR.' : "")
+      );
     }
     console.log(`[EXULU] 
 \u2713 Document processing completed successfully`);
     console.log(`[EXULU] Total pages: ${json.length}`);
     console.log(`[EXULU] Output file: ${paths.json}`);
-    if (config?.vlm?.model) {
-      console.error("[EXULU] VLM validation is only supported when docling is enabled, skipping validation.");
-    }
     const vlmModel = config?.vlm?.model ? await resolveVlmModel(config) : void 0;
     if (vlmModel && json.length > 0) {
       json = await validateWithVLM(
@@ -34096,9 +34493,6 @@ async function documentProcessor({
     } = await loadFile(file, name, tempDir);
     let supportedTypes = [];
     switch (config?.processor.name) {
-      case "docling":
-        supportedTypes = ["pdf", "docx", "doc", "txt", "md", "jpg", "jpeg", "png", "gif", "webp"];
-        break;
       case "officeparser":
         supportedTypes = ["docx", "pptx", "xlsx", "odt", "odp", "ods", "pdf", "rtf", "csv", "md", "html"];
         break;