@exulu/backend 3.7.4 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/{chunk-27K2CO47.js → chunk-QMN6MVHQ.js} +1 -1
  2. package/dist/{chunk-AWMU6QXB.js → chunk-RBEWHG7I.js} +297 -39
  3. package/dist/cli/start-whisper.js +1 -1
  4. package/dist/{convert-exulu-tools-to-ai-sdk-tools-XNQ6Q3X6.js → convert-exulu-tools-to-ai-sdk-tools-6RU4IZMI.js} +1 -1
  5. package/dist/index.cjs +566 -217
  6. package/dist/index.d.cts +3 -1
  7. package/dist/index.d.ts +3 -1
  8. package/dist/index.js +250 -180
  9. package/dist/{python-setup-JZGHWQCG.js → python-setup-DRJ3QX5F.js} +1 -1
  10. package/ee/LICENSE.md +2 -2
  11. package/ee/agentic-retrieval/pipeline/config.test.ts +18 -1
  12. package/ee/agentic-retrieval/pipeline/config.ts +15 -0
  13. package/ee/agentic-retrieval/pipeline/index.test.ts +73 -0
  14. package/ee/agentic-retrieval/pipeline/index.ts +67 -13
  15. package/ee/agentic-retrieval/pipeline/memory.test.ts +59 -0
  16. package/ee/agentic-retrieval/pipeline/memory.ts +181 -11
  17. package/ee/agentic-retrieval/pipeline/pin-rerun.test.ts +17 -0
  18. package/ee/agentic-retrieval/pipeline/pin-rerun.ts +29 -0
  19. package/ee/agentic-retrieval/pipeline/routing.test.ts +34 -0
  20. package/ee/agentic-retrieval/pipeline/routing.ts +96 -5
  21. package/ee/agentic-retrieval/pipeline/search.ts +9 -6
  22. package/ee/agentic-retrieval/pipeline/timing.test.ts +24 -0
  23. package/ee/agentic-retrieval/pipeline/timing.ts +26 -0
  24. package/ee/agentic-retrieval/pipeline/types.ts +2 -0
  25. package/ee/python/documents/processing/README.md +2 -3
  26. package/ee/python/documents/processing/doc_processor.ts +21 -61
  27. package/ee/python/documents/processing/split_pdf.py +25 -30
  28. package/ee/python/documents/processing/tests/__init__.py +0 -0
  29. package/ee/python/documents/processing/tests/test_split_pdf.py +230 -0
  30. package/ee/python/requirements.txt +12 -2
  31. package/ee/python/setup.sh +40 -1
  32. package/ee/python/transcription/pipeline.py +109 -15
  33. package/ee/python/transcription/tests/test_align_model_licensing.py +184 -0
  34. package/ee/workers.ts +2 -7
  35. package/license.md +2 -2
  36. package/package.json +3 -4
  37. package/scripts/postinstall.cjs +52 -1
  38. package/ee/python/documents/processing/document_to_markdown.py +0 -413
@@ -171,7 +171,7 @@ async function validatePythonEnvironment(packageRoot, checkPackages = true) {
171
171
  };
172
172
  }
173
173
  if (checkPackages) {
174
- const criticalPackages = ["docling", "transformers"];
174
+ const criticalPackages = ["pypdf", "transformers"];
175
175
  const missingPackages = [];
176
176
  for (const pkg of criticalPackages) {
177
177
  try {
@@ -1716,7 +1716,7 @@ var ExuluTool = class _ExuluTool {
1716
1716
  if (!agent) {
1717
1717
  throw new Error("Agent not found.");
1718
1718
  }
1719
- const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-XNQ6Q3X6.js");
1719
+ const { convertExuluToolsToAiSdkTools: convertExuluToolsToAiSdkTools2 } = await import("./convert-exulu-tools-to-ai-sdk-tools-6RU4IZMI.js");
1720
1720
  const tools = await convertExuluToolsToAiSdkTools2(
1721
1721
  [this],
1722
1722
  [],
@@ -2068,7 +2068,19 @@ var tuningSchema = z3.object({
2068
2068
  pinBoost: z3.number().min(0).max(1).default(0.15),
2069
2069
  identifierBoost: z3.number().min(0).max(1).default(0.15),
2070
2070
  pageWindow: z3.number().int().min(0).default(1),
2071
- maxQueriesPerContext: z3.number().int().positive().default(5)
2071
+ maxQueriesPerContext: z3.number().int().positive().default(5),
2072
+ /**
2073
+ * Orchestration engine. "v1" is the sequential flow every agent ran before 2026-09;
2074
+ * "v2" merges the phase-1 LLM hops and runs identifier pins in parallel. Per agent,
2075
+ * so a candidate agent can run v2 while the production agent stays on v1.
2076
+ */
2077
+ engine: z3.enum(["v1", "v2"]).default("v1"),
2078
+ /** v2 sub-features; each can be switched off on its own to bisect a regression. */
2079
+ v2: z3.object({
2080
+ mergedMemoryCall: z3.boolean().default(true),
2081
+ mergedRoutingCall: z3.boolean().default(true),
2082
+ parallelPins: z3.boolean().default(true)
2083
+ }).default({ mergedMemoryCall: true, mergedRoutingCall: true, parallelPins: true })
2072
2084
  });
2073
2085
  var boolVal = (v) => v === true || v === "true" || v === 1;
2074
2086
  var strVal = (v, fallback) => typeof v === "string" && v.length > 0 ? v : fallback;
@@ -2291,6 +2303,17 @@ async function microCall(args) {
2291
2303
  );
2292
2304
  }
2293
2305
 
2306
+ // ee/agentic-retrieval/pipeline/timing.ts
2307
+ async function withTiming(sink, key, work, now = Date.now) {
2308
+ if (!sink) return work();
2309
+ const started = now();
2310
+ try {
2311
+ return await work();
2312
+ } finally {
2313
+ sink[key] = Math.max(0, now() - started);
2314
+ }
2315
+ }
2316
+
2294
2317
  // ee/agentic-retrieval/pipeline/prefilter.ts
2295
2318
  import Fuse from "fuse.js";
2296
2319
  import { z as z4 } from "zod";
@@ -2606,6 +2629,49 @@ Only set pageNumber when the user names a specific page.
2606
2629
  Return hasFilenameHint/hasPageHint false (and omit the hint fields) when neither
2607
2630
  is present. When in doubt, return false.
2608
2631
  `;
2632
+ async function runMergedRoutingCall({
2633
+ model,
2634
+ question,
2635
+ knownIdentifiers,
2636
+ enabledContexts,
2637
+ routingRules,
2638
+ extraInstructions
2639
+ }) {
2640
+ const kbListing = enabledContexts.map((c) => `- ${c.id}: ${c.name}${c.description ? " \u2014 " + c.description : ""}`).join("\n");
2641
+ const rulesLines = routingRules.map((r) => `- ${r.id} (${r.label}): ${r.description}`).join("\n");
2642
+ const system = `You analyse the user's request and answer ${routingRules.length ? "three" : "two"} questions in one go.
2643
+
2644
+ A. DOCUMENT / PAGE REFERENCE (docPage):
2645
+ ${buildDocPagePrompt(knownIdentifiers)}
2646
+
2647
+ B. EXPLICIT KNOWLEDGE BASE REQUEST (explicitlyRequestedKnowledgeBases): check if the user has EXPLICITLY asked you to search in one or multiple of the following knowledge bases:
2648
+ ${kbListing}
2649
+ EXPLICIT means the user names a knowledge base or clearly commands searching a specific source (e.g. "search in the tickets", "look this up in the manuals KB"). A question that merely CONCERNS a topic related to a knowledge base's name or contents (e.g. asking about software changes, norms, or a product) is NOT an explicit request. When in doubt, return an empty array. If explicit, return the knowledge base ids.
2650
+ ` + (routingRules.length ? `
2651
+ C. CLASSIFICATION (classification): classify the request into exactly one of these categories:
2652
+ ${rulesLines}` + (extraInstructions ? `
2653
+ <instructions>
2654
+ ${extraInstructions}
2655
+ </instructions>` : "") : "");
2656
+ const ids = enabledContexts.map((c) => c.id);
2657
+ const schema = z5.object({
2658
+ docPage: z5.object({
2659
+ hasFilenameHint: z5.boolean(),
2660
+ filenameHints: z5.array(z5.string()).optional(),
2661
+ hasPageHint: z5.boolean(),
2662
+ pageNumber: z5.number().int().nullable().optional()
2663
+ }),
2664
+ explicitlyRequestedKnowledgeBases: z5.array(z5.enum(ids)),
2665
+ ...routingRules.length ? {
2666
+ classification: z5.object({
2667
+ ruleId: z5.enum(routingRules.map((r) => r.id)),
2668
+ reason: z5.string()
2669
+ })
2670
+ } : {}
2671
+ });
2672
+ const { output } = await microCall({ model, system, messages: [{ role: "user", content: question }], schema });
2673
+ return output;
2674
+ }
2609
2675
  async function runRoutingPhase(opts) {
2610
2676
  const {
2611
2677
  question,
@@ -2615,6 +2681,7 @@ async function runRoutingPhase(opts) {
2615
2681
  preselectedItems,
2616
2682
  extraInstructions,
2617
2683
  knownIdentifiers = [],
2684
+ mergedCall = false,
2618
2685
  model
2619
2686
  } = opts;
2620
2687
  try {
@@ -2634,7 +2701,23 @@ async function runRoutingPhase(opts) {
2634
2701
  ` + enabledContexts.map((c) => `- ${c.id}: ${c.name}${c.description ? " \u2014 " + c.description : ""}`).join("\n") + `
2635
2702
  EXPLICIT means the user names a knowledge base or clearly commands searching a specific source (e.g. "search in the tickets", "look this up in the manuals KB"). A question that merely CONCERNS a topic related to a knowledge base's name or contents (e.g. asking about software changes, norms, or a product) is NOT an explicit request \u2014 classification routing handles those. When in doubt, return an empty array.
2636
2703
  If explicit, return the knowledge base ids. If not, return an empty array.`;
2637
- const [docPageRaw, explicitKBRaw] = await Promise.all([
2704
+ const wantsClassification = routingRules.length > 0 && preselectedItems.size === 0;
2705
+ const merged = mergedCall ? await withTiming(
2706
+ opts.timings,
2707
+ "routing.mergedMs",
2708
+ () => runMergedRoutingCall({
2709
+ model,
2710
+ question,
2711
+ knownIdentifiers,
2712
+ enabledContexts,
2713
+ routingRules: wantsClassification ? routingRules : [],
2714
+ extraInstructions
2715
+ }).catch((err) => {
2716
+ console.warn("[EXULU pipeline] merged routing call failed \u2014 falling back to the v1 hops.", err);
2717
+ return null;
2718
+ })
2719
+ ) : null;
2720
+ const [docPageRaw, explicitKBRaw] = merged ? [{ output: merged.docPage }, { output: { explicitlyRequestedKnowledgeBases: merged.explicitlyRequestedKnowledgeBases } }] : await withTiming(opts.timings, "routing.detectMs", () => Promise.all([
2638
2721
  (async () => {
2639
2722
  try {
2640
2723
  return await microCall({
@@ -2676,7 +2759,7 @@ If explicit, return the knowledge base ids. If not, return an empty array.`;
2676
2759
  return { output: { explicitlyRequestedKnowledgeBases: [] } };
2677
2760
  }
2678
2761
  })()
2679
- ]);
2762
+ ]));
2680
2763
  const userPinnedItemIdsByContext = /* @__PURE__ */ new Map();
2681
2764
  let userRequestedPage = null;
2682
2765
  if (docPageRaw.output.hasFilenameHint && docPageRaw.output.filenameHints?.length) {
@@ -2754,7 +2837,7 @@ ${extraInstructions}
2754
2837
  </instructions>`;
2755
2838
  }
2756
2839
  try {
2757
- const { output: classified } = await microCall({
2840
+ const { output: classified } = merged?.classification ? { output: merged.classification } : await withTiming(opts.timings, "routing.classifyMs", () => microCall({
2758
2841
  model,
2759
2842
  system: classifyPrompt,
2760
2843
  messages: [{ role: "user", content: question }],
@@ -2762,7 +2845,7 @@ ${extraInstructions}
2762
2845
  ruleId: z5.enum(ruleIds),
2763
2846
  reason: z5.string()
2764
2847
  })
2765
- });
2848
+ }));
2766
2849
  const matchedRule = routingRules.find((r) => r.id === classified.ruleId);
2767
2850
  if (matchedRule) {
2768
2851
  const main = matchedRule.main.filter((id) => enabledIds.has(id));
@@ -2929,7 +3012,8 @@ async function recallMemoryByKeywords({
2929
3012
  importantKeyword,
2930
3013
  user,
2931
3014
  role,
2932
- memoryContext
3015
+ memoryContext,
3016
+ timings
2933
3017
  }) {
2934
3018
  const allKeywords = [
2935
3019
  ...new Set(
@@ -2946,7 +3030,7 @@ async function recallMemoryByKeywords({
2946
3030
  ...new Set(allKeywords.flatMap(deriveKeywordVariants).map(stripSeparators))
2947
3031
  ].filter((v) => v.length >= 4);
2948
3032
  if (!allVariants.length) return [];
2949
- const items = await loadMemoryItems(memoryContext);
3033
+ const items = await withTiming(timings, "memory.keywordRecall.itemsMs", () => loadMemoryItems(memoryContext));
2950
3034
  const scored = [];
2951
3035
  for (const item of items) {
2952
3036
  const haystack = stripSeparators(
@@ -2968,14 +3052,14 @@ async function recallMemoryByKeywords({
2968
3052
  "[EXULU pipeline] keyword-triggered memory matches:",
2969
3053
  topMatches.map((s) => `${s.name} (hits=${s.hits}, important=${s.importantHit})`)
2970
3054
  );
2971
- const chunks = await singleSearch({
3055
+ const chunks = await withTiming(timings, "memory.keywordRecall.searchMs", () => singleSearch({
2972
3056
  query: allKeywords.join(", "),
2973
- config: { method: "hybridSearch", cutoffs: void 0, limit: 50 },
3057
+ config: { method: "tsvector", cutoffs: void 0, limit: 50 },
2974
3058
  user,
2975
3059
  role,
2976
3060
  pinnedItemIds: topMatches.map((s) => s.id),
2977
3061
  context: memoryContext
2978
- });
3062
+ }));
2979
3063
  return chunks;
2980
3064
  }
2981
3065
  function neutralResult(question, keywords, importantKeyword, steps = []) {
@@ -2989,7 +3073,103 @@ function neutralResult(question, keywords, importantKeyword, steps = []) {
2989
3073
  steps
2990
3074
  };
2991
3075
  }
3076
+ async function runMergedMemoryCall({
3077
+ model,
3078
+ retrievedMemory,
3079
+ question,
3080
+ keywords,
3081
+ importantKeyword,
3082
+ memoryConfig,
3083
+ glossary
3084
+ }) {
3085
+ const glossaryText = glossary.length > 0 ? `
3086
+ The organization's documents use the following abbreviations/terms:
3087
+ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
3088
+ const system = `
3089
+ You review the shared company memory for the user's question and answer FOUR questions in one go.
3090
+
3091
+ 1. RELEVANCE (relevantChunkIds): return the chunk_ids of chunks containing information relevant to the
3092
+ question, or an empty array. Be generous: include chunks that are topically related, share key
3093
+ terminology, describe the same symptom from a different angle, or could plausibly help diagnose the
3094
+ issue \u2014 even if they don't answer the question directly. Memory entries are deliberately broad,
3095
+ hand-curated hints written by domain experts; the user's wording will rarely match the memory verbatim.
3096
+ When in doubt, include the chunk.
3097
+
3098
+ 2. OVERRIDE (override): decide whether ONE of the relevant chunks should become the AUTHORITATIVE basis
3099
+ of the answer, taking precedence over the official documentation even if the documents state something
3100
+ different. This is a deliberately STRICT check. Set overrides=true ONLY if a single chunk, on its own,
3101
+ contains a DIRECT and SUFFICIENT answer to exactly what the user asked. Being topically related, sharing
3102
+ terminology, describing the same component, or only partially addressing the question is NOT sufficient:
3103
+ then set overrides=false. When in doubt, set overrides=false. Memory entries may capture field experience
3104
+ that the manuals get wrong, so a confident, direct match is meant to win over the documents.
3105
+ ${memoryConfig.override ? "" : "(Override is disabled for this agent: return overrides=false.)"}
3106
+
3107
+ 3. FILE PRIORITIZATION (filePrioritization): set shouldPrioritizeFiles=true only if a relevant memory
3108
+ entry explicitly says to look in, prioritize, prefer, or always search a particular document, file, or
3109
+ file family (for example "When asked about X, always search in Y-Dateien first"). General background
3110
+ facts, glossaries, or synonyms are NOT a file prioritization instruction. When true, return
3111
+ fileNameHints exactly as referenced in the memory, bare names without folder paths.
3112
+ ${memoryConfig.filePrioritization ? "" : "(File prioritization is disabled for this agent: return false.)"}
3113
+
3114
+ 4. QUERY AUGMENTATION (augmentation): if, and only if, the relevant memory (or the glossary below)
3115
+ contains synonyms or similar terms for what the user asked, return the user question and keywords
3116
+ updated to include those synonyms \u2014 always keeping the original wording as well. Otherwise return the
3117
+ original question, keywords and important keyword unchanged.
3118
+ ${memoryConfig.queryAugmentation ? "" : "(Query augmentation is disabled for this agent: return the originals.)"}
3119
+
3120
+ <memory_chunks>
3121
+ ${retrievedMemory.map((chunk) => `- ${chunk.chunk_id}: ${chunk.item_name} - ${chunk.chunk_content}`).join("\n")}
3122
+ </memory_chunks>
3123
+ ${glossaryText}
3124
+ `;
3125
+ const { output } = await microCall({
3126
+ model,
3127
+ system,
3128
+ messages: [
3129
+ {
3130
+ role: "user",
3131
+ content: `
3132
+ <user_question>${question}</user_question>
3133
+ <relevant_keywords>${keywords.join(", ")}</relevant_keywords>
3134
+ <important_keyword>${importantKeyword}</important_keyword>
3135
+ `
3136
+ }
3137
+ ],
3138
+ schema: z6.object({
3139
+ relevantChunkIds: z6.array(z6.string()).describe("chunk_ids (UUIDs at the start of each bullet) of relevant chunks; empty array if none."),
3140
+ override: z6.object({
3141
+ overrides: z6.boolean().describe("True ONLY if a chunk directly and sufficiently answers the question."),
3142
+ confidence: z6.enum(["high", "medium", "low"]),
3143
+ authoritativeChunkIds: z6.array(z6.string()).describe("chunk_ids that directly answer the question; empty if overrides is false."),
3144
+ reason: z6.string().describe("One short sentence.")
3145
+ }),
3146
+ filePrioritization: z6.object({
3147
+ shouldPrioritizeFiles: z6.boolean(),
3148
+ fileNameHints: z6.array(z6.string()).optional()
3149
+ }),
3150
+ augmentation: z6.object({
3151
+ updatedUserQuestion: z6.string(),
3152
+ updatedRelevantKeywords: z6.array(z6.string()),
3153
+ updatedImportantKeyword: z6.string()
3154
+ })
3155
+ })
3156
+ });
3157
+ return output;
3158
+ }
3159
+ function mergedFollowups(merged, memoryConfig, hasAugmentationContent, question, importantKeyword) {
3160
+ const overrideResult = {
3161
+ output: memoryConfig.override ? merged.override : { overrides: false, confidence: "low", authoritativeChunkIds: [], reason: "" }
3162
+ };
3163
+ const fileResult = {
3164
+ output: memoryConfig.filePrioritization ? merged.filePrioritization : { shouldPrioritizeFiles: false, fileNameHints: [] }
3165
+ };
3166
+ const queryResult = {
3167
+ output: memoryConfig.queryAugmentation && hasAugmentationContent ? merged.augmentation : { updatedUserQuestion: question, updatedRelevantKeywords: [], updatedImportantKeyword: importantKeyword }
3168
+ };
3169
+ return [overrideResult, fileResult, queryResult];
3170
+ }
2992
3171
  async function runMemoryPhase({
3172
+ timings,
2993
3173
  memoryChunks,
2994
3174
  memoryContext,
2995
3175
  question,
@@ -3000,7 +3180,8 @@ async function runMemoryPhase({
3000
3180
  model,
3001
3181
  memoryConfig,
3002
3182
  glossary,
3003
- documentContexts
3183
+ documentContexts,
3184
+ mergedCall = false
3004
3185
  }) {
3005
3186
  try {
3006
3187
  if (!memoryConfig.enabled || memoryChunks.length === 0 && !memoryContext) {
@@ -3010,13 +3191,14 @@ async function runMemoryPhase({
3010
3191
  let retrieved_memory = [...memoryChunks];
3011
3192
  if (memoryContext) {
3012
3193
  try {
3013
- const keywordMatched = await recallMemoryByKeywords({
3194
+ const keywordMatched = await withTiming(timings, "memory.keywordRecallMs", () => recallMemoryByKeywords({
3014
3195
  keywords,
3015
3196
  importantKeyword,
3016
3197
  user,
3017
3198
  role,
3018
- memoryContext
3019
- });
3199
+ memoryContext,
3200
+ timings
3201
+ }));
3020
3202
  if (keywordMatched.length > 0) {
3021
3203
  const seen = new Set(retrieved_memory.map((c) => c.chunk_id));
3022
3204
  const additions = keywordMatched.filter((c) => !seen.has(c.chunk_id));
@@ -3041,8 +3223,24 @@ async function runMemoryPhase({
3041
3223
  </memory_chunks>
3042
3224
  `;
3043
3225
  let relevantMemoryChunks = [];
3226
+ let mergedOutput;
3044
3227
  try {
3045
- const { output: output_relevant_memory } = await microCall({
3228
+ if (mergedCall) {
3229
+ mergedOutput = await withTiming(
3230
+ timings,
3231
+ "memory.mergedMs",
3232
+ () => runMergedMemoryCall({
3233
+ model,
3234
+ retrievedMemory: retrieved_memory,
3235
+ question,
3236
+ keywords,
3237
+ importantKeyword,
3238
+ memoryConfig,
3239
+ glossary
3240
+ })
3241
+ );
3242
+ }
3243
+ const { output: output_relevant_memory } = mergedOutput ? { output: { relevantChunkIds: mergedOutput.relevantChunkIds } } : await withTiming(timings, "memory.relevanceMs", () => microCall({
3046
3244
  model,
3047
3245
  system: CHECK_MEMORIES_FOR_RELEVANT_INFORMATION,
3048
3246
  messages: [
@@ -3060,7 +3258,7 @@ async function runMemoryPhase({
3060
3258
  "The chunk_ids (UUIDs at the start of each bullet) of chunks containing information relevant to the user's question. Empty array if none are relevant."
3061
3259
  )
3062
3260
  })
3063
- });
3261
+ }));
3064
3262
  const ids = new Set(output_relevant_memory?.relevantChunkIds ?? []);
3065
3263
  relevantMemoryChunks = ids.size === 0 ? [] : retrieved_memory.filter((c) => ids.has(c.chunk_id));
3066
3264
  } catch (e) {
@@ -3150,7 +3348,7 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
3150
3348
 
3151
3349
  Otherwise, return the original user question, relevant keywords and important keyword.
3152
3350
  `;
3153
- const [overrideResult, fileResult, queryResult] = await Promise.all([
3351
+ const [overrideResult, fileResult, queryResult] = mergedOutput ? mergedFollowups(mergedOutput, memoryConfig, hasAugmentationContent, question, importantKeyword) : await withTiming(timings, "memory.followupsMs", () => Promise.all([
3154
3352
  // Override check: strict gate to decide if memory should be authoritative
3155
3353
  memoryConfig.override ? microCall({
3156
3354
  model,
@@ -3231,7 +3429,7 @@ ${glossary.map((g) => `${g.term} : ${g.meaning}`).join("\n")}` : "";
3231
3429
  updatedImportantKeyword: importantKeyword
3232
3430
  }
3233
3431
  })
3234
- ]);
3432
+ ]));
3235
3433
  const overrideIds = new Set(overrideResult.output?.authoritativeChunkIds ?? []);
3236
3434
  const authoritativeChunks = memoryChunksForAnswer.filter(
3237
3435
  (c) => c.chunk_id && overrideIds.has(c.chunk_id)
@@ -3407,7 +3605,7 @@ async function searchContexts(opts) {
3407
3605
  skipPrefilter
3408
3606
  } = opts;
3409
3607
  const chunkArrays = await Promise.all(
3410
- contextIds.map(async (ctxId) => {
3608
+ contextIds.map((ctxId) => withTiming(opts.timings, `${opts.timingPrefix ?? "search"}.${ctxId}Ms`, async () => {
3411
3609
  try {
3412
3610
  const ctx = contextsById.get(ctxId);
3413
3611
  if (!ctx) return [];
@@ -3445,13 +3643,13 @@ async function searchContexts(opts) {
3445
3643
  if (multiQuery) {
3446
3644
  let hydePassage = null;
3447
3645
  if (hyde) {
3448
- hydePassage = await generateHydePassage({
3646
+ hydePassage = await withTiming(opts.timings, `${opts.timingPrefix ?? "search"}.${ctxId}.hydeMs`, () => generateHydePassage({
3449
3647
  originalQuestion: question,
3450
3648
  relevantKeywords: keywords,
3451
3649
  importantKeyword,
3452
3650
  styleHint,
3453
3651
  model
3454
- });
3652
+ }));
3455
3653
  }
3456
3654
  const candidates = [
3457
3655
  question,
@@ -3466,14 +3664,14 @@ async function searchContexts(opts) {
3466
3664
  }
3467
3665
  if (kind === "conversations") {
3468
3666
  if (keywordPrefilter && pinnedItemIds.length === 0) {
3469
- const prefiltered = await fuzzyPrefilter({
3667
+ const prefiltered = await withTiming(opts.timings, `${opts.timingPrefix ?? "search"}.${ctxId}.prefilterMs`, () => fuzzyPrefilter({
3470
3668
  cacheKey: `conversations:${ctxId}`,
3471
3669
  relevantKeywords: keywords,
3472
3670
  importantKeyword,
3473
3671
  context: ctx,
3474
3672
  fields: ["name", "id", "external_id", "description"],
3475
3673
  normalize: (i) => [i.name, i.description].filter(Boolean).join(": ")
3476
- });
3674
+ }));
3477
3675
  pinnedItemIds = prefiltered.map((r) => r.id);
3478
3676
  }
3479
3677
  const keywordQuery = keywords.length ? keywords.join(" ") + " " + importantKeyword : question;
@@ -3488,7 +3686,7 @@ async function searchContexts(opts) {
3488
3686
  console.warn(`[EXULU pipeline] searchContexts failed for context "${ctxId}":`, err);
3489
3687
  return [];
3490
3688
  }
3491
- })
3689
+ }))
3492
3690
  );
3493
3691
  return { chunks: chunkArrays.flat() };
3494
3692
  }
@@ -3633,6 +3831,27 @@ async function rerankResults(opts) {
3633
3831
  return { limited_results, sorted_reranked_results: sorted, rerank_score_max_genuine };
3634
3832
  }
3635
3833
 
3834
+ // ee/agentic-retrieval/pipeline/pin-rerun.ts
3835
+ function needsPinRerun(originalQuestion, updatedQuestion) {
3836
+ if (originalQuestion === updatedQuestion) return false;
3837
+ const before = designations(originalQuestion);
3838
+ for (const token of designations(updatedQuestion)) {
3839
+ if (!before.has(token)) return true;
3840
+ }
3841
+ return false;
3842
+ }
3843
+ function designations(text) {
3844
+ const out = /* @__PURE__ */ new Set();
3845
+ for (const raw of text.split(/[\s,;:()\[\]"'?!]+/)) {
3846
+ const token = raw.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
3847
+ if (!token) continue;
3848
+ const hasDigit = /\p{N}/u.test(token);
3849
+ const capitals = (token.match(/\p{Lu}/gu) ?? []).length;
3850
+ if (hasDigit || capitals >= 2) out.add(token.toLowerCase());
3851
+ }
3852
+ return out;
3853
+ }
3854
+
3636
3855
  // ee/agentic-retrieval/pipeline/index.ts
3637
3856
  function addChunks(result, chunks) {
3638
3857
  const seen = new Set(result.chunks.map((c) => c.chunk_id));
@@ -3798,6 +4017,14 @@ function createAgenticRetrievalTool(opts) {
3798
4017
  usage: [],
3799
4018
  totalTokens: 0
3800
4019
  };
4020
+ const t0 = Date.now();
4021
+ const timings = {};
4022
+ let tPhase = t0;
4023
+ const lap = (name) => {
4024
+ const now = Date.now();
4025
+ timings[name] = (timings[name] ?? 0) + (now - tPhase);
4026
+ tPhase = now;
4027
+ };
3801
4028
  try {
3802
4029
  let enabledContexts = contexts.filter(
3803
4030
  (ctx) => cfg.knowledgeBases[ctx.id]?.enabled !== false
@@ -3870,8 +4097,19 @@ function createAgenticRetrievalTool(opts) {
3870
4097
  resolvedProject && projectScope?.customInstructions ? `Instructions for the attached project "${projectScope.name}":
3871
4098
  ${projectScope.customInstructions}` : ""
3872
4099
  ].filter(Boolean).join("\n");
3873
- const [memResult, routResult] = await Promise.all([
3874
- runMemoryPhase({
4100
+ const engineV2 = cfg.tuning.engine === "v2";
4101
+ const v2 = cfg.tuning.v2;
4102
+ const pinsFor = (question) => resolveIdentifierPins({
4103
+ question,
4104
+ identifierSets: cfg.vocabulary.identifiers,
4105
+ contextsById,
4106
+ kbKindById,
4107
+ model: utilityModel
4108
+ });
4109
+ const [memResult, routResult, parallelPins] = await Promise.all([
4110
+ withTiming(timings, "memoryMs", () => runMemoryPhase({
4111
+ timings,
4112
+ mergedCall: engineV2 && v2.mergedMemoryCall,
3875
4113
  memoryChunks: memoryItems ?? [],
3876
4114
  memoryContext,
3877
4115
  question: userQuery,
@@ -3883,8 +4121,10 @@ ${projectScope.customInstructions}` : ""
3883
4121
  memoryConfig: cfg.memory,
3884
4122
  glossary: cfg.vocabulary.glossary,
3885
4123
  documentContexts
3886
- }),
3887
- runRoutingPhase({
4124
+ })),
4125
+ withTiming(timings, "routingMs", () => runRoutingPhase({
4126
+ timings,
4127
+ mergedCall: engineV2 && v2.mergedRoutingCall,
3888
4128
  question: userQuery,
3889
4129
  enabledContexts,
3890
4130
  documentContexts,
@@ -3895,8 +4135,12 @@ ${projectScope.customInstructions}` : ""
3895
4135
  // detector must never treat these as filename hints.
3896
4136
  knownIdentifiers: cfg.vocabulary.identifiers.flatMap((i) => i.examples),
3897
4137
  model: utilityModel
3898
- })
4138
+ })),
4139
+ // engine v2: identifier pins depend only on the question, so they run alongside
4140
+ // memory and routing instead of after them (re-run below if memory rewrote the question).
4141
+ engineV2 && v2.parallelPins ? withTiming(timings, "pinsParallelMs", () => pinsFor(userQuery)) : Promise.resolve(null)
3899
4142
  ]);
4143
+ lap("memoryRoutingMs");
3900
4144
  for (const step of [...memResult.steps, ...routResult.steps]) {
3901
4145
  result.steps.push({
3902
4146
  stepNumber: 1,
@@ -3946,13 +4190,8 @@ ${projectScope.customInstructions}` : ""
3946
4190
  memoryPinnedItemIdsByContext,
3947
4191
  memoryOverride
3948
4192
  } = memResult;
3949
- const { pinsByContext: identifierPinsByContext, exactPinsByContext, steps: pinSteps } = await resolveIdentifierPins({
3950
- question: updatedQuestion,
3951
- identifierSets: cfg.vocabulary.identifiers,
3952
- contextsById,
3953
- kbKindById,
3954
- model: utilityModel
3955
- });
4193
+ const { pinsByContext: identifierPinsByContext, exactPinsByContext, steps: pinSteps } = parallelPins && !needsPinRerun(userQuery, updatedQuestion) ? parallelPins : await pinsFor(updatedQuestion);
4194
+ lap("pinsMs");
3956
4195
  for (const step of pinSteps) {
3957
4196
  result.steps.push({
3958
4197
  stepNumber: 1,
@@ -3982,6 +4221,8 @@ ${projectScope.customInstructions}` : ""
3982
4221
  rewrites: cfg.vocabulary.rewrites,
3983
4222
  styleHint: cfg.vocabulary.styleHint,
3984
4223
  maxQueries: cfg.tuning.maxQueriesPerContext,
4224
+ timings,
4225
+ timingPrefix: "search.main",
3985
4226
  skipPrefilter: false
3986
4227
  }),
3987
4228
  fallbackContextsToSearch.length > 0 && !hasExplicitDocAndPage ? searchContexts({
@@ -4002,9 +4243,12 @@ ${projectScope.customInstructions}` : ""
4002
4243
  rewrites: cfg.vocabulary.rewrites,
4003
4244
  styleHint: cfg.vocabulary.styleHint,
4004
4245
  maxQueries: cfg.tuning.maxQueriesPerContext,
4246
+ timings,
4247
+ timingPrefix: "search.fallback",
4005
4248
  skipPrefilter: true
4006
4249
  }) : Promise.resolve({ chunks: [] })
4007
4250
  ]);
4251
+ lap("searchMs");
4008
4252
  const pinnedItemIds = /* @__PURE__ */ new Set([
4009
4253
  ...(function* () {
4010
4254
  for (const s of memoryPinnedItemIdsByContext.values()) yield* s;
@@ -4067,6 +4311,7 @@ ${projectScope.customInstructions}` : ""
4067
4311
  chunks: [],
4068
4312
  tokens: 0
4069
4313
  });
4314
+ lap("rerankMs");
4070
4315
  addChunks(result, mainRerank.limited_results);
4071
4316
  yield { result: serializeOutput(result) };
4072
4317
  const literalLookupSatisfied = hasExplicitDocAndPage && mainRerank.limited_results.length > 0 && mainRerank.limited_results.some((r) => {
@@ -4115,6 +4360,7 @@ ${projectScope.customInstructions}` : ""
4115
4360
  });
4116
4361
  result.reasoning.push({ text: "Fallback results reranked", tools: [] });
4117
4362
  addChunks(result, fallbackRerank.limited_results);
4363
+ lap("fallbackRerankMs");
4118
4364
  yield { result: serializeOutput(result) };
4119
4365
  }
4120
4366
  if (memoryOverride.active) {
@@ -4137,9 +4383,19 @@ Verified answer:
4137
4383
  addChunks(result, memoryOverride.chunks);
4138
4384
  yield { result: serializeOutput(result) };
4139
4385
  }
4386
+ timings.totalMs = Date.now() - t0;
4387
+ result.timings = timings;
4388
+ result.steps.push({
4389
+ stepNumber: 1,
4390
+ text: `Timing: memory+routing ${timings.memoryRoutingMs ?? 0}ms (memory ${timings.memoryMs ?? 0}ms, routing ${timings.routingMs ?? 0}ms), pins ${timings.pinsMs ?? 0}ms, search ${timings.searchMs ?? 0}ms, rerank ${timings.rerankMs ?? 0}ms` + (timings.fallbackRerankMs !== void 0 ? `, fallback rerank ${timings.fallbackRerankMs}ms` : "") + `, total ${timings.totalMs}ms`,
4391
+ toolCalls: [],
4392
+ chunks: [],
4393
+ tokens: 0
4394
+ });
4140
4395
  if (cfg.logging) {
4141
- console.log("[EXULU pipeline] final result:", JSON.stringify({ steps: result.steps.length, chunks: result.chunks.length }));
4396
+ console.log("[EXULU pipeline] final result:", JSON.stringify({ steps: result.steps.length, chunks: result.chunks.length, timings }));
4142
4397
  }
4398
+ yield { result: serializeOutput(result) };
4143
4399
  return { result: serializeOutput(result) };
4144
4400
  } catch (err) {
4145
4401
  console.warn("[EXULU pipeline] retrieval pipeline failed:", err);
@@ -6052,7 +6308,7 @@ async function downloadKeyIntoSandbox(opts) {
6052
6308
  }
6053
6309
  async function resolvePythonVenvPath() {
6054
6310
  try {
6055
- const { getPythonVenvPath } = await import("./python-setup-JZGHWQCG.js");
6311
+ const { getPythonVenvPath } = await import("./python-setup-DRJ3QX5F.js");
6056
6312
  return getPythonVenvPath();
6057
6313
  } catch (err) {
6058
6314
  console.warn("[SKILLS] Could not resolve the Python venv for the session sandbox; skill scripts fall back to the system python.", err);
@@ -8248,6 +8504,7 @@ export {
8248
8504
  sanitizeToolName,
8249
8505
  KB_EDITOR_TOOL_ID,
8250
8506
  createKbEditorPickerTool,
8507
+ isIgnoredArtifactPath,
8251
8508
  reportSystemDependencies,
8252
8509
  downloadKeyIntoSandbox,
8253
8510
  truncateToolOutput,
@@ -8263,6 +8520,7 @@ export {
8263
8520
  guardExtractedFileText,
8264
8521
  SCRUBBED_CREDENTIAL_TEXT,
8265
8522
  SCRUBBED_OAUTH_TEXT,
8523
+ sessionFilePrefix,
8266
8524
  PreviewRenderError,
8267
8525
  getPdfPreviewBytes,
8268
8526
  imageAttachmentGuard,
@@ -2,7 +2,7 @@
2
2
  import "dotenv/config";
3
3
  import {
4
4
  getPackageRoot
5
- } from "../chunk-27K2CO47.js";
5
+ } from "../chunk-QMN6MVHQ.js";
6
6
 
7
7
  // src/cli/start-whisper.ts
8
8
  import "dotenv/config";
@@ -2,7 +2,7 @@ import "dotenv/config";
2
2
  import {
3
3
  convertExuluToolsToAiSdkTools,
4
4
  hydrateVariables
5
- } from "./chunk-AWMU6QXB.js";
5
+ } from "./chunk-RBEWHG7I.js";
6
6
  import "./chunk-4PDWNVNT.js";
7
7
  export {
8
8
  convertExuluToolsToAiSdkTools,