@exulu/backend 1.69.3 → 2.0.0

package/ee/agentic-retrieval/pipeline/config.test.ts

@@ -0,0 +1,81 @@
+import { parsePipelineConfig, effectiveKbSettings, KIND_PRESETS } from "./config";
+
+describe("parsePipelineConfig", () => {
+  it("returns full defaults for an empty/missing config", () => {
+    const cfg = parsePipelineConfig(undefined);
+    expect(cfg.tuning).toEqual({ topK: 5, fallbackThreshold: 0.95, pinBoost: 0.15,
+      identifierBoost: 0.15, pageWindow: 1, maxQueriesPerContext: 5 });
+    expect(cfg.memory).toEqual({ enabled: true, override: false, filePrioritization: false, queryAugmentation: true });
+    expect(cfg.routing.rules).toEqual([]);
+    expect(cfg.knowledgeBases).toEqual({});
+    expect(cfg.managedContext).toBe(false);
+    expect(cfg.reranker).toBe("none");
+  });
+
+  it("accepts parsed objects and JSON strings for json options", () => {
+    const cfg = parsePipelineConfig({
+      routing: { rules: [{ id: "tech", label: "T", description: "d", main: ["a"], fallback: [] }] },
+      tuning: '{"topK": 8}',
+      managed_context: "true",
+    });
+    expect(cfg.routing.rules[0].id).toBe("tech");
+    expect(cfg.tuning.topK).toBe(8);
+    expect(cfg.tuning.fallbackThreshold).toBe(0.95); // partial json keeps other defaults
+    expect(cfg.managedContext).toBe(true);
+  });
+
+  it("falls back to defaults on malformed json values", () => {
+    const warn = jest.spyOn(console, "warn").mockImplementation(() => {});
+    const cfg = parsePipelineConfig({ vocabulary: "{oops", memory: 42 });
+    expect(cfg.vocabulary.glossary).toEqual([]);
+    expect(cfg.memory.enabled).toBe(true);
+    warn.mockRestore();
+  });
+
+  it("never throws on any invalid input — structural guarantee", () => {
+    const warn = jest.spyOn(console, "warn").mockImplementation(() => {});
+    // All config keys set to garbage values
+    const cfg = parsePipelineConfig({
+      instructions: 123,
+      reranker: [],
+      managed_context: "invalid",
+      require_preselected_contexts: {},
+      logging: null,
+      utility_model: false,
+      knowledge_bases: 999,
+      routing: "not json",
+      vocabulary: [1, 2, 3],
+      memory: "broken{json",
+      tuning: Symbol("bad"),
+    });
+    // Must return a valid PipelineConfig with all required fields
+    expect(cfg).toHaveProperty("instructions");
+    expect(cfg).toHaveProperty("reranker");
+    expect(cfg).toHaveProperty("tuning");
+    expect(cfg.tuning.topK).toBe(5); // defaults are applied
+    expect(cfg.routing.rules).toEqual([]);
+    expect(cfg.knowledgeBases).toEqual({});
+    warn.mockRestore();
+  });
+});
+
+describe("effectiveKbSettings", () => {
+  it("applies kind presets", () => {
+    const s = effectiveKbSettings({ enabled: true, kind: "documents", instructions: "", overrides: {} }, {});
+    expect(s).toMatchObject({ limit: 100, expand: { before: 7, after: 7 }, multiQuery: true, hyde: true });
+    expect(KIND_PRESETS.conversations.keywordPrefilter).toBe(true);
+  });
+
+  it("precedence: overrides > context.configuration > preset", () => {
+    const ctx = { configuration: { expand: { before: 3, after: 3 }, cutoffs: { hybrid: 1.1 }, maxRetrievalResults: 40 } };
+    const s = effectiveKbSettings({ enabled: true, kind: "documents", instructions: "", overrides: { limit: 60 } }, ctx);
+    expect(s.limit).toBe(60);                       // override wins
+    expect(s.expand).toEqual({ before: 3, after: 3 }); // context config beats preset
+    expect(s.cutoffs).toEqual({ hybrid: 1.1 });
+  });
+
+  it("defaults a missing profile to enabled documents", () => {
+    const s = effectiveKbSettings(undefined, {});
+    expect(s.kind).toBe("documents");
+  });
+});

package/ee/agentic-retrieval/pipeline/config.ts

@@ -0,0 +1,189 @@
+import { z } from "zod";
+
+export const KB_KINDS = ["documents", "conversations", "records"] as const;
+export type KbKind = (typeof KB_KINDS)[number];
+
+export const DEFAULT_PREFILTER_CUTOFF = 2.5;
+export const RRF_K = 60;
+export const CHUNK_GROUP_MAX = 10;
+
+const kbProfileSchema = z.object({
+  enabled: z.boolean().default(true),
+  kind: z.enum(KB_KINDS).default("documents"),
+  instructions: z.string().default(""),
+  overrides: z
+    .object({
+      limit: z.number().int().positive().optional(),
+      expand: z.number().int().min(0).optional(),
+      multiQuery: z.boolean().optional(),
+      hyde: z.boolean().optional(),
+    })
+    .default({}),
+});
+const knowledgeBasesSchema = z.record(z.string(), kbProfileSchema);
+
+const routingRuleSchema = z.object({
+  id: z.string(),
+  label: z.string(),
+  description: z.string(),
+  main: z.array(z.string()),
+  fallback: z.array(z.string()).default([]),
+});
+const routingSchema = z.object({ rules: z.array(routingRuleSchema).default([]) });
+
+const identifierSetSchema = z.object({
+  name: z.string(),
+  description: z.string().default(""),
+  examples: z.array(z.string()).default([]),
+  strategy: z.enum(["fuzzy", "exact"]),
+  contexts: z.array(z.string()).default([]),
+});
+const vocabularySchema = z.object({
+  glossary: z.array(z.object({ term: z.string(), meaning: z.string() })).default([]),
+  identifiers: z.array(identifierSetSchema).default([]),
+  rewrites: z.array(z.object({ find: z.string(), replace: z.string() })).default([]),
+  styleHint: z.string().default(""),
+});
+const memorySchema = z.object({
+  enabled: z.boolean().default(true),
+  override: z.boolean().default(false),
+  filePrioritization: z.boolean().default(false),
+  queryAugmentation: z.boolean().default(true),
+});
+const tuningSchema = z.object({
+  topK: z.number().int().positive().default(5),
+  fallbackThreshold: z.number().min(0).max(1).default(0.95),
+  pinBoost: z.number().min(0).max(1).default(0.15),
+  identifierBoost: z.number().min(0).max(1).default(0.15),
+  pageWindow: z.number().int().min(0).default(1),
+  maxQueriesPerContext: z.number().int().positive().default(5),
+});
+
+export type KbProfile = z.infer<typeof kbProfileSchema>;
+export type RoutingRule = z.infer<typeof routingRuleSchema>;
+export type IdentifierSet = z.infer<typeof identifierSetSchema>;
+
+export type PipelineConfig = {
+  instructions: string;
+  reranker: string;
+  managedContext: boolean;
+  requirePreselectedContexts: boolean;
+  logging: boolean;
+  utilityModel: string;
+  knowledgeBases: Record<string, KbProfile>;
+  routing: z.infer<typeof routingSchema>;
+  vocabulary: z.infer<typeof vocabularySchema>;
+  memory: z.infer<typeof memorySchema>;
+  tuning: z.infer<typeof tuningSchema>;
+};
+
+const boolVal = (v: unknown): boolean => v === true || v === "true" || v === 1;
+const strVal = (v: unknown, fallback: string): string =>
+  typeof v === "string" && v.length > 0 ? v : fallback;
+
+/** Parse one json-typed option: accepts an object (already hydrated), a JSON string,
+ * or anything else (→ schema default). Schema failures fall back to defaults with a warning. */
+function jsonVal<S extends z.ZodTypeAny>(name: string, schema: S, v: unknown): z.infer<S> {
+  let candidate: unknown = v;
+  if (typeof v === "string" && v.trim().length > 0) {
+    try {
+      candidate = JSON.parse(v);
+    } catch (err) {
+      console.warn(`[EXULU pipeline] config "${name}" is not valid JSON — using defaults.`, err);
+      candidate = undefined;
+    }
+  } else if (typeof v !== "object" || v === null) {
+    candidate = undefined;
+  }
+  if (candidate !== undefined) {
+    const parsed = schema.safeParse(candidate);
+    if (parsed.success) return parsed.data;
+    console.warn(`[EXULU pipeline] config "${name}" failed validation — using defaults.`, parsed.error.message);
+  }
+  // Final fallback: try defaultFor, then undefined, then cast as last resort (never throw)
+  try {
+    return schema.parse(defaultFor(name));
+  } catch (err) {
+    console.warn(`[EXULU pipeline] failed to parse fallback for option "${name}" — falling back to schema.parse(undefined).`, err);
+    try {
+      return schema.parse(undefined);
+    } catch {
+      return defaultFor(name) as z.infer<S>;
+    }
+  }
+}
+
+function defaultFor(name: string): unknown {
+  switch (name) {
+    case "knowledge_bases": return {};
+    case "routing": return { rules: [] };
+    case "vocabulary": return { glossary: [], identifiers: [], rewrites: [], styleHint: "" };
+    case "memory": return {};
+    case "tuning": return {};
+    default: return {};
+  }
+}
+
+export function parsePipelineConfig(raw?: Record<string, unknown>): PipelineConfig {
+  const r = raw ?? {};
+  return {
+    instructions: strVal(r["instructions"], ""),
+    reranker: strVal(r["reranker"], "none"),
+    managedContext: boolVal(r["managed_context"]),
+    requirePreselectedContexts: boolVal(r["require_preselected_contexts"]),
+    logging: boolVal(r["logging"]),
+    utilityModel: strVal(r["utility_model"], ""),
+    knowledgeBases: jsonVal("knowledge_bases", knowledgeBasesSchema, r["knowledge_bases"]),
+    routing: jsonVal("routing", routingSchema, r["routing"]),
+    vocabulary: jsonVal("vocabulary", vocabularySchema, r["vocabulary"]),
+    memory: jsonVal("memory", memorySchema, r["memory"]),
+    tuning: jsonVal("tuning", tuningSchema, r["tuning"]),
+  };
+}
+
+export const KIND_PRESETS: Record<
+  KbKind,
+  { limit: number; expand: number; multiQuery: boolean; hyde: boolean; keywordPrefilter: boolean }
+> = {
+  documents: { limit: 100, expand: 7, multiQuery: true, hyde: true, keywordPrefilter: false },
+  conversations: { limit: 20, expand: 5, multiQuery: false, hyde: false, keywordPrefilter: true },
+  records: { limit: 20, expand: 2, multiQuery: false, hyde: false, keywordPrefilter: false },
+};
+
+export type EffectiveKbSettings = {
+  kind: KbKind;
+  instructions: string;
+  limit: number;
+  expand: { before: number; after: number } | undefined;
+  cutoffs: { cosineDistance?: number; tsvector?: number; hybrid?: number } | undefined;
+  multiQuery: boolean;
+  hyde: boolean;
+  keywordPrefilter: boolean;
+};
+
+/** Precedence per setting: profile.overrides > ctx.configuration > kind preset. */
+export function effectiveKbSettings(
+  profile: KbProfile | undefined,
+  ctx: { configuration?: { expand?: { before?: number; after?: number }; cutoffs?: any; maxRetrievalResults?: number } },
+): EffectiveKbSettings {
+  const kind: KbKind = profile?.kind ?? "documents";
+  const preset = KIND_PRESETS[kind];
+  const o = profile?.overrides ?? {};
+  const conf = ctx.configuration ?? {};
+  const expandN = o.expand ?? undefined;
+  const expand = expandN !== undefined
+    ? expandN === 0 ? undefined : { before: expandN, after: expandN }
+    : conf.expand && (conf.expand.before || conf.expand.after)
+      ? { before: conf.expand.before ?? 0, after: conf.expand.after ?? 0 }
+      : { before: preset.expand, after: preset.expand };
+  return {
+    kind,
+    instructions: profile?.instructions ?? "",
+    limit: o.limit ?? conf.maxRetrievalResults ?? preset.limit,
+    expand,
+    cutoffs: conf.cutoffs ?? undefined,
+    multiQuery: o.multiQuery ?? preset.multiQuery,
+    hyde: o.hyde ?? preset.hyde,
+    keywordPrefilter: preset.keywordPrefilter,
+  };
+}

package/ee/agentic-retrieval/pipeline/hyde.test.ts

@@ -0,0 +1,55 @@
+import { generateHydePassage, clearHydeCache } from "./hyde";
+
+jest.mock("ai", () => ({ generateText: jest.fn() }));
+import { generateText } from "ai";
+
+beforeEach(() => {
+  clearHydeCache();
+  (generateText as jest.Mock).mockReset();
+});
+
+describe("generateHydePassage", () => {
+  it("returns the generated passage and includes the styleHint in the prompt", async () => {
+    (generateText as jest.Mock).mockResolvedValue({ text: "A passage." });
+    const p = await generateHydePassage({
+      originalQuestion: "How do I lock door A?",
+      relevantKeywords: ["door"],
+      importantKeyword: "FST-2XT",
+      styleHint: "German elevator manuals",
+      model: {},
+    });
+    expect(p).toBe("A passage.");
+    const prompt = (generateText as jest.Mock).mock.calls[0][0].prompt as string;
+    expect(prompt).toContain("German elevator manuals");
+    expect(prompt).toContain("FST-2XT");
+    expect(prompt).toContain("same language as the question");
+  });
+
+  it("memoizes per question (one LLM call for two invocations)", async () => {
+    (generateText as jest.Mock).mockResolvedValue({ text: "A passage." });
+    const opts = { originalQuestion: "q", relevantKeywords: [], styleHint: "", model: {} };
+    await generateHydePassage(opts);
+    await generateHydePassage(opts);
+    expect(generateText).toHaveBeenCalledTimes(1);
+  });
+
+  it("returns null on failure without caching the failure", async () => {
+    (generateText as jest.Mock)
+      .mockRejectedValueOnce(new Error("x"))
+      .mockResolvedValueOnce({ text: "ok" });
+    const opts = { originalQuestion: "q2", relevantKeywords: [], styleHint: "", model: {} };
+    expect(await generateHydePassage(opts)).toBeNull();
+    expect(await generateHydePassage(opts)).toBe("ok");
+  });
+
+  it("returns null when no model is provided", async () => {
+    expect(
+      await generateHydePassage({
+        originalQuestion: "q",
+        relevantKeywords: [],
+        styleHint: "",
+        model: undefined,
+      })
+    ).toBeNull();
+  });
+});

package/ee/agentic-retrieval/pipeline/hyde.ts

@@ -0,0 +1,133 @@
+import { generateText } from "ai";
+import { extractIdentifierTokens } from "./text-utils";
+
+// Per-question memoization of the HyDE passage. A single search invokes
+// generateHydePassage several times with the same question, so without this we would pay
+// one LLM call per invocation. The PROMISE is cached (not just the resolved value) so
+// concurrent invocations within one request share a single generation. A failed/empty
+// result is evicted so a transient blip does not disable HyDE for that question.
+// FIFO-capped to bound memory.
+const hydeCache = new Map<string, Promise<string | null>>();
+const HYDE_CACHE_MAX = 200;
+
+function hydeCacheKey(
+  originalQuestion: string,
+  relevantKeywords: string[],
+  styleHint: string,
+  importantKeyword?: string,
+): string {
+  return JSON.stringify([originalQuestion, importantKeyword ?? "", relevantKeywords, styleHint]);
+}
+
+/**
+ * HyDE (Hypothetical Document Embeddings): generate a short passage that answers the
+ * question in the style and vocabulary of the knowledge base. Embedding this passage
+ * bridges the gap between the user's wording and the document's wording, which is what
+ * lets the relevant chunk enter the candidate pool at all.
+ *
+ * Returns null on any failure (incl. no model) so the caller degrades gracefully.
+ * Memoized per question (see hydeCache).
+ */
+export function generateHydePassage({
+  originalQuestion,
+  relevantKeywords,
+  importantKeyword,
+  styleHint,
+  model,
+}: {
+  originalQuestion: string;
+  relevantKeywords: string[];
+  importantKeyword?: string;
+  styleHint: string;
+  model: any;
+}): Promise<string | null> {
+  if (!model) return Promise.resolve(null);
+
+  const key = hydeCacheKey(originalQuestion, relevantKeywords, styleHint, importantKeyword);
+  const cached = hydeCache.get(key);
+  if (cached) return cached;
+
+  const passagePromise = generateHydePassageUncached({
+    originalQuestion,
+    relevantKeywords,
+    importantKeyword,
+    styleHint,
+    model,
+  })
+    .then((passage) => {
+      // Don't persist failures/empties — allow a retry on the next call.
+      if (passage === null) hydeCache.delete(key);
+      return passage;
+    })
+    .catch((e) => {
+      console.warn("[EXULU] HyDE passage generation failed, skipping:", e);
+      hydeCache.delete(key);
+      return null;
+    });
+
+  if (hydeCache.size >= HYDE_CACHE_MAX) {
+    const oldest = hydeCache.keys().next().value;
+    if (oldest !== undefined) hydeCache.delete(oldest);
+  }
+  hydeCache.set(key, passagePromise);
+  return passagePromise;
+}
+
+export function clearHydeCache(): void {
+  hydeCache.clear();
+}
+
+async function generateHydePassageUncached({
+  originalQuestion,
+  relevantKeywords,
+  importantKeyword,
+  styleHint,
+  model,
+}: {
+  originalQuestion: string;
+  relevantKeywords: string[];
+  importantKeyword?: string;
+  styleHint: string;
+  model: any;
+}): Promise<string | null> {
+  const modelHint =
+    importantKeyword ||
+    extractIdentifierTokens([importantKeyword, ...relevantKeywords, originalQuestion])[0] ||
+    "";
+
+  let prompt = `You are a technical writer producing content in the style of this organization's knowledge base.${
+    styleHint ? `\nThe documents look like this: ${styleHint}` : ""
+  }
+Write a SHORT hypothetical passage (3-6 sentences) that answers the question below the way the
+original document would state it — using the domain's typical terminology (menu paths, parameters,
+codes, section names). Write in the same language as the question.
+`;
+
+  if (modelHint) {
+    prompt += `
+IMPORTANT:
+- Refer exactly to the mentioned product/model: "${modelHint}". Mention this exact designation and NO other variant.
+- Invent plausible, domain-appropriate terms and structure; factual accuracy is not required — the text is only a search anchor.
+- Output ONLY the passage, no preamble, no markdown.
+`;
+  } else {
+    prompt += `
+IMPORTANT:
+- Invent plausible, domain-appropriate terms and structure; factual accuracy is not required — the text is only a search anchor.
+- Output ONLY the passage, no preamble, no markdown.
+`;
+  }
+
+  prompt += `
+Question: "${originalQuestion}"
+Relevant keywords: ${relevantKeywords.join(", ")}`;
+
+  const { text } = await generateText({
+    model,
+    prompt,
+    temperature: 0.3,
+    maxOutputTokens: 500,
+  });
+  const passage = (text || "").trim();
+  return passage.length > 0 ? passage : null;
+}

package/ee/agentic-retrieval/pipeline/index.test.ts

@@ -0,0 +1,140 @@
+// ee/agentic-retrieval/pipeline/index.test.ts
+import { createAgenticRetrievalTool, parsePreselectedItems } from "./index";
+
+jest.mock("@EE/entitlements", () => ({ checkLicense: () => ({ "agentic-retrieval": true }) }));
+jest.mock("@SRC/exulu/resolve-reranker", () => ({ resolveReranker: jest.fn(async () => ({ model: "m", rerank: async (_q: any, c: any) => c })) }));
+jest.mock("@SRC/exulu/resolve-model", () => ({ resolveModel: jest.fn() }));
+jest.mock("@SRC/exulu/app/singleton", () => ({ exuluApp: { get: () => ({ providers: [] }) } }));
+jest.mock("./routing", () => ({ runRoutingPhase: jest.fn(async () => ({
+  mainContexts: ["docs"], fallbackContexts: [], userPinnedItemIdsByContext: new Map(),
+  userRequestedPage: null, hasExplicitDocAndPage: false, steps: [{ text: "routed" }] })) }));
+jest.mock("./memory", () => ({ runMemoryPhase: jest.fn(async () => ({
+  memoryChunksForAnswer: [], memoryOverride: { active: false, chunks: [], reason: "" },
+  memoryPinnedItemIds: new Set(), updatedQuestion: "q", updatedKeywords: ["k"],
+  updatedImportantKeyword: "k", steps: [] })) }));
+jest.mock("./prefilter", () => ({ resolveIdentifierPins: jest.fn(async () => ({
+  pinsByContext: new Map(), exactPinsByContext: new Map(), steps: [] })) }));
+jest.mock("./search", () => ({ searchContexts: jest.fn(async () => ({ chunks: [] })) }));
+jest.mock("./rerank", () => ({ rerankResults: jest.fn(async () => ({
+  limited_results: [], sorted_reranked_results: [], rerank_score_max_genuine: 1 })) }));
+
+const drain = async (gen: AsyncGenerator<any>) => {
+  const out: any[] = [];
+  for await (const v of gen) out.push(v);
+  return out;
+};
+const ctx = (id: string) => ({ id, name: id, description: "", configuration: {} }) as any;
+const makeTool = (config?: Record<string, unknown>, extra: any = {}) => {
+  const tool = createAgenticRetrievalTool({ contexts: [ctx("docs"), ctx("tickets")], model: {} as any, ...extra })!;
+  const exec = (tool.tool as any).execute as (i: any, o?: any) => AsyncGenerator<any>;
+  return (inputs: any) => exec({ toolVariablesConfig: config ?? {}, ...inputs });
+};
+const inputs = { userQuery: "q", relevantKeywords: ["k"], importantKeyword: "k" };
+
+describe("createAgenticRetrievalTool", () => {
+  it("declares the static config surface (no per-context keys)", () => {
+    const tool = createAgenticRetrievalTool({ contexts: [ctx("docs")], model: {} as any })!;
+    const names = tool.config.map((c) => c.name).sort();
+    expect(names).toEqual([
+      "instructions", "knowledge_bases", "logging", "managed_context", "memory",
+      "max_steps", "require_preselected_contexts", "reranker", "routing", "tuning", "utility_model", "vocabulary",
+    ].sort());
+    expect(tool.config.filter((c) => c.type === "json").map((c) => c.name).sort())
+      .toEqual(["knowledge_bases", "memory", "routing", "tuning", "vocabulary"].sort());
+    expect(tool.id).toBe("agentic_context_search");
+  });
+
+  it("short-circuits managed_context without preselected items", async () => {
+    const run = makeTool({ managed_context: true });
+    const out = await drain(run(inputs));
+    expect(out[out.length - 1].result).toContain("preselect");
+  });
+
+  it("yields a message (not a throw) when requested KBs fall outside the preselection", async () => {
+    const { runRoutingPhase } = jest.requireMock("./routing");
+    runRoutingPhase.mockResolvedValueOnce({
+      mainContexts: ["tickets"], fallbackContexts: [], userPinnedItemIdsByContext: new Map(),
+      userRequestedPage: null, hasExplicitDocAndPage: false, steps: [] });
+    const run = makeTool({}, { preselected: ["docs/item1"] });
+    const out = await drain(run(inputs));
+    expect(out[out.length - 1].result).toContain("not part of the preselected");
+  });
+
+  it("streams cumulative AgenticRetrievalOutput snapshots and runs the full pipeline", async () => {
+    const run = makeTool({});
+    const out = await drain(run(inputs));
+    expect(out.length).toBeGreaterThanOrEqual(2);
+    const last = JSON.parse(out[out.length - 1].result);
+    expect(last).toMatchObject({ steps: expect.any(Array), reasoning: expect.any(Array), chunks: [] });
+    expect(last.steps.map((s: any) => s.text)).toContain("routed");
+  });
+
+  it("accumulates top-level chunks across memory, main, and fallback evidence", async () => {
+    const { runMemoryPhase } = jest.requireMock("./memory");
+    const { rerankResults } = jest.requireMock("./rerank");
+    runMemoryPhase.mockResolvedValueOnce({
+      memoryChunksForAnswer: [{ chunk_id: "m1" }],
+      memoryOverride: { active: false, chunks: [], reason: "" },
+      memoryPinnedItemIds: new Set(),
+      updatedQuestion: "q",
+      updatedKeywords: ["k"],
+      updatedImportantKeyword: "k",
+      steps: [],
+    });
+    rerankResults.mockResolvedValueOnce({
+      limited_results: [{ chunk_id: "r1" }, { chunk_id: "m1" }],
+      sorted_reranked_results: [],
+      rerank_score_max_genuine: 1,
+    });
+    const run = makeTool({});
+    const out = await drain(run(inputs));
+    const last = JSON.parse(out[out.length - 1].result);
+    const ids = last.chunks.map((c: any) => c.chunk_id);
+    expect(ids).toContain("m1");
+    expect(ids).toContain("r1");
+    // dedup: "m1" appears in both memory and rerank results but must appear only once
+    expect(ids.filter((id: string) => id === "m1").length).toBe(1);
+  });
+
+  it("filters contexts by knowledge_bases enabled=false", async () => {
+    const { searchContexts } = jest.requireMock("./search");
+    const { runRoutingPhase } = jest.requireMock("./routing");
+    const run = makeTool({ knowledge_bases: { tickets: { enabled: false } } });
+    await drain(run(inputs));
+    const routingCall = runRoutingPhase.mock.calls[runRoutingPhase.mock.calls.length - 1][0];
+    expect(routingCall.enabledContexts.map((c: any) => c.id)).toEqual(["docs"]);
+    expect(searchContexts).toHaveBeenCalled();
+  });
+});
+
+describe("payload deduplication", () => {
+  it("strips chunk_content from step chunks in the serialized payload; top-level keeps it", async () => {
+    const { runMemoryPhase } = jest.requireMock("./memory");
+    const memChunk = { chunk_id: "m1", chunk_content: "FULL MEMORY CONTENT", item_id: "i1", item_name: "Mem" };
+    runMemoryPhase.mockResolvedValueOnce({
+      memoryChunksForAnswer: [memChunk],
+      memoryOverride: { active: false, chunks: [], reason: "" },
+      memoryPinnedItemIds: new Set(), updatedQuestion: "q", updatedKeywords: ["k"],
+      updatedImportantKeyword: "k",
+      steps: [{ text: "memory step", chunks: [memChunk] }],
+    });
+    const run = makeTool({});
+    const out = await drain(run(inputs));
+    const last = JSON.parse(out[out.length - 1].result);
+    const stepWithChunks = last.steps.find((s: any) => s.chunks?.length > 0);
+    expect(stepWithChunks).toBeDefined();
+    expect(stepWithChunks.chunks[0].chunk_content).toBeUndefined();
+    expect(stepWithChunks.chunks[0].item_name).toBe("Mem");
+    const topLevel = last.chunks.find((c: any) => c.chunk_id === "m1");
+    expect(topLevel).toBeDefined();
+    expect(topLevel.chunk_content).toBe("FULL MEMORY CONTENT");
+  });
+});
+
+describe("parsePreselectedItems", () => {
+  it("parses ctx/item pairs and whole-context entries (null wins)", () => {
+    const m = parsePreselectedItems(["a/1", "a/2", "b", "b/3"]);
+    expect(m.get("a")).toEqual(["1", "2"]);
+    expect(m.get("b")).toBeNull();
+  });
+});