@exulu/backend 1.70.0 → 2.0.1

package/ee/agentic-retrieval/pipeline/rerank.test.ts

@@ -0,0 +1,128 @@
+import { splitChunksIntoGroups, rerankResults } from "./rerank";
+
+const chunk = (id: string, itemId: string, index: number, extra: any = {}) => ({
+  chunk_id: id, item_id: itemId, item_name: "item " + itemId, chunk_index: index,
+  chunk_content: "c" + id, chunk_hybrid_score: 1, ...extra,
+}) as any;
+const state = (over: Partial<any> = {}) => ({
+  pinnedItemIds: new Set<string>(), userPinnedItemIds: new Set<string>(),
+  userRequestedPage: null, keywords: [], importantKeyword: "", ...over,
+});
+const tuning = { topK: 5, pinBoost: 0.15, identifierBoost: 0.15, pageWindow: 1 };
+
+describe("splitChunksIntoGroups", () => {
+  it("returns empty for empty input", () => {
+    expect(splitChunksIntoGroups([])).toEqual([]);
+  });
+
+  it("splits at index gaps and at 10 chunks", () => {
+    const run = Array.from({ length: 12 }, (_, i) => chunk("a" + i, "A", i + 1));
+    expect(splitChunksIntoGroups(run).map((g) => g.length)).toEqual([10, 2]);
+    const gap = [chunk("b1", "B", 1), chunk("b2", "B", 5)];
+    expect(splitChunksIntoGroups(gap)).toHaveLength(2);
+  });
+
+  it("groups consecutive chunks together", () => {
+    const chunks = [chunk("x1", "X", 0), chunk("x2", "X", 1), chunk("x3", "X", 2)];
+    const groups = splitChunksIntoGroups(chunks);
+    expect(groups).toHaveLength(1);
+    expect(groups[0]).toHaveLength(3);
+  });
+});
+
+describe("rerankResults", () => {
+  it("reranks, applies pin boost, limits to topK, reports genuine max before boosts", async () => {
+    const chunks = Array.from({ length: 8 }, (_, i) => chunk("c" + i, "I" + i, 1));
+    const reranker = { model: "m", rerank: jest.fn(async (_q: string, items: any[]) =>
+      items.map((it, i) => ({ ...it, rerank_score: 0.8 - i * 0.05 }))) } as any;
+    const r = await rerankResults({ chunks, query: "q", state: state({ pinnedItemIds: new Set(["I7"]) }), reranker, tuning });
+    expect(r.rerank_score_max_genuine).toBeCloseTo(0.8);
+    expect(r.limited_results.some((c: any) => c.item_id === "I7")).toBe(true);
+    expect(r.limited_results.length).toBe(6);
+    const pinned = r.sorted_reranked_results.find((c: any) => c.item_id === "I7");
+    expect(pinned!.rerank_score).toBeCloseTo(0.8 - 7 * 0.05 + 0.15);
+  });
+
+  it("caps force-included pinned groups at topK (token-bomb regression)", async () => {
+    // 30 pinned items — an unbounded force-include would return 30+ groups.
+    const chunks = Array.from({ length: 30 }, (_, i) => chunk("c" + i, "P" + i, 1));
+    const reranker = { model: "m", rerank: jest.fn(async (_q: string, items: any[]) =>
+      items.map((it, i) => ({ ...it, rerank_score: 0.9 - i * 0.01 }))) } as any;
+    const r = await rerankResults({
+      chunks, query: "q",
+      state: state({ pinnedItemIds: new Set(Array.from({ length: 30 }, (_, i) => "P" + i)) }),
+      reranker, tuning,
+    });
+    expect(r.limited_results.length).toBeLessThanOrEqual(2 * tuning.topK);
+    // The best-scoring pinned groups survive the cap
+    expect(r.limited_results.some((c: any) => c.item_id === "P0")).toBe(true);
+  });
+
+  it("preserves the source-KB context tag through chunk grouping", async () => {
+    const chunks = [
+      { ...chunk("c1", "A", 1), context: { id: "docs", name: "Tech Doc" } },
+      { ...chunk("c2", "A", 2), context: { id: "docs", name: "Tech Doc" } },
+    ];
+    const reranker = { model: "m", rerank: jest.fn(async (_q: string, items: any[]) =>
+      items.map((it) => ({ ...it, rerank_score: 0.5 }))) } as any;
+    const r = await rerankResults({ chunks, query: "q", state: state(), reranker, tuning });
+    expect((r.limited_results[0] as any).context).toEqual({ id: "docs", name: "Tech Doc" });
+  });
+
+  it("boosts items whose name matches an identifier token", async () => {
+    const chunks = [chunk("c1", "A", 1), chunk("c2", "B", 1)];
+    chunks[1].item_name = "hb_FST-2XT_manual";
+    const reranker = { model: "m", rerank: jest.fn(async (_q: string, items: any[]) =>
+      items.map((it) => ({ ...it, rerank_score: 0.5 }))) } as any;
+    const r = await rerankResults({ chunks, query: "q", state: state({ importantKeyword: "FST-2XT" }), reranker, tuning });
+    expect(r.sorted_reranked_results[0].item_name).toBe("hb_FST-2XT_manual");
+  });
+
+  it("filters to the requested page window when a page was asked for", async () => {
+    const chunks = [
+      chunk("c1", "A", 1, { chunk_metadata: { page: 12 } }),
+      chunk("c2", "A", 2, { chunk_metadata: { page: 40 } }),
+    ];
+    const reranker = { model: "m", rerank: jest.fn(async (_q: string, items: any[]) =>
+      items.map((it) => ({ ...it, rerank_score: 0.5 }))) } as any;
+    const r = await rerankResults({ chunks, query: "q", state: state({ userRequestedPage: 12 }), reranker, tuning });
+    expect(r.limited_results).toHaveLength(1);
+    expect((r.limited_results[0] as any).chunk_metadata.page).toBe(12);
+  });
+
+  it("without a reranker falls back to hybrid-score ordering with genuine max 0", async () => {
+    const chunks = [chunk("c1", "A", 1, { chunk_hybrid_score: 0.2 }), chunk("c2", "B", 1, { chunk_hybrid_score: 0.9 })];
+    const r = await rerankResults({ chunks, query: "q", state: state(), reranker: undefined, tuning });
+    expect(r.sorted_reranked_results[0].chunk_id).toBe("c2");
+    expect(r.rerank_score_max_genuine).toBe(0);
+  });
+
+  it("treats an empty rerank response for non-empty input as the fallback ordering", async () => {
+    const chunks = [chunk("c1", "A", 1)];
+    const reranker = { model: "m", rerank: jest.fn(async () => []) } as any;
+    const r = await rerankResults({ chunks, query: "q", state: state(), reranker, tuning });
+    expect(r.sorted_reranked_results).toHaveLength(1);
+    expect(r.rerank_score_max_genuine).toBe(0);
+    expect((r.sorted_reranked_results[0] as any).rerank_score).toBeUndefined();
+  });
+
+  it("returns empty result for empty chunk input", async () => {
+    const r = await rerankResults({ chunks: [], query: "q", state: state(), reranker: undefined, tuning });
+    expect(r.limited_results).toHaveLength(0);
+    expect(r.sorted_reranked_results).toHaveLength(0);
+    expect(r.rerank_score_max_genuine).toBe(0);
+  });
+
+  it("merges consecutive chunks from the same item before reranking", async () => {
+    const chunks = [
+      chunk("c1", "A", 0, { chunk_content: "hello" }),
+      chunk("c2", "A", 1, { chunk_content: "world" }),
+    ];
+    const reranker = { model: "m", rerank: jest.fn(async (_q: string, items: any[]) =>
+      items.map((it) => ({ ...it, rerank_score: 0.5 }))) } as any;
+    await rerankResults({ chunks, query: "q", state: state(), reranker, tuning });
+    const passedItems: any[] = reranker.rerank.mock.calls[0][1];
+    expect(passedItems).toHaveLength(1);
+    expect(passedItems[0].chunk_content).toBe("hello\nworld");
+  });
+});

package/ee/agentic-retrieval/pipeline/rerank.ts

@@ -0,0 +1,178 @@
+import type { ResolvedReranker } from "@SRC/exulu/resolve-reranker";
+import { extractIdentifierTokens, itemMatchesIdentifierToken } from "./text-utils";
+import { CHUNK_GROUP_MAX } from "./config";
+import type { Chunk, ChunkWithScore, RerankState, RerankResult } from "./types";
+
+export function splitChunksIntoGroups(chunks: Chunk[]): Chunk[][] {
+  if (chunks.length === 0) return [];
+  const groups: Chunk[][] = [];
+  let currentGroup: Chunk[] = [chunks[0]!];
+  for (let i = 1; i < chunks.length; i++) {
+    const prev = chunks[i - 1]!;
+    const curr = chunks[i]!;
+    const hasGap = (curr.chunk_index || 0) - (prev.chunk_index || 0) > 1;
+    if (currentGroup.length >= CHUNK_GROUP_MAX || hasGap) {
+      groups.push(currentGroup);
+      currentGroup = [curr];
+    } else {
+      currentGroup.push(curr);
+    }
+  }
+  if (currentGroup.length > 0) groups.push(currentGroup);
+  return groups;
+}
+
+function buildRerankObjects(chunks: Chunk[]): Chunk[] {
+  const itemsMap = new Map<string, Chunk[]>();
+  for (const chunk of chunks) {
+    if (!itemsMap.has(chunk.item_id)) itemsMap.set(chunk.item_id, []);
+    itemsMap.get(chunk.item_id)!.push(chunk);
+  }
+  for (const items of itemsMap.values()) {
+    items.sort((a, b) => (a.chunk_index || 0) - (b.chunk_index || 0));
+  }
+  const objects: Chunk[] = [];
+  for (const items of itemsMap.values()) {
+    if (!items[0]) continue;
+    for (const group of splitChunksIntoGroups(items)) {
+      if (!group[0]) continue;
+      objects.push({
+        chunk_content: group.map(c => c.chunk_content).join('\n'),
+        // Preserve the source-KB tag attached by searchContexts (citation attribution)
+        context: (group[0] as any).context,
+        chunk_index: group[0].chunk_index,
+        chunk_id: group[0].chunk_id,
+        chunk_source: group[0].chunk_source,
+        chunk_metadata: group[0].chunk_metadata,
+        chunk_created_at: group[0].chunk_created_at,
+        chunk_updated_at: group[0].chunk_updated_at,
+        item_id: group[0].item_id,
+        item_external_id: group[0].item_external_id,
+        item_name: group[0].item_name,
+        item_updated_at: group[0].item_updated_at,
+        item_created_at: group[0].item_created_at,
+      });
+    }
+  }
+  return objects;
+}
+
+export async function rerankResults(opts: {
+  chunks: Chunk[];
+  query: string;
+  state: RerankState;
+  reranker?: ResolvedReranker;
+  tuning: { topK: number; pinBoost: number; identifierBoost: number; pageWindow: number };
+}): Promise<RerankResult> {
+  const { chunks, query, state, reranker, tuning } = opts;
+
+  // Empty input fast path
+  if (chunks.length === 0) {
+    return { limited_results: [], sorted_reranked_results: [], rerank_score_max_genuine: 0 };
+  }
+
+  const rerankObjs = buildRerankObjects(chunks);
+
+  let sorted: ChunkWithScore[] = [];
+  let rerank_score_max_genuine = 0;
+  let useFallback = false;
+
+  if (reranker !== undefined) {
+    const reranked = await reranker.rerank(query, rerankObjs);
+    if (reranked.length === 0 && rerankObjs.length > 0) {
+      // reranker internal error path: non-empty input → empty output
+      console.warn("[EXULU pipeline] reranker returned [] for non-empty input — falling back to hybrid-score ordering");
+      useFallback = true;
+    } else {
+      sorted = (reranked as ChunkWithScore[]).sort((a, b) => (b.rerank_score || 0) - (a.rerank_score || 0));
+      rerank_score_max_genuine = sorted[0]?.rerank_score ?? 0;
+    }
+  } else {
+    useFallback = true;
+  }
+
+  if (useFallback) {
+    // Sort groups by max chunk_hybrid_score from original chunks (per item)
+    const maxHybridByItem = new Map<string, number>();
+    for (const c of chunks) {
+      const cur = maxHybridByItem.get(c.item_id) ?? 0;
+      maxHybridByItem.set(c.item_id, Math.max(cur, c.chunk_hybrid_score ?? 0));
+    }
+    sorted = [...rerankObjs].sort((a, b) =>
+      (maxHybridByItem.get(b.item_id) ?? 0) - (maxHybridByItem.get(a.item_id) ?? 0)
+    ) as ChunkWithScore[];
+    // DO NOT set rerank_score; rerank_score_max_genuine stays 0
+  }
+
+  // Pin boost — no-op in fallback (rerank_score is undefined on fallback chunks)
+  if (state.pinnedItemIds.size > 0) {
+    for (const r of sorted) {
+      if (r.item_id && state.pinnedItemIds.has(r.item_id) && r.rerank_score !== undefined) {
+        r.rerank_score = Math.min(1, r.rerank_score + tuning.pinBoost);
+      }
+    }
+    if (!useFallback) {
+      sorted.sort((a, b) => (b.rerank_score || 0) - (a.rerank_score || 0));
+    }
+  }
+
+  // Identify pinned groups for force-inclusion in limited_results. Force-inclusion is
+  // capped at topK groups (taken in current score order, i.e. the best-scoring pins):
+  // an unbounded cap made the output size proportional to the pin count — with a broad
+  // pin set the tool returned 30+ concatenated chunk-groups per call (hundreds of
+  // thousands of tokens), overflowing the calling agent's context window. Worst case
+  // is now ~2×topK groups regardless of how many items are pinned.
+  const pinnedGroups: ChunkWithScore[] = [];
+  if (state.pinnedItemIds.size > 0) {
+    const seen = new Set<string>();
+    for (const r of sorted) {
+      if (pinnedGroups.length >= tuning.topK) break;
+      if (r.item_id && state.pinnedItemIds.has(r.item_id) && !seen.has(r.item_id)) {
+        seen.add(r.item_id);
+        pinnedGroups.push(r);
+      }
+    }
+  }
+
+  // Identifier boost — no-op in fallback (rerank_score is undefined)
+  const identifierTokens = extractIdentifierTokens([state.importantKeyword, ...state.keywords, query]);
+  if (identifierTokens.length > 0) {
+    let boosted = 0;
+    for (const r of sorted) {
+      if (itemMatchesIdentifierToken(r.item_name, identifierTokens) && r.rerank_score !== undefined) {
+        r.rerank_score = Math.min(1, r.rerank_score + tuning.identifierBoost);
+        boosted++;
+      }
+    }
+    if (boosted > 0 && !useFallback) {
+      sorted.sort((a, b) => (b.rerank_score || 0) - (a.rerank_score || 0));
+    }
+  }
+
+  // Top-K selection with pinned-item force-inclusion
+  const { topK } = tuning;
+  let limited_results: ChunkWithScore[] = (() => {
+    const top = sorted.slice(0, topK);
+    if (pinnedGroups.length === 0) return top;
+    const byId = new Map<string, ChunkWithScore>();
+    for (const r of [...pinnedGroups, ...top]) {
+      if (r.chunk_id && !byId.has(r.chunk_id)) byId.set(r.chunk_id, r);
+    }
+    return Array.from(byId.values())
+      .sort((a, b) => (b.rerank_score || 0) - (a.rerank_score || 0))
+      .slice(0, Math.max(topK, pinnedGroups.length + topK));
+  })();
+
+  // Page filter — applies to all paths; scoped to userPinnedItemIds when non-empty
+  if (state.userRequestedPage !== null) {
+    const scopeToUserPinned = state.userPinnedItemIds.size > 0;
+    const pageMatched = sorted.filter(r => {
+      if (scopeToUserPinned && !(r.item_id && state.userPinnedItemIds.has(r.item_id))) return false;
+      const p = (r.chunk_metadata as { page?: unknown } | undefined)?.page;
+      return typeof p === 'number' && Math.abs(p - state.userRequestedPage!) <= tuning.pageWindow;
+    });
+    if (pageMatched.length > 0) limited_results = pageMatched.slice(0, topK);
+  }
+
+  return { limited_results, sorted_reranked_results: sorted, rerank_score_max_genuine };
+}

package/ee/agentic-retrieval/pipeline/routing.test.ts

@@ -0,0 +1,144 @@
+// ee/agentic-retrieval/pipeline/routing.test.ts
+import { runRoutingPhase } from "./routing";
+
+jest.mock("ai", () => ({ generateText: jest.fn(), Output: { object: (x: any) => x } }));
+jest.mock("./prefilter", () => ({ fuzzyPrefilter: jest.fn(async () => []) }));
+import { generateText } from "ai";
+import { fuzzyPrefilter } from "./prefilter";
+
+const enabled = [
+  { id: "docs", name: "Docs", description: "manuals" },
+  { id: "tickets", name: "Tickets", description: "support" },
+];
+const noHints = { output: { hasFilenameHint: false, filenameHints: [], hasPageHint: false, pageNumber: null } };
+const noExplicit = { output: { explicitlyRequestedKnowledgeBases: [] } };
+
+beforeEach(() => { (generateText as jest.Mock).mockReset(); (fuzzyPrefilter as jest.Mock).mockClear(); });
+
+describe("runRoutingPhase", () => {
+  it("explicit KB request wins and yields no fallback", async () => {
+    (generateText as jest.Mock)
+      .mockResolvedValueOnce(noHints)
+      .mockResolvedValueOnce({ output: { explicitlyRequestedKnowledgeBases: ["tickets"] } });
+    const r = await runRoutingPhase({
+      question: "search tickets for X", enabledContexts: enabled, documentContexts: [],
+      routingRules: [{ id: "t", label: "T", description: "d", main: ["docs"], fallback: ["tickets"] }],
+      preselectedItems: new Map(), model: {},
+    });
+    expect(r.mainContexts).toEqual(["tickets"]);
+    expect(r.fallbackContexts).toEqual([]);
+    expect(generateText).toHaveBeenCalledTimes(2); // no classification call
+  });
+
+  it("classifies against configured rules when nothing is explicit", async () => {
+    (generateText as jest.Mock)
+      .mockResolvedValueOnce(noHints)
+      .mockResolvedValueOnce(noExplicit)
+      .mockResolvedValueOnce({ output: { ruleId: "t", reason: "because" } });
+    const r = await runRoutingPhase({
+      question: "how do I fix the door?", enabledContexts: enabled, documentContexts: [],
+      routingRules: [{ id: "t", label: "T", description: "d", main: ["docs"], fallback: ["tickets"] }],
+      preselectedItems: new Map(), model: {},
+    });
+    expect(r.mainContexts).toEqual(["docs"]);
+    expect(r.fallbackContexts).toEqual(["tickets"]);
+  });
+
+  it("with no rules everything enabled becomes main", async () => {
+    (generateText as jest.Mock).mockResolvedValueOnce(noHints).mockResolvedValueOnce(noExplicit);
+    const r = await runRoutingPhase({
+      question: "q", enabledContexts: enabled, documentContexts: [], routingRules: [],
+      preselectedItems: new Map(), model: {},
+    });
+    expect(r.mainContexts).toEqual(["docs", "tickets"]);
+    expect(r.fallbackContexts).toEqual([]);
+    expect(generateText).toHaveBeenCalledTimes(2);
+  });
+
+  it("discards doc-reference pins when the hint matches too many files (token-bomb regression)", async () => {
+    (generateText as jest.Mock)
+      .mockResolvedValueOnce({ output: { hasFilenameHint: true, filenameHints: ["FST-2XT"], hasPageHint: false, pageNumber: null } })
+      .mockResolvedValueOnce(noExplicit);
+    (fuzzyPrefilter as jest.Mock).mockResolvedValue(
+      Array.from({ length: 30 }, (_, i) => ({ id: "f" + i, name: "file" + i + ".pdf", key: "k" + i })),
+    );
+    const r = await runRoutingPhase({
+      question: "Was bedeutet der Fehler S2 CMP Input beim FST-2XT?",
+      enabledContexts: enabled, documentContexts: [{ id: "docs" }],
+      routingRules: [], preselectedItems: new Map(), model: {},
+    });
+    expect(r.userPinnedItemIdsByContext.size).toBe(0);
+    expect(r.hasExplicitDocAndPage).toBe(false);
+    expect(r.steps.some((s) => s.text.includes("too broad"))).toBe(true);
+  });
+
+  it("doc-reference prompt carries the file-marker rule and configured identifier examples", async () => {
+    (generateText as jest.Mock).mockResolvedValueOnce(noHints).mockResolvedValueOnce(noExplicit);
+    await runRoutingPhase({
+      question: "q", enabledContexts: enabled, documentContexts: [], routingRules: [],
+      preselectedItems: new Map(), knownIdentifiers: ["FST", "ECO", "CBM-2"], model: {},
+    });
+    const docCall = (generateText as jest.Mock).mock.calls.find(([args]) =>
+      String(args?.system ?? "").includes("filename hint"),
+    );
+    expect(docCall).toBeDefined();
+    expect(docCall![0].system).toContain("STRICT RULE");
+    expect(docCall![0].system).toContain("FST, ECO, CBM-2");
+    expect(docCall![0].system).toContain("NOT be treated as filenames");
+  });
+
+  it("explicit-KB prompt carries the topical-false-positive guard (newlkiag gate regression)", async () => {
+    (generateText as jest.Mock).mockResolvedValueOnce(noHints).mockResolvedValueOnce(noExplicit);
+    await runRoutingPhase({
+      question: "Welche Software-Änderungen gab es zuletzt?", enabledContexts: enabled,
+      documentContexts: [], routingRules: [], preselectedItems: new Map(), model: {},
+    });
+    const kbCall = (generateText as jest.Mock).mock.calls.find(([args]) =>
+      String(args?.system ?? "").includes("EXPLICITLY asked"),
+    );
+    expect(kbCall).toBeDefined();
+    expect(kbCall![0].system).toContain("is NOT an explicit request");
+    expect(kbCall![0].system).toContain("When in doubt, return");
+  });
+
+  it("resolves filename hints against document contexts and reports the page", async () => {
+    (generateText as jest.Mock)
+      .mockResolvedValueOnce({ output: { hasFilenameHint: true, filenameHints: ["manual X3"], hasPageHint: true, pageNumber: 12 } })
+      .mockResolvedValueOnce(noExplicit);
+    (fuzzyPrefilter as jest.Mock).mockResolvedValue([{ id: "42", name: "Manual X3", key: "k" }]);
+    const docCtx = { id: "docs" };
+    const r = await runRoutingPhase({
+      question: "in manual X3 on page 12", enabledContexts: enabled, documentContexts: [docCtx],
+      routingRules: [], preselectedItems: new Map(), model: {},
+    });
+    expect([...r.userPinnedItemIdsByContext.get("docs")!]).toEqual(["42"]);
+    expect(r.userRequestedPage).toBe(12);
+    expect(r.hasExplicitDocAndPage).toBe(true);
+  });
+
+  it("degrades to no pins when document-reference resolution fails", async () => {
+    (generateText as jest.Mock)
+      .mockResolvedValueOnce({ output: { hasFilenameHint: true, filenameHints: ["manual X3"], hasPageHint: false, pageNumber: null } })
+      .mockResolvedValueOnce(noExplicit);
+    (fuzzyPrefilter as jest.Mock).mockRejectedValueOnce(new Error("getItems failed"));
+    const r = await runRoutingPhase({
+      question: "in manual X3", enabledContexts: enabled, documentContexts: [{ id: "docs" }],
+      routingRules: [], preselectedItems: new Map(), model: {},
+    });
+    expect(r.userPinnedItemIdsByContext.size).toBe(0);
+    expect(r.steps.some((s) => s.text.includes("continuing without file pins"))).toBe(true);
+  });
+
+  it("degrades to all-main when the classifier throws", async () => {
+    (generateText as jest.Mock)
+      .mockResolvedValueOnce(noHints)
+      .mockResolvedValueOnce(noExplicit)
+      .mockRejectedValue(new Error("down")); // withRetry exhausts on classification
+    const r = await runRoutingPhase({
+      question: "q", enabledContexts: enabled, documentContexts: [],
+      routingRules: [{ id: "t", label: "T", description: "d", main: ["docs"], fallback: [] }],
+      preselectedItems: new Map(), model: {},
+    });
+    expect(r.mainContexts).toEqual(["docs", "tickets"]);
+  }, 30000); // withRetry backs off 2s+4s before exhausting
+});