@exulu/backend 1.70.0 → 2.0.0

package/ee/agentic-retrieval/pipeline/text-utils.test.ts

@@ -0,0 +1,43 @@
+import {
+  normalizeFileName, deriveKeywordVariants, extractIdentifierTokens,
+  itemMatchesIdentifierToken, applyRewrites, stripSeparators,
+} from "./text-utils";
+
+describe("text-utils", () => {
+  it("normalizeFileName strips the bucket segment, separators, and extension", () => {
+    expect(normalizeFileName("/bucket/folder/hb_PAM-E4_2023.de.pdf")).toBe("folder hbpame42023 de pdf");
+  });
+
+  it("normalizeFileName drops the first segment for paths without leading slash", () => {
+    expect(normalizeFileName("bucket/folder/hb_PAM-E4_2023.de.pdf")).toBe("folder hbpame42023 de pdf");
+  });
+
+  it("normalizeFileName preserves bare filenames without dropping segments", () => {
+    expect(normalizeFileName("hb_FST-2XT_manual.pdf")).toBe("hbfst2xtmanual pdf");
+  });
+
+  it("deriveKeywordVariants yields lowercased, separator- and digit-stripped forms ≥4 chars", () => {
+    expect(deriveKeywordVariants("FST-2XT").sort()).toEqual(["fst-2xt", "fst2xt"].sort());
+    expect(deriveKeywordVariants("MISCEL6")).toEqual(expect.arrayContaining(["miscel6", "miscel"]));
+    expect(deriveKeywordVariants("ab")).toEqual([]); // too short
+  });
+
+  it("extractIdentifierTokens keeps ≥4-char tokens containing a digit and a letter", () => {
+    expect(extractIdentifierTokens(["FST-2XT", "sperren", "S2", undefined])).toEqual(["fst2xt"]);
+  });
+
+  it("itemMatchesIdentifierToken matches separator-insensitively against the filename", () => {
+    expect(itemMatchesIdentifierToken("hb_FST-2XT_manual.pdf", ["fst2xt"])).toBe(true);
+    expect(itemMatchesIdentifierToken("hb_FST2_manual.pdf", ["fst2xt"])).toBe(false);
+  });
+
+  it("applyRewrites returns one variant per matching rule, none when nothing matches", () => {
+    const rules = [{ find: "bypass", replace: "override" }, { find: "zzz", replace: "yyy" }];
+    expect(applyRewrites("how to bypass the door", rules)).toEqual(["how to override the door"]);
+    expect(applyRewrites("hello", rules)).toEqual([]);
+  });
+
+  it("stripSeparators lowers and removes separators", () => {
+    expect(stripSeparators("FST-2 XT.a")).toBe("fst2xta");
+  });
+});

package/ee/agentic-retrieval/pipeline/text-utils.ts

@@ -0,0 +1,85 @@
+export const normalizeFileName = (fileName: string): string => {
+  // 1. Remove the first path segment (e.g., "/aufzugsperipherie")
+  const parts = fileName.split('/').filter(p => p); // filter out empty strings
+  let normalized = parts.length > 1 ? parts.slice(1).join('/') : (parts[0] || '');
+
+  // 2. Replace the remaining slashes with a space
+  normalized = normalized.replace(/\//g, ' ');
+
+  // 3. Convert to lowercase
+  normalized = normalized.toLowerCase();
+
+  // 4. PRE-CLEANUP: Insert a space before the file extension period.
+  // This looks for a dot followed by 3-4 letters/digits at the end of the string.
+  // The captured group ($1) puts the extension back with a space before it.
+  normalized = normalized.replace(/(\.[a-z0-9]{3,4})$/g, ' $1');
+
+  // 5. Final cleanup: Remove all characters that are NOT a lowercase letter, digit, space, or a period.
+  // We keep the period here temporarily to ensure the extension is preserved.
+  normalized = normalized.replace(/[^a-z0-9\. ]/g, '');
+
+  // 6. Final step: Replace any space/dot/space pattern with just a space
+  // and trim excess spaces. This handles the space inserted in step 4.
+  // The regex /\s*\.\s*/g finds any combination of space-dot-space and replaces it with a single space.
+  normalized = normalized.replace(/\s*\.\s*/g, ' ');
+
+  // 7. Clean up multiple spaces and leading/trailing spaces
+  normalized = normalized.replace(/ +/g, ' ').trim();
+
+  return normalized;
+};
+
+export const normalizeText = (text: string): string => {
+  return text.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+
+export const deriveKeywordVariants = (keyword: string): string[] => {
+    const lower = keyword.trim().toLowerCase();
+    if (!lower) return [];
+    const variants = new Set<string>();
+    variants.add(lower);
+    variants.add(lower.replace(/[-_\.\s]/g, ''));
+    variants.add(lower.replace(/\d+$/, ''));
+    variants.add(lower.replace(/[-_\.\s]/g, '').replace(/\d+$/, ''));
+    return [...variants].filter(v => v.length >= 4);
+}
+
+export function extractIdentifierTokens(parts: Array<string | undefined>): string[] {
+    const tokens = new Set<string>();
+    for (const part of parts) {
+        if (!part) continue;
+        // Split on whitespace so multi-word strings (the raw question) yield candidates.
+        for (const raw of part.split(/\s+/)) {
+            const norm = raw.toLowerCase().replace(/[^a-z0-9]/g, "");
+            if (norm.length >= 4 && /\d/.test(norm) && /[a-z]/.test(norm)) {
+                tokens.add(norm);
+            }
+        }
+    }
+    return [...tokens];
+}
+
+/** True if the document filename contains any of the identifier tokens (separators ignored). */
+export function itemMatchesIdentifierToken(itemName: string | undefined, tokens: string[]): boolean {
+    if (!itemName || tokens.length === 0) return false;
+    const normName = normalizeFileName(itemName).toLowerCase().replace(/[^a-z0-9]/g, "");
+    return tokens.some(t => normName.includes(t));
+}
+
+export const stripSeparators = (s: string): string => s.toLowerCase().replace(/[-_\.\s]/g, "");
+
+/** Apply configured find→replace rules (case-insensitive, all occurrences).
+ * Returns one rewritten query per rule that changed the input; deduped. */
+export function applyRewrites(
+  question: string,
+  rewrites: { find: string; replace: string }[],
+): string[] {
+  const out = new Set<string>();
+  for (const rule of rewrites) {
+    if (!rule.find) continue;
+    const escaped = rule.find.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const rewritten = question.replace(new RegExp(escaped, "gi"), rule.replace);
+    if (rewritten !== question) out.add(rewritten);
+  }
+  return [...out];
+}

package/ee/agentic-retrieval/pipeline/types.ts

@@ -0,0 +1,59 @@
+import type { VectorSearchChunkResult } from "@SRC/graphql/resolvers/vector-search";
+
+export type Chunk = VectorSearchChunkResult;
+export type ChunkWithScore = Chunk & { rerank_score?: number; context?: { id: string; name: string } };
+
+export type PhaseStep = {
+  text: string;
+  chunks?: ChunkWithScore[];
+  toolCalls?: Array<{ name: string; id: string; input: unknown }>;
+};
+
+export type RoutingPhaseResult = {
+  mainContexts: string[];
+  fallbackContexts: string[];
+  userPinnedItemIdsByContext: Map<string, Set<string>>;
+  userRequestedPage: number | null;
+  hasExplicitDocAndPage: boolean;
+  steps: PhaseStep[];
+};
+
+export type MemoryPhaseResult = {
+  memoryChunksForAnswer: ChunkWithScore[];
+  memoryOverride: { active: boolean; chunks: ChunkWithScore[]; reason: string };
+  memoryPinnedItemIds: Set<string>;
+  updatedQuestion: string;
+  updatedKeywords: string[];
+  updatedImportantKeyword: string;
+  steps: PhaseStep[];
+};
+
+export type SearchContextsResult = { chunks: Chunk[] };
+
+export type RerankState = {
+  pinnedItemIds: Set<string>;
+  userPinnedItemIds: Set<string>;
+  userRequestedPage: number | null;
+  keywords: string[];
+  importantKeyword: string;
+};
+export type RerankResult = {
+  limited_results: ChunkWithScore[];
+  sorted_reranked_results: ChunkWithScore[];
+  rerank_score_max_genuine: number;
+};
+
+export type RetrievalStep = {
+  stepNumber: number;
+  text: string;
+  toolCalls: Array<{ name: string; id: string; input: unknown }>;
+  chunks: ChunkWithScore[];
+  tokens: number;
+};
+export type AgenticRetrievalOutput = {
+  steps: RetrievalStep[];
+  reasoning: { text: string; tools: unknown[] }[];
+  chunks: ChunkWithScore[];
+  usage: unknown[];
+  totalTokens: number;
+};

package/ee/python/documents/processing/doc_processor.ts

@@ -834,7 +834,7 @@ async function processPdf(
 
       const splitResult = await executePythonScript({
         scriptPath: 'ee/python/documents/processing/split_pdf.py',
-        args: [paths.source, chunksDir, '--chunk-size', String(maxPagesPerChunk)],
+        args: [paths.source, chunksDir, '--chunk-size', String(maxPagesPerChunk), '--max-size-mb', '25'],
         timeout: 5 * 60 * 1000,
       });
 

package/ee/python/documents/processing/split_pdf.py

@@ -6,13 +6,13 @@ Outputs a JSON array to stdout, each element:
   { "path": "<absolute-path>", "start_page": <int>, "end_page": <int> }
 
 start_page is 0-indexed, end_page is exclusive (Python-slice convention).
-If the document fits within chunk_size, a single entry pointing to the
-original file is returned (no copy made).
+If the document fits within chunk_size AND within max_size_bytes, a single
+entry pointing to the original file is returned (no copy made).
 
 Progress and diagnostics go to stderr so stdout stays clean JSON.
 
 Usage:
-    split_pdf.py <input_pdf> <output_dir> [--chunk-size N]
+    split_pdf.py <input_pdf> <output_dir> [--chunk-size N] [--max-size-mb F]
 """
 
 import sys
@@ -23,7 +23,54 @@ import argparse
 import fitz  # PyMuPDF — installed as a docling transitive dependency
 
 
-def split_pdf(input_path: str, output_dir: str, chunk_size: int) -> list[dict]:
+def _write_chunk(
+    doc: fitz.Document,
+    output_dir: str,
+    chunk_start: int,
+    chunk_end: int,
+    max_size_bytes: int | None,
+) -> list[dict]:
+    """Write pages [chunk_start, chunk_end) to a file.
+
+    If the result exceeds max_size_bytes and contains more than one page,
+    delete it and recurse with the range bisected. Single-page chunks that
+    still exceed the limit are kept with a warning — they cannot be split
+    further without re-encoding.
+    """
+    chunk_path = os.path.join(output_dir, f"chunk_{chunk_start}_{chunk_end - 1}.pdf")
+
+    sub = fitz.open()
+    sub.insert_pdf(doc, from_page=chunk_start, to_page=chunk_end - 1)
+    sub.save(chunk_path)
+    sub.close()
+
+    chunk_bytes = os.path.getsize(chunk_path)
+    n_pages = chunk_end - chunk_start
+
+    if max_size_bytes and chunk_bytes > max_size_bytes and n_pages > 1:
+        os.remove(chunk_path)
+        mid = chunk_start + n_pages // 2
+        return (
+            _write_chunk(doc, output_dir, chunk_start, mid, max_size_bytes)
+            + _write_chunk(doc, output_dir, mid, chunk_end, max_size_bytes)
+        )
+
+    if max_size_bytes and chunk_bytes > max_size_bytes:
+        print(
+            f"[split_pdf] WARNING: single-page chunk {chunk_start} is {chunk_bytes:,} bytes — "
+            "exceeds size limit but cannot be split further",
+            file=sys.stderr,
+        )
+
+    return [{"path": os.path.abspath(chunk_path), "start_page": chunk_start, "end_page": chunk_end}]
+
+
+def split_pdf(
+    input_path: str,
+    output_dir: str,
+    chunk_size: int,
+    max_size_bytes: int | None = None,
+) -> list[dict]:
     doc = fitz.open(input_path)
 
     # Some PDFs are saved with an empty owner/user password by certain writers
@@ -39,9 +86,17 @@ def split_pdf(input_path: str, output_dir: str, chunk_size: int) -> list[dict]:
         print("[split_pdf] Authenticated with empty password (phantom-password PDF)", file=sys.stderr)
 
     total_pages = len(doc)
-    print(f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}", file=sys.stderr)
+    file_size = os.path.getsize(input_path)
+    print(
+        f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}, "
+        f"file size: {file_size:,} bytes"
+        + (f", max chunk size: {max_size_bytes:,} bytes" if max_size_bytes else ""),
+        file=sys.stderr,
+    )
 
-    if total_pages <= chunk_size:
+    needs_split = total_pages > chunk_size or (max_size_bytes and file_size > max_size_bytes)
+
+    if not needs_split:
         print("[split_pdf] No split needed — returning original path", file=sys.stderr)
         doc.close()
         return [{
@@ -55,23 +110,14 @@ def split_pdf(input_path: str, output_dir: str, chunk_size: int) -> list[dict]:
     chunks = []
     for start_page in range(0, total_pages, chunk_size):
         end_page = min(start_page + chunk_size, total_pages)
-        chunk_filename = f"chunk_{start_page}_{end_page - 1}.pdf"
-        chunk_path = os.path.join(output_dir, chunk_filename)
-
-        chunk_doc = fitz.open()
-        chunk_doc.insert_pdf(doc, from_page=start_page, to_page=end_page - 1)
-        chunk_doc.save(chunk_path)
-        chunk_doc.close()
-
-        chunks.append({
-            "path": os.path.abspath(chunk_path),
-            "start_page": start_page,
-            "end_page": end_page,
-        })
-        print(
-            f"[split_pdf] Chunk {len(chunks)}: pages {start_page}–{end_page - 1} → {chunk_filename}",
-            file=sys.stderr,
-        )
+        sub_chunks = _write_chunk(doc, output_dir, start_page, end_page, max_size_bytes)
+        for c in sub_chunks:
+            print(
+                f"[split_pdf] Chunk {len(chunks) + 1}: pages {c['start_page']}–{c['end_page'] - 1} "
+                f"({os.path.getsize(c['path']):,} bytes) → {os.path.basename(c['path'])}",
+                file=sys.stderr,
+            )
+        chunks.extend(sub_chunks)
 
     doc.close()
     return chunks
@@ -87,10 +133,18 @@ if __name__ == "__main__":
         default=25,
         help="Maximum pages per chunk (default: 25)",
     )
+    parser.add_argument(
+        "--max-size-mb",
+        type=float,
+        default=None,
+        help="Maximum chunk file size in MB — chunks exceeding this are bisected by page count (default: no limit)",
+    )
     args = parser.parse_args()
 
+    max_size_bytes = int(args.max_size_mb * 1024 * 1024) if args.max_size_mb is not None else None
+
     try:
-        chunks = split_pdf(args.input_pdf, args.output_dir, args.chunk_size)
+        chunks = split_pdf(args.input_pdf, args.output_dir, args.chunk_size, max_size_bytes)
         print(json.dumps(chunks))
     except Exception as e:
         print(f"[split_pdf] ERROR: {e}", file=sys.stderr)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.70.0",
+  "version": "2.0.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {
@@ -132,6 +132,7 @@
     "express": "^5.1.0",
     "express-http-proxy": "^2.1.2",
     "franc": "^6.2.0",
+    "fuse.js": "^7.4.2",
     "graphql": "^16.11.0",
     "graphql-tools": "^9.0.18",
     "graphql-type-json": "^0.3.2",