@exulu/backend 1.68.0 → 1.69.0

package/ee/python/documents/processing/doc_processor.ts

@@ -40,6 +40,13 @@ type DocumentProcessorConfig = {
      * vertex_ai) can be switched without code changes.
      */
     model?: string
+    /**
+     * Maximum pages per OCR request for the "mistral" processor.
+     * Vertex AI OCR rejects documents over 30 pages; the PDF is split into
+     * chunks of this size and each chunk is OCR'd independently.
+     * Defaults to 25 (safely under the Vertex AI 30-page limit).
+     */
+    maxPagesPerChunk?: number
   }
   /**
    * Optional cost-attribution context, forwarded to LiteLLM as spend tags
@@ -816,17 +823,61 @@ async function processPdf(
         ...config.attribution,
       });
 
-      // Wait a randomn time between 1 and 5 seconds to prevent rate limiting
-      await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 4000) + 1000));
+      // Split the PDF into ≤ N-page chunks before sending to OCR.
+      // Vertex AI (and some other providers) reject documents over 30 pages.
+      // We use PyMuPDF via a Python helper because it handles edge cases that
+      // trip up JS PDF libraries — in particular "phantom password" PDFs that
+      // are technically encrypted with an empty string (the OS opens them
+      // transparently, but libraries throw without the empty-string fallback).
+      const maxPagesPerChunk = config.processor.maxPagesPerChunk ?? 25;
+      const chunksDir = path.join(path.dirname(paths.json), 'ocr_chunks');
+
+      const splitResult = await executePythonScript({
+        scriptPath: 'ee/python/documents/processing/split_pdf.py',
+        args: [paths.source, chunksDir, '--chunk-size', String(maxPagesPerChunk)],
+        timeout: 5 * 60 * 1000,
+      });
+
+      const pdfChunks: Array<{ path: string; start_page: number; end_page: number }> =
+        JSON.parse(splitResult.stdout);
+
+      console.log(`[EXULU] PDF split into ${pdfChunks.length} chunk(s) for OCR (max ${maxPagesPerChunk} pages each)`);
+
+      // Process chunks in parallel with a concurrency cap to respect rate limits.
+      // Each chunk gets a small random jitter to spread out requests.
+      const chunkLimit = pLimit(3);
 
-      const base64Pdf = buffer.toString('base64');
+      const chunkResults = await Promise.all(
+        pdfChunks.map((chunk, i) =>
+          chunkLimit(async () => {
+            // Yield to the event loop so BullMQ can renew job locks during long runs
+            await new Promise(resolve => setImmediate(resolve));
+            await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 1000) + 200));
 
-      const ocrResponse = await withRetry(async () => {
-        return await resolved.ocr({
-          type: "document_url",
-          document_url: "data:application/pdf;base64," + base64Pdf,
-        }, { includeImageBase64: false });
-      }, 10);
+            console.log(`[EXULU] OCR chunk ${i + 1}/${pdfChunks.length}: pages ${chunk.start_page}–${chunk.end_page - 1}`);
+
+            const chunkBuffer = await fs.promises.readFile(chunk.path);
+            const chunkBase64 = chunkBuffer.toString('base64');
+
+            const chunkResponse = await withRetry(async () => {
+              return await resolved.ocr({
+                type: "document_url",
+                document_url: "data:application/pdf;base64," + chunkBase64,
+              }, { includeImageBase64: false });
+            }, 10);
+
+            return { pages: chunkResponse.pages, offset: chunk.start_page };
+          })
+        )
+      );
+
+      // Merge all chunk pages in document order, offsetting indices to their
+      // original positions in the full document.
+      const mergedPages = chunkResults
+        .sort((a, b) => a.offset - b.offset)
+        .flatMap(({ pages, offset }) =>
+          pages.map(p => ({ ...p, index: p.index + offset }))
+        );
 
       const parser = new LiteParse();
       const screenshots = await parser.screenshot(paths.source, undefined);
@@ -841,7 +892,7 @@ async function processPdf(
         screenshot.imagePath = path.join(paths.images, `${screenshot.pageNum}.png`);
       }
 
-      json = ocrResponse.pages.map(page => ({
+      json = mergedPages.map(page => ({
         page: page.index + 1,
         content: page.markdown,
         image: screenshots.find(s => s.pageNum === page.index + 1)?.imagePath,

package/ee/python/documents/processing/split_pdf.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""
+PDF Splitter — splits a PDF into fixed-size page chunks using PyMuPDF.
+
+Outputs a JSON array to stdout, each element:
+  { "path": "<absolute-path>", "start_page": <int>, "end_page": <int> }
+
+start_page is 0-indexed, end_page is exclusive (Python-slice convention).
+If the document fits within chunk_size, a single entry pointing to the
+original file is returned (no copy made).
+
+Progress and diagnostics go to stderr so stdout stays clean JSON.
+
+Usage:
+    split_pdf.py <input_pdf> <output_dir> [--chunk-size N]
+"""
+
+import sys
+import os
+import json
+import argparse
+
+import fitz  # PyMuPDF — installed as a docling transitive dependency
+
+
+def split_pdf(input_path: str, output_dir: str, chunk_size: int) -> list[dict]:
+    doc = fitz.open(input_path)
+
+    # Some PDFs are saved with an empty owner/user password by certain writers
+    # (e.g. older Adobe Acrobat exports). The OS opens them transparently by
+    # trying "" first, but most libraries raise immediately. We replicate that
+    # OS-level behaviour here.
+    if doc.needs_pass:
+        authenticated = doc.authenticate("")
+        if not authenticated:
+            raise ValueError(
+                "PDF requires a non-empty password and cannot be opened automatically."
+            )
+        print("[split_pdf] Authenticated with empty password (phantom-password PDF)", file=sys.stderr)
+
+    total_pages = len(doc)
+    print(f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}", file=sys.stderr)
+
+    if total_pages <= chunk_size:
+        print("[split_pdf] No split needed — returning original path", file=sys.stderr)
+        doc.close()
+        return [{
+            "path": os.path.abspath(input_path),
+            "start_page": 0,
+            "end_page": total_pages,
+        }]
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    chunks = []
+    for start_page in range(0, total_pages, chunk_size):
+        end_page = min(start_page + chunk_size, total_pages)
+        chunk_filename = f"chunk_{start_page}_{end_page - 1}.pdf"
+        chunk_path = os.path.join(output_dir, chunk_filename)
+
+        chunk_doc = fitz.open()
+        chunk_doc.insert_pdf(doc, from_page=start_page, to_page=end_page - 1)
+        chunk_doc.save(chunk_path)
+        chunk_doc.close()
+
+        chunks.append({
+            "path": os.path.abspath(chunk_path),
+            "start_page": start_page,
+            "end_page": end_page,
+        })
+        print(
+            f"[split_pdf] Chunk {len(chunks)}: pages {start_page}–{end_page - 1} → {chunk_filename}",
+            file=sys.stderr,
+        )
+
+    doc.close()
+    return chunks
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Split a PDF into fixed-size page chunks.")
+    parser.add_argument("input_pdf", help="Path to the input PDF")
+    parser.add_argument("output_dir", help="Directory to write chunk PDFs into")
+    parser.add_argument(
+        "--chunk-size",
+        type=int,
+        default=25,
+        help="Maximum pages per chunk (default: 25)",
+    )
+    args = parser.parse_args()
+
+    try:
+        chunks = split_pdf(args.input_pdf, args.output_dir, args.chunk_size)
+        print(json.dumps(chunks))
+    except Exception as e:
+        print(f"[split_pdf] ERROR: {e}", file=sys.stderr)
+        sys.exit(1)

package/ee/queues/queues.ts

@@ -1,5 +1,6 @@
 import { Queue } from "bullmq";
 import { redisServer } from "./server";
+import { guardRedisStartup, logRedisErrors } from "./redis-startup";
 import { BullMQOtel } from "bullmq-otel";
 import type { ExuluQueueConfig } from "@EXULU_TYPES/queue-config";
 import { checkLicense } from "@EE/entitlements";
@@ -115,6 +116,15 @@ class ExuluQueues {
         },
         telemetry: new BullMQOtel("simple-guide"),
       });
+      // Surface connection errors and FAIL FAST instead of hanging silently when Redis is down:
+      // wait for the connection to be ready (bounded by REDIS_STARTUP_TIMEOUT_MS) before using it.
+      logRedisErrors(newQueue, `queue "${name}"`);
+      try {
+        await guardRedisStartup(`queue "${name}"`, () => newQueue.waitUntilReady(), newQueue);
+      } catch (err) {
+        void newQueue.close().catch(() => { /* best-effort cleanup; we are aborting startup anyway */ });
+        throw err;
+      }
       await newQueue.setGlobalConcurrency(queueConcurrency);
       this.queues.push({
         queue: newQueue,

package/ee/queues/redis-startup.ts

@@ -0,0 +1,121 @@
+import { redisServer } from "./server";
+
+/**
+ * Loud, bounded Redis startup helpers.
+ *
+ * Without these, a Redis-down boot hangs SILENTLY: the BullMQ queue/worker connections retry
+ * forever with no error listener and no timeout, so `exulu()` never returns and nothing is logged
+ * (you just see repeated `connect ETIMEDOUT 127.0.0.1:6379` from the socket layer, if anything).
+ *
+ * These helpers make a Redis-dependent startup step:
+ *   1. announce the target host:port it is connecting to,
+ *   2. surface the (otherwise swallowed) connection errors with address + code,
+ *   3. warn every few seconds while it is still blocked, and
+ *   4. FAIL FAST with a clear error after REDIS_STARTUP_TIMEOUT_MS instead of hanging forever.
+ */
+
+/** Hard ceiling on a single Redis-dependent startup step before we abort instead of hanging. */
+export const REDIS_STARTUP_TIMEOUT_MS = 60_000;
+/** How often, while still blocked, to remind the operator that startup is stuck on Redis. */
+const WATCHDOG_INTERVAL_MS = 10_000;
+/** Throttle for the permanent per-connection error logger so a retry storm can't flood the log. */
+const ERROR_LOG_THROTTLE_MS = 30_000;
+
+const log = (line: string): void => console.log(`[EXULU-REDIS] ${line}`);
+const warn = (line: string): void => console.warn(`[EXULU-REDIS] ${line}`);
+const errorLog = (line: string): void => console.error(`[EXULU-REDIS] ${line}`);
+
+/** The configured Redis target as `host:port` (with `(unset)` placeholders) for log/error messages. */
+export const redisAddress = (): string =>
+  `${redisServer.host || "(unset)"}:${redisServer.port || "(unset)"}`;
+
+/** One-line, human-readable description of a Redis/socket error (code first, message head, no stack). */
+const describeError = (e: unknown): string => {
+  const any = e as any;
+  const head = any?.message ? String(any.message).split("\n")[0] : undefined;
+  if (any?.code) return head && head !== any.code ? `${any.code} (${head})` : `${any.code}`;
+  return head ?? String(e);
+};
+
+/** Minimal structural shape shared by ioredis, node-redis clients, and BullMQ Queue/Worker. */
+type RedisErrorSource = {
+  on(event: "error", cb: (err: unknown) => void): unknown;
+  off?(event: "error", cb: (err: unknown) => void): unknown;
+};
+
+/**
+ * Attach a PERMANENT `error` listener that logs connection errors with the target address (the first
+ * immediately, then at most once per throttle window). Also prevents node-redis/ioredis from treating
+ * an `error` event as unhandled. Safe to call once per long-lived connection.
+ */
+export function logRedisErrors(source: RedisErrorSource, label: string): void {
+  let count = 0;
+  let lastLoggedAt = 0;
+  source.on("error", (err) => {
+    count += 1;
+    const now = Date.now();
+    if (count === 1 || now - lastLoggedAt >= ERROR_LOG_THROTTLE_MS) {
+      errorLog(`${label} connection error (${redisAddress()}): ${describeError(err)}${count > 1 ? ` (x${count})` : ""}`);
+      lastLoggedAt = now;
+    }
+  });
+}
+
+/**
+ * Run a Redis-dependent startup step with loud logging + a hard timeout. Transparent on success
+ * (returns `run()`'s value). While `run()` is pending it warns every WATCHDOG_INTERVAL_MS that startup
+ * is blocked; if it does not settle within REDIS_STARTUP_TIMEOUT_MS it REJECTS with a clear error
+ * (citing the address + last surfaced connection error) so the caller can fail the boot instead of
+ * hanging forever. `source`, if given, is observed only to capture the latest error for that message.
+ */
+export async function guardRedisStartup<T>(
+  label: string,
+  run: () => Promise<T>,
+  source?: RedisErrorSource,
+): Promise<T> {
+  const addr = redisAddress();
+  log(`Connecting to Redis (${addr}) for ${label}…`);
+  const startedAt = Date.now();
+
+  let lastError: unknown;
+  const onError = (err: unknown): void => { lastError = err; };
+  source?.on("error", onError);
+
+  const watchdog = setInterval(() => {
+    const secs = Math.round((Date.now() - startedAt) / 1000);
+    warn(
+      `⚠ Still waiting for Redis at ${addr} after ${secs}s — ${label} startup is blocked. ` +
+        `Is Redis running? (aborting at ${REDIS_STARTUP_TIMEOUT_MS / 1000}s)`,
+    );
+  }, WATCHDOG_INTERVAL_MS);
+  // Don't let the watchdog timer keep the event loop alive on its own; the timeout below holds it.
+  (watchdog as { unref?: () => void }).unref?.();
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(
+        new Error(
+          `[EXULU-REDIS] Redis unreachable at ${addr} after ${REDIS_STARTUP_TIMEOUT_MS / 1000}s — aborting ${label} startup. ` +
+            `Last error: ${lastError ? describeError(lastError) : "none surfaced"}. ` +
+            `Check REDIS_HOST/REDIS_PORT and that a Redis server is reachable at ${addr}.`,
+        ),
+      );
+    }, REDIS_STARTUP_TIMEOUT_MS);
+  });
+
+  // Mark the work promise handled so a rejection that lands AFTER the timeout already won the race
+  // does not surface as an unhandledRejection.
+  const runPromise = Promise.resolve().then(run);
+  runPromise.catch(() => { /* handled via the race below or intentionally ignored post-timeout */ });
+
+  try {
+    const result = await Promise.race([runPromise, timeout]);
+    log(`Redis ready; ${label} initialized (${addr}, ${((Date.now() - startedAt) / 1000).toFixed(1)}s).`);
+    return result as T;
+  } finally {
+    clearInterval(watchdog);
+    if (timer) clearTimeout(timer);
+    source?.off?.("error", onError);
+  }
+}

package/ee/workers.ts

@@ -1,5 +1,6 @@
 import IORedis from "ioredis";
 import { redisServer } from "@EE/queues/server.ts";
+import { guardRedisStartup, logRedisErrors } from "@EE/queues/redis-startup.ts";
 import { Job, Worker, type JobState } from "bullmq";
 import { bullmq } from "@SRC/validators/bullmq.ts";
 import { getEnabledTools } from "@SRC/utils/enabled-tools.ts";
@@ -158,6 +159,11 @@ export const createWorkers = async (
       },
       maxRetriesPerRequest: null,
     });
+    // Surface connection errors and FAIL FAST instead of hanging silently when Redis is down:
+    // confirm the connection is actually reachable (bounded by REDIS_STARTUP_TIMEOUT_MS) before
+    // handing it to the workers, which would otherwise block on their first operation forever.
+    logRedisErrors(redisConnection, "worker");
+    await guardRedisStartup("workers", () => redisConnection.ping().then(() => undefined), redisConnection);
   }
 
   const workers = queues.map((queue) => {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.68.0",
+  "version": "1.69.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {