@exulu/backend 1.67.0 → 1.69.0

package/ee/agentic-retrieval/v3/agent-loop.ts

@@ -2,7 +2,7 @@ import { generateText, stepCountIs, tool } from "ai";
 import type { LanguageModel, Tool as AITool, ModelMessage } from "ai";
 import { z } from "zod";
 import { withRetry } from "@SRC/utils/with-retry";
-import type { ExuluReranker } from "@SRC/exulu/reranker";
+import type { ResolvedReranker } from "@SRC/exulu/resolve-reranker";
 import type { AgenticRetrievalOutput, ChunkResult, ClassificationResult } from "./types";
 import type { StrategyConfig } from "./strategies";
 import { createDynamicTools } from "./dynamic-tools";
@@ -69,7 +69,7 @@ export async function* runAgentLoop(params: {
   strategy: StrategyConfig;
   tools: Record<string, AITool>;
   model: LanguageModel;
-  reranker?: ExuluReranker;
+  reranker?: ResolvedReranker;
   contextGuidance?: string;
   customInstructions?: string;
   classification: ClassificationResult;
@@ -171,8 +171,8 @@ export async function* runAgentLoop(params: {
 
     // Rerank if reranker is available
     if (reranker && stepChunks.length > 0) {
-      console.log(`[EXULU] v3 reranking ${stepChunks.length} chunks with ${reranker.name}`);
-      stepChunks = await reranker.run(query, stepChunks as any);
+      console.log(`[EXULU] v3 reranking ${stepChunks.length} chunks with ${reranker.model}`);
+      stepChunks = await reranker.rerank(query, stepChunks);
     }
 
     // Create dynamic tools (browse adjacent pages, load specific pages)

package/ee/agentic-retrieval/v3/index.ts

@@ -2,7 +2,8 @@ import { z } from "zod";
 import { createBashTool } from "bash-tool";
 import type { LanguageModel, Tool } from "ai";
 import type { ExuluContext } from "@SRC/exulu/context";
-import type { ExuluReranker } from "@SRC/exulu/reranker";
+import { resolveReranker } from "@SRC/exulu/resolve-reranker";
+import type { ResolvedReranker } from "@SRC/exulu/resolve-reranker";
 import { ExuluTool } from "@SRC/exulu/tool";
 import type { User } from "@EXULU_TYPES/models/user";
 import { checkLicense } from "@EE/entitlements";
@@ -34,7 +35,7 @@ async function* executeV3({
 }: {
   query: string;
   contexts: ExuluContext[];
-  reranker?: ExuluReranker;
+  reranker?: ResolvedReranker;
   toolVariablesConfig?: Record<string, any>;
   model: LanguageModel;
   user?: User;
@@ -189,7 +190,6 @@ async function* executeV3({
 export function createAgenticRetrievalToolV3({
   contexts,
   instructions: adminInstructions,
-  rerankers,
   user,
   role,
   model,
@@ -197,7 +197,6 @@ export function createAgenticRetrievalToolV3({
   memoryItems
 }: {
   contexts: ExuluContext[];
-  rerankers: ExuluReranker[];
   user?: User;
   role?: string;
   model?: LanguageModel;
@@ -355,7 +354,7 @@ export function createAgenticRetrievalToolV3({
       }
 
       let activeContexts = contexts;
-      let configuredReranker: ExuluReranker | undefined;
+      let configuredReranker: ResolvedReranker | undefined;
       let configInstructions = "";
       let logTrajectory = false;
       let requiresPreselectedContexts = false;
@@ -382,7 +381,22 @@ export function createAgenticRetrievalToolV3({
         const rerankerId = toolVariablesConfig["reranker"];
 
         if (rerankerId && rerankerId !== "none") {
-          configuredReranker = rerankers.find((r) => r.id === rerankerId);
+          // rerankerId is a LiteLLM model_name from config.litellm.yaml
+          // (model_info.type: reranker). Resolution is best-effort: a
+          // misconfigured model or an unready proxy must not break retrieval —
+          // it just runs unreranked, matching the old find()→undefined path.
+          try {
+            configuredReranker = await resolveReranker({
+              model: rerankerId,
+              user,
+              roleId: role,
+            });
+          } catch (err) {
+            console.warn(
+              `[EXULU] v3 — could not resolve reranker "${rerankerId}", continuing without reranking:`,
+              err,
+            );
+          }
         }
       }
 

package/ee/python/documents/processing/doc_processor.ts

@@ -14,17 +14,48 @@ import { checkLicense } from '@EE/entitlements';
 import { executePythonScript } from '@SRC/utils/python-executor';
 import { setupPythonEnvironment, validatePythonEnvironment } from '@SRC/utils/python-setup';
 import { LiteParse } from '@llamaindex/liteparse';
-import { Mistral } from '@mistralai/mistralai';
-import { ExuluVariables } from '@SRC/index';
+import { resolveOcr } from '@SRC/exulu/resolve-ocr';
+import type { ResolveOcrInput } from '@SRC/exulu/resolve-ocr';
+import { resolveModel } from '@SRC/exulu/resolve-model';
 
 type DocumentProcessorConfig = {
   vlm?: {
-    model: LanguageModel;
+    /**
+     * LiteLLM model_name for the VLM page-validation pass (declared in
+     * config.litellm.yaml, e.g. "vertex-gemini-2.5-flash"). Resolved via
+     * resolveModel() so the VLM pass shares the same tag-based cost controls
+     * and provider-switching as chat / embeddings / OCR, and the underlying
+     * provider can be swapped without code changes.
+     */
+    model: string;
     concurrency: number;
   },
   processor: {
     name: "docling" | "liteparse" | "mistral" | "officeparser"
+    /**
+     * LiteLLM model_name for the "mistral" OCR processor (declared in
+     * config.litellm.yaml). Defaults to "mistral-ocr". OCR is routed through
+     * the LiteLLM proxy so it shares the same tag-based cost controls as chat
+     * and embeddings, and the underlying provider (mistral / azure_ai /
+     * vertex_ai) can be switched without code changes.
+     */
+    model?: string
+    /**
+     * Maximum pages per OCR request for the "mistral" processor.
+     * Vertex AI OCR rejects documents over 30 pages; the PDF is split into
+     * chunks of this size and each chunk is OCR'd independently.
+     * Defaults to 25 (safely under the Vertex AI 30-page limit).
+     */
+    maxPagesPerChunk?: number
   }
+  /**
+   * Optional cost-attribution context, forwarded to LiteLLM as spend tags
+   * (user / role / project / context) for both the OCR pass (resolveOcr) and
+   * the VLM page-validation pass (resolveModel). Not yet populated by callers;
+   * the wiring is in place so per-user/per-context budgets work the moment
+   * attribution is threaded through.
+   */
+  attribution?: Omit<ResolveOcrInput, "model">
   debugging?: {
     deleteTempFiles?: boolean;
   }
@@ -94,6 +125,38 @@ async function processWord(file: Buffer): Promise<ProcessorOutput> {
   }
 }
 
+/**
+ * Resolve the dev-supplied VLM `model` string (a LiteLLM model_name from
+ * config.litellm.yaml, e.g. "vertex-gemini-2.5-flash") into an `ai` SDK
+ * LanguageModel via resolveModel. This routes the VLM page-validation pass
+ * through the LiteLLM proxy — same tag-based cost controls and provider
+ * switching as chat / embeddings / OCR — and keeps the internal VLM helpers
+ * (validateWithVLM / validatePageWithVLM) working with a LanguageModel.
+ *
+ * Returns undefined when no VLM model is configured. Attribution (user /
+ * project / agent / routine) is forwarded for spend tagging when callers
+ * populate config.attribution; rbacBypass is set because this is a background
+ * package call where model-level access control is delegated to LiteLLM.
+ */
+async function resolveVlmModel(
+  config?: DocumentProcessorConfig,
+): Promise<LanguageModel | undefined> {
+  const modelId = config?.vlm?.model;
+  if (!modelId) return undefined;
+
+  const { languageModel } = await resolveModel({
+    modelId,
+    providers: [], // unused in LiteLLM mode; resolveModel ignores it there
+    user: config?.attribution?.user,
+    project: config?.attribution?.project,
+    agent: config?.attribution?.agent,
+    routine: config?.attribution?.routine,
+    rbacBypass: true,
+  });
+
+  return languageModel;
+}
+
 /**
  * Processes a standalone image file by optionally extracting content using VLM
  */
@@ -122,14 +185,15 @@ async function processImage(
     }];
 
     // If VLM is enabled, use it to extract content from the image
-    if (config?.vlm?.model) {
+    const vlmModel = await resolveVlmModel(config);
+    if (vlmModel) {
       console.log('[EXULU] Extracting content from image using VLM...');
 
       json = await validateWithVLM(
         json,
-        config.vlm.model,
+        vlmModel,
         verbose,
-        config.vlm.concurrency
+        config!.vlm!.concurrency
       );
 
       // Save the processed result
@@ -679,15 +743,6 @@ async function processDocument(
   };
 }
 
-const getMistralApiKey = async () => {
-  if (process.env.MISTRAL_API_KEY) {
-    return process.env.MISTRAL_API_KEY;
-  } else {
-    const variable = await ExuluVariables.get("MISTRAL_API_KEY");
-    return variable;
-  }
-}
-
 async function processPdf(
   buffer: Buffer,
   paths: ProcessingPaths,
@@ -759,29 +814,70 @@ async function processPdf(
 
     } else if (config?.processor.name === "mistral") {
 
-      const MISTRAL_API_KEY = await getMistralApiKey();
-      if (!MISTRAL_API_KEY) {
-        throw new Error('[EXULU] MISTRAL_API_KEY is not set, please set it in the environment variable via process.env or via an Exulu variable named "MISTRAL_API_KEY".');
-      }
+      // OCR is routed through the LiteLLM proxy's Mistral-compatible /v1/ocr
+      // endpoint (see resolveOcr) rather than the Mistral SDK directly. This
+      // gives us tag-based cost control and lets us switch the OCR provider
+      // (mistral / azure_ai / vertex_ai) from config.litellm.yaml.
+      const resolved = await resolveOcr({
+        model: config.processor.model ?? "mistral-ocr",
+        ...config.attribution,
+      });
 
-      // Wait a randomn time between 1 and 5 seconds to prevent rate limiting
-      await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 4000) + 1000));
+      // Split the PDF into ≤ N-page chunks before sending to OCR.
+      // Vertex AI (and some other providers) reject documents over 30 pages.
+      // We use PyMuPDF via a Python helper because it handles edge cases that
+      // trip up JS PDF libraries — in particular "phantom password" PDFs that
+      // are technically encrypted with an empty string (the OS opens them
+      // transparently, but libraries throw without the empty-string fallback).
+      const maxPagesPerChunk = config.processor.maxPagesPerChunk ?? 25;
+      const chunksDir = path.join(path.dirname(paths.json), 'ocr_chunks');
+
+      const splitResult = await executePythonScript({
+        scriptPath: 'ee/python/documents/processing/split_pdf.py',
+        args: [paths.source, chunksDir, '--chunk-size', String(maxPagesPerChunk)],
+        timeout: 5 * 60 * 1000,
+      });
 
-      const base64Pdf = buffer.toString('base64');
-      const client = new Mistral({ apiKey: MISTRAL_API_KEY });
+      const pdfChunks: Array<{ path: string; start_page: number; end_page: number }> =
+        JSON.parse(splitResult.stdout);
 
-      const ocrResponse = await withRetry(async () => {
-        type MistralOCRResponse = Awaited<ReturnType<typeof client.ocr.process>>;
-        const ocrResponse: MistralOCRResponse = await client.ocr.process({
-          document: {
-            type: "document_url",
-            documentUrl: "data:application/pdf;base64," + base64Pdf
-          },
-          model: "mistral-ocr-latest",
-          includeImageBase64: false
-        });
-        return ocrResponse;
-      }, 10);
+      console.log(`[EXULU] PDF split into ${pdfChunks.length} chunk(s) for OCR (max ${maxPagesPerChunk} pages each)`);
+
+      // Process chunks in parallel with a concurrency cap to respect rate limits.
+      // Each chunk gets a small random jitter to spread out requests.
+      const chunkLimit = pLimit(3);
+
+      const chunkResults = await Promise.all(
+        pdfChunks.map((chunk, i) =>
+          chunkLimit(async () => {
+            // Yield to the event loop so BullMQ can renew job locks during long runs
+            await new Promise(resolve => setImmediate(resolve));
+            await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 1000) + 200));
+
+            console.log(`[EXULU] OCR chunk ${i + 1}/${pdfChunks.length}: pages ${chunk.start_page}–${chunk.end_page - 1}`);
+
+            const chunkBuffer = await fs.promises.readFile(chunk.path);
+            const chunkBase64 = chunkBuffer.toString('base64');
+
+            const chunkResponse = await withRetry(async () => {
+              return await resolved.ocr({
+                type: "document_url",
+                document_url: "data:application/pdf;base64," + chunkBase64,
+              }, { includeImageBase64: false });
+            }, 10);
+
+            return { pages: chunkResponse.pages, offset: chunk.start_page };
+          })
+        )
+      );
+
+      // Merge all chunk pages in document order, offsetting indices to their
+      // original positions in the full document.
+      const mergedPages = chunkResults
+        .sort((a, b) => a.offset - b.offset)
+        .flatMap(({ pages, offset }) =>
+          pages.map(p => ({ ...p, index: p.index + offset }))
+        );
 
       const parser = new LiteParse();
       const screenshots = await parser.screenshot(paths.source, undefined);
@@ -796,7 +892,7 @@ async function processPdf(
         screenshot.imagePath = path.join(paths.images, `${screenshot.pageNum}.png`);
       }
 
-      json = ocrResponse.pages.map(page => ({
+      json = mergedPages.map(page => ({
         page: page.index + 1,
         content: page.markdown,
         image: screenshots.find(s => s.pageNum === page.index + 1)?.imagePath,
@@ -838,13 +934,14 @@ async function processPdf(
     }
 
     // Apply VLM validation if enabled
-    if (config?.vlm?.model && json.length > 0) {
+    const vlmModel = config?.vlm?.model ? await resolveVlmModel(config) : undefined;
+    if (vlmModel && json.length > 0) {
 
       json = await validateWithVLM(
         json,
-        config.vlm.model,
+        vlmModel,
         verbose,
-        config.vlm.concurrency
+        config!.vlm!.concurrency
       );
 
       console.log('[EXULU] \n📊 Processing Summary:');
@@ -1046,7 +1143,6 @@ export async function documentProcessor({
   } catch (error) {
     console.error('Error during chunking:', error);
     throw error;
-
   } finally {
     if (config?.debugging?.deleteTempFiles !== false) {
       // Delete the temp directory using the local array to avoid race conditions

package/ee/python/documents/processing/split_pdf.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""
+PDF Splitter — splits a PDF into fixed-size page chunks using PyMuPDF.
+
+Outputs a JSON array to stdout, each element:
+  { "path": "<absolute-path>", "start_page": <int>, "end_page": <int> }
+
+start_page is 0-indexed, end_page is exclusive (Python-slice convention).
+If the document fits within chunk_size, a single entry pointing to the
+original file is returned (no copy made).
+
+Progress and diagnostics go to stderr so stdout stays clean JSON.
+
+Usage:
+    split_pdf.py <input_pdf> <output_dir> [--chunk-size N]
+"""
+
+import sys
+import os
+import json
+import argparse
+
+import fitz  # PyMuPDF — installed as a docling transitive dependency
+
+
+def split_pdf(input_path: str, output_dir: str, chunk_size: int) -> list[dict]:
+    doc = fitz.open(input_path)
+
+    # Some PDFs are saved with an empty owner/user password by certain writers
+    # (e.g. older Adobe Acrobat exports). The OS opens them transparently by
+    # trying "" first, but most libraries raise immediately. We replicate that
+    # OS-level behaviour here.
+    if doc.needs_pass:
+        authenticated = doc.authenticate("")
+        if not authenticated:
+            raise ValueError(
+                "PDF requires a non-empty password and cannot be opened automatically."
+            )
+        print("[split_pdf] Authenticated with empty password (phantom-password PDF)", file=sys.stderr)
+
+    total_pages = len(doc)
+    print(f"[split_pdf] Total pages: {total_pages}, chunk size: {chunk_size}", file=sys.stderr)
+
+    if total_pages <= chunk_size:
+        print("[split_pdf] No split needed — returning original path", file=sys.stderr)
+        doc.close()
+        return [{
+            "path": os.path.abspath(input_path),
+            "start_page": 0,
+            "end_page": total_pages,
+        }]
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    chunks = []
+    for start_page in range(0, total_pages, chunk_size):
+        end_page = min(start_page + chunk_size, total_pages)
+        chunk_filename = f"chunk_{start_page}_{end_page - 1}.pdf"
+        chunk_path = os.path.join(output_dir, chunk_filename)
+
+        chunk_doc = fitz.open()
+        chunk_doc.insert_pdf(doc, from_page=start_page, to_page=end_page - 1)
+        chunk_doc.save(chunk_path)
+        chunk_doc.close()
+
+        chunks.append({
+            "path": os.path.abspath(chunk_path),
+            "start_page": start_page,
+            "end_page": end_page,
+        })
+        print(
+            f"[split_pdf] Chunk {len(chunks)}: pages {start_page}–{end_page - 1} → {chunk_filename}",
+            file=sys.stderr,
+        )
+
+    doc.close()
+    return chunks
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Split a PDF into fixed-size page chunks.")
+    parser.add_argument("input_pdf", help="Path to the input PDF")
+    parser.add_argument("output_dir", help="Directory to write chunk PDFs into")
+    parser.add_argument(
+        "--chunk-size",
+        type=int,
+        default=25,
+        help="Maximum pages per chunk (default: 25)",
+    )
+    args = parser.parse_args()
+
+    try:
+        chunks = split_pdf(args.input_pdf, args.output_dir, args.chunk_size)
+        print(json.dumps(chunks))
+    except Exception as e:
+        print(f"[split_pdf] ERROR: {e}", file=sys.stderr)
+        sys.exit(1)

package/ee/queues/queues.ts

@@ -1,5 +1,6 @@
 import { Queue } from "bullmq";
 import { redisServer } from "./server";
+import { guardRedisStartup, logRedisErrors } from "./redis-startup";
 import { BullMQOtel } from "bullmq-otel";
 import type { ExuluQueueConfig } from "@EXULU_TYPES/queue-config";
 import { checkLicense } from "@EE/entitlements";
@@ -115,6 +116,15 @@ class ExuluQueues {
         },
         telemetry: new BullMQOtel("simple-guide"),
       });
+      // Surface connection errors and FAIL FAST instead of hanging silently when Redis is down:
+      // wait for the connection to be ready (bounded by REDIS_STARTUP_TIMEOUT_MS) before using it.
+      logRedisErrors(newQueue, `queue "${name}"`);
+      try {
+        await guardRedisStartup(`queue "${name}"`, () => newQueue.waitUntilReady(), newQueue);
+      } catch (err) {
+        void newQueue.close().catch(() => { /* best-effort cleanup; we are aborting startup anyway */ });
+        throw err;
+      }
       await newQueue.setGlobalConcurrency(queueConcurrency);
       this.queues.push({
         queue: newQueue,

package/ee/queues/redis-startup.ts

@@ -0,0 +1,121 @@
+import { redisServer } from "./server";
+
+/**
+ * Loud, bounded Redis startup helpers.
+ *
+ * Without these, a Redis-down boot hangs SILENTLY: the BullMQ queue/worker connections retry
+ * forever with no error listener and no timeout, so `exulu()` never returns and nothing is logged
+ * (you just see repeated `connect ETIMEDOUT 127.0.0.1:6379` from the socket layer, if anything).
+ *
+ * These helpers make a Redis-dependent startup step:
+ *   1. announce the target host:port it is connecting to,
+ *   2. surface the (otherwise swallowed) connection errors with address + code,
+ *   3. warn every few seconds while it is still blocked, and
+ *   4. FAIL FAST with a clear error after REDIS_STARTUP_TIMEOUT_MS instead of hanging forever.
+ */
+
+/** Hard ceiling on a single Redis-dependent startup step before we abort instead of hanging. */
+export const REDIS_STARTUP_TIMEOUT_MS = 60_000;
+/** How often, while still blocked, to remind the operator that startup is stuck on Redis. */
+const WATCHDOG_INTERVAL_MS = 10_000;
+/** Throttle for the permanent per-connection error logger so a retry storm can't flood the log. */
+const ERROR_LOG_THROTTLE_MS = 30_000;
+
+const log = (line: string): void => console.log(`[EXULU-REDIS] ${line}`);
+const warn = (line: string): void => console.warn(`[EXULU-REDIS] ${line}`);
+const errorLog = (line: string): void => console.error(`[EXULU-REDIS] ${line}`);
+
+/** The configured Redis target as `host:port` (with `(unset)` placeholders) for log/error messages. */
+export const redisAddress = (): string =>
+  `${redisServer.host || "(unset)"}:${redisServer.port || "(unset)"}`;
+
+/** One-line, human-readable description of a Redis/socket error (code first, message head, no stack). */
+const describeError = (e: unknown): string => {
+  const any = e as any;
+  const head = any?.message ? String(any.message).split("\n")[0] : undefined;
+  if (any?.code) return head && head !== any.code ? `${any.code} (${head})` : `${any.code}`;
+  return head ?? String(e);
+};
+
+/** Minimal structural shape shared by ioredis, node-redis clients, and BullMQ Queue/Worker. */
+type RedisErrorSource = {
+  on(event: "error", cb: (err: unknown) => void): unknown;
+  off?(event: "error", cb: (err: unknown) => void): unknown;
+};
+
+/**
+ * Attach a PERMANENT `error` listener that logs connection errors with the target address (the first
+ * immediately, then at most once per throttle window). Also prevents node-redis/ioredis from treating
+ * an `error` event as unhandled. Safe to call once per long-lived connection.
+ */
+export function logRedisErrors(source: RedisErrorSource, label: string): void {
+  let count = 0;
+  let lastLoggedAt = 0;
+  source.on("error", (err) => {
+    count += 1;
+    const now = Date.now();
+    if (count === 1 || now - lastLoggedAt >= ERROR_LOG_THROTTLE_MS) {
+      errorLog(`${label} connection error (${redisAddress()}): ${describeError(err)}${count > 1 ? ` (x${count})` : ""}`);
+      lastLoggedAt = now;
+    }
+  });
+}
+
+/**
+ * Run a Redis-dependent startup step with loud logging + a hard timeout. Transparent on success
+ * (returns `run()`'s value). While `run()` is pending it warns every WATCHDOG_INTERVAL_MS that startup
+ * is blocked; if it does not settle within REDIS_STARTUP_TIMEOUT_MS it REJECTS with a clear error
+ * (citing the address + last surfaced connection error) so the caller can fail the boot instead of
+ * hanging forever. `source`, if given, is observed only to capture the latest error for that message.
+ */
+export async function guardRedisStartup<T>(
+  label: string,
+  run: () => Promise<T>,
+  source?: RedisErrorSource,
+): Promise<T> {
+  const addr = redisAddress();
+  log(`Connecting to Redis (${addr}) for ${label}…`);
+  const startedAt = Date.now();
+
+  let lastError: unknown;
+  const onError = (err: unknown): void => { lastError = err; };
+  source?.on("error", onError);
+
+  const watchdog = setInterval(() => {
+    const secs = Math.round((Date.now() - startedAt) / 1000);
+    warn(
+      `⚠ Still waiting for Redis at ${addr} after ${secs}s — ${label} startup is blocked. ` +
+        `Is Redis running? (aborting at ${REDIS_STARTUP_TIMEOUT_MS / 1000}s)`,
+    );
+  }, WATCHDOG_INTERVAL_MS);
+  // Don't let the watchdog timer keep the event loop alive on its own; the timeout below holds it.
+  (watchdog as { unref?: () => void }).unref?.();
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(() => {
+      reject(
+        new Error(
+          `[EXULU-REDIS] Redis unreachable at ${addr} after ${REDIS_STARTUP_TIMEOUT_MS / 1000}s — aborting ${label} startup. ` +
+            `Last error: ${lastError ? describeError(lastError) : "none surfaced"}. ` +
+            `Check REDIS_HOST/REDIS_PORT and that a Redis server is reachable at ${addr}.`,
+        ),
+      );
+    }, REDIS_STARTUP_TIMEOUT_MS);
+  });
+
+  // Mark the work promise handled so a rejection that lands AFTER the timeout already won the race
+  // does not surface as an unhandledRejection.
+  const runPromise = Promise.resolve().then(run);
+  runPromise.catch(() => { /* handled via the race below or intentionally ignored post-timeout */ });
+
+  try {
+    const result = await Promise.race([runPromise, timeout]);
+    log(`Redis ready; ${label} initialized (${addr}, ${((Date.now() - startedAt) / 1000).toFixed(1)}s).`);
+    return result as T;
+  } finally {
+    clearInterval(watchdog);
+    if (timer) clearTimeout(timer);
+    source?.off?.("error", onError);
+  }
+}