@exulu/backend 1.68.0 → 1.69.1

package/dist/index.js

@@ -43,8 +43,10 @@ import {
   exchangeCodeForTokens,
   exuluApp,
   getBudgetSettings,
+  getChunkEntitiesTableName,
   getChunksTableName,
   getEntitiesForItem,
+  getEntitiesTableName,
   getPresignedUrl,
   getS3ObjectBytes,
   getS3ObjectContent,
@@ -66,6 +68,7 @@ import {
   postgresClient,
   provisionDefaultUserBudget,
   reportSystemDependencies,
+  resolveEmbedder,
   resolveEntityModel,
   resolveLiteLLMConfigPath,
   resolveModel,
@@ -85,7 +88,7 @@ import {
   vectorSearch,
   waitForLiteLLMReady,
   withRetry
-} from "./chunk-VPSLTGZF.js";
+} from "./chunk-PJSDLFXL.js";
 import {
   findLiteLLMModel
 } from "./chunk-YCE44CMU.js";
@@ -104,6 +107,72 @@ var redisServer = {
   username: process.env.REDIS_USER || ""
 };
 
+// ee/queues/redis-startup.ts
+var REDIS_STARTUP_TIMEOUT_MS = 6e4;
+var WATCHDOG_INTERVAL_MS = 1e4;
+var ERROR_LOG_THROTTLE_MS = 3e4;
+var log = (line) => console.log(`[EXULU-REDIS] ${line}`);
+var warn = (line) => console.warn(`[EXULU-REDIS] ${line}`);
+var errorLog = (line) => console.error(`[EXULU-REDIS] ${line}`);
+var redisAddress = () => `${redisServer.host || "(unset)"}:${redisServer.port || "(unset)"}`;
+var describeError = (e) => {
+  const any = e;
+  const head = any?.message ? String(any.message).split("\n")[0] : void 0;
+  if (any?.code) return head && head !== any.code ? `${any.code} (${head})` : `${any.code}`;
+  return head ?? String(e);
+};
+function logRedisErrors(source, label) {
+  let count = 0;
+  let lastLoggedAt = 0;
+  source.on("error", (err) => {
+    count += 1;
+    const now = Date.now();
+    if (count === 1 || now - lastLoggedAt >= ERROR_LOG_THROTTLE_MS) {
+      errorLog(`${label} connection error (${redisAddress()}): ${describeError(err)}${count > 1 ? ` (x${count})` : ""}`);
+      lastLoggedAt = now;
+    }
+  });
+}
+async function guardRedisStartup(label, run, source) {
+  const addr = redisAddress();
+  log(`Connecting to Redis (${addr}) for ${label}\u2026`);
+  const startedAt = Date.now();
+  let lastError;
+  const onError = (err) => {
+    lastError = err;
+  };
+  source?.on("error", onError);
+  const watchdog = setInterval(() => {
+    const secs = Math.round((Date.now() - startedAt) / 1e3);
+    warn(
+      `\u26A0 Still waiting for Redis at ${addr} after ${secs}s \u2014 ${label} startup is blocked. Is Redis running? (aborting at ${REDIS_STARTUP_TIMEOUT_MS / 1e3}s)`
+    );
+  }, WATCHDOG_INTERVAL_MS);
+  watchdog.unref?.();
+  let timer2;
+  const timeout = new Promise((_resolve, reject) => {
+    timer2 = setTimeout(() => {
+      reject(
+        new Error(
+          `[EXULU-REDIS] Redis unreachable at ${addr} after ${REDIS_STARTUP_TIMEOUT_MS / 1e3}s \u2014 aborting ${label} startup. Last error: ${lastError ? describeError(lastError) : "none surfaced"}. Check REDIS_HOST/REDIS_PORT and that a Redis server is reachable at ${addr}.`
+        )
+      );
+    }, REDIS_STARTUP_TIMEOUT_MS);
+  });
+  const runPromise = Promise.resolve().then(run);
+  runPromise.catch(() => {
+  });
+  try {
+    const result = await Promise.race([runPromise, timeout]);
+    log(`Redis ready; ${label} initialized (${addr}, ${((Date.now() - startedAt) / 1e3).toFixed(1)}s).`);
+    return result;
+  } finally {
+    clearInterval(watchdog);
+    if (timer2) clearTimeout(timer2);
+    source?.off?.("error", onError);
+  }
+}
+
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
@@ -121,9 +190,11 @@ async function redisClient() {
       client["exulu"] = createClient({
         url
       });
-      await client["exulu"].connect();
+      logRedisErrors(client["exulu"], "client");
+      await guardRedisStartup("client", () => client["exulu"].connect().then(() => void 0), client["exulu"]);
     } catch (error) {
       console.error(`[EXULU] error connecting to redis:`, error);
+      delete client["exulu"];
       return { client: null };
     }
   }
@@ -141,7 +212,8 @@ import "express";
 // src/validators/requests.ts
 var requestValidators = {
   authenticate: async (req) => {
-    const apikey = req.headers["exulu-api-key"] || null;
+    const rawApiKey = req.headers["exulu-api-key"] || req.headers["x-api-key"];
+    const apikey = rawApiKey?.replace(/^Bearer\s+/i, "") || null;
     const { db } = await postgresClient();
     let authtoken = null;
     if (typeof apikey !== "string") {
@@ -405,6 +477,14 @@ var ExuluQueues = class {
         },
         telemetry: new BullMQOtel("simple-guide")
       });
+      logRedisErrors(newQueue, `queue "${name}"`);
+      try {
+        await guardRedisStartup(`queue "${name}"`, () => newQueue.waitUntilReady(), newQueue);
+      } catch (err) {
+        void newQueue.close().catch(() => {
+        });
+        throw err;
+      }
       await newQueue.setGlobalConcurrency(queueConcurrency);
       this.queues.push({
         queue: newQueue,
@@ -2490,6 +2570,8 @@ var createWorkers = async (providers, queues2, config, contexts, evals, tools, t
       },
       maxRetriesPerRequest: null
     });
+    logRedisErrors(redisConnection, "worker");
+    await guardRedisStartup("workers", () => redisConnection.ping().then(() => void 0), redisConnection);
   }
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue.queue.name}.`);
@@ -3660,7 +3742,7 @@ var renderTranscript = (segments, speakers) => {
 
 // src/exulu/transcription/service.ts
 var TABLE = "transcription_jobs";
-var log = (msg) => console.log(`[EXULU-TRANSCRIPTION] ${msg}`);
+var log2 = (msg) => console.log(`[EXULU-TRANSCRIPTION] ${msg}`);
 var parseJsonField = (v) => {
   if (v == null) return null;
   if (typeof v === "string") {
@@ -3731,7 +3813,7 @@ var transcriptionService = {
         error: err.message,
         updatedAt: /* @__PURE__ */ new Date()
       }).returning("*");
-      log(`Failed to dispatch job ${row.id}: ${err.message}`);
+      log2(`Failed to dispatch job ${row.id}: ${err.message}`);
       return this._rowFromDb(failed);
     }
   },
@@ -3759,9 +3841,9 @@ var transcriptionService = {
             updatedAt: /* @__PURE__ */ new Date()
           });
         } else if (err instanceof TranscriptionServerUnavailable) {
-          log(`Whisper server unreachable while polling ${row.id}; will retry`);
+          log2(`Whisper server unreachable while polling ${row.id}; will retry`);
         } else {
-          log(`Error polling job ${row.id}: ${err.message}`);
+          log2(`Error polling job ${row.id}: ${err.message}`);
         }
       }
     }
@@ -3811,7 +3893,7 @@ var transcriptionService = {
       } catch (err) {
         const code = err.code;
         if (code !== "JOB_NOT_FOUND") {
-          log(`Best-effort cancel of whisper job failed: ${err.message}`);
+          log2(`Best-effort cancel of whisper job failed: ${err.message}`);
         }
       }
     }
@@ -3897,7 +3979,7 @@ var transcriptionService = {
           []
         );
       } catch (err) {
-        log(`RBAC update failed for item ${itemId}: ${err.message}`);
+        log2(`RBAC update failed for item ${itemId}: ${err.message}`);
       }
     }
     const projectId = input.project_id ?? row.project_id ?? null;
@@ -4185,7 +4267,7 @@ var durationFromSegments = (segments) => {
 // src/exulu/recall/service.ts
 var TABLE2 = "transcription_jobs";
 var DEFAULT_BOT_NAME = "Exulu Notetaker";
-var log2 = (msg) => console.log(`[EXULU-RECALL] ${msg}`);
+var log3 = (msg) => console.log(`[EXULU-RECALL] ${msg}`);
 var parseJson = (v) => {
   if (v == null) return null;
   if (typeof v === "string") {
@@ -4282,7 +4364,7 @@ var recallService = {
         error: err.message,
         updatedAt: /* @__PURE__ */ new Date()
       }).returning("*");
-      log2(`createBot failed for job ${inserted.id}: ${err.message}`);
+      log3(`createBot failed for job ${inserted.id}: ${err.message}`);
       return this._row(failed);
     }
   },
@@ -4296,7 +4378,7 @@ var recallService = {
     const name = event?.event ?? "";
     const job = await this._findJob(ids.botId, ids.recordingId, ids.transcriptId);
     if (!job) {
-      log2(`No job for event ${name} (bot=${ids.botId}); ignoring.`);
+      log3(`No job for event ${name} (bot=${ids.botId}); ignoring.`);
       return;
     }
     if (name.startsWith("bot.")) {
@@ -4321,12 +4403,12 @@ var recallService = {
         await this._fail(job.id, ids.subCode || ids.code || "transcript failed");
         return;
       default:
-        log2(`Unhandled event ${name} for job ${job.id}`);
+        log3(`Unhandled event ${name} for job ${job.id}`);
     }
   },
   async _onRecordingDone(jobId, recordingId) {
     if (!recordingId) {
-      log2(`recording.done for job ${jobId} had no recording id`);
+      log3(`recording.done for job ${jobId} had no recording id`);
       return;
     }
     const { db } = await postgresClient();
@@ -4336,7 +4418,7 @@ var recallService = {
       updatedAt: /* @__PURE__ */ new Date()
     });
     if (!claimed) {
-      log2(`recording.done for job ${jobId} already handled; skipping.`);
+      log3(`recording.done for job ${jobId} already handled; skipping.`);
       return;
     }
     try {
@@ -4356,7 +4438,7 @@ var recallService = {
     if (!dbRow) return;
     const job = this._row(dbRow);
     if ((job.status === "awaiting_review" || job.status === "saved") && job.raw_segments && job.raw_segments.length > 0) {
-      log2(`transcript.done for job ${jobId} already processed; skipping.`);
+      log3(`transcript.done for job ${jobId} already processed; skipping.`);
       return;
     }
     try {
@@ -4375,7 +4457,7 @@ var recallService = {
           const recDuration = recordingDurationSeconds(rec);
           if (recDuration != null) duration = recDuration;
         } catch (err) {
-          log2(`could not fetch recording duration for job ${jobId}: ${err.message}`);
+          log3(`could not fetch recording duration for job ${jobId}: ${err.message}`);
         }
       }
       await this._update(jobId, {
@@ -4415,11 +4497,11 @@ var recallService = {
     const prompts = job.post_processing_prompts ?? [];
     if (prompts.length === 0) return [];
     if (job.post_processing_outputs && job.post_processing_outputs.length > 0) {
-      log2(`post-processing for job ${jobId} already ran; skipping.`);
+      log3(`post-processing for job ${jobId} already ran; skipping.`);
       return job.post_processing_outputs;
     }
     if (!job.raw_segments || job.raw_segments.length === 0) {
-      log2(`post-processing for job ${jobId} skipped: no transcript.`);
+      log3(`post-processing for job ${jobId} skipped: no transcript.`);
       return [];
     }
     const outputs = [];
@@ -4492,7 +4574,7 @@ ${transcriptText}`,
         ran_at: ranAt
       };
     } catch (err) {
-      log2(`post-processing prompt ${promptId} failed for job ${job.id}: ${err.message}`);
+      log3(`post-processing prompt ${promptId} failed for job ${job.id}: ${err.message}`);
       return {
         prompt_id: promptId,
         agent_id: agentId,
@@ -8644,6 +8726,52 @@ var verifyRecallRequest = (headers, rawBody, secret = recallVerificationSecret()
   return passed ? { ok: true } : { ok: false, reason: "signature mismatch" };
 };
 
+// src/exulu/shared-artifacts.ts
+import bcrypt3 from "bcryptjs";
+var normalizeS3Key = (key, bucket) => {
+  const segments = key.split("/").filter((s, i) => !(i === 0 && s === "")).map((s) => decodeURIComponent(s));
+  if (segments[0] === bucket) segments.shift();
+  return segments.join("/");
+};
+var isHtmlKey = (key) => /\.html?$/i.test(key);
+var deriveFilename = (key) => {
+  const base = key.split("/").pop() ?? key;
+  return base.split("_EXULU_").pop() ?? base;
+};
+var slugifyShareName = (input) => deriveFilename(input).toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+var isExpired = (expiresAt, now) => {
+  if (!expiresAt) return false;
+  return new Date(expiresAt).getTime() <= now.getTime();
+};
+var validateCreateInput = (input, now) => {
+  if (!input.s3key) return { ok: false, message: "s3key is required." };
+  if (!input.name) return { ok: false, message: "name is required." };
+  const mode = input.auth_mode;
+  if (mode !== "public" && mode !== "password" && mode !== "regular") {
+    return { ok: false, message: "auth_mode must be public, password, or regular." };
+  }
+  if (mode === "password" && !input.password) {
+    return { ok: false, message: "A password is required for password mode." };
+  }
+  if (input.expires_at && Number.isNaN(new Date(input.expires_at).getTime())) {
+    return { ok: false, message: "expires_at is not a valid date." };
+  }
+  if (input.expires_at && isExpired(input.expires_at, now)) {
+    return { ok: false, message: "expires_at must be in the future." };
+  }
+  return { ok: true };
+};
+var hashSharePassword = (password) => bcrypt3.hash(password, 10);
+var verifySharePassword = (password, hash) => bcrypt3.compare(password, hash);
+var contentHeadersFor = (key, contentType, filename) => {
+  if (isHtmlKey(key)) return { contentType: "text/html; charset=utf-8" };
+  return {
+    contentType: contentType || "application/octet-stream",
+    disposition: `attachment; filename="${filename.replace(/"/g, "")}"`
+  };
+};
+var getSharedArtifactByName = (db, name) => db("shared_artifacts").where({ name }).first();
+
 // src/exulu/routes.ts
 var REQUEST_SIZE_LIMIT = "50mb";
 var getExuluVersionNumber = async () => {
@@ -11678,6 +11806,126 @@ ${style.markdown}` : params.prompt;
     }
     res.status(204).send();
   });
+  app.post("/shared-artifacts", async (req, res) => {
+    const { db } = await postgresClient();
+    const auth = await requestValidators.authenticate(req);
+    if (!auth.user?.id) {
+      res.status(401).json({ detail: "Authentication required." });
+      return;
+    }
+    const now = /* @__PURE__ */ new Date();
+    const valid = validateCreateInput(req.body, now);
+    if (!valid.ok) {
+      res.status(400).json({ detail: valid.message });
+      return;
+    }
+    const bucket = config.fileUploads?.s3Bucket ?? "";
+    const s3key = normalizeS3Key(req.body.s3key, bucket);
+    const name = slugifyShareName(req.body.name);
+    if (!name) {
+      res.status(400).json({ detail: "name must contain url-safe characters." });
+      return;
+    }
+    const existing = await getSharedArtifactByName(db, name);
+    if (existing) {
+      res.status(409).json({ detail: "That share name is already taken." });
+      return;
+    }
+    const auth_mode = req.body.auth_mode;
+    const password_hash = auth_mode === "password" ? await hashSharePassword(req.body.password) : null;
+    const rights_mode = auth_mode === "regular" ? req.body.rights_mode ?? "private" : "public";
+    const [row] = await db("shared_artifacts").insert({
+      name,
+      s3key,
+      auth_mode,
+      password_hash,
+      expires_at: req.body.expires_at ?? null,
+      content_type: req.body.content_type ?? null,
+      rights_mode,
+      created_by: auth.user.id
+    }).returning("*");
+    if (auth_mode === "regular" && req.body.rbac) {
+      await handleRBACUpdate(db, "shared_artifact", row.id, req.body.rbac, []);
+    }
+    res.status(201).json({ name: row.name });
+  });
+  app.get("/shared-artifacts/:name/meta", async (req, res) => {
+    const { db } = await postgresClient();
+    const row = await getSharedArtifactByName(db, req.params.name ?? "");
+    if (!row) {
+      res.status(404).json({ detail: "Not found." });
+      return;
+    }
+    if (isExpired(row.expires_at, /* @__PURE__ */ new Date())) {
+      res.status(410).json({ detail: "This link has expired." });
+      return;
+    }
+    res.json({
+      auth_mode: row.auth_mode,
+      expires_at: row.expires_at,
+      filename: deriveFilename(row.s3key),
+      content_type: row.content_type,
+      is_html: /\.html?$/i.test(row.s3key)
+    });
+  });
+  app.get("/shared-artifacts/:name/content", async (req, res) => {
+    const { db } = await postgresClient();
+    const row = await getSharedArtifactByName(db, req.params.name ?? "");
+    if (!row) {
+      res.status(404).json({ detail: "Not found." });
+      return;
+    }
+    if (isExpired(row.expires_at, /* @__PURE__ */ new Date())) {
+      res.status(410).json({ detail: "This link has expired." });
+      return;
+    }
+    if (row.auth_mode === "password") {
+      const pw = req.headers["x-share-password"] || "";
+      if (!row.password_hash || !await verifySharePassword(pw, row.password_hash)) {
+        res.status(401).json({ detail: "Incorrect password." });
+        return;
+      }
+    } else if (row.auth_mode === "regular") {
+      const viewer = await requestValidators.authenticate(req);
+      if (!viewer.user?.id) {
+        res.status(401).json({ detail: "Authentication required." });
+        return;
+      }
+      const rbac = await RBACResolver(
+        db,
+        "shared_artifact",
+        row.id,
+        row.rights_mode || "private"
+      );
+      const ok = await checkRecordAccess({ ...row, RBAC: rbac }, "read", viewer.user);
+      if (!ok) {
+        res.status(403).json({ detail: "You don't have access to this artifact." });
+        return;
+      }
+    }
+    let bytes;
+    try {
+      bytes = await getS3ObjectBytes(row.s3key, config);
+    } catch (e) {
+      if (e?.name === "NoSuchKey" || e?.name === "NotFound" || e?.$metadata?.httpStatusCode === 404) {
+        res.status(404).json({ detail: "Artifact file not found." });
+        return;
+      }
+      console.error("[EXULU] shared-artifact content read failed", e);
+      res.status(500).json({ detail: "Failed to read artifact." });
+      return;
+    }
+    const headers = contentHeadersFor(
+      row.s3key,
+      row.content_type,
+      deriveFilename(row.s3key)
+    );
+    res.setHeader("Content-Type", headers.contentType);
+    res.setHeader("X-Content-Type-Options", "nosniff");
+    res.setHeader("Cache-Control", "private, no-store");
+    if (headers.disposition) res.setHeader("Content-Disposition", headers.disposition);
+    res.send(bytes);
+  });
   app.use(express2.static("public"));
   await registerOpenAIGatewayRoutes(app, providers, tools, contexts, config);
   return app;
@@ -15373,6 +15621,73 @@ var RecursiveChunker = class _RecursiveChunker extends BaseChunker {
   }
 };
 
+// src/exulu/read-api.ts
+var authorizedRead = async (context, user, role, opts = {}) => {
+  if (!opts.itemIds?.length && !opts.externalIds?.length) {
+    throw new Error("authorizedRead requires itemIds or externalIds to constrain the read.");
+  }
+  const { db } = await postgresClient();
+  const table = convertContextToTableDefinition(context);
+  const itemsTable = getTableName(context.id);
+  const chunksTable = getChunksTableName(context.id);
+  const acUser = role && (!user.role || user.role.id !== role) ? { ...user, role: { ...user.role ?? {}, id: role } } : user;
+  let q = db(chunksTable + " as chunks").select([
+    "chunks.id as chunk_id",
+    "chunks.source as chunk_source",
+    "chunks.content as chunk_content",
+    "chunks.chunk_index",
+    "chunks.metadata as chunk_metadata",
+    db.raw('chunks."createdAt" as chunk_created_at'),
+    db.raw('chunks."updatedAt" as chunk_updated_at'),
+    "items.id as item_id",
+    "items.name as item_name",
+    "items.external_id as item_external_id",
+    db.raw('items."createdAt" as item_created_at'),
+    db.raw('items."updatedAt" as item_updated_at')
+  ]);
+  q = q.leftJoin(itemsTable + " as items", "chunks.source", "items.id");
+  if (opts.itemIds?.length) q = q.whereIn("items.id", opts.itemIds);
+  if (opts.externalIds?.length) q = q.whereIn("items.external_id", opts.externalIds);
+  if (opts.chunkIndexRange) {
+    const { from, to } = opts.chunkIndexRange;
+    if (typeof from === "number") q = q.where("chunks.chunk_index", ">=", from);
+    if (typeof to === "number") q = q.where("chunks.chunk_index", "<=", to);
+  }
+  q = applyAccessControl(table, q, acUser, "items");
+  q = q.orderBy("chunks.source").orderBy("chunks.chunk_index");
+  return await q;
+};
+var entitiesAvailable = async (context) => {
+  if (!context.entities) return false;
+  const { db } = await postgresClient();
+  const table = getChunkEntitiesTableName(context.id);
+  const res = await db.raw(
+    "SELECT to_regclass(?) IS NOT NULL AS exists",
+    [table]
+  );
+  return res.rows?.[0]?.exists === true;
+};
+var embedQuery = async (context, text, opts = {}) => {
+  const resolved = await resolveEmbedder({
+    model: context.embedder.model,
+    contextId: context.id,
+    contextName: context.name,
+    user: opts.user,
+    roleId: opts.role
+  });
+  const [vector] = await resolved.embed([text], { inputType: opts.inputType ?? "query" });
+  return vector ?? [];
+};
+var ExuluReadApi = {
+  getTableName,
+  getChunksTableName,
+  getEntitiesTableName,
+  getChunkEntitiesTableName,
+  entitiesAvailable,
+  authorizedRead,
+  embedQuery
+};
+
 // src/postgres/init-exulu-db.ts
 var {
   agentsSchema: agentsSchema2,
@@ -15400,7 +15715,8 @@ var {
   promptFavoritesSchema: promptFavoritesSchema2,
   transcriptionJobsSchema: transcriptionJobsSchema2,
   imageGenerationsSchema,
-  oauthTokensSchema
+  oauthTokensSchema,
+  sharedArtifactsSchema
 } = coreSchemas.get();
 var addMissingFields = async (knex, tableName, fields, skipFields = []) => {
   for (const field of fields) {
@@ -15444,6 +15760,7 @@ var up = async function(knex) {
     transcriptionJobsSchema2(),
     imageGenerationsSchema(),
     oauthTokensSchema(),
+    sharedArtifactsSchema(),
     rbacSchema2(),
     agentsSchema2(),
     feedbackSchema2(),
@@ -15729,7 +16046,7 @@ var checkLiteLLMDatabaseSafety = (configPath) => {
 
 // src/exulu/litellm/db-init.ts
 var WARNING_BANNER = "\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550";
-var warn = (lines) => {
+var warn2 = (lines) => {
   console.warn(`
 ${WARNING_BANNER}`);
   console.warn("\u26A0  [EXULU-LITELLM] CONFIGURATION WARNING");
@@ -15737,13 +16054,13 @@ ${WARNING_BANNER}`);
   console.warn(`${WARNING_BANNER}
 `);
 };
-var log3 = (line) => console.log(`[EXULU-LITELLM] ${line}`);
+var log4 = (line) => console.log(`[EXULU-LITELLM] ${line}`);
 var initLiteLLMDatabase = async (packageRoot) => {
   const configPath = process.env.LITELLM_CONFIG_PATH ?? resolve2(process.cwd(), "./config.litellm.yaml");
   const safety = checkLiteLLMDatabaseSafety(configPath);
   if (safety.ok && safety.reason === "no-litellm-db-mode") return;
   if (!safety.ok && safety.reason === "unparseable-url") {
-    warn([
+    warn2([
       `LiteLLM's database_url is not a valid postgres URL:`,
       `   ${safety.rawUrl}`,
       `Expected postgres://user:pass@host:port/database. Skipping setup.`
@@ -15752,7 +16069,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
   }
   if (!safety.ok && safety.reason === "shared-with-exulu") {
     const { exuluTarget } = safety;
-    warn([
+    warn2([
       `LiteLLM's database_url points to the SAME database Exulu uses:`,
       `   ${exuluTarget.host}:${exuluTarget.port}/${exuluTarget.database}`,
       ``,
@@ -15771,7 +16088,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
     return;
   }
   const target = "litellmTarget" in safety ? safety.litellmTarget : void 0;
-  log3(
+  log4(
     `LiteLLM database mode detected (${target?.host}:${target?.port}/${target?.database}).`
   );
   const ensureDatabaseExists = async () => {
@@ -15783,7 +16100,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
     } catch (err) {
       const code = err?.code;
       if (code !== "3D000") {
-        warn([
+        warn2([
           `Could not connect to LiteLLM's target database:`,
           `   ${err instanceof Error ? err.message : String(err)}`,
           ``,
@@ -15795,13 +16112,13 @@ var initLiteLLMDatabase = async (packageRoot) => {
       const url = new URL(litellmUrl);
       const targetDbName = url.pathname.replace(/^\//, "");
       if (!targetDbName) {
-        warn([`LiteLLM database_url has no database name; cannot auto-create.`]);
+        warn2([`LiteLLM database_url has no database name; cannot auto-create.`]);
         return false;
       }
       url.pathname = "/postgres";
-      log3(`Target database "${targetDbName}" does not exist; creating it\u2026`);
+      log4(`Target database "${targetDbName}" does not exist; creating it\u2026`);
       if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(targetDbName)) {
-        warn([
+        warn2([
           `Refusing to auto-create database "${targetDbName}" \u2014 name`,
           `contains characters that would require quoting. Create it`,
           `manually: createdb -h ${url.hostname} -U ${url.username} ${targetDbName}`
@@ -15812,10 +16129,10 @@ var initLiteLLMDatabase = async (packageRoot) => {
       try {
         await admin.connect();
         await admin.query(`CREATE DATABASE "${targetDbName}"`);
-        log3(`\u2713 Created database "${targetDbName}".`);
+        log4(`\u2713 Created database "${targetDbName}".`);
         return true;
       } catch (createErr) {
-        warn([
+        warn2([
           `Failed to auto-create database "${targetDbName}":`,
           `   ${createErr instanceof Error ? createErr.message : String(createErr)}`,
           ``,
@@ -15834,7 +16151,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
     }
   };
   if (!await ensureDatabaseExists()) return;
-  log3("Checking that the target database is safe to push into\u2026");
+  log4("Checking that the target database is safe to push into\u2026");
   const client2 = new Client({ connectionString: litellmUrl });
   let foreignTables = [];
   try {
@@ -15850,7 +16167,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
     );
     foreignTables = res.rows.map((r) => r.table_name);
   } catch (err) {
-    warn([
+    warn2([
       `Could not query LiteLLM's target database to verify it is safe`,
       `to push into:`,
       `   ${err instanceof Error ? err.message : String(err)}`,
@@ -15866,7 +16183,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
     }
   }
   if (foreignTables.length > 0) {
-    warn([
+    warn2([
       `LiteLLM's target database contains ${foreignTables.length} table(s) that are NOT`,
       `part of LiteLLM's schema:`,
       ...foreignTables.slice(0, 10).map((t) => `   - ${t}`),
@@ -15882,7 +16199,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
   const venvLibDir = resolve2(packageRoot, "ee/python/.venv/lib");
   const pythonVersionDir = existsSync5(venvLibDir) ? readdirSync(venvLibDir).find((entry) => /^python3\.\d+$/.test(entry)) : void 0;
   if (!pythonVersionDir) {
-    warn([
+    warn2([
       `Could not find a python3.* directory under ${venvLibDir}.`,
       `Run \`npm run python:setup\` to create the venv.`,
       `Skipping LiteLLM database setup.`
@@ -15896,7 +16213,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
   );
   const schemaPath = resolve2(litellmProxyDir, "schema.prisma");
   if (!existsSync5(prismaCli)) {
-    warn([
+    warn2([
       `Prisma CLI not found at ${prismaCli}.`,
       `Run \`npm run python:setup\` to create the venv and install prisma.`,
       `Skipping LiteLLM database setup.`
@@ -15904,13 +16221,13 @@ var initLiteLLMDatabase = async (packageRoot) => {
     return;
   }
   if (!existsSync5(schemaPath)) {
-    warn([
+    warn2([
       `LiteLLM Prisma schema not found at ${schemaPath}.`,
       `Re-run \`npm run python:setup\`. Skipping LiteLLM database setup.`
     ]);
     return;
   }
-  log3("Running `prisma db push` against LiteLLM's schema\u2026");
+  log4("Running `prisma db push` against LiteLLM's schema\u2026");
   const result = spawnSync(prismaCli, ["db", "push", "--skip-generate"], {
     cwd: litellmProxyDir,
     env: {
@@ -15924,14 +16241,14 @@ var initLiteLLMDatabase = async (packageRoot) => {
     encoding: "utf8"
   });
   if (result.error) {
-    warn([
+    warn2([
       `Failed to launch prisma: ${result.error.message}`,
       `Skipping LiteLLM database setup.`
     ]);
     return;
   }
   if (result.status !== 0) {
-    warn([
+    warn2([
       `prisma db push exited with status ${result.status}.`,
       `stdout:`,
       ...(result.stdout || "(empty)").split("\n").map((l) => `   ${l}`),
@@ -15940,7 +16257,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
     ]);
     return;
   }
-  log3("\u2713 LiteLLM database ready.");
+  log4("\u2713 LiteLLM database ready.");
 };
 
 // src/postgres/init-litellm-db.ts
@@ -17207,14 +17524,37 @@ ${setupResult.output || ""}`);
         model: config.processor.model ?? "mistral-ocr",
         ...config.attribution
       });
-      await new Promise((resolve4) => setTimeout(resolve4, Math.floor(Math.random() * 4e3) + 1e3));
-      const base64Pdf = buffer.toString("base64");
-      const ocrResponse = await withRetry(async () => {
-        return await resolved.ocr({
-          type: "document_url",
-          document_url: "data:application/pdf;base64," + base64Pdf
-        }, { includeImageBase64: false });
-      }, 10);
+      const maxPagesPerChunk = config.processor.maxPagesPerChunk ?? 25;
+      const chunksDir = path.join(path.dirname(paths.json), "ocr_chunks");
+      const splitResult = await executePythonScript({
+        scriptPath: "ee/python/documents/processing/split_pdf.py",
+        args: [paths.source, chunksDir, "--chunk-size", String(maxPagesPerChunk)],
+        timeout: 5 * 60 * 1e3
+      });
+      const pdfChunks = JSON.parse(splitResult.stdout);
+      console.log(`[EXULU] PDF split into ${pdfChunks.length} chunk(s) for OCR (max ${maxPagesPerChunk} pages each)`);
+      const chunkLimit = pLimit(3);
+      const chunkResults = await Promise.all(
+        pdfChunks.map(
+          (chunk, i) => chunkLimit(async () => {
+            await new Promise((resolve4) => setImmediate(resolve4));
+            await new Promise((resolve4) => setTimeout(resolve4, Math.floor(Math.random() * 1e3) + 200));
+            console.log(`[EXULU] OCR chunk ${i + 1}/${pdfChunks.length}: pages ${chunk.start_page}\u2013${chunk.end_page - 1}`);
+            const chunkBuffer = await fs3.promises.readFile(chunk.path);
+            const chunkBase64 = chunkBuffer.toString("base64");
+            const chunkResponse = await withRetry(async () => {
+              return await resolved.ocr({
+                type: "document_url",
+                document_url: "data:application/pdf;base64," + chunkBase64
+              }, { includeImageBase64: false });
+            }, 10);
+            return { pages: chunkResponse.pages, offset: chunk.start_page };
+          })
+        )
+      );
+      const mergedPages = chunkResults.sort((a, b) => a.offset - b.offset).flatMap(
+        ({ pages, offset }) => pages.map((p) => ({ ...p, index: p.index + offset }))
+      );
       const parser = new LiteParse();
       const screenshots = await parser.screenshot(paths.source, void 0);
       await fs3.promises.mkdir(paths.images, { recursive: true });
@@ -17228,7 +17568,7 @@ ${setupResult.output || ""}`);
         );
         screenshot.imagePath = path.join(paths.images, `${screenshot.pageNum}.png`);
       }
-      json = ocrResponse.pages.map((page) => ({
+      json = mergedPages.map((page) => ({
         page: page.index + 1,
         content: page.markdown,
         image: screenshots.find((s) => s.pageNum === page.index + 1)?.imagePath,
@@ -17557,9 +17897,11 @@ export {
   ExuluProvider,
   ExuluPython,
   queues as ExuluQueues,
+  ExuluReadApi,
   ExuluReranker,
   ExuluTool,
   trajectoryRegistry as ExuluTrajectoryRegistry,
   ExuluVariables,
-  defaultChunker
+  defaultChunker,
+  postgresClient
 };