@exulu/backend 3.7.2 → 3.7.4

package/dist/index.cjs

@@ -3727,6 +3727,40 @@ Current item: ${JSON.stringify(summary)}`
   }
 });
 
+// ee/invoke-skills/artifact-filter.ts
+function isIgnoredArtifactPath(relativePath) {
+  return relativePath.split(/[\\/]+/).some((segment) => IGNORED_SEGMENTS.has(segment));
+}
+function capArtifacts(artifacts, max = DEFAULT_ARTIFACT_CAP) {
+  if (artifacts.length <= max) return { kept: artifacts, omitted: 0 };
+  return { kept: artifacts.slice(0, max), omitted: artifacts.length - max };
+}
+function needsDownload(localSize, remoteSize) {
+  return localSize === void 0 || localSize !== remoteSize;
+}
+var IGNORED_SEGMENTS, DEFAULT_ARTIFACT_CAP;
+var init_artifact_filter = __esm({
+  "ee/invoke-skills/artifact-filter.ts"() {
+    "use strict";
+    init_cjs_shims();
+    IGNORED_SEGMENTS = /* @__PURE__ */ new Set([
+      "venv",
+      ".venv",
+      "env",
+      "node_modules",
+      "__pycache__",
+      "site-packages",
+      "dist-packages",
+      ".cache",
+      ".git",
+      ".pytest_cache",
+      ".mypy_cache",
+      ".ipynb_checkpoints"
+    ]);
+    DEFAULT_ARTIFACT_CAP = 50;
+  }
+});
+
 // src/exulu/system-dependencies.ts
 async function getNpmGlobalRoot() {
   if (cachedNpmGlobalRoot !== void 0) return cachedNpmGlobalRoot;
@@ -3856,6 +3890,229 @@ var init_variable = __esm({
   }
 });
 
+// src/utils/python-setup.ts
+var python_setup_exports = {};
+__export(python_setup_exports, {
+  getPackageRoot: () => getPackageRoot,
+  getPythonSetupInstructions: () => getPythonSetupInstructions,
+  getPythonVenvPath: () => getPythonVenvPath,
+  isPythonEnvironmentSetup: () => isPythonEnvironmentSetup,
+  setupPythonEnvironment: () => setupPythonEnvironment,
+  validatePythonEnvironment: () => validatePythonEnvironment
+});
+function getPackageRoot() {
+  const currentFile = (0, import_url.fileURLToPath)(importMetaUrl);
+  let currentDir = (0, import_path.dirname)(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
+    if ((0, import_fs.existsSync)(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse((0, import_fs.readFileSync)(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = (0, import_path.resolve)(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  const fallback = (0, import_path.resolve)((0, import_path.dirname)((0, import_url.fileURLToPath)(importMetaUrl)), "../..");
+  return fallback;
+}
+function getSetupScriptPath(packageRoot) {
+  return (0, import_path.resolve)(packageRoot, "ee/python/setup.sh");
+}
+function getVenvPath(packageRoot) {
+  return (0, import_path.resolve)(packageRoot, "ee/python/.venv");
+}
+function getPythonVenvPath(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  return isPythonEnvironmentSetup(root) ? getVenvPath(root) : void 0;
+}
+function isPythonEnvironmentSetup(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
+  return (0, import_fs.existsSync)(venvPath) && (0, import_fs.existsSync)(pythonPath);
+}
+async function setupPythonEnvironment(options = {}) {
+  const {
+    packageRoot = getPackageRoot(),
+    force = false,
+    verbose = false,
+    timeout = 6e5
+    // 10 minutes
+  } = options;
+  if (!force && isPythonEnvironmentSetup(packageRoot)) {
+    if (verbose) {
+      console.log("\u2713 Python environment already set up");
+    }
+    return {
+      success: true,
+      message: "Python environment already exists",
+      alreadyExists: true
+    };
+  }
+  const setupScriptPath = getSetupScriptPath(packageRoot);
+  if (!(0, import_fs.existsSync)(setupScriptPath)) {
+    return {
+      success: false,
+      message: `Setup script not found at: ${setupScriptPath}`,
+      alreadyExists: false
+    };
+  }
+  try {
+    if (verbose) {
+      console.log("Setting up Python environment...");
+    }
+    const { stdout, stderr } = await execAsync2(`bash "${setupScriptPath}"`, {
+      cwd: packageRoot,
+      timeout,
+      env: {
+        ...process.env,
+        // Ensure script can write to the directory
+        PYTHONDONTWRITEBYTECODE: "1"
+      },
+      maxBuffer: 10 * 1024 * 1024
+      // 10MB buffer
+    });
+    const output = stdout + stderr;
+    const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
+    const pythonVersion = versionMatch ? versionMatch[1] : void 0;
+    if (verbose) {
+      console.log(output);
+    }
+    return {
+      success: true,
+      message: "Python environment set up successfully",
+      alreadyExists: false,
+      pythonVersion,
+      output
+    };
+  } catch (error) {
+    const errorOutput = error.stdout + error.stderr;
+    return {
+      success: false,
+      message: `Setup failed: ${error.message}`,
+      alreadyExists: false,
+      output: errorOutput
+    };
+  }
+}
+function getPythonSetupInstructions() {
+  return `
+Python environment not set up. Please run one of the following commands:
+
+Option 1 (Automatic):
+  import { setupPythonEnvironment } from '@exulu/backend';
+  await setupPythonEnvironment();
+
+Option 2 (Manual - for package consumers):
+  npx @exulu/backend setup-python
+
+Option 3 (Manual - for contributors):
+  npm run python:setup
+
+These commands will automatically create a Python virtual environment (.venv)
+in the @exulu/backend package and install all required dependencies.
+
+Requirements:
+  - Python 3.10 or higher must be installed
+  - pip must be available
+  - venv module must be available (for creating virtual environments)
+
+If Python dependencies are not installed, install them first, then run one of the commands above:
+  - macOS: brew install python@3.12
+  - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
+  - Alpine Linux: apk add python3 py3-pip python3-dev
+  - Windows: Download from https://www.python.org/downloads/
+
+Note: In Docker containers, ensure you install all three components:
+  Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
+  Alpine: apk add python3 py3-pip python3-dev
+`.trim();
+}
+async function validatePythonEnvironment(packageRoot, checkPackages = true) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
+  if (!(0, import_fs.existsSync)(venvPath)) {
+    return {
+      valid: false,
+      message: getPythonSetupInstructions()
+    };
+  }
+  if (!(0, import_fs.existsSync)(pythonPath)) {
+    return {
+      valid: false,
+      message: "Python virtual environment is corrupted. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+  try {
+    await execAsync2(`"${pythonPath}" --version`, { cwd: root });
+  } catch {
+    return {
+      valid: false,
+      message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+  if (checkPackages) {
+    const criticalPackages = ["docling", "transformers"];
+    const missingPackages = [];
+    for (const pkg of criticalPackages) {
+      try {
+        await execAsync2(`"${pythonPath}" -c "import ${pkg}"`, {
+          cwd: root,
+          timeout: 1e4
+          // 10 second timeout per import check
+        });
+      } catch {
+        missingPackages.push(pkg);
+      }
+    }
+    if (missingPackages.length > 0) {
+      return {
+        valid: false,
+        message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
+
+This usually happens when:
+1. The .venv folder was copied but dependencies were not installed
+2. The package was installed via npm but setup script was not run
+
+Please run:
+  await setupPythonEnvironment({ force: true })
+
+Or manually run the setup script:
+  bash ` + getSetupScriptPath(root)
+      };
+    }
+  }
+  return {
+    valid: true,
+    message: "Python environment is valid"
+  };
+}
+var import_child_process, import_util, import_path, import_fs, import_url, execAsync2;
+var init_python_setup = __esm({
+  "src/utils/python-setup.ts"() {
+    "use strict";
+    init_cjs_shims();
+    import_child_process = require("child_process");
+    import_util = require("util");
+    import_path = require("path");
+    import_fs = require("fs");
+    import_url = require("url");
+    execAsync2 = (0, import_util.promisify)(import_child_process.exec);
+  }
+});
+
 // ee/invoke-skills/create-sandbox.ts
 function probeSandboxSupport() {
   if (sandboxProbePromise) return sandboxProbePromise;
@@ -3941,7 +4198,7 @@ function resolveSessionPath(inputPath, sessionDir) {
   }
   return resolved;
 }
-async function restoreArtifactsFromS3(sessionDir, sessionId, userId, config) {
+async function restoreArtifactsFromS3(sessionDir, sessionId, userId, config, opts = {}) {
   const userPrefix = `user_${userId}/sessions/${sessionId}/`;
   let objects;
   try {
@@ -3961,7 +4218,17 @@ async function restoreArtifactsFromS3(sessionDir, sessionId, userId, config) {
     const idx = obj.key.indexOf(userPrefix);
     const relativePath = idx >= 0 ? obj.key.slice(idx + userPrefix.length) : "";
     if (!relativePath) continue;
+    if (isIgnoredArtifactPath(relativePath)) continue;
     const localPath = (0, import_node_path4.join)(sessionDir, relativePath);
+    if (opts.onlyMissing) {
+      let localSize;
+      try {
+        localSize = (await (0, import_promises.stat)(localPath)).size;
+      } catch {
+        localSize = void 0;
+      }
+      if (!needsDownload(localSize, obj.size)) continue;
+    }
     try {
       const bytes = await getS3ObjectBytes(obj.key, config);
       await (0, import_promises.mkdir)((0, import_node_path4.dirname)(localPath), { recursive: true });
@@ -3995,6 +4262,15 @@ async function downloadKeyIntoSandbox(opts) {
   await (0, import_promises.writeFile)(localPath, bytes);
   return { written: true, localPath };
 }
+async function resolvePythonVenvPath() {
+  try {
+    const { getPythonVenvPath: getPythonVenvPath2 } = await Promise.resolve().then(() => (init_python_setup(), python_setup_exports));
+    return getPythonVenvPath2();
+  } catch (err) {
+    console.warn("[SKILLS] Could not resolve the Python venv for the session sandbox; skill scripts fall back to the system python.", err);
+    return void 0;
+  }
+}
 async function createSessionSandbox(sessionId, skills, config, userId) {
   const cached = sandboxCache.get(sessionId);
   if (cached) {
@@ -4008,6 +4284,13 @@ async function createSessionSandbox(sessionId, skills, config, userId) {
       await downloadSkill(skill, skillsDirectory2, config);
       cached.installedSkills.set(skill.id, skill.current_version);
     }
+    if (userId && config.fileUploads) {
+      try {
+        await restoreArtifactsFromS3(cached.handle.sessionDir, sessionId, userId, config, { onlyMissing: true });
+      } catch (err) {
+        console.error(`[SKILLS] Failed to re-sync S3 session files for session ${sessionId}; continuing.`, err);
+      }
+    }
     return cached.handle;
   }
   const sessionDir = (0, import_node_path4.join)("/tmp", "exulu-sessions", sessionId);
@@ -4025,8 +4308,8 @@ async function createSessionSandbox(sessionId, skills, config, userId) {
       `[SKILLS] S3 artifact persistence disabled for session ${sessionId} (userId=${userId ?? "missing"}, fileUploads=${config.fileUploads ? "configured" : "missing"})`
     );
   }
-  if (userId && config.fileUploads && !dirExisted) {
-    await restoreArtifactsFromS3(sessionDir, sessionId, userId, config);
+  if (userId && config.fileUploads) {
+    await restoreArtifactsFromS3(sessionDir, sessionId, userId, config, { onlyMissing: dirExisted });
   }
   const probe = await probeSandboxSupport();
   const useDirectExec = !probe.canSandbox;
@@ -4061,6 +4344,7 @@ Probe error: ${probe.reason ?? "(no detail)"}`
     await import_sandbox_runtime.SandboxManager.initialize(baselineSandboxConfig);
   }
   const npmGlobalRoot = await getNpmGlobalRoot();
+  const pythonVenvPath = await resolvePythonVenvPath();
   const sessionSandboxConfig = {
     network: {
       allowedDomains: [],
@@ -4075,7 +4359,8 @@ Probe error: ${probe.reason ?? "(no detail)"}`
         // Allow Node to read globally-installed packages from inside
         // the sandbox. Without this, `require('docx')` fails with
         // EPERM even when NODE_PATH points the resolver here.
-        ...npmGlobalRoot ? [npmGlobalRoot] : []
+        ...npmGlobalRoot ? [npmGlobalRoot] : [],
+        ...pythonVenvPath ? [pythonVenvPath] : []
       ],
       allowWrite: [sessionDir],
       denyWrite: []
@@ -4093,7 +4378,8 @@ Probe error: ${probe.reason ?? "(no detail)"}`
   const sandboxedExecEnv = {
     ...configuredVariables,
     ...process.env,
-    ...npmGlobalRoot ? { NODE_PATH: npmGlobalRoot } : {}
+    ...npmGlobalRoot ? { NODE_PATH: npmGlobalRoot } : {},
+    ...pythonVenvPath ? { PATH: `${(0, import_node_path4.join)(pythonVenvPath, "bin")}:${process.env.PATH ?? ""}`, VIRTUAL_ENV: pythonVenvPath } : {}
   };
   const wrapIfNeeded = async (command) => {
     if (useDirectExec) return command;
@@ -4102,7 +4388,7 @@ Probe error: ${probe.reason ?? "(no detail)"}`
   const runWrapped = async (command) => {
     const wrapped = await wrapIfNeeded(command);
     try {
-      const { stdout, stderr } = await execAsync2(wrapped, {
+      const { stdout, stderr } = await execAsync3(wrapped, {
         maxBuffer: EXEC_MAX_BUFFER,
         shell: "/bin/bash",
         env: sandboxedExecEnv
@@ -4218,6 +4504,7 @@ Probe error: ${probe.reason ?? "(no detail)"}`
       for (const entry of entries) {
         const full = (0, import_node_path4.join)(dir, entry.name);
         if (full === skillsDir) continue;
+        if (isIgnoredArtifactPath((0, import_node_path4.relative)(sessionDir, full))) continue;
         if (entry.isDirectory()) {
           await walk(full);
         } else if (entry.isFile()) {
@@ -4313,19 +4600,24 @@ Probe error: ${probe.reason ?? "(no detail)"}`
         }
       }
       let stdout = result?.stdout ?? "";
-      const withUrls = artifacts.filter((a) => a.url);
+      const { kept, omitted } = capArtifacts(artifacts);
+      const withUrls = kept.filter((a) => a.url);
       if (withUrls.length > 0) {
         const lines = ["", "[exulu-artifacts]"];
         for (const a of withUrls) {
           lines.push(`  ${a.relativePath}: ${a.url}`);
         }
+        if (omitted > 0) {
+          lines.push(`  \u2026 ${omitted} more file(s) were created and mirrored but are not listed here.`);
+        }
         stdout = `${stdout}
 ${lines.join("\n")}`;
       }
       return {
         ...result,
         stdout,
-        artifacts
+        artifacts: kept,
+        ...omitted > 0 ? { artifactsOmitted: omitted } : {}
       };
     }
   });
@@ -4345,7 +4637,7 @@ ${lines.join("\n")}`;
   sandboxCache.set(sessionId, { handle, installedSkills });
   return handle;
 }
-var import_sandbox_runtime, import_promises, import_node_fs5, import_node_path4, import_node_child_process3, import_node_util2, import_bash_tool, import_ai2, import_zod6, import_crypto_js3, getAllExuluVariables, execAsync2, EXEC_MAX_BUFFER, sandboxProbePromise, SANDBOX_FALLBACK_INSTRUCTIONS, degradedModeLogged, sandboxCache;
+var import_sandbox_runtime, import_promises, import_node_fs5, import_node_path4, import_node_child_process3, import_node_util2, import_bash_tool, import_ai2, import_zod6, import_crypto_js3, getAllExuluVariables, execAsync3, EXEC_MAX_BUFFER, sandboxProbePromise, SANDBOX_FALLBACK_INSTRUCTIONS, degradedModeLogged, sandboxCache;
 var init_create_sandbox = __esm({
   "ee/invoke-skills/create-sandbox.ts"() {
     "use strict";
@@ -4357,6 +4649,7 @@ var init_create_sandbox = __esm({
     import_node_child_process3 = require("child_process");
     import_node_util2 = require("util");
     init_uppy();
+    init_artifact_filter();
     init_system_dependencies();
     import_bash_tool = require("bash-tool");
     import_ai2 = require("ai");
@@ -4390,7 +4683,7 @@ var init_create_sandbox = __esm({
       }
       return out;
     };
-    execAsync2 = (0, import_node_util2.promisify)(import_node_child_process3.exec);
+    execAsync3 = (0, import_node_util2.promisify)(import_node_child_process3.exec);
     EXEC_MAX_BUFFER = 32 * 1024 * 1024;
     SANDBOX_FALLBACK_INSTRUCTIONS = 'Skill sandboxing is running in DEGRADED mode: bwrap cannot create user namespaces on this host, so commands\nexecute directly. The container remains the isolation boundary and resolveSessionPath still scopes\nreadFile/writeFile to the session directory at the JS layer, but bash commands are NOT kernel-sandboxed.\n\nTo restore full sandboxing on Ubuntu 23.10+ / 24.04+ hosts (kernel 6.5+):\n  sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0\n  echo "kernel.apparmor_restrict_unprivileged_userns=0" | sudo tee /etc/sysctl.d/60-userns.conf\n  sudo sysctl --system\n\nOn Debian hosts where the same symptom appears:\n  sudo sysctl -w kernel.unprivileged_userns_clone=1\n\nSet EXULU_REQUIRE_SANDBOX=1 to fail startup instead of degrading.';
     degradedModeLogged = false;
@@ -4629,6 +4922,18 @@ var init_auth_tool_model_output = __esm({
   }
 });
 
+// src/exulu/session-files.ts
+function sessionFilePrefix(ownerId, sessionId, s3prefix) {
+  const general = s3prefix ? `${s3prefix.replace(/\/+$/, "")}/` : "";
+  return `${general}user_${ownerId}/sessions/${sessionId}/`;
+}
+var init_session_files = __esm({
+  "src/exulu/session-files.ts"() {
+    "use strict";
+    init_cjs_shims();
+  }
+});
+
 // src/templates/tools/session-file-read-tool.ts
 var import_zod7, DEFAULT_LIMIT, MAX_CONTENT_CHARS, createSessionFileReadTool;
 var init_session_file_read_tool = __esm({
@@ -4636,6 +4941,7 @@ var init_session_file_read_tool = __esm({
     "use strict";
     init_cjs_shims();
     import_zod7 = require("zod");
+    init_session_files();
     init_tool();
     init_uppy();
     DEFAULT_LIMIT = 250;
@@ -4643,7 +4949,8 @@ var init_session_file_read_tool = __esm({
     createSessionFileReadTool = ({
       sessionID,
       user,
-      exuluConfig
+      exuluConfig,
+      ownerId
     }) => {
       if (!sessionID || !exuluConfig?.fileUploads?.s3Bucket) return void 0;
       const readSessionFileExecute = async ({ filename, offset, limit }) => {
@@ -4654,8 +4961,7 @@ var init_session_file_read_tool = __esm({
           };
         }
         const uploads = exuluConfig.fileUploads;
-        const generalPrefix = uploads.s3prefix ? `${uploads.s3prefix.replace(/\/$/, "")}/` : "";
-        const key = `${generalPrefix}user_${user?.id ?? "api"}/sessions/${sessionID}/${safeName}`;
+        const key = `${sessionFilePrefix(ownerId ?? user?.id ?? "api", sessionID, uploads.s3prefix)}${safeName}`;
         try {
           const url = await getPresignedUrl(uploads.s3Bucket, key, exuluConfig);
           const res = await fetch(url);
@@ -4759,6 +5065,18 @@ var init_document_render_helpers = __esm({
 });
 
 // src/templates/tools/parse-document-tool.ts
+function looksLikeGarbledTextLayer(text) {
+  let control = 0;
+  let visible = 0;
+  for (const ch of text) {
+    const code = ch.charCodeAt(0);
+    if (code === 9 || code === 10 || code === 12 || code === 13 || code === 32) continue;
+    visible++;
+    if (code < 32 || code === 127) control++;
+  }
+  if (visible < 40) return false;
+  return control / visible > 0.01;
+}
 var import_zod8, import_node_path6, import_officeparser, DEFAULT_LIMIT2, MAX_CONTENT_CHARS2, MIN_CHARS_PER_PAGE, OFFICE_EXTENSIONS, pagesPattern, createParseDocumentTool;
 var init_parse_document_tool = __esm({
   "src/templates/tools/parse-document-tool.ts"() {
@@ -4770,6 +5088,7 @@ var init_parse_document_tool = __esm({
     init_tool();
     init_uppy();
     init_document_render_helpers();
+    init_session_files();
     DEFAULT_LIMIT2 = 250;
     MAX_CONTENT_CHARS2 = 16e3;
     MIN_CHARS_PER_PAGE = 20;
@@ -4789,7 +5108,8 @@ var init_parse_document_tool = __esm({
     createParseDocumentTool = ({
       sessionID,
       user,
-      exuluConfig
+      exuluConfig,
+      ownerId
     }) => {
       if (!sessionID || !exuluConfig?.fileUploads?.s3Bucket) return void 0;
       const parseDocumentExecute = async ({
@@ -4814,8 +5134,7 @@ var init_parse_document_tool = __esm({
           return { error: `The pages option is only supported for PDF files \u2014 "${ext}" documents are extracted whole.` };
         }
         const uploads = exuluConfig.fileUploads;
-        const generalPrefix = uploads.s3prefix ? `${uploads.s3prefix.replace(/\/$/, "")}/` : "";
-        const key = `${generalPrefix}user_${user?.id ?? "api"}/sessions/${sessionID}/${safeName}`;
+        const key = `${sessionFilePrefix(ownerId ?? user?.id ?? "api", sessionID, uploads.s3prefix)}${safeName}`;
         try {
           const url = await getPresignedUrl(uploads.s3Bucket, key, exuluConfig);
           const res = await fetch(url);
@@ -4830,6 +5149,11 @@ var init_parse_document_tool = __esm({
             const pageTexts = raw.replace(/\f$/, "").split("\f");
             totalPages = pageTexts.length;
             const nonWhitespace = raw.replace(/\s/g, "").length;
+            if (looksLikeGarbledTextLayer(raw)) {
+              return {
+                error: `"${safeName}" has a text layer that is unreadable (its font encoding maps glyphs to the wrong characters, so words and especially numbers come out wrong or vanish). Do not use extracted text from this file. Use view_document_page to read the pages visually, or suggest the user add the document to a knowledge base with a document processor for full OCR.`
+              };
+            }
             if (nonWhitespace < totalPages * MIN_CHARS_PER_PAGE) {
               return {
                 error: `"${safeName}" has no extractable text layer (likely a scan or image-based PDF). Use view_document_page to look at pages visually, or suggest the user add the document to a knowledge base with a document processor for full OCR.`
@@ -4928,7 +5252,7 @@ async function getPdfPreviewBytes(opts) {
         const bytes = await getS3ObjectBytes(sourceKey, config);
         await (0, import_promises3.writeFile)(inputPath, bytes);
         try {
-          await execAsync3(
+          await execAsync4(
             `soffice --headless --convert-to pdf "${inputPath}" --outdir "${CACHE_OUT}"`,
             { timeout: 6e4, maxBuffer: 16 * 1024 * 1024 }
           );
@@ -4954,7 +5278,7 @@ async function getPdfPreviewBytes(opts) {
   inFlight.set(safeEtag, promise);
   return promise;
 }
-var import_node_child_process5, import_node_fs6, import_promises3, import_node_path7, import_node_util4, execAsync3, CACHE_ROOT, CACHE_IN, CACHE_OUT, inFlight, PreviewRenderError;
+var import_node_child_process5, import_node_fs6, import_promises3, import_node_path7, import_node_util4, execAsync4, CACHE_ROOT, CACHE_IN, CACHE_OUT, inFlight, PreviewRenderError;
 var init_pdf_preview_cache = __esm({
   "src/sessions/pdf-preview-cache.ts"() {
     "use strict";
@@ -4965,7 +5289,7 @@ var init_pdf_preview_cache = __esm({
     import_node_path7 = require("path");
     import_node_util4 = require("util");
     init_uppy();
-    execAsync3 = (0, import_node_util4.promisify)(import_node_child_process5.exec);
+    execAsync4 = (0, import_node_util4.promisify)(import_node_child_process5.exec);
     CACHE_ROOT = "/tmp/exulu-pdf-cache";
     CACHE_IN = (0, import_node_path7.join)(CACHE_ROOT, "_in");
     CACHE_OUT = (0, import_node_path7.join)(CACHE_ROOT, "_out");
@@ -5165,6 +5489,7 @@ var init_view_document_page_tool = __esm({
     "use strict";
     init_cjs_shims();
     import_zod9 = require("zod");
+    init_session_files();
     import_node_path8 = require("path");
     init_tool();
     init_uppy();
@@ -5197,7 +5522,8 @@ var init_view_document_page_tool = __esm({
     createViewDocumentPageTool = ({
       sessionID,
       user,
-      exuluConfig
+      exuluConfig,
+      ownerId
     }) => {
       if (!sessionID || !exuluConfig?.fileUploads?.s3Bucket) return void 0;
       const viewDocumentPageExecute = async ({ filename, page, model }, options) => {
@@ -5230,8 +5556,7 @@ var init_view_document_page_tool = __esm({
           }
         }
         const uploads = exuluConfig.fileUploads;
-        const generalPrefix = uploads.s3prefix ? `${uploads.s3prefix.replace(/\/$/, "")}/` : "";
-        const key = `${generalPrefix}user_${user?.id ?? "api"}/sessions/${sessionID}/${safeName}`;
+        const key = `${sessionFilePrefix(ownerId ?? user?.id ?? "api", sessionID, uploads.s3prefix)}${safeName}`;
         const pageNumber = page ?? 1;
         try {
           let imageBytes;
@@ -5305,13 +5630,13 @@ var init_view_document_page_tool = __esm({
 });
 
 // src/exulu/audit/config.ts
-var import_os, import_path, normalizePrefix, hasAllS3Fields, resolveAuditConfig;
+var import_os, import_path2, normalizePrefix, hasAllS3Fields, resolveAuditConfig;
 var init_config = __esm({
   "src/exulu/audit/config.ts"() {
     "use strict";
     init_cjs_shims();
     import_os = __toESM(require("os"), 1);
-    import_path = __toESM(require("path"), 1);
+    import_path2 = __toESM(require("path"), 1);
     normalizePrefix = (p) => {
       const raw = (p ?? "audit").trim().replace(/^\/+|\/+$/g, "");
       return `${raw || "audit"}/`;
@@ -5343,7 +5668,7 @@ var init_config = __esm({
         retentionDays: a.retentionDays,
         manageLifecycle: a.manageLifecycle ?? !usingSharedFileUploadsBucket,
         usingSharedFileUploadsBucket,
-        spoolDir: a.spoolDir ?? import_path.default.join(import_os.default.tmpdir(), "exulu-audit-spool"),
+        spoolDir: a.spoolDir ?? import_path2.default.join(import_os.default.tmpdir(), "exulu-audit-spool"),
         flush: {
           maxRecords: a.flush?.maxRecords ?? 100,
           maxIntervalMs: a.flush?.maxIntervalMs ?? 5e3
@@ -5464,29 +5789,29 @@ ${JSON.stringify(config, null, 2)}`
 });
 
 // src/exulu/audit/sink.ts
-var import_crypto2, import_fs, import_path2, createFsSpoolStore, pad, AuditSink;
+var import_crypto2, import_fs2, import_path3, createFsSpoolStore, pad, AuditSink;
 var init_sink = __esm({
   "src/exulu/audit/sink.ts"() {
     "use strict";
     init_cjs_shims();
     import_crypto2 = require("crypto");
-    import_fs = require("fs");
-    import_path2 = __toESM(require("path"), 1);
+    import_fs2 = require("fs");
+    import_path3 = __toESM(require("path"), 1);
     createFsSpoolStore = (dir) => ({
       write: async (name, body) => {
-        await import_fs.promises.mkdir(dir, { recursive: true });
-        await import_fs.promises.writeFile(import_path2.default.join(dir, name), body, "utf8");
+        await import_fs2.promises.mkdir(dir, { recursive: true });
+        await import_fs2.promises.writeFile(import_path3.default.join(dir, name), body, "utf8");
       },
       list: async () => {
         try {
-          return (await import_fs.promises.readdir(dir)).filter((f) => f.endsWith(".ndjson"));
+          return (await import_fs2.promises.readdir(dir)).filter((f) => f.endsWith(".ndjson"));
         } catch {
           return [];
         }
       },
-      read: async (name) => import_fs.promises.readFile(import_path2.default.join(dir, name), "utf8"),
+      read: async (name) => import_fs2.promises.readFile(import_path3.default.join(dir, name), "utf8"),
       remove: async (name) => {
-        await import_fs.promises.rm(import_path2.default.join(dir, name), { force: true });
+        await import_fs2.promises.rm(import_path3.default.join(dir, name), { force: true });
       }
     });
     pad = (n) => String(n).padStart(2, "0");
@@ -6032,7 +6357,7 @@ var init_convert_exulu_tools_to_ai_sdk_tools = __esm({
       await Promise.all(promises2);
       return tool4;
     };
-    convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approvedTools, allExuluTools, configs, contexts, user, exuluConfig, sessionID, req, project, sessionItems, model, agent, memoryItems, contextWindow, disabledTools) => {
+    convertExuluToolsToAiSdkTools = async (currentTools, currentSkills, approvedTools, allExuluTools, configs, contexts, user, exuluConfig, sessionID, req, project, sessionItems, model, agent, memoryItems, contextWindow, disabledTools, sessionOwnerId) => {
       if (!currentTools) return {};
       if (!allExuluTools) {
         allExuluTools = [];
@@ -6049,7 +6374,7 @@ var init_convert_exulu_tools_to_ai_sdk_tools = __esm({
             sessionID,
             currentSkills || [],
             exuluConfig,
-            user?.id
+            sessionOwnerId ?? user?.id
           );
         } catch (err) {
           console.error(
@@ -6114,15 +6439,15 @@ var init_convert_exulu_tools_to_ai_sdk_tools = __esm({
           currentTools.push(sessionItemsRetrievalTool);
         }
       }
-      const sessionFileReadTool = createSessionFileReadTool({ sessionID, user, exuluConfig });
+      const sessionFileReadTool = createSessionFileReadTool({ sessionID, user, exuluConfig, ownerId: sessionOwnerId });
       if (sessionFileReadTool && !disabled.has(sessionFileReadTool.id)) {
         currentTools.push(sessionFileReadTool);
       }
-      const parseDocumentTool = createParseDocumentTool({ sessionID, user, exuluConfig });
+      const parseDocumentTool = createParseDocumentTool({ sessionID, user, exuluConfig, ownerId: sessionOwnerId });
       if (parseDocumentTool && !disabled.has(parseDocumentTool.id)) {
         currentTools.push(parseDocumentTool);
       }
-      const viewDocumentPageTool = createViewDocumentPageTool({ sessionID, user, exuluConfig });
+      const viewDocumentPageTool = createViewDocumentPageTool({ sessionID, user, exuluConfig, ownerId: sessionOwnerId });
       if (viewDocumentPageTool && !disabled.has(viewDocumentPageTool.id)) {
         currentTools.push(viewDocumentPageTool);
       }
@@ -16786,7 +17111,59 @@ function dropEmptyMessages(messages) {
   return kept.length === messages.length ? messages : kept;
 }
 
+// src/exulu/stored-file-url.ts
+init_cjs_shims();
+var stripTrailingSlash = (s) => s.replace(/\/+$/, "");
+function decodeKey(rawPath) {
+  const key = rawPath.replace(/^\/+/, "");
+  try {
+    return decodeURIComponent(key);
+  } catch {
+    return key;
+  }
+}
+function parseStoredFileUrl(url, store) {
+  if (!url || !store.bucket) return void 0;
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return void 0;
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return void 0;
+  if (store.endpoint) {
+    let endpoint;
+    try {
+      endpoint = new URL(store.endpoint);
+    } catch {
+      return void 0;
+    }
+    if (parsed.host !== endpoint.host) return void 0;
+    const base = stripTrailingSlash(endpoint.pathname);
+    const prefix = `${base}/${store.bucket}/`;
+    if (!parsed.pathname.startsWith(prefix)) return void 0;
+    const key2 = decodeKey(parsed.pathname.slice(prefix.length));
+    return key2 ? { bucket: store.bucket, key: key2 } : void 0;
+  }
+  if (!parsed.host.startsWith(`${store.bucket}.s3`) || !parsed.host.endsWith(".amazonaws.com")) {
+    return void 0;
+  }
+  const key = decodeKey(parsed.pathname);
+  return key ? { bucket: store.bucket, key } : void 0;
+}
+async function resolveFreshFileUrl(url, opts) {
+  const location = parseStoredFileUrl(url, opts);
+  if (!location) return url;
+  try {
+    return await opts.sign(location.bucket, location.key);
+  } catch (err) {
+    console.warn(`[EXULU] could not re-sign stored file URL for key "${location.key}":`, err);
+    return url;
+  }
+}
+
 // src/exulu/generate-stream.ts
+init_uppy();
 var import_ai6 = require("ai");
 
 // src/exulu/context-guard.ts
@@ -17104,7 +17481,12 @@ var processFilePartsInMessages = async (messages, offloadCtx) => {
             return part;
           }
           console.log(`[EXULU] Processing part`, part);
-          const { url } = part;
+          const uploads = offloadCtx.exuluConfig?.fileUploads;
+          const url = uploads?.s3Bucket ? await resolveFreshFileUrl(part.url, {
+            endpoint: uploads.s3endpoint,
+            bucket: uploads.s3Bucket,
+            sign: (bucket, key) => getPresignedUrl(bucket, key, offloadCtx.exuluConfig)
+          }) : part.url;
           const classified = classifyFilePart(part);
           console.log(`[EXULU] File part classified as ${classified.kind}: ${classified.filename}`);
           if (classified.kind === "image") {
@@ -17235,10 +17617,12 @@ var generateSync = async ({
   }
   let project;
   let sessionItems;
+  let sessionOwnerId;
   if (session) {
     const sessionData = await getSession({ sessionID: session });
     sessionItems = sessionData.session_items;
     project = sessionData.project;
+    sessionOwnerId = sessionData.user ?? void 0;
   }
   const model = languageModel;
   console.log("[EXULU] Model created for generating sync.");
@@ -17342,7 +17726,8 @@ var generateSync = async ({
     agent,
     memoryItems,
     contextWindow,
-    disabledTools
+    disabledTools,
+    sessionOwnerId
   );
   const agenticEntry = currentTools?.find((t) => t.id === "agentic_context_search");
   const agenticToolKey = agenticEntry ? sanitizeToolName(agenticEntry.name) : void 0;
@@ -17577,10 +17962,12 @@ var generateStream = async ({
   let previousMessagesContent = previousMessages || [];
   let project;
   let sessionItems;
+  let sessionOwnerId;
   if (session) {
     const sessionData = await getSession({ sessionID: session });
     project = sessionData.project;
     sessionItems = sessionData.session_items;
+    sessionOwnerId = sessionData.user ?? void 0;
     console.log("[EXULU] loading previous messages from session: " + session);
     const previousMessages2 = await getAgentMessages({
       session,
@@ -17760,7 +18147,8 @@ When a tool execution is not approved by the user, do not retry it unless explic
     agent,
     memoryItems,
     contextWindow,
-    disabledTools
+    disabledTools,
+    sessionOwnerId
   );
   console.log("[EXULU] Converted tools", Object.keys(tools));
   const includesContextSearchTool = currentTools?.some(
@@ -19920,7 +20308,7 @@ var mapRoutineRunRow = (row, routineById) => {
 
 // src/graphql/schemas/index.ts
 init_entitlements();
-var import_fs2 = require("fs");
+var import_fs3 = require("fs");
 
 // src/exulu/transcription/service.ts
 init_cjs_shims();
@@ -23926,7 +24314,7 @@ var import_utils5 = require("@apollo/utils.keyvaluecache");
 var import_body_parser = __toESM(require("body-parser"), 1);
 var import_crypto_js7 = require("crypto-js");
 var import_openai = require("openai");
-var import_fs3 = __toESM(require("fs"), 1);
+var import_fs4 = __toESM(require("fs"), 1);
 var import_node_crypto15 = require("crypto");
 var import_api2 = require("@opentelemetry/api");
 var import_jszip3 = __toESM(require("jszip"), 1);
@@ -25263,7 +25651,7 @@ var REQUEST_SIZE_LIMIT = "50mb";
 var getExuluVersionNumber = async () => {
   try {
     const path4 = process.cwd();
-    const packageJson = import_fs3.default.readFileSync(path4 + "/package.json", "utf8");
+    const packageJson = import_fs4.default.readFileSync(path4 + "/package.json", "utf8");
     const packageData = JSON.parse(packageJson);
     const exuluVersion = packageData.dependencies["@exulu/backend"];
     console.log(`[EXULU] Installed exulu-backend version: ${exuluVersion}`);
@@ -30244,210 +30632,7 @@ init_singleton();
 init_entitlements();
 init_system_dependencies();
 init_supervisor();
-
-// src/utils/python-setup.ts
-init_cjs_shims();
-var import_child_process = require("child_process");
-var import_util = require("util");
-var import_path3 = require("path");
-var import_fs4 = require("fs");
-var import_url = require("url");
-var execAsync4 = (0, import_util.promisify)(import_child_process.exec);
-function getPackageRoot() {
-  const currentFile = (0, import_url.fileURLToPath)(importMetaUrl);
-  let currentDir = (0, import_path3.dirname)(currentFile);
-  let attempts = 0;
-  const maxAttempts = 10;
-  while (attempts < maxAttempts) {
-    const packageJsonPath = (0, import_path3.join)(currentDir, "package.json");
-    if ((0, import_fs4.existsSync)(packageJsonPath)) {
-      try {
-        const packageJson = JSON.parse((0, import_fs4.readFileSync)(packageJsonPath, "utf-8"));
-        if (packageJson.name === "@exulu/backend") {
-          return currentDir;
-        }
-      } catch {
-      }
-    }
-    const parentDir = (0, import_path3.resolve)(currentDir, "..");
-    if (parentDir === currentDir) {
-      break;
-    }
-    currentDir = parentDir;
-    attempts++;
-  }
-  const fallback = (0, import_path3.resolve)((0, import_path3.dirname)((0, import_url.fileURLToPath)(importMetaUrl)), "../..");
-  return fallback;
-}
-function getSetupScriptPath(packageRoot) {
-  return (0, import_path3.resolve)(packageRoot, "ee/python/setup.sh");
-}
-function getVenvPath(packageRoot) {
-  return (0, import_path3.resolve)(packageRoot, "ee/python/.venv");
-}
-function isPythonEnvironmentSetup(packageRoot) {
-  const root = packageRoot ?? getPackageRoot();
-  const venvPath = getVenvPath(root);
-  const pythonPath = (0, import_path3.join)(venvPath, "bin", "python");
-  return (0, import_fs4.existsSync)(venvPath) && (0, import_fs4.existsSync)(pythonPath);
-}
-async function setupPythonEnvironment(options = {}) {
-  const {
-    packageRoot = getPackageRoot(),
-    force = false,
-    verbose = false,
-    timeout = 6e5
-    // 10 minutes
-  } = options;
-  if (!force && isPythonEnvironmentSetup(packageRoot)) {
-    if (verbose) {
-      console.log("\u2713 Python environment already set up");
-    }
-    return {
-      success: true,
-      message: "Python environment already exists",
-      alreadyExists: true
-    };
-  }
-  const setupScriptPath = getSetupScriptPath(packageRoot);
-  if (!(0, import_fs4.existsSync)(setupScriptPath)) {
-    return {
-      success: false,
-      message: `Setup script not found at: ${setupScriptPath}`,
-      alreadyExists: false
-    };
-  }
-  try {
-    if (verbose) {
-      console.log("Setting up Python environment...");
-    }
-    const { stdout, stderr } = await execAsync4(`bash "${setupScriptPath}"`, {
-      cwd: packageRoot,
-      timeout,
-      env: {
-        ...process.env,
-        // Ensure script can write to the directory
-        PYTHONDONTWRITEBYTECODE: "1"
-      },
-      maxBuffer: 10 * 1024 * 1024
-      // 10MB buffer
-    });
-    const output = stdout + stderr;
-    const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
-    const pythonVersion = versionMatch ? versionMatch[1] : void 0;
-    if (verbose) {
-      console.log(output);
-    }
-    return {
-      success: true,
-      message: "Python environment set up successfully",
-      alreadyExists: false,
-      pythonVersion,
-      output
-    };
-  } catch (error) {
-    const errorOutput = error.stdout + error.stderr;
-    return {
-      success: false,
-      message: `Setup failed: ${error.message}`,
-      alreadyExists: false,
-      output: errorOutput
-    };
-  }
-}
-function getPythonSetupInstructions() {
-  return `
-Python environment not set up. Please run one of the following commands:
-
-Option 1 (Automatic):
-  import { setupPythonEnvironment } from '@exulu/backend';
-  await setupPythonEnvironment();
-
-Option 2 (Manual - for package consumers):
-  npx @exulu/backend setup-python
-
-Option 3 (Manual - for contributors):
-  npm run python:setup
-
-These commands will automatically create a Python virtual environment (.venv)
-in the @exulu/backend package and install all required dependencies.
-
-Requirements:
-  - Python 3.10 or higher must be installed
-  - pip must be available
-  - venv module must be available (for creating virtual environments)
-
-If Python dependencies are not installed, install them first, then run one of the commands above:
-  - macOS: brew install python@3.12
-  - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
-  - Alpine Linux: apk add python3 py3-pip python3-dev
-  - Windows: Download from https://www.python.org/downloads/
-
-Note: In Docker containers, ensure you install all three components:
-  Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
-  Alpine: apk add python3 py3-pip python3-dev
-`.trim();
-}
-async function validatePythonEnvironment(packageRoot, checkPackages = true) {
-  const root = packageRoot ?? getPackageRoot();
-  const venvPath = getVenvPath(root);
-  const pythonPath = (0, import_path3.join)(venvPath, "bin", "python");
-  if (!(0, import_fs4.existsSync)(venvPath)) {
-    return {
-      valid: false,
-      message: getPythonSetupInstructions()
-    };
-  }
-  if (!(0, import_fs4.existsSync)(pythonPath)) {
-    return {
-      valid: false,
-      message: "Python virtual environment is corrupted. Please run:\n  await setupPythonEnvironment({ force: true })"
-    };
-  }
-  try {
-    await execAsync4(`"${pythonPath}" --version`, { cwd: root });
-  } catch {
-    return {
-      valid: false,
-      message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
-    };
-  }
-  if (checkPackages) {
-    const criticalPackages = ["docling", "transformers"];
-    const missingPackages = [];
-    for (const pkg of criticalPackages) {
-      try {
-        await execAsync4(`"${pythonPath}" -c "import ${pkg}"`, {
-          cwd: root,
-          timeout: 1e4
-          // 10 second timeout per import check
-        });
-      } catch {
-        missingPackages.push(pkg);
-      }
-    }
-    if (missingPackages.length > 0) {
-      return {
-        valid: false,
-        message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
-
-This usually happens when:
-1. The .venv folder was copied but dependencies were not installed
-2. The package was installed via npm but setup script was not run
-
-Please run:
-  await setupPythonEnvironment({ force: true })
-
-Or manually run the setup script:
-  bash ` + getSetupScriptPath(root)
-      };
-    }
-  }
-  return {
-    valid: true,
-    message: "Python environment is valid"
-  };
-}
+init_python_setup();
 
 // src/templates/contexts/index.ts
 init_cjs_shims();
@@ -32436,6 +32621,7 @@ var initLiteLLMDatabase = async (packageRoot) => {
 };
 
 // src/postgres/init-litellm-db.ts
+init_python_setup();
 var initLitellmDb = async () => {
   await initLiteLLMDatabase(getPackageRoot());
   console.log("[EXULU] LiteLLM database initialized.");
@@ -32970,6 +33156,9 @@ var MarkdownChunker = class {
   }
 };
 
+// src/index.ts
+init_python_setup();
+
 // ee/python/documents/processing/doc_processor.ts
 init_cjs_shims();
 var fs4 = __toESM(require("fs"), 1);
@@ -32992,6 +33181,7 @@ var import_util3 = require("util");
 var import_path4 = require("path");
 var import_fs5 = require("fs");
 var import_url2 = require("url");
+init_python_setup();
 var execAsync5 = (0, import_util3.promisify)(import_child_process2.exec);
 function getPackageRoot2() {
   const currentFile = (0, import_url2.fileURLToPath)(importMetaUrl);
@@ -33127,6 +33317,7 @@ ${command}`;
 }
 
 // ee/python/documents/processing/doc_processor.ts
+init_python_setup();
 var import_liteparse = require("@llamaindex/liteparse");
 
 // src/exulu/resolve-ocr.ts