@exulu/backend 1.48.2 → 1.49.0

package/dist/index.js

@@ -877,7 +877,7 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
       if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
         if (attempt < maxRetries) {
           const backoffMs = Math.pow(2, attempt) * 1e3;
-          await new Promise((resolve) => setTimeout(resolve, backoffMs));
+          await new Promise((resolve3) => setTimeout(resolve3, backoffMs));
           s3Client = void 0;
           getS3Client(config);
           continue;
@@ -4607,7 +4607,7 @@ async function withRetry(generateFn, maxRetries = 3) {
       if (attempt === maxRetries) {
         throw error;
       }
-      await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 1e3));
+      await new Promise((resolve3) => setTimeout(resolve3, Math.pow(2, attempt) * 1e3));
     }
   }
   throw lastError;
@@ -5263,8 +5263,8 @@ async function* executeAgenticRetrieval({
   try {
     let finishResolver;
     let finishRejector;
-    const finishPromise = new Promise((resolve, reject) => {
-      finishResolver = resolve;
+    const finishPromise = new Promise((resolve3, reject) => {
+      finishResolver = resolve3;
       finishRejector = reject;
     });
     const timeoutId = setTimeout(() => {
@@ -7343,7 +7343,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
               } = await validateWorkflowPayload(data, providers);
               const retries = 3;
               let attempts = 0;
-              const promise = new Promise(async (resolve, reject) => {
+              const promise = new Promise(async (resolve3, reject) => {
                 while (attempts < retries) {
                   try {
                     const messages2 = await processUiMessagesFlow({
@@ -7358,7 +7358,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                       config,
                       variables: data.inputs
                     });
-                    resolve(messages2);
+                    resolve3(messages2);
                     break;
                   } catch (error) {
                     console.error(
@@ -7369,7 +7369,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                     if (attempts >= retries) {
                       reject(new Error(error instanceof Error ? error.message : String(error)));
                     }
-                    await new Promise((resolve2) => setTimeout((resolve3) => resolve3(true), 2e3));
+                    await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
                   }
                 }
               });
@@ -7419,7 +7419,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
               } = await validateEvalPayload(data, providers);
               const retries = 3;
               let attempts = 0;
-              const promise = new Promise(async (resolve, reject) => {
+              const promise = new Promise(async (resolve3, reject) => {
                 while (attempts < retries) {
                   try {
                     const messages2 = await processUiMessagesFlow({
@@ -7433,7 +7433,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                       tools,
                       config
                     });
-                    resolve(messages2);
+                    resolve3(messages2);
                     break;
                   } catch (error) {
                     console.error(
@@ -7444,7 +7444,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                     if (attempts >= retries) {
                       reject(new Error(error instanceof Error ? error.message : String(error)));
                     }
-                    await new Promise((resolve2) => setTimeout((resolve3) => resolve3(true), 2e3));
+                    await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
                   }
                 }
               });
@@ -7859,7 +7859,7 @@ var pollJobResult = async ({
     attempts++;
     const job = await Job.fromId(queue.queue, jobId);
     if (!job) {
-      await new Promise((resolve) => setTimeout((resolve2) => resolve2(true), 2e3));
+      await new Promise((resolve3) => setTimeout((resolve4) => resolve4(true), 2e3));
       continue;
     }
     const elapsedTime = Date.now() - startTime;
@@ -7889,7 +7889,7 @@ var pollJobResult = async ({
       console.log(`[EXULU] eval function ${job.id} result: ${result}`);
       break;
     }
-    await new Promise((resolve) => setTimeout((resolve2) => resolve2(true), 2e3));
+    await new Promise((resolve3) => setTimeout((resolve4) => resolve4(true), 2e3));
   }
   return result;
 };
@@ -7997,7 +7997,7 @@ var processUiMessagesFlow = async ({
       label: agent.name,
       trigger: "agent"
     };
-    messageHistory = await new Promise(async (resolve, reject) => {
+    messageHistory = await new Promise(async (resolve3, reject) => {
       const startTime = Date.now();
       try {
         const result = await provider.generateStream({
@@ -8074,7 +8074,7 @@ var processUiMessagesFlow = async ({
                 })
               ] : []
             ]);
-            resolve({
+            resolve3({
               messages,
               metadata: {
                 tokens: {
@@ -8822,7 +8822,7 @@ type PageInfo {
         } = await validateWorkflowPayload(jobData, providers);
         const retries = 3;
         let attempts = 0;
-        const promise = new Promise(async (resolve, reject) => {
+        const promise = new Promise(async (resolve3, reject) => {
           while (attempts < retries) {
             try {
               const messages2 = await processUiMessagesFlow({
@@ -8837,7 +8837,7 @@ type PageInfo {
                 config,
                 variables: args.variables
               });
-              resolve(messages2);
+              resolve3(messages2);
               break;
             } catch (error) {
               console.error(
@@ -8851,7 +8851,7 @@ type PageInfo {
               if (attempts >= retries) {
                 reject(error instanceof Error ? error : new Error(String(error)));
               }
-              await new Promise((resolve2) => setTimeout((resolve3) => resolve3(true), 2e3));
+              await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
             }
           }
         });
@@ -9104,10 +9104,10 @@ type PageInfo {
       contexts.map(async (context2) => {
         let processor = null;
         if (context2.processor) {
-          processor = await new Promise(async (resolve, reject) => {
+          processor = await new Promise(async (resolve3, reject) => {
             const config2 = context2.processor?.config;
             const queue = await config2?.queue;
-            resolve({
+            resolve3({
               name: context2.processor.name,
               description: context2.processor.description,
               queue: queue?.queue?.name || void 0,
@@ -9188,10 +9188,10 @@ type PageInfo {
     }
     let processor = null;
     if (data.processor) {
-      processor = await new Promise(async (resolve, reject) => {
+      processor = await new Promise(async (resolve3, reject) => {
         const config2 = data.processor?.config;
         const queue = await config2?.queue;
-        resolve({
+        resolve3({
           name: data.processor.name,
           description: data.processor.description,
           queue: queue?.queue?.name || void 0,
@@ -13086,7 +13086,7 @@ var internetSearchTool = new ExuluTool({
       } catch (error) {
         if (error instanceof Perplexity.RateLimitError && attempt < maxRetries - 1) {
           const delay = Math.pow(2, attempt) * 1e3 + Math.random() * 1e3;
-          await new Promise((resolve) => setTimeout(resolve, delay));
+          await new Promise((resolve3) => setTimeout(resolve3, delay));
           continue;
         }
         throw error;
@@ -15283,7 +15283,7 @@ var create = ({
 // src/index.ts
 import CryptoJS9 from "crypto-js";
 
-// ee/markdown.ts
+// ee/chunking/markdown.ts
 var extractPageTag = (text) => {
   let match;
   let lastPageNumber = void 0;
@@ -15710,9 +15710,172 @@ var MarkdownChunker = class {
   }
 };
 
-// ee/documents/processing/doc_processor.ts
+// src/utils/python-setup.ts
 import { exec } from "child_process";
 import { promisify } from "util";
+import { resolve, join, dirname } from "path";
+import { existsSync, readFileSync } from "fs";
+import { fileURLToPath } from "url";
+var execAsync = promisify(exec);
+function getPackageRoot() {
+  const currentFile = fileURLToPath(import.meta.url);
+  let currentDir = dirname(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = join(currentDir, "package.json");
+    if (existsSync(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = resolve(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  const fallback = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+  return fallback;
+}
+function getSetupScriptPath(packageRoot) {
+  return resolve(packageRoot, "ee/python/setup.sh");
+}
+function getVenvPath(packageRoot) {
+  return resolve(packageRoot, "ee/python/.venv");
+}
+function isPythonEnvironmentSetup(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = join(venvPath, "bin", "python");
+  return existsSync(venvPath) && existsSync(pythonPath);
+}
+async function setupPythonEnvironment(options = {}) {
+  const {
+    packageRoot = getPackageRoot(),
+    force = false,
+    verbose = false,
+    timeout = 6e5
+    // 10 minutes
+  } = options;
+  if (!force && isPythonEnvironmentSetup(packageRoot)) {
+    if (verbose) {
+      console.log("\u2713 Python environment already set up");
+    }
+    return {
+      success: true,
+      message: "Python environment already exists",
+      alreadyExists: true
+    };
+  }
+  const setupScriptPath = getSetupScriptPath(packageRoot);
+  if (!existsSync(setupScriptPath)) {
+    return {
+      success: false,
+      message: `Setup script not found at: ${setupScriptPath}`,
+      alreadyExists: false
+    };
+  }
+  try {
+    if (verbose) {
+      console.log("Setting up Python environment...");
+    }
+    const { stdout, stderr } = await execAsync(`bash "${setupScriptPath}"`, {
+      cwd: packageRoot,
+      timeout,
+      env: {
+        ...process.env,
+        // Ensure script can write to the directory
+        PYTHONDONTWRITEBYTECODE: "1"
+      },
+      maxBuffer: 10 * 1024 * 1024
+      // 10MB buffer
+    });
+    const output = stdout + stderr;
+    const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
+    const pythonVersion = versionMatch ? versionMatch[1] : void 0;
+    if (verbose) {
+      console.log(output);
+    }
+    return {
+      success: true,
+      message: "Python environment set up successfully",
+      alreadyExists: false,
+      pythonVersion,
+      output
+    };
+  } catch (error) {
+    const errorOutput = error.stdout + error.stderr;
+    return {
+      success: false,
+      message: `Setup failed: ${error.message}`,
+      alreadyExists: false,
+      output: errorOutput
+    };
+  }
+}
+function getPythonSetupInstructions() {
+  return `
+Python environment not set up. Please run one of the following:
+
+Option 1 (Automatic):
+  import { setupPythonEnvironment } from '@exulu/backend';
+  await setupPythonEnvironment();
+
+Option 2 (Manual - for package consumers):
+  npx @exulu/backend setup-python
+
+Option 3 (Manual - for contributors):
+  npm run python:setup
+
+Requirements:
+  - Python 3.10 or higher must be installed
+  - pip must be available
+
+Installing Python:
+  - macOS: brew install python@3.12
+  - Ubuntu/Debian: sudo apt-get install python3.12
+  - Windows: Download from https://www.python.org/downloads/
+`.trim();
+}
+async function validatePythonEnvironment(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = join(venvPath, "bin", "python");
+  if (!existsSync(venvPath)) {
+    return {
+      valid: false,
+      message: getPythonSetupInstructions()
+    };
+  }
+  if (!existsSync(pythonPath)) {
+    return {
+      valid: false,
+      message: "Python virtual environment is corrupted. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+  try {
+    await execAsync(`"${pythonPath}" --version`, { cwd: root });
+    return {
+      valid: true,
+      message: "Python environment is valid"
+    };
+  } catch {
+    return {
+      valid: false,
+      message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+}
+
+// ee/python/documents/processing/doc_processor.ts
+import "child_process";
+import "util";
 import * as fs2 from "fs";
 import * as path from "path";
 import { generateText as generateText3, Output as Output2 } from "ai";
@@ -15723,7 +15886,131 @@ import * as mammoth from "mammoth";
 import TurndownService from "turndown";
 import WordExtractor from "word-extractor";
 import { parseOfficeAsync as parseOfficeAsync2 } from "officeparser";
-var execAsync = promisify(exec);
+
+// src/utils/python-executor.ts
+import { exec as exec2 } from "child_process";
+import { promisify as promisify2 } from "util";
+import { resolve as resolve2, join as join2, dirname as dirname2 } from "path";
+import { existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
+import { fileURLToPath as fileURLToPath2 } from "url";
+var execAsync2 = promisify2(exec2);
+function getPackageRoot2() {
+  const currentFile = fileURLToPath2(import.meta.url);
+  let currentDir = dirname2(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = join2(currentDir, "package.json");
+    if (existsSync2(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse(readFileSync2(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = resolve2(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  return resolve2(dirname2(fileURLToPath2(import.meta.url)), "../..");
+}
+var PythonEnvironmentError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "PythonEnvironmentError";
+  }
+};
+var PythonExecutionError = class extends Error {
+  stdout;
+  stderr;
+  exitCode;
+  constructor(message, stdout, stderr, exitCode) {
+    super(message);
+    this.name = "PythonExecutionError";
+    this.stdout = stdout;
+    this.stderr = stderr;
+    this.exitCode = exitCode;
+  }
+};
+function getVenvPath2(packageRoot) {
+  return resolve2(packageRoot, "ee/python/.venv");
+}
+function getPythonExecutable(packageRoot) {
+  const venvPath = getVenvPath2(packageRoot);
+  return join2(venvPath, "bin", "python");
+}
+async function validatePythonEnvironmentForExecution(packageRoot) {
+  const validation = await validatePythonEnvironment(packageRoot);
+  if (!validation.valid) {
+    throw new PythonEnvironmentError(validation.message);
+  }
+}
+async function executePythonScript(config) {
+  const {
+    scriptPath,
+    args = [],
+    packageRoot = getPackageRoot2(),
+    cwd = process.cwd(),
+    timeout = 3e5,
+    // 5 minutes default
+    env = {},
+    validateEnvironment = true
+  } = config;
+  if (validateEnvironment) {
+    await validatePythonEnvironmentForExecution(packageRoot);
+  }
+  const resolvedScriptPath = resolve2(packageRoot, scriptPath);
+  if (!existsSync2(resolvedScriptPath)) {
+    throw new PythonExecutionError(
+      `Python script not found: ${resolvedScriptPath}`,
+      "",
+      "",
+      1
+    );
+  }
+  const pythonExecutable = getPythonExecutable(packageRoot);
+  const quotedArgs = args.map((arg) => {
+    return arg.includes(" ") ? `"${arg}"` : arg;
+  });
+  const command = `${pythonExecutable} "${resolvedScriptPath}" ${quotedArgs.join(" ")}`;
+  try {
+    const { stdout, stderr } = await execAsync2(command, {
+      cwd,
+      timeout,
+      env: {
+        ...process.env,
+        ...env,
+        // Ensure Python doesn't write bytecode files
+        PYTHONDONTWRITEBYTECODE: "1"
+      },
+      maxBuffer: 10 * 1024 * 1024
+      // 10MB buffer for output
+    });
+    return {
+      stdout: stdout.trim(),
+      stderr: stderr.trim(),
+      exitCode: 0,
+      success: true
+    };
+  } catch (error) {
+    const stdout = error.stdout?.toString() ?? "";
+    const stderr = error.stderr?.toString() ?? "";
+    const exitCode = error.code ?? 1;
+    throw new PythonExecutionError(
+      `Python script execution failed: ${error.message}`,
+      stdout,
+      stderr,
+      exitCode
+    );
+  }
+}
+
+// ee/python/documents/processing/doc_processor.ts
 async function processDocx(file) {
   const html = await mammoth.convertToHtml({ buffer: file });
   const turndownService = new TurndownService();
@@ -15973,16 +16260,24 @@ async function processPdf(buffer, paths, config, verbose = false) {
   try {
     let json;
     if (config?.docling) {
-      const script = `modal run modal_script.py --pdf-path "${paths.source}" --output "${paths.json}" --images-dir "${paths.images}"`;
-      console.log(`[EXULU] Running python script: ${script}`);
-      const { stderr } = await execAsync(
-        // todo replace python3 with the compiled executable
-        script,
-        { maxBuffer: 2e3 * 1024 * 1024 }
-        // 2000 MB buffer for large outputs
-      );
-      if (stderr) {
-        console.log("Processing info:", stderr.trim());
+      console.log(`[EXULU] Processing document with document_to_markdown.py`);
+      const result = await executePythonScript({
+        scriptPath: "ee/python/documents/processing/document_to_markdown.py",
+        args: [
+          paths.source,
+          "-o",
+          paths.json,
+          "--images-dir",
+          paths.images
+        ],
+        timeout: 30 * 60 * 1e3
+        // 30 minutes for large documents
+      });
+      if (result.stderr) {
+        console.log("Processing info:", result.stderr.trim());
+      }
+      if (!result.success) {
+        throw new Error(`Document processing failed: ${result.stderr}`);
       }
       const jsonContent = await fs2.promises.readFile(paths.json, "utf-8");
       json = JSON.parse(jsonContent);
@@ -15997,10 +16292,10 @@ async function processPdf(buffer, paths, config, verbose = false) {
         headings: []
       }];
     }
-    console.log(`
+    console.log(`[EXULU] 
 \u2713 Document processing completed successfully`);
-    console.log(`  Total pages: ${json.length}`);
-    console.log(`  Output file: ${paths.json}`);
+    console.log(`[EXULU] Total pages: ${json.length}`);
+    console.log(`[EXULU] Output file: ${paths.json}`);
     if (!config?.docling && config?.vlm?.model) {
       console.error("[EXULU] VLM validation is only supported when docling is enabled, skipping validation.");
     }
@@ -16213,6 +16508,12 @@ var ExuluChunkers = {
     rules: RecursiveRules
   }
 };
+var ExuluPython = {
+  setup: setupPythonEnvironment,
+  check: isPythonEnvironmentSetup,
+  validate: validatePythonEnvironment,
+  instructions: getPythonSetupInstructions
+};
 export {
   JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM,
   STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM,
@@ -16228,6 +16529,7 @@ export {
   ExuluJobs,
   ExuluOtel,
   ExuluProvider,
+  ExuluPython,
   queues as ExuluQueues,
   ExuluReranker,
   ExuluTool,

package/ee/{markdown.ts → chunking/markdown.ts}

@@ -1,5 +1,5 @@
-import { ExuluTokenizer } from "./tokenizer.ts";
-import { checkLicense } from "./entitlements";
+import { ExuluTokenizer } from "../tokenizer.ts";
+import { checkLicense } from "../entitlements";
 
 type CurrentHeaders = {
     h1?: {