@exulu/backend 1.48.1 → 1.49.0

package/README.md

@@ -1,3 +1,5 @@
+**Exulu IMP** (Intelligence Management Platform) is an open-source TypeScript framework for building production-ready AI agent applications. It provides the backend infrastructure for managing agents, semantic search contexts, background job processing, and tool integrations.
+
 <a href="https://exulu.com/"><img width="100%" src="https://mintlify.s3.us-west-1.amazonaws.com/exulu/images/create_agents.png" alt="Exulu IMP - Create, deploy and manage AI agents" /></a>
 
 <br />
@@ -26,8 +28,6 @@
 > [!IMPORTANT]
 > ⭐ Star this repo to follow development and updates
 
-**Exulu IMP** (Intelligence Management Platform) is an open-source TypeScript framework for building production-ready AI agent applications. It provides the backend infrastructure for managing agents, semantic search contexts, background job processing, and tool integrations.
-
 <h3>✨ Why Exulu IMP?</h3>
 
 <ul>

package/dist/index.cjs

@@ -43,12 +43,20 @@ __export(index_exports, {
   ExuluEval: () => ExuluEval,
   ExuluJobs: () => ExuluJobs,
   ExuluOtel: () => ExuluOtel,
+  ExuluProvider: () => ExuluProvider,
+  ExuluPython: () => ExuluPython,
   ExuluQueues: () => queues,
   ExuluReranker: () => ExuluReranker,
   ExuluTool: () => ExuluTool,
   ExuluVariables: () => ExuluVariables
 });
 module.exports = __toCommonJS(index_exports);
+
+// node_modules/tsup/assets/cjs_shims.js
+var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
+var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
+
+// src/index.ts
 var import_config = require("dotenv/config");
 
 // src/redis/client.ts
@@ -910,7 +918,7 @@ var uploadFile = async (file, fileName, config, options = {}, user, customBucket
       if (error.name === "SignatureDoesNotMatch" || error.name === "InvalidAccessKeyId" || error.name === "AccessDenied") {
         if (attempt < maxRetries) {
           const backoffMs = Math.pow(2, attempt) * 1e3;
-          await new Promise((resolve) => setTimeout(resolve, backoffMs));
+          await new Promise((resolve3) => setTimeout(resolve3, backoffMs));
           s3Client = void 0;
           getS3Client(config);
           continue;
@@ -4640,7 +4648,7 @@ async function withRetry(generateFn, maxRetries = 3) {
       if (attempt === maxRetries) {
         throw error;
       }
-      await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 1e3));
+      await new Promise((resolve3) => setTimeout(resolve3, Math.pow(2, attempt) * 1e3));
     }
   }
   throw lastError;
@@ -5296,8 +5304,8 @@ async function* executeAgenticRetrieval({
   try {
     let finishResolver;
     let finishRejector;
-    const finishPromise = new Promise((resolve, reject) => {
-      finishResolver = resolve;
+    const finishPromise = new Promise((resolve3, reject) => {
+      finishResolver = resolve3;
       finishRejector = reject;
     });
     const timeoutId = setTimeout(() => {
@@ -7376,7 +7384,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
               } = await validateWorkflowPayload(data, providers);
               const retries = 3;
               let attempts = 0;
-              const promise = new Promise(async (resolve, reject) => {
+              const promise = new Promise(async (resolve3, reject) => {
                 while (attempts < retries) {
                   try {
                     const messages2 = await processUiMessagesFlow({
@@ -7391,7 +7399,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                       config,
                       variables: data.inputs
                     });
-                    resolve(messages2);
+                    resolve3(messages2);
                     break;
                   } catch (error) {
                     console.error(
@@ -7402,7 +7410,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                     if (attempts >= retries) {
                       reject(new Error(error instanceof Error ? error.message : String(error)));
                     }
-                    await new Promise((resolve2) => setTimeout((resolve3) => resolve3(true), 2e3));
+                    await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
                   }
                 }
               });
@@ -7452,7 +7460,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
               } = await validateEvalPayload(data, providers);
               const retries = 3;
               let attempts = 0;
-              const promise = new Promise(async (resolve, reject) => {
+              const promise = new Promise(async (resolve3, reject) => {
                 while (attempts < retries) {
                   try {
                     const messages2 = await processUiMessagesFlow({
@@ -7466,7 +7474,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                       tools,
                       config
                     });
-                    resolve(messages2);
+                    resolve3(messages2);
                     break;
                   } catch (error) {
                     console.error(
@@ -7477,7 +7485,7 @@ var createWorkers = async (providers, queues2, config, contexts, rerankers, eval
                     if (attempts >= retries) {
                       reject(new Error(error instanceof Error ? error.message : String(error)));
                     }
-                    await new Promise((resolve2) => setTimeout((resolve3) => resolve3(true), 2e3));
+                    await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
                   }
                 }
               });
@@ -7892,7 +7900,7 @@ var pollJobResult = async ({
     attempts++;
     const job = await import_bullmq3.Job.fromId(queue.queue, jobId);
     if (!job) {
-      await new Promise((resolve) => setTimeout((resolve2) => resolve2(true), 2e3));
+      await new Promise((resolve3) => setTimeout((resolve4) => resolve4(true), 2e3));
       continue;
     }
     const elapsedTime = Date.now() - startTime;
@@ -7922,7 +7930,7 @@ var pollJobResult = async ({
       console.log(`[EXULU] eval function ${job.id} result: ${result}`);
       break;
     }
-    await new Promise((resolve) => setTimeout((resolve2) => resolve2(true), 2e3));
+    await new Promise((resolve3) => setTimeout((resolve4) => resolve4(true), 2e3));
   }
   return result;
 };
@@ -8030,7 +8038,7 @@ var processUiMessagesFlow = async ({
       label: agent.name,
       trigger: "agent"
     };
-    messageHistory = await new Promise(async (resolve, reject) => {
+    messageHistory = await new Promise(async (resolve3, reject) => {
       const startTime = Date.now();
       try {
         const result = await provider.generateStream({
@@ -8107,7 +8115,7 @@ var processUiMessagesFlow = async ({
                 })
               ] : []
             ]);
-            resolve({
+            resolve3({
               messages,
               metadata: {
                 tokens: {
@@ -8855,7 +8863,7 @@ type PageInfo {
         } = await validateWorkflowPayload(jobData, providers);
         const retries = 3;
         let attempts = 0;
-        const promise = new Promise(async (resolve, reject) => {
+        const promise = new Promise(async (resolve3, reject) => {
           while (attempts < retries) {
             try {
               const messages2 = await processUiMessagesFlow({
@@ -8870,7 +8878,7 @@ type PageInfo {
                 config,
                 variables: args.variables
               });
-              resolve(messages2);
+              resolve3(messages2);
               break;
             } catch (error) {
               console.error(
@@ -8884,7 +8892,7 @@ type PageInfo {
               if (attempts >= retries) {
                 reject(error instanceof Error ? error : new Error(String(error)));
               }
-              await new Promise((resolve2) => setTimeout((resolve3) => resolve3(true), 2e3));
+              await new Promise((resolve4) => setTimeout((resolve5) => resolve5(true), 2e3));
             }
           }
         });
@@ -9137,10 +9145,10 @@ type PageInfo {
       contexts.map(async (context2) => {
         let processor = null;
         if (context2.processor) {
-          processor = await new Promise(async (resolve, reject) => {
+          processor = await new Promise(async (resolve3, reject) => {
             const config2 = context2.processor?.config;
             const queue = await config2?.queue;
-            resolve({
+            resolve3({
               name: context2.processor.name,
               description: context2.processor.description,
               queue: queue?.queue?.name || void 0,
@@ -9221,10 +9229,10 @@ type PageInfo {
     }
     let processor = null;
     if (data.processor) {
-      processor = await new Promise(async (resolve, reject) => {
+      processor = await new Promise(async (resolve3, reject) => {
         const config2 = data.processor?.config;
         const queue = await config2?.queue;
-        resolve({
+        resolve3({
           name: data.processor.name,
           description: data.processor.description,
           queue: queue?.queue?.name || void 0,
@@ -13112,7 +13120,7 @@ var internetSearchTool = new ExuluTool({
       } catch (error) {
         if (error instanceof import_perplexity_ai.default.RateLimitError && attempt < maxRetries - 1) {
           const delay = Math.pow(2, attempt) * 1e3 + Math.random() * 1e3;
-          await new Promise((resolve) => setTimeout(resolve, delay));
+          await new Promise((resolve3) => setTimeout(resolve3, delay));
           continue;
         }
         throw error;
@@ -15309,7 +15317,7 @@ var create = ({
 // src/index.ts
 var import_crypto_js9 = __toESM(require("crypto-js"), 1);
 
-// ee/markdown.ts
+// ee/chunking/markdown.ts
 var extractPageTag = (text) => {
   let match;
   let lastPageNumber = void 0;
@@ -15736,9 +15744,172 @@ var MarkdownChunker = class {
   }
 };
 
-// ee/documents/processing/doc_processor.ts
+// src/utils/python-setup.ts
 var import_child_process = require("child_process");
 var import_util2 = require("util");
+var import_path = require("path");
+var import_fs2 = require("fs");
+var import_url = require("url");
+var execAsync = (0, import_util2.promisify)(import_child_process.exec);
+function getPackageRoot() {
+  const currentFile = (0, import_url.fileURLToPath)(importMetaUrl);
+  let currentDir = (0, import_path.dirname)(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = (0, import_path.join)(currentDir, "package.json");
+    if ((0, import_fs2.existsSync)(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse((0, import_fs2.readFileSync)(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = (0, import_path.resolve)(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  const fallback = (0, import_path.resolve)((0, import_path.dirname)((0, import_url.fileURLToPath)(importMetaUrl)), "../..");
+  return fallback;
+}
+function getSetupScriptPath(packageRoot) {
+  return (0, import_path.resolve)(packageRoot, "ee/python/setup.sh");
+}
+function getVenvPath(packageRoot) {
+  return (0, import_path.resolve)(packageRoot, "ee/python/.venv");
+}
+function isPythonEnvironmentSetup(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
+  return (0, import_fs2.existsSync)(venvPath) && (0, import_fs2.existsSync)(pythonPath);
+}
+async function setupPythonEnvironment(options = {}) {
+  const {
+    packageRoot = getPackageRoot(),
+    force = false,
+    verbose = false,
+    timeout = 6e5
+    // 10 minutes
+  } = options;
+  if (!force && isPythonEnvironmentSetup(packageRoot)) {
+    if (verbose) {
+      console.log("\u2713 Python environment already set up");
+    }
+    return {
+      success: true,
+      message: "Python environment already exists",
+      alreadyExists: true
+    };
+  }
+  const setupScriptPath = getSetupScriptPath(packageRoot);
+  if (!(0, import_fs2.existsSync)(setupScriptPath)) {
+    return {
+      success: false,
+      message: `Setup script not found at: ${setupScriptPath}`,
+      alreadyExists: false
+    };
+  }
+  try {
+    if (verbose) {
+      console.log("Setting up Python environment...");
+    }
+    const { stdout, stderr } = await execAsync(`bash "${setupScriptPath}"`, {
+      cwd: packageRoot,
+      timeout,
+      env: {
+        ...process.env,
+        // Ensure script can write to the directory
+        PYTHONDONTWRITEBYTECODE: "1"
+      },
+      maxBuffer: 10 * 1024 * 1024
+      // 10MB buffer
+    });
+    const output = stdout + stderr;
+    const versionMatch = output.match(/Python (\d+\.\d+\.\d+)/);
+    const pythonVersion = versionMatch ? versionMatch[1] : void 0;
+    if (verbose) {
+      console.log(output);
+    }
+    return {
+      success: true,
+      message: "Python environment set up successfully",
+      alreadyExists: false,
+      pythonVersion,
+      output
+    };
+  } catch (error) {
+    const errorOutput = error.stdout + error.stderr;
+    return {
+      success: false,
+      message: `Setup failed: ${error.message}`,
+      alreadyExists: false,
+      output: errorOutput
+    };
+  }
+}
+function getPythonSetupInstructions() {
+  return `
+Python environment not set up. Please run one of the following:
+
+Option 1 (Automatic):
+  import { setupPythonEnvironment } from '@exulu/backend';
+  await setupPythonEnvironment();
+
+Option 2 (Manual - for package consumers):
+  npx @exulu/backend setup-python
+
+Option 3 (Manual - for contributors):
+  npm run python:setup
+
+Requirements:
+  - Python 3.10 or higher must be installed
+  - pip must be available
+
+Installing Python:
+  - macOS: brew install python@3.12
+  - Ubuntu/Debian: sudo apt-get install python3.12
+  - Windows: Download from https://www.python.org/downloads/
+`.trim();
+}
+async function validatePythonEnvironment(packageRoot) {
+  const root = packageRoot ?? getPackageRoot();
+  const venvPath = getVenvPath(root);
+  const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
+  if (!(0, import_fs2.existsSync)(venvPath)) {
+    return {
+      valid: false,
+      message: getPythonSetupInstructions()
+    };
+  }
+  if (!(0, import_fs2.existsSync)(pythonPath)) {
+    return {
+      valid: false,
+      message: "Python virtual environment is corrupted. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+  try {
+    await execAsync(`"${pythonPath}" --version`, { cwd: root });
+    return {
+      valid: true,
+      message: "Python environment is valid"
+    };
+  } catch {
+    return {
+      valid: false,
+      message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
+    };
+  }
+}
+
+// ee/python/documents/processing/doc_processor.ts
+var import_child_process3 = require("child_process");
+var import_util4 = require("util");
 var fs2 = __toESM(require("fs"), 1);
 var path = __toESM(require("path"), 1);
 var import_ai7 = require("ai");
@@ -15749,7 +15920,131 @@ var mammoth = __toESM(require("mammoth"), 1);
 var import_turndown = __toESM(require("turndown"), 1);
 var import_word_extractor = __toESM(require("word-extractor"), 1);
 var import_officeparser2 = require("officeparser");
-var execAsync = (0, import_util2.promisify)(import_child_process.exec);
+
+// src/utils/python-executor.ts
+var import_child_process2 = require("child_process");
+var import_util3 = require("util");
+var import_path2 = require("path");
+var import_fs3 = require("fs");
+var import_url2 = require("url");
+var execAsync2 = (0, import_util3.promisify)(import_child_process2.exec);
+function getPackageRoot2() {
+  const currentFile = (0, import_url2.fileURLToPath)(importMetaUrl);
+  let currentDir = (0, import_path2.dirname)(currentFile);
+  let attempts = 0;
+  const maxAttempts = 10;
+  while (attempts < maxAttempts) {
+    const packageJsonPath = (0, import_path2.join)(currentDir, "package.json");
+    if ((0, import_fs3.existsSync)(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse((0, import_fs3.readFileSync)(packageJsonPath, "utf-8"));
+        if (packageJson.name === "@exulu/backend") {
+          return currentDir;
+        }
+      } catch {
+      }
+    }
+    const parentDir = (0, import_path2.resolve)(currentDir, "..");
+    if (parentDir === currentDir) {
+      break;
+    }
+    currentDir = parentDir;
+    attempts++;
+  }
+  return (0, import_path2.resolve)((0, import_path2.dirname)((0, import_url2.fileURLToPath)(importMetaUrl)), "../..");
+}
+var PythonEnvironmentError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "PythonEnvironmentError";
+  }
+};
+var PythonExecutionError = class extends Error {
+  stdout;
+  stderr;
+  exitCode;
+  constructor(message, stdout, stderr, exitCode) {
+    super(message);
+    this.name = "PythonExecutionError";
+    this.stdout = stdout;
+    this.stderr = stderr;
+    this.exitCode = exitCode;
+  }
+};
+function getVenvPath2(packageRoot) {
+  return (0, import_path2.resolve)(packageRoot, "ee/python/.venv");
+}
+function getPythonExecutable(packageRoot) {
+  const venvPath = getVenvPath2(packageRoot);
+  return (0, import_path2.join)(venvPath, "bin", "python");
+}
+async function validatePythonEnvironmentForExecution(packageRoot) {
+  const validation = await validatePythonEnvironment(packageRoot);
+  if (!validation.valid) {
+    throw new PythonEnvironmentError(validation.message);
+  }
+}
+async function executePythonScript(config) {
+  const {
+    scriptPath,
+    args = [],
+    packageRoot = getPackageRoot2(),
+    cwd = process.cwd(),
+    timeout = 3e5,
+    // 5 minutes default
+    env = {},
+    validateEnvironment = true
+  } = config;
+  if (validateEnvironment) {
+    await validatePythonEnvironmentForExecution(packageRoot);
+  }
+  const resolvedScriptPath = (0, import_path2.resolve)(packageRoot, scriptPath);
+  if (!(0, import_fs3.existsSync)(resolvedScriptPath)) {
+    throw new PythonExecutionError(
+      `Python script not found: ${resolvedScriptPath}`,
+      "",
+      "",
+      1
+    );
+  }
+  const pythonExecutable = getPythonExecutable(packageRoot);
+  const quotedArgs = args.map((arg) => {
+    return arg.includes(" ") ? `"${arg}"` : arg;
+  });
+  const command = `${pythonExecutable} "${resolvedScriptPath}" ${quotedArgs.join(" ")}`;
+  try {
+    const { stdout, stderr } = await execAsync2(command, {
+      cwd,
+      timeout,
+      env: {
+        ...process.env,
+        ...env,
+        // Ensure Python doesn't write bytecode files
+        PYTHONDONTWRITEBYTECODE: "1"
+      },
+      maxBuffer: 10 * 1024 * 1024
+      // 10MB buffer for output
+    });
+    return {
+      stdout: stdout.trim(),
+      stderr: stderr.trim(),
+      exitCode: 0,
+      success: true
+    };
+  } catch (error) {
+    const stdout = error.stdout?.toString() ?? "";
+    const stderr = error.stderr?.toString() ?? "";
+    const exitCode = error.code ?? 1;
+    throw new PythonExecutionError(
+      `Python script execution failed: ${error.message}`,
+      stdout,
+      stderr,
+      exitCode
+    );
+  }
+}
+
+// ee/python/documents/processing/doc_processor.ts
 async function processDocx(file) {
   const html = await mammoth.convertToHtml({ buffer: file });
   const turndownService = new import_turndown.default();
@@ -15894,15 +16189,15 @@ Specific notes and guidelines:
   };
   return validation;
 }
-async function validateWithVLM(document, model, verbose = false, concurrency = 10) {
-  console.log(`[EXULU] Starting VLM validation for docling output, ${document.length} pages...`);
+async function validateWithVLM(document2, model, verbose = false, concurrency = 10) {
+  console.log(`[EXULU] Starting VLM validation for docling output, ${document2.length} pages...`);
   console.log(
     `[EXULU] Concurrency limit: ${concurrency}`
   );
   let validatedCount = 0;
   let correctedCount = 0;
   const limit = (0, import_p_limit.default)(concurrency);
-  const validationTasks = document.map(
+  const validationTasks = document2.map(
     (page) => limit(async () => {
       const imagePath = page.image;
       if (!imagePath) {
@@ -15947,7 +16242,7 @@ async function validateWithVLM(document, model, verbose = false, concurrency = 1
   console.log(`[EXULU] VLM validation complete:`);
   console.log(`[EXULU] Validated: ${validatedCount} chunks`);
   console.log(`[EXULU] Corrected: ${correctedCount} chunks`);
-  return document;
+  return document2;
 }
 async function processDocument(filePath, fileType, buffer, tempDir, config, verbose = false) {
   console.log("Starting document processing...");
@@ -15999,16 +16294,24 @@ async function processPdf(buffer, paths, config, verbose = false) {
   try {
     let json;
     if (config?.docling) {
-      const script = `modal run modal_script.py --pdf-path "${paths.source}" --output "${paths.json}" --images-dir "${paths.images}"`;
-      console.log(`[EXULU] Running python script: ${script}`);
-      const { stderr } = await execAsync(
-        // todo replace python3 with the compiled executable
-        script,
-        { maxBuffer: 2e3 * 1024 * 1024 }
-        // 2000 MB buffer for large outputs
-      );
-      if (stderr) {
-        console.log("Processing info:", stderr.trim());
+      console.log(`[EXULU] Processing document with document_to_markdown.py`);
+      const result = await executePythonScript({
+        scriptPath: "ee/python/documents/processing/document_to_markdown.py",
+        args: [
+          paths.source,
+          "-o",
+          paths.json,
+          "--images-dir",
+          paths.images
+        ],
+        timeout: 30 * 60 * 1e3
+        // 30 minutes for large documents
+      });
+      if (result.stderr) {
+        console.log("Processing info:", result.stderr.trim());
+      }
+      if (!result.success) {
+        throw new Error(`Document processing failed: ${result.stderr}`);
       }
       const jsonContent = await fs2.promises.readFile(paths.json, "utf-8");
       json = JSON.parse(jsonContent);
@@ -16023,10 +16326,10 @@ async function processPdf(buffer, paths, config, verbose = false) {
         headings: []
       }];
     }
-    console.log(`
+    console.log(`[EXULU] 
 \u2713 Document processing completed successfully`);
-    console.log(`  Total pages: ${json.length}`);
-    console.log(`  Output file: ${paths.json}`);
+    console.log(`[EXULU] Total pages: ${json.length}`);
+    console.log(`[EXULU] Output file: ${paths.json}`);
     if (!config?.docling && config?.vlm?.model) {
       console.error("[EXULU] VLM validation is only supported when docling is enabled, skipping validation.");
     }
@@ -16239,6 +16542,12 @@ var ExuluChunkers = {
     rules: RecursiveRules
   }
 };
+var ExuluPython = {
+  setup: setupPythonEnvironment,
+  check: isPythonEnvironmentSetup,
+  validate: validatePythonEnvironment,
+  instructions: getPythonSetupInstructions
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   EXULU_JOB_STATUS_ENUM,
@@ -16254,6 +16563,8 @@ var ExuluChunkers = {
   ExuluEval,
   ExuluJobs,
   ExuluOtel,
+  ExuluProvider,
+  ExuluPython,
   ExuluQueues,
   ExuluReranker,
   ExuluTool,