@exulu/backend 1.49.1 → 1.49.2

package/dist/index.cjs

@@ -10139,6 +10139,7 @@ var ExuluProvider = class {
               {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
   
               IMPORTANT formatting rules:
+              - Do NOT reference just chunks like "Looking at chunk_index 5 and chunk_index 0 from the search result", always use the JSON format above.
               - Use the exact format shown above, all on ONE line
               - Do NOT use quotes around field names or values
               - Use the context ID from the tool result
@@ -15855,7 +15856,7 @@ async function setupPythonEnvironment(options = {}) {
 }
 function getPythonSetupInstructions() {
   return `
-Python environment not set up. Please run one of the following:
+Python environment not set up. Please run one of the following commands:
 
 Option 1 (Automatic):
   import { setupPythonEnvironment } from '@exulu/backend';
@@ -15867,17 +15868,26 @@ Option 2 (Manual - for package consumers):
 Option 3 (Manual - for contributors):
   npm run python:setup
 
+These commands will automatically create a Python virtual environment (.venv)
+in the @exulu/backend package and install all required dependencies.
+
 Requirements:
   - Python 3.10 or higher must be installed
   - pip must be available
+  - venv module must be available (for creating virtual environments)
 
-Installing Python:
+If Python dependencies are not installed, install them first, then run one of the commands above:
   - macOS: brew install python@3.12
-  - Ubuntu/Debian: sudo apt-get install python3.12
+  - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
+  - Alpine Linux: apk add python3 py3-pip python3-dev
   - Windows: Download from https://www.python.org/downloads/
+
+Note: In Docker containers, ensure you install all three components:
+  Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
+  Alpine: apk add python3 py3-pip python3-dev
 `.trim();
 }
-async function validatePythonEnvironment(packageRoot) {
+async function validatePythonEnvironment(packageRoot, checkPackages = true) {
   const root = packageRoot ?? getPackageRoot();
   const venvPath = getVenvPath(root);
   const pythonPath = (0, import_path.join)(venvPath, "bin", "python");
@@ -15895,21 +15905,50 @@ async function validatePythonEnvironment(packageRoot) {
   }
   try {
     await execAsync(`"${pythonPath}" --version`, { cwd: root });
-    return {
-      valid: true,
-      message: "Python environment is valid"
-    };
   } catch {
     return {
       valid: false,
       message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
     };
   }
+  if (checkPackages) {
+    const criticalPackages = ["docling", "transformers"];
+    const missingPackages = [];
+    for (const pkg of criticalPackages) {
+      try {
+        await execAsync(`"${pythonPath}" -c "import ${pkg}"`, {
+          cwd: root,
+          timeout: 1e4
+          // 10 second timeout per import check
+        });
+      } catch {
+        missingPackages.push(pkg);
+      }
+    }
+    if (missingPackages.length > 0) {
+      return {
+        valid: false,
+        message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
+
+This usually happens when:
+1. The .venv folder was copied but dependencies were not installed
+2. The package was installed via npm but setup script was not run
+
+Please run:
+  await setupPythonEnvironment({ force: true })
+
+Or manually run the setup script:
+  bash ` + getSetupScriptPath(root)
+      };
+    }
+  }
+  return {
+    valid: true,
+    message: "Python environment is valid"
+  };
 }
 
 // ee/python/documents/processing/doc_processor.ts
-var import_child_process3 = require("child_process");
-var import_util4 = require("util");
 var fs2 = __toESM(require("fs"), 1);
 var path = __toESM(require("path"), 1);
 var import_ai7 = require("ai");
@@ -16022,8 +16061,8 @@ async function executePythonScript(config) {
         // Ensure Python doesn't write bytecode files
         PYTHONDONTWRITEBYTECODE: "1"
       },
-      maxBuffer: 10 * 1024 * 1024
-      // 10MB buffer for output
+      maxBuffer: 50 * 1024 * 1024
+      // 50MB buffer for output (increased from 10MB)
     });
     return {
       stdout: stdout.trim(),
@@ -16035,8 +16074,25 @@ async function executePythonScript(config) {
     const stdout = error.stdout?.toString() ?? "";
     const stderr = error.stderr?.toString() ?? "";
     const exitCode = error.code ?? 1;
+    let errorMessage = `Python script execution failed: ${error.message}`;
+    if (stderr) {
+      errorMessage += `
+
+Stderr:
+${stderr}`;
+    }
+    if (stdout) {
+      errorMessage += `
+
+Stdout:
+${stdout}`;
+    }
+    errorMessage += `
+
+Command executed:
+${command}`;
     throw new PythonExecutionError(
-      `Python script execution failed: ${error.message}`,
+      errorMessage,
       stdout,
       stderr,
       exitCode
@@ -16294,6 +16350,25 @@ async function processPdf(buffer, paths, config, verbose = false) {
   try {
     let json;
     if (config?.docling) {
+      console.log(`[EXULU] Validating Python environment...`);
+      const validation = await validatePythonEnvironment(void 0, true);
+      if (!validation.valid) {
+        console.log(`[EXULU] Python environment not ready, setting up automatically...`);
+        console.log(`[EXULU] Reason: ${validation.message}`);
+        const setupResult = await setupPythonEnvironment({
+          verbose: true,
+          force: false
+          // Only setup if not already done
+        });
+        if (!setupResult.success) {
+          throw new Error(`Failed to setup Python environment: ${setupResult.message}
+
+${setupResult.output || ""}`);
+        }
+        console.log(`[EXULU] Python environment setup completed successfully`);
+      } else {
+        console.log(`[EXULU] Python environment is valid`);
+      }
       console.log(`[EXULU] Processing document with document_to_markdown.py`);
       const result = await executePythonScript({
         scriptPath: "ee/python/documents/processing/document_to_markdown.py",

package/dist/index.d.cts

@@ -1945,6 +1945,8 @@ interface PythonSetupResult {
 }
 /**
  * Check if Python environment is already set up
+ * Note: This only checks if the venv exists, not if packages are installed.
+ * Use validatePythonEnvironment() for a more thorough check.
  */
 declare function isPythonEnvironmentSetup(packageRoot?: string): boolean;
 /**
@@ -1979,9 +1981,10 @@ declare function getPythonSetupInstructions(): string;
  * Validate Python environment and provide helpful error messages
  *
  * @param packageRoot - Package root directory
+ * @param checkPackages - Whether to verify critical packages are installed
  * @returns Object with validation status and message
  */
-declare function validatePythonEnvironment(packageRoot?: string): Promise<{
+declare function validatePythonEnvironment(packageRoot?: string, checkPackages?: boolean): Promise<{
     valid: boolean;
     message: string;
 }>;

package/dist/index.d.ts

@@ -1945,6 +1945,8 @@ interface PythonSetupResult {
 }
 /**
  * Check if Python environment is already set up
+ * Note: This only checks if the venv exists, not if packages are installed.
+ * Use validatePythonEnvironment() for a more thorough check.
  */
 declare function isPythonEnvironmentSetup(packageRoot?: string): boolean;
 /**
@@ -1979,9 +1981,10 @@ declare function getPythonSetupInstructions(): string;
  * Validate Python environment and provide helpful error messages
  *
  * @param packageRoot - Package root directory
+ * @param checkPackages - Whether to verify critical packages are installed
  * @returns Object with validation status and message
  */
-declare function validatePythonEnvironment(packageRoot?: string): Promise<{
+declare function validatePythonEnvironment(packageRoot?: string, checkPackages?: boolean): Promise<{
     valid: boolean;
     message: string;
 }>;

package/dist/index.js

@@ -10105,6 +10105,7 @@ var ExuluProvider = class {
               {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
   
               IMPORTANT formatting rules:
+              - Do NOT reference just chunks like "Looking at chunk_index 5 and chunk_index 0 from the search result", always use the JSON format above.
               - Use the exact format shown above, all on ONE line
               - Do NOT use quotes around field names or values
               - Use the context ID from the tool result
@@ -15821,7 +15822,7 @@ async function setupPythonEnvironment(options = {}) {
 }
 function getPythonSetupInstructions() {
   return `
-Python environment not set up. Please run one of the following:
+Python environment not set up. Please run one of the following commands:
 
 Option 1 (Automatic):
   import { setupPythonEnvironment } from '@exulu/backend';
@@ -15833,17 +15834,26 @@ Option 2 (Manual - for package consumers):
 Option 3 (Manual - for contributors):
   npm run python:setup
 
+These commands will automatically create a Python virtual environment (.venv)
+in the @exulu/backend package and install all required dependencies.
+
 Requirements:
   - Python 3.10 or higher must be installed
   - pip must be available
+  - venv module must be available (for creating virtual environments)
 
-Installing Python:
+If Python dependencies are not installed, install them first, then run one of the commands above:
   - macOS: brew install python@3.12
-  - Ubuntu/Debian: sudo apt-get install python3.12
+  - Ubuntu/Debian: sudo apt-get install python3.12 python3-pip python3-venv
+  - Alpine Linux: apk add python3 py3-pip python3-dev
   - Windows: Download from https://www.python.org/downloads/
+
+Note: In Docker containers, ensure you install all three components:
+  Ubuntu/Debian: apt-get install -y python3 python3-pip python3-venv
+  Alpine: apk add python3 py3-pip python3-dev
 `.trim();
 }
-async function validatePythonEnvironment(packageRoot) {
+async function validatePythonEnvironment(packageRoot, checkPackages = true) {
   const root = packageRoot ?? getPackageRoot();
   const venvPath = getVenvPath(root);
   const pythonPath = join(venvPath, "bin", "python");
@@ -15861,21 +15871,50 @@ async function validatePythonEnvironment(packageRoot) {
   }
   try {
     await execAsync(`"${pythonPath}" --version`, { cwd: root });
-    return {
-      valid: true,
-      message: "Python environment is valid"
-    };
   } catch {
     return {
       valid: false,
       message: "Python executable is not working. Please run:\n  await setupPythonEnvironment({ force: true })"
     };
   }
+  if (checkPackages) {
+    const criticalPackages = ["docling", "transformers"];
+    const missingPackages = [];
+    for (const pkg of criticalPackages) {
+      try {
+        await execAsync(`"${pythonPath}" -c "import ${pkg}"`, {
+          cwd: root,
+          timeout: 1e4
+          // 10 second timeout per import check
+        });
+      } catch {
+        missingPackages.push(pkg);
+      }
+    }
+    if (missingPackages.length > 0) {
+      return {
+        valid: false,
+        message: `Python environment exists but required packages are not installed: ${missingPackages.join(", ")}
+
+This usually happens when:
+1. The .venv folder was copied but dependencies were not installed
+2. The package was installed via npm but setup script was not run
+
+Please run:
+  await setupPythonEnvironment({ force: true })
+
+Or manually run the setup script:
+  bash ` + getSetupScriptPath(root)
+      };
+    }
+  }
+  return {
+    valid: true,
+    message: "Python environment is valid"
+  };
 }
 
 // ee/python/documents/processing/doc_processor.ts
-import "child_process";
-import "util";
 import * as fs2 from "fs";
 import * as path from "path";
 import { generateText as generateText3, Output as Output2 } from "ai";
@@ -15988,8 +16027,8 @@ async function executePythonScript(config) {
         // Ensure Python doesn't write bytecode files
         PYTHONDONTWRITEBYTECODE: "1"
       },
-      maxBuffer: 10 * 1024 * 1024
-      // 10MB buffer for output
+      maxBuffer: 50 * 1024 * 1024
+      // 50MB buffer for output (increased from 10MB)
     });
     return {
       stdout: stdout.trim(),
@@ -16001,8 +16040,25 @@ async function executePythonScript(config) {
     const stdout = error.stdout?.toString() ?? "";
     const stderr = error.stderr?.toString() ?? "";
     const exitCode = error.code ?? 1;
+    let errorMessage = `Python script execution failed: ${error.message}`;
+    if (stderr) {
+      errorMessage += `
+
+Stderr:
+${stderr}`;
+    }
+    if (stdout) {
+      errorMessage += `
+
+Stdout:
+${stdout}`;
+    }
+    errorMessage += `
+
+Command executed:
+${command}`;
     throw new PythonExecutionError(
-      `Python script execution failed: ${error.message}`,
+      errorMessage,
       stdout,
       stderr,
       exitCode
@@ -16260,6 +16316,25 @@ async function processPdf(buffer, paths, config, verbose = false) {
   try {
     let json;
     if (config?.docling) {
+      console.log(`[EXULU] Validating Python environment...`);
+      const validation = await validatePythonEnvironment(void 0, true);
+      if (!validation.valid) {
+        console.log(`[EXULU] Python environment not ready, setting up automatically...`);
+        console.log(`[EXULU] Reason: ${validation.message}`);
+        const setupResult = await setupPythonEnvironment({
+          verbose: true,
+          force: false
+          // Only setup if not already done
+        });
+        if (!setupResult.success) {
+          throw new Error(`Failed to setup Python environment: ${setupResult.message}
+
+${setupResult.output || ""}`);
+        }
+        console.log(`[EXULU] Python environment setup completed successfully`);
+      } else {
+        console.log(`[EXULU] Python environment is valid`);
+      }
       console.log(`[EXULU] Processing document with document_to_markdown.py`);
       const result = await executePythonScript({
         scriptPath: "ee/python/documents/processing/document_to_markdown.py",

package/ee/python/documents/processing/doc_processor.ts

@@ -1,5 +1,3 @@
-import { exec } from 'child_process';
-import { promisify } from 'util';
 import * as fs from 'fs';
 import * as path from 'path';
 import { generateText, Output } from 'ai';
@@ -14,6 +12,7 @@ import WordExtractor from 'word-extractor';
 import { parseOfficeAsync } from "officeparser";
 import { checkLicense } from '@EE/entitlements';
 import { executePythonScript } from '@SRC/utils/python-executor';
+import { setupPythonEnvironment, validatePythonEnvironment } from '@SRC/utils/python-setup';
 
 type DocumentProcessorConfig = {
   vlm?: {
@@ -432,6 +431,29 @@ async function processPdf(
     // Call the PDF processor script
     if (config?.docling) {
 
+      // Validate Python environment and setup if needed
+      console.log(`[EXULU] Validating Python environment...`);
+      const validation = await validatePythonEnvironment(undefined, true);
+
+      if (!validation.valid) {
+        console.log(`[EXULU] Python environment not ready, setting up automatically...`);
+        console.log(`[EXULU] Reason: ${validation.message}`);
+
+        const setupResult = await setupPythonEnvironment({
+          verbose: true,
+          force: false, // Only setup if not already done
+        });
+
+        
+        if (!setupResult.success) {
+          throw new Error(`Failed to setup Python environment: ${setupResult.message}\n\n${setupResult.output || ''}`);
+        }
+
+        console.log(`[EXULU] Python environment setup completed successfully`);
+      } else {
+        console.log(`[EXULU] Python environment is valid`);
+      }
+
       console.log(`[EXULU] Processing document with document_to_markdown.py`);
 
       const result = await executePythonScript({

package/ee/python/setup.sh

@@ -94,21 +94,59 @@ print_success "Found Python $PYTHON_VERSION at $(which $PYTHON_CMD)"
 echo ""
 echo "Step 2: Checking pip installation..."
 if ! $PYTHON_CMD -m pip --version &> /dev/null; then
-    print_error "pip is not installed"
-    echo ""
-    echo "Please install pip:"
-    echo "  curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
-    echo "  $PYTHON_CMD get-pip.py"
-    echo ""
-    exit 1
+    print_warning "pip is not installed, attempting to bootstrap..."
+
+    # Try to use ensurepip to bootstrap pip
+    if $PYTHON_CMD -m ensurepip --version &> /dev/null; then
+        print_info "Using ensurepip to install pip..."
+        $PYTHON_CMD -m ensurepip --default-pip || {
+            print_error "Failed to bootstrap pip using ensurepip"
+            echo ""
+            echo "Please install pip manually:"
+            echo "  Ubuntu/Debian: sudo apt-get install python3-pip"
+            echo "  Alpine: apk add py3-pip"
+            echo "  Or using get-pip.py:"
+            echo "    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
+            echo "    $PYTHON_CMD get-pip.py"
+            echo ""
+            exit 1
+        }
+        print_success "pip bootstrapped successfully using ensurepip"
+    else
+        print_error "pip is not installed and ensurepip is not available"
+        echo ""
+        echo "Please install pip manually:"
+        echo "  Ubuntu/Debian: sudo apt-get install python3-pip"
+        echo "  Alpine: apk add py3-pip"
+        echo "  Or using get-pip.py:"
+        echo "    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
+        echo "    $PYTHON_CMD get-pip.py"
+        echo ""
+        exit 1
+    fi
 fi
 
 PIP_VERSION=$($PYTHON_CMD -m pip --version | awk '{print $2}')
 print_success "Found pip $PIP_VERSION"
 
-# Step 3: Create or update virtual environment
+# Step 3: Check for venv module
+echo ""
+echo "Step 3: Checking venv module..."
+if ! $PYTHON_CMD -m venv --help &> /dev/null; then
+    print_error "venv module is not available"
+    echo ""
+    echo "The venv module is required to create virtual environments."
+    echo "Please install it:"
+    echo "  Ubuntu/Debian: sudo apt-get install python3-venv"
+    echo "  Alpine: apk add python3-dev"
+    echo ""
+    exit 1
+fi
+print_success "venv module is available"
+
+# Step 4: Create or update virtual environment
 echo ""
-echo "Step 3: Setting up virtual environment..."
+echo "Step 4: Setting up virtual environment..."
 if [ -d "$VENV_DIR" ]; then
     # Check if virtual environment is valid
     if [ -f "$VENV_DIR/bin/activate" ] && [ -f "$VENV_DIR/bin/python" ]; then
@@ -117,18 +155,32 @@ if [ -d "$VENV_DIR" ]; then
     else
         print_warning "Virtual environment is corrupted, recreating..."
         rm -rf "$VENV_DIR"
-        $PYTHON_CMD -m venv "$VENV_DIR"
+        $PYTHON_CMD -m venv "$VENV_DIR" || {
+            print_error "Failed to create virtual environment"
+            echo ""
+            echo "This usually means the venv module is not properly installed."
+            echo "Try installing: sudo apt-get install python3-venv python3-dev"
+            echo ""
+            exit 1
+        }
         print_success "Virtual environment created"
     fi
 else
     print_info "Creating virtual environment at $VENV_DIR"
-    $PYTHON_CMD -m venv "$VENV_DIR"
+    $PYTHON_CMD -m venv "$VENV_DIR" || {
+        print_error "Failed to create virtual environment"
+        echo ""
+        echo "This usually means the venv module is not properly installed."
+        echo "Try installing: sudo apt-get install python3-venv python3-dev"
+        echo ""
+        exit 1
+    }
     print_success "Virtual environment created"
 fi
 
-# Step 4: Activate virtual environment and upgrade pip
+# Step 5: Activate virtual environment and upgrade pip
 echo ""
-echo "Step 4: Activating virtual environment..."
+echo "Step 5: Activating virtual environment..."
 source "$VENV_DIR/bin/activate"
 print_success "Virtual environment activated"
 
@@ -137,9 +189,9 @@ print_info "Upgrading pip in virtual environment..."
 pip install --upgrade pip > /dev/null 2>&1
 print_success "pip upgraded to latest version"
 
-# Step 5: Install dependencies
+# Step 6: Install dependencies
 echo ""
-echo "Step 5: Installing Python dependencies..."
+echo "Step 6: Installing Python dependencies..."
 if [ ! -f "$REQUIREMENTS_FILE" ]; then
     print_error "Requirements file not found: $REQUIREMENTS_FILE"
     exit 1
@@ -151,16 +203,16 @@ pip install -r "$REQUIREMENTS_FILE"
 
 print_success "All dependencies installed successfully"
 
-# Step 6: Validate installation
+# Step 7: Validate installation
 echo ""
-echo "Step 6: Validating installation..."
+echo "Step 7: Validating installation..."
 
 # Test critical imports
 print_info "Testing critical imports..."
 $PYTHON_CMD -c "import docling" 2>/dev/null && print_success "docling imported successfully" || print_error "Failed to import docling"
 $PYTHON_CMD -c "import transformers" 2>/dev/null && print_success "transformers imported successfully" || print_error "Failed to import transformers"
 
-# Step 7: Display summary
+# Step 8: Display summary
 echo ""
 echo -e "${GREEN}========================================${NC}"
 echo -e "${GREEN}  Setup Complete!${NC}"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.49.1",
+  "version": "1.49.2",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {