@exulu/backend 1.48.2 → 1.49.0

package/ee/python/README.md

@@ -0,0 +1,295 @@
+# Exulu Python Integration
+
+This directory contains Python scripts and utilities used by the Exulu backend. The integration is designed to be seamless for TypeScript developers, requiring minimal Python knowledge.
+
+## Quick Start
+
+### First-Time Setup
+
+Run the setup command to configure your Python environment:
+
+```bash
+npm run python:setup
+```
+
+This will:
+- ✅ Validate Python 3.10+ is installed
+- ✅ Create a virtual environment at `ee/python/.venv`
+- ✅ Install all required dependencies
+- ✅ Verify the installation
+
+**That's it!** You're ready to use Python scripts from TypeScript.
+
+## Available npm Scripts
+
+| Command | Description |
+|---------|-------------|
+| `npm run python:setup` | Initial setup - creates venv and installs dependencies |
+| `npm run python:install` | Install/update Python dependencies |
+| `npm run python:validate` | Verify Python environment is working |
+| `npm run python:clean` | Clean Python cache and virtual environment |
+| `npm run python:rebuild` | Clean and rebuild Python environment from scratch |
+
+## Using Python Scripts from TypeScript
+
+The `python-executor` utility provides a type-safe interface for calling Python scripts:
+
+### Basic Example
+
+```typescript
+import { executePythonScript } from '../utils/python-executor';
+
+// Execute a Python script
+const result = await executePythonScript({
+  scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
+  args: [
+    '/path/to/document.pdf',
+    '-o', '/output/processed.json',
+    '--images-dir', '/output/images'
+  ]
+});
+
+if (result.success) {
+  console.log('Script executed successfully!');
+  console.log('Output:', result.stdout);
+} else {
+  console.error('Script failed:', result.stderr);
+}
+```
+
+### Simple Usage (throws on error)
+
+```typescript
+import { executePythonScriptSimple } from '../utils/python-executor';
+
+// Get stdout directly, throws on error
+const output = await executePythonScriptSimple({
+  scriptPath: 'ee/python/my_script.py',
+  args: ['arg1', 'arg2']
+});
+
+console.log('Output:', output);
+```
+
+### Advanced Configuration
+
+```typescript
+import { executePythonScript } from '../utils/python-executor';
+
+const result = await executePythonScript({
+  scriptPath: 'ee/python/my_script.py',
+  args: ['--verbose'],
+  cwd: process.cwd(),
+  timeout: 600000, // 10 minutes
+  env: {
+    CUSTOM_VAR: 'value'
+  },
+  validateEnvironment: true // default
+});
+```
+
+### Error Handling
+
+```typescript
+import {
+  executePythonScript,
+  PythonEnvironmentError,
+  PythonExecutionError
+} from '../utils/python-executor';
+
+try {
+  const result = await executePythonScript({
+    scriptPath: 'ee/python/my_script.py'
+  });
+
+  // Handle success
+  console.log(result.stdout);
+
+} catch (error) {
+  if (error instanceof PythonEnvironmentError) {
+    // Python environment not set up
+    console.error('Please run: npm run python:setup');
+  } else if (error instanceof PythonExecutionError) {
+    // Script execution failed
+    console.error('Script error:', error.stderr);
+    console.error('Exit code:', error.exitCode);
+  }
+}
+```
+
+### Check Environment Status
+
+```typescript
+import { isPythonEnvironmentReady } from '../utils/python-executor';
+
+if (await isPythonEnvironmentReady()) {
+  console.log('Python environment is ready!');
+} else {
+  console.log('Please run: npm run python:setup');
+}
+```
+
+## Directory Structure
+
+```
+ee/python/
+├── .venv/                    # Virtual environment (gitignored)
+├── requirements.txt          # Python dependencies
+├── setup.sh                  # Setup script
+├── README.md                 # This file
+└── documents/
+    └── processing/
+        ├── document_to_markdown.py
+        └── ...
+```
+
+## Adding New Python Scripts
+
+1. **Create your script** in an appropriate subdirectory under `ee/python/`
+
+2. **Add dependencies** to `requirements.txt`:
+   ```
+   your-package==1.2.3
+   ```
+
+3. **Update dependencies**:
+   ```bash
+   npm run python:install
+   ```
+
+4. **Use from TypeScript**:
+   ```typescript
+   import { executePythonScript } from '../utils/python-executor';
+
+   const result = await executePythonScript({
+     scriptPath: 'ee/python/your-module/your-script.py',
+     args: ['arg1', 'arg2']
+   });
+   ```
+
+## Troubleshooting
+
+### Python environment not found
+
+```bash
+npm run python:setup
+```
+
+### Dependencies not installing
+
+```bash
+npm run python:rebuild
+```
+
+### Script execution fails
+
+1. **Validate environment**:
+   ```bash
+   npm run python:validate
+   ```
+
+2. **Check Python version**:
+   ```bash
+   source ee/python/.venv/bin/activate
+   python --version  # Should be 3.10+
+   ```
+
+3. **Test manually**:
+   ```bash
+   source ee/python/.venv/bin/activate
+   python ee/python/your-script.py --help
+   ```
+
+### Import errors in Python scripts
+
+Make sure all required packages are in `requirements.txt` and run:
+```bash
+npm run python:install
+```
+
+## Requirements
+
+- **Python**: 3.10 or higher
+- **pip**: Latest version (auto-upgraded during setup)
+- **Operating System**: macOS, Linux, or Windows with WSL
+
+### Installing Python
+
+**macOS:**
+```bash
+brew install python@3.12
+```
+
+**Ubuntu/Debian:**
+```bash
+sudo apt-get update
+sudo apt-get install python3.12 python3.12-venv python3-pip
+```
+
+**Windows:**
+Download from [python.org](https://www.python.org/downloads/)
+
+## CI/CD Integration
+
+### GitHub Actions Example
+
+```yaml
+- name: Setup Python Environment
+  run: npm run python:setup
+
+- name: Validate Python Environment
+  run: npm run python:validate
+
+- name: Run Tests
+  run: npm test
+```
+
+### Caching Virtual Environment
+
+```yaml
+- uses: actions/cache@v3
+  with:
+    path: ee/python/.venv
+    key: ${{ runner.os }}-python-${{ hashFiles('ee/python/requirements.txt') }}
+```
+
+## Best Practices
+
+1. **Always use the TypeScript wrapper** - Don't call Python directly with `exec()`
+2. **Pin dependency versions** in `requirements.txt` for reproducibility
+3. **Handle errors gracefully** - Use try/catch with specific error types
+4. **Set appropriate timeouts** - Long-running scripts should have higher timeouts
+5. **Log errors properly** - Both stdout and stderr should be logged
+6. **Test environment setup** - Run `npm run python:validate` in CI
+
+## Available Python Modules
+
+### Document Processing
+
+**Script:** `documents/processing/document_to_markdown.py`
+
+Converts documents (PDF, DOCX, etc.) to structured JSON with markdown content.
+
+**Usage:**
+```typescript
+import { executePythonScript } from '../utils/python-executor';
+
+const result = await executePythonScript({
+  scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
+  args: [
+    '/path/to/document.pdf',
+    '-o', '/output/processed.json',
+    '--images-dir', '/output/images'
+  ]
+});
+
+const pages = JSON.parse(result.stdout);
+```
+
+## Support
+
+For issues or questions:
+1. Check this README
+2. Run `npm run python:validate`
+3. Check the [troubleshooting](#troubleshooting) section
+4. Open an issue with logs from `npm run python:validate`

package/ee/python/documents/processing/README.md

@@ -0,0 +1,155 @@
+# Document Processing Module
+
+Python scripts for processing documents (PDF, DOCX, etc.) into structured formats.
+
+## document_to_markdown.py
+
+Converts documents to structured JSON with page-separated markdown content and extracted images.
+
+### Features
+
+- ✅ PDF, DOCX, PPTX, and other document formats
+- ✅ Page-by-page content extraction
+- ✅ Hierarchical heading structure
+- ✅ Image extraction with high resolution
+- ✅ Table preservation in markdown format
+- ✅ Normalized whitespace handling
+
+### Usage from TypeScript
+
+```typescript
+import { executePythonScript } from '../../../../src/utils/python-executor';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+async function processDocument(documentPath: string, outputDir: string) {
+  try {
+    // Execute the document processor
+    const result = await executePythonScript({
+      scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
+      args: [
+        documentPath,
+        '-o', join(outputDir, 'processed.json'),
+        '--images-dir', join(outputDir, 'images')
+      ],
+      timeout: 600000 // 10 minutes for large documents
+    });
+
+    if (!result.success) {
+      throw new Error(`Processing failed: ${result.stderr}`);
+    }
+
+    // Read the processed JSON
+    const processedData = JSON.parse(
+      await readFile(join(outputDir, 'processed.json'), 'utf-8')
+    );
+
+    return processedData;
+  } catch (error) {
+    console.error('Document processing error:', error);
+    throw error;
+  }
+}
+
+// Example usage
+const pages = await processDocument(
+  '/path/to/document.pdf',
+  '/output/directory'
+);
+
+// Access page content
+pages.forEach((page, index) => {
+  console.log(`Page ${page.page}:`);
+  console.log(`Content: ${page.content.substring(0, 100)}...`);
+  console.log(`Image: ${page.image || 'None'}`);
+  console.log(`Headings:`, page.headings);
+});
+```
+
+### Command-Line Usage
+
+```bash
+# Activate virtual environment
+source ee/python/.venv/bin/activate
+
+# Process a document
+python ee/python/documents/processing/document_to_markdown.py \
+  /path/to/document.pdf \
+  -o /output/processed.json \
+  --images-dir /output/images
+```
+
+### Output Format
+
+The script outputs a JSON array with page objects:
+
+```json
+[
+  {
+    "page": 1,
+    "content": "# Document Title\n\nFirst paragraph...",
+    "image": "/output/images/page_1.png",
+    "headings": {
+      "Document Title": null
+    }
+  },
+  {
+    "page": 2,
+    "content": "## Section 1\n\nMore content...",
+    "image": "/output/images/page_2.png",
+    "headings": {
+      "Document Title": {
+        "Section 1": null
+      }
+    }
+  }
+]
+```
+
+### Arguments
+
+| Argument | Description | Required |
+|----------|-------------|----------|
+| `pdf_path` | Path to the document file | Yes |
+| `-o, --output` | Output path for JSON file | No (default: `<document_name>/processed.json`) |
+| `--images-dir` | Directory to save page images | No (default: `<output_dir>/images`) |
+
+### Configuration
+
+You can modify these constants in the script:
+
+```python
+IMAGE_RESOLUTION_SCALE = 2.0  # Image resolution multiplier
+```
+
+### Dependencies
+
+This script requires the following Python packages (installed via `npm run python:setup`):
+
+- `docling` - Document conversion
+- `docling-hierarchical-pdf` - Hierarchical heading processing
+- `transformers` - ML-based text processing
+- `PIL` - Image handling
+
+### Troubleshooting
+
+**Issue: ImportError for docling**
+```bash
+npm run python:install
+```
+
+**Issue: Script timeout for large documents**
+```typescript
+// Increase timeout
+const result = await executePythonScript({
+  scriptPath: '...',
+  timeout: 1200000 // 20 minutes
+});
+```
+
+**Issue: Low-quality images**
+
+Increase `IMAGE_RESOLUTION_SCALE` in the script:
+```python
+IMAGE_RESOLUTION_SCALE = 3.0  # Higher quality
+```

package/ee/{documents → python/documents}/processing/doc_processor.ts

@@ -13,8 +13,7 @@ import TurndownService from 'turndown';
 import WordExtractor from 'word-extractor';
 import { parseOfficeAsync } from "officeparser";
 import { checkLicense } from '@EE/entitlements';
-
-const execAsync = promisify(exec);
+import { executePythonScript } from '@SRC/utils/python-executor';
 
 type DocumentProcessorConfig = {
   vlm?: {
@@ -430,19 +429,28 @@ async function processPdf(
 ): Promise<ProcessorOutput> {
   try {
     let json: ProcessedDocument;
-    // Call the PDF processor executable
+    // Call the PDF processor script
     if (config?.docling) {
-      /* `python3 pdf_to_markdown.py "${paths.source}" --output "${paths.json}" --images-dir "${paths.images}"` */
-      const script = `modal run modal_script.py --pdf-path "${paths.source}" --output "${paths.json}" --images-dir "${paths.images}"`
-      console.log(`[EXULU] Running python script: ${script}`);
-      const { stderr } = await execAsync(
-        // todo replace python3 with the compiled executable
-        script,
-        { maxBuffer: 2000 * 1024 * 1024 } // 2000 MB buffer for large outputs
-      );
-      // Log stderr (processing info, not errors)
-      if (stderr) {
-        console.log('Processing info:', stderr.trim());
+
+      console.log(`[EXULU] Processing document with document_to_markdown.py`);
+
+      const result = await executePythonScript({
+        scriptPath: 'ee/python/documents/processing/document_to_markdown.py',
+        args: [
+          paths.source,
+          '-o', paths.json,
+          '--images-dir', paths.images
+        ],
+        timeout: 30 * 60 * 1000, // 30 minutes for large documents
+      });
+
+      // Log processing info from stderr
+      if (result.stderr) {
+        console.log('Processing info:', result.stderr.trim());
+      }
+
+      if (!result.success) {
+        throw new Error(`Document processing failed: ${result.stderr}`);
       }
 
       // Read the generated JSON file
@@ -460,9 +468,9 @@ async function processPdf(
       }];
     }
 
-    console.log(`\n✓ Document processing completed successfully`);
-    console.log(`  Total pages: ${json.length}`);
-    console.log(`  Output file: ${paths.json}`);
+    console.log(`[EXULU] \n✓ Document processing completed successfully`);
+    console.log(`[EXULU] Total pages: ${json.length}`);
+    console.log(`[EXULU] Output file: ${paths.json}`);
 
     if (!config?.docling && config?.vlm?.model) {
       console.error('[EXULU] VLM validation is only supported when docling is enabled, skipping validation.');

package/ee/{documents/processing/pdf_to_markdown.py → python/documents/processing/document_to_markdown.py}

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 """
-PDF to Markdown Converter using Docling
-Converts a PDF to JSON with page-separated markdown and images.
+Document to Markdown Converter using Docling
+Converts a document to JSON with page-separated markdown and images.
 
 Usage:
-    pdf_to_markdown.py <pdf_file_path> [-o OUTPUT_PATH] [--max-tokens MAX_TOKENS]
+    document_to_markdown.py <document_file_path> [-o OUTPUT_PATH] [--images-dir IMAGES_DIR]
 """
 
 import sys
@@ -346,13 +346,6 @@ def main():
         help='Directory to save page images (default: <pdf_name>_images/)'
     )
 
-    parser.add_argument(
-        '--max-tokens',
-        type=int,
-        dest='max_tokens',
-        help='Maximum number of tokens (currently not used, reserved for future use)'
-    )
-
     # Parse arguments
     args = parser.parse_args()