@aj-archipelago/cortex 1.3.67 → 1.4.0

package/config.js

@@ -423,6 +423,33 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
             },
+            "replicate-qwen-image": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/qwen/qwen-image/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-qwen-image-edit-plus": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/qwen/qwen-image-edit-plus/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-seedream-4": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/bytedance/seedream-4/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
             "azure-video-translate": {
                 "type": "AZURE-VIDEO-TRANSLATE",
                 "url": "https://eastus.api.cognitive.microsoft.com/videotranslation",

package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/__init__.py

@@ -0,0 +1,3 @@
+"""Azure Function entry point for document to PDF conversion."""
+import azure.functions as func
+from function_app import app

package/helper-apps/cortex-doc-to-pdf/DocToPdfFunction/function.json

@@ -0,0 +1,20 @@
+{
+    "scriptFile": "__init__.py",
+    "bindings": [
+        {
+            "authLevel": "function",
+            "type": "httpTrigger",
+            "direction": "in",
+            "name": "req",
+            "methods": [
+                "get",
+                "post"
+            ]
+        },
+        {
+            "type": "http",
+            "direction": "out",
+            "name": "$return"
+        }
+    ]
+}

package/helper-apps/cortex-doc-to-pdf/Dockerfile

@@ -0,0 +1,46 @@
+# Use official Azure Functions Python base image
+FROM mcr.microsoft.com/azure-functions/python:4-python3.11
+
+# Install LibreOffice + UNO Python bindings and unoconv for fast conversions
+RUN apt-get update && \
+    apt-get install -y \
+    libreoffice \
+    libreoffice-writer \
+    libreoffice-calc \
+    libreoffice-impress \
+    python3-uno \
+    unoconv \
+    fonts-liberation \
+    fonts-dejavu \
+    fonts-liberation2 \
+    fonts-noto \
+    fonts-noto-cjk \
+    fonts-noto-color-emoji \
+    curl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set LibreOffice to use headless mode by default
+ENV SAL_USE_VCLPLUGIN=svp
+
+# Set the working directory
+ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
+    AzureFunctionsJobHost__Logging__Console__IsEnabled=true \
+    FUNCTIONS_WORKER_RUNTIME=python
+
+# Copy requirements first for better layer caching
+WORKDIR /home/site/wwwroot
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy function app files
+COPY . .
+
+# Expose port for standalone server mode
+EXPOSE 8080
+
+# Start the application directly (no UNO listener)
+CMD ["python", "function_app.py"]

package/helper-apps/cortex-doc-to-pdf/README.md

@@ -0,0 +1,408 @@
+# Document to PDF Converter - Azure Container App
+
+A comprehensive document-to-PDF conversion service that runs as both an Azure Function and a standalone HTTP server. Built with LibreOffice, it supports **40+ document formats** including Word, Excel, PowerPoint, text files, HTML, and more.
+
+## Quick Start
+
+### Using Docker Compose (Easiest)
+
+```bash
+# Build and start the service
+docker compose up --build -d
+
+# Test the service
+curl "http://localhost:8080/convert?uri=https://file-examples.com/storage/fe783f04fc66761fd44fb46/2017/02/file-sample_100kB.doc" -o test.pdf
+
+# Check health
+curl http://localhost:8080/health
+
+# View logs
+docker compose logs -f
+
+# Stop the service
+docker compose down
+```
+
+## Supported Formats
+
+### Microsoft Office
+- **Word**: `.doc`, `.docx`, `.docm`, `.dot`, `.dotx`, `.dotm`
+- **Excel**: `.xls`, `.xlsx`, `.xlsm`, `.xlt`, `.xltx`, `.xltm`, `.csv`
+- **PowerPoint**: `.ppt`, `.pptx`, `.pptm`, `.pot`, `.potx`, `.potm`, `.pps`, `.ppsx`, `.ppsm`
+
+### OpenDocument
+- **Text**: `.odt`, `.ott`
+- **Spreadsheet**: `.ods`, `.ots`
+- **Presentation**: `.odp`, `.otp`
+- **Graphics**: `.odg`, `.otg`
+
+### Web & Text
+- **Web**: `.html`, `.htm`, `.xhtml`
+- **Text**: `.txt`, `.rtf`, `.xml`
+
+### Legacy Formats
+- WordPerfect, Lotus 1-2-3, dBase files, and more
+
+## API Usage
+
+### Endpoints
+
+**Standalone Server Mode** (Docker):
+- `GET/POST /convert` - Convert document to PDF
+- `GET /health` - Health check
+
+**Azure Function Mode**:
+- `GET/POST /api/convert` - Convert document to PDF
+
+### Convert Document
+
+```bash
+# GET request
+curl "http://localhost:8080/convert?uri=https://example.com/document.docx" -o output.pdf
+
+# POST request
+curl -X POST http://localhost:8080/convert \
+  -H "Content-Type: application/json" \
+  -d '{"uri": "https://example.com/document.xlsx"}' \
+  -o output.pdf
+```
+
+### Response
+
+**Success (200)**:
+- Content-Type: `application/pdf`
+- Body: PDF binary data
+
+**Error (400/500)**:
+```json
+{
+  "error": "Error type",
+  "details": "Error details"
+}
+```
+
+## Testing
+
+### Run Tests
+
+```bash
+# Run conversion tests
+python3 tests/test_conversion.py
+
+# Run streaming tests
+python3 tests/test_streaming.py
+
+# Or run in Docker
+docker compose run --rm doc-to-pdf python3 tests/test_streaming.py
+```
+
+Tests verify:
+- ✅ File upload streaming (memory efficient)
+- ✅ URI-based conversion
+- ✅ Streaming downloads
+- ✅ Concurrent conversions
+- ✅ Error handling
+- ✅ All document formats
+
+Sample files are in the `samples/` directory.
+
+## Deployment
+
+### Azure Container Apps (Recommended)
+
+```bash
+# Create resources
+az group create --name cortex-rg --location eastus
+az acr create --resource-group cortex-rg --name cortexregistry --sku Basic
+
+# Build and push
+az acr build --registry cortexregistry --image cortex-doc-to-pdf:latest .
+
+# Create container app environment
+az containerapp env create \
+  --name cortex-env \
+  --resource-group cortex-rg \
+  --location eastus
+
+# Deploy
+az containerapp create \
+  --name cortex-doc-to-pdf \
+  --resource-group cortex-rg \
+  --environment cortex-env \
+  --image cortexregistry.azurecr.io/cortex-doc-to-pdf:latest \
+  --target-port 8080 \
+  --ingress external \
+  --min-replicas 1 \
+  --max-replicas 10 \
+  --cpu 1.0 \
+  --memory 2.0Gi \
+  --env-vars PORT=8080 \
+  --command python function_app.py
+```
+
+### Azure Function App
+
+```bash
+# Create Function App
+az functionapp create \
+  --resource-group cortex-rg \
+  --name cortex-doc-to-pdf-func \
+  --storage-account cortexstorage \
+  --runtime python \
+  --runtime-version 3.11 \
+  --functions-version 4 \
+  --os-type Linux
+
+# Deploy
+func azure functionapp publish cortex-doc-to-pdf-func
+```
+
+## Local Development
+
+### With Docker (Recommended)
+
+```bash
+# Start with auto-reload
+docker compose up
+
+# Rebuild after code changes
+docker compose up --build
+```
+
+### Without Docker
+
+```bash
+# Install LibreOffice
+# macOS: brew install --cask libreoffice
+# Ubuntu: sudo apt-get install libreoffice
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Run server
+python function_app.py
+```
+
+## Configuration
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `PORT` | HTTP server port | `8080` |
+| `AzureWebJobsStorage` | Azure Storage (Function mode) | - |
+| `FUNCTIONS_WORKER_RUNTIME` | Azure Functions runtime | `python` |
+
+### Conversion Timeout
+
+Adjust in `host.json`:
+```json
+{
+  "functionTimeout": "00:10:00"
+}
+```
+
+## Performance
+
+### Typical Conversion Times
+- Simple documents: 1-3 seconds
+- Complex documents: 3-10 seconds
+- Large presentations: 10-30 seconds
+
+### Resource Requirements
+- **CPU**: 1.0 vCPU minimum
+- **Memory**: 2.0 GB minimum  
+- **Disk**: Ephemeral storage
+
+## Troubleshooting
+
+### Docker Build Issues
+
+**Platform mismatch error** (Apple Silicon Macs):
+```bash
+# Already configured in docker-compose.yml
+platform: linux/amd64
+```
+
+**Disk space error**:
+```bash
+docker system prune -a -f
+```
+
+### Conversion Failures
+
+**Check logs**:
+```bash
+docker compose logs -f
+```
+
+**Test with known-good file**:
+```bash
+curl "http://localhost:8080/convert?uri=https://file-examples.com/storage/fe783f04fc66761fd44fb46/2017/02/file-sample_100kB.doc" -o test.pdf
+```
+
+## Project Structure
+
+```
+cortex-doc-to-pdf/
+├── function_app.py          # Main entry point & routing
+├── request_handlers.py      # HTTP request/response handling
+├── document_converter.py    # Conversion business logic
+├── converter.py             # LibreOffice wrapper
+├── tests/                   # Test suite
+│   ├── test_streaming.py   # Streaming tests
+│   ├── test_conversion.py  # Conversion tests
+│   └── run_tests.sh        # Test runner
+├── samples/                 # Sample documents
+├── Dockerfile              # Container image
+├── docker-compose.yml      # Local orchestration
+├── requirements.txt        # Dependencies
+└── README.md              # Documentation
+```
+
+## Examples
+
+### URI-Based Conversion
+
+```bash
+# Word document via URI
+curl -X POST http://localhost:8080/convert \
+  -H "Content-Type: application/json" \
+  -d '{"uri": "https://example.com/document.docx"}' \
+  -o output.pdf
+
+# Or with GET (URL-encode special characters)
+curl "http://localhost:8080/convert?uri=https://example.com/file.xlsx" -o output.pdf
+```
+
+### File Upload (Recommended - Streaming)
+
+Upload local files directly - **no need for remote URI**. Files are streamed in 8KB chunks for memory efficiency.
+
+```bash
+# Upload file directly (streams upload & download)
+curl -X POST http://localhost:8080/convert \
+  -F "file=@document.xlsx" \
+  -o output.pdf
+
+# Works with any supported format
+curl -X POST http://localhost:8080/convert \
+  -F "file=@presentation.pptx" \
+  -o slides.pdf
+
+# Large files are handled efficiently (no memory bloat)
+curl -X POST http://localhost:8080/convert \
+  -F "file=@large-spreadsheet.xlsx" \
+  -o output.pdf
+```
+
+**Why file upload is recommended:**
+- ✅ **Streaming**: Chunked upload (8KB) and download
+- ✅ **Memory efficient**: Handles large files without RAM bloat
+- ✅ **Direct**: No need to host files on a server first
+- ✅ **Fast**: No download step required
+
+### JavaScript/Node.js Example
+
+**See `examples/` folder for complete working examples!**
+
+```javascript
+const axios = require('axios');
+const fs = require('fs');
+const FormData = require('form-data');
+
+// Upload file with streaming (both upload & download)
+async function convertToPDF(inputFile, outputFile) {
+    const form = new FormData();
+    form.append('file', fs.createReadStream(inputFile));
+    
+    const response = await axios({
+        method: 'POST',
+        url: 'http://localhost:8080/',  // Can use / or /convert
+        data: form,
+        headers: form.getHeaders(),
+        responseType: 'stream',  // Stream the response
+    });
+    
+    // Stream PDF to file
+    const writer = fs.createWriteStream(outputFile);
+    response.data.pipe(writer);
+    
+    return new Promise((resolve, reject) => {
+        writer.on('finish', resolve);
+        writer.on('error', reject);
+    });
+}
+
+// Usage
+await convertToPDF('document.docx', 'output.pdf');
+```
+
+**Features:**
+- ✅ Streaming upload (memory efficient)
+- ✅ Streaming download (direct to file)
+- ✅ Progress tracking support
+- ✅ Works in Node.js and Browser
+
+**Full example:** See `examples/nodejs-client.js`
+
+### Python Example
+
+```python
+import requests
+
+# Method 1: Upload local file (Recommended - Streaming)
+def convert_file_to_pdf(file_path):
+    with open(file_path, 'rb') as f:
+        files = {'file': f}
+        response = requests.post(
+            'http://localhost:8080/convert',
+            files=files
+        )
+    
+    if response.status_code == 200:
+        with open('output.pdf', 'wb') as f:
+            f.write(response.content)
+        print('✓ PDF created successfully')
+    else:
+        print(f'✗ Error: {response.json()}')
+
+# Method 2: Convert from URI
+def convert_url_to_pdf(document_url):
+    response = requests.post(
+        'http://localhost:8080/convert',
+        json={'uri': document_url}
+    )
+    
+    if response.status_code == 200:
+        with open('output.pdf', 'wb') as f:
+            f.write(response.content)
+        print('✓ PDF created successfully')
+    else:
+        print(f'✗ Error: {response.json()}')
+
+# Upload local file (streams efficiently)
+convert_file_to_pdf('document.docx')
+
+# Or convert from URL
+convert_url_to_pdf('https://example.com/document.docx')
+```
+
+## Security Considerations
+
+- **Authentication**: Add API keys or OAuth for production
+- **Rate Limiting**: Implement at API gateway level
+- **Input Validation**: URI format and allowlisting
+- **HTTPS**: Use reverse proxy or Azure ingress
+- **Resource Limits**: Configure memory and CPU limits
+
+## License
+
+This project is part of the Cortex project.
+
+## Support
+
+For issues or questions:
+1. Check the logs: `docker compose logs -f`
+2. Run tests: `./test_in_docker.sh`
+3. Verify LibreOffice: `docker compose run --rm --entrypoint soffice doc-to-pdf --version`

package/helper-apps/cortex-doc-to-pdf/converter.py

@@ -0,0 +1,157 @@
+import os
+import subprocess
+import logging
+import time
+from pathlib import Path
+from typing import Optional, List
+try:
+    import uno  # type: ignore
+    from com.sun.star.beans import PropertyValue  # type: ignore
+except Exception:
+    uno = None
+    PropertyValue = None
+
+class DocumentConverter:
+    """
+    Optimized document converter using LibreOffice with maximum performance settings.
+    Conversions typically take 2-4 seconds depending on file size and complexity.
+    """
+    
+    SUPPORTED_FORMATS = {
+        '.doc', '.docx', '.docm', '.dot', '.dotx', '.dotm',
+        '.xls', '.xlsx', '.xlsm', '.xlt', '.xltx', '.xltm', '.csv',
+        '.ppt', '.pptx', '.pptm', '.pot', '.potx', '.potm', '.pps', '.ppsx', '.ppsm',
+        '.odt', '.ott', '.ods', '.ots', '.odp', '.otp', '.odg', '.otg', '.odf',
+        '.txt', '.rtf',
+        '.html', '.htm', '.xhtml',
+        '.xml', '.wpd', '.wps',
+        '.wk1', '.wks', '.123', '.dif', '.dbf',
+    }
+    
+    def __init__(self, libreoffice_path: Optional[str] = None):
+        """Initialize with optimized LibreOffice settings."""
+        self.libreoffice_path = libreoffice_path or self._find_libreoffice()
+        if not self.libreoffice_path:
+            raise RuntimeError("LibreOffice not found!")
+        
+        logging.info(f"Using LibreOffice at: {self.libreoffice_path}")
+    
+    def _find_libreoffice(self) -> Optional[str]:
+        """Find LibreOffice installation."""
+        for cmd in ['soffice', 'libreoffice']:
+            try:
+                result = subprocess.run(
+                    ['which', cmd],
+                    capture_output=True,
+                    text=True,
+                    timeout=5
+                )
+                if result.returncode == 0 and result.stdout.strip():
+                    return result.stdout.strip()
+            except:
+                pass
+        
+        common_paths = [
+            '/usr/bin/libreoffice',
+            '/usr/bin/soffice',
+            '/Applications/LibreOffice.app/Contents/MacOS/soffice',
+        ]
+        
+        for path in common_paths:
+            if os.path.exists(path):
+                return path
+        
+        return None
+    
+    def is_supported_format(self, file_extension: str) -> bool:
+        """Check if file extension is supported."""
+        if not file_extension.startswith('.'):
+            file_extension = '.' + file_extension
+        return file_extension.lower() in self.SUPPORTED_FORMATS
+    
+    def get_supported_formats(self) -> List[str]:
+        """Get list of supported formats."""
+        return sorted(list(self.SUPPORTED_FORMATS))
+    
+    def convert_to_pdf(
+        self,
+        input_file: str,
+        output_dir: Optional[str] = None,
+        timeout: int = 30
+    ) -> Optional[str]:
+        """
+        Convert document to PDF with maximum speed optimizations.
+        
+        Performance: 2-4 seconds typical conversion time
+        - Small files (< 100KB): ~2.5s
+        - Medium files (100KB-1MB): ~3-4s  
+        - Large files (> 1MB): ~3-4s
+        
+        Note: LibreOffice has baseline processing overhead that cannot be eliminated.
+        This is already optimized with minimal flags and headless backend.
+        """
+        if not os.path.exists(input_file):
+            raise FileNotFoundError(f"Input file not found: {input_file}")
+        
+        input_path = Path(input_file)
+        if output_dir is None:
+            output_dir = str(input_path.parent)
+        
+        pdf_filename = input_path.stem + ".pdf"
+        output_path = os.path.join(output_dir, pdf_filename)
+        
+        start_time = time.time()
+        logging.info(f"Converting {input_file} to PDF...")
+        
+        # Direct soffice path only
+
+        try:
+            # Optimized LibreOffice command - minimal flags for maximum speed
+            cmd = [
+                self.libreoffice_path,
+                '--headless',
+                '--invisible',
+                '--nocrashreport',
+                '--nodefault',
+                '--nofirststartwizard',
+                '--nolockcheck',
+                '--nologo',
+                '--norestore',
+                '--convert-to', 'pdf:writer_pdf_Export',
+                '--outdir', output_dir,
+                input_file
+            ]
+
+            env = {**os.environ, 'HOME': output_dir, 'SAL_USE_VCLPLUGIN': 'svp'}
+
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+                env=env,
+                stdin=subprocess.DEVNULL
+            )
+            
+            elapsed = time.time() - start_time
+            
+            if result.returncode != 0:
+                logging.error(f"Conversion failed ({elapsed:.2f}s): {result.stderr}")
+                return None
+            
+            if not os.path.exists(output_path):
+                logging.error(f"PDF not created ({elapsed:.2f}s): {output_path}")
+                return None
+            
+            file_size = os.path.getsize(output_path)
+            logging.info(f"✓ Converted in {elapsed:.2f}s ({file_size/1024:.1f}KB): {output_path}")
+            return output_path
+            
+        except subprocess.TimeoutExpired:
+            elapsed = time.time() - start_time
+            logging.error(f"Conversion timed out after {elapsed:.2f}s")
+            return None
+        except Exception as e:
+            elapsed = time.time() - start_time
+            logging.error(f"Conversion error ({elapsed:.2f}s): {e}", exc_info=True)
+            return None

package/helper-apps/cortex-doc-to-pdf/docker-compose.yml

@@ -0,0 +1,23 @@
+services:
+  doc-to-pdf:
+    build:
+      context: .
+    platform: linux/amd64  # Force AMD64 for Azure compatibility (works on Mac via emulation)
+    ports:
+      - "8080:8080"
+    environment:
+      - PORT=8080
+      
+    tmpfs:
+      - /tmp:size=256m
+    volumes:
+      # Mount local code for development (optional)
+      - ./function_app.py:/home/site/wwwroot/function_app.py
+      - ./converter.py:/home/site/wwwroot/converter.py
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s