@aj-archipelago/cortex 1.3.67 → 1.4.0

package/helper-apps/cortex-doc-to-pdf/document_converter.py

@@ -0,0 +1,181 @@
+"""Document conversion logic - handles both URI and stream-based conversions."""
+
+import logging
+import tempfile
+import os
+import urllib.request
+import urllib.parse
+from pathlib import Path
+import shutil
+from converter import DocumentConverter
+
+
+async def convert_from_uri(uri: str) -> dict:
+    """
+    Convert a document from a URI to PDF.
+    
+    Args:
+        uri: URL of the document to convert
+        
+    Returns:
+        dict with 'success', 'data' (PDF bytes), 'filename', or 'error' keys
+    """
+    try:
+        converter = DocumentConverter()
+        
+        logging.info(f"Downloading document from: {uri}")
+        
+        # Parse filename from URI
+        parsed_url = urllib.parse.urlparse(uri)
+        filename = os.path.basename(parsed_url.path)
+        if not filename or '.' not in filename:
+            filename = "document.pdf"
+        
+        # Create temporary directory for processing
+        temp_dir = tempfile.mkdtemp()
+        
+        try:
+            # Download file
+            input_path = os.path.join(temp_dir, filename)
+            
+            try:
+                urllib.request.urlretrieve(uri, input_path)
+            except Exception as e:
+                logging.error(f"Failed to download file: {str(e)}")
+                return {
+                    "success": False,
+                    "error": "Failed to download document",
+                    "details": str(e)
+                }
+            
+            logging.info(f"Document downloaded to: {input_path}")
+            
+            # Check if file extension is supported
+            file_ext = Path(input_path).suffix.lower()
+            if not converter.is_supported_format(file_ext):
+                return {
+                    "success": False,
+                    "error": "Unsupported file format",
+                    "format": file_ext,
+                    "supported_formats": converter.get_supported_formats()
+                }
+            
+            # Convert to PDF
+            logging.info(f"Converting {file_ext} document to PDF...")
+            pdf_path = converter.convert_to_pdf(input_path, temp_dir)
+            
+            if not pdf_path or not os.path.exists(pdf_path):
+                return {
+                    "success": False,
+                    "error": "Conversion failed",
+                    "message": "The document could not be converted to PDF"
+                }
+            
+            # Read the PDF file
+            with open(pdf_path, 'rb') as pdf_file:
+                pdf_data = pdf_file.read()
+            
+            # Generate output filename
+            output_filename = Path(filename).stem + ".pdf"
+            
+            logging.info(f"Conversion successful. PDF size: {len(pdf_data)} bytes")
+            
+            return {
+                "success": True,
+                "data": pdf_data,
+                "filename": output_filename
+            }
+        
+        finally:
+            # Cleanup temp directory
+            try:
+                shutil.rmtree(temp_dir, ignore_errors=True)
+            except Exception as e:
+                logging.warning(f"Failed to cleanup temp directory: {e}")
+    
+    except Exception as e:
+        logging.error(f"Error during conversion: {str(e)}", exc_info=True)
+        return {
+            "success": False,
+            "error": "Conversion error",
+            "details": str(e)
+        }
+
+
+async def convert_from_stream(file_data: bytes, filename: str) -> dict:
+    """
+    Convert a document from uploaded file data to PDF.
+    
+    Args:
+        file_data: Binary file data
+        filename: Original filename (used to determine format)
+        
+    Returns:
+        dict with 'success', 'data' (PDF bytes), 'filename', or 'error' keys
+    """
+    try:
+        converter = DocumentConverter()
+        
+        logging.info(f"Converting uploaded file: {filename} ({len(file_data)} bytes)")
+        
+        # Create temporary directory for processing
+        temp_dir = tempfile.mkdtemp()
+        
+        try:
+            # Save uploaded file
+            input_path = os.path.join(temp_dir, filename)
+            with open(input_path, 'wb') as f:
+                f.write(file_data)
+            
+            logging.info(f"Saved uploaded file to: {input_path}")
+            
+            # Check if file extension is supported
+            file_ext = Path(input_path).suffix.lower()
+            if not converter.is_supported_format(file_ext):
+                return {
+                    "success": False,
+                    "error": "Unsupported file format",
+                    "format": file_ext,
+                    "supported_formats": converter.get_supported_formats()
+                }
+            
+            # Convert to PDF
+            logging.info(f"Converting {file_ext} document to PDF...")
+            pdf_path = converter.convert_to_pdf(input_path, temp_dir)
+            
+            if not pdf_path or not os.path.exists(pdf_path):
+                return {
+                    "success": False,
+                    "error": "Conversion failed",
+                    "message": "The document could not be converted to PDF"
+                }
+            
+            # Read the PDF file
+            with open(pdf_path, 'rb') as pdf_file:
+                pdf_data = pdf_file.read()
+            
+            # Generate output filename
+            output_filename = Path(filename).stem + ".pdf"
+            
+            logging.info(f"Conversion successful. PDF size: {len(pdf_data)} bytes")
+            
+            return {
+                "success": True,
+                "data": pdf_data,
+                "filename": output_filename
+            }
+        
+        finally:
+            # Cleanup temp directory
+            try:
+                shutil.rmtree(temp_dir, ignore_errors=True)
+            except Exception as e:
+                logging.warning(f"Failed to cleanup temp directory: {e}")
+    
+    except Exception as e:
+        logging.error(f"Error during conversion: {str(e)}", exc_info=True)
+        return {
+            "success": False,
+            "error": "Conversion error",
+            "details": str(e)
+        }

package/helper-apps/cortex-doc-to-pdf/examples/README.md

@@ -0,0 +1,252 @@
+# JavaScript Client Examples
+
+Complete examples showing how to use the Document to PDF Converter from JavaScript.
+
+## Node.js Client
+
+### Installation
+
+```bash
+cd examples
+npm install
+```
+
+### Usage
+
+```javascript
+const { convertFileToPDF, convertUriToURL, checkHealth } = require('./nodejs-client');
+
+// Check if service is running
+await checkHealth();
+
+// Convert a local file (streaming)
+await convertFileToURL(
+    'document.docx',
+    'output.pdf'
+);
+
+// Convert from URI
+await convertUriToURL(
+    'https://example.com/document.xlsx',
+    'output.pdf'
+);
+```
+
+### Run Example
+
+```bash
+npm run example
+```
+
+### Features
+
+- ✅ **Streaming upload** - Files are streamed in chunks (memory efficient)
+- ✅ **Streaming download** - PDF is streamed directly to file
+- ✅ **Progress tracking** - Monitor upload and download progress
+- ✅ **Batch conversion** - Convert multiple files efficiently
+- ✅ **Error handling** - Comprehensive error handling
+- ✅ **Large file support** - Handles files of any size
+
+### API Methods
+
+#### `convertFileToURL(inputPath, outputPath)`
+Convert a local file to PDF with streaming.
+
+```javascript
+await convertFileToURL('./document.docx', './output.pdf');
+```
+
+#### `convertUriToURL(uri, outputPath)`
+Convert a document from URL to PDF.
+
+```javascript
+await convertUriToURL('https://example.com/file.xlsx', './output.pdf');
+```
+
+#### `convertFileWithProgress(inputPath, outputPath)`
+Convert with upload/download progress tracking.
+
+```javascript
+await convertFileWithProgress('./large-file.docx', './output.pdf');
+// Shows: 📤 Upload: 75%
+//        📥 Downloaded: 523.45 KB
+```
+
+#### `convertMultipleFiles(files, outputDir)`
+Batch convert multiple files.
+
+```javascript
+await convertMultipleFiles(
+    ['file1.docx', 'file2.xlsx', 'file3.pptx'],
+    './output'
+);
+```
+
+#### `checkHealth()`
+Check if service is running.
+
+```javascript
+const isHealthy = await checkHealth();
+```
+
+## Advanced Examples
+
+### TypeScript Example
+
+```typescript
+import axios from 'axios';
+import * as fs from 'fs';
+import FormData from 'form-data';
+
+interface ConversionResult {
+    success: boolean;
+    outputPath: string;
+    error?: string;
+}
+
+async function convertFile(
+    inputPath: string,
+    outputPath: string
+): Promise<ConversionResult> {
+    try {
+        const form = new FormData();
+        form.append('file', fs.createReadStream(inputPath));
+        
+        const response = await axios.post('http://localhost:8080/', form, {
+            headers: form.getHeaders(),
+            responseType: 'stream',
+        });
+        
+        const writer = fs.createWriteStream(outputPath);
+        response.data.pipe(writer);
+        
+        return new Promise((resolve) => {
+            writer.on('finish', () => 
+                resolve({ success: true, outputPath })
+            );
+            writer.on('error', (error) => 
+                resolve({ success: false, outputPath, error: error.message })
+            );
+        });
+    } catch (error) {
+        return { 
+            success: false, 
+            outputPath, 
+            error: error.message 
+        };
+    }
+}
+```
+
+### React Example
+
+```jsx
+import React, { useState } from 'react';
+import axios from 'axios';
+
+function DocumentConverter() {
+    const [file, setFile] = useState(null);
+    const [progress, setProgress] = useState(0);
+    const [pdfUrl, setPdfUrl] = useState(null);
+    
+    const handleConvert = async () => {
+        if (!file) return;
+        
+        const formData = new FormData();
+        formData.append('file', file);
+        
+        try {
+            const response = await axios.post(
+                'http://localhost:8080/',
+                formData,
+                {
+                    responseType: 'blob',
+                    onUploadProgress: (e) => {
+                        setProgress(Math.round((e.loaded * 100) / e.total));
+                    }
+                }
+            );
+            
+            const url = URL.createObjectURL(response.data);
+            setPdfUrl(url);
+        } catch (error) {
+            console.error('Conversion failed:', error);
+        }
+    };
+    
+    return (
+        <div>
+            <input 
+                type="file" 
+                onChange={(e) => setFile(e.target.files[0])}
+            />
+            <button onClick={handleConvert}>Convert</button>
+            {progress > 0 && <progress value={progress} max="100" />}
+            {pdfUrl && <a href={pdfUrl} download="output.pdf">Download PDF</a>}
+        </div>
+    );
+}
+```
+
+## Service Endpoints
+
+Both Node.js and browser examples support these endpoints:
+
+- **POST /** - Convert file (root path)
+- **POST /convert** - Convert file (explicit path)
+- **GET /health** - Health check
+
+## Streaming Benefits
+
+### Upload Streaming
+- Files are sent in chunks (not loaded entirely in memory)
+- Supports large files without memory issues
+- Progress can be tracked
+
+### Download Streaming
+- PDF is streamed directly to destination
+- No intermediate buffering
+- Memory efficient
+- Can save directly to file (Node.js) or download (browser)
+
+## Error Handling
+
+```javascript
+try {
+    await convertFileToURL('document.docx', 'output.pdf');
+} catch (error) {
+    if (error.response) {
+        // Server responded with error
+        console.error('Server error:', error.response.data);
+    } else if (error.request) {
+        // No response received
+        console.error('No response from server');
+    } else {
+        // Other errors
+        console.error('Error:', error.message);
+    }
+}
+```
+
+## Performance Tips
+
+1. **Use streaming** - Always use `responseType: 'stream'` in Node.js
+2. **Batch processing** - Convert multiple files sequentially, not in parallel
+3. **Error recovery** - Implement retry logic for failed conversions
+4. **Progress feedback** - Show progress to users for better UX
+5. **File validation** - Validate file types before upload
+
+## Troubleshooting
+
+### CORS Issues (Browser)
+If you get CORS errors in the browser, the server needs to set appropriate headers. For development, you can use a proxy or run the service with CORS enabled.
+
+### Large Files
+The service handles large files efficiently with streaming. No special configuration needed!
+
+### Memory Usage
+Both upload and download use streaming, so memory usage stays low even with large files.
+
+## License
+
+Part of the Cortex project.

package/helper-apps/cortex-doc-to-pdf/examples/nodejs-client.js

@@ -0,0 +1,266 @@
+/**
+ * Node.js Client Example for Document to PDF Converter
+ * 
+ * This example shows how to:
+ * 1. Upload files with streaming
+ * 2. Download PDFs with streaming
+ * 3. Save to local file system
+ */
+
+const fs = require('fs');
+const path = require('path');
+const FormData = require('form-data');
+const axios = require('axios');
+
+const SERVICE_URL = process.env.DOC_TO_PDF_URL || 'http://localhost:8080';
+
+/**
+ * Method 1: Upload local file (Recommended - Streaming)
+ * Uses streams for memory efficiency
+ */
+async function convertFileToPDF(inputFilePath, outputFilePath) {
+    try {
+        console.log(`📤 Uploading: ${inputFilePath}`);
+
+        // Create form data with file stream
+        const form = new FormData();
+        const fileStream = fs.createReadStream(inputFilePath);
+        form.append('file', fileStream, path.basename(inputFilePath));
+
+        // Upload with streaming (both upload and download)
+        const response = await axios({
+            method: 'POST',
+            url: `${SERVICE_URL}/`,  // Can use / or /convert
+            data: form,
+            headers: {
+                ...form.getHeaders(),
+            },
+            responseType: 'stream',  // Stream the response
+            maxContentLength: Infinity,
+            maxBodyLength: Infinity,
+        });
+
+        // Stream the PDF response to file
+        const writer = fs.createWriteStream(outputFilePath);
+        response.data.pipe(writer);
+
+        return new Promise((resolve, reject) => {
+            writer.on('finish', () => {
+                console.log(`✅ Saved to: ${outputFilePath}`);
+                const stats = fs.statSync(outputFilePath);
+                console.log(`📄 Size: ${(stats.size / 1024).toFixed(2)} KB`);
+                resolve(outputFilePath);
+            });
+            writer.on('error', reject);
+        });
+
+    } catch (error) {
+        console.error('❌ Conversion failed:', error.message);
+        if (error.response) {
+            console.error('Response:', error.response.data);
+        }
+        throw error;
+    }
+}
+
+/**
+ * Method 2: Convert from URI
+ * Downloads from URL and converts to PDF
+ */
+async function convertUriToPDF(documentUri, outputFilePath) {
+    try {
+        console.log(`🌐 Converting from URI: ${documentUri}`);
+
+        const response = await axios({
+            method: 'POST',
+            url: `${SERVICE_URL}/convert`,
+            data: { uri: documentUri },
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            responseType: 'stream',  // Stream the response
+        });
+
+        // Stream to file
+        const writer = fs.createWriteStream(outputFilePath);
+        response.data.pipe(writer);
+
+        return new Promise((resolve, reject) => {
+            writer.on('finish', () => {
+                console.log(`✅ Saved to: ${outputFilePath}`);
+                resolve(outputFilePath);
+            });
+            writer.on('error', reject);
+        });
+
+    } catch (error) {
+        console.error('❌ Conversion failed:', error.message);
+        if (error.response) {
+            const chunks = [];
+            for await (const chunk of error.response.data) {
+                chunks.push(chunk);
+            }
+            console.error('Error:', JSON.parse(Buffer.concat(chunks).toString()));
+        }
+        throw error;
+    }
+}
+
+/**
+ * Method 3: Upload with progress tracking
+ * Shows upload and download progress
+ */
+async function convertFileWithProgress(inputFilePath, outputFilePath) {
+    try {
+        console.log(`📤 Uploading: ${inputFilePath}`);
+
+        const form = new FormData();
+        const fileStream = fs.createReadStream(inputFilePath);
+        const fileSize = fs.statSync(inputFilePath).size;
+
+        form.append('file', fileStream, path.basename(inputFilePath));
+
+        let uploadedBytes = 0;
+
+        const response = await axios({
+            method: 'POST',
+            url: `${SERVICE_URL}/`,
+            data: form,
+            headers: form.getHeaders(),
+            responseType: 'stream',
+            maxContentLength: Infinity,
+            maxBodyLength: Infinity,
+            onUploadProgress: (progressEvent) => {
+                uploadedBytes = progressEvent.loaded;
+                const percent = Math.round((progressEvent.loaded * 100) / fileSize);
+                process.stdout.write(`\r📤 Upload: ${percent}%`);
+            },
+        });
+
+        console.log('\n📥 Downloading PDF...');
+
+        const writer = fs.createWriteStream(outputFilePath);
+        let downloadedBytes = 0;
+
+        response.data.on('data', (chunk) => {
+            downloadedBytes += chunk.length;
+            process.stdout.write(`\r📥 Downloaded: ${(downloadedBytes / 1024).toFixed(2)} KB`);
+        });
+
+        response.data.pipe(writer);
+
+        return new Promise((resolve, reject) => {
+            writer.on('finish', () => {
+                console.log(`\n✅ Saved to: ${outputFilePath}`);
+                resolve(outputFilePath);
+            });
+            writer.on('error', reject);
+        });
+
+    } catch (error) {
+        console.error('\n❌ Conversion failed:', error.message);
+        throw error;
+    }
+}
+
+/**
+ * Method 4: Batch conversion
+ * Convert multiple files efficiently
+ */
+async function convertMultipleFiles(inputFiles, outputDir) {
+    console.log(`📦 Converting ${inputFiles.length} files...`);
+
+    const results = [];
+
+    for (const inputFile of inputFiles) {
+        try {
+            const fileName = path.basename(inputFile, path.extname(inputFile));
+            const outputFile = path.join(outputDir, `${fileName}.pdf`);
+
+            await convertFileToPDF(inputFile, outputFile);
+            results.push({ success: true, input: inputFile, output: outputFile });
+        } catch (error) {
+            results.push({ success: false, input: inputFile, error: error.message });
+        }
+    }
+
+    // Summary
+    const successful = results.filter(r => r.success).length;
+    console.log(`\n✅ Converted ${successful}/${inputFiles.length} files`);
+
+    return results;
+}
+
+/**
+ * Health check
+ */
+async function checkHealth() {
+    try {
+        const response = await axios.get(`${SERVICE_URL}/health`);
+        console.log('✅ Service is healthy:', response.data);
+        return true;
+    } catch (error) {
+        console.error('❌ Service is not available:', error.message);
+        return false;
+    }
+}
+
+// Example usage
+async function main() {
+    // Check if service is running
+    const isHealthy = await checkHealth();
+    if (!isHealthy) {
+        console.error('Service is not available. Please start it first.');
+        process.exit(1);
+    }
+
+    console.log('\n=== Document to PDF Converter - Node.js Client ===\n');
+
+    // Example 1: Convert a single file
+    await convertFileToPDF(
+        '../samples/data.txt',
+        './output/data.pdf'
+    );
+
+    console.log('\n---\n');
+
+    // Example 2: Convert with progress
+    await convertFileWithProgress(
+        '../samples/file-sample_1MB.docx',
+        './output/document.pdf'
+    );
+
+    console.log('\n---\n');
+
+    // Example 3: Convert from URI
+    // await convertUriToPDF(
+    //     'https://example.com/document.docx',
+    //     './output/from-uri.pdf'
+    // );
+
+    // Example 4: Batch conversion
+    // await convertMultipleFiles(
+    //     ['file1.docx', 'file2.xlsx', 'file3.pptx'],
+    //     './output'
+    // );
+}
+
+// Run if called directly
+if (require.main === module) {
+    // Create output directory
+    const outputDir = path.join(__dirname, 'output');
+    if (!fs.existsSync(outputDir)) {
+        fs.mkdirSync(outputDir, { recursive: true });
+    }
+
+    main().catch(console.error);
+}
+
+// Export for use in other modules
+module.exports = {
+    convertFileToPDF,
+    convertUriToPDF,
+    convertFileWithProgress,
+    convertMultipleFiles,
+    checkHealth,
+};