@minded-ai/mindedjs 3.0.7 → 3.1.7

package/dist/internalTools/documentExtraction/documentExtraction.js

@@ -1,736 +1,177 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DocumentProcessor = void 0;
-exports.extractFromDocument = extractFromDocument;
-const fs = __importStar(require("fs"));
-const path = __importStar(require("path"));
-const logger_1 = require("../../utils/logger");
-const os = __importStar(require("os"));
-const types_1 = require("./types");
+exports.DocumentProcessingMode = void 0;
+exports.parseDocumentAndExtractStructuredData = parseDocumentAndExtractStructuredData;
+exports.parseDocument = parseDocument;
+const extractStructuredData_1 = require("./extractStructuredData");
+const parseDocumentManaged_1 = require("./parseDocumentManaged");
+const parseDocumentLocal_1 = require("./parseDocumentLocal");
+/**
+ * Document processing mode
+ */
+var DocumentProcessingMode;
+(function (DocumentProcessingMode) {
+    /** Process documents using Minded cloud service (default) */
+    DocumentProcessingMode["MANAGED"] = "managed";
+    /** Process documents locally using LlamaCloud */
+    DocumentProcessingMode["LOCAL"] = "local";
+})(DocumentProcessingMode || (exports.DocumentProcessingMode = DocumentProcessingMode = {}));
 /**
- * Extract data from documents using AI or return raw text.
+ * Parse document and extract structured data using AI.
+ *
+ * This function provides a flexible way to process documents with optional AI-powered extraction:
+ * - Raw text extraction: Parse document without LLM processing
+ * - Structured extraction with schema: Extract data matching a Zod schema
+ * - Structured extraction with prompt: Guide extraction using custom prompts
+ * - Processing modes: Use DocumentProcessingMode.LOCAL (requires LlamaCloud API key) or DocumentProcessingMode.MANAGED (backend service)
  *
- * This function allows you to process various document types (PDFs, images, Word docs, etc.)
- * in multiple ways:
- * - With LLM + Schema: Extract structured data according to a Zod schema
- * - With LLM + System Prompt: Extract unstructured data based on prompt instructions
- * - Without LLM: Extract raw text content using LlamaParse
+ * @param options - Document processing options
+ * @param options.documentSource - URL or file path to the document
+ * @param options.processingMode - Document parsing mode: DocumentProcessingMode.MANAGED (default, backend service) or DocumentProcessingMode.LOCAL (requires llamaCloudApiKey)
+ * @param options.sessionId - Unique session identifier for logging and tracking
+ * @param options.llamaCloudApiKey - LlamaCloud API key for local processing. Required when processingMode is DocumentProcessingMode.LOCAL. Can be provided as parameter or via LLAMA_CLOUD_API_KEY environment variable
+ * @param options.returnStructuredOutput - Whether to extract structured data using LLM (true) or return raw text only (false). Defaults to false.
+ * @param options.llm - Language model instance for AI-powered extraction. Required when returnStructuredOutput is true
+ * @param options.outputSchema - Optional Zod schema defining the structure of extracted data
+ * @param options.outputSchemaPrompt - Optional prompt to guide the llm how to extract the data
  *
- * @param options - Document extraction options
- * @param options.llm - Optional language model for AI-powered extraction
- * @param options.documentPath - Path to the document file
- * @param options.documentContent - Document content as Buffer or string
- * @param options.documentUrl - URL to fetch the document from
- * @param options.schema - Optional Zod schema for structured data extraction
- * @param options.systemPrompt - Optional prompt for guiding extraction
- * @param options.config - Optional document processor configuration
+ * @returns Promise resolving to an object containing:
+ *   - rawContent: The raw extracted text from the document
+ *   - structuredContent: AI-extracted structured data (if returnStructuredOutput is true)
+ *   - metadata: Document metadata from processing
  *
- * @returns Promise resolving to extracted data/text and metadata
+ * @throws {Error} If documentSource is not provided
+ * @throws {Error} If returnStructuredOutput is true but llm is not provided
+ * @throws {Error} If document parsing or extraction fails
  *
  * @example
  * ```typescript
- * import { extractFromDocument } from '@minded-ai/mindedjs';
+ * import { parseDocumentAndExtractStructuredData, DocumentProcessingMode } from '@minded-ai/mindedjs';
  * import { z } from 'zod';
  *
- * // Extract structured data with schema
- * const result1 = await extractFromDocument({
+ * // Parse document and extract structured data using a schema
+ * const result1 = await parseDocumentAndExtractStructuredData({
+ *   documentSource: './invoice.pdf',
+ *   processingMode: DocumentProcessingMode.MANAGED,
+ *   sessionId: state.sessionId,
+ *   returnStructuredOutput: true,
  *   llm: agent.llm,
- *   documentPath: './invoice.pdf',
- *   schema: z.object({
+ *   outputSchema: z.object({
  *     invoiceNumber: z.string(),
- *     amount: z.number(),
- *   })
- * });
- *
- * // Extract unstructured data with prompt
- * const result2 = await extractFromDocument({
- *   llm: agent.llm,
- *   documentPath: './contract.pdf',
- *   systemPrompt: 'Extract all payment terms and conditions'
+ *     totalAmount: z.number(),
+ *     date: z.string(),
+ *   }),
  * });
  *
- * // Extract raw text without LLM
- * const result3 = await extractFromDocument({
- *   documentPath: './document.pdf'
+ * // Parse document only
+ * const result3 = await parseDocumentAndExtractStructuredData({
+ *   documentSource: './document.pdf',
+ *   processingMode: DocumentProcessingMode.MANAGED,
+ *   sessionId: state.sessionId,
+ *   returnStructuredOutput: false,
  * });
  * ```
  */
-async function extractFromDocument(options) {
-    // Create a document processor
-    const processor = new DocumentProcessor(options.config, options.llm);
-    // Extract from document using the processor
-    return processor.extractFromDocument({
-        documentPath: options.documentPath,
-        documentContent: options.documentContent,
-        documentUrl: options.documentUrl,
-        schema: options.schema,
-        systemPrompt: options.systemPrompt,
+async function parseDocumentAndExtractStructuredData({ documentSource, processingMode, sessionId, llamaCloudApiKey, returnStructuredOutput = false, llm, outputSchema, outputSchemaPrompt, }) {
+    // Parse document
+    const result = await parseDocument({
+        documentSource,
+        processingMode,
+        sessionId,
+        llamaCloudApiKey,
     });
+    if (!returnStructuredOutput || !result.rawContent) {
+        return result;
+    }
+    if (!llm) {
+        throw new Error('LLM instance is required when returnStructuredOutput is true. Please provide an LLM instance.');
+    }
+    // Extract structured data from the parsed document
+    const structuredContent = await (0, extractStructuredData_1.extractStructuredDataFromString)({
+        content: result.rawContent,
+        llm,
+        schema: outputSchema,
+        prompt: outputSchemaPrompt,
+        sessionId,
+    });
+    return {
+        ...result,
+        structuredContent,
+    };
 }
 /**
- * Generic document processor that can extract structured data from various document types
- * including images, PDFs, Word documents, spreadsheets, and more.
+ * Parse document and extract raw text content.
+ *
+ * This function processes various document types (PDFs, images, Word docs, etc.) and extracts
+ * raw text content using either local processing with LlamaCloud or managed backend service.
+ * Use this for raw text extraction without AI-powered data extraction.
+ *
+ * @param options - Document parsing options
+ * @param options.documentSource - URL or file path to the document
+ * @param options.processingMode - Parsing mode: DocumentProcessingMode.LOCAL (requires llamaCloudApiKey) or DocumentProcessingMode.MANAGED (backend service, default)
+ * @param options.sessionId - Unique session identifier for logging and tracking
+ * @param options.llamaCloudApiKey - LlamaCloud API key for local processing. Required when processingMode is DocumentProcessingMode.LOCAL. Can be provided as parameter or via LLAMA_CLOUD_API_KEY environment variable
+ *
+ * @returns Promise resolving to an object containing:
+ *   - rawContent: The raw extracted text from the document
+ *   - metadata: Document processing metadata (file size, type, processing time, content length)
+ *
+ * @throws {Error} If documentSource is not provided
+ * @throws {Error} If document processing fails
+ *
+ * @example
+ * ```typescript
+ * import { parseDocument, DocumentProcessingMode } from '@minded-ai/mindedjs';
+ *
+ * // Parse document using managed service
+ * const result1 = await parseDocument({
+ *   documentSource: 'https://example.com/invoice.pdf',
+ *   processingMode: DocumentProcessingMode.MANAGED,
+ *   sessionId: state.sessionId,
+ * });
+ * // result1: { rawContent: "Invoice text...", metadata: {...} }
+ *
+ * // Parse local document using LlamaCloud
+ * const result2 = await parseDocument({
+ *   documentSource: './contract.pdf',
+ *   processingMode: DocumentProcessingMode.LOCAL,
+ *   sessionId: state.sessionId,
+ *   llamaCloudApiKey: process.env.LLAMA_CLOUD_API_KEY,
+ * });
+ * // result2: { rawContent: "Contract text...", metadata: {...} }
+ * ```
  */
-class DocumentProcessor {
-    constructor(config = {}, llm) {
-        this.llm = null;
-        this.llamaCloudApiKey = null;
-        this.sharpModule = null;
-        this.sharpLoadAttempted = false;
-        this.config = {
-            maxImageWidth: 1200,
-            imageQuality: 85,
-            useBase64: false,
-            ...config,
-        };
-        this.llm = llm || null;
-        this.llamaCloudApiKey = this.config.llamaCloudApiKey || process.env.LLAMA_CLOUD_API_KEY || null;
-    }
-    /**
-     * Parse document using LlamaCloud REST API
-     */
-    async parseWithLlamaCloud(filePath) {
-        if (!this.llamaCloudApiKey) {
-            return null;
-        }
-        try {
-            // Step 1: Upload file and start parsing
-            const fileContent = fs.readFileSync(filePath);
-            const fileName = path.basename(filePath);
-            const mimeType = this.getMimeType(path.extname(filePath));
-            const formData = new FormData();
-            const blob = new Blob([fileContent], { type: mimeType });
-            formData.append('file', blob, fileName);
-            formData.append('premium_mode', 'true');
-            const uploadResponse = await fetch('https://api.cloud.llamaindex.ai/api/v1/parsing/upload', {
-                method: 'POST',
-                headers: {
-                    Accept: 'application/json',
-                    Authorization: `Bearer ${this.llamaCloudApiKey}`,
-                },
-                body: formData,
-            });
-            if (!uploadResponse.ok) {
-                const errorText = await uploadResponse.text();
-                throw new Error(`Failed to upload file: ${uploadResponse.status} - ${errorText}`);
-            }
-            const uploadResult = await uploadResponse.json();
-            const jobId = uploadResult.id || uploadResult.job_id;
-            if (!jobId) {
-                throw new Error('No job ID returned from upload');
-            }
-            logger_1.logger.info({
-                msg: '[DocumentProcessor] File uploaded to LlamaCloud',
-                jobId,
-                fileName,
-            });
-            // Step 2: Poll for job completion
-            let attempts = 0;
-            const maxAttempts = 60; // 60 attempts with 2 second delay = 2 minutes max
-            const pollDelay = 2000; // 2 seconds
-            while (attempts < maxAttempts) {
-                const statusResponse = await fetch(`https://api.cloud.llamaindex.ai/api/v1/parsing/job/${jobId}`, {
-                    method: 'GET',
-                    headers: {
-                        Accept: 'application/json',
-                        Authorization: `Bearer ${this.llamaCloudApiKey}`,
-                    },
-                });
-                if (!statusResponse.ok) {
-                    throw new Error(`Failed to check job status: ${statusResponse.status}`);
-                }
-                const statusResult = await statusResponse.json();
-                const status = statusResult.status || statusResult.job_status;
-                if (status === 'SUCCESS' || status === 'COMPLETED' || status === 'completed') {
-                    // Step 3: Retrieve results in Markdown
-                    // Create an AbortController for timeout
-                    const controller = new AbortController();
-                    const timeout = setTimeout(() => controller.abort(), 20000); // 20 second timeout
-                    let resultResponse;
-                    try {
-                        resultResponse = await fetch(`https://api.cloud.llamaindex.ai/api/v1/parsing/job/${jobId}/result/markdown`, {
-                            method: 'GET',
-                            headers: {
-                                Accept: 'application/json',
-                                Authorization: `Bearer ${this.llamaCloudApiKey}`,
-                            },
-                            signal: controller.signal,
-                        });
-                    }
-                    catch (fetchError) {
-                        clearTimeout(timeout);
-                        if (fetchError instanceof Error && fetchError.name === 'AbortError') {
-                            throw new Error('Timeout fetching results from LlamaCloud after 20 seconds');
-                        }
-                        throw fetchError;
-                    }
-                    clearTimeout(timeout);
-                    if (!resultResponse.ok) {
-                        const errorText = await resultResponse.text();
-                        throw new Error(`Failed to retrieve results: ${resultResponse.status} - ${errorText}`);
-                    }
-                    let resultData;
-                    try {
-                        // Read response using manual stream reading (more reliable than text())
-                        let responseText;
-                        if (resultResponse.body) {
-                            const reader = resultResponse.body.getReader();
-                            const chunks = [];
-                            let totalLength = 0;
-                            try {
-                                while (true) {
-                                    const { done, value } = await reader.read();
-                                    if (done)
-                                        break;
-                                    if (value) {
-                                        chunks.push(value);
-                                        totalLength += value.length;
-                                    }
-                                }
-                                // Combine chunks
-                                const combined = new Uint8Array(totalLength);
-                                let offset = 0;
-                                for (const chunk of chunks) {
-                                    combined.set(chunk, offset);
-                                    offset += chunk.length;
-                                }
-                                responseText = new TextDecoder().decode(combined);
-                            }
-                            finally {
-                                reader.releaseLock();
-                            }
-                        }
-                        else {
-                            responseText = await resultResponse.text();
-                        }
-                        // Try to parse as JSON, but if it fails, use the text directly
-                        try {
-                            resultData = JSON.parse(responseText);
-                        }
-                        catch (_a) {
-                            // If it's not JSON, assume it's the markdown content directly
-                            resultData = responseText;
-                        }
-                    }
-                    catch (textError) {
-                        logger_1.logger.error({
-                            msg: '[DocumentProcessor] Failed to read response text',
-                            jobId,
-                            error: textError instanceof Error ? textError.message : String(textError),
-                            stack: textError instanceof Error ? textError.stack : undefined,
-                        });
-                        throw new Error('Failed to read response from LlamaCloud');
-                    }
-                    logger_1.logger.debug({
-                        msg: '[DocumentProcessor] Result data structure',
-                        jobId,
-                        dataType: typeof resultData,
-                        keys: typeof resultData === 'object' && resultData !== null ? Object.keys(resultData) : [],
-                        hasMarkdown: typeof resultData === 'object' && 'markdown' in resultData,
-                        hasContent: typeof resultData === 'object' && 'content' in resultData,
-                        hasText: typeof resultData === 'object' && 'text' in resultData,
-                    });
-                    // The API might return the markdown directly as a string or nested in an object
-                    let markdownContent;
-                    if (typeof resultData === 'string') {
-                        markdownContent = resultData;
-                    }
-                    else {
-                        markdownContent = resultData.markdown || resultData.content || resultData.text || '';
-                    }
-                    if (!markdownContent) {
-                        logger_1.logger.error({
-                            msg: '[DocumentProcessor] No content in result',
-                            jobId,
-                            resultData: JSON.stringify(resultData).substring(0, 500),
-                        });
-                        throw new Error('No content returned from parsing');
-                    }
-                    logger_1.logger.info({
-                        msg: '[DocumentProcessor] Successfully parsed document with LlamaCloud',
-                        jobId,
-                        contentLength: markdownContent.length,
-                        preview: markdownContent.substring(0, 100),
-                    });
-                    logger_1.logger.debug({
-                        msg: '[DocumentProcessor] About to return markdown content',
-                        jobId,
-                    });
-                    return markdownContent;
-                }
-                else if (status === 'FAILED' || status === 'ERROR' || status === 'failed') {
-                    throw new Error(`Parsing job failed: ${statusResult.error || 'Unknown error'}`);
-                }
-                // Wait before next attempt
-                await new Promise((resolve) => setTimeout(resolve, pollDelay));
-                attempts++;
-            }
-            throw new Error('Parsing job timed out after 2 minutes');
-        }
-        catch (err) {
-            logger_1.logger.warn({
-                message: '[DocumentProcessor] LlamaCloud parsing failed',
-                err,
-            });
-            return null;
-        }
-        finally {
-            logger_1.logger.debug({
-                msg: '[DocumentProcessor] parseWithLlamaCloud finished',
-                filePath,
-            });
-        }
-    }
-    /**
-     * Get MIME type for file extension
-     */
-    getMimeType(fileExtension) {
-        const mimeTypes = {
-            '.pdf': 'application/pdf',
-            '.doc': 'application/msword',
-            '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-            '.txt': 'text/plain',
-            '.rtf': 'application/rtf',
-            '.jpg': 'image/jpeg',
-            '.jpeg': 'image/jpeg',
-            '.png': 'image/png',
-            '.gif': 'image/gif',
-            '.bmp': 'image/bmp',
-            '.webp': 'image/webp',
-            '.tiff': 'image/tiff',
-            '.xls': 'application/vnd.ms-excel',
-            '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-            '.csv': 'text/csv',
-            '.html': 'text/html',
-            '.htm': 'text/html',
-            '.xml': 'application/xml',
-            '.md': 'text/markdown',
-        };
-        return mimeTypes[fileExtension.toLowerCase()] || 'application/octet-stream';
+async function parseDocument({ documentSource, processingMode = DocumentProcessingMode.MANAGED, sessionId, llamaCloudApiKey, }) {
+    if (!documentSource) {
+        throw new Error('documentSource is required - provide a URL or file path');
     }
-    /**
-     * Extract data from a document - either structured data using AI or raw text
-     */
-    async extractFromDocument(options) {
-        logger_1.logger.debug({
-            msg: '[DocumentProcessor] extractFromDocument called',
-            hasDocumentPath: !!options.documentPath,
-            hasDocumentUrl: !!options.documentUrl,
-            hasDocumentContent: !!options.documentContent,
-            hasLLM: !!this.llm,
-            hasSchema: !!options.schema,
+    const isDocumentUrl = isUrl(documentSource);
+    if (processingMode === DocumentProcessingMode.MANAGED) {
+        return (0, parseDocumentManaged_1.parseDocumentWithManagedService)({
+            documentSource,
+            isDocumentUrl,
+            sessionId,
         });
-        const startTime = Date.now();
-        try {
-            logger_1.logger.debug({
-                msg: '[DocumentProcessor] Getting document content',
-            });
-            // Determine document source and content
-            const { content, fileType, fileSize } = await this.getDocumentContent(options);
-            // Process document content based on type
-            let processedContent;
-            if (this.isImageFile(fileType)) {
-                processedContent = await this.processImageDocument(content, fileType, options.documentPath);
-            }
-            else {
-                processedContent = await this.processTextDocument(content, options.documentPath, fileType);
-            }
-            logger_1.logger.info({
-                msg: '[DocumentProcessor] Document content processed',
-                fileType,
-                contentLength: processedContent.length,
-            });
-            const processingTime = Date.now() - startTime;
-            // If no LLM is provided, return the raw text content
-            if (!this.llm) {
-                return {
-                    data: processedContent,
-                    metadata: {
-                        fileSize,
-                        fileType,
-                        processingTime,
-                        contentLength: processedContent.length,
-                    },
-                };
-            }
-            // If LLM is provided, extract data (structured or unstructured)
-            const extractedData = await this.extractStructuredData(processedContent, options.schema, options.llmConfig, options.systemPrompt);
-            return {
-                data: extractedData,
-                metadata: {
-                    fileSize,
-                    fileType,
-                    processingTime,
-                    contentLength: processedContent.length,
-                },
-            };
-        }
-        catch (err) {
-            logger_1.logger.error({
-                message: '[DocumentProcessor] Document processing failed',
-                err,
-            });
-            throw new Error(`Document processing failed: ${err instanceof Error ? err.message : String(err)}`);
-        }
     }
-    /**
-     * Get document content from various sources
-     */
-    async getDocumentContent(options) {
-        logger_1.logger.debug({
-            msg: '[DocumentProcessor] getDocumentContent called',
-            hasPath: !!options.documentPath,
-            hasContent: !!options.documentContent,
-            hasUrl: !!options.documentUrl,
+    else {
+        return (0, parseDocumentLocal_1.parseDocumentWithLocalService)({
+            documentSource,
+            isDocumentUrl,
+            sessionId,
+            llamaCloudApiKey: llamaCloudApiKey !== null && llamaCloudApiKey !== void 0 ? llamaCloudApiKey : process.env.LLAMA_CLOUD_API_KEY,
         });
-        // From file path
-        if (options.documentPath) {
-            if (!fs.existsSync(options.documentPath)) {
-                throw new Error(`Document not found: ${options.documentPath}`);
-            }
-            const content = fs.readFileSync(options.documentPath);
-            const fileType = path.extname(options.documentPath).toLowerCase();
-            return {
-                content,
-                fileType,
-                fileSize: content.length,
-            };
-        }
-        // From provided content
-        if (options.documentContent) {
-            // Try to infer file type from content if it's a buffer
-            let fileType = '.unknown';
-            if (Buffer.isBuffer(options.documentContent)) {
-                fileType = this.inferFileTypeFromBuffer(options.documentContent);
-            }
-            else if (typeof options.documentContent === 'string') {
-                fileType = '.txt'; // Assume text content
-            }
-            return {
-                content: options.documentContent,
-                fileType,
-                fileSize: Buffer.isBuffer(options.documentContent) ? options.documentContent.length : Buffer.byteLength(options.documentContent),
-            };
-        }
-        // From URL
-        if (options.documentUrl) {
-            logger_1.logger.debug({
-                msg: '[DocumentProcessor] Fetching document from URL',
-                url: options.documentUrl,
-            });
-            const response = await fetch(options.documentUrl);
-            logger_1.logger.debug({
-                msg: '[DocumentProcessor] URL fetch response',
-                status: response.status,
-                ok: response.ok,
-            });
-            if (!response.ok) {
-                throw new Error(`Failed to fetch document from URL: ${response.statusText}`);
-            }
-            const arrayBuffer = await response.arrayBuffer();
-            const content = Buffer.from(arrayBuffer);
-            const fileType = this.inferFileTypeFromUrl(options.documentUrl) || this.inferFileTypeFromBuffer(content);
-            logger_1.logger.debug({
-                msg: '[DocumentProcessor] Document fetched from URL',
-                contentSize: content.length,
-                fileType,
-            });
-            return {
-                content,
-                fileType,
-                fileSize: content.length,
-            };
-        }
-        throw new Error('No document source provided. Specify documentPath, documentContent, or documentUrl.');
     }
-    /**
-     * Process image documents by converting them to a standardized format
-     */
-    async processImageDocument(content, fileType, filePath) {
-        try {
-            // First, try to use LlamaParser if available for text extraction
-            if (filePath && this.llamaCloudApiKey) {
-                logger_1.logger.debug({
-                    msg: '[DocumentProcessor] Calling parseWithLlamaCloud for image',
-                    filePath,
-                });
-                const parsedContent = await this.parseWithLlamaCloud(filePath);
-                logger_1.logger.debug({
-                    msg: '[DocumentProcessor] parseWithLlamaCloud returned for image',
-                    hasContent: !!parsedContent,
-                    contentLength: parsedContent === null || parsedContent === void 0 ? void 0 : parsedContent.length,
-                });
-                if (parsedContent) {
-                    return parsedContent;
-                }
-            }
-            // If no file path, create a temporary file for LlamaCloud parsing
-            if (!filePath && this.llamaCloudApiKey) {
-                const tempDir = os.tmpdir();
-                const tempFileName = `temp_${Date.now()}${fileType}`;
-                const tempFilePath = path.join(tempDir, tempFileName);
-                logger_1.logger.debug({
-                    msg: '[DocumentProcessor] Creating temp file for image',
-                    tempFilePath,
-                    contentSize: content.length,
-                });
-                try {
-                    fs.writeFileSync(tempFilePath, content);
-                    logger_1.logger.debug({
-                        msg: '[DocumentProcessor] Calling parseWithLlamaCloud for temp image',
-                        tempFilePath,
-                    });
-                    const parsedContent = await this.parseWithLlamaCloud(tempFilePath);
-                    logger_1.logger.debug({
-                        msg: '[DocumentProcessor] parseWithLlamaCloud returned for temp image',
-                        hasContent: !!parsedContent,
-                        contentLength: parsedContent === null || parsedContent === void 0 ? void 0 : parsedContent.length,
-                    });
-                    fs.unlinkSync(tempFilePath);
-                    if (parsedContent) {
-                        return parsedContent;
-                    }
-                }
-                catch (err) {
-                    // Clean up temp file on error
-                    if (fs.existsSync(tempFilePath)) {
-                        fs.unlinkSync(tempFilePath);
-                    }
-                    logger_1.logger.warn({ msg: '[DocumentProcessor] Failed to parse image with LlamaCloud', err });
-                }
-            }
-            // Fallback: Convert to image format for LLM processing
-            // For PDFs, convert first page to image
-            if (fileType === '.pdf') {
-                throw new Error('Failed to convert PDF to image');
-            }
-            logger_1.logger.warn({
-                msg: '[DocumentProcessor] Sharp module not available. Using original image without optimization.',
-                fileType,
-                contentSize: content.length,
-            });
-            // If sharp is not available, use the original image
-            if (this.config.useBase64) {
-                // Return original image as base64
-                const base64Image = content.toString('base64');
-                const mimeType = this.getMimeType(fileType);
-                return `data:${mimeType};base64,${base64Image}`;
-            }
-            else {
-                // Without sharp and without base64, we cannot process the image
-                return `[IMAGE CONTENT - ${fileType.toUpperCase()} file. Size: ${content.length} bytes. Consider using LLAMA_CLOUD_API_KEY for text extraction or set useBase64: true]`;
-            }
-        }
-        catch (err) {
-            throw new Error(`Failed to process image document: ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    /**
-     * Process text-based documents using LlamaParser or fallback methods
-     */
-    async processTextDocument(content, filePath, fileType) {
-        // Try LlamaCloud parsing if we have a file path
-        if (filePath && this.llamaCloudApiKey) {
-            const parsedContent = await this.parseWithLlamaCloud(filePath);
-            if (parsedContent) {
-                return parsedContent;
-            }
-        }
-        // If no file path but we have content and LlamaCloud API key, create a temp file
-        if (!filePath && this.llamaCloudApiKey && Buffer.isBuffer(content)) {
-            const tempDir = os.tmpdir();
-            const tempFileName = `temp_${Date.now()}${fileType || '.txt'}`;
-            const tempFilePath = path.join(tempDir, tempFileName);
-            try {
-                fs.writeFileSync(tempFilePath, content);
-                const parsedContent = await this.parseWithLlamaCloud(tempFilePath);
-                fs.unlinkSync(tempFilePath);
-                if (parsedContent) {
-                    return parsedContent;
-                }
-            }
-            catch (err) {
-                // Clean up temp file on error
-                if (fs.existsSync(tempFilePath)) {
-                    fs.unlinkSync(tempFilePath);
-                }
-                logger_1.logger.warn({ msg: '[DocumentProcessor] Failed to parse text document with LlamaCloud', err });
-            }
-        }
-        // Fallback: handle based on file type
-        if (typeof content === 'string') {
-            return content;
-        }
-        // For binary content, convert to text
-        if (fileType === '.pdf' && !this.llamaCloudApiKey) {
-            throw new Error('PDF processing requires LLAMA_CLOUD_API_KEY environment variable for LlamaParser');
-        }
-        // Basic text extraction for simple formats
-        if (['.txt', '.md', '.html', '.htm', '.xml', '.csv'].includes(fileType || '')) {
-            return content.toString('utf-8');
-        }
-        // For unsupported binary formats without LlamaParser
-        throw new Error(`Unsupported document type ${fileType}. Please provide LLAMA_CLOUD_API_KEY for advanced document processing.`);
-    }
-    /**
-     * Extract data using LLM - either structured with schema or unstructured with prompt
-     */
-    async extractStructuredData(content, schema, llmConfig, systemPrompt) {
-        if (!this.llm) {
-            throw new Error('LLM instance is required for data extraction. Please provide an LLM when creating the DocumentProcessor.');
-        }
-        // Note: llmConfig is ignored when using the provided LLM instance
-        // The LLM should already be configured with the desired model and temperature
-        const defaultSystemPrompt = 'You are an expert data-extraction assistant. ' +
-            'Extract the requested information from the provided document content. ' +
-            'If you cannot find a value for a required field, use "N/A" or a descriptive placeholder. ' +
-            'Be accurate and thorough in your extraction.';
-        const finalSystemPrompt = systemPrompt || defaultSystemPrompt;
-        try {
-            // If schema is provided, use structured output
-            if (schema) {
-                // Check if the LLM supports withStructuredOutput
-                if (!('withStructuredOutput' in this.llm)) {
-                    throw new Error('The provided LLM does not support structured output. Please use a compatible LLM instance.');
-                }
-                const structuredLlm = this.llm.withStructuredOutput(schema);
-                const result = await structuredLlm.invoke([
-                    {
-                        role: 'system',
-                        content: finalSystemPrompt,
-                    },
-                    {
-                        role: 'user',
-                        content: `Please extract the following information from this document:\n\n${content}`,
-                    },
-                ]);
-                logger_1.logger.debug({
-                    msg: '[DocumentProcessor] Structured data extraction completed',
-                    extractedData: JSON.stringify(result, null, 2),
-                });
-                return result;
-            }
-            else {
-                // Without schema, return the LLM's text response
-                const response = await this.llm.invoke([
-                    {
-                        role: 'system',
-                        content: finalSystemPrompt,
-                    },
-                    {
-                        role: 'user',
-                        content: `Please analyze and extract information from this document:\n\n${content}`,
-                    },
-                ]);
-                // Extract the text content from the response
-                let textContent;
-                if (typeof response.content === 'string') {
-                    textContent = response.content;
-                }
-                else if (Array.isArray(response.content) && response.content.length > 0) {
-                    // Handle array of content blocks
-                    textContent = response.content.map((block) => (typeof block === 'string' ? block : block.text || '')).join('\n');
-                }
-                else {
-                    textContent = String(response.content);
-                }
-                logger_1.logger.debug({
-                    msg: '[DocumentProcessor] Unstructured data extraction completed',
-                    contentLength: textContent.length,
-                });
-                return textContent;
-            }
-        }
-        catch (err) {
-            throw new Error(`LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    /**
-     * Check if file is an image type
-     */
-    isImageFile(fileType) {
-        const imageTypes = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff'];
-        return imageTypes.includes(fileType.toLowerCase());
-    }
-    /**
-     * Infer file type from buffer content
-     */
-    inferFileTypeFromBuffer(buffer) {
-        // Check common file signatures
-        const signatures = {
-            '89504E47': '.png',
-            FFD8FF: '.jpg',
-            '47494638': '.gif',
-            '25504446': '.pdf',
-            '504B0304': '.zip', // Also used by docx, xlsx, pptx
-            D0CF11E0: '.doc', // Also xls, ppt
-        };
-        const hex = buffer.toString('hex', 0, 4).toUpperCase();
-        for (const [signature, type] of Object.entries(signatures)) {
-            if (hex.startsWith(signature)) {
-                return type;
-            }
-        }
-        return '.unknown';
-    }
-    /**
-     * Infer file type from URL
-     */
-    inferFileTypeFromUrl(url) {
-        try {
-            const pathname = new URL(url).pathname;
-            const extension = path.extname(pathname).toLowerCase();
-            return extension || null;
-        }
-        catch (_a) {
-            return null;
-        }
-    }
-    /**
-     * Get list of supported document types
-     */
-    static getSupportedDocumentTypes() {
-        return [...types_1.SUPPORTED_DOCUMENT_TYPES];
+}
+/**
+ * Check if a string is a URL
+ */
+function isUrl(source) {
+    try {
+        const url = new URL(source);
+        return url.protocol === 'http:' || url.protocol === 'https:';
     }
-    /**
-     * Check if a file type is supported
-     */
-    static isDocumentTypeSupported(fileType) {
-        return types_1.SUPPORTED_DOCUMENT_TYPES.includes(fileType.toLowerCase());
+    catch (_a) {
+        return false;
     }
 }
-exports.DocumentProcessor = DocumentProcessor;
 //# sourceMappingURL=documentExtraction.js.map