@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/dist/internalTools/documentExtraction/documentExtraction.js

@@ -0,0 +1,736 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocumentProcessor = void 0;
+exports.extractFromDocument = extractFromDocument;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const logger_1 = require("../../utils/logger");
+const os = __importStar(require("os"));
+const types_1 = require("./types");
+/**
+ * Extract data from documents using AI or return raw text.
+ *
+ * This function allows you to process various document types (PDFs, images, Word docs, etc.)
+ * in multiple ways:
+ * - With LLM + Schema: Extract structured data according to a Zod schema
+ * - With LLM + System Prompt: Extract unstructured data based on prompt instructions
+ * - Without LLM: Extract raw text content using LlamaParse
+ *
+ * @param options - Document extraction options
+ * @param options.llm - Optional language model for AI-powered extraction
+ * @param options.documentPath - Path to the document file
+ * @param options.documentContent - Document content as Buffer or string
+ * @param options.documentUrl - URL to fetch the document from
+ * @param options.schema - Optional Zod schema for structured data extraction
+ * @param options.systemPrompt - Optional prompt for guiding extraction
+ * @param options.config - Optional document processor configuration
+ *
+ * @returns Promise resolving to extracted data/text and metadata
+ *
+ * @example
+ * ```typescript
+ * import { extractFromDocument } from '@minded-ai/mindedjs';
+ * import { z } from 'zod';
+ *
+ * // Extract structured data with schema
+ * const result1 = await extractFromDocument({
+ *   llm: agent.llm,
+ *   documentPath: './invoice.pdf',
+ *   schema: z.object({
+ *     invoiceNumber: z.string(),
+ *     amount: z.number(),
+ *   })
+ * });
+ *
+ * // Extract unstructured data with prompt
+ * const result2 = await extractFromDocument({
+ *   llm: agent.llm,
+ *   documentPath: './contract.pdf',
+ *   systemPrompt: 'Extract all payment terms and conditions'
+ * });
+ *
+ * // Extract raw text without LLM
+ * const result3 = await extractFromDocument({
+ *   documentPath: './document.pdf'
+ * });
+ * ```
+ */
+async function extractFromDocument(options) {
+    // Create a document processor
+    const processor = new DocumentProcessor(options.config, options.llm);
+    // Extract from document using the processor
+    return processor.extractFromDocument({
+        documentPath: options.documentPath,
+        documentContent: options.documentContent,
+        documentUrl: options.documentUrl,
+        schema: options.schema,
+        systemPrompt: options.systemPrompt,
+    });
+}
+/**
+ * Generic document processor that can extract structured data from various document types
+ * including images, PDFs, Word documents, spreadsheets, and more.
+ */
+class DocumentProcessor {
+    constructor(config = {}, llm) {
+        this.llm = null;
+        this.llamaCloudApiKey = null;
+        this.sharpModule = null;
+        this.sharpLoadAttempted = false;
+        this.config = {
+            maxImageWidth: 1200,
+            imageQuality: 85,
+            useBase64: false,
+            ...config,
+        };
+        this.llm = llm || null;
+        this.llamaCloudApiKey = this.config.llamaCloudApiKey || process.env.LLAMA_CLOUD_API_KEY || null;
+    }
+    /**
+     * Parse document using LlamaCloud REST API
+     */
+    async parseWithLlamaCloud(filePath) {
+        if (!this.llamaCloudApiKey) {
+            return null;
+        }
+        try {
+            // Step 1: Upload file and start parsing
+            const fileContent = fs.readFileSync(filePath);
+            const fileName = path.basename(filePath);
+            const mimeType = this.getMimeType(path.extname(filePath));
+            const formData = new FormData();
+            const blob = new Blob([fileContent], { type: mimeType });
+            formData.append('file', blob, fileName);
+            formData.append('premium_mode', 'true');
+            const uploadResponse = await fetch('https://api.cloud.llamaindex.ai/api/v1/parsing/upload', {
+                method: 'POST',
+                headers: {
+                    Accept: 'application/json',
+                    Authorization: `Bearer ${this.llamaCloudApiKey}`,
+                },
+                body: formData,
+            });
+            if (!uploadResponse.ok) {
+                const errorText = await uploadResponse.text();
+                throw new Error(`Failed to upload file: ${uploadResponse.status} - ${errorText}`);
+            }
+            const uploadResult = await uploadResponse.json();
+            const jobId = uploadResult.id || uploadResult.job_id;
+            if (!jobId) {
+                throw new Error('No job ID returned from upload');
+            }
+            logger_1.logger.info({
+                msg: '[DocumentProcessor] File uploaded to LlamaCloud',
+                jobId,
+                fileName,
+            });
+            // Step 2: Poll for job completion
+            let attempts = 0;
+            const maxAttempts = 60; // 60 attempts with 2 second delay = 2 minutes max
+            const pollDelay = 2000; // 2 seconds
+            while (attempts < maxAttempts) {
+                const statusResponse = await fetch(`https://api.cloud.llamaindex.ai/api/v1/parsing/job/${jobId}`, {
+                    method: 'GET',
+                    headers: {
+                        Accept: 'application/json',
+                        Authorization: `Bearer ${this.llamaCloudApiKey}`,
+                    },
+                });
+                if (!statusResponse.ok) {
+                    throw new Error(`Failed to check job status: ${statusResponse.status}`);
+                }
+                const statusResult = await statusResponse.json();
+                const status = statusResult.status || statusResult.job_status;
+                if (status === 'SUCCESS' || status === 'COMPLETED' || status === 'completed') {
+                    // Step 3: Retrieve results in Markdown
+                    // Create an AbortController for timeout
+                    const controller = new AbortController();
+                    const timeout = setTimeout(() => controller.abort(), 20000); // 20 second timeout
+                    let resultResponse;
+                    try {
+                        resultResponse = await fetch(`https://api.cloud.llamaindex.ai/api/v1/parsing/job/${jobId}/result/markdown`, {
+                            method: 'GET',
+                            headers: {
+                                Accept: 'application/json',
+                                Authorization: `Bearer ${this.llamaCloudApiKey}`,
+                            },
+                            signal: controller.signal,
+                        });
+                    }
+                    catch (fetchError) {
+                        clearTimeout(timeout);
+                        if (fetchError instanceof Error && fetchError.name === 'AbortError') {
+                            throw new Error('Timeout fetching results from LlamaCloud after 20 seconds');
+                        }
+                        throw fetchError;
+                    }
+                    clearTimeout(timeout);
+                    if (!resultResponse.ok) {
+                        const errorText = await resultResponse.text();
+                        throw new Error(`Failed to retrieve results: ${resultResponse.status} - ${errorText}`);
+                    }
+                    let resultData;
+                    try {
+                        // Read response using manual stream reading (more reliable than text())
+                        let responseText;
+                        if (resultResponse.body) {
+                            const reader = resultResponse.body.getReader();
+                            const chunks = [];
+                            let totalLength = 0;
+                            try {
+                                while (true) {
+                                    const { done, value } = await reader.read();
+                                    if (done)
+                                        break;
+                                    if (value) {
+                                        chunks.push(value);
+                                        totalLength += value.length;
+                                    }
+                                }
+                                // Combine chunks
+                                const combined = new Uint8Array(totalLength);
+                                let offset = 0;
+                                for (const chunk of chunks) {
+                                    combined.set(chunk, offset);
+                                    offset += chunk.length;
+                                }
+                                responseText = new TextDecoder().decode(combined);
+                            }
+                            finally {
+                                reader.releaseLock();
+                            }
+                        }
+                        else {
+                            responseText = await resultResponse.text();
+                        }
+                        // Try to parse as JSON, but if it fails, use the text directly
+                        try {
+                            resultData = JSON.parse(responseText);
+                        }
+                        catch (_a) {
+                            // If it's not JSON, assume it's the markdown content directly
+                            resultData = responseText;
+                        }
+                    }
+                    catch (textError) {
+                        logger_1.logger.error({
+                            msg: '[DocumentProcessor] Failed to read response text',
+                            jobId,
+                            error: textError instanceof Error ? textError.message : String(textError),
+                            stack: textError instanceof Error ? textError.stack : undefined,
+                        });
+                        throw new Error('Failed to read response from LlamaCloud');
+                    }
+                    logger_1.logger.debug({
+                        msg: '[DocumentProcessor] Result data structure',
+                        jobId,
+                        dataType: typeof resultData,
+                        keys: typeof resultData === 'object' && resultData !== null ? Object.keys(resultData) : [],
+                        hasMarkdown: typeof resultData === 'object' && 'markdown' in resultData,
+                        hasContent: typeof resultData === 'object' && 'content' in resultData,
+                        hasText: typeof resultData === 'object' && 'text' in resultData,
+                    });
+                    // The API might return the markdown directly as a string or nested in an object
+                    let markdownContent;
+                    if (typeof resultData === 'string') {
+                        markdownContent = resultData;
+                    }
+                    else {
+                        markdownContent = resultData.markdown || resultData.content || resultData.text || '';
+                    }
+                    if (!markdownContent) {
+                        logger_1.logger.error({
+                            msg: '[DocumentProcessor] No content in result',
+                            jobId,
+                            resultData: JSON.stringify(resultData).substring(0, 500),
+                        });
+                        throw new Error('No content returned from parsing');
+                    }
+                    logger_1.logger.info({
+                        msg: '[DocumentProcessor] Successfully parsed document with LlamaCloud',
+                        jobId,
+                        contentLength: markdownContent.length,
+                        preview: markdownContent.substring(0, 100),
+                    });
+                    logger_1.logger.debug({
+                        msg: '[DocumentProcessor] About to return markdown content',
+                        jobId,
+                    });
+                    return markdownContent;
+                }
+                else if (status === 'FAILED' || status === 'ERROR' || status === 'failed') {
+                    throw new Error(`Parsing job failed: ${statusResult.error || 'Unknown error'}`);
+                }
+                // Wait before next attempt
+                await new Promise((resolve) => setTimeout(resolve, pollDelay));
+                attempts++;
+            }
+            throw new Error('Parsing job timed out after 2 minutes');
+        }
+        catch (err) {
+            logger_1.logger.warn({
+                message: '[DocumentProcessor] LlamaCloud parsing failed',
+                err,
+            });
+            return null;
+        }
+        finally {
+            logger_1.logger.debug({
+                msg: '[DocumentProcessor] parseWithLlamaCloud finished',
+                filePath,
+            });
+        }
+    }
+    /**
+     * Get MIME type for file extension
+     */
+    getMimeType(fileExtension) {
+        const mimeTypes = {
+            '.pdf': 'application/pdf',
+            '.doc': 'application/msword',
+            '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            '.txt': 'text/plain',
+            '.rtf': 'application/rtf',
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.gif': 'image/gif',
+            '.bmp': 'image/bmp',
+            '.webp': 'image/webp',
+            '.tiff': 'image/tiff',
+            '.xls': 'application/vnd.ms-excel',
+            '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            '.csv': 'text/csv',
+            '.html': 'text/html',
+            '.htm': 'text/html',
+            '.xml': 'application/xml',
+            '.md': 'text/markdown',
+        };
+        return mimeTypes[fileExtension.toLowerCase()] || 'application/octet-stream';
+    }
+    /**
+     * Extract data from a document - either structured data using AI or raw text
+     */
+    async extractFromDocument(options) {
+        logger_1.logger.debug({
+            msg: '[DocumentProcessor] extractFromDocument called',
+            hasDocumentPath: !!options.documentPath,
+            hasDocumentUrl: !!options.documentUrl,
+            hasDocumentContent: !!options.documentContent,
+            hasLLM: !!this.llm,
+            hasSchema: !!options.schema,
+        });
+        const startTime = Date.now();
+        try {
+            logger_1.logger.debug({
+                msg: '[DocumentProcessor] Getting document content',
+            });
+            // Determine document source and content
+            const { content, fileType, fileSize } = await this.getDocumentContent(options);
+            // Process document content based on type
+            let processedContent;
+            if (this.isImageFile(fileType)) {
+                processedContent = await this.processImageDocument(content, fileType, options.documentPath);
+            }
+            else {
+                processedContent = await this.processTextDocument(content, options.documentPath, fileType);
+            }
+            logger_1.logger.info({
+                msg: '[DocumentProcessor] Document content processed',
+                fileType,
+                contentLength: processedContent.length,
+            });
+            const processingTime = Date.now() - startTime;
+            // If no LLM is provided, return the raw text content
+            if (!this.llm) {
+                return {
+                    data: processedContent,
+                    metadata: {
+                        fileSize,
+                        fileType,
+                        processingTime,
+                        contentLength: processedContent.length,
+                    },
+                };
+            }
+            // If LLM is provided, extract data (structured or unstructured)
+            const extractedData = await this.extractStructuredData(processedContent, options.schema, options.llmConfig, options.systemPrompt);
+            return {
+                data: extractedData,
+                metadata: {
+                    fileSize,
+                    fileType,
+                    processingTime,
+                    contentLength: processedContent.length,
+                },
+            };
+        }
+        catch (err) {
+            logger_1.logger.error({
+                message: '[DocumentProcessor] Document processing failed',
+                err,
+            });
+            throw new Error(`Document processing failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /**
+     * Get document content from various sources
+     */
+    async getDocumentContent(options) {
+        logger_1.logger.debug({
+            msg: '[DocumentProcessor] getDocumentContent called',
+            hasPath: !!options.documentPath,
+            hasContent: !!options.documentContent,
+            hasUrl: !!options.documentUrl,
+        });
+        // From file path
+        if (options.documentPath) {
+            if (!fs.existsSync(options.documentPath)) {
+                throw new Error(`Document not found: ${options.documentPath}`);
+            }
+            const content = fs.readFileSync(options.documentPath);
+            const fileType = path.extname(options.documentPath).toLowerCase();
+            return {
+                content,
+                fileType,
+                fileSize: content.length,
+            };
+        }
+        // From provided content
+        if (options.documentContent) {
+            // Try to infer file type from content if it's a buffer
+            let fileType = '.unknown';
+            if (Buffer.isBuffer(options.documentContent)) {
+                fileType = this.inferFileTypeFromBuffer(options.documentContent);
+            }
+            else if (typeof options.documentContent === 'string') {
+                fileType = '.txt'; // Assume text content
+            }
+            return {
+                content: options.documentContent,
+                fileType,
+                fileSize: Buffer.isBuffer(options.documentContent) ? options.documentContent.length : Buffer.byteLength(options.documentContent),
+            };
+        }
+        // From URL
+        if (options.documentUrl) {
+            logger_1.logger.debug({
+                msg: '[DocumentProcessor] Fetching document from URL',
+                url: options.documentUrl,
+            });
+            const response = await fetch(options.documentUrl);
+            logger_1.logger.debug({
+                msg: '[DocumentProcessor] URL fetch response',
+                status: response.status,
+                ok: response.ok,
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch document from URL: ${response.statusText}`);
+            }
+            const arrayBuffer = await response.arrayBuffer();
+            const content = Buffer.from(arrayBuffer);
+            const fileType = this.inferFileTypeFromUrl(options.documentUrl) || this.inferFileTypeFromBuffer(content);
+            logger_1.logger.debug({
+                msg: '[DocumentProcessor] Document fetched from URL',
+                contentSize: content.length,
+                fileType,
+            });
+            return {
+                content,
+                fileType,
+                fileSize: content.length,
+            };
+        }
+        throw new Error('No document source provided. Specify documentPath, documentContent, or documentUrl.');
+    }
+    /**
+     * Process image documents by converting them to a standardized format
+     */
+    async processImageDocument(content, fileType, filePath) {
+        try {
+            // First, try to use LlamaParser if available for text extraction
+            if (filePath && this.llamaCloudApiKey) {
+                logger_1.logger.debug({
+                    msg: '[DocumentProcessor] Calling parseWithLlamaCloud for image',
+                    filePath,
+                });
+                const parsedContent = await this.parseWithLlamaCloud(filePath);
+                logger_1.logger.debug({
+                    msg: '[DocumentProcessor] parseWithLlamaCloud returned for image',
+                    hasContent: !!parsedContent,
+                    contentLength: parsedContent === null || parsedContent === void 0 ? void 0 : parsedContent.length,
+                });
+                if (parsedContent) {
+                    return parsedContent;
+                }
+            }
+            // If no file path, create a temporary file for LlamaCloud parsing
+            if (!filePath && this.llamaCloudApiKey) {
+                const tempDir = os.tmpdir();
+                const tempFileName = `temp_${Date.now()}${fileType}`;
+                const tempFilePath = path.join(tempDir, tempFileName);
+                logger_1.logger.debug({
+                    msg: '[DocumentProcessor] Creating temp file for image',
+                    tempFilePath,
+                    contentSize: content.length,
+                });
+                try {
+                    fs.writeFileSync(tempFilePath, content);
+                    logger_1.logger.debug({
+                        msg: '[DocumentProcessor] Calling parseWithLlamaCloud for temp image',
+                        tempFilePath,
+                    });
+                    const parsedContent = await this.parseWithLlamaCloud(tempFilePath);
+                    logger_1.logger.debug({
+                        msg: '[DocumentProcessor] parseWithLlamaCloud returned for temp image',
+                        hasContent: !!parsedContent,
+                        contentLength: parsedContent === null || parsedContent === void 0 ? void 0 : parsedContent.length,
+                    });
+                    fs.unlinkSync(tempFilePath);
+                    if (parsedContent) {
+                        return parsedContent;
+                    }
+                }
+                catch (err) {
+                    // Clean up temp file on error
+                    if (fs.existsSync(tempFilePath)) {
+                        fs.unlinkSync(tempFilePath);
+                    }
+                    logger_1.logger.warn({ msg: '[DocumentProcessor] Failed to parse image with LlamaCloud', err });
+                }
+            }
+            // Fallback: Convert to image format for LLM processing
+            // For PDFs, convert first page to image
+            if (fileType === '.pdf') {
+                throw new Error('Failed to convert PDF to image');
+            }
+            logger_1.logger.warn({
+                msg: '[DocumentProcessor] Sharp module not available. Using original image without optimization.',
+                fileType,
+                contentSize: content.length,
+            });
+            // If sharp is not available, use the original image
+            if (this.config.useBase64) {
+                // Return original image as base64
+                const base64Image = content.toString('base64');
+                const mimeType = this.getMimeType(fileType);
+                return `data:${mimeType};base64,${base64Image}`;
+            }
+            else {
+                // Without sharp and without base64, we cannot process the image
+                return `[IMAGE CONTENT - ${fileType.toUpperCase()} file. Size: ${content.length} bytes. Consider using LLAMA_CLOUD_API_KEY for text extraction or set useBase64: true]`;
+            }
+        }
+        catch (err) {
+            throw new Error(`Failed to process image document: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /**
+     * Process text-based documents using LlamaParser or fallback methods
+     */
+    async processTextDocument(content, filePath, fileType) {
+        // Try LlamaCloud parsing if we have a file path
+        if (filePath && this.llamaCloudApiKey) {
+            const parsedContent = await this.parseWithLlamaCloud(filePath);
+            if (parsedContent) {
+                return parsedContent;
+            }
+        }
+        // If no file path but we have content and LlamaCloud API key, create a temp file
+        if (!filePath && this.llamaCloudApiKey && Buffer.isBuffer(content)) {
+            const tempDir = os.tmpdir();
+            const tempFileName = `temp_${Date.now()}${fileType || '.txt'}`;
+            const tempFilePath = path.join(tempDir, tempFileName);
+            try {
+                fs.writeFileSync(tempFilePath, content);
+                const parsedContent = await this.parseWithLlamaCloud(tempFilePath);
+                fs.unlinkSync(tempFilePath);
+                if (parsedContent) {
+                    return parsedContent;
+                }
+            }
+            catch (err) {
+                // Clean up temp file on error
+                if (fs.existsSync(tempFilePath)) {
+                    fs.unlinkSync(tempFilePath);
+                }
+                logger_1.logger.warn({ msg: '[DocumentProcessor] Failed to parse text document with LlamaCloud', err });
+            }
+        }
+        // Fallback: handle based on file type
+        if (typeof content === 'string') {
+            return content;
+        }
+        // For binary content, convert to text
+        if (fileType === '.pdf' && !this.llamaCloudApiKey) {
+            throw new Error('PDF processing requires LLAMA_CLOUD_API_KEY environment variable for LlamaParser');
+        }
+        // Basic text extraction for simple formats
+        if (['.txt', '.md', '.html', '.htm', '.xml', '.csv'].includes(fileType || '')) {
+            return content.toString('utf-8');
+        }
+        // For unsupported binary formats without LlamaParser
+        throw new Error(`Unsupported document type ${fileType}. Please provide LLAMA_CLOUD_API_KEY for advanced document processing.`);
+    }
+    /**
+     * Extract data using LLM - either structured with schema or unstructured with prompt
+     */
+    async extractStructuredData(content, schema, llmConfig, systemPrompt) {
+        if (!this.llm) {
+            throw new Error('LLM instance is required for data extraction. Please provide an LLM when creating the DocumentProcessor.');
+        }
+        // Note: llmConfig is ignored when using the provided LLM instance
+        // The LLM should already be configured with the desired model and temperature
+        const defaultSystemPrompt = 'You are an expert data-extraction assistant. ' +
+            'Extract the requested information from the provided document content. ' +
+            'If you cannot find a value for a required field, use "N/A" or a descriptive placeholder. ' +
+            'Be accurate and thorough in your extraction.';
+        const finalSystemPrompt = systemPrompt || defaultSystemPrompt;
+        try {
+            // If schema is provided, use structured output
+            if (schema) {
+                // Check if the LLM supports withStructuredOutput
+                if (!('withStructuredOutput' in this.llm)) {
+                    throw new Error('The provided LLM does not support structured output. Please use a compatible LLM instance.');
+                }
+                const structuredLlm = this.llm.withStructuredOutput(schema);
+                const result = await structuredLlm.invoke([
+                    {
+                        role: 'system',
+                        content: finalSystemPrompt,
+                    },
+                    {
+                        role: 'user',
+                        content: `Please extract the following information from this document:\n\n${content}`,
+                    },
+                ]);
+                logger_1.logger.debug({
+                    msg: '[DocumentProcessor] Structured data extraction completed',
+                    extractedData: JSON.stringify(result, null, 2),
+                });
+                return result;
+            }
+            else {
+                // Without schema, return the LLM's text response
+                const response = await this.llm.invoke([
+                    {
+                        role: 'system',
+                        content: finalSystemPrompt,
+                    },
+                    {
+                        role: 'user',
+                        content: `Please analyze and extract information from this document:\n\n${content}`,
+                    },
+                ]);
+                // Extract the text content from the response
+                let textContent;
+                if (typeof response.content === 'string') {
+                    textContent = response.content;
+                }
+                else if (Array.isArray(response.content) && response.content.length > 0) {
+                    // Handle array of content blocks
+                    textContent = response.content.map((block) => (typeof block === 'string' ? block : block.text || '')).join('\n');
+                }
+                else {
+                    textContent = String(response.content);
+                }
+                logger_1.logger.debug({
+                    msg: '[DocumentProcessor] Unstructured data extraction completed',
+                    contentLength: textContent.length,
+                });
+                return textContent;
+            }
+        }
+        catch (err) {
+            throw new Error(`LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /**
+     * Check if file is an image type
+     */
+    isImageFile(fileType) {
+        const imageTypes = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff'];
+        return imageTypes.includes(fileType.toLowerCase());
+    }
+    /**
+     * Infer file type from buffer content
+     */
+    inferFileTypeFromBuffer(buffer) {
+        // Check common file signatures
+        const signatures = {
+            '89504E47': '.png',
+            FFD8FF: '.jpg',
+            '47494638': '.gif',
+            '25504446': '.pdf',
+            '504B0304': '.zip', // Also used by docx, xlsx, pptx
+            D0CF11E0: '.doc', // Also xls, ppt
+        };
+        const hex = buffer.toString('hex', 0, 4).toUpperCase();
+        for (const [signature, type] of Object.entries(signatures)) {
+            if (hex.startsWith(signature)) {
+                return type;
+            }
+        }
+        return '.unknown';
+    }
+    /**
+     * Infer file type from URL
+     */
+    inferFileTypeFromUrl(url) {
+        try {
+            const pathname = new URL(url).pathname;
+            const extension = path.extname(pathname).toLowerCase();
+            return extension || null;
+        }
+        catch (_a) {
+            return null;
+        }
+    }
+    /**
+     * Get list of supported document types
+     */
+    static getSupportedDocumentTypes() {
+        return [...types_1.SUPPORTED_DOCUMENT_TYPES];
+    }
+    /**
+     * Check if a file type is supported
+     */
+    static isDocumentTypeSupported(fileType) {
+        return types_1.SUPPORTED_DOCUMENT_TYPES.includes(fileType.toLowerCase());
+    }
+}
+exports.DocumentProcessor = DocumentProcessor;
+//# sourceMappingURL=documentExtraction.js.map