@dcyfr/ai-rag 1.0.0 → 1.1.0

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2026 DCYFR
+Copyright (c) 2025-2026 DCYFR Labs (https://www.dcyfr.ai)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,13 +1,135 @@
 # @dcyfr/ai-rag
 
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/dcyfr/dcyfr-ai-rag)
+
 > **RAG (Retrieval-Augmented Generation) framework for Node.js and TypeScript**
 
 Build production-ready RAG systems with document loading, embedding, vector stores, and semantic search.
 
 [![npm version](https://img.shields.io/npm/v/@dcyfr/ai-rag.svg)](https://www.npmjs.com/package/@dcyfr/ai-rag)
-[![TypeScript](https://img.shields.io/badge/TypeScript-5.3+-blue.svg)](https://www.typescriptlang.org/)
+[![TypeScript](https://img.shields.io/badge/TypeScript-5.7+-blue.svg)](https://www.typescriptlang.org/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
+---
+- [Document Conversion](#-document-conversion-markitdown)
+- **Document Conversion** - Convert 15+ file formats (PDF, DOCX, PPTX, XLSX, images, etc.) to Markdown via MarkItDown Python library
+---
+
+## 📄 Document Conversion (MarkItDown)
+
+Convert diverse document formats to LLM-optimized Markdown:
+
+```typescript
+import { convertToMarkdown, convertBatch } from '@dcyfr/ai-rag/ingestion';
+
+// Single document conversion
+const result = await convertToMarkdown('/path/to/document.pdf', {
+  timeout: 45000, // 45 seconds
+  maxFileSize: 50 * 1024 * 1024, // 50 MB
+  enableLLMDescriptions: true, // Use GPT-4 Vision for image descriptions
+});
+
+console.log(result.markdown); // Converted markdown content
+console.log(result.metadata); // File size, duration, page count, etc.
+
+// Batch conversion (parallel, concurrency-controlled)
+const files = ['/docs/report.pdf', '/slides/deck.pptx', '/data/sheet.xlsx'];
+const results = await convertBatch(files, { timeout: 60000 });
+
+results.forEach((r, i) => {
+  if (r.success) {
+    console.log(`✅ ${files[i]}: ${r.markdown.length} chars`);
+  } else {
+    console.error(`❌ ${files[i]}: ${r.error}`);
+  }
+});
+```
+
+**Supported Formats:**
+- **Documents:** PDF, DOCX, PPTX, XLSX, CSV, TXT, Markdown
+- **Web:** HTML, XML, JSON
+- **Images:** PNG, JPG, JPEG, GIF, WEBP (with optional LLM-powered OCR)
+- **Audio:** MP3, WAV, M4A (transcription)
+- **Archives:** EPUB, ZIP
+
+**Installation:**
+```bash
+# Python environment required (workspace already configured)
+pip install markitdown>=0.1.5
+
+# Or use workspace .venv (pre-configured)
+source /path/to/workspace/.venv/bin/activate
+```
+
+**Performance:**
+- **Latency:** 200-500ms per document (PDF/Office), <100ms (text/HTML)
+- **Concurrency:** Max 3 parallel conversions (configurable)
+- **Memory:** ~50-200 MB per conversion (temp files auto-cleaned)
+
+**Error Handling:**
+```typescript
+import { ConversionError, ConversionErrorType } from '@dcyfr/ai-rag/ingestion';
+
+try {
+  const result = await convertToMarkdown('/path/to/file.pdf');
+} catch (error) {
+  if (error instanceof ConversionError) {
+    switch (error.type) {
+      case ConversionErrorType.TIMEOUT:
+        console.error('Conversion timed out - file too large?');
+        break;
+      case ConversionErrorType.FILE_TOO_LARGE:
+        console.error(`File exceeds ${error.details?.maxFileSize} bytes`);
+        break;
+      case ConversionErrorType.UNSUPPORTED_FORMAT:
+        console.error('File format not supported by MarkItDown');
+        break;
+      default:
+        console.error(`Conversion failed: ${error.message}`);
+    }
+  }
+}
+```
+
+**LLM Integration (Optional):**
+```typescript
+// Enable GPT-4 Vision or Claude for image descriptions
+const result = await convertToMarkdown('/path/to/presentation.pptx', {
+  enableLLMDescriptions: true,
+  llmModel: 'gpt-4-vision-preview', // or 'claude-3-opus-20240229'
+});
+
+// Requires environment variables:
+// OPENAI_API_KEY=sk-...
+// ANTHROPIC_API_KEY=sk-ant-...
+```
+
+
+## ⚡ 30-Second Quick Start
+
+```bash
+# Install package
+npm install @dcyfr/ai-rag
+
+# Basic usage
+import { TextLoader, InMemoryVectorStore } from '@dcyfr/ai-rag';
+
+const loader = new TextLoader();
+const store = new InMemoryVectorStore();
+# ✅ RAG system ready for document ingestion
+```
+
+---
+
+## 🧭 Related Packages
+
+| Package | Purpose | Type |
+|---------|---------|------|
+| [@dcyfr/ai](../dcyfr-ai) | Core AI harness | npm package |
+| [@dcyfr/ai-agents](../dcyfr-ai-agents) | Autonomous agents | Template |
+| [@dcyfr/ai-chatbot](../dcyfr-ai-chatbot) | Chatbot template | Template |
+| [dcyfr-labs](../dcyfr-labs) | Production Next.js app | Application |
+
 ---
 
 ## ✨ Features
@@ -57,7 +179,7 @@ import {
 // 1. Setup components
 const loader = new TextLoader();
 const embedder = new SimpleEmbeddingGenerator({ dimensions: 384 });
-const store = new InMemory VectorStore({
+const store = new InMemoryVectorStore({
   collectionName: 'my-docs',
   embeddingDimensions: 384,
 });

package/dist/index.d.ts

@@ -5,4 +5,5 @@ export type * from './types/index.js';
 export * from './loaders/index.js';
 export * from './stores/index.js';
 export * from './pipeline/index.js';
+export * from './ingestion/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,mBAAmB,kBAAkB,CAAC;AAGtC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,mBAAmB,CAAC;AAGlC,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,mBAAmB,kBAAkB,CAAC;AAGtC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,mBAAmB,CAAC;AAGlC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,sBAAsB,CAAC"}

package/dist/index.js

@@ -7,4 +7,6 @@ export * from './loaders/index.js';
 export * from './stores/index.js';
 // Pipelines
 export * from './pipeline/index.js';
+// Document ingestion / conversion
+export * from './ingestion/index.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,UAAU;AACV,cAAc,oBAAoB,CAAC;AAEnC,SAAS;AACT,cAAc,mBAAmB,CAAC;AAElC,YAAY;AACZ,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,UAAU;AACV,cAAc,oBAAoB,CAAC;AAEnC,SAAS;AACT,cAAc,mBAAmB,CAAC;AAElC,YAAY;AACZ,cAAc,qBAAqB,CAAC;AAEpC,kCAAkC;AAClC,cAAc,sBAAsB,CAAC"}

package/dist/ingestion/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Document ingestion module for MarkItDown integration
+ * @module @dcyfr/ai-rag/ingestion
+ */
+export { convertToMarkdown, convertBatch, checkMarkItDownInstalled } from './markitdown-bridge.js';
+export type { ConversionOptions, ConversionResult, SupportedFormat, SubprocessMessage, } from './types.js';
+export { ConversionError, ConversionErrorType } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/ingestion/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ingestion/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AACnG,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,EACf,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC"}

package/dist/ingestion/index.js

@@ -0,0 +1,7 @@
+/**
+ * Document ingestion module for MarkItDown integration
+ * @module @dcyfr/ai-rag/ingestion
+ */
+export { convertToMarkdown, convertBatch, checkMarkItDownInstalled } from './markitdown-bridge.js';
+export { ConversionError, ConversionErrorType } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/ingestion/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ingestion/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAOnG,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC"}

package/dist/ingestion/markitdown-bridge.d.ts

@@ -0,0 +1,37 @@
+/**
+ * TypeScript bridge to Python MarkItDown document converter
+ * @module @dcyfr/ai-rag/ingestion/markitdown-bridge
+ */
+import type { ConversionOptions, ConversionResult } from './types.js';
+/**
+ * Convert document to Markdown using Python MarkItDown subprocess
+ *
+ * @param filePath - Absolute path to file to convert
+ * @param options - Conversion options
+ * @returns Conversion result with markdown and metadata
+ *
+ * @throws {ConversionError} If conversion fails
+ *
+ * @example
+ * ```typescript
+ * const result = await convertToMarkdown('/path/to/document.pdf', {
+ *   timeout: 45000,
+ *   enableLLMDescriptions: true
+ * });
+ * console.log(result.markdown);
+ * ```
+ */
+export declare function convertToMarkdown(filePath: string, options?: ConversionOptions): Promise<ConversionResult>;
+/**
+ * Batch convert multiple documents
+ *
+ * @param filePaths - Array of file paths to convert
+ * @param options - Shared conversion options
+ * @returns Array of conversion results (same order as input)
+ */
+export declare function convertBatch(filePaths: string[], options?: ConversionOptions): Promise<ConversionResult[]>;
+/**
+ * Check if Python MarkItDown is installed and accessible
+ */
+export declare function checkMarkItDownInstalled(): Promise<boolean>;
+//# sourceMappingURL=markitdown-bridge.d.ts.map

package/dist/ingestion/markitdown-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markitdown-bridge.d.ts","sourceRoot":"","sources":["../../src/ingestion/markitdown-bridge.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EACV,iBAAiB,EACjB,gBAAgB,EAGjB,MAAM,YAAY,CAAC;AAwRpB;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAoE3B;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EAAE,EACnB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAkC7B;AAED;;GAEG;AACH,wBAAsB,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC,CAmBjE"}

package/dist/ingestion/markitdown-bridge.js

@@ -0,0 +1,325 @@
+/**
+ * TypeScript bridge to Python MarkItDown document converter
+ * @module @dcyfr/ai-rag/ingestion/markitdown-bridge
+ */
+import { spawn } from 'node:child_process';
+import { existsSync, promises as fs } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve, basename, extname, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ConversionError, ConversionErrorType } from './types.js';
+// ES module __dirname equivalent
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+/**
+ * Default conversion options
+ */
+const DEFAULT_OPTIONS = {
+    timeout: 30000, // 30 seconds
+    maxFileSize: 52428800, // 50MB
+    enableLLMDescriptions: false,
+    preserveMetadata: true,
+};
+/**
+ * Supported file extensions mapping to format types
+ */
+const EXTENSION_MAP = {
+    '.pdf': 'pdf',
+    '.docx': 'docx',
+    '.pptx': 'pptx',
+    '.xlsx': 'xlsx',
+    '.csv': 'csv',
+    '.html': 'html',
+    '.htm': 'htm',
+    '.xml': 'xml',
+    '.json': 'json',
+    '.png': 'png',
+    '.jpg': 'jpg',
+    '.jpeg': 'jpeg',
+    '.gif': 'gif',
+    '.webp': 'webp',
+    '.mp3': 'mp3',
+    '.wav': 'wav',
+    '.m4a': 'm4a',
+    '.epub': 'epub',
+    '.zip': 'zip',
+};
+/**
+ * Detect file format from extension
+ */
+function detectFormat(filePath) {
+    const ext = extname(filePath).toLowerCase();
+    const format = EXTENSION_MAP[ext];
+    if (!format) {
+        throw new ConversionError(ConversionErrorType.UNSUPPORTED_FORMAT, `Unsupported file format: ${ext}`, { filePath, extension: ext });
+    }
+    return format;
+}
+/**
+ * Create temporary directory for conversion workspace
+ */
+async function createTempDir() {
+    try {
+        const tempDir = join(tmpdir(), `markitdown-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`);
+        await fs.mkdir(tempDir, { recursive: true });
+        return tempDir;
+    }
+    catch (error) {
+        throw new ConversionError(ConversionErrorType.TEMP_DIR_ERROR, 'Failed to create temporary directory', { error: error instanceof Error ? error.message : String(error) });
+    }
+}
+/**
+ * Clean up temporary directory
+ */
+async function cleanupTempDir(tempDir) {
+    try {
+        await fs.rm(tempDir, { recursive: true, force: true });
+    }
+    catch (error) {
+        // Log warning but don't throw - cleanup failure shouldn't break the flow
+        console.warn(`Failed to cleanup temp directory ${tempDir}:`, error);
+    }
+}
+/**
+ * Validate file accessibility and size
+ */
+async function validateFile(filePath, maxFileSize) {
+    try {
+        const stats = await fs.stat(filePath);
+        if (!stats.isFile()) {
+            throw new ConversionError(ConversionErrorType.FILE_NOT_FOUND, 'Path is not a file', { filePath });
+        }
+        if (stats.size > maxFileSize) {
+            throw new ConversionError(ConversionErrorType.FILE_TOO_LARGE, `File size ${stats.size} bytes exceeds limit ${maxFileSize} bytes`, { filePath, fileSize: stats.size, maxFileSize });
+        }
+    }
+    catch (error) {
+        if (error instanceof ConversionError) {
+            throw error;
+        }
+        throw new ConversionError(ConversionErrorType.FILE_NOT_FOUND, `File not found or inaccessible: ${filePath}`, { error: error instanceof Error ? error.message : String(error) });
+    }
+}
+/**
+ * Find Python executable with worktree-aware fallback order
+ */
+function getPythonExecutable() {
+    const explicitPython = process.env.PYTHON_EXECUTABLE;
+    if (explicitPython && existsSync(explicitPython)) {
+        return explicitPython;
+    }
+    const activeVenv = process.env.VIRTUAL_ENV;
+    if (activeVenv) {
+        const activeVenvPython = join(activeVenv, 'bin', 'python');
+        if (existsSync(activeVenvPython)) {
+            return activeVenvPython;
+        }
+    }
+    const workspaceRoot = resolve(__dirname, '../../../..');
+    const candidatePaths = [
+        join(workspaceRoot, '.venv', 'bin', 'python'),
+        join(workspaceRoot, '..', 'dcyfr-workspace', '.venv', 'bin', 'python'),
+        '/usr/bin/python3',
+    ];
+    for (const candidate of candidatePaths) {
+        if (existsSync(candidate)) {
+            return candidate;
+        }
+    }
+    return 'python3';
+}
+/**
+ * Execute Python subprocess with timeout handling
+ */
+async function executeSubprocess(python, args, options) {
+    const child = spawn(python, args, {
+        cwd: options.cwd,
+        timeout: options.timeout,
+        env: options.env,
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout?.on('data', (data) => {
+        stdout += data.toString();
+    });
+    child.stderr?.on('data', (data) => {
+        stderr += data.toString();
+    });
+    const exitCode = await new Promise((resolve, reject) => {
+        const timeoutId = setTimeout(() => {
+            child.kill('SIGTERM');
+            setTimeout(() => child.kill('SIGKILL'), 1000);
+            reject(new ConversionError(ConversionErrorType.TIMEOUT, `Conversion exceeded timeout of ${options.timeout}ms`, { filePath: options.filePath, timeout: options.timeout }));
+        }, options.timeout);
+        child.on('exit', (code) => {
+            clearTimeout(timeoutId);
+            resolve(code);
+        });
+        child.on('error', (error) => {
+            clearTimeout(timeoutId);
+            reject(new ConversionError(ConversionErrorType.SUBPROCESS_ERROR, `Python subprocess failed: ${error.message}`, { error: error.message, python }));
+        });
+    });
+    return { stdout, stderr, exitCode };
+}
+/**
+ * Build conversion result metadata
+ */
+async function buildMetadata(fileName, resolvedPath, format, durationMs, stderr, opts) {
+    const stats = await fs.stat(resolvedPath);
+    const metadata = {
+        fileName,
+        fileSize: stats.size,
+        format,
+        convertedAt: new Date().toISOString(),
+        durationMs,
+        usedLLMDescriptions: opts.enableLLMDescriptions,
+    };
+    const pageRegex = /(\p{N}+)\s+pages?/iu;
+    const pageMatch = pageRegex.exec(stderr);
+    if (pageMatch) {
+        metadata.pageCount = Number.parseInt(pageMatch[1], 10);
+    }
+    return metadata;
+}
+/**
+ * Validate subprocess execution result
+ */
+function validateSubprocessResult(output, resolvedPath) {
+    if (output.exitCode !== 0) {
+        throw new ConversionError(ConversionErrorType.SUBPROCESS_ERROR, `MarkItDown conversion failed with exit code ${output.exitCode}`, { exitCode: output.exitCode, stderr: output.stderr, stdout: output.stdout, filePath: resolvedPath });
+    }
+    const markdown = output.stdout.trim();
+    if (!markdown) {
+        throw new ConversionError(ConversionErrorType.SUBPROCESS_ERROR, 'MarkItDown returned empty output', { stderr: output.stderr, filePath: resolvedPath });
+    }
+    return markdown;
+}
+/**
+ * Convert document to Markdown using Python MarkItDown subprocess
+ *
+ * @param filePath - Absolute path to file to convert
+ * @param options - Conversion options
+ * @returns Conversion result with markdown and metadata
+ *
+ * @throws {ConversionError} If conversion fails
+ *
+ * @example
+ * ```typescript
+ * const result = await convertToMarkdown('/path/to/document.pdf', {
+ *   timeout: 45000,
+ *   enableLLMDescriptions: true
+ * });
+ * console.log(result.markdown);
+ * ```
+ */
+export async function convertToMarkdown(filePath, options = {}) {
+    const resolvedPath = resolve(filePath);
+    const opts = {
+        ...DEFAULT_OPTIONS,
+        ...options,
+    };
+    await validateFile(resolvedPath, opts.maxFileSize);
+    const format = detectFormat(resolvedPath);
+    const fileName = basename(resolvedPath);
+    let tempDir = null;
+    try {
+        tempDir = await createTempDir();
+        const python = getPythonExecutable();
+        const startMs = Date.now();
+        const output = await executeSubprocess(python, ['-m', 'markitdown', resolvedPath], {
+            cwd: tempDir,
+            timeout: opts.timeout,
+            env: {
+                ...process.env,
+                ...(opts.enableLLMDescriptions && {
+                    OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+                    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
+                    LLM_MODEL: opts.llmModel || 'gpt-4-vision-preview',
+                }),
+            },
+            filePath: resolvedPath,
+        });
+        const markdown = validateSubprocessResult(output, resolvedPath);
+        const durationMs = Date.now() - startMs;
+        const metadata = await buildMetadata(fileName, resolvedPath, format, durationMs, output.stderr, opts);
+        return {
+            markdown,
+            metadata,
+            success: true,
+            warnings: output.stderr ? [output.stderr] : undefined,
+        };
+    }
+    catch (error) {
+        if (error instanceof ConversionError) {
+            throw error;
+        }
+        throw new ConversionError(ConversionErrorType.SUBPROCESS_ERROR, `Unexpected error during conversion: ${error instanceof Error ? error.message : String(error)}`, { error: error instanceof Error ? error.message : String(error), filePath: resolvedPath });
+    }
+    finally {
+        if (tempDir) {
+            await cleanupTempDir(tempDir);
+        }
+    }
+}
+/**
+ * Batch convert multiple documents
+ *
+ * @param filePaths - Array of file paths to convert
+ * @param options - Shared conversion options
+ * @returns Array of conversion results (same order as input)
+ */
+export async function convertBatch(filePaths, options = {}) {
+    // Process conversions in parallel with concurrency limit
+    const MAX_CONCURRENT = 3;
+    const results = [];
+    for (let i = 0; i < filePaths.length; i += MAX_CONCURRENT) {
+        const batch = filePaths.slice(i, i + MAX_CONCURRENT);
+        const batchResults = await Promise.allSettled(batch.map((path) => convertToMarkdown(path, options)));
+        for (const result of batchResults) {
+            if (result.status === 'fulfilled') {
+                results.push(result.value);
+            }
+            else {
+                // Convert rejected promise to failed ConversionResult
+                const error = result.reason;
+                results.push({
+                    markdown: '',
+                    metadata: {
+                        fileName: '',
+                        fileSize: 0,
+                        format: 'pdf',
+                        convertedAt: new Date().toISOString(),
+                        durationMs: 0,
+                    },
+                    success: false,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    }
+    return results;
+}
+/**
+ * Check if Python MarkItDown is installed and accessible
+ */
+export async function checkMarkItDownInstalled() {
+    try {
+        const python = getPythonExecutable();
+        const child = spawn(python, ['-c', 'import markitdown; print("ok")'], {
+            timeout: 5000,
+        });
+        return new Promise((resolve) => {
+            child.on('exit', (code) => {
+                resolve(code === 0);
+            });
+            child.on('error', () => {
+                resolve(false);
+            });
+        });
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=markitdown-bridge.js.map

package/dist/ingestion/markitdown-bridge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markitdown-bridge.js","sourceRoot":"","sources":["../../src/ingestion/markitdown-bridge.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAOzC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAElE,iCAAiC;AACjC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAEtC;;GAEG;AACH,MAAM,eAAe,GAA+G;IAClI,OAAO,EAAE,KAAK,EAAE,aAAa;IAC7B,WAAW,EAAE,QAAQ,EAAE,OAAO;IAC9B,qBAAqB,EAAE,KAAK;IAC5B,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF;;GAEG;AACH,MAAM,aAAa,GAAoC;IACrD,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;CACd,CAAC;AAEF;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB;IACpC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,kBAAkB,EACtC,4BAA4B,GAAG,EAAE,EACjC,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,CAC7B,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa;IAC1B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACrG,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,cAAc,EAClC,sCAAsC,EACtC,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC3C,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yEAAyE;QACzE,OAAO,CAAC,IAAI,CAAC,oCAAoC,OAAO,GAAG,EAAE,KAAK,CAAC,CAAC;IACtE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,WAAmB;IAC/D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEtC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,cAAc,EAClC,oBAAoB,EACpB,EAAE,QAAQ,EAAE,CACb,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,GAAG,WAAW,EAAE,CAAC;YAC7B,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,cAAc,EAClC,aAAa,KAAK,CAAC,IAAI,wBAAwB,WAAW,QAAQ,EAClE,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,CAChD,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;YACrC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,cAAc,EAClC,mCAAmC,QAAQ,EAAE,EAC7C,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB;IAC1B,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IACrD,IAAI,cAAc,IAAI,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QACjD,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;IAC3C,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,gBAAgB,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC3D,IAAI,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACjC,OAAO,gBAAgB,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACxD,MAAM,cAAc,GAAG;QACrB,IAAI,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC;QAC7C,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC;QACtE,kBAAkB;KACnB,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,cAAc,EAAE,CAAC;QACvC,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAWD;;GAEG;AACH,KAAK,UAAU,iBAAiB,CAC9B,MAAc,EACd,IAAc,EACd,OAKC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE;QAChC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IAEH,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,MAAM,GAAG,EAAE,CAAC;IAEhB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAChC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAChC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,IAAI,eAAe,CACxB,mBAAmB,CAAC,OAAO,EAC3B,kCAAkC,OAAO,CAAC,OAAO,IAAI,EACrD,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CACzD,CAAC,CAAC;QACL,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpB,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,CAAC,IAAI,eAAe,CACxB,mBAAmB,CAAC,gBAAgB,EACpC,6BAA6B,KAAK,CAAC,OAAO,EAAE,EAC5C,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,CACjC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAC1B,QAAgB,EAChB,YAAoB,EACpB,MAAuB,EACvB,UAAkB,EAClB,MAAc,EACd,IAAgE;IAEhE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAqB;QACjC,QAAQ;QACR,QAAQ,EAAE,KAAK,CAAC,IAAI;QACpB,MAAM;QACN,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU;QACV,mBAAmB,EAAE,IAAI,CAAC,qBAAqB;KAChD,CAAC;IAEF,MAAM,SAAS,GAAG,qBAAqB,CAAC;IACxC,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,SAAS,EAAE,CAAC;QACd,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,wBAAwB,CAC/B,MAAwB,EACxB,YAAoB;IAEpB,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,gBAAgB,EACpC,+CAA+C,MAAM,CAAC,QAAQ,EAAE,EAChE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,CACpG,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACtC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,gBAAgB,EACpC,kCAAkC,EAClC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,CAClD,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,UAA6B,EAAE;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG;QACX,GAAG,eAAe;QAClB,GAAG,OAAO;KACX,CAAC;IAEF,MAAM,YAAY,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IAExC,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CACpC,MAAM,EACN,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,EAClC;YACE,GAAG,EAAE,OAAO;YACZ,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,EAAE;gBACH,GAAG,OAAO,CAAC,GAAG;gBACd,GAAG,CAAC,IAAI,CAAC,qBAAqB,IAAI;oBAChC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;oBAC1C,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;oBAChD,SAAS,EAAE,IAAI,CAAC,QAAQ,IAAI,sBAAsB;iBACnD,CAAC;aACH;YACD,QAAQ,EAAE,YAAY;SACvB,CACF,CAAC;QAEF,MAAM,QAAQ,GAAG,wBAAwB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACxC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAClC,QAAQ,EACR,YAAY,EACZ,MAAM,EACN,UAAU,EACV,MAAM,CAAC,MAAM,EACb,IAAI,CACL,CAAC;QAEF,OAAO;YACL,QAAQ;YACR,QAAQ;YACR,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;SACtD,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;YACrC,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,IAAI,eAAe,CACvB,mBAAmB,CAAC,gBAAgB,EACpC,uCAAuC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAC1F,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAmB,EACnB,UAA6B,EAAE;IAE/B,yDAAyD;IACzD,MAAM,cAAc,GAAG,CAAC,CAAC;IACzB,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CACtD,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,QAAQ,EAAE,EAAE;oBACZ,QAAQ,EAAE;wBACR,QAAQ,EAAE,EAAE;wBACZ,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,KAAK;wBACb,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACrC,UAAU,EAAE,CAAC;qBACd;oBACD,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,gCAAgC,CAAC,EAAE;YACpE,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACrB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/ingestion/types.d.ts

@@ -0,0 +1,158 @@
+/**
+ * MarkItDown document conversion types
+ * @module @dcyfr/ai-rag/ingestion
+ */
+/**
+ * Supported file formats for conversion
+ */
+export type SupportedFormat = 'pdf' | 'docx' | 'pptx' | 'xlsx' | 'csv' | 'html' | 'htm' | 'xml' | 'json' | 'png' | 'jpg' | 'jpeg' | 'gif' | 'webp' | 'mp3' | 'wav' | 'm4a' | 'epub' | 'zip';
+/**
+ * Options for document conversion
+ */
+export interface ConversionOptions {
+    /**
+     * Maximum time to wait for conversion (milliseconds)
+     * @default 30000 (30 seconds)
+     */
+    timeout?: number;
+    /**
+     * Maximum file size to process (bytes)
+     * @default 52428800 (50MB)
+     */
+    maxFileSize?: number;
+    /**
+     * Enable LLM-powered image descriptions (requires API key)
+     * Supports: OpenAI GPT-4 Vision, Anthropic Claude
+     * @default false
+     */
+    enableLLMDescriptions?: boolean;
+    /**
+     * LLM model to use for image descriptions
+     * @default "gpt-4-vision-preview" or "claude-3-opus-20240229"
+     */
+    llmModel?: string;
+    /**
+     * Preserve original file metadata in result
+     * @default true
+     */
+    preserveMetadata?: boolean;
+    /**
+     * Working directory for temporary files
+     * Auto-created and cleaned up after conversion
+     * @default system temp directory
+     */
+    workDir?: string;
+}
+/**
+ * Metadata about the converted document
+ */
+export interface DocumentMetadata {
+    /**
+     * Original file name
+     */
+    fileName: string;
+    /**
+     * File size in bytes
+     */
+    fileSize: number;
+    /**
+     * Detected or specified file format
+     */
+    format: SupportedFormat;
+    /**
+     * Conversion timestamp (ISO 8601)
+     */
+    convertedAt: string;
+    /**
+     * Conversion duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Number of pages (PDF, DOCX, PPTX) or sections
+     */
+    pageCount?: number;
+    /**
+     * Whether LLM descriptions were used
+     */
+    usedLLMDescriptions?: boolean;
+    /**
+     * Additional format-specific metadata
+     */
+    [key: string]: unknown;
+}
+/**
+ * Result of document conversion
+ */
+export interface ConversionResult {
+    /**
+     * Converted markdown content
+     */
+    markdown: string;
+    /**
+     * Document metadata
+     */
+    metadata: DocumentMetadata;
+    /**
+     * Conversion success status
+     */
+    success: boolean;
+    /**
+     * Error message if conversion failed
+     */
+    error?: string;
+    /**
+     * Warning messages (non-fatal issues)
+     */
+    warnings?: string[];
+}
+/**
+ * Error types for conversion failures
+ */
+export declare enum ConversionErrorType {
+    /** File not found or inaccessible */
+    FILE_NOT_FOUND = "FILE_NOT_FOUND",
+    /** File exceeds maximum size limit */
+    FILE_TOO_LARGE = "FILE_TOO_LARGE",
+    /** File format not supported by MarkItDown */
+    UNSUPPORTED_FORMAT = "UNSUPPORTED_FORMAT",
+    /** Conversion timeout exceeded */
+    TIMEOUT = "TIMEOUT",
+    /** Python subprocess failed to start or crashed */
+    SUBPROCESS_ERROR = "SUBPROCESS_ERROR",
+    /** Temporary directory creation/cleanup failed */
+    TEMP_DIR_ERROR = "TEMP_DIR_ERROR",
+    /** Invalid conversion options provided */
+    INVALID_OPTIONS = "INVALID_OPTIONS",
+    /** Python environment or MarkItDown not installed */
+    PYTHON_ENV_ERROR = "PYTHON_ENV_ERROR",
+    /** LLM API call failed (if enableLLMDescriptions=true) */
+    LLM_API_ERROR = "LLM_API_ERROR"
+}
+/**
+ * Conversion error with typed error code
+ */
+export declare class ConversionError extends Error {
+    readonly type: ConversionErrorType;
+    readonly details?: Record<string, unknown> | undefined;
+    constructor(type: ConversionErrorType, message: string, details?: Record<string, unknown> | undefined);
+}
+/**
+ * Internal subprocess communication message format
+ */
+export interface SubprocessMessage {
+    /** Message type: request or response */
+    type: 'request' | 'response';
+    /** File path to convert */
+    filePath?: string;
+    /** Conversion options */
+    options?: ConversionOptions;
+    /** Conversion result */
+    result?: ConversionResult;
+    /** Error information */
+    error?: {
+        type: string;
+        message: string;
+        details?: Record<string, unknown>;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/ingestion/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GACxC,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,GAC/B,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GACvC,KAAK,GAAG,KAAK,GAAG,KAAK,GACrB,MAAM,GAAG,KAAK,CAAC;AAEnB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B;;OAEG;IACH,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,gBAAgB,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,oBAAY,mBAAmB;IAC7B,qCAAqC;IACrC,cAAc,mBAAmB;IAEjC,sCAAsC;IACtC,cAAc,mBAAmB;IAEjC,8CAA8C;IAC9C,kBAAkB,uBAAuB;IAEzC,kCAAkC;IAClC,OAAO,YAAY;IAEnB,mDAAmD;IACnD,gBAAgB,qBAAqB;IAErC,kDAAkD;IAClD,cAAc,mBAAmB;IAEjC,0CAA0C;IAC1C,eAAe,oBAAoB;IAEnC,qDAAqD;IACrD,gBAAgB,qBAAqB;IAErC,0DAA0D;IAC1D,aAAa,kBAAkB;CAChC;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,KAAK;aAEtB,IAAI,EAAE,mBAAmB;aAEzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;gBAFjC,IAAI,EAAE,mBAAmB,EACzC,OAAO,EAAE,MAAM,EACC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,YAAA;CAKpD;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,IAAI,EAAE,SAAS,GAAG,UAAU,CAAC;IAE7B,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,yBAAyB;IACzB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAE5B,wBAAwB;IACxB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAE1B,wBAAwB;IACxB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC;CACH"}

package/dist/ingestion/types.js

@@ -0,0 +1,42 @@
+/**
+ * MarkItDown document conversion types
+ * @module @dcyfr/ai-rag/ingestion
+ */
+/**
+ * Error types for conversion failures
+ */
+export var ConversionErrorType;
+(function (ConversionErrorType) {
+    /** File not found or inaccessible */
+    ConversionErrorType["FILE_NOT_FOUND"] = "FILE_NOT_FOUND";
+    /** File exceeds maximum size limit */
+    ConversionErrorType["FILE_TOO_LARGE"] = "FILE_TOO_LARGE";
+    /** File format not supported by MarkItDown */
+    ConversionErrorType["UNSUPPORTED_FORMAT"] = "UNSUPPORTED_FORMAT";
+    /** Conversion timeout exceeded */
+    ConversionErrorType["TIMEOUT"] = "TIMEOUT";
+    /** Python subprocess failed to start or crashed */
+    ConversionErrorType["SUBPROCESS_ERROR"] = "SUBPROCESS_ERROR";
+    /** Temporary directory creation/cleanup failed */
+    ConversionErrorType["TEMP_DIR_ERROR"] = "TEMP_DIR_ERROR";
+    /** Invalid conversion options provided */
+    ConversionErrorType["INVALID_OPTIONS"] = "INVALID_OPTIONS";
+    /** Python environment or MarkItDown not installed */
+    ConversionErrorType["PYTHON_ENV_ERROR"] = "PYTHON_ENV_ERROR";
+    /** LLM API call failed (if enableLLMDescriptions=true) */
+    ConversionErrorType["LLM_API_ERROR"] = "LLM_API_ERROR";
+})(ConversionErrorType || (ConversionErrorType = {}));
+/**
+ * Conversion error with typed error code
+ */
+export class ConversionError extends Error {
+    type;
+    details;
+    constructor(type, message, details) {
+        super(message);
+        this.type = type;
+        this.details = details;
+        this.name = 'ConversionError';
+    }
+}
+//# sourceMappingURL=types.js.map

package/dist/ingestion/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAkIH;;GAEG;AACH,MAAM,CAAN,IAAY,mBA2BX;AA3BD,WAAY,mBAAmB;IAC7B,qCAAqC;IACrC,wDAAiC,CAAA;IAEjC,sCAAsC;IACtC,wDAAiC,CAAA;IAEjC,8CAA8C;IAC9C,gEAAyC,CAAA;IAEzC,kCAAkC;IAClC,0CAAmB,CAAA;IAEnB,mDAAmD;IACnD,4DAAqC,CAAA;IAErC,kDAAkD;IAClD,wDAAiC,CAAA;IAEjC,0CAA0C;IAC1C,0DAAmC,CAAA;IAEnC,qDAAqD;IACrD,4DAAqC,CAAA;IAErC,0DAA0D;IAC1D,sDAA+B,CAAA;AACjC,CAAC,EA3BW,mBAAmB,KAAnB,mBAAmB,QA2B9B;AAED;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAEtB;IAEA;IAHlB,YACkB,IAAyB,EACzC,OAAe,EACC,OAAiC;QAEjD,KAAK,CAAC,OAAO,CAAC,CAAC;QAJC,SAAI,GAAJ,IAAI,CAAqB;QAEzB,YAAO,GAAP,OAAO,CAA0B;QAGjD,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF"}

package/dist/pipeline/ingestion/pipeline.d.ts

@@ -7,21 +7,33 @@ import type { DocumentLoader, EmbeddingGenerator, VectorStore, IngestionOptions,
  * Pipeline for ingesting documents into vector store
  */
 export declare class IngestionPipeline {
-    private loader;
-    private embedder;
-    private store;
+    private readonly loader;
+    private readonly embedder;
+    private readonly store;
     constructor(loader: DocumentLoader, embedder: EmbeddingGenerator, store: VectorStore);
     /**
      * Ingest one or more documents
      */
     ingest(filePaths: string | string[], options?: IngestionOptions): Promise<IngestionResult>;
+    private loadDocumentsForPath;
+    private processAndStoreDocuments;
+    private extractErrorType;
+    private updateMemoryMetrics;
     /**
      * Process documents: chunk and embed
      */
     private processDocuments;
     /**
-     * Simple chunking (override with loader-specific chunking if available)
+     * Fixed-size chunking with overlap
      */
-    private chunkDocument;
+    private fixedChunkDocument;
+    /**
+     * Semantic chunking preserving heading-based sections where possible
+     */
+    private semanticChunkDocument;
+    /**
+     * Infer document type from file extension
+     */
+    private inferDocumentType;
 }
 //# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline/ingestion/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAGV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAE9B;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,KAAK,CAAc;gBAGzB,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,EAAE,WAAW;IAOpB;;OAEG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IA6D3B;;OAEG;YACW,gBAAgB;IAqB9B;;OAEG;IACH,OAAO,CAAC,aAAa;CAgCtB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAGV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAG9B;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;gBAGlC,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,EAAE,WAAW;IAOpB;;OAEG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;YA+Fb,oBAAoB;YA0CpB,wBAAwB;IAsCtC,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,mBAAmB;IAW3B;;OAEG;YACW,gBAAgB;IAmC9B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsC7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAS1B"}

package/dist/pipeline/ingestion/pipeline.js

@@ -2,6 +2,7 @@
  * Document ingestion pipeline
  * Orchestrates loading, embedding, and storage
  */
+import { convertToMarkdown } from '../../ingestion/index.js';
 /**
  * Pipeline for ingesting documents into vector store
  */
@@ -20,58 +21,150 @@ export class IngestionPipeline {
     async ingest(filePaths, options) {
         const paths = Array.isArray(filePaths) ? filePaths : [filePaths];
         const startTime = Date.now();
-        const { batchSize = 32, onProgress, loaderConfig, } = options ?? {};
+        const { batchSize = 32, onProgress, loaderConfig, enableDocumentConversion = false, conversionTimeoutMs = 30000, conversionMaxFileSize = 50 * 1024 * 1024, enableLLMDescriptions = false, chunkingStrategy = 'fixed', fixedChunkSize, fixedChunkOverlap, onConversionError, } = options ?? {};
         let totalDocuments = 0;
         let totalChunks = 0;
+        let successCount = 0;
+        let failureCount = 0;
         const errors = [];
+        const conversionDurationsMs = [];
+        let peakHeapUsedBytes = process.memoryUsage().heapUsed;
+        const warnings = [];
         for (let i = 0; i < paths.length; i++) {
             const path = paths[i];
             try {
-                // Load documents
-                const documents = await this.loader.load(path, loaderConfig);
+                const documents = await this.loadDocumentsForPath(path, {
+                    enableDocumentConversion,
+                    conversionTimeoutMs,
+                    conversionMaxFileSize,
+                    enableLLMDescriptions,
+                    loaderConfig,
+                    conversionDurationsMs,
+                });
                 totalDocuments += documents.length;
-                // Chunk and embed in batches
-                const allChunks = [];
-                for (let j = 0; j < documents.length; j += batchSize) {
-                    const batch = documents.slice(j, j + batchSize);
-                    const chunks = await this.processDocuments(batch);
-                    allChunks.push(...chunks);
-                    if (onProgress) {
-                        onProgress(i + 1, paths.length, {
-                            currentFile: path,
-                            documentsProcessed: Math.min(j + batchSize, documents.length),
-                            totalDocuments: documents.length,
-                            chunksGenerated: allChunks.length,
-                        });
-                    }
-                }
-                // Store chunks
-                await this.store.addDocuments(allChunks);
-                totalChunks += allChunks.length;
+                const chunkCount = await this.processAndStoreDocuments(path, documents, {
+                    batchSize,
+                    chunkingStrategy,
+                    fixedChunkSize,
+                    fixedChunkOverlap,
+                    onProgress,
+                    fileIndex: i,
+                    totalFiles: paths.length,
+                });
+                totalChunks += chunkCount;
+                successCount++;
             }
             catch (error) {
-                errors.push({
-                    file: path,
-                    error: error instanceof Error ? error.message : String(error),
-                });
+                failureCount++;
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                errors.push({ file: path, error: errorMessage });
+                if (enableDocumentConversion && onConversionError) {
+                    onConversionError({
+                        file: path,
+                        error: errorMessage,
+                        errorType: this.extractErrorType(error),
+                    });
+                }
             }
+            peakHeapUsedBytes = this.updateMemoryMetrics(path, peakHeapUsedBytes, warnings);
         }
         const endTime = Date.now();
+        const durationMs = endTime - startTime;
+        const averageConversionMs = conversionDurationsMs.length > 0
+            ? conversionDurationsMs.reduce((sum, ms) => sum + ms, 0) / conversionDurationsMs.length
+            : undefined;
+        const documentsPerSecond = durationMs > 0 ? (successCount / durationMs) * 1000 : 0;
         return {
             documentsProcessed: totalDocuments,
+            successCount,
+            failureCount,
             chunksGenerated: totalChunks,
             errors,
-            durationMs: endTime - startTime,
+            metrics: {
+                averageConversionMs,
+                documentsPerSecond,
+                peakHeapUsedMb: peakHeapUsedBytes / (1024 * 1024),
+                warnings,
+            },
+            durationMs,
         };
     }
+    async loadDocumentsForPath(path, config) {
+        if (!config.enableDocumentConversion) {
+            return this.loader.load(path, config.loaderConfig);
+        }
+        const conversionStart = Date.now();
+        const converted = await convertToMarkdown(path, {
+            timeout: config.conversionTimeoutMs,
+            maxFileSize: config.conversionMaxFileSize,
+            enableLLMDescriptions: config.enableLLMDescriptions,
+            preserveMetadata: true,
+        });
+        config.conversionDurationsMs.push(Date.now() - conversionStart);
+        return [
+            {
+                id: `${path}-${Date.now()}`,
+                content: converted.markdown,
+                metadata: {
+                    source: path,
+                    type: this.inferDocumentType(path),
+                    createdAt: new Date(),
+                    conversionTimestamp: converted.metadata.convertedAt,
+                    originalFileType: converted.metadata.format,
+                    pageCount: converted.metadata.pageCount,
+                    conversionDurationMs: converted.metadata.durationMs,
+                    usedLLMDescriptions: converted.metadata.usedLLMDescriptions,
+                },
+            },
+        ];
+    }
+    async processAndStoreDocuments(path, documents, config) {
+        const allChunks = [];
+        for (let j = 0; j < documents.length; j += config.batchSize) {
+            const batch = documents.slice(j, j + config.batchSize);
+            const chunks = await this.processDocuments(batch, {
+                chunkingStrategy: config.chunkingStrategy,
+                fixedChunkSize: config.fixedChunkSize,
+                fixedChunkOverlap: config.fixedChunkOverlap,
+            });
+            allChunks.push(...chunks);
+            if (config.onProgress) {
+                config.onProgress(config.fileIndex + 1, config.totalFiles, {
+                    currentFile: path,
+                    documentsProcessed: Math.min(j + config.batchSize, documents.length),
+                    totalDocuments: documents.length,
+                    chunksGenerated: allChunks.length,
+                });
+            }
+        }
+        await this.store.addDocuments(allChunks);
+        return allChunks.length;
+    }
+    extractErrorType(error) {
+        if (typeof error !== 'object' || error === null || !('type' in error)) {
+            return undefined;
+        }
+        const rawType = error.type;
+        return typeof rawType === 'string' ? rawType : undefined;
+    }
+    updateMemoryMetrics(path, currentPeak, warnings) {
+        const currentHeap = process.memoryUsage().heapUsed;
+        const nextPeak = Math.max(currentHeap, currentPeak);
+        if (currentHeap > 512 * 1024 * 1024) {
+            warnings.push(`High memory usage detected while processing ${path}: ${(currentHeap / (1024 * 1024)).toFixed(1)}MB`);
+        }
+        return nextPeak;
+    }
     /**
      * Process documents: chunk and embed
      */
-    async processDocuments(documents) {
+    async processDocuments(documents, options) {
         const chunks = [];
         for (const doc of documents) {
-            // Split into chunks (using simple text splitting)
-            const docChunks = this.chunkDocument(doc);
+            // Split into chunks
+            const docChunks = options?.chunkingStrategy === 'semantic'
+                ? this.semanticChunkDocument(doc)
+                : this.fixedChunkDocument(doc, options?.fixedChunkSize ?? 1000, options?.fixedChunkOverlap ?? 200);
             chunks.push(...docChunks);
         }
         // Generate embeddings
@@ -84,11 +177,9 @@ export class IngestionPipeline {
         return chunks;
     }
     /**
-     * Simple chunking (override with loader-specific chunking if available)
+     * Fixed-size chunking with overlap
      */
-    chunkDocument(doc) {
-        const chunkSize = 1000;
-        const overlap = 200;
+    fixedChunkDocument(doc, chunkSize, overlap) {
         const chunks = [];
         const content = doc.content;
         for (let i = 0; i < content.length; i += chunkSize - overlap) {
@@ -114,5 +205,57 @@ export class IngestionPipeline {
         }
         return chunks;
     }
+    /**
+     * Semantic chunking preserving heading-based sections where possible
+     */
+    semanticChunkDocument(doc) {
+        const chunks = [];
+        const sections = doc.content
+            .split(/\n(?=#{1,6}\s)/g)
+            .filter((section) => section.trim().length > 0);
+        if (sections.length <= 1) {
+            return this.fixedChunkDocument(doc, 1000, 200);
+        }
+        let cursor = 0;
+        for (let i = 0; i < sections.length; i++) {
+            const section = sections[i];
+            const startChar = doc.content.indexOf(section, cursor);
+            const endChar = startChar + section.length;
+            const chunkId = `${doc.id}-chunk-${i}`;
+            chunks.push({
+                id: chunkId,
+                documentId: doc.id,
+                content: section,
+                index: i,
+                metadata: {
+                    chunkIndex: i,
+                    chunkCount: sections.length,
+                    startChar,
+                    endChar,
+                    chunkingStrategy: 'semantic',
+                    ...doc.metadata,
+                },
+            });
+            cursor = endChar;
+        }
+        return chunks;
+    }
+    /**
+     * Infer document type from file extension
+     */
+    inferDocumentType(path) {
+        const lowerPath = path.toLowerCase();
+        if (lowerPath.endsWith('.pdf'))
+            return 'pdf';
+        if (lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown'))
+            return 'markdown';
+        if (lowerPath.endsWith('.html') || lowerPath.endsWith('.htm'))
+            return 'html';
+        if (lowerPath.endsWith('.json'))
+            return 'json';
+        if (lowerPath.endsWith('.txt'))
+            return 'text';
+        return 'other';
+    }
 }
 //# sourceMappingURL=pipeline.js.map

package/dist/pipeline/ingestion/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAYH;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAiB;IACvB,QAAQ,CAAqB;IAC7B,KAAK,CAAc;IAE3B,YACE,MAAsB,EACtB,QAA4B,EAC5B,KAAkB;QAElB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,SAA4B,EAC5B,OAA0B;QAE1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,EACJ,SAAS,GAAG,EAAE,EACd,UAAU,EACV,YAAY,GACb,GAAG,OAAO,IAAI,EAAE,CAAC;QAElB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,MAAM,GAA2C,EAAE,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,CAAC;gBACH,iBAAiB;gBACjB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC7D,cAAc,IAAI,SAAS,CAAC,MAAM,CAAC;gBAEnC,6BAA6B;gBAC7B,MAAM,SAAS,GAAoB,EAAE,CAAC;gBAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;oBACrD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;oBAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBAClD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAE1B,IAAI,UAAU,EAAE,CAAC;wBACf,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE;4BAC9B,WAAW,EAAE,IAAI;4BACjB,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC;4BAC7D,cAAc,EAAE,SAAS,CAAC,MAAM;4BAChC,eAAe,EAAE,SAAS,CAAC,MAAM;yBAClC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,eAAe;gBACf,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,OAAO;YACL,kBAAkB,EAAE,cAAc;YAClC,eAAe,EAAE,WAAW;YAC5B,MAAM;YACN,UAAU,EAAE,OAAO,GAAG,SAAS;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,SAAqB;QAClD,MAAM,MAAM,GAAoB,EAAE,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,kDAAkD;YAClD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpD,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,GAAa;QACjC,MAAM,SAAS,GAAG,IAAI,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,CAAC;QACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,QAAQ,EAAE;oBACR,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,UAAU,EAAE,CAAC,EAAE,oBAAoB;oBACnC,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC;oBAChD,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACX,MAAM,CAAiB;IACvB,QAAQ,CAAqB;IAC7B,KAAK,CAAc;IAEpC,YACE,MAAsB,EACtB,QAA4B,EAC5B,KAAkB;QAElB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,SAA4B,EAC5B,OAA0B;QAE1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,EACJ,SAAS,GAAG,EAAE,EACd,UAAU,EACV,YAAY,EACZ,wBAAwB,GAAG,KAAK,EAChC,mBAAmB,GAAG,KAAK,EAC3B,qBAAqB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EACxC,qBAAqB,GAAG,KAAK,EAC7B,gBAAgB,GAAG,OAAO,EAC1B,cAAc,EACd,iBAAiB,EACjB,iBAAiB,GAClB,GAAG,OAAO,IAAI,EAAE,CAAC;QAElB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,MAAM,GAA2C,EAAE,CAAC;QAC1D,MAAM,qBAAqB,GAAa,EAAE,CAAC;QAC3C,IAAI,iBAAiB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;QACvD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE;oBACtD,wBAAwB;oBACxB,mBAAmB;oBACnB,qBAAqB;oBACrB,qBAAqB;oBACrB,YAAY;oBACZ,qBAAqB;iBACtB,CAAC,CAAC;gBAEH,cAAc,IAAI,SAAS,CAAC,MAAM,CAAC;gBAEnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,SAAS,EAAE;oBACtE,SAAS;oBACT,gBAAgB;oBAChB,cAAc;oBACd,iBAAiB;oBACjB,UAAU;oBACV,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,KAAK,CAAC,MAAM;iBACzB,CAAC,CAAC;gBAEH,WAAW,IAAI,UAAU,CAAC;gBAC1B,YAAY,EAAE,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;gBAEjD,IAAI,wBAAwB,IAAI,iBAAiB,EAAE,CAAC;oBAClD,iBAAiB,CAAC;wBAChB,IAAI,EAAE,IAAI;wBACV,KAAK,EAAE,YAAY;wBACnB,SAAS,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;qBACxC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,OAAO,GAAG,SAAS,CAAC;QACvC,MAAM,mBAAmB,GACvB,qBAAqB,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,qBAAqB,CAAC,MAAM;YACvF,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,kBAAkB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnF,OAAO;YACL,kBAAkB,EAAE,cAAc;YAClC,YAAY;YACZ,YAAY;YACZ,eAAe,EAAE,WAAW;YAC5B,MAAM;YACN,OAAO,EAAE;gBACP,mBAAmB;gBACnB,kBAAkB;gBAClB,cAAc,EAAE,iBAAiB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;gBACjD,QAAQ;aACT;YACD,UAAU;SACX,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,IAAY,EACZ,MAOC;QAED,IAAI,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE;YAC9C,OAAO,EAAE,MAAM,CAAC,mBAAmB;YACnC,WAAW,EAAE,MAAM,CAAC,qBAAqB;YACzC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;YACnD,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;QACH,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QAEhE,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC3B,OAAO,EAAE,SAAS,CAAC,QAAQ;gBAC3B,QAAQ,EAAE;oBACR,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAClC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACrB,mBAAmB,EAAE,SAAS,CAAC,QAAQ,CAAC,WAAW;oBACnD,gBAAgB,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM;oBAC3C,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,SAAS;oBACvC,oBAAoB,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU;oBACnD,mBAAmB,EAAE,SAAS,CAAC,QAAQ,CAAC,mBAAmB;iBAC5D;aACF;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,IAAY,EACZ,SAAqB,EACrB,MAQC;QAED,MAAM,SAAS,GAAoB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE;gBAChD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;aAC5C,CAAC,CAAC;YACH,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAE1B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE;oBACzD,WAAW,EAAE,IAAI;oBACjB,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC;oBACpE,cAAc,EAAE,SAAS,CAAC,MAAM;oBAChC,eAAe,EAAE,SAAS,CAAC,MAAM;iBAClC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,SAAS,CAAC,MAAM,CAAC;IAC1B,CAAC;IAEO,gBAAgB,CAAC,KAAc;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAI,KAA4B,CAAC,IAAI,CAAC;QACnD,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,CAAC;IAEO,mBAAmB,CAAC,IAAY,EAAE,WAAmB,EAAE,QAAkB;QAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACpD,IAAI,WAAW,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CACX,+CAA+C,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACrG,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAC5B,SAAqB,EACrB,OAIC;QAED,MAAM,MAAM,GAAoB,EAAE,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,oBAAoB;YACpB,MAAM,SAAS,GACb,OAAO,EAAE,gBAAgB,KAAK,UAAU;gBACtC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC;gBACjC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CACrB,GAAG,EACH,OAAO,EAAE,cAAc,IAAI,IAAI,EAC/B,OAAO,EAAE,iBAAiB,IAAI,GAAG,CAClC,CAAC;YACR,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpD,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,GAAa,EAAE,SAAiB,EAAE,OAAe;QAC1E,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,QAAQ,EAAE;oBACR,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,UAAU,EAAE,CAAC,EAAE,oBAAoB;oBACnC,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC;oBAChD,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,GAAa;QACzC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO;aACzB,KAAK,CAAC,iBAAiB,CAAC;aACxB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAElD,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;YAC3C,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YAEvC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE;oBACR,UAAU,EAAE,CAAC;oBACb,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,SAAS;oBACT,OAAO;oBACP,gBAAgB,EAAE,UAAU;oBAC5B,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC,CAAC;YAEH,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAAY;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,UAAU,CAAC;QACpF,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAC7E,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,OAAO,MAAM,CAAC;QAC/C,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAC9C,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/types/index.d.ts

@@ -232,6 +232,26 @@ export interface IngestionOptions {
     batchSize?: number;
     /** Loader configuration */
     loaderConfig?: LoaderConfig;
+    /** Enable MarkItDown preprocessing for non-text documents */
+    enableDocumentConversion?: boolean;
+    /** Conversion timeout in milliseconds */
+    conversionTimeoutMs?: number;
+    /** Conversion file size limit in bytes */
+    conversionMaxFileSize?: number;
+    /** Enable LLM image descriptions during conversion */
+    enableLLMDescriptions?: boolean;
+    /** Chunking strategy */
+    chunkingStrategy?: 'semantic' | 'fixed';
+    /** Fixed chunk size (characters) */
+    fixedChunkSize?: number;
+    /** Fixed chunk overlap (characters) */
+    fixedChunkOverlap?: number;
+    /** Callback for conversion errors */
+    onConversionError?: (error: {
+        file: string;
+        error: string;
+        errorType?: string;
+    }) => void;
     /** Progress callback */
     onProgress?: (current: number, total: number, details?: {
         currentFile: string;
@@ -246,6 +266,10 @@ export interface IngestionOptions {
 export interface IngestionResult {
     /** Number of documents processed */
     documentsProcessed: number;
+    /** Number of input files successfully processed */
+    successCount: number;
+    /** Number of input files that failed processing */
+    failureCount: number;
     /** Number of chunks generated */
     chunksGenerated: number;
     /** Errors during ingestion */
@@ -253,6 +277,17 @@ export interface IngestionResult {
         file: string;
         error: string;
     }>;
+    /** Performance and resource metrics */
+    metrics?: {
+        /** Average conversion duration per file in milliseconds */
+        averageConversionMs?: number;
+        /** Documents converted per second */
+        documentsPerSecond?: number;
+        /** Peak heap usage in MB during ingestion */
+        peakHeapUsedMb?: number;
+        /** Warnings generated during ingestion */
+        warnings?: string[];
+    };
     /** Total duration in milliseconds */
     durationMs: number;
 }

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,IAAI,EAAE,KAAK,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9D,yBAAyB;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,yBAAyB;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,uBAAuB;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,gCAAgC;IAChC,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,mCAAmC;IACnC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,+BAA+B;IAC/B,aAAa,IAAI,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,cAAc,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,CAAC;IAChD,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,iCAAiC;IACjC,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,mCAAmC;IACnC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACnG,6BAA6B;IAC7B,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,sBAAsB;IACtB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,yBAAyB;IACzB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IACvD,0BAA0B;IAC1B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe;IACf,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;IACnE,uBAAuB;IACvB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,iCAAiC;IACjC,WAAW,EAAE,iBAAiB,CAAC;IAC/B,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2BAA2B;IAC3B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gCAAgC;IAChC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uBAAuB;IACvB,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,kCAAkC;IAClC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qBAAqB;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,EAAE;QACR,6BAA6B;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,UAAU,EAAE,MAAM,CAAC;QACnB,8BAA8B;QAC9B,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,wBAAwB;IACxB,UAAU,CAAC,EAAE,CACX,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB,KACE,IAAI,CAAC;CACX;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,IAAI,EAAE,KAAK,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9D,yBAAyB;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,yBAAyB;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,uBAAuB;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,gCAAgC;IAChC,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,mCAAmC;IACnC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,+BAA+B;IAC/B,aAAa,IAAI,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,cAAc,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,CAAC;IAChD,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,iCAAiC;IACjC,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,mCAAmC;IACnC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACnG,6BAA6B;IAC7B,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,sBAAsB;IACtB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,yBAAyB;IACzB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IACvD,0BAA0B;IAC1B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe;IACf,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;IACnE,uBAAuB;IACvB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,iCAAiC;IACjC,WAAW,EAAE,iBAAiB,CAAC;IAC/B,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2BAA2B;IAC3B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gCAAgC;IAChC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uBAAuB;IACvB,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,kCAAkC;IAClC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qBAAqB;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,EAAE;QACR,6BAA6B;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,UAAU,EAAE,MAAM,CAAC;QACnB,8BAA8B;QAC9B,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,6DAA6D;IAC7D,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,yCAAyC;IACzC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,0CAA0C;IAC1C,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sDAAsD;IACtD,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,wBAAwB;IACxB,gBAAgB,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC;IACxC,oCAAoC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uCAAuC;IACvC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qCAAqC;IACrC,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE;QAC1B,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,KAAK,IAAI,CAAC;IACX,wBAAwB;IACxB,UAAU,CAAC,EAAE,CACX,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB,KACE,IAAI,CAAC;CACX;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,uCAAuC;IACvC,OAAO,CAAC,EAAE;QACR,2DAA2D;QAC3D,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,qCAAqC;QACrC,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,6CAA6C;QAC7C,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,0CAA0C;QAC1C,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dcyfr/ai-rag",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "RAG (Retrieval-Augmented Generation) system template - DCYFR AI starter",
   "type": "module",
   "main": "./dist/index.js",
@@ -77,7 +77,7 @@
     "zod": "^3.22.4"
   },
   "peerDependencies": {
-    "@dcyfr/ai": "^1.0.0",
+    "@dcyfr/ai": "^2.1.0",
     "chromadb": "^1.8.0"
   },
   "peerDependenciesMeta": {
@@ -90,14 +90,13 @@
   },
   "devDependencies": {
     "@changesets/changelog-github": "^0.5.2",
-    "@changesets/cli": "^2.29.8",
-    "@types/node": "^20.11.0",
-    "@typescript-eslint/eslint-plugin": "^6.19.0",
-    "@typescript-eslint/parser": "^6.19.0",
+    "@changesets/cli": "^2.30.0",
+    "@types/node": "^22.0.0",
     "@vitest/coverage-v8": "^4.0.18",
-    "eslint": "^8.56.0",
+    "eslint": "^9.39.2",
     "tsx": "^4.7.0",
     "typescript": "^5.3.3",
+    "typescript-eslint": "^8.56.0",
     "vitest": "^4.0.18"
   },
   "engines": {