@dcyfr/ai-rag 0.2.0 → 1.1.0

package/dist/ingestion/types.d.ts

@@ -0,0 +1,158 @@
+/**
+ * MarkItDown document conversion types
+ * @module @dcyfr/ai-rag/ingestion
+ */
+/**
+ * Supported file formats for conversion
+ */
+export type SupportedFormat = 'pdf' | 'docx' | 'pptx' | 'xlsx' | 'csv' | 'html' | 'htm' | 'xml' | 'json' | 'png' | 'jpg' | 'jpeg' | 'gif' | 'webp' | 'mp3' | 'wav' | 'm4a' | 'epub' | 'zip';
+/**
+ * Options for document conversion
+ */
+export interface ConversionOptions {
+    /**
+     * Maximum time to wait for conversion (milliseconds)
+     * @default 30000 (30 seconds)
+     */
+    timeout?: number;
+    /**
+     * Maximum file size to process (bytes)
+     * @default 52428800 (50MB)
+     */
+    maxFileSize?: number;
+    /**
+     * Enable LLM-powered image descriptions (requires API key)
+     * Supports: OpenAI GPT-4 Vision, Anthropic Claude
+     * @default false
+     */
+    enableLLMDescriptions?: boolean;
+    /**
+     * LLM model to use for image descriptions
+     * @default "gpt-4-vision-preview" or "claude-3-opus-20240229"
+     */
+    llmModel?: string;
+    /**
+     * Preserve original file metadata in result
+     * @default true
+     */
+    preserveMetadata?: boolean;
+    /**
+     * Working directory for temporary files
+     * Auto-created and cleaned up after conversion
+     * @default system temp directory
+     */
+    workDir?: string;
+}
+/**
+ * Metadata about the converted document
+ */
+export interface DocumentMetadata {
+    /**
+     * Original file name
+     */
+    fileName: string;
+    /**
+     * File size in bytes
+     */
+    fileSize: number;
+    /**
+     * Detected or specified file format
+     */
+    format: SupportedFormat;
+    /**
+     * Conversion timestamp (ISO 8601)
+     */
+    convertedAt: string;
+    /**
+     * Conversion duration in milliseconds
+     */
+    durationMs: number;
+    /**
+     * Number of pages (PDF, DOCX, PPTX) or sections
+     */
+    pageCount?: number;
+    /**
+     * Whether LLM descriptions were used
+     */
+    usedLLMDescriptions?: boolean;
+    /**
+     * Additional format-specific metadata
+     */
+    [key: string]: unknown;
+}
+/**
+ * Result of document conversion
+ */
+export interface ConversionResult {
+    /**
+     * Converted markdown content
+     */
+    markdown: string;
+    /**
+     * Document metadata
+     */
+    metadata: DocumentMetadata;
+    /**
+     * Conversion success status
+     */
+    success: boolean;
+    /**
+     * Error message if conversion failed
+     */
+    error?: string;
+    /**
+     * Warning messages (non-fatal issues)
+     */
+    warnings?: string[];
+}
+/**
+ * Error types for conversion failures
+ */
+export declare enum ConversionErrorType {
+    /** File not found or inaccessible */
+    FILE_NOT_FOUND = "FILE_NOT_FOUND",
+    /** File exceeds maximum size limit */
+    FILE_TOO_LARGE = "FILE_TOO_LARGE",
+    /** File format not supported by MarkItDown */
+    UNSUPPORTED_FORMAT = "UNSUPPORTED_FORMAT",
+    /** Conversion timeout exceeded */
+    TIMEOUT = "TIMEOUT",
+    /** Python subprocess failed to start or crashed */
+    SUBPROCESS_ERROR = "SUBPROCESS_ERROR",
+    /** Temporary directory creation/cleanup failed */
+    TEMP_DIR_ERROR = "TEMP_DIR_ERROR",
+    /** Invalid conversion options provided */
+    INVALID_OPTIONS = "INVALID_OPTIONS",
+    /** Python environment or MarkItDown not installed */
+    PYTHON_ENV_ERROR = "PYTHON_ENV_ERROR",
+    /** LLM API call failed (if enableLLMDescriptions=true) */
+    LLM_API_ERROR = "LLM_API_ERROR"
+}
+/**
+ * Conversion error with typed error code
+ */
+export declare class ConversionError extends Error {
+    readonly type: ConversionErrorType;
+    readonly details?: Record<string, unknown> | undefined;
+    constructor(type: ConversionErrorType, message: string, details?: Record<string, unknown> | undefined);
+}
+/**
+ * Internal subprocess communication message format
+ */
+export interface SubprocessMessage {
+    /** Message type: request or response */
+    type: 'request' | 'response';
+    /** File path to convert */
+    filePath?: string;
+    /** Conversion options */
+    options?: ConversionOptions;
+    /** Conversion result */
+    result?: ConversionResult;
+    /** Error information */
+    error?: {
+        type: string;
+        message: string;
+        details?: Record<string, unknown>;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/ingestion/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GACxC,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,GAC/B,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GACvC,KAAK,GAAG,KAAK,GAAG,KAAK,GACrB,MAAM,GAAG,KAAK,CAAC;AAEnB;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,eAAe,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B;;OAEG;IACH,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,gBAAgB,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,oBAAY,mBAAmB;IAC7B,qCAAqC;IACrC,cAAc,mBAAmB;IAEjC,sCAAsC;IACtC,cAAc,mBAAmB;IAEjC,8CAA8C;IAC9C,kBAAkB,uBAAuB;IAEzC,kCAAkC;IAClC,OAAO,YAAY;IAEnB,mDAAmD;IACnD,gBAAgB,qBAAqB;IAErC,kDAAkD;IAClD,cAAc,mBAAmB;IAEjC,0CAA0C;IAC1C,eAAe,oBAAoB;IAEnC,qDAAqD;IACrD,gBAAgB,qBAAqB;IAErC,0DAA0D;IAC1D,aAAa,kBAAkB;CAChC;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,KAAK;aAEtB,IAAI,EAAE,mBAAmB;aAEzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;gBAFjC,IAAI,EAAE,mBAAmB,EACzC,OAAO,EAAE,MAAM,EACC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,YAAA;CAKpD;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,IAAI,EAAE,SAAS,GAAG,UAAU,CAAC;IAE7B,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,yBAAyB;IACzB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAE5B,wBAAwB;IACxB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAE1B,wBAAwB;IACxB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC;CACH"}

package/dist/ingestion/types.js

@@ -0,0 +1,42 @@
+/**
+ * MarkItDown document conversion types
+ * @module @dcyfr/ai-rag/ingestion
+ */
+/**
+ * Error types for conversion failures
+ */
+export var ConversionErrorType;
+(function (ConversionErrorType) {
+    /** File not found or inaccessible */
+    ConversionErrorType["FILE_NOT_FOUND"] = "FILE_NOT_FOUND";
+    /** File exceeds maximum size limit */
+    ConversionErrorType["FILE_TOO_LARGE"] = "FILE_TOO_LARGE";
+    /** File format not supported by MarkItDown */
+    ConversionErrorType["UNSUPPORTED_FORMAT"] = "UNSUPPORTED_FORMAT";
+    /** Conversion timeout exceeded */
+    ConversionErrorType["TIMEOUT"] = "TIMEOUT";
+    /** Python subprocess failed to start or crashed */
+    ConversionErrorType["SUBPROCESS_ERROR"] = "SUBPROCESS_ERROR";
+    /** Temporary directory creation/cleanup failed */
+    ConversionErrorType["TEMP_DIR_ERROR"] = "TEMP_DIR_ERROR";
+    /** Invalid conversion options provided */
+    ConversionErrorType["INVALID_OPTIONS"] = "INVALID_OPTIONS";
+    /** Python environment or MarkItDown not installed */
+    ConversionErrorType["PYTHON_ENV_ERROR"] = "PYTHON_ENV_ERROR";
+    /** LLM API call failed (if enableLLMDescriptions=true) */
+    ConversionErrorType["LLM_API_ERROR"] = "LLM_API_ERROR";
+})(ConversionErrorType || (ConversionErrorType = {}));
+/**
+ * Conversion error with typed error code
+ */
+export class ConversionError extends Error {
+    type;
+    details;
+    constructor(type, message, details) {
+        super(message);
+        this.type = type;
+        this.details = details;
+        this.name = 'ConversionError';
+    }
+}
+//# sourceMappingURL=types.js.map

package/dist/ingestion/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAkIH;;GAEG;AACH,MAAM,CAAN,IAAY,mBA2BX;AA3BD,WAAY,mBAAmB;IAC7B,qCAAqC;IACrC,wDAAiC,CAAA;IAEjC,sCAAsC;IACtC,wDAAiC,CAAA;IAEjC,8CAA8C;IAC9C,gEAAyC,CAAA;IAEzC,kCAAkC;IAClC,0CAAmB,CAAA;IAEnB,mDAAmD;IACnD,4DAAqC,CAAA;IAErC,kDAAkD;IAClD,wDAAiC,CAAA;IAEjC,0CAA0C;IAC1C,0DAAmC,CAAA;IAEnC,qDAAqD;IACrD,4DAAqC,CAAA;IAErC,0DAA0D;IAC1D,sDAA+B,CAAA;AACjC,CAAC,EA3BW,mBAAmB,KAAnB,mBAAmB,QA2B9B;AAED;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAEtB;IAEA;IAHlB,YACkB,IAAyB,EACzC,OAAe,EACC,OAAiC;QAEjD,KAAK,CAAC,OAAO,CAAC,CAAC;QAJC,SAAI,GAAJ,IAAI,CAAqB;QAEzB,YAAO,GAAP,OAAO,CAA0B;QAGjD,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF"}

package/dist/pipeline/ingestion/pipeline.d.ts

@@ -7,21 +7,33 @@ import type { DocumentLoader, EmbeddingGenerator, VectorStore, IngestionOptions,
  * Pipeline for ingesting documents into vector store
  */
 export declare class IngestionPipeline {
-    private loader;
-    private embedder;
-    private store;
+    private readonly loader;
+    private readonly embedder;
+    private readonly store;
     constructor(loader: DocumentLoader, embedder: EmbeddingGenerator, store: VectorStore);
     /**
      * Ingest one or more documents
      */
     ingest(filePaths: string | string[], options?: IngestionOptions): Promise<IngestionResult>;
+    private loadDocumentsForPath;
+    private processAndStoreDocuments;
+    private extractErrorType;
+    private updateMemoryMetrics;
     /**
      * Process documents: chunk and embed
      */
     private processDocuments;
     /**
-     * Simple chunking (override with loader-specific chunking if available)
+     * Fixed-size chunking with overlap
      */
-    private chunkDocument;
+    private fixedChunkDocument;
+    /**
+     * Semantic chunking preserving heading-based sections where possible
+     */
+    private semanticChunkDocument;
+    /**
+     * Infer document type from file extension
+     */
+    private inferDocumentType;
 }
 //# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline/ingestion/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAGV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAE9B;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,KAAK,CAAc;gBAGzB,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,EAAE,WAAW;IAOpB;;OAEG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IA6D3B;;OAEG;YACW,gBAAgB;IAqB9B;;OAEG;IACH,OAAO,CAAC,aAAa;CAgCtB"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAGV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAG9B;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;gBAGlC,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,EAAE,WAAW;IAOpB;;OAEG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,EAC5B,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;YA+Fb,oBAAoB;YA0CpB,wBAAwB;IAsCtC,OAAO,CAAC,gBAAgB;IAQxB,OAAO,CAAC,mBAAmB;IAW3B;;OAEG;YACW,gBAAgB;IAmC9B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsC7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAS1B"}

package/dist/pipeline/ingestion/pipeline.js

@@ -2,6 +2,7 @@
  * Document ingestion pipeline
  * Orchestrates loading, embedding, and storage
  */
+import { convertToMarkdown } from '../../ingestion/index.js';
 /**
  * Pipeline for ingesting documents into vector store
  */
@@ -20,58 +21,150 @@ export class IngestionPipeline {
     async ingest(filePaths, options) {
         const paths = Array.isArray(filePaths) ? filePaths : [filePaths];
         const startTime = Date.now();
-        const { batchSize = 32, onProgress, loaderConfig, } = options ?? {};
+        const { batchSize = 32, onProgress, loaderConfig, enableDocumentConversion = false, conversionTimeoutMs = 30000, conversionMaxFileSize = 50 * 1024 * 1024, enableLLMDescriptions = false, chunkingStrategy = 'fixed', fixedChunkSize, fixedChunkOverlap, onConversionError, } = options ?? {};
         let totalDocuments = 0;
         let totalChunks = 0;
+        let successCount = 0;
+        let failureCount = 0;
         const errors = [];
+        const conversionDurationsMs = [];
+        let peakHeapUsedBytes = process.memoryUsage().heapUsed;
+        const warnings = [];
         for (let i = 0; i < paths.length; i++) {
             const path = paths[i];
             try {
-                // Load documents
-                const documents = await this.loader.load(path, loaderConfig);
+                const documents = await this.loadDocumentsForPath(path, {
+                    enableDocumentConversion,
+                    conversionTimeoutMs,
+                    conversionMaxFileSize,
+                    enableLLMDescriptions,
+                    loaderConfig,
+                    conversionDurationsMs,
+                });
                 totalDocuments += documents.length;
-                // Chunk and embed in batches
-                const allChunks = [];
-                for (let j = 0; j < documents.length; j += batchSize) {
-                    const batch = documents.slice(j, j + batchSize);
-                    const chunks = await this.processDocuments(batch);
-                    allChunks.push(...chunks);
-                    if (onProgress) {
-                        onProgress(i + 1, paths.length, {
-                            currentFile: path,
-                            documentsProcessed: Math.min(j + batchSize, documents.length),
-                            totalDocuments: documents.length,
-                            chunksGenerated: allChunks.length,
-                        });
-                    }
-                }
-                // Store chunks
-                await this.store.addDocuments(allChunks);
-                totalChunks += allChunks.length;
+                const chunkCount = await this.processAndStoreDocuments(path, documents, {
+                    batchSize,
+                    chunkingStrategy,
+                    fixedChunkSize,
+                    fixedChunkOverlap,
+                    onProgress,
+                    fileIndex: i,
+                    totalFiles: paths.length,
+                });
+                totalChunks += chunkCount;
+                successCount++;
             }
             catch (error) {
-                errors.push({
-                    file: path,
-                    error: error instanceof Error ? error.message : String(error),
-                });
+                failureCount++;
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                errors.push({ file: path, error: errorMessage });
+                if (enableDocumentConversion && onConversionError) {
+                    onConversionError({
+                        file: path,
+                        error: errorMessage,
+                        errorType: this.extractErrorType(error),
+                    });
+                }
             }
+            peakHeapUsedBytes = this.updateMemoryMetrics(path, peakHeapUsedBytes, warnings);
         }
         const endTime = Date.now();
+        const durationMs = endTime - startTime;
+        const averageConversionMs = conversionDurationsMs.length > 0
+            ? conversionDurationsMs.reduce((sum, ms) => sum + ms, 0) / conversionDurationsMs.length
+            : undefined;
+        const documentsPerSecond = durationMs > 0 ? (successCount / durationMs) * 1000 : 0;
         return {
             documentsProcessed: totalDocuments,
+            successCount,
+            failureCount,
             chunksGenerated: totalChunks,
             errors,
-            durationMs: endTime - startTime,
+            metrics: {
+                averageConversionMs,
+                documentsPerSecond,
+                peakHeapUsedMb: peakHeapUsedBytes / (1024 * 1024),
+                warnings,
+            },
+            durationMs,
         };
     }
+    async loadDocumentsForPath(path, config) {
+        if (!config.enableDocumentConversion) {
+            return this.loader.load(path, config.loaderConfig);
+        }
+        const conversionStart = Date.now();
+        const converted = await convertToMarkdown(path, {
+            timeout: config.conversionTimeoutMs,
+            maxFileSize: config.conversionMaxFileSize,
+            enableLLMDescriptions: config.enableLLMDescriptions,
+            preserveMetadata: true,
+        });
+        config.conversionDurationsMs.push(Date.now() - conversionStart);
+        return [
+            {
+                id: `${path}-${Date.now()}`,
+                content: converted.markdown,
+                metadata: {
+                    source: path,
+                    type: this.inferDocumentType(path),
+                    createdAt: new Date(),
+                    conversionTimestamp: converted.metadata.convertedAt,
+                    originalFileType: converted.metadata.format,
+                    pageCount: converted.metadata.pageCount,
+                    conversionDurationMs: converted.metadata.durationMs,
+                    usedLLMDescriptions: converted.metadata.usedLLMDescriptions,
+                },
+            },
+        ];
+    }
+    async processAndStoreDocuments(path, documents, config) {
+        const allChunks = [];
+        for (let j = 0; j < documents.length; j += config.batchSize) {
+            const batch = documents.slice(j, j + config.batchSize);
+            const chunks = await this.processDocuments(batch, {
+                chunkingStrategy: config.chunkingStrategy,
+                fixedChunkSize: config.fixedChunkSize,
+                fixedChunkOverlap: config.fixedChunkOverlap,
+            });
+            allChunks.push(...chunks);
+            if (config.onProgress) {
+                config.onProgress(config.fileIndex + 1, config.totalFiles, {
+                    currentFile: path,
+                    documentsProcessed: Math.min(j + config.batchSize, documents.length),
+                    totalDocuments: documents.length,
+                    chunksGenerated: allChunks.length,
+                });
+            }
+        }
+        await this.store.addDocuments(allChunks);
+        return allChunks.length;
+    }
+    extractErrorType(error) {
+        if (typeof error !== 'object' || error === null || !('type' in error)) {
+            return undefined;
+        }
+        const rawType = error.type;
+        return typeof rawType === 'string' ? rawType : undefined;
+    }
+    updateMemoryMetrics(path, currentPeak, warnings) {
+        const currentHeap = process.memoryUsage().heapUsed;
+        const nextPeak = Math.max(currentHeap, currentPeak);
+        if (currentHeap > 512 * 1024 * 1024) {
+            warnings.push(`High memory usage detected while processing ${path}: ${(currentHeap / (1024 * 1024)).toFixed(1)}MB`);
+        }
+        return nextPeak;
+    }
     /**
      * Process documents: chunk and embed
      */
-    async processDocuments(documents) {
+    async processDocuments(documents, options) {
         const chunks = [];
         for (const doc of documents) {
-            // Split into chunks (using simple text splitting)
-            const docChunks = this.chunkDocument(doc);
+            // Split into chunks
+            const docChunks = options?.chunkingStrategy === 'semantic'
+                ? this.semanticChunkDocument(doc)
+                : this.fixedChunkDocument(doc, options?.fixedChunkSize ?? 1000, options?.fixedChunkOverlap ?? 200);
             chunks.push(...docChunks);
         }
         // Generate embeddings
@@ -84,11 +177,9 @@ export class IngestionPipeline {
         return chunks;
     }
     /**
-     * Simple chunking (override with loader-specific chunking if available)
+     * Fixed-size chunking with overlap
      */
-    chunkDocument(doc) {
-        const chunkSize = 1000;
-        const overlap = 200;
+    fixedChunkDocument(doc, chunkSize, overlap) {
         const chunks = [];
         const content = doc.content;
         for (let i = 0; i < content.length; i += chunkSize - overlap) {
@@ -114,5 +205,57 @@ export class IngestionPipeline {
         }
         return chunks;
     }
+    /**
+     * Semantic chunking preserving heading-based sections where possible
+     */
+    semanticChunkDocument(doc) {
+        const chunks = [];
+        const sections = doc.content
+            .split(/\n(?=#{1,6}\s)/g)
+            .filter((section) => section.trim().length > 0);
+        if (sections.length <= 1) {
+            return this.fixedChunkDocument(doc, 1000, 200);
+        }
+        let cursor = 0;
+        for (let i = 0; i < sections.length; i++) {
+            const section = sections[i];
+            const startChar = doc.content.indexOf(section, cursor);
+            const endChar = startChar + section.length;
+            const chunkId = `${doc.id}-chunk-${i}`;
+            chunks.push({
+                id: chunkId,
+                documentId: doc.id,
+                content: section,
+                index: i,
+                metadata: {
+                    chunkIndex: i,
+                    chunkCount: sections.length,
+                    startChar,
+                    endChar,
+                    chunkingStrategy: 'semantic',
+                    ...doc.metadata,
+                },
+            });
+            cursor = endChar;
+        }
+        return chunks;
+    }
+    /**
+     * Infer document type from file extension
+     */
+    inferDocumentType(path) {
+        const lowerPath = path.toLowerCase();
+        if (lowerPath.endsWith('.pdf'))
+            return 'pdf';
+        if (lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown'))
+            return 'markdown';
+        if (lowerPath.endsWith('.html') || lowerPath.endsWith('.htm'))
+            return 'html';
+        if (lowerPath.endsWith('.json'))
+            return 'json';
+        if (lowerPath.endsWith('.txt'))
+            return 'text';
+        return 'other';
+    }
 }
 //# sourceMappingURL=pipeline.js.map

package/dist/pipeline/ingestion/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAYH;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAiB;IACvB,QAAQ,CAAqB;IAC7B,KAAK,CAAc;IAE3B,YACE,MAAsB,EACtB,QAA4B,EAC5B,KAAkB;QAElB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,SAA4B,EAC5B,OAA0B;QAE1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,EACJ,SAAS,GAAG,EAAE,EACd,UAAU,EACV,YAAY,GACb,GAAG,OAAO,IAAI,EAAE,CAAC;QAElB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,MAAM,GAA2C,EAAE,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,CAAC;gBACH,iBAAiB;gBACjB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBAC7D,cAAc,IAAI,SAAS,CAAC,MAAM,CAAC;gBAEnC,6BAA6B;gBAC7B,MAAM,SAAS,GAAoB,EAAE,CAAC;gBAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;oBACrD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;oBAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBAClD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAE1B,IAAI,UAAU,EAAE,CAAC;wBACf,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE;4BAC9B,WAAW,EAAE,IAAI;4BACjB,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC;4BAC7D,cAAc,EAAE,SAAS,CAAC,MAAM;4BAChC,eAAe,EAAE,SAAS,CAAC,MAAM;yBAClC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,eAAe;gBACf,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;gBACzC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,OAAO;YACL,kBAAkB,EAAE,cAAc;YAClC,eAAe,EAAE,WAAW;YAC5B,MAAM;YACN,UAAU,EAAE,OAAO,GAAG,SAAS;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,SAAqB;QAClD,MAAM,MAAM,GAAoB,EAAE,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,kDAAkD;YAClD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpD,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,GAAa;QACjC,MAAM,SAAS,GAAG,IAAI,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,CAAC;QACpB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,QAAQ,EAAE;oBACR,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,UAAU,EAAE,CAAC,EAAE,oBAAoB;oBACnC,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC;oBAChD,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/pipeline/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACX,MAAM,CAAiB;IACvB,QAAQ,CAAqB;IAC7B,KAAK,CAAc;IAEpC,YACE,MAAsB,EACtB,QAA4B,EAC5B,KAAkB;QAElB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,SAA4B,EAC5B,OAA0B;QAE1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,EACJ,SAAS,GAAG,EAAE,EACd,UAAU,EACV,YAAY,EACZ,wBAAwB,GAAG,KAAK,EAChC,mBAAmB,GAAG,KAAK,EAC3B,qBAAqB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EACxC,qBAAqB,GAAG,KAAK,EAC7B,gBAAgB,GAAG,OAAO,EAC1B,cAAc,EACd,iBAAiB,EACjB,iBAAiB,GAClB,GAAG,OAAO,IAAI,EAAE,CAAC;QAElB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,MAAM,GAA2C,EAAE,CAAC;QAC1D,MAAM,qBAAqB,GAAa,EAAE,CAAC;QAC3C,IAAI,iBAAiB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;QACvD,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE;oBACtD,wBAAwB;oBACxB,mBAAmB;oBACnB,qBAAqB;oBACrB,qBAAqB;oBACrB,YAAY;oBACZ,qBAAqB;iBACtB,CAAC,CAAC;gBAEH,cAAc,IAAI,SAAS,CAAC,MAAM,CAAC;gBAEnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,SAAS,EAAE;oBACtE,SAAS;oBACT,gBAAgB;oBAChB,cAAc;oBACd,iBAAiB;oBACjB,UAAU;oBACV,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,KAAK,CAAC,MAAM;iBACzB,CAAC,CAAC;gBAEH,WAAW,IAAI,UAAU,CAAC;gBAC1B,YAAY,EAAE,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;gBAEjD,IAAI,wBAAwB,IAAI,iBAAiB,EAAE,CAAC;oBAClD,iBAAiB,CAAC;wBAChB,IAAI,EAAE,IAAI;wBACV,KAAK,EAAE,YAAY;wBACnB,SAAS,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;qBACxC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,EAAE,QAAQ,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,OAAO,GAAG,SAAS,CAAC;QACvC,MAAM,mBAAmB,GACvB,qBAAqB,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,qBAAqB,CAAC,MAAM;YACvF,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,kBAAkB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnF,OAAO;YACL,kBAAkB,EAAE,cAAc;YAClC,YAAY;YACZ,YAAY;YACZ,eAAe,EAAE,WAAW;YAC5B,MAAM;YACN,OAAO,EAAE;gBACP,mBAAmB;gBACnB,kBAAkB;gBAClB,cAAc,EAAE,iBAAiB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;gBACjD,QAAQ;aACT;YACD,UAAU;SACX,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,IAAY,EACZ,MAOC;QAED,IAAI,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE;YAC9C,OAAO,EAAE,MAAM,CAAC,mBAAmB;YACnC,WAAW,EAAE,MAAM,CAAC,qBAAqB;YACzC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;YACnD,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;QACH,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;QAEhE,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC3B,OAAO,EAAE,SAAS,CAAC,QAAQ;gBAC3B,QAAQ,EAAE;oBACR,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAClC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACrB,mBAAmB,EAAE,SAAS,CAAC,QAAQ,CAAC,WAAW;oBACnD,gBAAgB,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM;oBAC3C,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,SAAS;oBACvC,oBAAoB,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU;oBACnD,mBAAmB,EAAE,SAAS,CAAC,QAAQ,CAAC,mBAAmB;iBAC5D;aACF;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,wBAAwB,CACpC,IAAY,EACZ,SAAqB,EACrB,MAQC;QAED,MAAM,SAAS,GAAoB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE;gBAChD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;aAC5C,CAAC,CAAC;YACH,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAE1B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE;oBACzD,WAAW,EAAE,IAAI;oBACjB,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC;oBACpE,cAAc,EAAE,SAAS,CAAC,MAAM;oBAChC,eAAe,EAAE,SAAS,CAAC,MAAM;iBAClC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,SAAS,CAAC,MAAM,CAAC;IAC1B,CAAC;IAEO,gBAAgB,CAAC,KAAc;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,OAAO,GAAI,KAA4B,CAAC,IAAI,CAAC;QACnD,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3D,CAAC;IAEO,mBAAmB,CAAC,IAAY,EAAE,WAAmB,EAAE,QAAkB;QAC/E,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACpD,IAAI,WAAW,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CACX,+CAA+C,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACrG,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAC5B,SAAqB,EACrB,OAIC;QAED,MAAM,MAAM,GAAoB,EAAE,CAAC;QAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,oBAAoB;YACpB,MAAM,SAAS,GACb,OAAO,EAAE,gBAAgB,KAAK,UAAU;gBACtC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC;gBACjC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CACrB,GAAG,EACH,OAAO,EAAE,cAAc,IAAI,IAAI,EAC/B,OAAO,EAAE,iBAAiB,IAAI,GAAG,CAClC,CAAC;YACR,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEpD,oBAAoB;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,GAAa,EAAE,SAAiB,EAAE,OAAe;QAC1E,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,QAAQ,EAAE;oBACR,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,UAAU,EAAE,CAAC,EAAE,oBAAoB;oBACnC,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC;oBAChD,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC,CAAC;QACL,CAAC;QAED,sBAAsB;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,GAAa;QACzC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO;aACzB,KAAK,CAAC,iBAAiB,CAAC;aACxB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAElD,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;YAC3C,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YAEvC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO;gBACX,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,OAAO;gBAChB,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE;oBACR,UAAU,EAAE,CAAC;oBACb,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,SAAS;oBACT,OAAO;oBACP,gBAAgB,EAAE,UAAU;oBAC5B,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC,CAAC;YAEH,MAAM,GAAG,OAAO,CAAC;QACnB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,IAAY;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,UAAU,CAAC;QACpF,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAC7E,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,OAAO,MAAM,CAAC;QAC/C,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;QAC9C,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/dist/types/index.d.ts

@@ -232,6 +232,26 @@ export interface IngestionOptions {
     batchSize?: number;
     /** Loader configuration */
     loaderConfig?: LoaderConfig;
+    /** Enable MarkItDown preprocessing for non-text documents */
+    enableDocumentConversion?: boolean;
+    /** Conversion timeout in milliseconds */
+    conversionTimeoutMs?: number;
+    /** Conversion file size limit in bytes */
+    conversionMaxFileSize?: number;
+    /** Enable LLM image descriptions during conversion */
+    enableLLMDescriptions?: boolean;
+    /** Chunking strategy */
+    chunkingStrategy?: 'semantic' | 'fixed';
+    /** Fixed chunk size (characters) */
+    fixedChunkSize?: number;
+    /** Fixed chunk overlap (characters) */
+    fixedChunkOverlap?: number;
+    /** Callback for conversion errors */
+    onConversionError?: (error: {
+        file: string;
+        error: string;
+        errorType?: string;
+    }) => void;
     /** Progress callback */
     onProgress?: (current: number, total: number, details?: {
         currentFile: string;
@@ -246,6 +266,10 @@ export interface IngestionOptions {
 export interface IngestionResult {
     /** Number of documents processed */
     documentsProcessed: number;
+    /** Number of input files successfully processed */
+    successCount: number;
+    /** Number of input files that failed processing */
+    failureCount: number;
     /** Number of chunks generated */
     chunksGenerated: number;
     /** Errors during ingestion */
@@ -253,6 +277,17 @@ export interface IngestionResult {
         file: string;
         error: string;
     }>;
+    /** Performance and resource metrics */
+    metrics?: {
+        /** Average conversion duration per file in milliseconds */
+        averageConversionMs?: number;
+        /** Documents converted per second */
+        documentsPerSecond?: number;
+        /** Peak heap usage in MB during ingestion */
+        peakHeapUsedMb?: number;
+        /** Warnings generated during ingestion */
+        warnings?: string[];
+    };
     /** Total duration in milliseconds */
     durationMs: number;
 }

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,IAAI,EAAE,KAAK,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9D,yBAAyB;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,yBAAyB;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,uBAAuB;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,gCAAgC;IAChC,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,mCAAmC;IACnC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,+BAA+B;IAC/B,aAAa,IAAI,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,cAAc,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,CAAC;IAChD,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,iCAAiC;IACjC,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,mCAAmC;IACnC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACnG,6BAA6B;IAC7B,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,sBAAsB;IACtB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,yBAAyB;IACzB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IACvD,0BAA0B;IAC1B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe;IACf,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;IACnE,uBAAuB;IACvB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,iCAAiC;IACjC,WAAW,EAAE,iBAAiB,CAAC;IAC/B,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2BAA2B;IAC3B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gCAAgC;IAChC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uBAAuB;IACvB,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,kCAAkC;IAClC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qBAAqB;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,EAAE;QACR,6BAA6B;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,UAAU,EAAE,MAAM,CAAC;QACnB,8BAA8B;QAC9B,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,wBAAwB;IACxB,UAAU,CAAC,EAAE,CACX,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB,KACE,IAAI,CAAC;CACX;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB;IACpB,IAAI,EAAE,KAAK,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9D,yBAAyB;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uBAAuB;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,yBAAyB;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,uBAAuB;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,gCAAgC;IAChC,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,mCAAmC;IACnC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,+BAA+B;IAC/B,aAAa,IAAI,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,cAAc,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,CAAC;IAChD,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,iCAAiC;IACjC,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,mCAAmC;IACnC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACnG,6BAA6B;IAC7B,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,sBAAsB;IACtB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,yBAAyB;IACzB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IACvD,0BAA0B;IAC1B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe;IACf,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;IACnE,uBAAuB;IACvB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,8BAA8B;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,iCAAiC;IACjC,WAAW,EAAE,iBAAiB,CAAC;IAC/B,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2BAA2B;IAC3B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gCAAgC;IAChC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uBAAuB;IACvB,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,kCAAkC;IAClC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qBAAqB;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,EAAE;QACR,6BAA6B;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,UAAU,EAAE,MAAM,CAAC;QACnB,8BAA8B;QAC9B,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,6DAA6D;IAC7D,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,yCAAyC;IACzC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,0CAA0C;IAC1C,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sDAAsD;IACtD,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,wBAAwB;IACxB,gBAAgB,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC;IACxC,oCAAoC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,uCAAuC;IACvC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qCAAqC;IACrC,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE;QAC1B,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,KAAK,IAAI,CAAC;IACX,wBAAwB;IACxB,UAAU,CAAC,EAAE,CACX,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB,KACE,IAAI,CAAC;CACX;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,uCAAuC;IACvC,OAAO,CAAC,EAAE;QACR,2DAA2D;QAC3D,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,qCAAqC;QACrC,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,6CAA6C;QAC7C,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,0CAA0C;QAC1C,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;CACpB"}