@dcyfr/ai-rag 0.1.0

package/dist/loaders/html/index.js

@@ -0,0 +1,106 @@
+/**
+ * HTML document loader
+ * Handles HTML files (.html, .htm)
+ */
+import { promises as fs } from 'node:fs';
+import { basename } from 'node:path';
+/**
+ * Load HTML documents
+ */
+export class HTMLLoader {
+    supportedExtensions = ['.html', '.htm'];
+    async load(source, config) {
+        try {
+            const content = await fs.readFile(source, 'utf-8');
+            const stats = await fs.stat(source);
+            // Extract title from <title> tag if present
+            const titleMatch = content.match(/<title>(.*?)<\/title>/i);
+            const title = titleMatch ? titleMatch[1] : basename(source);
+            // Extract text content
+            const textContent = config?.preserveFormatting
+                ? content
+                : this.extractText(content);
+            const document = {
+                id: this.generateId(source),
+                content: textContent,
+                metadata: {
+                    source,
+                    type: 'html',
+                    createdAt: stats.birthtime,
+                    updatedAt: stats.mtime,
+                    title,
+                    ...config?.metadata,
+                },
+            };
+            // Apply chunking if configured
+            if (config?.chunkSize) {
+                return this.chunkDocument(document, config);
+            }
+            return [document];
+        }
+        catch (error) {
+            throw new Error(`Failed to load HTML file ${source}: ${error}`);
+        }
+    }
+    /**
+     * Extract text content from HTML
+     * This is a simple implementation - for production use a proper HTML parser
+     */
+    extractText(html) {
+        return html
+            // Remove script and style tags
+            .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
+            .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
+            // Remove HTML comments
+            .replace(/<!--[\s\S]*?-->/g, '')
+            // Remove HTML tags
+            .replace(/<[^>]+>/g, ' ')
+            // Decode HTML entities (basic)
+            .replace(/&nbsp;/g, ' ')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&amp;/g, '&')
+            .replace(/&quot;/g, '"')
+            .replace(/&#39;/g, "'")
+            // Clean up whitespace
+            .replace(/\s+/g, ' ')
+            .replace(/\n{3,}/g, '\n\n')
+            .trim();
+    }
+    /**
+     * Split document into chunks
+     */
+    chunkDocument(document, config) {
+        const chunkSize = config.chunkSize ?? 1000;
+        const chunkOverlap = config.chunkOverlap ?? 200;
+        const content = document.content;
+        const chunks = [];
+        let start = 0;
+        let chunkIndex = 0;
+        while (start < content.length) {
+            const end = Math.min(start + chunkSize, content.length);
+            const chunkContent = content.slice(start, end);
+            chunks.push({
+                id: `${document.id}-chunk-${chunkIndex}`,
+                content: chunkContent,
+                metadata: {
+                    ...document.metadata,
+                    chunkIndex,
+                    startChar: start,
+                    endChar: end,
+                    parentDocumentId: document.id,
+                },
+            });
+            start += chunkSize - chunkOverlap;
+            chunkIndex++;
+        }
+        return chunks;
+    }
+    /**
+     * Generate document ID from source
+     */
+    generateId(source) {
+        return `html-${Buffer.from(source).toString('base64').slice(0, 16)}`;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/loaders/html/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/loaders/html/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC;;GAEG;AACH,MAAM,OAAO,UAAU;IACrB,mBAAmB,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAExC,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAqB;QAC9C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEpC,4CAA4C;YAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE5D,uBAAuB;YACvB,MAAM,WAAW,GAAG,MAAM,EAAE,kBAAkB;gBAC5C,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAa;gBACzB,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC3B,OAAO,EAAE,WAAW;gBACpB,QAAQ,EAAE;oBACR,MAAM;oBACN,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,SAAS,EAAE,KAAK,CAAC,KAAK;oBACtB,KAAK;oBACL,GAAG,MAAM,EAAE,QAAQ;iBACpB;aACF,CAAC;YAEF,+BAA+B;YAC/B,IAAI,MAAM,EAAE,SAAS,EAAE,CAAC;gBACtB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC9C,CAAC;YAED,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,IAAY;QAC9B,OAAO,IAAI;YACT,+BAA+B;aAC9B,OAAO,CAAC,qDAAqD,EAAE,EAAE,CAAC;aAClE,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC;YAChE,uBAAuB;aACtB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;YAChC,mBAAmB;aAClB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;YACzB,+BAA+B;aAC9B,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;YACvB,sBAAsB;aACrB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,IAAI,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAkB,EAAE,MAAoB;QAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;QACjC,MAAM,MAAM,GAAe,EAAE,CAAC;QAE9B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACxD,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAE/C,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,GAAG,QAAQ,CAAC,EAAE,UAAU,UAAU,EAAE;gBACxC,OAAO,EAAE,YAAY;gBACrB,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU;oBACV,SAAS,EAAE,KAAK;oBAChB,OAAO,EAAE,GAAG;oBACZ,gBAAgB,EAAE,QAAQ,CAAC,EAAE;iBAC9B;aACF,CAAC,CAAC;YAEH,KAAK,IAAI,SAAS,GAAG,YAAY,CAAC;YAClC,UAAU,EAAE,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,MAAc;QAC/B,OAAO,QAAQ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACvE,CAAC;CACF"}

package/dist/loaders/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Document loaders
+ */
+export { TextLoader } from './text/index.js';
+export { MarkdownLoader } from './markdown/index.js';
+export { HTMLLoader } from './html/index.js';
+export type { Document, DocumentLoader, LoaderConfig } from '../types/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/loaders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,YAAY,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/loaders/index.js

@@ -0,0 +1,7 @@
+/**
+ * Document loaders
+ */
+export { TextLoader } from './text/index.js';
+export { MarkdownLoader } from './markdown/index.js';
+export { HTMLLoader } from './html/index.js';
+//# sourceMappingURL=index.js.map

package/dist/loaders/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/markdown/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Markdown document loader
+ * Handles Markdown files (.md, .markdown)
+ */
+import type { Document, DocumentLoader, LoaderConfig } from '../../types/index.js';
+/**
+ * Load Markdown documents
+ */
+export declare class MarkdownLoader implements DocumentLoader {
+    supportedExtensions: string[];
+    load(source: string, config?: LoaderConfig): Promise<Document[]>;
+    /**
+     * Remove markdown formatting for pure text
+     */
+    private removeFormatting;
+    /**
+     * Chunk document by sections (headings)
+     */
+    private chunkBySection;
+    /**
+     * Split content by headings
+     */
+    private splitByHeadings;
+    /**
+     * Chunk text by size with overlap
+     */
+    private chunkText;
+    /**
+     * Generate document ID from source
+     */
+    private generateId;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/loaders/markdown/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/loaders/markdown/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAInF;;GAEG;AACH,qBAAa,cAAe,YAAW,cAAc;IACnD,mBAAmB,WAAwB;IAErC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAiCtE;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAsBxB;;OAEG;IACH,OAAO,CAAC,cAAc;IAsCtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAwBvB;;OAEG;IACH,OAAO,CAAC,SAAS;IAejB;;OAEG;IACH,OAAO,CAAC,UAAU;CAGnB"}

package/dist/loaders/markdown/index.js

@@ -0,0 +1,150 @@
+/**
+ * Markdown document loader
+ * Handles Markdown files (.md, .markdown)
+ */
+import { promises as fs } from 'node:fs';
+import { basename } from 'node:path';
+/**
+ * Load Markdown documents
+ */
+export class MarkdownLoader {
+    supportedExtensions = ['.md', '.markdown'];
+    async load(source, config) {
+        try {
+            const content = await fs.readFile(source, 'utf-8');
+            const stats = await fs.stat(source);
+            // Extract title from first heading if present
+            const titleMatch = content.match(/^#\s+(.+)$/m);
+            const title = titleMatch ? titleMatch[1] : basename(source);
+            const document = {
+                id: this.generateId(source),
+                content: config?.preserveFormatting ? content : this.removeFormatting(content),
+                metadata: {
+                    source,
+                    type: 'markdown',
+                    createdAt: stats.birthtime,
+                    updatedAt: stats.mtime,
+                    title,
+                    ...config?.metadata,
+                },
+            };
+            // Apply chunking if configured
+            if (config?.chunkSize) {
+                return this.chunkBySection(document, config);
+            }
+            return [document];
+        }
+        catch (error) {
+            throw new Error(`Failed to load markdown file ${source}: ${error}`);
+        }
+    }
+    /**
+     * Remove markdown formatting for pure text
+     */
+    removeFormatting(content) {
+        return content
+            // Remove code blocks
+            .replace(/```[\s\S]*?```/g, '')
+            // Remove inline code
+            .replace(/`([^`]+)`/g, '$1')
+            // Remove bold/italic
+            .replace(/(\*\*|__)(.*?)\1/g, '$2')
+            .replace(/(\*|_)(.*?)\1/g, '$2')
+            // Remove links but keep text
+            .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+            // Remove images
+            .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+            // Remove headings markers
+            .replace(/^#{1,6}\s+/gm, '')
+            // Remove horizontal rules
+            .replace(/^[-*_]{3,}$/gm, '')
+            // Clean up extra whitespace
+            .replace(/\n{3,}/g, '\n\n')
+            .trim();
+    }
+    /**
+     * Chunk document by sections (headings)
+     */
+    chunkBySection(document, config) {
+        const content = document.content;
+        const sections = this.splitByHeadings(content);
+        const chunks = [];
+        sections.forEach((section, index) => {
+            // Further chunk if section is too large
+            if (config.chunkSize && section.content.length > config.chunkSize) {
+                const subChunks = this.chunkText(section.content, config);
+                subChunks.forEach((chunk, subIndex) => {
+                    chunks.push({
+                        id: `${document.id}-${index}-${subIndex}`,
+                        content: chunk,
+                        metadata: {
+                            ...document.metadata,
+                            section: section.title,
+                            chunkIndex: chunks.length,
+                            parentDocumentId: document.id,
+                        },
+                    });
+                });
+            }
+            else {
+                chunks.push({
+                    id: `${document.id}-${index}`,
+                    content: section.content,
+                    metadata: {
+                        ...document.metadata,
+                        section: section.title,
+                        chunkIndex: index,
+                        parentDocumentId: document.id,
+                    },
+                });
+            }
+        });
+        return chunks;
+    }
+    /**
+     * Split content by headings
+     */
+    splitByHeadings(content) {
+        const sections = [];
+        const lines = content.split('\n');
+        let currentSection = { title: 'Introduction', content: '' };
+        for (const line of lines) {
+            const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+            if (headingMatch) {
+                if (currentSection.content.trim()) {
+                    sections.push(currentSection);
+                }
+                currentSection = { title: headingMatch[2], content: '' };
+            }
+            else {
+                currentSection.content += line + '\n';
+            }
+        }
+        if (currentSection.content.trim()) {
+            sections.push(currentSection);
+        }
+        return sections;
+    }
+    /**
+     * Chunk text by size with overlap
+     */
+    chunkText(text, config) {
+        const chunkSize = config.chunkSize ?? 1000;
+        const chunkOverlap = config.chunkOverlap ?? 200;
+        const chunks = [];
+        let start = 0;
+        while (start < text.length) {
+            const end = Math.min(start + chunkSize, text.length);
+            chunks.push(text.slice(start, end));
+            start += chunkSize - chunkOverlap;
+        }
+        return chunks;
+    }
+    /**
+     * Generate document ID from source
+     */
+    generateId(source) {
+        return `md-${Buffer.from(source).toString('base64').slice(0, 16)}`;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/loaders/markdown/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/loaders/markdown/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC;;GAEG;AACH,MAAM,OAAO,cAAc;IACzB,mBAAmB,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAqB;QAC9C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEpC,8CAA8C;YAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE5D,MAAM,QAAQ,GAAa;gBACzB,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC3B,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBAC9E,QAAQ,EAAE;oBACR,MAAM;oBACN,IAAI,EAAE,UAAU;oBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,SAAS,EAAE,KAAK,CAAC,KAAK;oBACtB,KAAK;oBACL,GAAG,MAAM,EAAE,QAAQ;iBACpB;aACF,CAAC;YAEF,+BAA+B;YAC/B,IAAI,MAAM,EAAE,SAAS,EAAE,CAAC;gBACtB,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC/C,CAAC;YAED,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,OAAO,OAAO;YACZ,qBAAqB;aACpB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;YAC/B,qBAAqB;aACpB,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;YAC5B,qBAAqB;aACpB,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;aAClC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;YAChC,6BAA6B;aAC5B,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC;YACxC,gBAAgB;aACf,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC;YACvC,0BAA0B;aACzB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;YAC5B,0BAA0B;aACzB,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;YAC7B,4BAA4B;aAC3B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,IAAI,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,QAAkB,EAAE,MAAoB;QAC7D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAe,EAAE,CAAC;QAE9B,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,wCAAwC;YACxC,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;gBAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC1D,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;oBACpC,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,GAAG,QAAQ,CAAC,EAAE,IAAI,KAAK,IAAI,QAAQ,EAAE;wBACzC,OAAO,EAAE,KAAK;wBACd,QAAQ,EAAE;4BACR,GAAG,QAAQ,CAAC,QAAQ;4BACpB,OAAO,EAAE,OAAO,CAAC,KAAK;4BACtB,UAAU,EAAE,MAAM,CAAC,MAAM;4BACzB,gBAAgB,EAAE,QAAQ,CAAC,EAAE;yBAC9B;qBACF,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,GAAG,QAAQ,CAAC,EAAE,IAAI,KAAK,EAAE;oBAC7B,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,QAAQ,EAAE;wBACR,GAAG,QAAQ,CAAC,QAAQ;wBACpB,OAAO,EAAE,OAAO,CAAC,KAAK;wBACtB,UAAU,EAAE,KAAK;wBACjB,gBAAgB,EAAE,QAAQ,CAAC,EAAE;qBAC9B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,OAAe;QACrC,MAAM,QAAQ,GAA8C,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,cAAc,GAAG,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YACrD,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;oBAClC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAChC,CAAC;gBACD,cAAc,GAAG,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;YACxC,CAAC;QACH,CAAC;QAED,IAAI,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,IAAY,EAAE,MAAoB;QAClD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACpC,KAAK,IAAI,SAAS,GAAG,YAAY,CAAC;QACpC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,MAAc;QAC/B,OAAO,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACrE,CAAC;CACF"}

package/dist/loaders/text/index.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Text document loader
+ * Handles plain text files (.txt)
+ */
+import type { Document, DocumentLoader, LoaderConfig } from '../../types/index.js';
+/**
+ * Load plain text documents
+ */
+export declare class TextLoader implements DocumentLoader {
+    supportedExtensions: string[];
+    load(source: string, config?: LoaderConfig): Promise<Document[]>;
+    /**
+     * Split document into chunks
+     */
+    private chunkDocument;
+    /**
+     * Generate document ID from source
+     */
+    private generateId;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/loaders/text/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/loaders/text/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAInF;;GAEG;AACH,qBAAa,UAAW,YAAW,cAAc;IAC/C,mBAAmB,WAAqB;IAElC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAkCtE;;OAEG;IACH,OAAO,CAAC,aAAa;IAgCrB;;OAEG;IACH,OAAO,CAAC,UAAU;CAGnB"}

package/dist/loaders/text/index.js

@@ -0,0 +1,78 @@
+/**
+ * Text document loader
+ * Handles plain text files (.txt)
+ */
+import { promises as fs } from 'node:fs';
+import { basename } from 'node:path';
+/**
+ * Load plain text documents
+ */
+export class TextLoader {
+    supportedExtensions = ['.txt', '.text'];
+    async load(source, config) {
+        try {
+            const content = await fs.readFile(source, 'utf-8');
+            const stats = await fs.stat(source);
+            const document = {
+                id: this.generateId(source),
+                content,
+                metadata: {
+                    source,
+                    type: 'text',
+                    createdAt: stats.birthtime,
+                    updatedAt: stats.mtime,
+                    title: basename(source),
+                    ...config?.metadata,
+                },
+            };
+            // Skip empty documents
+            if (!content.trim()) {
+                return [];
+            }
+            // Apply chunking if configured
+            if (config?.chunkSize) {
+                return this.chunkDocument(document, config);
+            }
+            return [document];
+        }
+        catch (error) {
+            throw new Error(`Failed to load text file ${source}: ${error}`);
+        }
+    }
+    /**
+     * Split document into chunks
+     */
+    chunkDocument(document, config) {
+        const chunkSize = config.chunkSize ?? 1000;
+        const chunkOverlap = Math.min(config.chunkOverlap ?? 200, chunkSize - 1);
+        const content = document.content;
+        const chunks = [];
+        let start = 0;
+        let chunkIndex = 0;
+        while (start < content.length) {
+            const end = Math.min(start + chunkSize, content.length);
+            const chunkContent = content.slice(start, end);
+            chunks.push({
+                id: `${document.id}-chunk-${chunkIndex}`,
+                content: chunkContent,
+                metadata: {
+                    ...document.metadata,
+                    chunkIndex,
+                    startChar: start,
+                    endChar: end,
+                    parentDocumentId: document.id,
+                },
+            });
+            start += chunkSize - chunkOverlap;
+            chunkIndex++;
+        }
+        return chunks;
+    }
+    /**
+     * Generate document ID from source
+     */
+    generateId(source) {
+        return `text-${Buffer.from(source).toString('base64').slice(0, 16)}`;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/loaders/text/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/loaders/text/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC;;GAEG;AACH,MAAM,OAAO,UAAU;IACrB,mBAAmB,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAExC,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,MAAqB;QAC9C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEpC,MAAM,QAAQ,GAAa;gBACzB,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC3B,OAAO;gBACP,QAAQ,EAAE;oBACR,MAAM;oBACN,IAAI,EAAE,MAAM;oBACZ,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,SAAS,EAAE,KAAK,CAAC,KAAK;oBACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC;oBACvB,GAAG,MAAM,EAAE,QAAQ;iBACpB;aACF,CAAC;YAEF,uBAAuB;YACvB,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,+BAA+B;YAC/B,IAAI,MAAM,EAAE,SAAS,EAAE,CAAC;gBACtB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC9C,CAAC;YAED,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,QAAkB,EAAE,MAAoB;QAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;QACjC,MAAM,MAAM,GAAe,EAAE,CAAC;QAE9B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACxD,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAE/C,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,GAAG,QAAQ,CAAC,EAAE,UAAU,UAAU,EAAE;gBACxC,OAAO,EAAE,YAAY;gBACrB,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU;oBACV,SAAS,EAAE,KAAK;oBAChB,OAAO,EAAE,GAAG;oBACZ,gBAAgB,EAAE,QAAQ,CAAC,EAAE;iBAC9B;aACF,CAAC,CAAC;YAEH,KAAK,IAAI,SAAS,GAAG,YAAY,CAAC;YAClC,UAAU,EAAE,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,MAAc;QAC/B,OAAO,QAAQ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACvE,CAAC;CACF"}

package/dist/pipeline/embedding/generator.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Simple embedding generator (placeholder implementation)
+ * In production, integrate with OpenAI, Anthropic, or local models
+ */
+import type { EmbeddingGenerator, EmbeddingConfig } from '../../types/index.js';
+export interface SimpleEmbeddingOptions {
+    dimensions?: number;
+}
+/**
+ * Simple embedding generator using text hashing
+ * For demonstration purposes - use real embeddings in production
+ */
+export declare class SimpleEmbeddingGenerator implements EmbeddingGenerator {
+    private dimensions;
+    constructor(options?: SimpleEmbeddingOptions);
+    embed(texts: string[], _config?: EmbeddingConfig): Promise<number[][]>;
+    getDimensions(): number;
+    /**
+     * Generate simple embedding using character-based hashing
+     * This is NOT a real embedding - use OpenAI/Anthropic/local models in production
+     */
+    private generateEmbedding;
+}
+//# sourceMappingURL=generator.d.ts.map

package/dist/pipeline/embedding/generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../../src/pipeline/embedding/generator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEhF,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,qBAAa,wBAAyB,YAAW,kBAAkB;IACjE,OAAO,CAAC,UAAU,CAAS;gBAEf,OAAO,GAAE,sBAA2B;IAI1C,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAI5E,aAAa,IAAI,MAAM;IAIvB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;CAoB1B"}

package/dist/pipeline/embedding/generator.js

@@ -0,0 +1,42 @@
+/**
+ * Simple embedding generator (placeholder implementation)
+ * In production, integrate with OpenAI, Anthropic, or local models
+ */
+/**
+ * Simple embedding generator using text hashing
+ * For demonstration purposes - use real embeddings in production
+ */
+export class SimpleEmbeddingGenerator {
+    dimensions;
+    constructor(options = {}) {
+        this.dimensions = options.dimensions ?? 384; // Common embedding size
+    }
+    async embed(texts, _config) {
+        return texts.map((text) => this.generateEmbedding(text));
+    }
+    getDimensions() {
+        return this.dimensions;
+    }
+    /**
+     * Generate simple embedding using character-based hashing
+     * This is NOT a real embedding - use OpenAI/Anthropic/local models in production
+     */
+    generateEmbedding(text) {
+        const embedding = new Array(this.dimensions).fill(0);
+        // Simple character-based features
+        for (let i = 0; i < text.length; i++) {
+            const charCode = text.charCodeAt(i);
+            const index = charCode % this.dimensions;
+            embedding[index] += 1;
+        }
+        // Normalize vector
+        const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
+        if (magnitude > 0) {
+            for (let i = 0; i < embedding.length; i++) {
+                embedding[i] /= magnitude;
+            }
+        }
+        return embedding;
+    }
+}
+//# sourceMappingURL=generator.js.map

package/dist/pipeline/embedding/generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generator.js","sourceRoot":"","sources":["../../../src/pipeline/embedding/generator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH;;;GAGG;AACH,MAAM,OAAO,wBAAwB;IAC3B,UAAU,CAAS;IAE3B,YAAY,UAAkC,EAAE;QAC9C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,wBAAwB;IACvE,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe,EAAE,OAAyB;QACpD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,iBAAiB,CAAC,IAAY;QACpC,MAAM,SAAS,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAErD,kCAAkC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,KAAK,GAAG,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;YACzC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAED,mBAAmB;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAI,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACjF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}

package/dist/pipeline/embedding/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Embedding module exports
+ */
+export { SimpleEmbeddingGenerator } from './generator.js';
+export { EmbeddingPipeline } from './pipeline.js';
+export type { SimpleEmbeddingOptions } from './generator.js';
+export type { EmbeddingPipelineOptions } from './pipeline.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/pipeline/embedding/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/pipeline/embedding/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,YAAY,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC"}

package/dist/pipeline/embedding/index.js

@@ -0,0 +1,6 @@
+/**
+ * Embedding module exports
+ */
+export { SimpleEmbeddingGenerator } from './generator.js';
+export { EmbeddingPipeline } from './pipeline.js';
+//# sourceMappingURL=index.js.map

package/dist/pipeline/embedding/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/pipeline/embedding/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC"}

package/dist/pipeline/embedding/pipeline.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Embedding pipeline for processing documents
+ */
+import type { Document, DocumentChunk, EmbeddingGenerator } from '../../types/index.js';
+export interface EmbeddingPipelineOptions {
+    /** Batch size for embedding generation */
+    batchSize?: number;
+    /** Progress callback */
+    onProgress?: (processed: number, total: number) => void;
+}
+/**
+ * Pipeline for generating embeddings from documents
+ */
+export declare class EmbeddingPipeline {
+    private embedder;
+    constructor(embedder: EmbeddingGenerator);
+    /**
+     * Process documents and generate embeddings
+     */
+    process(documents: Document[], options?: EmbeddingPipelineOptions): Promise<DocumentChunk[]>;
+    /**
+     * Generate embedding for a single query
+     */
+    embedQuery(query: string): Promise<number[]>;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/pipeline/embedding/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../src/pipeline/embedding/pipeline.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAExF,MAAM,WAAW,wBAAwB;IACvC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACzD;AAED;;GAEG;AACH,qBAAa,iBAAiB;IAChB,OAAO,CAAC,QAAQ;gBAAR,QAAQ,EAAE,kBAAkB;IAEhD;;OAEG;IACG,OAAO,CACX,SAAS,EAAE,QAAQ,EAAE,EACrB,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,aAAa,EAAE,CAAC;IA0C3B;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAInD"}

package/dist/pipeline/embedding/pipeline.js

@@ -0,0 +1,59 @@
+/**
+ * Embedding pipeline for processing documents
+ */
+/**
+ * Pipeline for generating embeddings from documents
+ */
+export class EmbeddingPipeline {
+    embedder;
+    constructor(embedder) {
+        this.embedder = embedder;
+    }
+    /**
+     * Process documents and generate embeddings
+     */
+    async process(documents, options = {}) {
+        const batchSize = options.batchSize ?? 32;
+        const chunks = [];
+        // Convert documents to chunks
+        for (let i = 0; i < documents.length; i++) {
+            const doc = documents[i];
+            const chunk = {
+                id: doc.id,
+                documentId: doc.id,
+                content: doc.content,
+                index: i,
+                metadata: {
+                    chunkIndex: i,
+                    chunkCount: documents.length,
+                    startChar: 0,
+                    endChar: doc.content.length,
+                    ...doc.metadata,
+                },
+            };
+            chunks.push(chunk);
+        }
+        // Generate embeddings in batches
+        for (let i = 0; i < chunks.length; i += batchSize) {
+            const batch = chunks.slice(i, i + batchSize);
+            const texts = batch.map((chunk) => chunk.content);
+            const embeddings = await this.embedder.embed(texts);
+            // Assign embeddings to chunks
+            for (let j = 0; j < batch.length; j++) {
+                batch[j].embedding = embeddings[j];
+            }
+            if (options.onProgress) {
+                options.onProgress(Math.min(i + batchSize, chunks.length), chunks.length);
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Generate embedding for a single query
+     */
+    async embedQuery(query) {
+        const embeddings = await this.embedder.embed([query]);
+        return embeddings[0];
+    }
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/pipeline/embedding/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../../src/pipeline/embedding/pipeline.ts"],"names":[],"mappings":"AAAA;;GAEG;AAWH;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACR;IAApB,YAAoB,QAA4B;QAA5B,aAAQ,GAAR,QAAQ,CAAoB;IAAG,CAAC;IAEpD;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,SAAqB,EACrB,UAAoC,EAAE;QAEtC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAoB,EAAE,CAAC;QAEnC,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,KAAK,GAAkB;gBAC3B,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE;oBACR,UAAU,EAAE,CAAC;oBACb,UAAU,EAAE,SAAS,CAAC,MAAM;oBAC5B,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM;oBAC3B,GAAG,GAAG,CAAC,QAAQ;iBAChB;aACF,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,iCAAiC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAEpD,8BAA8B;YAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAa;QAC5B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;CACF"}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Pipeline exports
+ */
+export * from './embedding/index.js';
+export * from './ingestion/index.js';
+export * from './retrieval/index.js';
+//# sourceMappingURL=index.d.ts.map