@cogitator-ai/rag 0.1.0 → 0.1.4

package/README.md

@@ -64,7 +64,7 @@ for (const r of results) {
 | ---------------- | --------------------- | ------------ | ----------------------------- |
 | `TextLoader`     | `.txt`                | —            | Files and directories         |
 | `MarkdownLoader` | `.md`                 | —            | Strips frontmatter by default |
-| `JSONLoader`     | `.json`               | —            | Configurable JSON path        |
+| `JSONLoader`     | `.json`               | —            | Configurable content field    |
 | `CSVLoader`      | `.csv`                | `papaparse`  | Column selection, row mapping |
 | `HTMLLoader`     | `.html`, `.htm`       | `cheerio`    | CSS selector support          |
 | `PDFLoader`      | `.pdf`                | `pdf-parse`  | Text extraction from PDFs     |
@@ -75,7 +75,7 @@ import { MarkdownLoader, WebLoader, CSVLoader } from '@cogitator-ai/rag';
 
 const md = new MarkdownLoader({ stripFrontmatter: true });
 const web = new WebLoader({ selector: 'article' });
-const csv = new CSVLoader({ columns: ['title', 'body'] });
+const csv = new CSVLoader({ contentColumn: 'body', metadataColumns: ['title'] });
 ```
 
 ---

package/dist/chunkers/create-chunker.d.ts

@@ -0,0 +1,3 @@
+import type { ChunkingConfig, Chunker, AsyncChunker, EmbeddingService } from '@cogitator-ai/types';
+export declare function createChunker(config: ChunkingConfig, embeddingService?: EmbeddingService): Chunker | AsyncChunker;
+//# sourceMappingURL=create-chunker.d.ts.map

package/dist/chunkers/create-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAKnG,wBAAgB,aAAa,CAC3B,MAAM,EAAE,cAAc,EACtB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,GAAG,YAAY,CA8BxB"}

package/dist/chunkers/create-chunker.js

@@ -0,0 +1,32 @@
+import { FixedSizeChunker } from './fixed-chunker';
+import { RecursiveChunker } from './recursive-chunker';
+import { SemanticChunker } from './semantic-chunker';
+export function createChunker(config, embeddingService) {
+    switch (config.strategy) {
+        case 'fixed':
+            return new FixedSizeChunker({
+                chunkSize: config.chunkSize,
+                chunkOverlap: config.chunkOverlap,
+            });
+        case 'recursive':
+            return new RecursiveChunker({
+                chunkSize: config.chunkSize,
+                chunkOverlap: config.chunkOverlap,
+                separators: config.separators,
+            });
+        case 'semantic': {
+            if (!embeddingService) {
+                throw new Error('embeddingService is required for semantic chunking strategy');
+            }
+            return new SemanticChunker({
+                embeddingService,
+                maxChunkSize: config.chunkSize,
+            });
+        }
+        default: {
+            const _exhaustive = config.strategy;
+            throw new Error(`Unknown chunking strategy: ${_exhaustive}`);
+        }
+    }
+}
+//# sourceMappingURL=create-chunker.js.map

package/dist/chunkers/create-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC,QAAQ,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/chunkers/fixed-chunker.d.ts

@@ -0,0 +1,12 @@
+import type { Chunker, DocumentChunk } from '@cogitator-ai/types';
+export interface FixedSizeChunkerOptions {
+    chunkSize: number;
+    chunkOverlap: number;
+}
+export declare class FixedSizeChunker implements Chunker {
+    private readonly chunkSize;
+    private readonly chunkOverlap;
+    constructor(options: FixedSizeChunkerOptions);
+    chunk(text: string, documentId: string): DocumentChunk[];
+}
+//# sourceMappingURL=fixed-chunker.d.ts.map

package/dist/chunkers/fixed-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixed-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,uBAAuB;IAc5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;CAuBzD"}

package/dist/chunkers/fixed-chunker.js

@@ -0,0 +1,40 @@
+import { nanoid } from 'nanoid';
+export class FixedSizeChunker {
+    chunkSize;
+    chunkOverlap;
+    constructor(options) {
+        if (options.chunkSize <= 0) {
+            throw new Error('chunkSize must be a positive number');
+        }
+        if (options.chunkOverlap < 0) {
+            throw new Error('chunkOverlap must be non-negative');
+        }
+        if (options.chunkOverlap >= options.chunkSize) {
+            throw new Error('chunkOverlap must be less than chunkSize');
+        }
+        this.chunkSize = options.chunkSize;
+        this.chunkOverlap = options.chunkOverlap;
+    }
+    chunk(text, documentId) {
+        if (text.length === 0)
+            return [];
+        const step = this.chunkSize - this.chunkOverlap;
+        const chunks = [];
+        let order = 0;
+        for (let start = 0; start < text.length; start += step) {
+            const end = Math.min(start + this.chunkSize, text.length);
+            chunks.push({
+                id: nanoid(),
+                documentId,
+                content: text.slice(start, end),
+                startOffset: start,
+                endOffset: end,
+                order: order++,
+            });
+            if (end === text.length)
+                break;
+        }
+        return chunks;
+    }
+}
+//# sourceMappingURL=fixed-chunker.js.map

package/dist/chunkers/fixed-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAQhC,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC/B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,GAAG;gBACd,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;gBAAE,MAAM;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/chunkers/index.d.ts

@@ -0,0 +1,5 @@
+export { FixedSizeChunker, type FixedSizeChunkerOptions } from './fixed-chunker';
+export { RecursiveChunker, type RecursiveChunkerOptions } from './recursive-chunker';
+export { SemanticChunker, type SemanticChunkerOptions } from './semantic-chunker';
+export { createChunker } from './create-chunker';
+//# sourceMappingURL=index.d.ts.map

package/dist/chunkers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/chunkers/index.js

@@ -0,0 +1,5 @@
+export { FixedSizeChunker } from './fixed-chunker';
+export { RecursiveChunker } from './recursive-chunker';
+export { SemanticChunker } from './semantic-chunker';
+export { createChunker } from './create-chunker';
+//# sourceMappingURL=index.js.map

package/dist/chunkers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAgC,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAgC,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,eAAe,EAA+B,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/chunkers/recursive-chunker.d.ts

@@ -0,0 +1,18 @@
+import type { Chunker, DocumentChunk } from '@cogitator-ai/types';
+export interface RecursiveChunkerOptions {
+    chunkSize: number;
+    chunkOverlap: number;
+    separators?: string[];
+}
+export declare class RecursiveChunker implements Chunker {
+    private readonly chunkSize;
+    private readonly chunkOverlap;
+    private readonly separators;
+    constructor(options: RecursiveChunkerOptions);
+    chunk(text: string, documentId: string): DocumentChunk[];
+    private splitText;
+    private charSplit;
+    private mergeWithOverlap;
+    private mergeWithOverlapSimple;
+}
+//# sourceMappingURL=recursive-chunker.d.ts.map

package/dist/chunkers/recursive-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,SAAS;IA+CjB,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,gBAAgB;IAwFxB,OAAO,CAAC,sBAAsB;CAwB/B"}

package/dist/chunkers/recursive-chunker.js

@@ -0,0 +1,161 @@
+import { nanoid } from 'nanoid';
+const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' ', ''];
+export class RecursiveChunker {
+    chunkSize;
+    chunkOverlap;
+    separators;
+    constructor(options) {
+        if (options.chunkSize <= 0) {
+            throw new Error('chunkSize must be a positive number');
+        }
+        if (options.chunkOverlap < 0) {
+            throw new Error('chunkOverlap must be non-negative');
+        }
+        if (options.chunkOverlap >= options.chunkSize) {
+            throw new Error('chunkOverlap must be less than chunkSize');
+        }
+        this.chunkSize = options.chunkSize;
+        this.chunkOverlap = options.chunkOverlap;
+        this.separators = options.separators ?? DEFAULT_SEPARATORS;
+    }
+    chunk(text, documentId) {
+        if (text.length === 0)
+            return [];
+        const texts = this.splitText(text, 0);
+        return this.mergeWithOverlap(texts, documentId);
+    }
+    splitText(text, separatorIndex) {
+        if (text.length <= this.chunkSize)
+            return [text];
+        if (separatorIndex >= this.separators.length) {
+            return this.charSplit(text);
+        }
+        const separator = this.separators[separatorIndex];
+        if (separator === '') {
+            return this.charSplit(text);
+        }
+        const parts = text.split(separator);
+        if (parts.length === 1) {
+            return this.splitText(text, separatorIndex + 1);
+        }
+        const merged = [];
+        let current = '';
+        for (const part of parts) {
+            if (part.length === 0)
+                continue;
+            const candidate = current ? current + separator + part : part;
+            if (candidate.length <= this.chunkSize) {
+                current = candidate;
+            }
+            else {
+                if (current)
+                    merged.push(current);
+                if (part.length > this.chunkSize) {
+                    const subParts = this.splitText(part, separatorIndex + 1);
+                    merged.push(...subParts);
+                    current = '';
+                }
+                else {
+                    current = part;
+                }
+            }
+        }
+        if (current)
+            merged.push(current);
+        return merged;
+    }
+    charSplit(text) {
+        const result = [];
+        for (let i = 0; i < text.length; i += this.chunkSize) {
+            result.push(text.slice(i, i + this.chunkSize));
+        }
+        return result;
+    }
+    mergeWithOverlap(texts, documentId) {
+        if (this.chunkOverlap === 0) {
+            let offset = 0;
+            return texts.map((content, i) => {
+                const chunk = {
+                    id: nanoid(),
+                    documentId,
+                    content,
+                    startOffset: offset,
+                    endOffset: offset + content.length,
+                    order: i,
+                };
+                offset += content.length;
+                return chunk;
+            });
+        }
+        const joined = texts.join('');
+        const chunks = [];
+        let order = 0;
+        let textIndex = 0;
+        let i = 0;
+        while (i < texts.length) {
+            let content = texts[i];
+            let consumed = 1;
+            while (i + consumed < texts.length) {
+                const next = content + texts[i + consumed];
+                if (next.length <= this.chunkSize) {
+                    content = next;
+                    consumed++;
+                }
+                else {
+                    break;
+                }
+            }
+            const startOffset = joined.indexOf(content, textIndex > 0 ? Math.max(0, textIndex - this.chunkOverlap) : 0);
+            chunks.push({
+                id: nanoid(),
+                documentId,
+                content,
+                startOffset: startOffset >= 0 ? startOffset : textIndex,
+                endOffset: (startOffset >= 0 ? startOffset : textIndex) + content.length,
+                order: order++,
+            });
+            textIndex += content.length;
+            const overlapChars = Math.min(this.chunkOverlap, content.length);
+            const overlapText = content.slice(content.length - overlapChars);
+            i += consumed;
+            if (i < texts.length) {
+                const remaining = texts.slice(i);
+                const withOverlap = overlapText + remaining.join('');
+                if (withOverlap.length <= this.chunkSize) {
+                    chunks.push({
+                        id: nanoid(),
+                        documentId,
+                        content: withOverlap,
+                        startOffset: textIndex - overlapChars,
+                        endOffset: textIndex - overlapChars + withOverlap.length,
+                        order: order++,
+                    });
+                    break;
+                }
+                const reSplit = this.splitText(withOverlap, 0);
+                const subChunks = this.mergeWithOverlapSimple(reSplit, documentId, textIndex - overlapChars, order);
+                chunks.push(...subChunks);
+                break;
+            }
+        }
+        return chunks;
+    }
+    mergeWithOverlapSimple(texts, documentId, baseOffset, startOrder) {
+        const chunks = [];
+        let offset = baseOffset;
+        let order = startOrder;
+        for (const content of texts) {
+            chunks.push({
+                id: nanoid(),
+                documentId,
+                content,
+                startOffset: offset,
+                endOffset: offset + content.length,
+                order: order++,
+            });
+            offset += content.length;
+        }
+        return chunks;
+    }
+}
+//# sourceMappingURL=recursive-chunker.js.map

package/dist/chunkers/recursive-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,cAAsB;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAElD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9D,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACzB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,KAAe,EAAE,UAAkB;QAC1D,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;gBAC9B,MAAM,KAAK,GAAkB;oBAC3B,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,MAAM;oBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;oBAClC,KAAK,EAAE,CAAC;iBACT,CAAC;gBACF,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,OAAO,CAAC,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC3C,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClC,OAAO,GAAG,IAAI,CAAC;oBACf,QAAQ,EAAE,CAAC;gBACb,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAChC,OAAO,EACP,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;YACF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;gBACvD,SAAS,EAAE,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM;gBACxE,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;YAE5B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;YAEjE,CAAC,IAAI,QAAQ,CAAC;YAEd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAErD,IAAI,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,WAAW;wBACpB,WAAW,EAAE,SAAS,GAAG,YAAY;wBACrC,SAAS,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,CAAC,MAAM;wBACxD,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;oBACH,MAAM;gBACR,CAAC;gBAED,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAC3C,OAAO,EACP,UAAU,EACV,SAAS,GAAG,YAAY,EACxB,KAAK,CACN,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,sBAAsB,CAC5B,KAAe,EACf,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,UAAU,CAAC;QACxB,IAAI,KAAK,GAAG,UAAU,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,MAAM;gBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;gBAClC,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;QAC3B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/chunkers/semantic-chunker.d.ts

@@ -0,0 +1,19 @@
+import type { AsyncChunker, DocumentChunk, EmbeddingService } from '@cogitator-ai/types';
+export interface SemanticChunkerOptions {
+    embeddingService: EmbeddingService;
+    breakpointThreshold?: number;
+    minChunkSize?: number;
+    maxChunkSize?: number;
+}
+export declare class SemanticChunker implements AsyncChunker {
+    private readonly embeddingService;
+    private readonly breakpointThreshold;
+    private readonly minChunkSize;
+    private readonly maxChunkSize;
+    constructor(options: SemanticChunkerOptions);
+    chunk(text: string, documentId: string): Promise<DocumentChunk[]>;
+    private mergeSmallGroups;
+    private buildChunks;
+    private splitLargeGroup;
+}
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunkers/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAyBD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAOrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,eAAe;CA8DxB"}

package/dist/chunkers/semantic-chunker.js

@@ -0,0 +1,176 @@
+import { nanoid } from 'nanoid';
+function cosineSimilarity(a, b) {
+    const len = Math.min(a.length, b.length);
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < len; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dot / denom;
+}
+const SENTENCE_SPLIT_RE = /(?<=[.?!])\s+/;
+function splitSentences(text) {
+    return text
+        .split(SENTENCE_SPLIT_RE)
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+export class SemanticChunker {
+    embeddingService;
+    breakpointThreshold;
+    minChunkSize;
+    maxChunkSize;
+    constructor(options) {
+        this.embeddingService = options.embeddingService;
+        this.breakpointThreshold = options.breakpointThreshold ?? 0.5;
+        this.minChunkSize = options.minChunkSize ?? 100;
+        this.maxChunkSize = options.maxChunkSize ?? 2000;
+    }
+    async chunk(text, documentId) {
+        if (!text || text.trim().length === 0)
+            return [];
+        const sentences = splitSentences(text);
+        if (sentences.length === 0)
+            return [];
+        if (sentences.length === 1) {
+            return this.buildChunks([sentences], text, documentId);
+        }
+        const embeddings = await this.embeddingService.embedBatch(sentences);
+        const similarities = [];
+        for (let i = 0; i < embeddings.length - 1; i++) {
+            similarities.push(cosineSimilarity(embeddings[i], embeddings[i + 1]));
+        }
+        const groups = [];
+        let currentGroup = [sentences[0]];
+        for (let i = 0; i < similarities.length; i++) {
+            if (similarities[i] < this.breakpointThreshold) {
+                groups.push(currentGroup);
+                currentGroup = [sentences[i + 1]];
+            }
+            else {
+                currentGroup.push(sentences[i + 1]);
+            }
+        }
+        groups.push(currentGroup);
+        const merged = this.mergeSmallGroups(groups);
+        return this.buildChunks(merged, text, documentId);
+    }
+    mergeSmallGroups(groups) {
+        if (groups.length <= 1)
+            return groups;
+        const result = [];
+        let current = groups[0];
+        for (let i = 1; i < groups.length; i++) {
+            const currentText = current.join(' ');
+            const nextText = groups[i].join(' ');
+            const combinedLength = currentText.length + 1 + nextText.length;
+            if (currentText.length < this.minChunkSize && combinedLength <= this.maxChunkSize) {
+                current = [...current, ...groups[i]];
+            }
+            else {
+                result.push(current);
+                current = groups[i];
+            }
+        }
+        if (current.length > 0) {
+            const currentText = current.join(' ');
+            if (currentText.length < this.minChunkSize && result.length > 0) {
+                const last = result[result.length - 1];
+                const lastText = last.join(' ');
+                if (lastText.length + 1 + currentText.length <= this.maxChunkSize) {
+                    result[result.length - 1] = [...last, ...current];
+                }
+                else {
+                    result.push(current);
+                }
+            }
+            else {
+                result.push(current);
+            }
+        }
+        return result;
+    }
+    buildChunks(groups, originalText, documentId) {
+        const chunks = [];
+        let searchFrom = 0;
+        for (const group of groups) {
+            const content = group.join(' ');
+            if (content.length <= this.maxChunkSize) {
+                const startOffset = originalText.indexOf(group[0], searchFrom);
+                const lastSentence = group[group.length - 1];
+                const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
+                const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
+                chunks.push({
+                    id: nanoid(),
+                    documentId,
+                    content,
+                    startOffset: Math.max(0, startOffset),
+                    endOffset,
+                    order: chunks.length,
+                });
+                searchFrom = endOffset;
+            }
+            else {
+                const subChunks = this.splitLargeGroup(group, originalText, documentId, searchFrom, chunks.length);
+                chunks.push(...subChunks);
+                if (subChunks.length > 0) {
+                    searchFrom = subChunks[subChunks.length - 1].endOffset;
+                }
+            }
+        }
+        return chunks;
+    }
+    splitLargeGroup(sentences, originalText, documentId, searchFrom, startOrder) {
+        const chunks = [];
+        let currentSentences = [];
+        let currentLength = 0;
+        let order = startOrder;
+        let offset = searchFrom;
+        for (const sentence of sentences) {
+            const newLength = currentLength + (currentSentences.length > 0 ? 1 : 0) + sentence.length;
+            if (newLength > this.maxChunkSize && currentSentences.length > 0) {
+                const content = currentSentences.join(' ');
+                const startOffset = originalText.indexOf(currentSentences[0], offset);
+                const lastSentence = currentSentences[currentSentences.length - 1];
+                const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
+                const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
+                chunks.push({
+                    id: nanoid(),
+                    documentId,
+                    content,
+                    startOffset: Math.max(0, startOffset),
+                    endOffset,
+                    order: order++,
+                });
+                offset = endOffset;
+                currentSentences = [sentence];
+                currentLength = sentence.length;
+            }
+            else {
+                currentSentences.push(sentence);
+                currentLength = newLength;
+            }
+        }
+        if (currentSentences.length > 0) {
+            const content = currentSentences.join(' ');
+            const startOffset = originalText.indexOf(currentSentences[0], offset);
+            const lastSentence = currentSentences[currentSentences.length - 1];
+            const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
+            const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
+            chunks.push({
+                id: nanoid(),
+                documentId,
+                content,
+                startOffset: Math.max(0, startOffset),
+                endOffset,
+                order: order++,
+            });
+        }
+        return chunks;
+    }
+}
+//# sourceMappingURL=semantic-chunker.js.map

package/dist/chunkers/semantic-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEhC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBAEH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,KAAK,EAAE;iBACf,CAAC,CAAC;gBAEH,MAAM,GAAG,SAAS,CAAC;gBACnB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAElF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;gBACrC,SAAS;gBACT,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export declare const VERSION = "0.1.0";
+export { RAGPipeline, type RAGPipelineDeps } from './rag-pipeline';
+export { RAGPipelineBuilder } from './rag-builder';
+export * from './loaders';
+export * from './chunkers';
+export * from './retrievers';
+export * from './rerankers';
+export * from './schema';
+export { createSearchTool, createIngestTool, ragTools, type RAGTool } from './tools';
+export type { RAGDocument, DocumentChunk, DocumentLoader, Chunker, AsyncChunker, ChunkingConfig, ChunkingStrategy, Retriever, RetrievalConfig, RetrievalStrategy, RetrievalResult, Reranker, RAGPipelineConfig, EmbeddingService, EmbeddingAdapter, } from '@cogitator-ai/types';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAE5B,cAAc,UAAU,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,KAAK,OAAO,EAAE,MAAM,SAAS,CAAC;AAErF,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC"}

package/dist/index.js

@@ -0,0 +1,10 @@
+export const VERSION = '0.1.0';
+export { RAGPipeline } from './rag-pipeline';
+export { RAGPipelineBuilder } from './rag-builder';
+export * from './loaders';
+export * from './chunkers';
+export * from './retrievers';
+export * from './rerankers';
+export * from './schema';
+export { createSearchTool, createIngestTool, ragTools } from './tools';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAwB,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAE5B,cAAc,UAAU,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAgB,MAAM,SAAS,CAAC"}

package/dist/loaders/csv-loader.d.ts

@@ -0,0 +1,16 @@
+import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
+export interface CSVLoaderOptions {
+    contentColumn?: string;
+    metadataColumns?: string[];
+    delimiter?: string;
+}
+export declare class CSVLoader implements DocumentLoader {
+    readonly supportedTypes: string[];
+    private readonly contentColumn?;
+    private readonly metadataColumns?;
+    private readonly delimiter?;
+    constructor(options?: CSVLoaderOptions);
+    load(source: string): Promise<RAGDocument[]>;
+    private extractMetadata;
+}
+//# sourceMappingURL=csv-loader.d.ts.map

package/dist/loaders/csv-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA4BlD,OAAO,CAAC,eAAe;CAWxB"}