@cogitator-ai/rag 0.1.10 → 0.2.0

package/README.md

@@ -52,23 +52,42 @@ await pipeline.ingest('./docs');
 const results = await pipeline.query('How does authentication work?');
 
 for (const r of results) {
-  console.log(`[${r.score.toFixed(3)}] ${r.content.slice(0, 100)}...`);
+  console.log(`[${r.score.toFixed(3)}] ${r.source}: ${r.content.slice(0, 100)}...`);
 }
 ```
 
+Every stored chunk carries the document's loader metadata (e.g. Markdown frontmatter, PDF page numbers, CSV metadata columns, page title/URL) plus `documentId`, `source`, `sourceType`, `order`, `startOffset` and `endOffset`. Retrieval results expose the document path/URL as `result.source` and the rest in `result.metadata`.
+
 ---
 
 ## Document Loaders
 
-| Loader           | Formats               | Optional Dep | Notes                         |
-| ---------------- | --------------------- | ------------ | ----------------------------- |
-| `TextLoader`     | `.txt`                | —            | Files and directories         |
-| `MarkdownLoader` | `.md`                 | —            | Strips frontmatter by default |
-| `JSONLoader`     | `.json`               | —            | Configurable content field    |
-| `CSVLoader`      | `.csv`                | `papaparse`  | Column selection, row mapping |
-| `HTMLLoader`     | `.html`, `.htm`       | `cheerio`    | CSS selector support          |
-| `PDFLoader`      | `.pdf`                | `pdf-parse`  | Text extraction from PDFs     |
-| `WebLoader`      | `http://`, `https://` | `cheerio`    | Fetches and parses web pages  |
+| Loader           | Formats               | Optional Dep | Notes                                                  |
+| ---------------- | --------------------- | ------------ | ------------------------------------------------------ |
+| `TextLoader`     | `.txt`                | —            | Files and directories                                  |
+| `MarkdownLoader` | `.md`, `.mdx`         | —            | Optional frontmatter -> metadata                       |
+| `JSONLoader`     | `.json`               | —            | Configurable content field                             |
+| `CSVLoader`      | `.csv`                | `papaparse`  | Column selection, `row` number in metadata             |
+| `HTMLLoader`     | `.html`, `.htm`       | `cheerio`    | CSS selector, scripts/styles removed, block-aware text |
+| `PDFLoader`      | `.pdf`                | `pdf-parse`  | Whole document or one document per page                |
+| `WebLoader`      | `http://`, `https://` | `cheerio`    | HTML, plain text and JSON pages; SSRF-protected        |
+
+Directory sources (`TextLoader`, `MarkdownLoader`) are loaded in sorted file-name order.
+
+### WebLoader security
+
+`WebLoader` blocks loopback, private, link-local, CGNAT, multicast and cloud-metadata addresses — including IPv4-mapped IPv6 forms and hostnames that **resolve** to such addresses. The check runs at connect time for every redirect hop, so DNS rebinding cannot bypass it. To ingest an intranet site, opt in explicitly:
+
+```typescript
+const intranet = new WebLoader({
+  allowPrivateNetwork: true,
+  headers: { Authorization: `Bearer ${process.env.WIKI_TOKEN}` },
+  timeoutMs: 15_000,
+  maxResponseBytes: 10 * 1024 * 1024,
+});
+```
+
+Responses are decompressed (gzip, deflate, br), decoded using the declared charset, and limited to `maxResponseBytes` (50MB by default). Non-text content types are rejected.
 
 ```typescript
 import { MarkdownLoader, WebLoader, CSVLoader } from '@cogitator-ai/rag';
@@ -101,7 +120,7 @@ const chunks = chunker.chunk(text, documentId);
 
 ### Recursive
 
-Splits on configurable separators (`\n\n`, `\n`, `. `, ` `) trying to keep paragraphs and sentences intact.
+Splits on configurable separators (`\n\n`, `\n`, `. `, ` `) trying to keep paragraphs and sentences intact. Chunk offsets always point into the original text, chunks never exceed `chunkSize`, sentence punctuation is kept and overlap never produces a chunk fully contained in the previous one.
 
 ```typescript
 import { RecursiveChunker } from '@cogitator-ai/rag';
@@ -115,7 +134,7 @@ const chunker = new RecursiveChunker({
 
 ### Semantic
 
-Uses embedding similarity between sentences to find natural breakpoints. Async — requires an `EmbeddingService`.
+Uses embedding similarity between sentences (and blank-line separated blocks) to find natural breakpoints. Async — requires an `EmbeddingService`.
 
 ```typescript
 import { SemanticChunker } from '@cogitator-ai/rag';
@@ -152,6 +171,29 @@ const chunker = createChunker(
 | `hybrid`      | `HybridRetriever`     | Combines BM25 keyword search with vector search (RRF)         |
 | `multi-query` | `MultiQueryRetriever` | Expands query into variants, merges results                   |
 
+`RAGPipelineBuilder` builds the retriever matching `retrieval.strategy` when no custom `withRetriever()` is given:
+
+```typescript
+const pipeline = new RAGPipelineBuilder()
+  .withLoader(loader)
+  .withEmbeddingService(embeddingService)
+  .withEmbeddingAdapter(embeddingAdapter)
+  .withHybridSearch(new HybridSearch({ embeddingAdapter, embeddingService })) // 'hybrid'
+  .withQueryExpander(expandWithLLM) // 'multi-query'
+  .withConfig({
+    chunking: { strategy: 'recursive', chunkSize: 500, chunkOverlap: 50 },
+    retrieval: { strategy: 'multi-query', multiQueryCount: 3, topK: 5 },
+  })
+  .build();
+```
+
+| Strategy      | Builder requirement                        | Config used                                                                                                     |
+| ------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------- |
+| `similarity`  | —                                          | `topK`, `threshold`                                                                                             |
+| `mmr`         | —                                          | `mmrLambda`, `topK`, `threshold`                                                                                |
+| `hybrid`      | `withHybridSearch(hybridSearch, weights?)` | `topK`, `threshold`                                                                                             |
+| `multi-query` | `withQueryExpander(fn)`                    | `multiQueryCount` (max variants), `topK`; base is hybrid when `withHybridSearch()` is set, similarity otherwise |
+
 ### Similarity
 
 ```typescript
@@ -169,7 +211,7 @@ const results = await retriever.retrieve('What is TypeScript?');
 
 ### MMR
 
-Reduces redundancy by penalizing results that are too similar to already-selected ones.
+Reduces redundancy by penalizing results that are too similar to already-selected ones. If the vector store does not return stored vectors (e.g. Qdrant), candidates are re-embedded to compute diversity.
 
 ```typescript
 import { MMRRetriever } from '@cogitator-ai/rag';
@@ -184,7 +226,7 @@ const retriever = new MMRRetriever({
 
 ### Hybrid
 
-Requires `HybridSearch` from `@cogitator-ai/memory`.
+Requires `HybridSearch` from `@cogitator-ai/memory`. Chunks ingested through `RAGPipeline` are added to the HybridSearch BM25 index automatically (any retriever implementing `ChunkIndexer.indexChunk()` is notified on ingest), and keyword-only hits are mapped back to their chunk, document and metadata.
 
 ```typescript
 import { HybridRetriever } from '@cogitator-ai/rag';
@@ -198,7 +240,7 @@ const retriever = new HybridRetriever({
 
 ### Multi-Query
 
-Generates query variations and merges results. You provide the expansion function (typically an LLM call).
+Generates query variations and merges results (best score per chunk). You provide the expansion function (typically an LLM call). The original query is always searched; variants are de-duplicated and capped by `defaultMaxQueries` / `multiQueryCount`. If expansion fails the original query is used alone; if every retrieval fails the error is thrown.
 
 ```typescript
 import { MultiQueryRetriever } from '@cogitator-ai/rag';
@@ -211,6 +253,7 @@ const retriever = new MultiQueryRetriever({
     );
     return response.split('\n').filter(Boolean);
   },
+  defaultMaxQueries: 3,
 });
 ```
 
@@ -279,9 +322,8 @@ Use `ragTools()` to give a Cogitator agent access to your knowledge base.
 
 ```typescript
 import { Agent, tool } from '@cogitator-ai/core';
-import { RAGPipelineBuilder, TextLoader, createSearchTool } from '@cogitator-ai/rag';
+import { RAGPipelineBuilder, TextLoader, ragTools } from '@cogitator-ai/rag';
 import { InMemoryEmbeddingAdapter, OpenAIEmbeddingService } from '@cogitator-ai/memory';
-import { z } from 'zod';
 
 const pipeline = new RAGPipelineBuilder()
   .withLoader(new TextLoader())
@@ -295,27 +337,20 @@ const pipeline = new RAGPipelineBuilder()
 
 await pipeline.ingest('./knowledge-base');
 
-const ragSearch = createSearchTool(pipeline);
-
-const searchKB = tool({
-  name: ragSearch.name,
-  description: ragSearch.description,
-  parameters: z.object({
-    query: z.string().describe('Search query'),
-    limit: z.number().int().positive().optional(),
-    threshold: z.number().min(0).max(1).optional(),
-  }),
-  execute: async (params) => ragSearch.execute(params),
-});
+const [ragSearch, ragIngest] = ragTools(pipeline, { allowedRoots: ['./knowledge-base'] });
 
 const agent = new Agent({
   name: 'docs-assistant',
-  model: 'gpt-4o',
+  model: 'openai/gpt-4o',
   instructions: 'Use rag_search to find information before answering.',
-  tools: [searchKB],
+  tools: [tool(ragSearch), tool(ragIngest)],
 });
 ```
 
+`RAGTool` objects carry Zod parameter schemas and can be passed to `tool()` directly. `rag_search` falls back to the pipeline's configured `topK`/`threshold` when the model omits `limit`/`threshold`.
+
+**Security:** `rag_ingest` reads whatever source the model passes. When the tool is exposed to an LLM, restrict it with `allowedRoots` (paths are canonicalized, so `..` traversal and symlink escapes are rejected) and `allowUrls: false` if web ingestion is not needed. URL ingestion goes through the SSRF-protected `WebLoader` when the pipeline uses one.
+
 ---
 
 ## Pipeline Stats

package/dist/chunkers/create-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC,QAAQ,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC,QAAQ,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/chunkers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/chunkers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAgC,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAgC,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,eAAe,EAA+B,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAgC,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/chunkers/recursive-chunker.d.ts

@@ -10,9 +10,8 @@ export declare class RecursiveChunker implements Chunker {
     private readonly separators;
     constructor(options: RecursiveChunkerOptions);
     chunk(text: string, documentId: string): DocumentChunk[];
-    private mapPiecePositions;
     private mergePieces;
-    private splitText;
+    private splitSpan;
     private charSplit;
 }
 //# sourceMappingURL=recursive-chunker.d.ts.map

package/dist/chunkers/recursive-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAUxD,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,SAAS;IA+CjB,OAAO,CAAC,SAAS;CAOlB"}
+{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,WAAW;IAwCnB,OAAO,CAAC,SAAS;IAyDjB,OAAO,CAAC,SAAS;CAOlB"}

package/dist/chunkers/recursive-chunker.js

@@ -21,102 +21,112 @@ export class RecursiveChunker {
     chunk(text, documentId) {
         if (text.length === 0)
             return [];
-        const pieces = this.splitText(text, 0);
-        const piecePositions = this.mapPiecePositions(pieces, text);
-        return this.mergePieces(pieces, piecePositions, text, documentId);
+        const pieces = this.splitSpan(text, { start: 0, end: text.length }, 0);
+        return this.mergePieces(pieces, text, documentId);
     }
-    mapPiecePositions(pieces, originalText) {
-        const positions = [];
-        let searchFrom = 0;
-        for (const piece of pieces) {
-            const idx = originalText.indexOf(piece, searchFrom);
-            const start = idx >= 0 ? idx : searchFrom;
-            positions.push({ start, end: start + piece.length });
-            searchFrom = start + piece.length;
-        }
-        return positions;
-    }
-    mergePieces(pieces, positions, originalText, documentId) {
+    mergePieces(pieces, text, documentId) {
         const chunks = [];
-        let order = 0;
         let i = 0;
         while (i < pieces.length) {
-            const chunkStart = positions[i].start;
+            const chunkStart = pieces[i].start;
             let j = i + 1;
-            while (j < pieces.length && positions[j].end - chunkStart <= this.chunkSize) {
+            while (j < pieces.length && pieces[j].end - chunkStart <= this.chunkSize) {
                 j++;
             }
-            const chunkEnd = positions[j - 1].end;
+            const chunkEnd = pieces[j - 1].end;
             chunks.push({
                 id: nanoid(),
                 documentId,
-                content: originalText.slice(chunkStart, chunkEnd),
+                content: text.slice(chunkStart, chunkEnd),
                 startOffset: chunkStart,
                 endOffset: chunkEnd,
-                order: order++,
+                order: chunks.length,
             });
             if (j >= pieces.length)
                 break;
+            let next = j;
             if (this.chunkOverlap > 0) {
                 const overlapTarget = chunkEnd - this.chunkOverlap;
-                let k = j;
-                while (k > i + 1 && positions[k - 1].start >= overlapTarget) {
-                    k--;
+                while (next > i + 1 && pieces[next - 1].start >= overlapTarget) {
+                    next--;
+                }
+                while (next < j && pieces[j].end - pieces[next].start > this.chunkSize) {
+                    next++;
                 }
-                i = Math.max(k, i + 1);
-            }
-            else {
-                i = j;
             }
+            i = next;
         }
         return chunks;
     }
-    splitText(text, separatorIndex) {
-        if (text.length <= this.chunkSize)
-            return [text];
-        if (separatorIndex >= this.separators.length) {
-            return this.charSplit(text);
+    splitSpan(text, span, separatorIndex) {
+        const trimmed = trimSpan(text, span);
+        if (!trimmed)
+            return [];
+        if (trimmed.end - trimmed.start <= this.chunkSize)
+            return [trimmed];
+        let index = separatorIndex;
+        while (index < this.separators.length) {
+            const separator = this.separators[index];
+            if (separator === '')
+                break;
+            if (text.slice(trimmed.start, trimmed.end).includes(separator))
+                break;
+            index++;
         }
-        const separator = this.separators[separatorIndex];
-        if (separator === '') {
-            return this.charSplit(text);
+        const separator = this.separators[index];
+        if (separator === undefined || separator === '') {
+            return this.charSplit(trimmed);
         }
-        const parts = text.split(separator);
-        if (parts.length === 1) {
-            return this.splitText(text, separatorIndex + 1);
+        const kept = /^\S*/.exec(separator)[0].length;
+        const segments = [];
+        let cursor = trimmed.start;
+        for (;;) {
+            const at = text.indexOf(separator, cursor);
+            if (at === -1 || at >= trimmed.end) {
+                segments.push({ start: cursor, end: trimmed.end });
+                break;
+            }
+            segments.push({ start: cursor, end: Math.min(at + kept, trimmed.end) });
+            cursor = at + separator.length;
         }
         const merged = [];
-        let current = '';
-        for (const part of parts) {
-            if (part.length === 0)
+        let current = null;
+        for (const raw of segments) {
+            const segment = trimSpan(text, raw);
+            if (!segment)
                 continue;
-            const candidate = current ? current + separator + part : part;
-            if (candidate.length <= this.chunkSize) {
-                current = candidate;
+            if (current && segment.end - current.start <= this.chunkSize) {
+                current = { start: current.start, end: segment.end };
+                continue;
+            }
+            if (current)
+                merged.push(current);
+            if (segment.end - segment.start > this.chunkSize) {
+                merged.push(...this.splitSpan(text, segment, index + 1));
+                current = null;
             }
             else {
-                if (current)
-                    merged.push(current);
-                if (part.length > this.chunkSize) {
-                    const subParts = this.splitText(part, separatorIndex + 1);
-                    merged.push(...subParts);
-                    current = '';
-                }
-                else {
-                    current = part;
-                }
+                current = segment;
             }
         }
         if (current)
             merged.push(current);
         return merged;
     }
-    charSplit(text) {
+    charSplit(span) {
         const result = [];
-        for (let i = 0; i < text.length; i += this.chunkSize) {
-            result.push(text.slice(i, i + this.chunkSize));
+        for (let start = span.start; start < span.end; start += this.chunkSize) {
+            result.push({ start, end: Math.min(start + this.chunkSize, span.end) });
         }
         return result;
     }
 }
+function trimSpan(text, span) {
+    let { start, end } = span;
+    while (start < end && /\s/.test(text[start]))
+        start++;
+    while (end > start && /\s/.test(text[end - 1]))
+        end--;
+    return start < end ? { start, end } : null;
+}
 //# sourceMappingURL=recursive-chunker.js.map

package/dist/chunkers/recursive-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEvC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpE,CAAC;IAEO,iBAAiB,CACvB,MAAgB,EAChB,YAAoB;QAEpB,MAAM,SAAS,GAA0C,EAAE,CAAC;QAC5D,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YACpD,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;YAC1C,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACrD,UAAU,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,WAAW,CACjB,MAAgB,EAChB,SAAgD,EAChD,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAEtC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC5E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEtC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACjD,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,IAAI,CAAC,GAAG,CAAC,CAAC;gBACV,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAC5D,CAAC,EAAE,CAAC;gBACN,CAAC;gBACD,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,CAAC,GAAG,CAAC,CAAC;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,cAAsB;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAElD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9D,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACzB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,UAAkB;QAClE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC;YAEpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACzC,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChE,IAAI,EAAE,CAAC;gBACT,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzE,IAAI,EAAE,CAAC;gBACT,CAAC;YACH,CAAC;YACD,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,IAAU,EAAE,cAAsB;QAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,IAAI,KAAK,GAAG,cAAc,CAAC;QAC3B,OAAO,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAE,CAAC;YAC1C,IAAI,SAAS,KAAK,EAAE;gBAAE,MAAM;YAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,MAAM;YACtE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAW,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnD,MAAM;YACR,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,MAAM,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,IAAI,OAAO,GAAgB,IAAI,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7D,OAAO,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzD,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,OAAO,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAU;QAC1B,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAOD,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAU;IACxC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;QAAE,KAAK,EAAE,CAAC;IACvD,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IACvD,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}

package/dist/chunkers/semantic-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8BD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAqBrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IA2EvB,OAAO,CAAC,iBAAiB;CAqB1B"}
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8BD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAqBrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAuCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IA2EvB,OAAO,CAAC,iBAAiB;CAqB1B"}

package/dist/chunkers/semantic-chunker.js

@@ -15,7 +15,7 @@ function cosineSimilarity(a, b) {
     const denom = Math.sqrt(normA) * Math.sqrt(normB);
     return denom === 0 ? 0 : dot / denom;
 }
-const SENTENCE_SPLIT_RE = /(?<=[.?!])\s+/;
+const SENTENCE_SPLIT_RE = /(?<=[.?!])\s+|\n\s*\n/;
 function splitSentences(text) {
     return text
         .split(SENTENCE_SPLIT_RE)
@@ -52,6 +52,9 @@ export class SemanticChunker {
             return this.buildChunks([sentences], text, documentId);
         }
         const embeddings = await this.embeddingService.embedBatch(sentences);
+        if (embeddings.length !== sentences.length) {
+            throw new Error(`Embedding count mismatch: got ${embeddings.length} vectors for ${sentences.length} sentences`);
+        }
         const similarities = [];
         for (let i = 0; i < embeddings.length - 1; i++) {
             similarities.push(cosineSimilarity(embeddings[i], embeddings[i + 1]));

package/dist/chunkers/semantic-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CACpF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CACpF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export declare const VERSION = "0.1.0";
+export declare const VERSION: string;
 export { RAGPipeline, type RAGPipelineDeps } from './rag-pipeline.js';
 export { RAGPipelineBuilder } from './rag-builder.js';
 export * from './loaders/index.js';
@@ -6,6 +6,6 @@ export * from './chunkers/index.js';
 export * from './retrievers/index.js';
 export * from './rerankers/index.js';
 export * from './schema.js';
-export { createSearchTool, createIngestTool, ragTools, type RAGTool } from './tools.js';
+export { createSearchTool, createIngestTool, ragTools, type RAGTool, type IngestToolOptions, } from './tools.js';
 export type { RAGDocument, DocumentChunk, DocumentLoader, Chunker, AsyncChunker, ChunkingConfig, ChunkingStrategy, Retriever, RetrievalConfig, RetrievalStrategy, RetrievalResult, Reranker, RAGPipelineConfig, EmbeddingService, EmbeddingAdapter, } from '@cogitator-ai/types';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAE5B,cAAc,UAAU,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,KAAK,OAAO,EAAE,MAAM,SAAS,CAAC;AAErF,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AAErC,cAAc,aAAa,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,iBAAiB,GACvB,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC"}

package/dist/index.js

@@ -1,4 +1,6 @@
-export const VERSION = '0.1.0';
+import { createRequire } from 'node:module';
+const packageJson = createRequire(import.meta.url)('../package.json');
+export const VERSION = packageJson.version;
 export { RAGPipeline } from './rag-pipeline.js';
 export { RAGPipelineBuilder } from './rag-builder.js';
 export * from './loaders/index.js';
@@ -6,5 +8,5 @@ export * from './chunkers/index.js';
 export * from './retrievers/index.js';
 export * from './rerankers/index.js';
 export * from './schema.js';
-export { createSearchTool, createIngestTool, ragTools } from './tools.js';
+export { createSearchTool, createIngestTool, ragTools, } from './tools.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAwB,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAE5B,cAAc,UAAU,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAgB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAwB,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AAErC,cAAc,aAAa,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,GAGT,MAAM,YAAY,CAAC"}

package/dist/loaders/csv-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAkClD,OAAO,CAAC,eAAe;CAWxB"}
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAevE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmClD,OAAO,CAAC,eAAe;CAWxB"}

package/dist/loaders/csv-loader.js

@@ -2,12 +2,15 @@ import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { nanoid } from 'nanoid';
 async function loadPapaparse() {
+    let mod;
     try {
-        return await import('papaparse');
+        mod = (await import('papaparse'));
     }
     catch {
         throw new Error('papaparse is required for CSVLoader. Install it: pnpm add papaparse');
     }
+    const interop = mod.default;
+    return interop && typeof interop.parse === 'function' ? interop : mod;
 }
 export class CSVLoader {
     supportedTypes = ['csv'];
@@ -30,18 +33,20 @@ export class CSVLoader {
         });
         const headers = result.meta.fields ?? [];
         const contentCol = this.contentColumn ?? headers[0];
+        if (contentCol === undefined)
+            return [];
         if (this.contentColumn && !headers.includes(this.contentColumn)) {
             throw new Error(`CSVLoader: contentColumn "${this.contentColumn}" not found in CSV headers: ${headers.join(', ')}`);
         }
-        return result.data.map((row) => {
+        return result.data.map((row, index) => {
             const content = row[contentCol] ?? '';
-            const metadata = this.extractMetadata(row);
+            const metadata = { ...this.extractMetadata(row), row: index + 1 };
             return {
                 id: nanoid(),
                 content,
                 source: filePath,
                 sourceType: 'csv',
-                ...(metadata && Object.keys(metadata).length > 0 && { metadata }),
+                metadata,
             };
         });
     }

package/dist/loaders/csv-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,aAAa;IAC1B,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,aAAa,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAE3C,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
+{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKhC,KAAK,UAAU,aAAa;IAC1B,IAAI,GAA8B,CAAC;IACnC,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,CAA8B,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,OAAO,OAAO,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxE,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAExC,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,aAAa,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACpC,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;YAElE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,QAAQ;aACT,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/loaders/html-loader.d.ts

@@ -2,6 +2,10 @@ import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
 export interface HTMLLoaderOptions {
     selector?: string;
 }
+/**
+ * Extracts readable text from HTML: scripts, styles and other non-content elements are
+ * dropped, block elements become line breaks and whitespace is normalized.
+ */
 export declare class HTMLLoader implements DocumentLoader {
     readonly supportedTypes: string[];
     readonly selector: string;

package/dist/loaders/html-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAmB;IAC1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,OAAO,CAAC,EAAE,iBAAiB;IAIjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAM5C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC;CAkBhG"}
+{"version":3,"file":"html-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAgBD;;;GAGG;AACH,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAmB;IAC1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,OAAO,CAAC,EAAE,iBAAiB;IAIjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAM5C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC;CA+BhG"}