@cogitator-ai/rag 0.1.9 → 0.2.0

package/README.md

@@ -52,23 +52,42 @@ await pipeline.ingest('./docs');
 const results = await pipeline.query('How does authentication work?');
 
 for (const r of results) {
-  console.log(`[${r.score.toFixed(3)}] ${r.content.slice(0, 100)}...`);
+  console.log(`[${r.score.toFixed(3)}] ${r.source}: ${r.content.slice(0, 100)}...`);
 }
 ```
 
+Every stored chunk carries the document's loader metadata (e.g. Markdown frontmatter, PDF page numbers, CSV metadata columns, page title/URL) plus `documentId`, `source`, `sourceType`, `order`, `startOffset` and `endOffset`. Retrieval results expose the document path/URL as `result.source` and the rest in `result.metadata`.
+
 ---
 
 ## Document Loaders
 
-| Loader           | Formats               | Optional Dep | Notes                         |
-| ---------------- | --------------------- | ------------ | ----------------------------- |
-| `TextLoader`     | `.txt`                | —            | Files and directories         |
-| `MarkdownLoader` | `.md`                 | —            | Strips frontmatter by default |
-| `JSONLoader`     | `.json`               | —            | Configurable content field    |
-| `CSVLoader`      | `.csv`                | `papaparse`  | Column selection, row mapping |
-| `HTMLLoader`     | `.html`, `.htm`       | `cheerio`    | CSS selector support          |
-| `PDFLoader`      | `.pdf`                | `pdf-parse`  | Text extraction from PDFs     |
-| `WebLoader`      | `http://`, `https://` | `cheerio`    | Fetches and parses web pages  |
+| Loader           | Formats               | Optional Dep | Notes                                                  |
+| ---------------- | --------------------- | ------------ | ------------------------------------------------------ |
+| `TextLoader`     | `.txt`                | —            | Files and directories                                  |
+| `MarkdownLoader` | `.md`, `.mdx`         | —            | Optional frontmatter -> metadata                       |
+| `JSONLoader`     | `.json`               | —            | Configurable content field                             |
+| `CSVLoader`      | `.csv`                | `papaparse`  | Column selection, `row` number in metadata             |
+| `HTMLLoader`     | `.html`, `.htm`       | `cheerio`    | CSS selector, scripts/styles removed, block-aware text |
+| `PDFLoader`      | `.pdf`                | `pdf-parse`  | Whole document or one document per page                |
+| `WebLoader`      | `http://`, `https://` | `cheerio`    | HTML, plain text and JSON pages; SSRF-protected        |
+
+Directory sources (`TextLoader`, `MarkdownLoader`) are loaded in sorted file-name order.
+
+### WebLoader security
+
+`WebLoader` blocks loopback, private, link-local, CGNAT, multicast and cloud-metadata addresses — including IPv4-mapped IPv6 forms and hostnames that **resolve** to such addresses. The check runs at connect time for every redirect hop, so DNS rebinding cannot bypass it. To ingest an intranet site, opt in explicitly:
+
+```typescript
+const intranet = new WebLoader({
+  allowPrivateNetwork: true,
+  headers: { Authorization: `Bearer ${process.env.WIKI_TOKEN}` },
+  timeoutMs: 15_000,
+  maxResponseBytes: 10 * 1024 * 1024,
+});
+```
+
+Responses are decompressed (gzip, deflate, br), decoded using the declared charset, and limited to `maxResponseBytes` (50MB by default). Non-text content types are rejected.
 
 ```typescript
 import { MarkdownLoader, WebLoader, CSVLoader } from '@cogitator-ai/rag';
@@ -101,7 +120,7 @@ const chunks = chunker.chunk(text, documentId);
 
 ### Recursive
 
-Splits on configurable separators (`\n\n`, `\n`, `. `, ` `) trying to keep paragraphs and sentences intact.
+Splits on configurable separators (`\n\n`, `\n`, `. `, ` `) trying to keep paragraphs and sentences intact. Chunk offsets always point into the original text, chunks never exceed `chunkSize`, sentence punctuation is kept and overlap never produces a chunk fully contained in the previous one.
 
 ```typescript
 import { RecursiveChunker } from '@cogitator-ai/rag';
@@ -115,7 +134,7 @@ const chunker = new RecursiveChunker({
 
 ### Semantic
 
-Uses embedding similarity between sentences to find natural breakpoints. Async — requires an `EmbeddingService`.
+Uses embedding similarity between sentences (and blank-line separated blocks) to find natural breakpoints. Async — requires an `EmbeddingService`.
 
 ```typescript
 import { SemanticChunker } from '@cogitator-ai/rag';
@@ -152,6 +171,29 @@ const chunker = createChunker(
 | `hybrid`      | `HybridRetriever`     | Combines BM25 keyword search with vector search (RRF)         |
 | `multi-query` | `MultiQueryRetriever` | Expands query into variants, merges results                   |
 
+`RAGPipelineBuilder` builds the retriever matching `retrieval.strategy` when no custom `withRetriever()` is given:
+
+```typescript
+const pipeline = new RAGPipelineBuilder()
+  .withLoader(loader)
+  .withEmbeddingService(embeddingService)
+  .withEmbeddingAdapter(embeddingAdapter)
+  .withHybridSearch(new HybridSearch({ embeddingAdapter, embeddingService })) // 'hybrid'
+  .withQueryExpander(expandWithLLM) // 'multi-query'
+  .withConfig({
+    chunking: { strategy: 'recursive', chunkSize: 500, chunkOverlap: 50 },
+    retrieval: { strategy: 'multi-query', multiQueryCount: 3, topK: 5 },
+  })
+  .build();
+```
+
+| Strategy      | Builder requirement                        | Config used                                                                                                     |
+| ------------- | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------- |
+| `similarity`  | —                                          | `topK`, `threshold`                                                                                             |
+| `mmr`         | —                                          | `mmrLambda`, `topK`, `threshold`                                                                                |
+| `hybrid`      | `withHybridSearch(hybridSearch, weights?)` | `topK`, `threshold`                                                                                             |
+| `multi-query` | `withQueryExpander(fn)`                    | `multiQueryCount` (max variants), `topK`; base is hybrid when `withHybridSearch()` is set, similarity otherwise |
+
 ### Similarity
 
 ```typescript
@@ -169,7 +211,7 @@ const results = await retriever.retrieve('What is TypeScript?');
 
 ### MMR
 
-Reduces redundancy by penalizing results that are too similar to already-selected ones.
+Reduces redundancy by penalizing results that are too similar to already-selected ones. If the vector store does not return stored vectors (e.g. Qdrant), candidates are re-embedded to compute diversity.
 
 ```typescript
 import { MMRRetriever } from '@cogitator-ai/rag';
@@ -184,7 +226,7 @@ const retriever = new MMRRetriever({
 
 ### Hybrid
 
-Requires `HybridSearch` from `@cogitator-ai/memory`.
+Requires `HybridSearch` from `@cogitator-ai/memory`. Chunks ingested through `RAGPipeline` are added to the HybridSearch BM25 index automatically (any retriever implementing `ChunkIndexer.indexChunk()` is notified on ingest), and keyword-only hits are mapped back to their chunk, document and metadata.
 
 ```typescript
 import { HybridRetriever } from '@cogitator-ai/rag';
@@ -198,7 +240,7 @@ const retriever = new HybridRetriever({
 
 ### Multi-Query
 
-Generates query variations and merges results. You provide the expansion function (typically an LLM call).
+Generates query variations and merges results (best score per chunk). You provide the expansion function (typically an LLM call). The original query is always searched; variants are de-duplicated and capped by `defaultMaxQueries` / `multiQueryCount`. If expansion fails the original query is used alone; if every retrieval fails the error is thrown.
 
 ```typescript
 import { MultiQueryRetriever } from '@cogitator-ai/rag';
@@ -211,6 +253,7 @@ const retriever = new MultiQueryRetriever({
     );
     return response.split('\n').filter(Boolean);
   },
+  defaultMaxQueries: 3,
 });
 ```
 
@@ -279,9 +322,8 @@ Use `ragTools()` to give a Cogitator agent access to your knowledge base.
 
 ```typescript
 import { Agent, tool } from '@cogitator-ai/core';
-import { RAGPipelineBuilder, TextLoader, createSearchTool } from '@cogitator-ai/rag';
+import { RAGPipelineBuilder, TextLoader, ragTools } from '@cogitator-ai/rag';
 import { InMemoryEmbeddingAdapter, OpenAIEmbeddingService } from '@cogitator-ai/memory';
-import { z } from 'zod';
 
 const pipeline = new RAGPipelineBuilder()
   .withLoader(new TextLoader())
@@ -295,27 +337,20 @@ const pipeline = new RAGPipelineBuilder()
 
 await pipeline.ingest('./knowledge-base');
 
-const ragSearch = createSearchTool(pipeline);
-
-const searchKB = tool({
-  name: ragSearch.name,
-  description: ragSearch.description,
-  parameters: z.object({
-    query: z.string().describe('Search query'),
-    limit: z.number().int().positive().optional(),
-    threshold: z.number().min(0).max(1).optional(),
-  }),
-  execute: async (params) => ragSearch.execute(params),
-});
+const [ragSearch, ragIngest] = ragTools(pipeline, { allowedRoots: ['./knowledge-base'] });
 
 const agent = new Agent({
   name: 'docs-assistant',
-  model: 'gpt-4o',
+  model: 'openai/gpt-4o',
   instructions: 'Use rag_search to find information before answering.',
-  tools: [searchKB],
+  tools: [tool(ragSearch), tool(ragIngest)],
 });
 ```
 
+`RAGTool` objects carry Zod parameter schemas and can be passed to `tool()` directly. `rag_search` falls back to the pipeline's configured `topK`/`threshold` when the model omits `limit`/`threshold`.
+
+**Security:** `rag_ingest` reads whatever source the model passes. When the tool is exposed to an LLM, restrict it with `allowedRoots` (paths are canonicalized, so `..` traversal and symlink escapes are rejected) and `allowUrls: false` if web ingestion is not needed. URL ingestion goes through the SSRF-protected `WebLoader` when the pipeline uses one.
+
 ---
 
 ## Pipeline Stats

package/dist/chunkers/create-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC,QAAQ,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC,QAAQ,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/chunkers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/chunkers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAgC,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAgC,MAAM,qBAAqB,CAAC;AACrF,OAAO,EAAE,eAAe,EAA+B,MAAM,oBAAoB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunkers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAgC,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/chunkers/recursive-chunker.d.ts

@@ -10,9 +10,8 @@ export declare class RecursiveChunker implements Chunker {
     private readonly separators;
     constructor(options: RecursiveChunkerOptions);
     chunk(text: string, documentId: string): DocumentChunk[];
-    private splitText;
+    private mergePieces;
+    private splitSpan;
     private charSplit;
-    private mergeWithOverlap;
-    private mergeWithOverlapSimple;
 }
 //# sourceMappingURL=recursive-chunker.d.ts.map

package/dist/chunkers/recursive-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,SAAS;IA+CjB,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,gBAAgB;IAwFxB,OAAO,CAAC,sBAAsB;CAwB/B"}
+{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,WAAW;IAwCnB,OAAO,CAAC,SAAS;IAyDjB,OAAO,CAAC,SAAS;CAOlB"}

package/dist/chunkers/recursive-chunker.js

@@ -21,141 +21,112 @@ export class RecursiveChunker {
     chunk(text, documentId) {
         if (text.length === 0)
             return [];
-        const texts = this.splitText(text, 0);
-        return this.mergeWithOverlap(texts, documentId);
+        const pieces = this.splitSpan(text, { start: 0, end: text.length }, 0);
+        return this.mergePieces(pieces, text, documentId);
     }
-    splitText(text, separatorIndex) {
-        if (text.length <= this.chunkSize)
-            return [text];
-        if (separatorIndex >= this.separators.length) {
-            return this.charSplit(text);
+    mergePieces(pieces, text, documentId) {
+        const chunks = [];
+        let i = 0;
+        while (i < pieces.length) {
+            const chunkStart = pieces[i].start;
+            let j = i + 1;
+            while (j < pieces.length && pieces[j].end - chunkStart <= this.chunkSize) {
+                j++;
+            }
+            const chunkEnd = pieces[j - 1].end;
+            chunks.push({
+                id: nanoid(),
+                documentId,
+                content: text.slice(chunkStart, chunkEnd),
+                startOffset: chunkStart,
+                endOffset: chunkEnd,
+                order: chunks.length,
+            });
+            if (j >= pieces.length)
+                break;
+            let next = j;
+            if (this.chunkOverlap > 0) {
+                const overlapTarget = chunkEnd - this.chunkOverlap;
+                while (next > i + 1 && pieces[next - 1].start >= overlapTarget) {
+                    next--;
+                }
+                while (next < j && pieces[j].end - pieces[next].start > this.chunkSize) {
+                    next++;
+                }
+            }
+            i = next;
         }
-        const separator = this.separators[separatorIndex];
-        if (separator === '') {
-            return this.charSplit(text);
+        return chunks;
+    }
+    splitSpan(text, span, separatorIndex) {
+        const trimmed = trimSpan(text, span);
+        if (!trimmed)
+            return [];
+        if (trimmed.end - trimmed.start <= this.chunkSize)
+            return [trimmed];
+        let index = separatorIndex;
+        while (index < this.separators.length) {
+            const separator = this.separators[index];
+            if (separator === '')
+                break;
+            if (text.slice(trimmed.start, trimmed.end).includes(separator))
+                break;
+            index++;
         }
-        const parts = text.split(separator);
-        if (parts.length === 1) {
-            return this.splitText(text, separatorIndex + 1);
+        const separator = this.separators[index];
+        if (separator === undefined || separator === '') {
+            return this.charSplit(trimmed);
+        }
+        const kept = /^\S*/.exec(separator)[0].length;
+        const segments = [];
+        let cursor = trimmed.start;
+        for (;;) {
+            const at = text.indexOf(separator, cursor);
+            if (at === -1 || at >= trimmed.end) {
+                segments.push({ start: cursor, end: trimmed.end });
+                break;
+            }
+            segments.push({ start: cursor, end: Math.min(at + kept, trimmed.end) });
+            cursor = at + separator.length;
         }
         const merged = [];
-        let current = '';
-        for (const part of parts) {
-            if (part.length === 0)
+        let current = null;
+        for (const raw of segments) {
+            const segment = trimSpan(text, raw);
+            if (!segment)
                 continue;
-            const candidate = current ? current + separator + part : part;
-            if (candidate.length <= this.chunkSize) {
-                current = candidate;
+            if (current && segment.end - current.start <= this.chunkSize) {
+                current = { start: current.start, end: segment.end };
+                continue;
+            }
+            if (current)
+                merged.push(current);
+            if (segment.end - segment.start > this.chunkSize) {
+                merged.push(...this.splitSpan(text, segment, index + 1));
+                current = null;
             }
             else {
-                if (current)
-                    merged.push(current);
-                if (part.length > this.chunkSize) {
-                    const subParts = this.splitText(part, separatorIndex + 1);
-                    merged.push(...subParts);
-                    current = '';
-                }
-                else {
-                    current = part;
-                }
+                current = segment;
             }
         }
         if (current)
             merged.push(current);
         return merged;
     }
-    charSplit(text) {
+    charSplit(span) {
         const result = [];
-        for (let i = 0; i < text.length; i += this.chunkSize) {
-            result.push(text.slice(i, i + this.chunkSize));
+        for (let start = span.start; start < span.end; start += this.chunkSize) {
+            result.push({ start, end: Math.min(start + this.chunkSize, span.end) });
         }
         return result;
     }
-    mergeWithOverlap(texts, documentId) {
-        if (this.chunkOverlap === 0) {
-            let offset = 0;
-            return texts.map((content, i) => {
-                const chunk = {
-                    id: nanoid(),
-                    documentId,
-                    content,
-                    startOffset: offset,
-                    endOffset: offset + content.length,
-                    order: i,
-                };
-                offset += content.length;
-                return chunk;
-            });
-        }
-        const joined = texts.join('');
-        const chunks = [];
-        let order = 0;
-        let textIndex = 0;
-        let i = 0;
-        while (i < texts.length) {
-            let content = texts[i];
-            let consumed = 1;
-            while (i + consumed < texts.length) {
-                const next = content + texts[i + consumed];
-                if (next.length <= this.chunkSize) {
-                    content = next;
-                    consumed++;
-                }
-                else {
-                    break;
-                }
-            }
-            const startOffset = joined.indexOf(content, textIndex > 0 ? Math.max(0, textIndex - this.chunkOverlap) : 0);
-            chunks.push({
-                id: nanoid(),
-                documentId,
-                content,
-                startOffset: startOffset >= 0 ? startOffset : textIndex,
-                endOffset: (startOffset >= 0 ? startOffset : textIndex) + content.length,
-                order: order++,
-            });
-            textIndex += content.length;
-            const overlapChars = Math.min(this.chunkOverlap, content.length);
-            const overlapText = content.slice(content.length - overlapChars);
-            i += consumed;
-            if (i < texts.length) {
-                const remaining = texts.slice(i);
-                const withOverlap = overlapText + remaining.join('');
-                if (withOverlap.length <= this.chunkSize) {
-                    chunks.push({
-                        id: nanoid(),
-                        documentId,
-                        content: withOverlap,
-                        startOffset: textIndex - overlapChars,
-                        endOffset: textIndex - overlapChars + withOverlap.length,
-                        order: order++,
-                    });
-                    break;
-                }
-                const reSplit = this.splitText(withOverlap, 0);
-                const subChunks = this.mergeWithOverlapSimple(reSplit, documentId, textIndex - overlapChars, order);
-                chunks.push(...subChunks);
-                break;
-            }
-        }
-        return chunks;
-    }
-    mergeWithOverlapSimple(texts, documentId, baseOffset, startOrder) {
-        const chunks = [];
-        let offset = baseOffset;
-        let order = startOrder;
-        for (const content of texts) {
-            chunks.push({
-                id: nanoid(),
-                documentId,
-                content,
-                startOffset: offset,
-                endOffset: offset + content.length,
-                order: order++,
-            });
-            offset += content.length;
-        }
-        return chunks;
-    }
+}
+function trimSpan(text, span) {
+    let { start, end } = span;
+    while (start < end && /\s/.test(text[start]))
+        start++;
+    while (end > start && /\s/.test(text[end - 1]))
+        end--;
+    return start < end ? { start, end } : null;
 }
 //# sourceMappingURL=recursive-chunker.js.map

package/dist/chunkers/recursive-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,cAAsB;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAElD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9D,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACzB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,KAAe,EAAE,UAAkB;QAC1D,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;gBAC9B,MAAM,KAAK,GAAkB;oBAC3B,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,MAAM;oBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;oBAClC,KAAK,EAAE,CAAC;iBACT,CAAC;gBACF,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,OAAO,CAAC,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC3C,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClC,OAAO,GAAG,IAAI,CAAC;oBACf,QAAQ,EAAE,CAAC;gBACb,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAChC,OAAO,EACP,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;YACF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;gBACvD,SAAS,EAAE,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM;gBACxE,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;YAE5B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;YAEjE,CAAC,IAAI,QAAQ,CAAC;YAEd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAErD,IAAI,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,WAAW;wBACpB,WAAW,EAAE,SAAS,GAAG,YAAY;wBACrC,SAAS,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,CAAC,MAAM;wBACxD,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;oBACH,MAAM;gBACR,CAAC;gBAED,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAC3C,OAAO,EACP,UAAU,EACV,SAAS,GAAG,YAAY,EACxB,KAAK,CACN,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,sBAAsB,CAC5B,KAAe,EACf,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,UAAU,CAAC;QACxB,IAAI,KAAK,GAAG,UAAU,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,MAAM;gBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;gBAClC,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;QAC3B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,UAAkB;QAClE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC;YAEpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACzC,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChE,IAAI,EAAE,CAAC;gBACT,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzE,IAAI,EAAE,CAAC;gBACT,CAAC;YACH,CAAC;YACD,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,IAAU,EAAE,cAAsB;QAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,IAAI,KAAK,GAAG,cAAc,CAAC;QAC3B,OAAO,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAE,CAAC;YAC1C,IAAI,SAAS,KAAK,EAAE;gBAAE,MAAM;YAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,MAAM;YACtE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAW,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnD,MAAM;YACR,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,MAAM,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,IAAI,OAAO,GAAgB,IAAI,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7D,OAAO,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzD,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,OAAO,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAU;QAC1B,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAOD,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAU;IACxC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;QAAE,KAAK,EAAE,CAAC;IACvD,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IACvD,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}

package/dist/chunkers/semantic-chunker.d.ts

@@ -13,7 +13,9 @@ export declare class SemanticChunker implements AsyncChunker {
     constructor(options: SemanticChunkerOptions);
     chunk(text: string, documentId: string): Promise<DocumentChunk[]>;
     private mergeSmallGroups;
+    private findSentenceOffset;
     private buildChunks;
     private splitLargeGroup;
+    private emitSentenceGroup;
 }
 //# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunkers/semantic-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAyBD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAOrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,eAAe;CA8DxB"}
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8BD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAqBrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAuCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IA2EvB,OAAO,CAAC,iBAAiB;CAqB1B"}