abb-opencode-local-rag 0.1.0

package/dist/chunker/semantic-chunker.d.ts

@@ -0,0 +1,97 @@
+import type { AtomicTextRange, TextChunk } from './index.js';
+/**
+ * Semantic Chunker configuration
+ * Based on paper recommendations: hardThreshold=0.6, initConst=1.5, c=0.9
+ */
+export interface SemanticChunkerConfig {
+    /** Hard threshold for minimum similarity (default: 0.6) */
+    hardThreshold: number;
+    /** Initial constant for first sentence pair (default: 1.5) */
+    initConst: number;
+    /** Scaling constant for threshold calculation (default: 0.9) */
+    c: number;
+    /** Minimum chunk length in characters (default: 50) */
+    minChunkLength: number;
+}
+/**
+ * Embedder interface for generating embeddings
+ */
+export interface EmbedderInterface {
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+/**
+ * Check if a chunk is garbage (should be filtered out)
+ *
+ * Criteria (language-agnostic):
+ * 1. Empty after trimming
+ * 2. Contains alphanumeric -> valid content (keep)
+ * 3. Only decoration characters (----, ====, etc.) -> garbage
+ * 4. Single character repeated >80% of text -> garbage
+ *
+ * Note: Applied after minChunkLength filter
+ *
+ * @param text - Chunk text to check
+ * @returns true if chunk is garbage and should be removed
+ */
+export declare function isGarbageChunk(text: string): boolean;
+/** Default minimum chunk length in characters */
+export declare const DEFAULT_MIN_CHUNK_LENGTH = 50;
+/**
+ * Semantic chunker using Max-Min algorithm
+ *
+ * The algorithm groups consecutive sentences based on semantic similarity:
+ * 1. Split text into sentences
+ * 2. Generate embeddings for all sentences
+ * 3. For each sentence, decide whether to add to current chunk or start new chunk
+ * 4. Decision is based on comparing max similarity with new sentence vs min similarity within chunk
+ *
+ * Key insight: A sentence belongs to a chunk if its maximum similarity to any chunk member
+ * is greater than the minimum similarity between existing chunk members (with threshold adjustment)
+ */
+export declare class SemanticChunker {
+    private readonly config;
+    constructor(config?: Partial<SemanticChunkerConfig>);
+    /**
+     * Split text into semantically coherent chunks
+     *
+     * @param text - The text to chunk
+     * @param embedder - Embedder to generate sentence embeddings
+     * @returns Array of text chunks
+     */
+    chunkText(text: string, embedder: EmbedderInterface, atomicRanges?: readonly AtomicTextRange[]): Promise<TextChunk[]>;
+    /**
+     * Group sentences into chunks using Max-Min algorithm
+     */
+    private groupSentences;
+    /**
+     * Decide if a sentence should be added to the current chunk
+     * Based on Max-Min algorithm from the paper
+     */
+    private shouldAddToChunk;
+    /**
+     * Get minimum pairwise similarity within a chunk.
+     * Only compares the last WINDOW_SIZE sentences for O(1) complexity.
+     * This approximation is valid because recent sentences are most relevant
+     * for determining chunk coherence (per Max-Min paper's experimental setup).
+     */
+    private getMinSimilarity;
+    /**
+     * Get maximum similarity between a sentence and any sentence in the chunk
+     */
+    private getMaxSimilarity;
+    /**
+     * Calculate dynamic threshold based on chunk size
+     * threshold = max(c * minSim * sigmoid(|C|), hardThreshold)
+     */
+    private calculateThreshold;
+    /**
+     * Sigmoid function
+     */
+    private sigmoid;
+    /**
+     * Calculate cosine similarity between two vectors
+     * Public for testing
+     */
+    cosineSimilarity(vec1: number[], vec2: number[]): number;
+}
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunker/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunker/semantic-chunker.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAO5D;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,2DAA2D;IAC3D,aAAa,EAAE,MAAM,CAAA;IACrB,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAA;IACjB,gEAAgE;IAChE,CAAC,EAAE,MAAM,CAAA;IACT,uDAAuD;IACvD,cAAc,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;CACjD;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAmBpD;AAMD,iDAAiD;AACjD,eAAO,MAAM,wBAAwB,KAAK,CAAA;AAa1C;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAE9C,YAAY,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,EAEtD;IAED;;;;;;OAMG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,YAAY,GAAE,SAAS,eAAe,EAAO,GAC5C,OAAO,CAAC,SAAS,EAAE,CAAC,CAgDtB;IAED;;OAEG;IACH,OAAO,CAAC,cAAc;IAsEtB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAuBxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;OAEG;IACH,OAAO,CAAC,OAAO;IAIf;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAqBvD;CACF"}

package/dist/chunker/semantic-chunker.js

@@ -0,0 +1,294 @@
+// Semantic Chunker implementation using Max-Min algorithm
+// Based on: "Max–Min semantic chunking of documents for RAG application" (Springer, 2025)
+import { splitIntoSentenceUnits } from './sentence-splitter.js';
+// ============================================
+// Performance Optimization Constants
+// ============================================
+/**
+ * Number of recent sentences to compare in getMinSimilarity.
+ * Based on Max-Min paper's experimental conditions (median 5 sentences per chunk).
+ * Reduces complexity from O(k²) to O(WINDOW_SIZE²) = O(25) = O(1).
+ */
+const WINDOW_SIZE = 5;
+/**
+ * Maximum number of sentences per chunk before forced split.
+ * Safety limit to prevent computational explosion on homogeneous documents.
+ * Set to 3x the paper's median chunk size for reasonable margin.
+ */
+const MAX_SENTENCES = 15;
+/**
+ * Check if a chunk is garbage (should be filtered out)
+ *
+ * Criteria (language-agnostic):
+ * 1. Empty after trimming
+ * 2. Contains alphanumeric -> valid content (keep)
+ * 3. Only decoration characters (----, ====, etc.) -> garbage
+ * 4. Single character repeated >80% of text -> garbage
+ *
+ * Note: Applied after minChunkLength filter
+ *
+ * @param text - Chunk text to check
+ * @returns true if chunk is garbage and should be removed
+ */
+export function isGarbageChunk(text) {
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return true;
+    // If contains any alphanumeric, consider valid content
+    if (/[a-zA-Z0-9]/.test(trimmed))
+        return false;
+    // Decoration line patterns only (----, ====, ****, etc.)
+    if (/^[-=_.*#|~`@!%^&*()[\]{}\\/<>:+\s]+$/.test(trimmed))
+        return true;
+    // Excessive repetition of single character (>80%)
+    const charCounts = new Map();
+    for (const char of trimmed) {
+        charCounts.set(char, (charCounts.get(char) ?? 0) + 1);
+    }
+    const maxCount = Math.max(...charCounts.values());
+    if (maxCount / trimmed.length > 0.8)
+        return true;
+    return false;
+}
+// ============================================
+// Default Configuration
+// ============================================
+/** Default minimum chunk length in characters */
+export const DEFAULT_MIN_CHUNK_LENGTH = 50;
+const DEFAULT_SEMANTIC_CHUNKER_CONFIG = {
+    hardThreshold: 0.6,
+    initConst: 1.5,
+    c: 0.9,
+    minChunkLength: DEFAULT_MIN_CHUNK_LENGTH,
+};
+// ============================================
+// SemanticChunker Class
+// ============================================
+/**
+ * Semantic chunker using Max-Min algorithm
+ *
+ * The algorithm groups consecutive sentences based on semantic similarity:
+ * 1. Split text into sentences
+ * 2. Generate embeddings for all sentences
+ * 3. For each sentence, decide whether to add to current chunk or start new chunk
+ * 4. Decision is based on comparing max similarity with new sentence vs min similarity within chunk
+ *
+ * Key insight: A sentence belongs to a chunk if its maximum similarity to any chunk member
+ * is greater than the minimum similarity between existing chunk members (with threshold adjustment)
+ */
+export class SemanticChunker {
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_SEMANTIC_CHUNKER_CONFIG, ...config };
+    }
+    /**
+     * Split text into semantically coherent chunks
+     *
+     * @param text - The text to chunk
+     * @param embedder - Embedder to generate sentence embeddings
+     * @returns Array of text chunks
+     */
+    async chunkText(text, embedder, atomicRanges = []) {
+        // Handle empty input
+        if (!text || text.trim().length === 0) {
+            // Supplied ranges are programmer contracts and must fail fast even when
+            // ordinary empty text would otherwise return before sentence splitting.
+            if (atomicRanges.length > 0)
+                splitIntoSentenceUnits(text, atomicRanges);
+            return [];
+        }
+        // Split into sentences
+        const sentenceUnits = splitIntoSentenceUnits(text, atomicRanges);
+        if (sentenceUnits.length === 0) {
+            return [];
+        }
+        // Generate embeddings for all sentences
+        const embeddings = await embedder.embedBatch(sentenceUnits.map((unit) => unit.text));
+        // Apply Max-Min algorithm to group sentences into chunks
+        const sentenceGroups = this.groupSentences(sentenceUnits, embeddings);
+        // Convert groups to TextChunks
+        const chunks = [];
+        let chunkIndex = 0;
+        for (const group of sentenceGroups) {
+            const chunkText = group.map((unit) => unit.text).join(' ');
+            const containsAtomicUnit = group.some((unit) => unit.atomic);
+            // Filter out chunks that are too short or garbage
+            if ((containsAtomicUnit || chunkText.length >= this.config.minChunkLength) &&
+                !isGarbageChunk(chunkText)) {
+                const firstUnit = group[0];
+                const lastUnit = group[group.length - 1];
+                if (!firstUnit || !lastUnit)
+                    continue;
+                chunks.push({
+                    text: chunkText,
+                    index: chunkIndex,
+                    sourceStart: firstUnit.sourceStart,
+                    sourceEnd: lastUnit.sourceEnd,
+                });
+                chunkIndex++;
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Group sentences into chunks using Max-Min algorithm
+     */
+    groupSentences(sentences, embeddings) {
+        if (sentences.length === 0)
+            return [];
+        if (sentences.length === 1) {
+            const sentence = sentences[0];
+            return sentence ? [[sentence]] : [];
+        }
+        const groups = [];
+        let currentGroup = [];
+        let currentGroupEmbeddings = [];
+        for (let i = 0; i < sentences.length; i++) {
+            const sentence = sentences[i];
+            const embedding = embeddings[i];
+            if (!sentence || !embedding)
+                continue;
+            if (currentGroup.length === 0) {
+                // Start new group with first sentence
+                currentGroup.push(sentence);
+                currentGroupEmbeddings.push(embedding);
+            }
+            else if (currentGroup.length === 1) {
+                // Special case for second sentence (init phase)
+                const firstEmbedding = currentGroupEmbeddings[0];
+                if (!firstEmbedding)
+                    continue;
+                const similarity = this.cosineSimilarity(firstEmbedding, embedding);
+                if (this.config.initConst * similarity > this.config.hardThreshold) {
+                    // Add to current group
+                    currentGroup.push(sentence);
+                    currentGroupEmbeddings.push(embedding);
+                }
+                else {
+                    // Start new group
+                    groups.push([...currentGroup]);
+                    currentGroup = [sentence];
+                    currentGroupEmbeddings = [embedding];
+                }
+            }
+            else {
+                // Force split if chunk reaches MAX_SENTENCES (safety limit for performance)
+                if (currentGroup.length >= MAX_SENTENCES) {
+                    groups.push([...currentGroup]);
+                    currentGroup = [sentence];
+                    currentGroupEmbeddings = [embedding];
+                    continue;
+                }
+                // Normal case: check if sentence should join current group
+                const shouldAdd = this.shouldAddToChunk(embedding, currentGroupEmbeddings);
+                if (shouldAdd) {
+                    currentGroup.push(sentence);
+                    currentGroupEmbeddings.push(embedding);
+                }
+                else {
+                    // Start new group
+                    groups.push([...currentGroup]);
+                    currentGroup = [sentence];
+                    currentGroupEmbeddings = [embedding];
+                }
+            }
+        }
+        // Don't forget the last group
+        if (currentGroup.length > 0) {
+            groups.push(currentGroup);
+        }
+        return groups;
+    }
+    /**
+     * Decide if a sentence should be added to the current chunk
+     * Based on Max-Min algorithm from the paper
+     */
+    shouldAddToChunk(newEmbedding, chunkEmbeddings) {
+        // Calculate min similarity within current chunk
+        const minSim = this.getMinSimilarity(chunkEmbeddings);
+        // Calculate max similarity between new sentence and chunk
+        const maxSim = this.getMaxSimilarity(newEmbedding, chunkEmbeddings);
+        // Calculate dynamic threshold
+        const threshold = this.calculateThreshold(minSim, chunkEmbeddings.length);
+        return maxSim > threshold;
+    }
+    /**
+     * Get minimum pairwise similarity within a chunk.
+     * Only compares the last WINDOW_SIZE sentences for O(1) complexity.
+     * This approximation is valid because recent sentences are most relevant
+     * for determining chunk coherence (per Max-Min paper's experimental setup).
+     */
+    getMinSimilarity(embeddings) {
+        if (embeddings.length < 2)
+            return 1.0;
+        // Only compare the last WINDOW_SIZE embeddings to reduce O(k²) to O(1)
+        const startIdx = Math.max(0, embeddings.length - WINDOW_SIZE);
+        const windowEmbeddings = embeddings.slice(startIdx);
+        let minSim = 1.0;
+        for (let i = 0; i < windowEmbeddings.length; i++) {
+            for (let j = i + 1; j < windowEmbeddings.length; j++) {
+                const embI = windowEmbeddings[i];
+                const embJ = windowEmbeddings[j];
+                if (!embI || !embJ)
+                    continue;
+                const sim = this.cosineSimilarity(embI, embJ);
+                if (sim < minSim) {
+                    minSim = sim;
+                }
+            }
+        }
+        return minSim;
+    }
+    /**
+     * Get maximum similarity between a sentence and any sentence in the chunk
+     */
+    getMaxSimilarity(embedding, chunkEmbeddings) {
+        let maxSim = -1.0;
+        for (const chunkEmb of chunkEmbeddings) {
+            const sim = this.cosineSimilarity(embedding, chunkEmb);
+            if (sim > maxSim) {
+                maxSim = sim;
+            }
+        }
+        return maxSim;
+    }
+    /**
+     * Calculate dynamic threshold based on chunk size
+     * threshold = max(c * minSim * sigmoid(|C|), hardThreshold)
+     */
+    calculateThreshold(minSim, chunkSize) {
+        const sigmoidValue = this.sigmoid(chunkSize);
+        const dynamicThreshold = this.config.c * minSim * sigmoidValue;
+        return Math.max(dynamicThreshold, this.config.hardThreshold);
+    }
+    /**
+     * Sigmoid function
+     */
+    sigmoid(x) {
+        return 1 / (1 + Math.exp(-x));
+    }
+    /**
+     * Calculate cosine similarity between two vectors
+     * Public for testing
+     */
+    cosineSimilarity(vec1, vec2) {
+        if (vec1.length !== vec2.length || vec1.length === 0) {
+            return 0;
+        }
+        let dotProduct = 0;
+        let norm1 = 0;
+        let norm2 = 0;
+        for (let i = 0; i < vec1.length; i++) {
+            const v1 = vec1[i] ?? 0;
+            const v2 = vec2[i] ?? 0;
+            dotProduct += v1 * v2;
+            norm1 += v1 * v1;
+            norm2 += v2 * v2;
+        }
+        const denominator = Math.sqrt(norm1) * Math.sqrt(norm2);
+        if (denominator === 0)
+            return 0;
+        return dotProduct / denominator;
+    }
+}
+//# sourceMappingURL=semantic-chunker.js.map

package/dist/chunker/semantic-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunker/semantic-chunker.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAC1D,0FAA0F;AAG1F,OAAO,EAAqB,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AA4BlF,+CAA+C;AAC/C,qCAAqC;AACrC,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,WAAW,GAAG,CAAC,CAAA;AAErB;;;;GAIG;AACH,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,uDAAuD;IACvD,IAAI,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAA;IAE7C,yDAAyD;IACzD,IAAI,sCAAsC,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IAErE,kDAAkD;IAClD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC5C,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,IAAI,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAA;IAEhD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAE/C,iDAAiD;AACjD,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAA;AAE1C,MAAM,+BAA+B,GAA0B;IAC7D,aAAa,EAAE,GAAG;IAClB,SAAS,EAAE,GAAG;IACd,CAAC,EAAE,GAAG;IACN,cAAc,EAAE,wBAAwB;CACzC,CAAA;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAE/C;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,eAAe;IACT,MAAM,CAAuB;IAE9C,YAAY,MAAM,GAAmC,EAAE;QACrD,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,+BAA+B,EAAE,GAAG,MAAM,EAAE,CAAA;IACjE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,QAA2B,EAC3B,YAAY,GAA+B,EAAE;QAE7C,qBAAqB;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,wEAAwE;YACxE,wEAAwE;YACxE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;gBAAE,sBAAsB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;YACvE,OAAO,EAAE,CAAA;QACX,CAAC;QAED,uBAAuB;QACvB,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAChE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,CAAA;QACX,CAAC;QAED,wCAAwC;QACxC,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAEpF,yDAAyD;QACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,UAAU,CAAC,CAAA;QAErE,+BAA+B;QAC/B,MAAM,MAAM,GAAgB,EAAE,CAAA;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAC1D,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAE5D,kDAAkD;YAClD,IACE,CAAC,kBAAkB,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBACtE,CAAC,cAAc,CAAC,SAAS,CAAC,EAC1B,CAAC;gBACD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;gBAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBACxC,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBACrC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,UAAU;oBACjB,WAAW,EAAE,SAAS,CAAC,WAAW;oBAClC,SAAS,EAAE,QAAQ,CAAC,SAAS;iBAC9B,CAAC,CAAA;gBACF,UAAU,EAAE,CAAA;YACd,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,SAAyB,EAAE,UAAsB;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAA;QACrC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YAC7B,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACrC,CAAC;QAED,MAAM,MAAM,GAAqB,EAAE,CAAA;QACnC,IAAI,YAAY,GAAmB,EAAE,CAAA;QACrC,IAAI,sBAAsB,GAAe,EAAE,CAAA;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YAC7B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS;gBAAE,SAAQ;YAErC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,sCAAsC;gBACtC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACxC,CAAC;iBAAM,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,gDAAgD;gBAChD,MAAM,cAAc,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAA;gBAChD,IAAI,CAAC,cAAc;oBAAE,SAAQ;gBAE7B,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAA;gBAEnE,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;oBACnE,uBAAuB;oBACvB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;oBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACxC,CAAC;qBAAM,CAAC;oBACN,kBAAkB;oBAClB,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;oBAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;oBACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;gBACtC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,4EAA4E;gBAC5E,IAAI,YAAY,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;oBAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;oBACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;oBACpC,SAAQ;gBACV,CAAC;gBAED,2DAA2D;gBAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAA;gBAE1E,IAAI,SAAS,EAAE,CAAC;oBACd,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;oBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACxC,CAAC;qBAAM,CAAC;oBACN,kBAAkB;oBAClB,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;oBAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;oBACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;gBACtC,CAAC;YACH,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAC3B,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,YAAsB,EAAE,eAA2B;QAC1E,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;QAErD,0DAA0D;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;QAEnE,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAA;QAEzE,OAAO,MAAM,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED;;;;;OAKG;IACK,gBAAgB,CAAC,UAAsB;QAC7C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,GAAG,CAAA;QAErC,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,WAAW,CAAC,CAAA;QAC7D,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAEnD,IAAI,MAAM,GAAG,GAAG,CAAA;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;gBAChC,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;gBAChC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI;oBAAE,SAAQ;gBAE5B,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC7C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,CAAA;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,SAAmB,EAAE,eAA2B;QACvE,IAAI,MAAM,GAAG,CAAC,GAAG,CAAA;QACjB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;YACtD,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,MAAc,EAAE,SAAiB;QAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QAC5C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,YAAY,CAAA;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACK,OAAO,CAAC,CAAS;QACvB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,IAAc,EAAE,IAAc;QAC7C,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,CAAA;QACV,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YACvB,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YACvB,UAAU,IAAI,EAAE,GAAG,EAAE,CAAA;YACrB,KAAK,IAAI,EAAE,GAAG,EAAE,CAAA;YAChB,KAAK,IAAI,EAAE,GAAG,EAAE,CAAA;QAClB,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACvD,IAAI,WAAW,KAAK,CAAC;YAAE,OAAO,CAAC,CAAA;QAE/B,OAAO,UAAU,GAAG,WAAW,CAAA;IACjC,CAAC;CACF"}

package/dist/chunker/sentence-splitter.d.ts

@@ -0,0 +1,28 @@
+import type { AtomicTextRange } from './index.js';
+export interface SentenceUnit {
+    text: string;
+    atomic: boolean;
+    sourceStart: number;
+    sourceEnd: number;
+}
+/**
+ * Split text into sentences using Intl.Segmenter
+ *
+ * Uses the Unicode Text Segmentation standard (UAX #29) via Intl.Segmenter.
+ * This provides multilingual support for sentence boundary detection.
+ *
+ * Note: Intl.Segmenter may split on abbreviations like "Mr." or "e.g."
+ * These edge cases are acceptable for semantic chunking as:
+ * 1. Short fragments will be grouped with adjacent sentences by similarity
+ * 2. Fragments below minChunkLength are filtered out
+ *
+ * @param text - The text to split into sentences
+ * @returns Array of sentences
+ */
+export declare function splitIntoSentences(text: string): string[];
+/**
+ * Split ordinary text while preserving the supplied positional ranges as
+ * indivisible sentence units.
+ */
+export declare function splitIntoSentenceUnits(text: string, atomicRanges?: readonly AtomicTextRange[]): SentenceUnit[];
+//# sourceMappingURL=sentence-splitter.d.ts.map

package/dist/chunker/sentence-splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-splitter.d.ts","sourceRoot":"","sources":["../../src/chunker/sentence-splitter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAyBjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;CAClB;AAoKD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAKzD;AAuBD;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,YAAY,GAAE,SAAS,eAAe,EAAO,GAC5C,YAAY,EAAE,CA0BhB"}

package/dist/chunker/sentence-splitter.js

@@ -0,0 +1,219 @@
+// Sentence Splitter for Semantic Chunking
+// Created: 2025-12-27
+// Purpose: Split text into sentences using Intl.Segmenter (Unicode standard)
+// ============================================
+// Constants
+// ============================================
+/**
+ * Placeholder for code blocks during processing
+ */
+const CODE_BLOCK_PLACEHOLDER = '\u0000CODE_BLOCK\u0000';
+/**
+ * Placeholder for inline code during processing
+ */
+const INLINE_CODE_PLACEHOLDER = '\u0000INLINE_CODE\u0000';
+// ============================================
+// Helper Functions
+// ============================================
+function maskCode(text, sourceStart) {
+    const blocks = [];
+    const fenced = [...text.matchAll(/```[\s\S]*?```/g)].flatMap((match) => match.index === undefined ? [] : [{ start: match.index, end: match.index + match[0].length }]);
+    const matchedRanges = [...fenced];
+    let gapStart = 0;
+    for (const fencedRange of [...fenced, { start: text.length, end: text.length }]) {
+        const gap = text.slice(gapStart, fencedRange.start);
+        for (const match of gap.matchAll(/`[^`]+`/g)) {
+            if (match.index === undefined)
+                continue;
+            matchedRanges.push({
+                start: gapStart + match.index,
+                end: gapStart + match.index + match[0].length,
+            });
+        }
+        gapStart = fencedRange.end;
+    }
+    matchedRanges.sort((left, right) => left.start - right.start);
+    let mappedText = '';
+    let cursor = 0;
+    const segments = [];
+    const append = (value, start, end, linear) => {
+        const mappedStart = mappedText.length;
+        mappedText += value;
+        segments.push({
+            mappedStart,
+            mappedEnd: mappedText.length,
+            sourceStart: sourceStart + start,
+            sourceEnd: sourceStart + end,
+            linear,
+        });
+    };
+    for (const [index, range] of matchedRanges.entries()) {
+        if (range.start > cursor)
+            append(text.slice(cursor, range.start), cursor, range.start, true);
+        const content = text.slice(range.start, range.end);
+        const placeholderPrefix = content.startsWith('```')
+            ? CODE_BLOCK_PLACEHOLDER
+            : INLINE_CODE_PLACEHOLDER;
+        const placeholder = `${placeholderPrefix}${index}${placeholderPrefix}`;
+        blocks.push({ placeholder, content });
+        append(placeholder, range.start, range.end, false);
+        cursor = range.end;
+    }
+    if (cursor < text.length)
+        append(text.slice(cursor), cursor, text.length, true);
+    return { mapped: { text: mappedText, segments }, blocks };
+}
+function sourceOffsetAt(mapped, offset, fallback) {
+    let low = 0;
+    let high = mapped.segments.length - 1;
+    while (low <= high) {
+        const middle = Math.floor((low + high) / 2);
+        const segment = mapped.segments[middle];
+        if (offset < segment.mappedStart) {
+            high = middle - 1;
+        }
+        else if (offset > segment.mappedEnd) {
+            low = middle + 1;
+        }
+        else if (offset === segment.mappedEnd) {
+            return segment.sourceEnd;
+        }
+        else {
+            return segment.linear
+                ? segment.sourceStart + (offset - segment.mappedStart)
+                : segment.sourceStart;
+        }
+    }
+    return fallback;
+}
+function restoreCode(text, blocks) {
+    let restored = text;
+    for (const block of blocks) {
+        restored = restored.replace(block.placeholder, () => block.content);
+    }
+    return restored;
+}
+function trimmedRange(text, start, end) {
+    const value = text.slice(start, end);
+    const trimmedStart = start + (value.length - value.trimStart().length);
+    const trimmedEnd = start + value.trimEnd().length;
+    return trimmedStart < trimmedEnd ? [trimmedStart, trimmedEnd] : null;
+}
+function splitOrdinaryRange(text, sourceStart, sourceEnd) {
+    if (sourceStart >= sourceEnd)
+        return [];
+    const { mapped, blocks } = maskCode(text.slice(sourceStart, sourceEnd), sourceStart);
+    // biome-ignore lint/suspicious/noControlCharactersInRegex: NUL delimiters identify masked code placeholders.
+    const paragraphSeparator = /\n{2,}|\n(?=\S)|(?<=\u0000)\n/g;
+    const paragraphRanges = [];
+    let cursor = 0;
+    for (const match of mapped.text.matchAll(paragraphSeparator)) {
+        const separatorStart = match.index;
+        if (separatorStart === undefined)
+            continue;
+        paragraphRanges.push([cursor, separatorStart]);
+        cursor = separatorStart + match[0].length;
+    }
+    paragraphRanges.push([cursor, mapped.text.length]);
+    const units = [];
+    const appendUnit = (start, end) => {
+        const range = trimmedRange(mapped.text, start, end);
+        if (!range)
+            return;
+        const [trimmedStart, trimmedEnd] = range;
+        const restored = restoreCode(mapped.text.slice(trimmedStart, trimmedEnd), blocks).trim();
+        if (!restored)
+            return;
+        units.push({
+            text: restored,
+            atomic: false,
+            sourceStart: sourceOffsetAt(mapped, trimmedStart, sourceStart),
+            sourceEnd: sourceOffsetAt(mapped, trimmedEnd, sourceEnd),
+        });
+    };
+    for (const [paragraphStart, paragraphEnd] of paragraphRanges) {
+        const range = trimmedRange(mapped.text, paragraphStart, paragraphEnd);
+        if (!range)
+            continue;
+        const [trimmedStart, trimmedEnd] = range;
+        const paragraph = mapped.text.slice(trimmedStart, trimmedEnd);
+        if (/^#{1,6}\s/.test(paragraph)) {
+            appendUnit(trimmedStart, trimmedEnd);
+            continue;
+        }
+        for (const segment of segmenter.segment(paragraph)) {
+            appendUnit(trimmedStart + segment.index, trimmedStart + segment.index + segment.segment.length);
+        }
+    }
+    return units;
+}
+// ============================================
+// Intl.Segmenter-based splitting
+// ============================================
+// Create segmenters for supported languages
+// Using 'und' (undetermined) as fallback for general Unicode support
+const segmenter = new Intl.Segmenter('und', { granularity: 'sentence' });
+/**
+ * Split text into sentences using Intl.Segmenter
+ *
+ * Uses the Unicode Text Segmentation standard (UAX #29) via Intl.Segmenter.
+ * This provides multilingual support for sentence boundary detection.
+ *
+ * Note: Intl.Segmenter may split on abbreviations like "Mr." or "e.g."
+ * These edge cases are acceptable for semantic chunking as:
+ * 1. Short fragments will be grouped with adjacent sentences by similarity
+ * 2. Fragments below minChunkLength are filtered out
+ *
+ * @param text - The text to split into sentences
+ * @returns Array of sentences
+ */
+export function splitIntoSentences(text) {
+    if (!text || text.trim().length === 0) {
+        return [];
+    }
+    return splitOrdinaryRange(text, 0, text.length).map((unit) => unit.text);
+}
+function validateAtomicRanges(text, atomicRanges) {
+    let previousEnd = 0;
+    for (const range of atomicRanges) {
+        const validOffsets = Number.isInteger(range.start) &&
+            Number.isInteger(range.end) &&
+            range.start >= 0 &&
+            range.start < range.end &&
+            range.end <= text.length;
+        const orderedAndNonOverlapping = range.start >= previousEnd;
+        if (!validOffsets || !orderedAndNonOverlapping) {
+            throw new Error(`Invalid atomic range [${range.start}, ${range.end}) for text length ${text.length}`);
+        }
+        previousEnd = range.end;
+    }
+}
+/**
+ * Split ordinary text while preserving the supplied positional ranges as
+ * indivisible sentence units.
+ */
+export function splitIntoSentenceUnits(text, atomicRanges = []) {
+    validateAtomicRanges(text, atomicRanges);
+    if (atomicRanges.length === 0) {
+        return splitOrdinaryRange(text, 0, text.length);
+    }
+    const units = [];
+    let cursor = 0;
+    for (const range of atomicRanges) {
+        units.push(...splitOrdinaryRange(text, cursor, range.start));
+        const atomicText = text.slice(range.start, range.end).trim();
+        if (!atomicText) {
+            throw new Error(`Invalid atomic range [${range.start}, ${range.end}): empty text`);
+        }
+        units.push({
+            text: atomicText,
+            atomic: true,
+            sourceStart: range.start,
+            sourceEnd: range.end,
+        });
+        cursor = range.end;
+    }
+    units.push(...splitOrdinaryRange(text, cursor, text.length));
+    return units;
+}
+//# sourceMappingURL=sentence-splitter.js.map