@robthepcguy/rag-vault 1.0.0

package/dist/chunker/index.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SemanticChunker = void 0;
+var semantic_chunker_js_1 = require("./semantic-chunker.js");
+Object.defineProperty(exports, "SemanticChunker", { enumerable: true, get: function () { return semantic_chunker_js_1.SemanticChunker; } });
+//# sourceMappingURL=index.js.map

package/dist/chunker/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunker/index.ts"],"names":[],"mappings":";;;AAUA,6DAAuD;AAA9C,sHAAA,eAAe,OAAA"}

package/dist/chunker/semantic-chunker.d.ts

@@ -0,0 +1,96 @@
+import type { TextChunk } from './index.js';
+/**
+ * Semantic Chunker configuration
+ * Based on paper recommendations: hardThreshold=0.6, initConst=1.5, c=0.9
+ */
+export interface SemanticChunkerConfig {
+    /** Hard threshold for minimum similarity (default: 0.6) */
+    hardThreshold: number;
+    /** Initial constant for first sentence pair (default: 1.5) */
+    initConst: number;
+    /** Scaling constant for threshold calculation (default: 0.9) */
+    c: number;
+    /** Minimum chunk length in characters (default: 50) */
+    minChunkLength: number;
+}
+/**
+ * Embedder interface for generating embeddings
+ */
+export interface EmbedderInterface {
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+/**
+ * Check if a chunk is garbage (should be filtered out)
+ *
+ * Criteria (language-agnostic):
+ * 1. Empty after trimming
+ * 2. Contains alphanumeric -> valid content (keep)
+ * 3. Only decoration characters (----, ====, etc.) -> garbage
+ * 4. Single character repeated >80% of text -> garbage
+ *
+ * Note: Applied after minChunkLength filter
+ *
+ * @param text - Chunk text to check
+ * @returns true if chunk is garbage and should be removed
+ */
+export declare function isGarbageChunk(text: string): boolean;
+export declare const DEFAULT_SEMANTIC_CHUNKER_CONFIG: SemanticChunkerConfig;
+/**
+ * Semantic chunker using Max-Min algorithm
+ *
+ * The algorithm groups consecutive sentences based on semantic similarity:
+ * 1. Split text into sentences
+ * 2. Generate embeddings for all sentences
+ * 3. For each sentence, decide whether to add to current chunk or start new chunk
+ * 4. Decision is based on comparing max similarity with new sentence vs min similarity within chunk
+ *
+ * Key insight: A sentence belongs to a chunk if its maximum similarity to any chunk member
+ * is greater than the minimum similarity between existing chunk members (with threshold adjustment)
+ */
+export declare class SemanticChunker {
+    private readonly config;
+    constructor(config?: Partial<SemanticChunkerConfig>);
+    /**
+     * Split text into semantically coherent chunks
+     *
+     * @param text - The text to chunk
+     * @param embedder - Embedder to generate sentence embeddings
+     * @returns Array of text chunks
+     */
+    chunkText(text: string, embedder: EmbedderInterface): Promise<TextChunk[]>;
+    /**
+     * Group sentences into chunks using Max-Min algorithm
+     */
+    private groupSentences;
+    /**
+     * Decide if a sentence should be added to the current chunk
+     * Based on Max-Min algorithm from the paper
+     */
+    private shouldAddToChunk;
+    /**
+     * Get minimum pairwise similarity within a chunk.
+     * Only compares the last WINDOW_SIZE sentences for O(1) complexity.
+     * This approximation is valid because recent sentences are most relevant
+     * for determining chunk coherence (per Max-Min paper's experimental setup).
+     */
+    private getMinSimilarity;
+    /**
+     * Get maximum similarity between a sentence and any sentence in the chunk
+     */
+    private getMaxSimilarity;
+    /**
+     * Calculate dynamic threshold based on chunk size
+     * threshold = max(c * minSim * sigmoid(|C|), hardThreshold)
+     */
+    private calculateThreshold;
+    /**
+     * Sigmoid function
+     */
+    private sigmoid;
+    /**
+     * Calculate cosine similarity between two vectors
+     * Public for testing - delegates to shared utility
+     */
+    cosineSimilarity(vec1: number[], vec2: number[]): number;
+}
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunker/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunker/semantic-chunker.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAQ3C;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,2DAA2D;IAC3D,aAAa,EAAE,MAAM,CAAA;IACrB,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAA;IACjB,gEAAgE;IAChE,CAAC,EAAE,MAAM,CAAA;IACT,uDAAuD;IACvD,cAAc,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;CACjD;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAmBpD;AAMD,eAAO,MAAM,+BAA+B,EAAE,qBAK7C,CAAA;AAMD;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;gBAElC,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM;IAIvD;;;;;;OAMG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAsChF;;OAEG;IACH,OAAO,CAAC,cAAc;IAmEtB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAuBxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;OAEG;IACH,OAAO,CAAC,OAAO;IAIf;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM;CAGzD"}

package/dist/chunker/semantic-chunker.js

@@ -0,0 +1,267 @@
+"use strict";
+// Semantic Chunker implementation using Max-Min algorithm
+// Based on: "Max–Min semantic chunking of documents for RAG application" (Springer, 2025)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SemanticChunker = exports.DEFAULT_SEMANTIC_CHUNKER_CONFIG = void 0;
+exports.isGarbageChunk = isGarbageChunk;
+const sentence_splitter_js_1 = require("./sentence-splitter.js");
+const math_js_1 = require("../utils/math.js");
+// ============================================
+// Performance Optimization Constants
+// ============================================
+/**
+ * Number of recent sentences to compare in getMinSimilarity.
+ * Based on Max-Min paper's experimental conditions (median 5 sentences per chunk).
+ * Reduces complexity from O(k²) to O(WINDOW_SIZE²) = O(25) = O(1).
+ */
+const WINDOW_SIZE = 5;
+/**
+ * Maximum number of sentences per chunk before forced split.
+ * Safety limit to prevent computational explosion on homogeneous documents.
+ * Set to 3x the paper's median chunk size for reasonable margin.
+ */
+const MAX_SENTENCES = 15;
+/**
+ * Check if a chunk is garbage (should be filtered out)
+ *
+ * Criteria (language-agnostic):
+ * 1. Empty after trimming
+ * 2. Contains alphanumeric -> valid content (keep)
+ * 3. Only decoration characters (----, ====, etc.) -> garbage
+ * 4. Single character repeated >80% of text -> garbage
+ *
+ * Note: Applied after minChunkLength filter
+ *
+ * @param text - Chunk text to check
+ * @returns true if chunk is garbage and should be removed
+ */
+function isGarbageChunk(text) {
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return true;
+    // If contains any alphanumeric, consider valid content
+    if (/[a-zA-Z0-9]/.test(trimmed))
+        return false;
+    // Decoration line patterns only (----, ====, ****, etc.)
+    if (/^[-=_.*#|~`@!%^&*()[\]{}\\/<>:+\s]+$/.test(trimmed))
+        return true;
+    // Excessive repetition of single character (>80%)
+    const charCounts = new Map();
+    for (const char of trimmed) {
+        charCounts.set(char, (charCounts.get(char) ?? 0) + 1);
+    }
+    const maxCount = Math.max(...charCounts.values());
+    if (maxCount / trimmed.length > 0.8)
+        return true;
+    return false;
+}
+// ============================================
+// Default Configuration
+// ============================================
+exports.DEFAULT_SEMANTIC_CHUNKER_CONFIG = {
+    hardThreshold: 0.6,
+    initConst: 1.5,
+    c: 0.9,
+    minChunkLength: 50,
+};
+// ============================================
+// SemanticChunker Class
+// ============================================
+/**
+ * Semantic chunker using Max-Min algorithm
+ *
+ * The algorithm groups consecutive sentences based on semantic similarity:
+ * 1. Split text into sentences
+ * 2. Generate embeddings for all sentences
+ * 3. For each sentence, decide whether to add to current chunk or start new chunk
+ * 4. Decision is based on comparing max similarity with new sentence vs min similarity within chunk
+ *
+ * Key insight: A sentence belongs to a chunk if its maximum similarity to any chunk member
+ * is greater than the minimum similarity between existing chunk members (with threshold adjustment)
+ */
+class SemanticChunker {
+    constructor(config = {}) {
+        this.config = { ...exports.DEFAULT_SEMANTIC_CHUNKER_CONFIG, ...config };
+    }
+    /**
+     * Split text into semantically coherent chunks
+     *
+     * @param text - The text to chunk
+     * @param embedder - Embedder to generate sentence embeddings
+     * @returns Array of text chunks
+     */
+    async chunkText(text, embedder) {
+        // Handle empty input
+        if (!text || text.trim().length === 0) {
+            return [];
+        }
+        // Split into sentences
+        const sentences = (0, sentence_splitter_js_1.splitIntoSentences)(text);
+        if (sentences.length === 0) {
+            return [];
+        }
+        // Generate embeddings for all sentences
+        const embeddings = await embedder.embedBatch(sentences);
+        // Apply Max-Min algorithm to group sentences into chunks
+        const sentenceGroups = this.groupSentences(sentences, embeddings);
+        // Convert groups to TextChunks
+        const chunks = [];
+        let chunkIndex = 0;
+        for (const group of sentenceGroups) {
+            const chunkText = group.join(' ');
+            // Filter out chunks that are too short or garbage
+            if (chunkText.length >= this.config.minChunkLength && !isGarbageChunk(chunkText)) {
+                chunks.push({
+                    text: chunkText,
+                    index: chunkIndex,
+                });
+                chunkIndex++;
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Group sentences into chunks using Max-Min algorithm
+     */
+    groupSentences(sentences, embeddings) {
+        if (sentences.length === 0)
+            return [];
+        if (sentences.length === 1)
+            return [[sentences[0] ?? '']];
+        const groups = [];
+        let currentGroup = [];
+        let currentGroupEmbeddings = [];
+        for (let i = 0; i < sentences.length; i++) {
+            const sentence = sentences[i];
+            const embedding = embeddings[i];
+            if (!sentence || !embedding)
+                continue;
+            if (currentGroup.length === 0) {
+                // Start new group with first sentence
+                currentGroup.push(sentence);
+                currentGroupEmbeddings.push(embedding);
+            }
+            else if (currentGroup.length === 1) {
+                // Special case for second sentence (init phase)
+                const firstEmbedding = currentGroupEmbeddings[0];
+                if (!firstEmbedding)
+                    continue;
+                const similarity = this.cosineSimilarity(firstEmbedding, embedding);
+                if (this.config.initConst * similarity > this.config.hardThreshold) {
+                    // Add to current group
+                    currentGroup.push(sentence);
+                    currentGroupEmbeddings.push(embedding);
+                }
+                else {
+                    // Start new group
+                    groups.push([...currentGroup]);
+                    currentGroup = [sentence];
+                    currentGroupEmbeddings = [embedding];
+                }
+            }
+            else {
+                // Force split if chunk reaches MAX_SENTENCES (safety limit for performance)
+                if (currentGroup.length >= MAX_SENTENCES) {
+                    groups.push([...currentGroup]);
+                    currentGroup = [sentence];
+                    currentGroupEmbeddings = [embedding];
+                    continue;
+                }
+                // Normal case: check if sentence should join current group
+                const shouldAdd = this.shouldAddToChunk(embedding, currentGroupEmbeddings);
+                if (shouldAdd) {
+                    currentGroup.push(sentence);
+                    currentGroupEmbeddings.push(embedding);
+                }
+                else {
+                    // Start new group
+                    groups.push([...currentGroup]);
+                    currentGroup = [sentence];
+                    currentGroupEmbeddings = [embedding];
+                }
+            }
+        }
+        // Don't forget the last group
+        if (currentGroup.length > 0) {
+            groups.push(currentGroup);
+        }
+        return groups;
+    }
+    /**
+     * Decide if a sentence should be added to the current chunk
+     * Based on Max-Min algorithm from the paper
+     */
+    shouldAddToChunk(newEmbedding, chunkEmbeddings) {
+        // Calculate min similarity within current chunk
+        const minSim = this.getMinSimilarity(chunkEmbeddings);
+        // Calculate max similarity between new sentence and chunk
+        const maxSim = this.getMaxSimilarity(newEmbedding, chunkEmbeddings);
+        // Calculate dynamic threshold
+        const threshold = this.calculateThreshold(minSim, chunkEmbeddings.length);
+        return maxSim > threshold;
+    }
+    /**
+     * Get minimum pairwise similarity within a chunk.
+     * Only compares the last WINDOW_SIZE sentences for O(1) complexity.
+     * This approximation is valid because recent sentences are most relevant
+     * for determining chunk coherence (per Max-Min paper's experimental setup).
+     */
+    getMinSimilarity(embeddings) {
+        if (embeddings.length < 2)
+            return 1.0;
+        // Only compare the last WINDOW_SIZE embeddings to reduce O(k²) to O(1)
+        const startIdx = Math.max(0, embeddings.length - WINDOW_SIZE);
+        const windowEmbeddings = embeddings.slice(startIdx);
+        let minSim = 1.0;
+        for (let i = 0; i < windowEmbeddings.length; i++) {
+            for (let j = i + 1; j < windowEmbeddings.length; j++) {
+                const embI = windowEmbeddings[i];
+                const embJ = windowEmbeddings[j];
+                if (!embI || !embJ)
+                    continue;
+                const sim = this.cosineSimilarity(embI, embJ);
+                if (sim < minSim) {
+                    minSim = sim;
+                }
+            }
+        }
+        return minSim;
+    }
+    /**
+     * Get maximum similarity between a sentence and any sentence in the chunk
+     */
+    getMaxSimilarity(embedding, chunkEmbeddings) {
+        let maxSim = -1.0;
+        for (const chunkEmb of chunkEmbeddings) {
+            const sim = this.cosineSimilarity(embedding, chunkEmb);
+            if (sim > maxSim) {
+                maxSim = sim;
+            }
+        }
+        return maxSim;
+    }
+    /**
+     * Calculate dynamic threshold based on chunk size
+     * threshold = max(c * minSim * sigmoid(|C|), hardThreshold)
+     */
+    calculateThreshold(minSim, chunkSize) {
+        const sigmoidValue = this.sigmoid(chunkSize);
+        const dynamicThreshold = this.config.c * minSim * sigmoidValue;
+        return Math.max(dynamicThreshold, this.config.hardThreshold);
+    }
+    /**
+     * Sigmoid function
+     */
+    sigmoid(x) {
+        return 1 / (1 + Math.exp(-x));
+    }
+    /**
+     * Calculate cosine similarity between two vectors
+     * Public for testing - delegates to shared utility
+     */
+    cosineSimilarity(vec1, vec2) {
+        return (0, math_js_1.cosineSimilarity)(vec1, vec2);
+    }
+}
+exports.SemanticChunker = SemanticChunker;
+//# sourceMappingURL=semantic-chunker.js.map

package/dist/chunker/semantic-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunker/semantic-chunker.ts"],"names":[],"mappings":";AAAA,0DAA0D;AAC1D,0FAA0F;;;AAgE1F,wCAmBC;AAhFD,iEAA2D;AAC3D,8CAA2E;AA4B3E,+CAA+C;AAC/C,qCAAqC;AACrC,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,WAAW,GAAG,CAAC,CAAA;AAErB;;;;GAIG;AACH,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB;;;;;;;;;;;;;GAaG;AACH,SAAgB,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,uDAAuD;IACvD,IAAI,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAA;IAE7C,yDAAyD;IACzD,IAAI,sCAAsC,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IAErE,kDAAkD;IAClD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC5C,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,IAAI,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAA;IAEhD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAElC,QAAA,+BAA+B,GAA0B;IACpE,aAAa,EAAE,GAAG;IAClB,SAAS,EAAE,GAAG;IACd,CAAC,EAAE,GAAG;IACN,cAAc,EAAE,EAAE;CACnB,CAAA;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAE/C;;;;;;;;;;;GAWG;AACH,MAAa,eAAe;IAG1B,YAAY,SAAyC,EAAE;QACrD,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,uCAA+B,EAAE,GAAG,MAAM,EAAE,CAAA;IACjE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,QAA2B;QACvD,qBAAqB;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAA;QACX,CAAC;QAED,uBAAuB;QACvB,MAAM,SAAS,GAAG,IAAA,yCAAkB,EAAC,IAAI,CAAC,CAAA;QAC1C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,CAAA;QACX,CAAC;QAED,wCAAwC;QACxC,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QAEvD,yDAAyD;QACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;QAEjE,+BAA+B;QAC/B,MAAM,MAAM,GAAgB,EAAE,CAAA;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEjC,kDAAkD;YAClD,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjF,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,UAAU;iBAClB,CAAC,CAAA;gBACF,UAAU,EAAE,CAAA;YACd,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,SAAmB,EAAE,UAAsB;QAChE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAA;QACrC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;QAEzD,MAAM,MAAM,GAAe,EAAE,CAAA;QAC7B,IAAI,YAAY,GAAa,EAAE,CAAA;QAC/B,IAAI,sBAAsB,GAAe,EAAE,CAAA;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YAC7B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS;gBAAE,SAAQ;YAErC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,sCAAsC;gBACtC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACxC,CAAC;iBAAM,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,gDAAgD;gBAChD,MAAM,cAAc,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAA;gBAChD,IAAI,CAAC,cAAc;oBAAE,SAAQ;gBAE7B,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAA;gBAEnE,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;oBACnE,uBAAuB;oBACvB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;oBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACxC,CAAC;qBAAM,CAAC;oBACN,kBAAkB;oBAClB,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;oBAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;oBACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;gBACtC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,4EAA4E;gBAC5E,IAAI,YAAY,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;oBAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;oBACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;oBACpC,SAAQ;gBACV,CAAC;gBAED,2DAA2D;gBAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAA;gBAE1E,IAAI,SAAS,EAAE,CAAC;oBACd,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;oBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACxC,CAAC;qBAAM,CAAC;oBACN,kBAAkB;oBAClB,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;oBAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;oBACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;gBACtC,CAAC;YACH,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAC3B,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,YAAsB,EAAE,eAA2B;QAC1E,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;QAErD,0DAA0D;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;QAEnE,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAA;QAEzE,OAAO,MAAM,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED;;;;;OAKG;IACK,gBAAgB,CAAC,UAAsB;QAC7C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,GAAG,CAAA;QAErC,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,WAAW,CAAC,CAAA;QAC7D,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAEnD,IAAI,MAAM,GAAG,GAAG,CAAA;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;gBAChC,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;gBAChC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI;oBAAE,SAAQ;gBAE5B,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC7C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,CAAA;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,SAAmB,EAAE,eAA2B;QACvE,IAAI,MAAM,GAAG,CAAC,GAAG,CAAA;QACjB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;YACtD,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,MAAc,EAAE,SAAiB;QAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QAC5C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,YAAY,CAAA;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACK,OAAO,CAAC,CAAS;QACvB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,IAAc,EAAE,IAAc;QAC7C,OAAO,IAAA,0BAAoB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IACzC,CAAC;CACF;AA9MD,0CA8MC"}

package/dist/chunker/sentence-splitter.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Split text into sentences using Intl.Segmenter
+ *
+ * Uses the Unicode Text Segmentation standard (UAX #29) via Intl.Segmenter.
+ * This provides multilingual support for sentence boundary detection.
+ *
+ * Note: Intl.Segmenter may split on abbreviations like "Mr." or "e.g."
+ * These edge cases are acceptable for semantic chunking as:
+ * 1. Short fragments will be grouped with adjacent sentences by similarity
+ * 2. Fragments below minChunkLength are filtered out
+ *
+ * @param text - The text to split into sentences
+ * @returns Array of sentences
+ */
+export declare function splitIntoSentences(text: string): string[];
+//# sourceMappingURL=sentence-splitter.d.ts.map

package/dist/chunker/sentence-splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-splitter.d.ts","sourceRoot":"","sources":["../../src/chunker/sentence-splitter.ts"],"names":[],"mappings":"AAoFA;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAwCzD"}

package/dist/chunker/sentence-splitter.js

@@ -0,0 +1,114 @@
+"use strict";
+// Sentence Splitter for Semantic Chunking
+// Created: 2025-12-27
+// Purpose: Split text into sentences using Intl.Segmenter (Unicode standard)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitIntoSentences = splitIntoSentences;
+// ============================================
+// Constants
+// ============================================
+/**
+ * Placeholder for code blocks during processing
+ */
+const CODE_BLOCK_PLACEHOLDER = '\u0000CODE_BLOCK\u0000';
+/**
+ * Placeholder for inline code during processing
+ */
+const INLINE_CODE_PLACEHOLDER = '\u0000INLINE_CODE\u0000';
+// ============================================
+// Helper Functions
+// ============================================
+/**
+ * Extract and replace code blocks with placeholders
+ */
+function extractCodeBlocks(text) {
+    const blocks = [];
+    let processedText = text;
+    // Extract fenced code blocks (```...```)
+    const codeBlockRegex = /```[\s\S]*?```/g;
+    let index = 0;
+    const codeBlockMatches = text.matchAll(codeBlockRegex);
+    for (const match of codeBlockMatches) {
+        const placeholder = `${CODE_BLOCK_PLACEHOLDER}${index}${CODE_BLOCK_PLACEHOLDER}`;
+        blocks.push({ placeholder, content: match[0] });
+        processedText = processedText.replace(match[0], placeholder);
+        index++;
+    }
+    // Extract inline code (`...`)
+    const inlineCodeRegex = /`[^`]+`/g;
+    const inlineMatches = processedText.matchAll(inlineCodeRegex);
+    for (const match of inlineMatches) {
+        const placeholder = `${INLINE_CODE_PLACEHOLDER}${index}${INLINE_CODE_PLACEHOLDER}`;
+        blocks.push({ placeholder, content: match[0] });
+        processedText = processedText.replace(match[0], placeholder);
+        index++;
+    }
+    return { text: processedText, blocks };
+}
+/**
+ * Restore code blocks from placeholders
+ */
+function restoreCodeBlocks(sentences, blocks) {
+    return sentences.map((sentence) => {
+        let restored = sentence;
+        for (const block of blocks) {
+            restored = restored.replace(block.placeholder, block.content);
+        }
+        return restored;
+    });
+}
+// ============================================
+// Intl.Segmenter-based splitting
+// ============================================
+// Create segmenters for supported languages
+// Using 'und' (undetermined) as fallback for general Unicode support
+const segmenter = new Intl.Segmenter('und', { granularity: 'sentence' });
+/**
+ * Split text into sentences using Intl.Segmenter
+ *
+ * Uses the Unicode Text Segmentation standard (UAX #29) via Intl.Segmenter.
+ * This provides multilingual support for sentence boundary detection.
+ *
+ * Note: Intl.Segmenter may split on abbreviations like "Mr." or "e.g."
+ * These edge cases are acceptable for semantic chunking as:
+ * 1. Short fragments will be grouped with adjacent sentences by similarity
+ * 2. Fragments below minChunkLength are filtered out
+ *
+ * @param text - The text to split into sentences
+ * @returns Array of sentences
+ */
+function splitIntoSentences(text) {
+    // Handle empty input
+    if (!text || text.trim().length === 0) {
+        return [];
+    }
+    // Extract code blocks to protect them from splitting
+    const { text: processedText, blocks } = extractCodeBlocks(text);
+    // Split on paragraph boundaries first
+    // biome-ignore lint/suspicious/noControlCharactersInRegex: Intentional use of NULL character as placeholder delimiter
+    const paragraphs = processedText.split(/\n{2,}|\n(?=\S)|(?<=\u0000)\n/);
+    const sentences = [];
+    for (const paragraph of paragraphs) {
+        const trimmedParagraph = paragraph.trim();
+        if (!trimmedParagraph)
+            continue;
+        // Check if it's a markdown heading (treat as single sentence)
+        if (/^#{1,6}\s/.test(trimmedParagraph)) {
+            sentences.push(trimmedParagraph);
+            continue;
+        }
+        // Use Intl.Segmenter for sentence splitting
+        const segments = segmenter.segment(trimmedParagraph);
+        for (const segment of segments) {
+            const trimmed = segment.segment.trim();
+            if (trimmed) {
+                sentences.push(trimmed);
+            }
+        }
+    }
+    // Restore code blocks
+    const restoredSentences = restoreCodeBlocks(sentences, blocks);
+    // Filter empty sentences and trim
+    return restoredSentences.map((s) => s.trim()).filter((s) => s.length > 0);
+}
+//# sourceMappingURL=sentence-splitter.js.map

package/dist/chunker/sentence-splitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-splitter.js","sourceRoot":"","sources":["../../src/chunker/sentence-splitter.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,sBAAsB;AACtB,6EAA6E;;AAgG7E,gDAwCC;AAtID,+CAA+C;AAC/C,YAAY;AACZ,+CAA+C;AAE/C;;GAEG;AACH,MAAM,sBAAsB,GAAG,wBAAwB,CAAA;AAEvD;;GAEG;AACH,MAAM,uBAAuB,GAAG,yBAAyB,CAAA;AAWzD,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,MAAM,GAAoB,EAAE,CAAA;IAClC,IAAI,aAAa,GAAG,IAAI,CAAA;IAExB,yCAAyC;IACzC,MAAM,cAAc,GAAG,iBAAiB,CAAA;IACxC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAA;IACtD,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,GAAG,sBAAsB,GAAG,KAAK,GAAG,sBAAsB,EAAE,CAAA;QAChF,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAC/C,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QAC5D,KAAK,EAAE,CAAA;IACT,CAAC;IAED,8BAA8B;IAC9B,MAAM,eAAe,GAAG,UAAU,CAAA;IAClC,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;IAC7D,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,GAAG,uBAAuB,GAAG,KAAK,GAAG,uBAAuB,EAAE,CAAA;QAClF,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAC/C,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QAC5D,KAAK,EAAE,CAAA;IACT,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAA;AACxC,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAmB,EAAE,MAAuB;IACrE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAChC,IAAI,QAAQ,GAAG,QAAQ,CAAA;QACvB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;QAC/D,CAAC;QACD,OAAO,QAAQ,CAAA;IACjB,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,+CAA+C;AAC/C,iCAAiC;AACjC,+CAA+C;AAE/C,4CAA4C;AAC5C,qEAAqE;AACrE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAA;AAExE;;;;;;;;;;;;;GAaG;AACH,SAAgB,kBAAkB,CAAC,IAAY;IAC7C,qBAAqB;IACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IAED,qDAAqD;IACrD,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAE/D,sCAAsC;IACtC,sHAAsH;IACtH,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAEvE,MAAM,SAAS,GAAa,EAAE,CAAA;IAE9B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAA;QACzC,IAAI,CAAC,gBAAgB;YAAE,SAAQ;QAE/B,8DAA8D;QAC9D,IAAI,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAChC,SAAQ;QACV,CAAC;QAED,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QACpD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;YACtC,IAAI,OAAO,EAAE,CAAC;gBACZ,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IAE9D,kCAAkC;IAClC,OAAO,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;AAC3E,CAAC"}

package/dist/embedder/index.d.ts

@@ -0,0 +1,55 @@
+export { EmbeddingError } from '../errors/index.js';
+/**
+ * Embedder configuration
+ */
+export interface EmbedderConfig {
+    /** HuggingFace model path */
+    modelPath: string;
+    /** Batch size */
+    batchSize: number;
+    /** Model cache directory */
+    cacheDir: string;
+}
+/**
+ * Embedding generation class using Transformers.js
+ *
+ * Responsibilities:
+ * - Generate embedding vectors (dimension depends on model)
+ * - Transformers.js wrapper
+ * - Batch processing (size 8)
+ */
+export declare class Embedder {
+    private model;
+    private initPromise;
+    private readonly config;
+    constructor(config: EmbedderConfig);
+    /**
+     * Get the model name/path
+     */
+    getModelName(): string;
+    /**
+     * Initialize Transformers.js model
+     */
+    initialize(): Promise<void>;
+    /**
+     * Ensure model is initialized (lazy initialization)
+     * This method is called automatically by embed() and embedBatch()
+     */
+    private ensureInitialized;
+    /**
+     * Convert single text to embedding vector
+     *
+     * @param text - Text
+     * @returns Embedding vector (dimension depends on model)
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Convert multiple texts to embedding vectors with batch processing
+     *
+     * @param texts - Array of texts
+     * @param signal - Optional AbortSignal for cancellation support
+     * @returns Array of embedding vectors (dimension depends on model)
+     */
+    embedBatch(texts: string[], signal?: AbortSignal): Promise<number[][]>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/embedder/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAMnD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;CACjB;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBjC;;;OAGG;YACW,iBAAiB;IA+B/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC5C;;;;;;OAMG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAmC7E"}