@ssmeo/mcp-local-rag 0.18.6

package/dist/bin/install-skills.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"install-skills.js","sourceRoot":"","sources":["../../src/bin/install-skills.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAExC,+CAA+C;AAC/C,YAAY;AACZ,+CAA+C;AAE/C,+DAA+D;AAC/D,0DAA0D;AAC1D,gEAAgE;AAChE,iEAAiE;AACjE,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACzD,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;AAE/E,kEAAkE;AAClE,oDAAoD;AACpD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAA;AAEzE,uBAAuB;AACvB,MAAM,OAAO,GAAG;IACd,qBAAqB,EAAE,gCAAgC;IACvD,oBAAoB,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC;IAC3E,eAAe,EAAE,+BAA+B;IAChD,cAAc,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,eAAe,CAAC;CACnD,CAAA;AAYV,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,OAAO,GAAY;QACvB,MAAM,EAAE,qBAAqB;QAC7B,IAAI,EAAE,KAAK;KACZ,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEnB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACP,OAAO,CAAC,IAAI,GAAG,IAAI,CAAA;gBACnB,MAAK;YAEP,KAAK,eAAe;gBAClB,0BAA0B;gBAC1B,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;oBAC/B,OAAO,CAAC,MAAM,GAAG,oBAAoB,CAAA;oBACrC,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACtB,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,MAAM,GAAG,qBAAqB,CAAA;gBACxC,CAAC;gBACD,MAAK;YAEP,KAAK,SAAS;gBACZ,uCAAuC;gBACvC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,WAAW,EAAE,CAAC;oBAChC,OAAO,CAAC,MAAM,GAAG,eAAe,CAAA;oBAChC,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACtB,CAAC;qBAAM,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,UAAU,EAAE,CAAC;oBACtC,OAAO,CAAC,MAAM,GAAG,cAAc,CAAA;oBAC/B,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACtB,CAAC;qBAAM,CAAC;oBACN,gDAAgD;oBAChD,OAAO,CAAC,MAAM,GAAG,cAAc,CAAA;gBACjC,CAAC;gBACD,MAAK;YAEP,KAAK,QAAQ,EAAE,CAAC;gBACd,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;oBACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAA;gBACzB,OAAO,CAAC,UAAU,GAAG,OAAO,CAAA;gBAC5B,CAAC,EAAE,CAAA,CAAC,gBAAgB;gBACpB,MAAK;YACP,CAAC;YAED;gBACE,IAAI,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAA;oBACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,+CAA+C;AAC/C,eAAe;AACf,+CAA+C;AAE/C,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgCb,CAAC,CAAA;AACF,CAAC;AAED,+CAA+C;AAC/C,eAAe;AACf,+CAA+C;AAE/C,SAAS,aAAa,CAAC,OAAgB;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;YACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IACrD,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,CAAC,kBAAkB,EAAE,mBAAmB,CAAU,CAAA;AAE7E;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,uBAAuB,EAAE,sBAAsB,CAAU,CAAA;AAErF,SAAS,OAAO,CAAC,UAAkB;IACjC,sBAAsB;IACtB,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,qCAAqC,aAAa,EAAE,CAAC,CAAA;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,0BAA0B;IAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACrC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACzC,OAAO,CAAC,GAAG,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IACvD,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEhD,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAA;QAC5C,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,OAAO,CAAC,CAAA;YACf,OAAO,CAAC,GAAG,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAA;IACrE,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAA;IAC/E,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,UAAU,GAAG,CAAC,IAAc;IAChC,6BAA6B;IAC7B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,SAAS,EAAE,CAAA;QACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAE/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,SAAS,EAAE,CAAA;QACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,CAAA;IAEzC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACxC,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,EAAE,CAAC,CAAA;IAClC,OAAO,CAAC,GAAG,EAAE,CAAA;IAEb,OAAO,CAAC,UAAU,CAAC,CAAA;IAEnB,OAAO,CAAC,GAAG,EAAE,CAAA;IACb,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAA;IACrC,OAAO,CAAC,GAAG,EAAE,CAAA;IACb,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAA;IACtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAA;IAC3C,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAA;IACvC,CAAC;AACH,CAAC"}

package/dist/chunker/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Indivisible UTF-16 text range in the source passed to semantic chunking.
+ */
+export interface AtomicTextRange {
+    /** Inclusive UTF-16 offset. */
+    start: number;
+    /** Exclusive UTF-16 offset. */
+    end: number;
+}
+/**
+ * Text chunk
+ */
+export interface TextChunk {
+    /** Chunk text */
+    text: string;
+    /** Chunk index (zero-based) */
+    index: number;
+    /** Inclusive UTF-16 offset in the source document. */
+    sourceStart: number;
+    /** Exclusive UTF-16 offset in the source document. */
+    sourceEnd: number;
+}
+export { DEFAULT_MIN_CHUNK_LENGTH, SemanticChunker } from './semantic-chunker.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/chunker/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunker/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,+BAA+B;IAC/B,GAAG,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAA;IACnB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,OAAO,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA"}

package/dist/chunker/index.js

@@ -0,0 +1,2 @@
+export { DEFAULT_MIN_CHUNK_LENGTH, SemanticChunker } from './semantic-chunker.js';
+//# sourceMappingURL=index.js.map

package/dist/chunker/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunker/index.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA"}

package/dist/chunker/semantic-chunker.d.ts

@@ -0,0 +1,96 @@
+import type { AtomicTextRange, TextChunk } from './index.js';
+/**
+ * Semantic Chunker configuration
+ * Based on paper recommendations: hardThreshold=0.6, initConst=1.5, c=0.9
+ */
+export interface SemanticChunkerConfig {
+    /** Hard threshold for minimum similarity (default: 0.6) */
+    hardThreshold: number;
+    /** Initial constant for first sentence pair (default: 1.5) */
+    initConst: number;
+    /** Scaling constant for threshold calculation (default: 0.9) */
+    c: number;
+    /** Minimum chunk length in characters (default: 50) */
+    minChunkLength: number;
+}
+/**
+ * Embedder interface for generating embeddings.
+ *
+ * `getTokenLimit` and `countTokens` are optional; without them the chunker
+ * skips token containment.
+ */
+export interface EmbedderInterface {
+    embedBatch(texts: string[]): Promise<number[][]>;
+    /** Resolved token cap, or `null` when no limit could be resolved. */
+    getTokenLimit?(): Promise<number | null>;
+    /** True, unclamped token lengths of each text. */
+    countTokens?(texts: string[]): Promise<number[]>;
+}
+/**
+ * Garbage chunks, language-agnostically: empty after trimming, only decoration
+ * characters (`----`, `====`), or one character repeated over 80% of the text.
+ * Anything alphanumeric is kept. Applied after the minChunkLength filter.
+ */
+export declare function isGarbageChunk(text: string): boolean;
+/** Default minimum chunk length in characters */
+export declare const DEFAULT_MIN_CHUNK_LENGTH = 50;
+/**
+ * Semantic chunker using the Max-Min algorithm: a sentence joins the current
+ * chunk when its maximum similarity to any member exceeds the minimum
+ * similarity between existing members, adjusted by the threshold.
+ */
+export declare class SemanticChunker {
+    private readonly config;
+    constructor(config?: Partial<SemanticChunkerConfig>);
+    /**
+     * Split text into semantically coherent chunks
+     */
+    chunkText(text: string, embedder: EmbedderInterface, atomicRanges?: readonly AtomicTextRange[]): Promise<TextChunk[]>;
+    /**
+     * Whether a sentence group becomes stored chunks. A group holding a
+     * containment piece is already admitted, judged before the split, so neither
+     * filter re-runs on a fragment of it. Otherwise garbage is rejected and the
+     * minimum length is waived for an atomic unit, which must stay whole.
+     */
+    private admitsGroup;
+    /**
+     * Group sentences into chunks using Max-Min algorithm
+     */
+    /**
+     * Whether a sentence continues the group being built: `initConst`-scaled
+     * similarity while the group holds one sentence, Max-Min beyond that, and
+     * never past `MAX_SENTENCES`.
+     */
+    private continuesGroup;
+    private groupSentences;
+    /**
+     * Decide if a sentence should be added to the current chunk
+     * Based on Max-Min algorithm from the paper
+     */
+    private shouldAddToChunk;
+    /**
+     * Minimum pairwise similarity within a chunk, over the last WINDOW_SIZE
+     * sentences only. The approximation follows the Max-Min paper: recent
+     * sentences are what determine coherence.
+     */
+    private getMinSimilarity;
+    /**
+     * Get maximum similarity between a sentence and any sentence in the chunk
+     */
+    private getMaxSimilarity;
+    /**
+     * Calculate dynamic threshold based on chunk size
+     * threshold = max(c * minSim * sigmoid(|C|), hardThreshold)
+     */
+    private calculateThreshold;
+    /**
+     * Sigmoid function
+     */
+    private sigmoid;
+    /**
+     * Calculate cosine similarity between two vectors
+     * Public for testing
+     */
+    cosineSimilarity(vec1: number[], vec2: number[]): number;
+}
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunker/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunker/semantic-chunker.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAY5D;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,2DAA2D;IAC3D,aAAa,EAAE,MAAM,CAAA;IACrB,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAA;IACjB,gEAAgE;IAChE,CAAC,EAAE,MAAM,CAAA;IACT,uDAAuD;IACvD,cAAc,EAAE,MAAM,CAAA;CACvB;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAChD,qEAAqE;IACrE,aAAa,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;IACxC,kDAAkD;IAClD,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CACjD;AAoBD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CA2BpD;AAMD,iDAAiD;AACjD,eAAO,MAAM,wBAAwB,KAAK,CAAA;AA6H1C;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAE9C,YAAY,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,EAEtD;IAED;;OAEG;IACG,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,YAAY,GAAE,SAAS,eAAe,EAAO,GAC5C,OAAO,CAAC,SAAS,EAAE,CAAC,CA8BtB;IAED;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAWnB;;OAEG;IACH;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,cAAc;IAgDtB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IA2BxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;OAEG;IACH,OAAO,CAAC,OAAO;IAIf;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAuBvD;CACF"}

package/dist/chunker/semantic-chunker.js

@@ -0,0 +1,362 @@
+// Semantic Chunker implementation using Max-Min algorithm
+// Based on: "Max–Min semantic chunking of documents for RAG application" (Springer, 2025)
+import { splitIntoSentenceUnits } from './sentence-splitter.js';
+import { splitUnitsIntoFittingRuns, splitUnitToFit, } from './token-containment.js';
+// ============================================
+// Performance Optimization Constants
+// ============================================
+/**
+ * Number of recent sentences to compare in getMinSimilarity.
+ * Based on Max-Min paper's experimental conditions (median 5 sentences per chunk).
+ * Reduces complexity from O(k²) to O(WINDOW_SIZE²) = O(25) = O(1).
+ */
+const WINDOW_SIZE = 5;
+/**
+ * Maximum number of sentences per chunk before forced split.
+ * Safety limit to prevent computational explosion on homogeneous documents.
+ * Set to 3x the paper's median chunk size for reasonable margin.
+ */
+const MAX_SENTENCES = 15;
+/**
+ * Garbage chunks, language-agnostically: empty after trimming, only decoration
+ * characters (`----`, `====`), or one character repeated over 80% of the text.
+ * Anything alphanumeric is kept. Applied after the minChunkLength filter.
+ */
+export function isGarbageChunk(text) {
+    const trimmed = text.trim();
+    if (trimmed.length === 0) {
+        return true;
+    }
+    // If contains any alphanumeric, consider valid content
+    if (/[a-zA-Z0-9]/.test(trimmed)) {
+        return false;
+    }
+    // Decoration line patterns only (----, ====, ****, etc.)
+    if (/^[-=_.*#|~`@!%^&*()[\]{}\\/<>:+\s]+$/.test(trimmed)) {
+        return true;
+    }
+    // Excessive repetition of single character (>80%)
+    const charCounts = new Map();
+    for (const char of trimmed) {
+        charCounts.set(char, (charCounts.get(char) ?? 0) + 1);
+    }
+    const maxCount = Math.max(...charCounts.values());
+    if (maxCount / trimmed.length > 0.8) {
+        return true;
+    }
+    return false;
+}
+// ============================================
+// Default Configuration
+// ============================================
+/** Default minimum chunk length in characters */
+export const DEFAULT_MIN_CHUNK_LENGTH = 50;
+// ============================================
+// Token Containment (stages A and C)
+// ============================================
+/** Joins a group's units into the chunk text, which is also what containment measures. */
+function joinUnits(units) {
+    return units.map((unit) => unit.text).join(' ');
+}
+/**
+ * The embedder's token budget, or `null` when the optional members are absent
+ * or no limit could be resolved. Containment is then skipped entirely.
+ */
+async function resolveContainmentBudget(embedder) {
+    const { getTokenLimit, countTokens } = embedder;
+    if (!getTokenLimit || !countTokens) {
+        return null;
+    }
+    const cap = await getTokenLimit.call(embedder);
+    if (cap === null) {
+        return null;
+    }
+    return { cap, countTokens: (texts) => countTokens.call(embedder, texts) };
+}
+/** One measurement pass over `texts`, rejecting a counter that drops inputs. */
+async function measureAll(texts, budget) {
+    const measured = await budget.countTokens(texts);
+    if (measured.length !== texts.length) {
+        throw new Error(`Token counter returned ${measured.length} measurements for ${texts.length} texts`);
+    }
+    return measured;
+}
+/**
+ * Reduce sentence units to pieces within the cap, before embedding.
+ *
+ * Admission is judged here, on the whole unit, and recorded on its pieces: a
+ * fragment can be noise by itself — a run of one character reads as pure
+ * repetition — so judging pieces would drop text the parent was admitted with.
+ */
+async function containUnits(units, budget) {
+    const measured = await measureAll(units.map((unit) => unit.text), budget);
+    const contained = [];
+    for (const [index, unit] of units.entries()) {
+        const unitTokens = measured[index] ?? 0;
+        if (unitTokens <= budget.cap) {
+            contained.push(unit);
+            continue;
+        }
+        const admitted = !isGarbageChunk(unit.text);
+        const pieces = await splitUnitToFit(unit, budget);
+        contained.push(...pieces.map((piece) => (admitted ? { ...piece, containmentSplit: true } : piece)));
+    }
+    return contained;
+}
+/**
+ * Reduce admitted groups to runs of whole units within the cap. A group that
+ * fits is returned untouched, so content inside the window keeps its boundaries.
+ */
+async function containGroups(groups, budget) {
+    const measured = await measureAll(groups.map(joinUnits), budget);
+    const contained = [];
+    for (const [index, group] of groups.entries()) {
+        if ((measured[index] ?? 0) <= budget.cap) {
+            contained.push(group);
+            continue;
+        }
+        contained.push(...(await splitUnitsIntoFittingRuns(group, { ...budget, joinUnits })));
+    }
+    return contained;
+}
+/**
+ * Build the stored chunks. Offsets come from the first and last unit, never from
+ * an index into the joined text: the join uses a single space while the source
+ * may hold newlines or repeated whitespace.
+ */
+function convertPiecesToChunks(pieces) {
+    const chunks = [];
+    for (const piece of pieces) {
+        const firstUnit = piece[0];
+        const lastUnit = piece[piece.length - 1];
+        if (!firstUnit || !lastUnit) {
+            continue;
+        }
+        chunks.push({
+            text: joinUnits(piece),
+            index: chunks.length,
+            sourceStart: firstUnit.sourceStart,
+            sourceEnd: lastUnit.sourceEnd,
+        });
+    }
+    return chunks;
+}
+const DEFAULT_SEMANTIC_CHUNKER_CONFIG = {
+    hardThreshold: 0.6,
+    initConst: 1.5,
+    c: 0.9,
+    minChunkLength: DEFAULT_MIN_CHUNK_LENGTH,
+};
+// ============================================
+// SemanticChunker Class
+// ============================================
+/**
+ * Semantic chunker using the Max-Min algorithm: a sentence joins the current
+ * chunk when its maximum similarity to any member exceeds the minimum
+ * similarity between existing members, adjusted by the threshold.
+ */
+export class SemanticChunker {
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_SEMANTIC_CHUNKER_CONFIG, ...config };
+    }
+    /**
+     * Split text into semantically coherent chunks
+     */
+    async chunkText(text, embedder, atomicRanges = []) {
+        // Handle empty input
+        if (!text || text.trim().length === 0) {
+            // Supplied ranges are programmer contracts and must fail fast even when
+            // ordinary empty text would otherwise return before sentence splitting.
+            if (atomicRanges.length > 0) {
+                splitIntoSentenceUnits(text, atomicRanges);
+            }
+            return [];
+        }
+        // Split into sentences
+        const sentenceUnits = splitIntoSentenceUnits(text, atomicRanges);
+        if (sentenceUnits.length === 0) {
+            return [];
+        }
+        const budget = await resolveContainmentBudget(embedder);
+        const units = budget ? await containUnits(sentenceUnits, budget) : sentenceUnits;
+        // Generate embeddings for all sentences
+        const embeddings = await embedder.embedBatch(units.map((unit) => unit.text));
+        // Apply Max-Min algorithm to group sentences into chunks
+        const sentenceGroups = this.groupSentences(units, embeddings);
+        const admitted = sentenceGroups.filter((group) => this.admitsGroup(group));
+        const pieces = budget ? await containGroups(admitted, budget) : admitted;
+        return convertPiecesToChunks(pieces);
+    }
+    /**
+     * Whether a sentence group becomes stored chunks. A group holding a
+     * containment piece is already admitted, judged before the split, so neither
+     * filter re-runs on a fragment of it. Otherwise garbage is rejected and the
+     * minimum length is waived for an atomic unit, which must stay whole.
+     */
+    admitsGroup(group) {
+        if (group.some((unit) => unit.containmentSplit)) {
+            return true;
+        }
+        const groupText = joinUnits(group);
+        if (isGarbageChunk(groupText)) {
+            return false;
+        }
+        return group.some((unit) => unit.atomic) || groupText.length >= this.config.minChunkLength;
+    }
+    /**
+     * Group sentences into chunks using Max-Min algorithm
+     */
+    /**
+     * Whether a sentence continues the group being built: `initConst`-scaled
+     * similarity while the group holds one sentence, Max-Min beyond that, and
+     * never past `MAX_SENTENCES`.
+     */
+    continuesGroup(embedding, groupEmbeddings) {
+        if (groupEmbeddings.length === 1) {
+            const firstEmbedding = groupEmbeddings[0];
+            if (!firstEmbedding) {
+                return false;
+            }
+            const similarity = this.cosineSimilarity(firstEmbedding, embedding);
+            return this.config.initConst * similarity > this.config.hardThreshold;
+        }
+        if (groupEmbeddings.length >= MAX_SENTENCES) {
+            return false;
+        }
+        return this.shouldAddToChunk(embedding, groupEmbeddings);
+    }
+    groupSentences(sentences, embeddings) {
+        if (sentences.length === 0) {
+            return [];
+        }
+        if (sentences.length === 1) {
+            const sentence = sentences[0];
+            return sentence ? [[sentence]] : [];
+        }
+        const groups = [];
+        let currentGroup = [];
+        let currentGroupEmbeddings = [];
+        for (let i = 0; i < sentences.length; i++) {
+            const sentence = sentences[i];
+            const embedding = embeddings[i];
+            if (!sentence || !embedding) {
+                continue;
+            }
+            if (currentGroup.length === 0) {
+                // Start new group with first sentence
+                currentGroup.push(sentence);
+                currentGroupEmbeddings.push(embedding);
+                continue;
+            }
+            if (this.continuesGroup(embedding, currentGroupEmbeddings)) {
+                currentGroup.push(sentence);
+                currentGroupEmbeddings.push(embedding);
+                continue;
+            }
+            // Start new group
+            groups.push([...currentGroup]);
+            currentGroup = [sentence];
+            currentGroupEmbeddings = [embedding];
+        }
+        // Don't forget the last group
+        if (currentGroup.length > 0) {
+            groups.push(currentGroup);
+        }
+        return groups;
+    }
+    /**
+     * Decide if a sentence should be added to the current chunk
+     * Based on Max-Min algorithm from the paper
+     */
+    shouldAddToChunk(newEmbedding, chunkEmbeddings) {
+        // Calculate min similarity within current chunk
+        const minSim = this.getMinSimilarity(chunkEmbeddings);
+        // Calculate max similarity between new sentence and chunk
+        const maxSim = this.getMaxSimilarity(newEmbedding, chunkEmbeddings);
+        // Calculate dynamic threshold
+        const threshold = this.calculateThreshold(minSim, chunkEmbeddings.length);
+        return maxSim > threshold;
+    }
+    /**
+     * Minimum pairwise similarity within a chunk, over the last WINDOW_SIZE
+     * sentences only. The approximation follows the Max-Min paper: recent
+     * sentences are what determine coherence.
+     */
+    getMinSimilarity(embeddings) {
+        if (embeddings.length < 2) {
+            return 1.0;
+        }
+        // Only compare the last WINDOW_SIZE embeddings to reduce O(k²) to O(1)
+        const startIdx = Math.max(0, embeddings.length - WINDOW_SIZE);
+        const windowEmbeddings = embeddings.slice(startIdx);
+        let minSim = 1.0;
+        for (let i = 0; i < windowEmbeddings.length; i++) {
+            for (let j = i + 1; j < windowEmbeddings.length; j++) {
+                const embI = windowEmbeddings[i];
+                const embJ = windowEmbeddings[j];
+                if (!embI || !embJ) {
+                    continue;
+                }
+                const sim = this.cosineSimilarity(embI, embJ);
+                if (sim < minSim) {
+                    minSim = sim;
+                }
+            }
+        }
+        return minSim;
+    }
+    /**
+     * Get maximum similarity between a sentence and any sentence in the chunk
+     */
+    getMaxSimilarity(embedding, chunkEmbeddings) {
+        let maxSim = -1.0;
+        for (const chunkEmb of chunkEmbeddings) {
+            const sim = this.cosineSimilarity(embedding, chunkEmb);
+            if (sim > maxSim) {
+                maxSim = sim;
+            }
+        }
+        return maxSim;
+    }
+    /**
+     * Calculate dynamic threshold based on chunk size
+     * threshold = max(c * minSim * sigmoid(|C|), hardThreshold)
+     */
+    calculateThreshold(minSim, chunkSize) {
+        const sigmoidValue = this.sigmoid(chunkSize);
+        const dynamicThreshold = this.config.c * minSim * sigmoidValue;
+        return Math.max(dynamicThreshold, this.config.hardThreshold);
+    }
+    /**
+     * Sigmoid function
+     */
+    sigmoid(x) {
+        return 1 / (1 + Math.exp(-x));
+    }
+    /**
+     * Calculate cosine similarity between two vectors
+     * Public for testing
+     */
+    cosineSimilarity(vec1, vec2) {
+        if (vec1.length !== vec2.length || vec1.length === 0) {
+            return 0;
+        }
+        let dotProduct = 0;
+        let norm1 = 0;
+        let norm2 = 0;
+        for (let i = 0; i < vec1.length; i++) {
+            const v1 = vec1[i] ?? 0;
+            const v2 = vec2[i] ?? 0;
+            dotProduct += v1 * v2;
+            norm1 += v1 * v1;
+            norm2 += v2 * v2;
+        }
+        const denominator = Math.sqrt(norm1) * Math.sqrt(norm2);
+        if (denominator === 0) {
+            return 0;
+        }
+        return dotProduct / denominator;
+    }
+}
+//# sourceMappingURL=semantic-chunker.js.map

package/dist/chunker/semantic-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunker/semantic-chunker.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAC1D,0FAA0F;AAG1F,OAAO,EAAqB,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAClF,OAAO,EAEL,yBAAyB,EACzB,cAAc,GACf,MAAM,wBAAwB,CAAA;AAmC/B,+CAA+C;AAC/C,qCAAqC;AACrC,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,WAAW,GAAG,CAAC,CAAA;AAErB;;;;GAIG;AACH,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,uDAAuD;IACvD,IAAI,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,yDAAyD;IACzD,IAAI,sCAAsC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,kDAAkD;IAClD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC5C,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,IAAI,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACpC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAE/C,iDAAiD;AACjD,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAA;AAE1C,+CAA+C;AAC/C,qCAAqC;AACrC,+CAA+C;AAE/C,0FAA0F;AAC1F,SAAS,SAAS,CAAC,KAA8B;IAC/C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,wBAAwB,CACrC,QAA2B;IAE3B,MAAM,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,QAAQ,CAAA;IAC/C,IAAI,CAAC,aAAa,IAAI,CAAC,WAAW,EAAE,CAAC;QACnC,OAAO,IAAI,CAAA;IACb,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC9C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAA;AAC3E,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,UAAU,CAAC,KAAe,EAAE,MAAyB;IAClE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;IAChD,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,0BAA0B,QAAQ,CAAC,MAAM,qBAAqB,KAAK,CAAC,MAAM,QAAQ,CACnF,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,YAAY,CACzB,KAAqB,EACrB,MAAyB;IAEzB,MAAM,QAAQ,GAAG,MAAM,UAAU,CAC/B,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAC9B,MAAM,CACP,CAAA;IACD,MAAM,SAAS,GAAmB,EAAE,CAAA;IACpC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACvC,IAAI,UAAU,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACpB,SAAQ;QACV,CAAC;QACD,MAAM,QAAQ,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACjD,SAAS,CAAC,IAAI,CACZ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CACpF,CAAA;IACH,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAC1B,MAAwB,EACxB,MAAyB;IAEzB,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAA;IAChE,MAAM,SAAS,GAAqB,EAAE,CAAA;IACtC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACzC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACrB,SAAQ;QACV,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,yBAAyB,CAAC,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAA;IACvF,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,MAAwB;IACrD,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACxC,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,SAAQ;QACV,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC;YACtB,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,SAAS,EAAE,QAAQ,CAAC,SAAS;SAC9B,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,+BAA+B,GAA0B;IAC7D,aAAa,EAAE,GAAG;IAClB,SAAS,EAAE,GAAG;IACd,CAAC,EAAE,GAAG;IACN,cAAc,EAAE,wBAAwB;CACzC,CAAA;AAED,+CAA+C;AAC/C,wBAAwB;AACxB,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,OAAO,eAAe;IACT,MAAM,CAAuB;IAE9C,YAAY,MAAM,GAAmC,EAAE;QACrD,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,+BAA+B,EAAE,GAAG,MAAM,EAAE,CAAA;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,QAA2B,EAC3B,YAAY,GAA+B,EAAE;QAE7C,qBAAqB;QACrB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,wEAAwE;YACxE,wEAAwE;YACxE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,sBAAsB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;YAC5C,CAAC;YACD,OAAO,EAAE,CAAA;QACX,CAAC;QAED,uBAAuB;QACvB,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAChE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,CAAA;QACX,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,QAAQ,CAAC,CAAA;QACvD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAA;QAEhF,wCAAwC;QACxC,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAE5E,yDAAyD;QACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAE7D,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAA;QAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;QAExE,OAAO,qBAAqB,CAAC,MAAM,CAAC,CAAA;IACtC,CAAC;IAED;;;;;OAKG;IACK,WAAW,CAAC,KAAqB;QACvC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAA;QAClC,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAA;IAC5F,CAAC;IAED;;OAEG;IACH;;;;OAIG;IACK,cAAc,CAAC,SAAmB,EAAE,eAA2B;QACrE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,OAAO,KAAK,CAAA;YACd,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAA;YACnE,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAA;QACvE,CAAC;QACD,IAAI,eAAe,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAA;QACd,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAC1D,CAAC;IAEO,cAAc,CAAC,SAAyB,EAAE,UAAsB;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,CAAA;QACX,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YAC7B,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACrC,CAAC;QAED,MAAM,MAAM,GAAqB,EAAE,CAAA;QACnC,IAAI,YAAY,GAAmB,EAAE,CAAA;QACrC,IAAI,sBAAsB,GAAe,EAAE,CAAA;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;YAC7B,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC5B,SAAQ;YACV,CAAC;YAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,sCAAsC;gBACtC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACtC,SAAQ;YACV,CAAC;YAED,IAAI,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,sBAAsB,CAAC,EAAE,CAAC;gBAC3D,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC3B,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBACtC,SAAQ;YACV,CAAC;YAED,kBAAkB;YAClB,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,CAAA;YAC9B,YAAY,GAAG,CAAC,QAAQ,CAAC,CAAA;YACzB,sBAAsB,GAAG,CAAC,SAAS,CAAC,CAAA;QACtC,CAAC;QAED,8BAA8B;QAC9B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QAC3B,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,YAAsB,EAAE,eAA2B;QAC1E,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAA;QAErD,0DAA0D;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;QAEnE,8BAA8B;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAA;QAEzE,OAAO,MAAM,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED;;;;OAIG;IACK,gBAAgB,CAAC,UAAsB;QAC7C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,GAAG,CAAA;QACZ,CAAC;QAED,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,WAAW,CAAC,CAAA;QAC7D,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAEnD,IAAI,MAAM,GAAG,GAAG,CAAA;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;gBAChC,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAA;gBAChC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACnB,SAAQ;gBACV,CAAC;gBAED,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC7C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;oBACjB,MAAM,GAAG,GAAG,CAAA;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,SAAmB,EAAE,eAA2B;QACvE,IAAI,MAAM,GAAG,CAAC,GAAG,CAAA;QACjB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;YACtD,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,kBAAkB,CAAC,MAAc,EAAE,SAAiB;QAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QAC5C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,YAAY,CAAA;QAC9D,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACK,OAAO,CAAC,CAAS;QACvB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,IAAc,EAAE,IAAc;QAC7C,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,CAAA;QACV,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YACvB,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YACvB,UAAU,IAAI,EAAE,GAAG,EAAE,CAAA;YACrB,KAAK,IAAI,EAAE,GAAG,EAAE,CAAA;YAChB,KAAK,IAAI,EAAE,GAAG,EAAE,CAAA;QAClB,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACvD,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAA;QACV,CAAC;QAED,OAAO,UAAU,GAAG,WAAW,CAAA;IACjC,CAAC;CACF"}

package/dist/chunker/sentence-splitter.d.ts

@@ -0,0 +1,32 @@
+import type { AtomicTextRange } from './index.js';
+export interface SentenceUnit {
+    text: string;
+    atomic: boolean;
+    sourceStart: number;
+    sourceEnd: number;
+    /**
+     * Set by token containment on pieces of a unit it admitted before splitting,
+     * so the group holding a piece is stored without the filters re-running on a
+     * fragment. Distinct from `atomic`, which means a range must stay whole; the
+     * splitter never sets this flag.
+     */
+    containmentSplit?: boolean;
+}
+/**
+ * Split text into sentences using Intl.Segmenter
+ *
+ * Uses the Unicode Text Segmentation standard (UAX #29) via Intl.Segmenter.
+ * This provides multilingual support for sentence boundary detection.
+ *
+ * Note: Intl.Segmenter may split on abbreviations like "Mr." or "e.g."
+ * These edge cases are acceptable for semantic chunking as:
+ * 1. Short fragments will be grouped with adjacent sentences by similarity
+ * 2. Fragments below minChunkLength are filtered out
+ */
+export declare function splitIntoSentences(text: string): string[];
+/**
+ * Split ordinary text while preserving the supplied positional ranges as
+ * indivisible sentence units.
+ */
+export declare function splitIntoSentenceUnits(text: string, atomicRanges?: readonly AtomicTextRange[]): SentenceUnit[];
+//# sourceMappingURL=sentence-splitter.d.ts.map

package/dist/chunker/sentence-splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence-splitter.d.ts","sourceRoot":"","sources":["../../src/chunker/sentence-splitter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAyBjD,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AAwLD;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAKzD;AAuBD;;;GAGG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,YAAY,GAAE,SAAS,eAAe,EAAO,GAC5C,YAAY,EAAE,CA6BhB"}