@remnic/core 9.57.5 → 9.57.6

package/src/chunking.ts

@@ -47,51 +47,136 @@ function estimateTokens(text: string): number {
 
 /**
  * Split text into sentences.
- * Handles common abbreviations and edge cases.
+ *
+ * The scan stays linear and does not use a backtracking regular expression.
+ * ASCII punctuation keeps its old whitespace rule. Unicode terminators also
+ * split when the next sentence starts immediately, as in CJK text.
  */
-function splitSentences(text: string): string[] {
-  // Split on sentence-ending punctuation (. ! ?) that is followed by whitespace
-  // or end of string; the punctuation stays with the sentence.
-  //
-  // Implemented as a single linear scan rather than a regex. Every regex form of
-  // this split is either polynomial (CodeQL js/polynomial-redos) or — once
-  // bounded/anchored to satisfy CodeQL — mishandles long runs or non-boundary
-  // punctuation (a global match silently drops a skipped prefix; a sticky match
-  // stops at the first interior `.` that is not a real boundary, e.g. "v1.2.3"
-  // or "example.com", emitting the whole document as one chunk). A character
-  // scan is O(n), allocation-free, drops nothing, and treats interior
-  // punctuation correctly. Normal prose splits identically to the previous
-  // /[^.!?]*[.!?]+(?:\s+|$)/g form.
-  const sentences: string[] = [];
+const UNICODE_SENTENCE_TERMINATORS = "。．！？؟۔।॥｡…";
+const ASCII_SENTENCE_TERMINATORS = ".!?";
+const CJK_NO_SPACE_TERMINATORS = "。．！？｡";
+const CLOSING_PUNCTUATION = "\"'”’»」』）］】〉》)]}";
+const DIGITS = "0123456789０１２３４５６７８９";
+const SENTENCE_SEGMENTER =
+  typeof Intl.Segmenter === "function"
+    ? new Intl.Segmenter(undefined, { granularity: "sentence" })
+    : undefined;
+
+function isSentenceTerminator(character: string): boolean {
+  return (
+    ASCII_SENTENCE_TERMINATORS.includes(character) ||
+    UNICODE_SENTENCE_TERMINATORS.includes(character)
+  );
+}
+
+function isCjk(character: string): boolean {
+  return /[\u3000-\u9fff\uf900-\ufaff\uac00-\ud7af]/u.test(character);
+}
+
+type SentenceList = string[] & { separators: string[] };
+
+function splitSentencesFallback(text: string): SentenceList {
+  const sentences = [] as unknown as SentenceList;
+  Object.defineProperty(sentences, "separators", {
+    value: [],
+    writable: true,
+  });
   let start = 0;
+  let separator = "";
   for (let i = 0; i < text.length; i++) {
     const ch = text[i];
-    if (ch !== "." && ch !== "!" && ch !== "?") continue;
-    // Consume a run of terminators (e.g. "?!", "...").
+    if (!isSentenceTerminator(ch)) continue;
+
     let end = i;
-    while (end + 1 < text.length) {
-      const n = text[end + 1];
-      if (n !== "." && n !== "!" && n !== "?") break;
+    let hasUnicodeTerminator = UNICODE_SENTENCE_TERMINATORS.includes(ch);
+    while (end + 1 < text.length && isSentenceTerminator(text[end + 1])) {
+      end++;
+      hasUnicodeTerminator ||= UNICODE_SENTENCE_TERMINATORS.includes(text[end]);
+    }
+
+    const isFullwidthDecimal =
+      ch === "．" &&
+      DIGITS.includes(text[i - 1] ?? "") &&
+      DIGITS.includes(text[end + 1] ?? "");
+    while (end + 1 < text.length && CLOSING_PUNCTUATION.includes(text[end + 1])) {
       end++;
     }
+
     const after = text[end + 1];
-    // A real boundary only if the terminator run ends the string or is followed
-    // by whitespace. Interior punctuation (no following whitespace) is left in
-    // place and the scan continues.
-    if (after === undefined || /\s/.test(after)) {
+    const boundary =
+      !isFullwidthDecimal &&
+      (hasUnicodeTerminator || after === undefined || /\s/u.test(after));
+    if (boundary) {
       const sentence = text.slice(start, end + 1).trim();
-      if (sentence.length > 0) sentences.push(sentence);
-      start = end + 1;
+      if (sentence.length > 0) {
+        sentences.push(sentence);
+        if (sentences.length > 1) sentences.separators.push(separator);
+      }
+      let nextStart = end + 1;
+      while (nextStart < text.length && /\s/u.test(text[nextStart])) nextStart++;
+      separator = text.slice(end + 1, nextStart);
+      start = nextStart;
     }
     i = end;
   }
-  // Trailing text without a closing terminator.
-  if (start < text.length) {
-    const remaining = text.slice(start).trim();
-    if (remaining.length > 0) sentences.push(remaining);
+
+  const remaining = text.slice(start).trim();
+  if (remaining.length > 0) {
+    sentences.push(remaining);
+    if (sentences.length > 1) sentences.separators.push(separator);
   }
   return sentences;
 }
+export function splitSentences(text: string): string[] {
+  const canUseSegmenter =
+    SENTENCE_SEGMENTER !== undefined &&
+    !/\s/u.test(text) &&
+    [...text].some((character) => UNICODE_SENTENCE_TERMINATORS.includes(character));
+  if (canUseSegmenter) {
+    const segments = Array.from(SENTENCE_SEGMENTER.segment(text), ({ segment }) =>
+      segment.trim(),
+    ).filter((segment) => segment.length > 0) as SentenceList;
+    if (
+      segments.length > 1 &&
+      segments.slice(0, -1).every((segment) =>
+        [...segment].some((character) =>
+          UNICODE_SENTENCE_TERMINATORS.includes(character),
+        ),
+      )
+    ) {
+      Object.defineProperty(segments, "separators", {
+        value: Array(segments.length - 1).fill(""),
+        writable: true,
+      });
+      return segments;
+    }
+  }
+  return splitSentencesFallback(text);
+}
+
+function endsWithoutSpace(text: string): boolean {
+  let index = text.length - 1;
+  while (index >= 0 && CLOSING_PUNCTUATION.includes(text[index])) index--;
+  const terminator = text[index] ?? "";
+  if (CJK_NO_SPACE_TERMINATORS.includes(terminator)) {
+    return isCjk(text[index - 1] ?? "");
+  }
+  return terminator === "…" && isCjk(text[index - 1] ?? "");
+}
+export function joinSentences(sentences: string[]): string {
+  let result = "";
+  const separators = (sentences as Partial<SentenceList>).separators;
+  for (let i = 0; i < sentences.length; i++) {
+    const sentence = sentences[i];
+    if (result.length > 0) {
+      const separator = separators?.[i - 1];
+      result += separator ?? (endsWithoutSpace(result) ? "" : " ");
+    }
+    result += sentence;
+  }
+  return result;
+}
+
 
 /**
  * Chunk content into overlapping segments at sentence boundaries.
@@ -152,7 +237,7 @@ export function chunkContent(
 
     if (atTarget || isLastSentence) {
       // Create chunk from accumulated sentences
-      const chunkContent = currentChunkSentences.join(" ");
+      const chunkContent = joinSentences(currentChunkSentences);
       chunks.push({
         content: chunkContent,
         index: chunkIndex,
@@ -216,14 +301,14 @@ export function reassembleChunks(chunks: string[]): string {
       const prevEnd = prevSentences.slice(-(j + 1));
       const currStart = currSentences.slice(0, j + 1);
 
-      if (prevEnd.join(" ") === currStart.join(" ")) {
+      if (joinSentences(prevEnd) === joinSentences(currStart)) {
         overlapCount = j + 1;
       }
     }
 
     // Add non-overlapping portion
     if (overlapCount > 0 && overlapCount < currSentences.length) {
-      result.push(currSentences.slice(overlapCount).join(" "));
+      result.push(joinSentences(currSentences.slice(overlapCount)));
     } else if (overlapCount === 0) {
       // No detected overlap, add full chunk
       result.push(currChunk);
@@ -231,5 +316,5 @@ export function reassembleChunks(chunks: string[]): string {
     // If overlapCount === currSentences.length, skip (fully contained)
   }
 
-  return result.join(" ");
+  return joinSentences(result);
 }

package/src/semantic-chunking.ts

@@ -6,7 +6,13 @@
  * natural topic boundaries, producing more coherent chunks.
  */
 
-import { chunkContent, type Chunk, type ChunkResult } from "./chunking.js";
+import {
+  chunkContent,
+  joinSentences,
+  splitSentences,
+  type Chunk,
+  type ChunkResult,
+} from "./chunking.js";
 
 // ---------------------------------------------------------------------------
 // Configuration
@@ -177,50 +183,6 @@ export function findLocalMinima(
 }
 
 // ---------------------------------------------------------------------------
-// Sentence tokenizer
-// ---------------------------------------------------------------------------
-
-/**
- * Split text into sentences at punctuation boundaries.
- * Preserves punctuation with the preceding sentence.
- */
-function splitSentences(text: string): string[] {
-  // Linear character scan instead of a regex. Every regex form of this split is
-  // either polynomial (CodeQL js/polynomial-redos) or — once bounded/anchored to
-  // satisfy CodeQL — mishandles long runs or interior punctuation (a global
-  // match drops a skipped prefix; a sticky match stops at the first non-boundary
-  // `.`, e.g. "v1.2.3" / "example.com", returning the whole document as one
-  // sentence and bypassing chunking). The scan is O(n), drops nothing, and
-  // handles interior punctuation correctly; normal prose splits identically to
-  // the previous /[^.!?]*[.!?]+(?:\s+|$)/g form.
-  const sentences: string[] = [];
-  let start = 0;
-  for (let i = 0; i < text.length; i++) {
-    const ch = text[i];
-    if (ch !== "." && ch !== "!" && ch !== "?") continue;
-    let end = i;
-    while (end + 1 < text.length) {
-      const n = text[end + 1];
-      if (n !== "." && n !== "!" && n !== "?") break;
-      end++;
-    }
-    const after = text[end + 1];
-    // A real boundary only if the terminator run ends the string or is followed
-    // by whitespace. Interior punctuation (no following whitespace) is left in
-    // place and the scan continues.
-    if (after === undefined || /\s/.test(after)) {
-      const sentence = text.slice(start, end + 1).trim();
-      if (sentence.length > 0) sentences.push(sentence);
-      start = end + 1;
-    }
-    i = end;
-  }
-  if (start < text.length) {
-    const remaining = text.slice(start).trim();
-    if (remaining.length > 0) sentences.push(remaining);
-  }
-  return sentences;
-}
 
 // ---------------------------------------------------------------------------
 // Token estimation
@@ -316,7 +278,7 @@ function mergeShortSegments(
 
   for (let i = 0; i < segments.length; i++) {
     buffer = [...buffer, ...segments[i]];
-    const tokenCount = estimateTokens(buffer.join(" "));
+    const tokenCount = estimateTokens(joinSentences(buffer));
 
     if (tokenCount >= minTokens || i === segments.length - 1) {
       merged.push(buffer);
@@ -344,7 +306,7 @@ function splitLongSegment(
   maxTokens: number,
   targetTokens: number,
 ): SemanticChunk[] {
-  const text = segment.join(" ");
+  const text = joinSentences(segment);
   // Cap targetTokens to maxTokens so recursive splitting never produces
   // segments larger than the configured maximum (Finding 2, PR #420).
   const cappedTarget = Math.min(targetTokens, maxTokens);
@@ -511,7 +473,7 @@ export async function semanticChunkContent(
 
   for (let segIdx = 0; segIdx < segments.length; segIdx++) {
     const segment = segments[segIdx];
-    const segText = segment.join(" ");
+    const segText = joinSentences(segment);
     const segTokens = estimateTokens(segText);
 
     if (segTokens > cfg.maxTokens) {

package/dist/chunk-P4BC54KI.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/semantic-chunking.ts"],"sourcesContent":["/**\n * Semantic Chunking with Smoothing-Based Topic Boundaries (Issue #368)\n *\n * An optional alternative to the recursive chunker in chunking.ts.\n * Uses sentence embeddings + cosine similarity + smoothing to detect\n * natural topic boundaries, producing more coherent chunks.\n */\n\nimport { chunkContent, type Chunk, type ChunkResult } from \"./chunking.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration\n// ---------------------------------------------------------------------------\n\nexport interface SemanticChunkingConfig {\n  /** Target tokens per chunk. Default: 200. */\n  targetTokens: number;\n  /** Minimum tokens for a segment before merging with neighbor. Default: 100. */\n  minTokens: number;\n  /** Maximum tokens for a segment before recursive splitting. Default: 400. */\n  maxTokens: number;\n  /** Window size for the moving-average smoothing filter. Default: 3. */\n  smoothingWindowSize: number;\n  /** How many standard deviations below the mean constitutes a boundary. Default: 1.0. */\n  boundaryThresholdStdDevs: number;\n  /** Batch size for embedding requests. Default: 32. */\n  embeddingBatchSize: number;\n  /** Fall back to recursive chunking when embeddings are unavailable. Default: true. */\n  fallbackToRecursive: boolean;\n}\n\nexport const DEFAULT_SEMANTIC_CHUNKING_CONFIG: SemanticChunkingConfig = {\n  targetTokens: 200,\n  minTokens: 100,\n  maxTokens: 400,\n  smoothingWindowSize: 3,\n  boundaryThresholdStdDevs: 1.0,\n  embeddingBatchSize: 32,\n  fallbackToRecursive: true,\n};\n\n// ---------------------------------------------------------------------------\n// Result types\n// ---------------------------------------------------------------------------\n\nexport interface SemanticChunk extends Chunk {\n  /** Optional topic hint derived from position. */\n  topicLabel?: string;\n  /** Cosine similarity score at the trailing boundary of this chunk. */\n  boundaryScore: number;\n}\n\nexport interface SemanticChunkResult {\n  /** Whether content was split into multiple chunks. */\n  chunked: boolean;\n  /** The chunks produced. */\n  chunks: SemanticChunk[];\n  /** Sentence indices where topic splits occurred. */\n  boundaries: number[];\n  /** Which algorithm produced the result. */\n  method: \"semantic\" | \"recursive-fallback\";\n}\n\n// ---------------------------------------------------------------------------\n// Embedding function signature\n// ---------------------------------------------------------------------------\n\n/** Caller-provided function that embeds an array of texts, returning vectors. */\nexport type EmbedFn = (texts: string[]) => Promise<number[][]>;\n\n// ---------------------------------------------------------------------------\n// Math utilities (exported for testing)\n// ---------------------------------------------------------------------------\n\n/**\n * Cosine similarity between two vectors.\n * Returns a value in [-1, 1]. Identical direction = 1, orthogonal = 0.\n *\n * NOTE: This duplicates cosineSimilarity in recall-mmr.ts and embedding-fallback.ts.\n * Consider extracting to a shared math utility in a future refactor.\n */\nexport function cosineSimilarity(a: number[], b: number[]): number {\n  if (a.length !== b.length) {\n    throw new Error(\n      `cosineSimilarity: vector length mismatch (${a.length} vs ${b.length})`,\n    );\n  }\n  if (a.length === 0) return 0;\n\n  let dot = 0;\n  let magA = 0;\n  let magB = 0;\n  for (let i = 0; i < a.length; i++) {\n    dot += a[i] * b[i];\n    magA += a[i] * a[i];\n    magB += b[i] * b[i];\n  }\n\n  const denom = Math.sqrt(magA) * Math.sqrt(magB);\n  if (denom === 0) return 0;\n  return dot / denom;\n}\n\n/**\n * Arithmetic mean of a numeric series.\n */\nexport function mean(series: number[]): number {\n  if (series.length === 0) return 0;\n  let sum = 0;\n  for (const v of series) sum += v;\n  return sum / series.length;\n}\n\n/**\n * Population standard deviation of a numeric series.\n */\nexport function stddev(series: number[]): number {\n  if (series.length === 0) return 0;\n  const m = mean(series);\n  let sumSq = 0;\n  for (const v of series) {\n    const d = v - m;\n    sumSq += d * d;\n  }\n  return Math.sqrt(sumSq / series.length);\n}\n\n/**\n * Simple moving average over a 1D series.\n * The window is centered: for window size W, each output[i] averages\n * series[i - floor(W/2) .. i + floor(W/2)], clamped to bounds.\n *\n * Even window sizes are rounded up to the next odd value so the window\n * is symmetric around the center point (Finding 4, PR #420).\n */\nexport function movingAverage(series: number[], windowSize: number): number[] {\n  if (series.length === 0) return [];\n  if (windowSize < 1) windowSize = 1;\n  // Round even values up to the next odd so the window is symmetric.\n  if (windowSize % 2 === 0) windowSize = windowSize + 1;\n\n  const halfW = Math.floor(windowSize / 2);\n  const result: number[] = new Array(series.length);\n\n  for (let i = 0; i < series.length; i++) {\n    const lo = Math.max(0, i - halfW);\n    const hi = Math.min(series.length - 1, i + halfW);\n    let sum = 0;\n    for (let j = lo; j <= hi; j++) sum += series[j];\n    result[i] = sum / (hi - lo + 1);\n  }\n  return result;\n}\n\n/**\n * Find indices in the series that are local minima AND below the threshold.\n * A local minimum is a point lower than both its immediate neighbors\n * (or lower-or-equal at series boundaries).\n */\nexport function findLocalMinima(\n  series: number[],\n  threshold: number,\n): number[] {\n  if (series.length <= 2) return [];\n\n  const minima: number[] = [];\n  for (let i = 1; i < series.length - 1; i++) {\n    if (\n      series[i] < series[i - 1] &&\n      series[i] < series[i + 1] &&\n      series[i] < threshold\n    ) {\n      minima.push(i);\n    }\n  }\n  return minima;\n}\n\n// ---------------------------------------------------------------------------\n// Sentence tokenizer\n// ---------------------------------------------------------------------------\n\n/**\n * Split text into sentences at punctuation boundaries.\n * Preserves punctuation with the preceding sentence.\n */\nfunction splitSentences(text: string): string[] {\n  // Linear character scan instead of a regex. Every regex form of this split is\n  // either polynomial (CodeQL js/polynomial-redos) or — once bounded/anchored to\n  // satisfy CodeQL — mishandles long runs or interior punctuation (a global\n  // match drops a skipped prefix; a sticky match stops at the first non-boundary\n  // `.`, e.g. \"v1.2.3\" / \"example.com\", returning the whole document as one\n  // sentence and bypassing chunking). The scan is O(n), drops nothing, and\n  // handles interior punctuation correctly; normal prose splits identically to\n  // the previous /[^.!?]*[.!?]+(?:\\s+|$)/g form.\n  const sentences: string[] = [];\n  let start = 0;\n  for (let i = 0; i < text.length; i++) {\n    const ch = text[i];\n    if (ch !== \".\" && ch !== \"!\" && ch !== \"?\") continue;\n    let end = i;\n    while (end + 1 < text.length) {\n      const n = text[end + 1];\n      if (n !== \".\" && n !== \"!\" && n !== \"?\") break;\n      end++;\n    }\n    const after = text[end + 1];\n    // A real boundary only if the terminator run ends the string or is followed\n    // by whitespace. Interior punctuation (no following whitespace) is left in\n    // place and the scan continues.\n    if (after === undefined || /\\s/.test(after)) {\n      const sentence = text.slice(start, end + 1).trim();\n      if (sentence.length > 0) sentences.push(sentence);\n      start = end + 1;\n    }\n    i = end;\n  }\n  if (start < text.length) {\n    const remaining = text.slice(start).trim();\n    if (remaining.length > 0) sentences.push(remaining);\n  }\n  return sentences;\n}\n\n// ---------------------------------------------------------------------------\n// Token estimation\n// ---------------------------------------------------------------------------\n\n/** Rough token estimate: ~4 chars per token for English. */\nfunction estimateTokens(text: string): number {\n  return Math.ceil(text.length / 4);\n}\n\n// ---------------------------------------------------------------------------\n// Core semantic chunking\n// ---------------------------------------------------------------------------\n\n/**\n * Batch-embed sentences using the provided embed function.\n * Respects the configured batch size.\n */\nasync function batchEmbed(\n  sentences: string[],\n  embedFn: EmbedFn,\n  batchSize: number,\n): Promise<number[][]> {\n  const allEmbeddings: number[][] = [];\n\n  for (let i = 0; i < sentences.length; i += batchSize) {\n    const batch = sentences.slice(i, i + batchSize);\n    const batchResult = await embedFn(batch);\n    for (const vec of batchResult) {\n      allEmbeddings.push(vec);\n    }\n  }\n\n  return allEmbeddings;\n}\n\nfunction findEmbeddingDimensionMismatch(\n  embeddings: number[][],\n): { expected: number; actual: number; index: number } | null {\n  if (embeddings.length <= 1) return null;\n  const expected = embeddings[0].length;\n  for (let i = 1; i < embeddings.length; i++) {\n    const actual = embeddings[i].length;\n    if (actual !== expected) {\n      return { expected, actual, index: i };\n    }\n  }\n  return null;\n}\n\n/**\n * Build segments from boundary indices.\n * boundaries are sentence indices at which splits occur (i.e., the split\n * happens AFTER the boundary index sentence).\n */\nfunction buildSegments(\n  sentences: string[],\n  boundaries: number[],\n): string[][] {\n  const sorted = [...boundaries].sort((a, b) => a - b);\n  const segments: string[][] = [];\n  let start = 0;\n\n  for (const b of sorted) {\n    // Split after sentence at index b: segment is [start .. b]\n    const splitPoint = b + 1;\n    if (splitPoint > start && splitPoint <= sentences.length) {\n      segments.push(sentences.slice(start, splitPoint));\n      start = splitPoint;\n    }\n  }\n\n  // Remaining sentences\n  if (start < sentences.length) {\n    segments.push(sentences.slice(start));\n  }\n\n  return segments;\n}\n\n/**\n * Merge short segments (below minTokens) with their neighbor.\n * Prefers merging forward; falls back to merging backward.\n */\nfunction mergeShortSegments(\n  segments: string[][],\n  minTokens: number,\n): string[][] {\n  if (segments.length <= 1) return segments;\n\n  const merged: string[][] = [];\n  let buffer: string[] = [];\n\n  for (let i = 0; i < segments.length; i++) {\n    buffer = [...buffer, ...segments[i]];\n    const tokenCount = estimateTokens(buffer.join(\" \"));\n\n    if (tokenCount >= minTokens || i === segments.length - 1) {\n      merged.push(buffer);\n      buffer = [];\n    }\n  }\n\n  // If the last merge left a dangling buffer, attach it to the last segment\n  if (buffer.length > 0) {\n    if (merged.length > 0) {\n      merged[merged.length - 1] = [...merged[merged.length - 1], ...buffer];\n    } else {\n      merged.push(buffer);\n    }\n  }\n\n  return merged;\n}\n\n/**\n * Split an oversized segment using recursive chunking.\n */\nfunction splitLongSegment(\n  segment: string[],\n  maxTokens: number,\n  targetTokens: number,\n): SemanticChunk[] {\n  const text = segment.join(\" \");\n  // Cap targetTokens to maxTokens so recursive splitting never produces\n  // segments larger than the configured maximum (Finding 2, PR #420).\n  const cappedTarget = Math.min(targetTokens, maxTokens);\n  const result: ChunkResult = chunkContent(text, {\n    targetTokens: cappedTarget,\n    minTokens: Math.min(cappedTarget, maxTokens),\n    overlapSentences: 0,\n  });\n\n  return result.chunks.map((c) => ({\n    content: c.content,\n    index: c.index,\n    tokenCount: c.tokenCount,\n    boundaryScore: 0,\n  }));\n}\n\n/**\n * Semantic chunking with smoothing-based topic boundary detection.\n *\n * @param content   - Full text to chunk.\n * @param embedFn   - Async function that embeds an array of texts.\n * @param config    - Optional partial config overrides.\n * @returns SemanticChunkResult\n */\nexport async function semanticChunkContent(\n  content: string,\n  embedFn: EmbedFn,\n  config?: Partial<SemanticChunkingConfig>,\n): Promise<SemanticChunkResult> {\n  const cfg: SemanticChunkingConfig = {\n    ...DEFAULT_SEMANTIC_CHUNKING_CONFIG,\n    ...config,\n  };\n\n  // Guard against non-positive batch size which would cause an infinite loop\n  const batchSize = Math.max(1, cfg.embeddingBatchSize);\n\n  // --- Empty / trivially short input ---\n  if (!content || content.trim().length === 0) {\n    return {\n      chunked: false,\n      chunks: [],\n      boundaries: [],\n      method: \"semantic\",\n    };\n  }\n\n  const sentences = splitSentences(content);\n\n  if (sentences.length <= 1) {\n    const tokenCount = estimateTokens(content);\n    return {\n      chunked: false,\n      chunks: [\n        {\n          content: content.trim(),\n          index: 0,\n          tokenCount,\n          boundaryScore: 1,\n        },\n      ],\n      boundaries: [],\n      method: \"semantic\",\n    };\n  }\n\n  // If total tokens is short enough, return as single chunk\n  const totalTokens = estimateTokens(content);\n  if (totalTokens <= cfg.minTokens) {\n    return {\n      chunked: false,\n      chunks: [\n        {\n          content: content.trim(),\n          index: 0,\n          tokenCount: totalTokens,\n          boundaryScore: 1,\n        },\n      ],\n      boundaries: [],\n      method: \"semantic\",\n    };\n  }\n\n  // --- Attempt embedding ---\n  let embeddings: number[][];\n  try {\n    embeddings = await batchEmbed(sentences, embedFn, batchSize);\n  } catch {\n    // Embedding failed — fall back if configured\n    if (cfg.fallbackToRecursive) {\n      return buildRecursiveFallback(content, cfg);\n    }\n    throw new Error(\n      \"Semantic chunking failed: embedding function threw and fallbackToRecursive is disabled\",\n    );\n  }\n\n  if (embeddings.length !== sentences.length) {\n    if (cfg.fallbackToRecursive) {\n      return buildRecursiveFallback(content, cfg);\n    }\n    throw new Error(\n      `Semantic chunking failed: expected ${sentences.length} embeddings but received ${embeddings.length}`,\n    );\n  }\n\n  const dimensionMismatch = findEmbeddingDimensionMismatch(embeddings);\n  if (dimensionMismatch) {\n    if (cfg.fallbackToRecursive) {\n      return buildRecursiveFallback(content, cfg);\n    }\n    throw new Error(\n      `Semantic chunking failed: embedding vectors have mismatched dimensions ` +\n        `(${dimensionMismatch.expected} vs ${dimensionMismatch.actual} at index ${dimensionMismatch.index})`,\n    );\n  }\n\n  // --- Compute pairwise cosine similarity ---\n  const similarities: number[] = [];\n  for (let i = 0; i < sentences.length - 1; i++) {\n    similarities.push(cosineSimilarity(embeddings[i], embeddings[i + 1]));\n  }\n\n  // If only one pair (2 sentences), nothing to smooth or split meaningfully.\n  // However, if the combined content exceeds maxTokens, apply recursive splitting.\n  if (similarities.length <= 1) {\n    if (totalTokens > cfg.maxTokens) {\n      return buildRecursiveFallback(content, cfg);\n    }\n    return {\n      chunked: false,\n      chunks: [\n        {\n          content: content.trim(),\n          index: 0,\n          tokenCount: totalTokens,\n          boundaryScore: similarities.length === 1 ? similarities[0] : 1,\n        },\n      ],\n      boundaries: [],\n      method: \"semantic\",\n    };\n  }\n\n  // --- Smooth the similarity series ---\n  const smoothed = movingAverage(similarities, cfg.smoothingWindowSize);\n\n  // --- Detect boundaries: local minima below (mean - k * stddev) ---\n  const m = mean(smoothed);\n  const s = stddev(smoothed);\n  const threshold = m - cfg.boundaryThresholdStdDevs * s;\n  const rawBoundaries = findLocalMinima(smoothed, threshold);\n\n  // --- Build segments, merge short, split long ---\n  let segments = buildSegments(sentences, rawBoundaries);\n  segments = mergeShortSegments(segments, cfg.minTokens);\n\n  // --- Convert segments to chunks, splitting oversized ones ---\n  const chunks: SemanticChunk[] = [];\n  const finalBoundaries: number[] = [];\n  let sentenceOffset = 0;\n\n  for (let segIdx = 0; segIdx < segments.length; segIdx++) {\n    const segment = segments[segIdx];\n    const segText = segment.join(\" \");\n    const segTokens = estimateTokens(segText);\n\n    if (segTokens > cfg.maxTokens) {\n      // Recursive split for oversized segment\n      const subChunks = splitLongSegment(segment, cfg.maxTokens, cfg.targetTokens);\n      for (const sc of subChunks) {\n        chunks.push({\n          ...sc,\n          index: chunks.length,\n        });\n      }\n    } else {\n      // Compute boundary score: the similarity at the trailing edge\n      const trailingSentenceIdx = sentenceOffset + segment.length - 1;\n      let bScore = 1;\n      if (\n        trailingSentenceIdx < similarities.length &&\n        segIdx < segments.length - 1\n      ) {\n        bScore = smoothed[trailingSentenceIdx] ?? similarities[trailingSentenceIdx] ?? 1;\n      }\n\n      chunks.push({\n        content: segText,\n        index: chunks.length,\n        tokenCount: segTokens,\n        boundaryScore: bScore,\n      });\n    }\n\n    // Record boundaries (all but the last segment produce a boundary)\n    if (segIdx < segments.length - 1) {\n      finalBoundaries.push(sentenceOffset + segment.length - 1);\n    }\n    sentenceOffset += segment.length;\n  }\n\n  return {\n    chunked: chunks.length > 1,\n    chunks,\n    boundaries: finalBoundaries,\n    method: \"semantic\",\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Recursive fallback helper\n// ---------------------------------------------------------------------------\n\nfunction buildRecursiveFallback(\n  content: string,\n  cfg: SemanticChunkingConfig,\n): SemanticChunkResult {\n  // Cap targetTokens to maxTokens so the recursive fallback path honours the\n  // same constraint as splitLongSegment (PR #439 post-merge cursor[bot] finding).\n  const cappedTarget = Math.min(cfg.targetTokens, cfg.maxTokens);\n  const result: ChunkResult = chunkContent(content, {\n    targetTokens: cappedTarget,\n    minTokens: Math.min(cfg.minTokens, cappedTarget),\n    overlapSentences: 0,\n  });\n\n  return {\n    chunked: result.chunked,\n    chunks: result.chunks.map((c) => ({\n      ...c,\n      boundaryScore: 0,\n    })),\n    boundaries: [],\n    method: \"recursive-fallback\",\n  };\n}\n"],"mappings":";;;;;AA+BO,IAAM,mCAA2D;AAAA,EACtE,cAAc;AAAA,EACd,WAAW;AAAA,EACX,WAAW;AAAA,EACX,qBAAqB;AAAA,EACrB,0BAA0B;AAAA,EAC1B,oBAAoB;AAAA,EACpB,qBAAqB;AACvB;AA0CO,SAAS,iBAAiB,GAAa,GAAqB;AACjE,MAAI,EAAE,WAAW,EAAE,QAAQ;AACzB,UAAM,IAAI;AAAA,MACR,6CAA6C,EAAE,MAAM,OAAO,EAAE,MAAM;AAAA,IACtE;AAAA,EACF;AACA,MAAI,EAAE,WAAW,EAAG,QAAO;AAE3B,MAAI,MAAM;AACV,MAAI,OAAO;AACX,MAAI,OAAO;AACX,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,WAAO,EAAE,CAAC,IAAI,EAAE,CAAC;AACjB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAClB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpB;AAEA,QAAM,QAAQ,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;AAC9C,MAAI,UAAU,EAAG,QAAO;AACxB,SAAO,MAAM;AACf;AAKO,SAAS,KAAK,QAA0B;AAC7C,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,MAAM;AACV,aAAW,KAAK,OAAQ,QAAO;AAC/B,SAAO,MAAM,OAAO;AACtB;AAKO,SAAS,OAAO,QAA0B;AAC/C,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,IAAI,KAAK,MAAM;AACrB,MAAI,QAAQ;AACZ,aAAW,KAAK,QAAQ;AACtB,UAAM,IAAI,IAAI;AACd,aAAS,IAAI;AAAA,EACf;AACA,SAAO,KAAK,KAAK,QAAQ,OAAO,MAAM;AACxC;AAUO,SAAS,cAAc,QAAkB,YAA8B;AAC5E,MAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AACjC,MAAI,aAAa,EAAG,cAAa;AAEjC,MAAI,aAAa,MAAM,EAAG,cAAa,aAAa;AAEpD,QAAM,QAAQ,KAAK,MAAM,aAAa,CAAC;AACvC,QAAM,SAAmB,IAAI,MAAM,OAAO,MAAM;AAEhD,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,KAAK,KAAK,IAAI,GAAG,IAAI,KAAK;AAChC,UAAM,KAAK,KAAK,IAAI,OAAO,SAAS,GAAG,IAAI,KAAK;AAChD,QAAI,MAAM;AACV,aAAS,IAAI,IAAI,KAAK,IAAI,IAAK,QAAO,OAAO,CAAC;AAC9C,WAAO,CAAC,IAAI,OAAO,KAAK,KAAK;AAAA,EAC/B;AACA,SAAO;AACT;AAOO,SAAS,gBACd,QACA,WACU;AACV,MAAI,OAAO,UAAU,EAAG,QAAO,CAAC;AAEhC,QAAM,SAAmB,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK;AAC1C,QACE,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,KACxB,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,KACxB,OAAO,CAAC,IAAI,WACZ;AACA,aAAO,KAAK,CAAC;AAAA,IACf;AAAA,EACF;AACA,SAAO;AACT;AAUA,SAAS,eAAe,MAAwB;AAS9C,QAAM,YAAsB,CAAC;AAC7B,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,KAAK,KAAK,CAAC;AACjB,QAAI,OAAO,OAAO,OAAO,OAAO,OAAO,IAAK;AAC5C,QAAI,MAAM;AACV,WAAO,MAAM,IAAI,KAAK,QAAQ;AAC5B,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,UAAI,MAAM,OAAO,MAAM,OAAO,MAAM,IAAK;AACzC;AAAA,IACF;AACA,UAAM,QAAQ,KAAK,MAAM,CAAC;AAI1B,QAAI,UAAU,UAAa,KAAK,KAAK,KAAK,GAAG;AAC3C,YAAM,WAAW,KAAK,MAAM,OAAO,MAAM,CAAC,EAAE,KAAK;AACjD,UAAI,SAAS,SAAS,EAAG,WAAU,KAAK,QAAQ;AAChD,cAAQ,MAAM;AAAA,IAChB;AACA,QAAI;AAAA,EACN;AACA,MAAI,QAAQ,KAAK,QAAQ;AACvB,UAAM,YAAY,KAAK,MAAM,KAAK,EAAE,KAAK;AACzC,QAAI,UAAU,SAAS,EAAG,WAAU,KAAK,SAAS;AAAA,EACpD;AACA,SAAO;AACT;AAOA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AAUA,eAAe,WACb,WACA,SACA,WACqB;AACrB,QAAM,gBAA4B,CAAC;AAEnC,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,WAAW;AACpD,UAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,SAAS;AAC9C,UAAM,cAAc,MAAM,QAAQ,KAAK;AACvC,eAAW,OAAO,aAAa;AAC7B,oBAAc,KAAK,GAAG;AAAA,IACxB;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,+BACP,YAC4D;AAC5D,MAAI,WAAW,UAAU,EAAG,QAAO;AACnC,QAAM,WAAW,WAAW,CAAC,EAAE;AAC/B,WAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAM,SAAS,WAAW,CAAC,EAAE;AAC7B,QAAI,WAAW,UAAU;AACvB,aAAO,EAAE,UAAU,QAAQ,OAAO,EAAE;AAAA,IACtC;AAAA,EACF;AACA,SAAO;AACT;AAOA,SAAS,cACP,WACA,YACY;AACZ,QAAM,SAAS,CAAC,GAAG,UAAU,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACnD,QAAM,WAAuB,CAAC;AAC9B,MAAI,QAAQ;AAEZ,aAAW,KAAK,QAAQ;AAEtB,UAAM,aAAa,IAAI;AACvB,QAAI,aAAa,SAAS,cAAc,UAAU,QAAQ;AACxD,eAAS,KAAK,UAAU,MAAM,OAAO,UAAU,CAAC;AAChD,cAAQ;AAAA,IACV;AAAA,EACF;AAGA,MAAI,QAAQ,UAAU,QAAQ;AAC5B,aAAS,KAAK,UAAU,MAAM,KAAK,CAAC;AAAA,EACtC;AAEA,SAAO;AACT;AAMA,SAAS,mBACP,UACA,WACY;AACZ,MAAI,SAAS,UAAU,EAAG,QAAO;AAEjC,QAAM,SAAqB,CAAC;AAC5B,MAAI,SAAmB,CAAC;AAExB,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAS,CAAC,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;AACnC,UAAM,aAAa,eAAe,OAAO,KAAK,GAAG,CAAC;AAElD,QAAI,cAAc,aAAa,MAAM,SAAS,SAAS,GAAG;AACxD,aAAO,KAAK,MAAM;AAClB,eAAS,CAAC;AAAA,IACZ;AAAA,EACF;AAGA,MAAI,OAAO,SAAS,GAAG;AACrB,QAAI,OAAO,SAAS,GAAG;AACrB,aAAO,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,OAAO,SAAS,CAAC,GAAG,GAAG,MAAM;AAAA,IACtE,OAAO;AACL,aAAO,KAAK,MAAM;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,iBACP,SACA,WACA,cACiB;AACjB,QAAM,OAAO,QAAQ,KAAK,GAAG;AAG7B,QAAM,eAAe,KAAK,IAAI,cAAc,SAAS;AACrD,QAAM,SAAsB,aAAa,MAAM;AAAA,IAC7C,cAAc;AAAA,IACd,WAAW,KAAK,IAAI,cAAc,SAAS;AAAA,IAC3C,kBAAkB;AAAA,EACpB,CAAC;AAED,SAAO,OAAO,OAAO,IAAI,CAAC,OAAO;AAAA,IAC/B,SAAS,EAAE;AAAA,IACX,OAAO,EAAE;AAAA,IACT,YAAY,EAAE;AAAA,IACd,eAAe;AAAA,EACjB,EAAE;AACJ;AAUA,eAAsB,qBACpB,SACA,SACA,QAC8B;AAC9B,QAAM,MAA8B;AAAA,IAClC,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAGA,QAAM,YAAY,KAAK,IAAI,GAAG,IAAI,kBAAkB;AAGpD,MAAI,CAAC,WAAW,QAAQ,KAAK,EAAE,WAAW,GAAG;AAC3C,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ,CAAC;AAAA,MACT,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAEA,QAAM,YAAY,eAAe,OAAO;AAExC,MAAI,UAAU,UAAU,GAAG;AACzB,UAAM,aAAa,eAAe,OAAO;AACzC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,QACN;AAAA,UACE,SAAS,QAAQ,KAAK;AAAA,UACtB,OAAO;AAAA,UACP;AAAA,UACA,eAAe;AAAA,QACjB;AAAA,MACF;AAAA,MACA,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,cAAc,eAAe,OAAO;AAC1C,MAAI,eAAe,IAAI,WAAW;AAChC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,QACN;AAAA,UACE,SAAS,QAAQ,KAAK;AAAA,UACtB,OAAO;AAAA,UACP,YAAY;AAAA,UACZ,eAAe;AAAA,QACjB;AAAA,MACF;AAAA,MACA,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,MAAI;AACJ,MAAI;AACF,iBAAa,MAAM,WAAW,WAAW,SAAS,SAAS;AAAA,EAC7D,QAAQ;AAEN,QAAI,IAAI,qBAAqB;AAC3B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI,WAAW,WAAW,UAAU,QAAQ;AAC1C,QAAI,IAAI,qBAAqB;AAC3B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,UAAM,IAAI;AAAA,MACR,sCAAsC,UAAU,MAAM,4BAA4B,WAAW,MAAM;AAAA,IACrG;AAAA,EACF;AAEA,QAAM,oBAAoB,+BAA+B,UAAU;AACnE,MAAI,mBAAmB;AACrB,QAAI,IAAI,qBAAqB;AAC3B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,UAAM,IAAI;AAAA,MACR,2EACM,kBAAkB,QAAQ,OAAO,kBAAkB,MAAM,aAAa,kBAAkB,KAAK;AAAA,IACrG;AAAA,EACF;AAGA,QAAM,eAAyB,CAAC;AAChC,WAAS,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,KAAK;AAC7C,iBAAa,KAAK,iBAAiB,WAAW,CAAC,GAAG,WAAW,IAAI,CAAC,CAAC,CAAC;AAAA,EACtE;AAIA,MAAI,aAAa,UAAU,GAAG;AAC5B,QAAI,cAAc,IAAI,WAAW;AAC/B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,QACN;AAAA,UACE,SAAS,QAAQ,KAAK;AAAA,UACtB,OAAO;AAAA,UACP,YAAY;AAAA,UACZ,eAAe,aAAa,WAAW,IAAI,aAAa,CAAC,IAAI;AAAA,QAC/D;AAAA,MACF;AAAA,MACA,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,WAAW,cAAc,cAAc,IAAI,mBAAmB;AAGpE,QAAM,IAAI,KAAK,QAAQ;AACvB,QAAM,IAAI,OAAO,QAAQ;AACzB,QAAM,YAAY,IAAI,IAAI,2BAA2B;AACrD,QAAM,gBAAgB,gBAAgB,UAAU,SAAS;AAGzD,MAAI,WAAW,cAAc,WAAW,aAAa;AACrD,aAAW,mBAAmB,UAAU,IAAI,SAAS;AAGrD,QAAM,SAA0B,CAAC;AACjC,QAAM,kBAA4B,CAAC;AACnC,MAAI,iBAAiB;AAErB,WAAS,SAAS,GAAG,SAAS,SAAS,QAAQ,UAAU;AACvD,UAAM,UAAU,SAAS,MAAM;AAC/B,UAAM,UAAU,QAAQ,KAAK,GAAG;AAChC,UAAM,YAAY,eAAe,OAAO;AAExC,QAAI,YAAY,IAAI,WAAW;AAE7B,YAAM,YAAY,iBAAiB,SAAS,IAAI,WAAW,IAAI,YAAY;AAC3E,iBAAW,MAAM,WAAW;AAC1B,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UACH,OAAO,OAAO;AAAA,QAChB,CAAC;AAAA,MACH;AAAA,IACF,OAAO;AAEL,YAAM,sBAAsB,iBAAiB,QAAQ,SAAS;AAC9D,UAAI,SAAS;AACb,UACE,sBAAsB,aAAa,UACnC,SAAS,SAAS,SAAS,GAC3B;AACA,iBAAS,SAAS,mBAAmB,KAAK,aAAa,mBAAmB,KAAK;AAAA,MACjF;AAEA,aAAO,KAAK;AAAA,QACV,SAAS;AAAA,QACT,OAAO,OAAO;AAAA,QACd,YAAY;AAAA,QACZ,eAAe;AAAA,MACjB,CAAC;AAAA,IACH;AAGA,QAAI,SAAS,SAAS,SAAS,GAAG;AAChC,sBAAgB,KAAK,iBAAiB,QAAQ,SAAS,CAAC;AAAA,IAC1D;AACA,sBAAkB,QAAQ;AAAA,EAC5B;AAEA,SAAO;AAAA,IACL,SAAS,OAAO,SAAS;AAAA,IACzB;AAAA,IACA,YAAY;AAAA,IACZ,QAAQ;AAAA,EACV;AACF;AAMA,SAAS,uBACP,SACA,KACqB;AAGrB,QAAM,eAAe,KAAK,IAAI,IAAI,cAAc,IAAI,SAAS;AAC7D,QAAM,SAAsB,aAAa,SAAS;AAAA,IAChD,cAAc;AAAA,IACd,WAAW,KAAK,IAAI,IAAI,WAAW,YAAY;AAAA,IAC/C,kBAAkB;AAAA,EACpB,CAAC;AAED,SAAO;AAAA,IACL,SAAS,OAAO;AAAA,IAChB,QAAQ,OAAO,OAAO,IAAI,CAAC,OAAO;AAAA,MAChC,GAAG;AAAA,MACH,eAAe;AAAA,IACjB,EAAE;AAAA,IACF,YAAY,CAAC;AAAA,IACb,QAAQ;AAAA,EACV;AACF;","names":[]}

package/dist/chunk-UQ7RN5HK.js

@@ -1,126 +0,0 @@
-// src/chunking.ts
-var DEFAULT_CHUNKING_CONFIG = {
-  targetTokens: 200,
-  minTokens: 150,
-  overlapSentences: 2
-};
-function estimateTokens(text) {
-  return Math.ceil(text.length / 4);
-}
-function splitSentences(text) {
-  const sentences = [];
-  let start = 0;
-  for (let i = 0; i < text.length; i++) {
-    const ch = text[i];
-    if (ch !== "." && ch !== "!" && ch !== "?") continue;
-    let end = i;
-    while (end + 1 < text.length) {
-      const n = text[end + 1];
-      if (n !== "." && n !== "!" && n !== "?") break;
-      end++;
-    }
-    const after = text[end + 1];
-    if (after === void 0 || /\s/.test(after)) {
-      const sentence = text.slice(start, end + 1).trim();
-      if (sentence.length > 0) sentences.push(sentence);
-      start = end + 1;
-    }
-    i = end;
-  }
-  if (start < text.length) {
-    const remaining = text.slice(start).trim();
-    if (remaining.length > 0) sentences.push(remaining);
-  }
-  return sentences;
-}
-function chunkContent(content, config = DEFAULT_CHUNKING_CONFIG) {
-  const totalTokens = estimateTokens(content);
-  if (totalTokens < config.minTokens) {
-    return {
-      chunked: false,
-      chunks: [{
-        content,
-        index: 0,
-        tokenCount: totalTokens
-      }]
-    };
-  }
-  const sentences = splitSentences(content);
-  if (sentences.length <= 1) {
-    return {
-      chunked: false,
-      chunks: [{
-        content,
-        index: 0,
-        tokenCount: totalTokens
-      }]
-    };
-  }
-  const chunks = [];
-  let currentChunkSentences = [];
-  let currentTokens = 0;
-  let chunkIndex = 0;
-  for (let i = 0; i < sentences.length; i++) {
-    const sentence = sentences[i];
-    const sentenceTokens = estimateTokens(sentence);
-    currentChunkSentences.push(sentence);
-    currentTokens += sentenceTokens;
-    const atTarget = currentTokens >= config.targetTokens;
-    const isLastSentence = i === sentences.length - 1;
-    if (atTarget || isLastSentence) {
-      const chunkContent2 = currentChunkSentences.join(" ");
-      chunks.push({
-        content: chunkContent2,
-        index: chunkIndex,
-        tokenCount: estimateTokens(chunkContent2)
-      });
-      chunkIndex++;
-      if (!isLastSentence) {
-        const overlapCount = Math.min(config.overlapSentences, currentChunkSentences.length);
-        if (overlapCount <= 0) {
-          currentChunkSentences = [];
-          currentTokens = 0;
-        } else {
-          currentChunkSentences = currentChunkSentences.slice(-overlapCount);
-          currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokens(s), 0);
-        }
-      }
-    }
-  }
-  return {
-    chunked: chunks.length > 1,
-    chunks
-  };
-}
-function reassembleChunks(chunks) {
-  if (chunks.length === 0) return "";
-  if (chunks.length === 1) return chunks[0];
-  const result = [chunks[0]];
-  for (let i = 1; i < chunks.length; i++) {
-    const prevChunk = chunks[i - 1];
-    const currChunk = chunks[i];
-    const prevSentences = splitSentences(prevChunk);
-    const currSentences = splitSentences(currChunk);
-    let overlapCount = 0;
-    for (let j = 0; j < Math.min(prevSentences.length, currSentences.length); j++) {
-      const prevEnd = prevSentences.slice(-(j + 1));
-      const currStart = currSentences.slice(0, j + 1);
-      if (prevEnd.join(" ") === currStart.join(" ")) {
-        overlapCount = j + 1;
-      }
-    }
-    if (overlapCount > 0 && overlapCount < currSentences.length) {
-      result.push(currSentences.slice(overlapCount).join(" "));
-    } else if (overlapCount === 0) {
-      result.push(currChunk);
-    }
-  }
-  return result.join(" ");
-}
-
-export {
-  DEFAULT_CHUNKING_CONFIG,
-  chunkContent,
-  reassembleChunks
-};
-//# sourceMappingURL=chunk-UQ7RN5HK.js.map

package/dist/chunk-UQ7RN5HK.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/chunking.ts"],"sourcesContent":["/**\n * Automatic Chunking with Overlap (Phase 2A)\n *\n * Sentence-boundary chunking for long memories.\n * Preserves coherent thoughts by never splitting mid-sentence.\n */\n\nexport interface ChunkingConfig {\n  /** Target tokens per chunk (default 200) */\n  targetTokens: number;\n  /** Minimum tokens to trigger chunking (default 150) */\n  minTokens: number;\n  /** Number of sentences to overlap between chunks (default 2) */\n  overlapSentences: number;\n}\n\nexport interface Chunk {\n  /** Chunk content */\n  content: string;\n  /** 0-based index */\n  index: number;\n  /** Approximate token count */\n  tokenCount: number;\n}\n\nexport interface ChunkResult {\n  /** Whether content was chunked */\n  chunked: boolean;\n  /** Array of chunks (length 1 if not chunked) */\n  chunks: Chunk[];\n}\n\n/** Default chunking configuration */\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n  targetTokens: 200,\n  minTokens: 150,\n  overlapSentences: 2,\n};\n\n/**\n * Estimate token count for text.\n * Rough approximation: ~4 characters per token for English.\n */\nfunction estimateTokens(text: string): number {\n  return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into sentences.\n * Handles common abbreviations and edge cases.\n */\nfunction splitSentences(text: string): string[] {\n  // Split on sentence-ending punctuation (. ! ?) that is followed by whitespace\n  // or end of string; the punctuation stays with the sentence.\n  //\n  // Implemented as a single linear scan rather than a regex. Every regex form of\n  // this split is either polynomial (CodeQL js/polynomial-redos) or — once\n  // bounded/anchored to satisfy CodeQL — mishandles long runs or non-boundary\n  // punctuation (a global match silently drops a skipped prefix; a sticky match\n  // stops at the first interior `.` that is not a real boundary, e.g. \"v1.2.3\"\n  // or \"example.com\", emitting the whole document as one chunk). A character\n  // scan is O(n), allocation-free, drops nothing, and treats interior\n  // punctuation correctly. Normal prose splits identically to the previous\n  // /[^.!?]*[.!?]+(?:\\s+|$)/g form.\n  const sentences: string[] = [];\n  let start = 0;\n  for (let i = 0; i < text.length; i++) {\n    const ch = text[i];\n    if (ch !== \".\" && ch !== \"!\" && ch !== \"?\") continue;\n    // Consume a run of terminators (e.g. \"?!\", \"...\").\n    let end = i;\n    while (end + 1 < text.length) {\n      const n = text[end + 1];\n      if (n !== \".\" && n !== \"!\" && n !== \"?\") break;\n      end++;\n    }\n    const after = text[end + 1];\n    // A real boundary only if the terminator run ends the string or is followed\n    // by whitespace. Interior punctuation (no following whitespace) is left in\n    // place and the scan continues.\n    if (after === undefined || /\\s/.test(after)) {\n      const sentence = text.slice(start, end + 1).trim();\n      if (sentence.length > 0) sentences.push(sentence);\n      start = end + 1;\n    }\n    i = end;\n  }\n  // Trailing text without a closing terminator.\n  if (start < text.length) {\n    const remaining = text.slice(start).trim();\n    if (remaining.length > 0) sentences.push(remaining);\n  }\n  return sentences;\n}\n\n/**\n * Chunk content into overlapping segments at sentence boundaries.\n *\n * @param content - The text content to chunk\n * @param config - Chunking configuration\n * @returns ChunkResult with chunks array\n */\nexport function chunkContent(\n  content: string,\n  config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG,\n): ChunkResult {\n  const totalTokens = estimateTokens(content);\n\n  // Don't chunk if below minimum threshold\n  if (totalTokens < config.minTokens) {\n    return {\n      chunked: false,\n      chunks: [{\n        content,\n        index: 0,\n        tokenCount: totalTokens,\n      }],\n    };\n  }\n\n  const sentences = splitSentences(content);\n\n  // If we couldn't split into multiple sentences, don't chunk\n  if (sentences.length <= 1) {\n    return {\n      chunked: false,\n      chunks: [{\n        content,\n        index: 0,\n        tokenCount: totalTokens,\n      }],\n    };\n  }\n\n  const chunks: Chunk[] = [];\n  let currentChunkSentences: string[] = [];\n  let currentTokens = 0;\n  let chunkIndex = 0;\n\n  for (let i = 0; i < sentences.length; i++) {\n    const sentence = sentences[i];\n    const sentenceTokens = estimateTokens(sentence);\n\n    // Add sentence to current chunk\n    currentChunkSentences.push(sentence);\n    currentTokens += sentenceTokens;\n\n    // Check if we've reached target size (with some flexibility)\n    // Allow going over by up to 50% to avoid tiny final chunks\n    const atTarget = currentTokens >= config.targetTokens;\n    const isLastSentence = i === sentences.length - 1;\n\n    if (atTarget || isLastSentence) {\n      // Create chunk from accumulated sentences\n      const chunkContent = currentChunkSentences.join(\" \");\n      chunks.push({\n        content: chunkContent,\n        index: chunkIndex,\n        tokenCount: estimateTokens(chunkContent),\n      });\n      chunkIndex++;\n\n      // Start new chunk with overlap (if not at end)\n      if (!isLastSentence) {\n        // Keep last N sentences for overlap.\n        // Guard: slice(-0) === slice(0), which returns the ENTIRE array\n        // (CLAUDE.md gotcha #27). When overlapSentences is 0, clear fully.\n        const overlapCount = Math.min(config.overlapSentences, currentChunkSentences.length);\n        if (overlapCount <= 0) {\n          currentChunkSentences = [];\n          currentTokens = 0;\n        } else {\n          currentChunkSentences = currentChunkSentences.slice(-overlapCount);\n          currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokens(s), 0);\n        }\n      }\n    }\n  }\n\n  // Only consider it \"chunked\" if we got multiple chunks\n  return {\n    chunked: chunks.length > 1,\n    chunks,\n  };\n}\n\n/**\n * Get parent content by reassembling chunks.\n * Useful for displaying full context when a chunk is retrieved.\n *\n * @param chunks - Array of chunk contents in order\n * @returns Reassembled parent content (with overlap removed)\n */\nexport function reassembleChunks(chunks: string[]): string {\n  if (chunks.length === 0) return \"\";\n  if (chunks.length === 1) return chunks[0];\n\n  // For overlapping chunks, we need to deduplicate\n  // Simple approach: use full first chunk, then non-overlapping parts of subsequent chunks\n  // This is imperfect but handles most cases\n  const result: string[] = [chunks[0]];\n\n  for (let i = 1; i < chunks.length; i++) {\n    const prevChunk = chunks[i - 1];\n    const currChunk = chunks[i];\n\n    // Find overlap by looking for common suffix/prefix\n    // Try to find where the previous chunk ends in the current chunk\n    const prevSentences = splitSentences(prevChunk);\n    const currSentences = splitSentences(currChunk);\n\n    // Find how many sentences from prev are at the start of curr\n    let overlapCount = 0;\n    for (let j = 0; j < Math.min(prevSentences.length, currSentences.length); j++) {\n      // Check if last N sentences of prev match first N sentences of curr\n      const prevEnd = prevSentences.slice(-(j + 1));\n      const currStart = currSentences.slice(0, j + 1);\n\n      if (prevEnd.join(\" \") === currStart.join(\" \")) {\n        overlapCount = j + 1;\n      }\n    }\n\n    // Add non-overlapping portion\n    if (overlapCount > 0 && overlapCount < currSentences.length) {\n      result.push(currSentences.slice(overlapCount).join(\" \"));\n    } else if (overlapCount === 0) {\n      // No detected overlap, add full chunk\n      result.push(currChunk);\n    }\n    // If overlapCount === currSentences.length, skip (fully contained)\n  }\n\n  return result.join(\" \");\n}\n"],"mappings":";AAiCO,IAAM,0BAA0C;AAAA,EACrD,cAAc;AAAA,EACd,WAAW;AAAA,EACX,kBAAkB;AACpB;AAMA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AAMA,SAAS,eAAe,MAAwB;AAa9C,QAAM,YAAsB,CAAC;AAC7B,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,KAAK,KAAK,CAAC;AACjB,QAAI,OAAO,OAAO,OAAO,OAAO,OAAO,IAAK;AAE5C,QAAI,MAAM;AACV,WAAO,MAAM,IAAI,KAAK,QAAQ;AAC5B,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,UAAI,MAAM,OAAO,MAAM,OAAO,MAAM,IAAK;AACzC;AAAA,IACF;AACA,UAAM,QAAQ,KAAK,MAAM,CAAC;AAI1B,QAAI,UAAU,UAAa,KAAK,KAAK,KAAK,GAAG;AAC3C,YAAM,WAAW,KAAK,MAAM,OAAO,MAAM,CAAC,EAAE,KAAK;AACjD,UAAI,SAAS,SAAS,EAAG,WAAU,KAAK,QAAQ;AAChD,cAAQ,MAAM;AAAA,IAChB;AACA,QAAI;AAAA,EACN;AAEA,MAAI,QAAQ,KAAK,QAAQ;AACvB,UAAM,YAAY,KAAK,MAAM,KAAK,EAAE,KAAK;AACzC,QAAI,UAAU,SAAS,EAAG,WAAU,KAAK,SAAS;AAAA,EACpD;AACA,SAAO;AACT;AASO,SAAS,aACd,SACA,SAAyB,yBACZ;AACb,QAAM,cAAc,eAAe,OAAO;AAG1C,MAAI,cAAc,OAAO,WAAW;AAClC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ,CAAC;AAAA,QACP;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,YAAY,eAAe,OAAO;AAGxC,MAAI,UAAU,UAAU,GAAG;AACzB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ,CAAC;AAAA,QACP;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,SAAkB,CAAC;AACzB,MAAI,wBAAkC,CAAC;AACvC,MAAI,gBAAgB;AACpB,MAAI,aAAa;AAEjB,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,UAAM,iBAAiB,eAAe,QAAQ;AAG9C,0BAAsB,KAAK,QAAQ;AACnC,qBAAiB;AAIjB,UAAM,WAAW,iBAAiB,OAAO;AACzC,UAAM,iBAAiB,MAAM,UAAU,SAAS;AAEhD,QAAI,YAAY,gBAAgB;AAE9B,YAAMA,gBAAe,sBAAsB,KAAK,GAAG;AACnD,aAAO,KAAK;AAAA,QACV,SAASA;AAAA,QACT,OAAO;AAAA,QACP,YAAY,eAAeA,aAAY;AAAA,MACzC,CAAC;AACD;AAGA,UAAI,CAAC,gBAAgB;AAInB,cAAM,eAAe,KAAK,IAAI,OAAO,kBAAkB,sBAAsB,MAAM;AACnF,YAAI,gBAAgB,GAAG;AACrB,kCAAwB,CAAC;AACzB,0BAAgB;AAAA,QAClB,OAAO;AACL,kCAAwB,sBAAsB,MAAM,CAAC,YAAY;AACjE,0BAAgB,sBAAsB,OAAO,CAAC,KAAK,MAAM,MAAM,eAAe,CAAC,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,SAAO;AAAA,IACL,SAAS,OAAO,SAAS;AAAA,IACzB;AAAA,EACF;AACF;AASO,SAAS,iBAAiB,QAA0B;AACzD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AAKxC,QAAM,SAAmB,CAAC,OAAO,CAAC,CAAC;AAEnC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,YAAY,OAAO,IAAI,CAAC;AAC9B,UAAM,YAAY,OAAO,CAAC;AAI1B,UAAM,gBAAgB,eAAe,SAAS;AAC9C,UAAM,gBAAgB,eAAe,SAAS;AAG9C,QAAI,eAAe;AACnB,aAAS,IAAI,GAAG,IAAI,KAAK,IAAI,cAAc,QAAQ,cAAc,MAAM,GAAG,KAAK;AAE7E,YAAM,UAAU,cAAc,MAAM,EAAE,IAAI,EAAE;AAC5C,YAAM,YAAY,cAAc,MAAM,GAAG,IAAI,CAAC;AAE9C,UAAI,QAAQ,KAAK,GAAG,MAAM,UAAU,KAAK,GAAG,GAAG;AAC7C,uBAAe,IAAI;AAAA,MACrB;AAAA,IACF;AAGA,QAAI,eAAe,KAAK,eAAe,cAAc,QAAQ;AAC3D,aAAO,KAAK,cAAc,MAAM,YAAY,EAAE,KAAK,GAAG,CAAC;AAAA,IACzD,WAAW,iBAAiB,GAAG;AAE7B,aAAO,KAAK,SAAS;AAAA,IACvB;AAAA,EAEF;AAEA,SAAO,OAAO,KAAK,GAAG;AACxB;","names":["chunkContent"]}