@rce-mcp/retrieval-core 0.1.1 → 0.1.3

package/src/chunking.ts

@@ -1,7 +1,7 @@
 import Parser from "tree-sitter";
 import Go from "tree-sitter-go";
-import JavaScript from "tree-sitter-javascript";
-import Python from "tree-sitter-python";
+import JavaScriptV023 from "tree-sitter-javascript-v023";
+import PythonV023 from "tree-sitter-python-v023";
 import TypeScript from "tree-sitter-typescript";
 
 export type ChunkingStrategy = "language_aware" | "sliding";
@@ -17,9 +17,15 @@ export interface ChunkingConfig {
   fallback_strategy: "sliding";
   target_chunk_tokens: number;
   chunk_overlap_tokens: number;
+  budget_tokenizer: "ranking" | "lightweight";
+  boundary_strictness: "legacy" | "semantic_js_ts";
   max_chunks_per_file: number;
   parse_timeout_ms: number;
   enabled_languages: string[];
+  recursive_semantic_chunking_enabled?: boolean;
+  semantic_merge_gap_lines?: number;
+  semantic_merge_max_span_lines?: number;
+  comment_forward_absorb_enabled?: boolean;
 }
 
 export interface ChunkingRawFile {
@@ -42,6 +48,7 @@ export interface ChunkingResult {
   language_aware_attempt_latency_ms?: number;
   fallback_path_latency_ms?: number;
   language?: string;
+  recursive_semantic_chunking_used?: boolean;
 }
 
 export type ParserLanguage = "typescript" | "tsx" | "javascript" | "jsx" | "python" | "go";
@@ -60,7 +67,7 @@ export interface ChunkingParserAvailabilitySnapshotEntry {
   error?: string;
 }
 
-const DEFAULT_BOUNDARY_NODE_TYPES: Record<ParserLanguage, Set<string>> = {
+const DEFAULT_BOUNDARY_NODE_TYPES_LEGACY: Record<ParserLanguage, Set<string>> = {
   typescript: new Set([
     "function_declaration",
     "generator_function_declaration",
@@ -77,11 +84,35 @@ const DEFAULT_BOUNDARY_NODE_TYPES: Record<ParserLanguage, Set<string>> = {
     "enum_declaration",
     "type_alias_declaration"
   ]),
-  javascript: new Set(["function_declaration", "generator_function_declaration", "class_declaration"]),
-  jsx: new Set(["function_declaration", "generator_function_declaration", "class_declaration"]),
+  javascript: new Set([
+    "function_declaration",
+    "generator_function_declaration",
+    "class_declaration",
+    "function_expression",
+    "arrow_function"
+  ]),
+  jsx: new Set([
+    "function_declaration",
+    "generator_function_declaration",
+    "class_declaration",
+    "function_expression",
+    "arrow_function"
+  ]),
   python: new Set(["function_definition", "class_definition"]),
   go: new Set(["function_declaration", "method_declaration", "type_declaration"])
 };
+const DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS: Record<ParserLanguage, Set<string>> = {
+  ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY,
+  typescript: new Set([
+    ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.typescript,
+    "function_expression",
+    "arrow_function",
+    "method_definition"
+  ]),
+  tsx: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.tsx, "function_expression", "arrow_function", "method_definition"]),
+  javascript: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.javascript, "method_definition"]),
+  jsx: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.jsx, "method_definition"])
+};
 
 const parserAvailabilityCache = new Map<ParserLanguage, ParserAvailability>();
 const parserInitAttempts = new Map<ParserLanguage, number>();
@@ -92,6 +123,15 @@ const CANONICAL_TO_PARSER_LANGUAGE: Record<string, ParserLanguage> = {
   python: "python",
   go: "go"
 };
+const JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES = new Set([
+  "assignment_expression",
+  "variable_declarator",
+  "pair",
+  "export_statement",
+  "public_field_definition",
+  "property_definition"
+]);
+const SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER = 1.35;
 
 function parserLanguageToCanonical(language: ParserLanguage): string {
   if (language === "tsx") {
@@ -131,7 +171,7 @@ function parserLanguageFromPath(path: string): ParserLanguage | undefined {
   if (normalized.endsWith(".tsx")) {
     return "tsx";
   }
-  if (normalized.endsWith(".ts")) {
+  if (normalized.endsWith(".ts") || normalized.endsWith(".mts") || normalized.endsWith(".cts")) {
     return "typescript";
   }
   if (normalized.endsWith(".jsx")) {
@@ -183,10 +223,11 @@ function loadParserLanguage(language: ParserLanguage): Parser.Language {
     return (TypeScript as unknown as { tsx: unknown }).tsx as Parser.Language;
   }
   if (language === "javascript" || language === "jsx") {
-    return resolveTreeSitterLanguageHandle(JavaScript);
+    // Bun is currently most reliable with tree-sitter 0.23-compatible JS/Python grammars.
+    return resolveTreeSitterLanguageHandle(JavaScriptV023);
   }
   if (language === "python") {
-    return resolveTreeSitterLanguageHandle(Python);
+    return resolveTreeSitterLanguageHandle(PythonV023);
   }
   return resolveTreeSitterLanguageHandle(Go);
 }
@@ -317,28 +358,66 @@ function trimLineRange(lines: string[], startRow: number, endRow: number): { sta
 
 function splitRangeWithBudget(input: {
   lines: string[];
+  lineTokenCounts: number[];
   startRow: number;
   endRow: number;
-  tokenize: (text: string) => string[];
   targetChunkTokens: number;
   overlapTokens: number;
   maxChunks: number;
+  preferSafeBoundarySplit?: boolean;
+  softMaxChunkTokens?: number;
 }): Array<{ startRow: number; endRow: number }> {
+  const rangeTokenCount = (startRow: number, endRow: number): number => {
+    let total = 0;
+    for (let row = startRow; row <= endRow; row += 1) {
+      total += input.lineTokenCounts[row] ?? 0;
+    }
+    return total;
+  };
+  const isSafeSplitBoundaryLine = (line: string): boolean => {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      return true;
+    }
+    return trimmed.endsWith(";") || trimmed.endsWith("}") || trimmed.endsWith("{");
+  };
   const segments: Array<{ startRow: number; endRow: number }> = [];
   let start = input.startRow;
 
   while (start <= input.endRow && segments.length < input.maxChunks) {
     let tokens = 0;
-    let end = start;
-    while (end <= input.endRow) {
-      tokens += input.tokenize(input.lines[end] ?? "").length;
-      if (tokens >= input.targetChunkTokens) {
+    let end = start - 1;
+    while (end < input.endRow) {
+      const nextEnd = end + 1;
+      tokens += input.lineTokenCounts[nextEnd] ?? 0;
+      end = nextEnd;
+      if (tokens >= input.targetChunkTokens && end >= start) {
         break;
       }
-      end += 1;
     }
 
-    const safeEnd = Math.min(end, input.endRow);
+    let safeEnd = Math.min(Math.max(start, end), input.endRow);
+    if (input.preferSafeBoundarySplit && safeEnd > start) {
+      let adjusted = safeEnd;
+      for (let row = safeEnd; row > start; row -= 1) {
+        if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+          adjusted = row;
+          break;
+        }
+      }
+      if (adjusted === safeEnd && typeof input.softMaxChunkTokens === "number" && input.softMaxChunkTokens > input.targetChunkTokens) {
+        for (let row = safeEnd + 1; row <= input.endRow; row += 1) {
+          if (rangeTokenCount(start, row) > input.softMaxChunkTokens) {
+            break;
+          }
+          if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+            adjusted = row;
+            break;
+          }
+        }
+      }
+      safeEnd = Math.max(start, adjusted);
+    }
     if (safeEnd >= start) {
       segments.push({ startRow: start, endRow: safeEnd });
     }
@@ -346,8 +425,18 @@ function splitRangeWithBudget(input: {
     if (safeEnd >= input.endRow) {
       break;
     }
-    const rewind = Math.max(1, Math.floor(input.overlapTokens / 4));
-    start = Math.max(start + 1, safeEnd - rewind + 1);
+
+    let nextStart = safeEnd + 1;
+    if (input.overlapTokens > 0) {
+      let overlap = 0;
+      let cursor = safeEnd;
+      while (cursor >= start && overlap < input.overlapTokens) {
+        overlap += input.lineTokenCounts[cursor] ?? 0;
+        cursor -= 1;
+      }
+      nextStart = Math.max(start + 1, cursor + 1);
+    }
+    start = Math.max(start + 1, nextStart);
   }
 
   return segments;
@@ -359,12 +448,14 @@ function buildSlidingChunks(input: {
   targetChunkTokens: number;
   overlapTokens: number;
   maxChunks: number;
+  lineTokenCounts?: number[];
 }): ChunkingOutput[] {
+  const lineTokenCounts = input.lineTokenCounts ?? computeLineTokenCounts(input.lines, input.tokenize);
   const rawSegments = splitRangeWithBudget({
     lines: input.lines,
+    lineTokenCounts,
     startRow: 0,
     endRow: Math.max(0, input.lines.length - 1),
-    tokenize: input.tokenize,
     targetChunkTokens: input.targetChunkTokens,
     overlapTokens: input.overlapTokens,
     maxChunks: input.maxChunks
@@ -395,6 +486,346 @@ function hasBoundaryAncestor(node: Parser.SyntaxNode, boundaryTypes: Set<string>
   return false;
 }
 
+function getBoundaryTypes(
+  parserLanguage: ParserLanguage,
+  boundaryStrictness: "legacy" | "semantic_js_ts"
+): Set<string> {
+  if (boundaryStrictness === "semantic_js_ts") {
+    return DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS[parserLanguage];
+  }
+  return DEFAULT_BOUNDARY_NODE_TYPES_LEGACY[parserLanguage];
+}
+
+function isExpressionBoundaryLanguage(parserLanguage: ParserLanguage, boundaryStrictness: "legacy" | "semantic_js_ts"): boolean {
+  if (boundaryStrictness === "semantic_js_ts") {
+    return (
+      parserLanguage === "javascript" ||
+      parserLanguage === "jsx" ||
+      parserLanguage === "typescript" ||
+      parserLanguage === "tsx"
+    );
+  }
+  return parserLanguage === "javascript" || parserLanguage === "jsx";
+}
+
+function isLanguageBoundaryCandidate(
+  parserLanguage: ParserLanguage,
+  node: Parser.SyntaxNode,
+  boundaryStrictness: "legacy" | "semantic_js_ts"
+): boolean {
+  if (!isExpressionBoundaryLanguage(parserLanguage, boundaryStrictness)) {
+    return true;
+  }
+  if (node.type !== "function_expression" && node.type !== "arrow_function") {
+    return true;
+  }
+  const parentType = node.parent?.type;
+  if (!parentType) {
+    return false;
+  }
+  return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(parentType);
+}
+
+export function __isChunkingBoundaryCandidateForTests(input: {
+  parserLanguage: ParserLanguage;
+  nodeType: string;
+  parentType?: string;
+  boundaryStrictness?: "legacy" | "semantic_js_ts";
+}): boolean {
+  const strictness = input.boundaryStrictness ?? "legacy";
+  if (!isExpressionBoundaryLanguage(input.parserLanguage, strictness)) {
+    return true;
+  }
+  if (input.nodeType !== "function_expression" && input.nodeType !== "arrow_function") {
+    return true;
+  }
+  if (!input.parentType) {
+    return false;
+  }
+  return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(input.parentType);
+}
+
+function computeLineTokenCounts(lines: string[], tokenize: (text: string) => string[]): number[] {
+  return lines.map((line) => tokenize(line ?? "").length);
+}
+
+function rangeTokenCount(lineTokenCounts: number[], startRow: number, endRow: number): number {
+  let total = 0;
+  for (let row = startRow; row <= endRow; row += 1) {
+    total += lineTokenCounts[row] ?? 0;
+  }
+  return total;
+}
+
+function listNamedChildren(node: Parser.SyntaxNode): Parser.SyntaxNode[] {
+  const children: Parser.SyntaxNode[] = [];
+  for (let index = 0; index < node.namedChildCount; index += 1) {
+    const child = node.namedChild(index);
+    if (child) {
+      children.push(child);
+    }
+  }
+  return children;
+}
+
+function normalizeNodeWindow(input: {
+  node: Parser.SyntaxNode;
+  lines: string[];
+  lastRow: number;
+}): { startRow: number; endRow: number } | undefined {
+  const startRow = Math.max(0, Math.min(input.lastRow, input.node.startPosition.row));
+  const endRow = Math.max(startRow, Math.min(input.lastRow, toInclusiveEndRow(input.node)));
+  const trimmed = trimLineRange(input.lines, startRow, endRow);
+  if (!trimmed) {
+    return undefined;
+  }
+  return {
+    startRow: trimmed.start,
+    endRow: trimmed.end
+  };
+}
+
+function buildRecursiveSemanticWindows(input: {
+  root: Parser.SyntaxNode;
+  lines: string[];
+  lineTokenCounts: number[];
+  targetChunkTokens: number;
+  maxChunks: number;
+  boundaryStrictness: "legacy" | "semantic_js_ts";
+}): Array<{ startRow: number; endRow: number }> {
+  const lastRow = Math.max(0, input.lines.length - 1);
+  const windows: Array<{ startRow: number; endRow: number }> = [];
+  const softMaxChunkTokens = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+  const seen = new Set<string>();
+
+  const pushSplitWindows = (startRow: number, endRow: number): void => {
+    if (startRow > endRow || windows.length >= input.maxChunks) {
+      return;
+    }
+    const segments = splitRangeWithBudget({
+      lines: input.lines,
+      lineTokenCounts: input.lineTokenCounts,
+      startRow,
+      endRow,
+      targetChunkTokens: input.targetChunkTokens,
+      overlapTokens: 0,
+      maxChunks: input.maxChunks - windows.length,
+      preferSafeBoundarySplit: input.boundaryStrictness === "semantic_js_ts",
+      softMaxChunkTokens
+    });
+    for (const segment of segments) {
+      const trimmed = trimLineRange(input.lines, segment.startRow, segment.endRow);
+      if (!trimmed) {
+        continue;
+      }
+      const key = `${trimmed.start}:${trimmed.end}`;
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+      windows.push({ startRow: trimmed.start, endRow: trimmed.end });
+      if (windows.length >= input.maxChunks) {
+        return;
+      }
+    }
+  };
+
+  const visitNode = (node: Parser.SyntaxNode): void => {
+    if (windows.length >= input.maxChunks) {
+      return;
+    }
+    const range = normalizeNodeWindow({
+      node,
+      lines: input.lines,
+      lastRow
+    });
+    if (!range) {
+      return;
+    }
+    const tokenCount = rangeTokenCount(input.lineTokenCounts, range.startRow, range.endRow);
+    if (tokenCount <= input.targetChunkTokens) {
+      const key = `${range.startRow}:${range.endRow}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        windows.push(range);
+      }
+      return;
+    }
+
+    const children = listNamedChildren(node)
+      .map((child) => ({
+        node: child,
+        range: normalizeNodeWindow({
+          node: child,
+          lines: input.lines,
+          lastRow
+        })
+      }))
+      .filter((child): child is { node: Parser.SyntaxNode; range: { startRow: number; endRow: number } } =>
+        Boolean(child.range)
+      )
+      .sort((a, b) => a.range.startRow - b.range.startRow || a.range.endRow - b.range.endRow);
+
+    if (children.length === 0) {
+      pushSplitWindows(range.startRow, range.endRow);
+      return;
+    }
+
+    let cursor = range.startRow;
+    for (const child of children) {
+      if (windows.length >= input.maxChunks) {
+        return;
+      }
+      if (child.range.endRow < cursor) {
+        continue;
+      }
+      if (child.range.startRow > cursor) {
+        pushSplitWindows(cursor, child.range.startRow - 1);
+      }
+
+      visitNode(child.node);
+      cursor = Math.max(cursor, child.range.endRow + 1);
+      if (cursor > range.endRow) {
+        return;
+      }
+    }
+    if (cursor <= range.endRow) {
+      pushSplitWindows(cursor, range.endRow);
+    }
+  };
+
+  visitNode(input.root);
+  return windows.sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+}
+
+function mergeSemanticWindows(input: {
+  windows: Array<{ startRow: number; endRow: number }>;
+  lineTokenCounts: number[];
+  targetChunkTokens: number;
+  semanticMergeGapLines: number;
+  semanticMergeMaxSpanLines: number;
+}): Array<{ startRow: number; endRow: number }> {
+  if (input.windows.length <= 1) {
+    return [...input.windows];
+  }
+  const ordered = [...input.windows].sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+  const merged: Array<{ startRow: number; endRow: number }> = [];
+  const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+  for (const window of ordered) {
+    const last = merged[merged.length - 1];
+    if (!last) {
+      merged.push({ ...window });
+      continue;
+    }
+    const gapLines = Math.max(0, window.startRow - last.endRow - 1);
+    const nextStartRow = Math.min(last.startRow, window.startRow);
+    const nextEndRow = Math.max(last.endRow, window.endRow);
+    const nextSpanLines = nextEndRow - nextStartRow + 1;
+    const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, nextStartRow, nextEndRow);
+    const canMerge =
+      gapLines <= input.semanticMergeGapLines &&
+      nextSpanLines <= input.semanticMergeMaxSpanLines &&
+      mergedTokenCount <= mergeTokenBudget;
+    if (!canMerge) {
+      merged.push({ ...window });
+      continue;
+    }
+    last.startRow = nextStartRow;
+    last.endRow = nextEndRow;
+  }
+  return merged;
+}
+
+function isCommentOnlyLine(line: string): boolean {
+  const trimmed = line.trim();
+  if (trimmed.length === 0) {
+    return true;
+  }
+  return (
+    trimmed.startsWith("//") ||
+    trimmed.startsWith("/*") ||
+    trimmed.startsWith("*") ||
+    trimmed.startsWith("*/") ||
+    trimmed.startsWith("#")
+  );
+}
+
+function windowLooksCommentOnly(input: { lines: string[]; startRow: number; endRow: number }): boolean {
+  for (let row = input.startRow; row <= input.endRow; row += 1) {
+    if (!isCommentOnlyLine(input.lines[row] ?? "")) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function absorbForwardCommentWindows(input: {
+  windows: Array<{ startRow: number; endRow: number }>;
+  lines: string[];
+  lineTokenCounts: number[];
+  targetChunkTokens: number;
+  semanticMergeMaxSpanLines: number;
+}): Array<{ startRow: number; endRow: number }> {
+  if (input.windows.length <= 1) {
+    return [...input.windows];
+  }
+  const output: Array<{ startRow: number; endRow: number }> = [];
+  const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+  for (let index = 0; index < input.windows.length; index += 1) {
+    const current = input.windows[index];
+    const next = input.windows[index + 1];
+    if (!current) {
+      continue;
+    }
+    if (!next) {
+      output.push({ ...current });
+      continue;
+    }
+    if (!windowLooksCommentOnly({ lines: input.lines, startRow: current.startRow, endRow: current.endRow })) {
+      output.push({ ...current });
+      continue;
+    }
+    const gapLines = Math.max(0, next.startRow - current.endRow - 1);
+    const nextSpanLines = next.endRow - current.startRow + 1;
+    const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, current.startRow, next.endRow);
+    const canAbsorb =
+      gapLines <= 1 && nextSpanLines <= input.semanticMergeMaxSpanLines && mergedTokenCount <= mergeTokenBudget;
+    if (!canAbsorb) {
+      output.push({ ...current });
+      continue;
+    }
+    output.push({
+      startRow: current.startRow,
+      endRow: next.endRow
+    });
+    index += 1;
+  }
+  return output;
+}
+
+function windowsToChunks(input: {
+  windows: Array<{ startRow: number; endRow: number }>;
+  lines: string[];
+  maxChunks: number;
+}): ChunkingOutput[] {
+  const chunks: ChunkingOutput[] = [];
+  for (const window of input.windows) {
+    if (chunks.length >= input.maxChunks) {
+      break;
+    }
+    const trimmed = trimLineRange(input.lines, window.startRow, window.endRow);
+    if (!trimmed) {
+      continue;
+    }
+    chunks.push({
+      start_line: trimmed.start + 1,
+      end_line: trimmed.end + 1,
+      snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+    });
+  }
+  return chunks;
+}
+
 function buildLanguageAwareChunks(input: {
   file: ChunkingRawFile;
   lines: string[];
@@ -403,6 +834,7 @@ function buildLanguageAwareChunks(input: {
   tokenize: (text: string) => string[];
 }): ChunkingResult {
   const languageAwareAttemptStart = Date.now();
+  const lineTokenCounts = computeLineTokenCounts(input.lines, input.tokenize);
   const parser = getParser(input.parserLanguage);
   if (!parser) {
     const fallbackStart = Date.now();
@@ -411,7 +843,8 @@ function buildLanguageAwareChunks(input: {
       tokenize: input.tokenize,
       targetChunkTokens: input.config.target_chunk_tokens,
       overlapTokens: input.config.chunk_overlap_tokens,
-      maxChunks: input.config.max_chunks_per_file
+      maxChunks: input.config.max_chunks_per_file,
+      lineTokenCounts
     });
     return {
       chunks,
@@ -435,7 +868,8 @@ function buildLanguageAwareChunks(input: {
         tokenize: input.tokenize,
         targetChunkTokens: input.config.target_chunk_tokens,
         overlapTokens: input.config.chunk_overlap_tokens,
-        maxChunks: input.config.max_chunks_per_file
+        maxChunks: input.config.max_chunks_per_file,
+        lineTokenCounts
       });
       return {
         chunks,
@@ -456,7 +890,8 @@ function buildLanguageAwareChunks(input: {
         tokenize: input.tokenize,
         targetChunkTokens: input.config.target_chunk_tokens,
         overlapTokens: input.config.chunk_overlap_tokens,
-        maxChunks: input.config.max_chunks_per_file
+        maxChunks: input.config.max_chunks_per_file,
+        lineTokenCounts
       });
       return {
         chunks,
@@ -469,77 +904,133 @@ function buildLanguageAwareChunks(input: {
       };
     }
 
-    const boundaryTypes = DEFAULT_BOUNDARY_NODE_TYPES[input.parserLanguage];
-    const candidates = root.descendantsOfType([...boundaryTypes]);
-    const boundaryNodes = candidates
-      .filter((node) => !hasBoundaryAncestor(node, boundaryTypes))
-      .sort((a, b) => a.startPosition.row - b.startPosition.row || a.startPosition.column - b.startPosition.column);
+    let chunks: ChunkingOutput[] = [];
+    let recursiveSemanticChunkingUsed = false;
 
-    if (boundaryNodes.length === 0) {
-      const fallbackStart = Date.now();
-      const chunks = buildSlidingChunks({
+    if (input.config.recursive_semantic_chunking_enabled) {
+      const semanticMergeGapLines = input.config.semantic_merge_gap_lines ?? 6;
+      const semanticMergeMaxSpanLines = input.config.semantic_merge_max_span_lines ?? 220;
+      const recursiveWindows = buildRecursiveSemanticWindows({
+        root,
         lines: input.lines,
-        tokenize: input.tokenize,
+        lineTokenCounts,
         targetChunkTokens: input.config.target_chunk_tokens,
-        overlapTokens: input.config.chunk_overlap_tokens,
+        maxChunks: input.config.max_chunks_per_file,
+        boundaryStrictness: input.config.boundary_strictness
+      });
+      const mergedWindows = mergeSemanticWindows({
+        windows: recursiveWindows,
+        lineTokenCounts,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        semanticMergeGapLines,
+        semanticMergeMaxSpanLines
+      });
+      const absorbedWindows =
+        input.config.comment_forward_absorb_enabled === false
+          ? mergedWindows
+          : absorbForwardCommentWindows({
+              windows: mergedWindows,
+              lines: input.lines,
+              lineTokenCounts,
+              targetChunkTokens: input.config.target_chunk_tokens,
+              semanticMergeMaxSpanLines
+            });
+      chunks = windowsToChunks({
+        windows: absorbedWindows,
+        lines: input.lines,
         maxChunks: input.config.max_chunks_per_file
       });
-      return {
-        chunks,
-        strategy: "sliding",
-        fallback_reason: "empty_language_boundaries",
-        parse_latency_ms: parseLatencyMs,
-        language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
-        fallback_path_latency_ms: Date.now() - fallbackStart,
-        language: parserLanguageToCanonical(input.parserLanguage)
-      };
-    }
+      recursiveSemanticChunkingUsed = chunks.length > 0;
+    } else {
+      const boundaryTypes = getBoundaryTypes(input.parserLanguage, input.config.boundary_strictness);
+      const candidates = root.descendantsOfType([...boundaryTypes]);
+      const boundaryNodes = candidates
+        .filter((node) => !hasBoundaryAncestor(node, boundaryTypes))
+        .filter((node) => isLanguageBoundaryCandidate(input.parserLanguage, node, input.config.boundary_strictness))
+        .sort((a, b) => a.startPosition.row - b.startPosition.row || a.startPosition.column - b.startPosition.column);
 
-    const segments: Array<{ startRow: number; endRow: number }> = [];
-    let cursor = 0;
-    const lastRow = Math.max(0, input.lines.length - 1);
-    for (const node of boundaryNodes) {
-      const startRow = Math.max(0, Math.min(lastRow, node.startPosition.row));
-      const endRow = Math.max(startRow, Math.min(lastRow, toInclusiveEndRow(node)));
-      if (startRow > cursor) {
-        segments.push({ startRow: cursor, endRow: startRow - 1 });
-      }
-      segments.push({ startRow, endRow });
-      cursor = endRow + 1;
-      if (cursor > lastRow) {
-        break;
+      if (boundaryNodes.length === 0) {
+        const fallbackStart = Date.now();
+        const fallbackChunks = buildSlidingChunks({
+          lines: input.lines,
+          tokenize: input.tokenize,
+          targetChunkTokens: input.config.target_chunk_tokens,
+          overlapTokens: input.config.chunk_overlap_tokens,
+          maxChunks: input.config.max_chunks_per_file,
+          lineTokenCounts
+        });
+        return {
+          chunks: fallbackChunks,
+          strategy: "sliding",
+          fallback_reason: "empty_language_boundaries",
+          parse_latency_ms: parseLatencyMs,
+          language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+          fallback_path_latency_ms: Date.now() - fallbackStart,
+          language: parserLanguageToCanonical(input.parserLanguage)
+        };
       }
-    }
-    if (cursor <= lastRow) {
-      segments.push({ startRow: cursor, endRow: lastRow });
-    }
 
-    const chunks: ChunkingOutput[] = [];
-    for (const segment of segments) {
-      if (segment.endRow < segment.startRow || chunks.length >= input.config.max_chunks_per_file) {
-        continue;
+      const segments: Array<{ startRow: number; endRow: number; boundary: boolean }> = [];
+      let cursor = 0;
+      const lastRow = Math.max(0, input.lines.length - 1);
+      for (const node of boundaryNodes) {
+        const startRow = Math.max(0, Math.min(lastRow, node.startPosition.row));
+        const endRow = Math.max(startRow, Math.min(lastRow, toInclusiveEndRow(node)));
+        if (startRow > cursor) {
+          segments.push({ startRow: cursor, endRow: startRow - 1, boundary: false });
+        }
+        segments.push({ startRow, endRow, boundary: true });
+        cursor = endRow + 1;
+        if (cursor > lastRow) {
+          break;
+        }
       }
-      const pieces = splitRangeWithBudget({
-        lines: input.lines,
-        startRow: segment.startRow,
-        endRow: segment.endRow,
-        tokenize: input.tokenize,
-        targetChunkTokens: input.config.target_chunk_tokens,
-        overlapTokens: input.config.chunk_overlap_tokens,
-        maxChunks: input.config.max_chunks_per_file - chunks.length
-      });
-      for (const piece of pieces) {
-        const trimmed = trimLineRange(input.lines, piece.startRow, piece.endRow);
-        if (!trimmed) {
+      if (cursor <= lastRow) {
+        segments.push({ startRow: cursor, endRow: lastRow, boundary: false });
+      }
+
+      for (const segment of segments) {
+        if (segment.endRow < segment.startRow || chunks.length >= input.config.max_chunks_per_file) {
           continue;
         }
-        chunks.push({
-          start_line: trimmed.start + 1,
-          end_line: trimmed.end + 1,
-          snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
-        });
-        if (chunks.length >= input.config.max_chunks_per_file) {
-          break;
+        const segmentTokenCount = lineTokenCounts
+          .slice(segment.startRow, segment.endRow + 1)
+          .reduce((sum, value) => sum + value, 0);
+        const enableSemanticBoundarySplits =
+          input.config.boundary_strictness === "semantic_js_ts" &&
+          (input.parserLanguage === "javascript" ||
+            input.parserLanguage === "jsx" ||
+            input.parserLanguage === "typescript" ||
+            input.parserLanguage === "tsx") &&
+          segment.boundary;
+        const softMaxChunkTokens = Math.floor(input.config.target_chunk_tokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+        const pieces =
+          enableSemanticBoundarySplits && segmentTokenCount <= softMaxChunkTokens
+            ? [{ startRow: segment.startRow, endRow: segment.endRow }]
+            : splitRangeWithBudget({
+                lines: input.lines,
+                lineTokenCounts,
+                startRow: segment.startRow,
+                endRow: segment.endRow,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file - chunks.length,
+                preferSafeBoundarySplit: enableSemanticBoundarySplits,
+                softMaxChunkTokens
+              });
+        for (const piece of pieces) {
+          const trimmed = trimLineRange(input.lines, piece.startRow, piece.endRow);
+          if (!trimmed) {
+            continue;
+          }
+          chunks.push({
+            start_line: trimmed.start + 1,
+            end_line: trimmed.end + 1,
+            snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+          });
+          if (chunks.length >= input.config.max_chunks_per_file) {
+            break;
+          }
         }
       }
     }
@@ -551,7 +1042,8 @@ function buildLanguageAwareChunks(input: {
         tokenize: input.tokenize,
         targetChunkTokens: input.config.target_chunk_tokens,
         overlapTokens: input.config.chunk_overlap_tokens,
-        maxChunks: input.config.max_chunks_per_file
+        maxChunks: input.config.max_chunks_per_file,
+        lineTokenCounts
       });
       return {
         chunks: slidingChunks,
@@ -569,7 +1061,8 @@ function buildLanguageAwareChunks(input: {
       strategy: "language_aware",
       parse_latency_ms: parseLatencyMs,
       language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
-      language: parserLanguageToCanonical(input.parserLanguage)
+      language: parserLanguageToCanonical(input.parserLanguage),
+      recursive_semantic_chunking_used: recursiveSemanticChunkingUsed
     };
   } catch {
     const fallbackStart = Date.now();
@@ -578,7 +1071,8 @@ function buildLanguageAwareChunks(input: {
       tokenize: input.tokenize,
       targetChunkTokens: input.config.target_chunk_tokens,
       overlapTokens: input.config.chunk_overlap_tokens,
-      maxChunks: input.config.max_chunks_per_file
+      maxChunks: input.config.max_chunks_per_file,
+      lineTokenCounts
     });
     return {
       chunks,