@cue-dev/retrieval-core 0.1.3

package/src/chunking.ts

@@ -0,0 +1,1187 @@
+import Parser from "tree-sitter";
+import Go from "tree-sitter-go";
+import JavaScriptV023 from "tree-sitter-javascript-v023";
+import Java from "tree-sitter-java";
+import PythonV023 from "tree-sitter-python-v023";
+import Rust from "tree-sitter-rust";
+import TypeScript from "tree-sitter-typescript";
+
+export type ChunkingStrategy = "language_aware" | "sliding";
+export type ChunkingFallbackReason =
+  | "unsupported_language"
+  | "parser_unavailable"
+  | "parse_error"
+  | "parse_timeout_exceeded"
+  | "empty_language_boundaries";
+
+export interface ChunkingConfig {
+  strategy: ChunkingStrategy;
+  fallback_strategy: "sliding";
+  target_chunk_tokens: number;
+  chunk_overlap_tokens: number;
+  budget_tokenizer: "ranking" | "lightweight";
+  boundary_strictness: "legacy" | "semantic_js_ts";
+  max_chunks_per_file: number;
+  parse_timeout_ms: number;
+  enabled_languages: string[];
+  recursive_semantic_chunking_enabled?: boolean;
+  semantic_merge_gap_lines?: number;
+  semantic_merge_max_span_lines?: number;
+  comment_forward_absorb_enabled?: boolean;
+}
+
+export interface ChunkingRawFile {
+  path: string;
+  content: string;
+  language?: string;
+}
+
+export interface ChunkingOutput {
+  start_line: number;
+  end_line: number;
+  snippet: string;
+}
+
+export interface ChunkingResult {
+  chunks: ChunkingOutput[];
+  strategy: ChunkingStrategy;
+  fallback_reason?: ChunkingFallbackReason;
+  parse_latency_ms?: number;
+  language_aware_attempt_latency_ms?: number;
+  fallback_path_latency_ms?: number;
+  language?: string;
+  recursive_semantic_chunking_used?: boolean;
+}
+
+export type ParserLanguage = "typescript" | "tsx" | "javascript" | "jsx" | "python" | "go" | "rust" | "java";
+
+type ParserAvailabilityStatus = "available" | "unavailable";
+
+interface ParserAvailability {
+  status: ParserAvailabilityStatus;
+  parser?: Parser;
+  error?: string;
+}
+
+export interface ChunkingParserAvailabilitySnapshotEntry {
+  language: string;
+  status: ParserAvailabilityStatus;
+  error?: string;
+}
+
+const DEFAULT_BOUNDARY_NODE_TYPES_LEGACY: Record<ParserLanguage, Set<string>> = {
+  typescript: new Set([
+    "function_declaration",
+    "generator_function_declaration",
+    "class_declaration",
+    "interface_declaration",
+    "enum_declaration",
+    "type_alias_declaration"
+  ]),
+  tsx: new Set([
+    "function_declaration",
+    "generator_function_declaration",
+    "class_declaration",
+    "interface_declaration",
+    "enum_declaration",
+    "type_alias_declaration"
+  ]),
+  javascript: new Set([
+    "function_declaration",
+    "generator_function_declaration",
+    "class_declaration",
+    "function_expression",
+    "arrow_function"
+  ]),
+  jsx: new Set([
+    "function_declaration",
+    "generator_function_declaration",
+    "class_declaration",
+    "function_expression",
+    "arrow_function"
+  ]),
+  python: new Set(["function_definition", "class_definition"]),
+  go: new Set(["function_declaration", "method_declaration", "type_declaration"]),
+  rust: new Set(["function_item", "impl_item", "trait_item", "struct_item", "enum_item"]),
+  java: new Set(["class_declaration", "interface_declaration", "enum_declaration", "method_declaration"])
+};
+const DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS: Record<ParserLanguage, Set<string>> = {
+  ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY,
+  typescript: new Set([
+    ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.typescript,
+    "function_expression",
+    "arrow_function",
+    "method_definition",
+    "class"
+  ]),
+  tsx: new Set([
+    ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.tsx,
+    "function_expression",
+    "arrow_function",
+    "method_definition",
+    "class"
+  ]),
+  javascript: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.javascript, "method_definition", "class"]),
+  jsx: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.jsx, "method_definition", "class"])
+};
+
+const parserAvailabilityCache = new Map<ParserLanguage, ParserAvailability>();
+const parserInitAttempts = new Map<ParserLanguage, number>();
+const parserLanguageLoaderOverrides = new Map<ParserLanguage, () => Parser.Language>();
+const CANONICAL_TO_PARSER_LANGUAGE: Record<string, ParserLanguage> = {
+  typescript: "typescript",
+  javascript: "javascript",
+  python: "python",
+  go: "go",
+  rust: "rust",
+  java: "java"
+};
+const JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES = new Set([
+  "assignment_expression",
+  "variable_declarator",
+  "pair",
+  "export_statement",
+  "public_field_definition",
+  "property_definition"
+]);
+const JAVASCRIPT_EXPRESSION_BOUNDARY_NODE_TYPES = new Set(["function_expression", "arrow_function", "class"]);
+const JAVASCRIPT_EXPRESSION_BOUNDARY_WRAPPER_TYPES = new Set([
+  "parenthesized_expression",
+  "as_expression",
+  "satisfies_expression",
+  "type_assertion",
+  "non_null_expression"
+]);
+const SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER = 1.35;
+
+function parserLanguageToCanonical(language: ParserLanguage): string {
+  if (language === "tsx") {
+    return "typescript";
+  }
+  if (language === "jsx") {
+    return "javascript";
+  }
+  return language;
+}
+
+function normalizeLanguageAlias(language: string): ParserLanguage | undefined {
+  const normalized = language.trim().toLowerCase();
+  if (normalized === "typescript" || normalized === "ts") {
+    return "typescript";
+  }
+  if (normalized === "tsx") {
+    return "tsx";
+  }
+  if (normalized === "javascript" || normalized === "js") {
+    return "javascript";
+  }
+  if (normalized === "jsx") {
+    return "jsx";
+  }
+  if (normalized === "python" || normalized === "py") {
+    return "python";
+  }
+  if (normalized === "go" || normalized === "golang") {
+    return "go";
+  }
+  if (normalized === "rust" || normalized === "rs") {
+    return "rust";
+  }
+  if (normalized === "java") {
+    return "java";
+  }
+  return undefined;
+}
+
+function parserLanguageFromPath(path: string): ParserLanguage | undefined {
+  const normalized = path.toLowerCase();
+  if (normalized.endsWith(".tsx")) {
+    return "tsx";
+  }
+  if (normalized.endsWith(".ts") || normalized.endsWith(".mts") || normalized.endsWith(".cts")) {
+    return "typescript";
+  }
+  if (normalized.endsWith(".jsx")) {
+    return "jsx";
+  }
+  if (normalized.endsWith(".js") || normalized.endsWith(".mjs") || normalized.endsWith(".cjs")) {
+    return "javascript";
+  }
+  if (normalized.endsWith(".py")) {
+    return "python";
+  }
+  if (normalized.endsWith(".go")) {
+    return "go";
+  }
+  if (normalized.endsWith(".rs")) {
+    return "rust";
+  }
+  if (normalized.endsWith(".java")) {
+    return "java";
+  }
+  return undefined;
+}
+
+function resolveParserLanguage(file: ChunkingRawFile): ParserLanguage | undefined {
+  const explicit = file.language ? normalizeLanguageAlias(file.language) : undefined;
+  if (explicit) {
+    return explicit;
+  }
+  return parserLanguageFromPath(file.path);
+}
+
+function formatErrorMessage(error: unknown): string {
+  if (error instanceof Error && error.message) {
+    return error.message;
+  }
+  return String(error);
+}
+
+function resolveTreeSitterLanguageHandle(moduleValue: unknown): Parser.Language {
+  if (moduleValue && typeof moduleValue === "object" && "language" in moduleValue) {
+    return (moduleValue as { language: unknown }).language as Parser.Language;
+  }
+  return moduleValue as Parser.Language;
+}
+
+function loadParserLanguage(language: ParserLanguage): Parser.Language {
+  const overrideLoader = parserLanguageLoaderOverrides.get(language);
+  if (overrideLoader) {
+    return overrideLoader();
+  }
+  if (language === "typescript") {
+    return (TypeScript as unknown as { typescript: unknown }).typescript as Parser.Language;
+  }
+  if (language === "tsx") {
+    return (TypeScript as unknown as { tsx: unknown }).tsx as Parser.Language;
+  }
+  if (language === "javascript" || language === "jsx") {
+    // Bun is currently most reliable with tree-sitter 0.23-compatible JS/Python grammars.
+    return resolveTreeSitterLanguageHandle(JavaScriptV023);
+  }
+  if (language === "python") {
+    return resolveTreeSitterLanguageHandle(PythonV023);
+  }
+  if (language === "rust") {
+    return resolveTreeSitterLanguageHandle(Rust);
+  }
+  if (language === "java") {
+    return resolveTreeSitterLanguageHandle(Java);
+  }
+  return resolveTreeSitterLanguageHandle(Go);
+}
+
+function getParserAvailability(language: ParserLanguage): ParserAvailability {
+  const cached = parserAvailabilityCache.get(language);
+  if (cached) {
+    return cached;
+  }
+  parserInitAttempts.set(language, (parserInitAttempts.get(language) ?? 0) + 1);
+  try {
+    const parser = new Parser();
+    parser.setLanguage(loadParserLanguage(language));
+    const availability: ParserAvailability = {
+      status: "available",
+      parser
+    };
+    parserAvailabilityCache.set(language, availability);
+    return availability;
+  } catch (error) {
+    const availability: ParserAvailability = {
+      status: "unavailable",
+      error: formatErrorMessage(error)
+    };
+    parserAvailabilityCache.set(language, availability);
+    return availability;
+  }
+}
+
+function getParser(language: ParserLanguage): Parser | undefined {
+  const availability = getParserAvailability(language);
+  if (availability.status !== "available") {
+    return undefined;
+  }
+  return availability.parser;
+}
+
+function toChunkingParserLanguage(canonicalLanguage: string): ParserLanguage | undefined {
+  return CANONICAL_TO_PARSER_LANGUAGE[canonicalLanguage];
+}
+
+export function getChunkingParserAvailabilitySnapshot(input?: {
+  enabled_languages?: string[];
+}): ChunkingParserAvailabilitySnapshotEntry[] {
+  const canonicalLanguages = new Set<string>();
+  if (input?.enabled_languages && input.enabled_languages.length > 0) {
+    for (const language of input.enabled_languages) {
+      const parsed = normalizeLanguageAlias(language);
+      if (!parsed) {
+        continue;
+      }
+      const canonical = parserLanguageToCanonical(parsed);
+      if (toChunkingParserLanguage(canonical)) {
+        canonicalLanguages.add(canonical);
+      }
+    }
+  }
+  if (canonicalLanguages.size === 0) {
+    for (const canonical of Object.keys(CANONICAL_TO_PARSER_LANGUAGE)) {
+      canonicalLanguages.add(canonical);
+    }
+  }
+
+  return [...canonicalLanguages]
+    .sort((a, b) => a.localeCompare(b))
+    .map((canonicalLanguage) => {
+      const parserLanguage = toChunkingParserLanguage(canonicalLanguage);
+      if (!parserLanguage) {
+        return {
+          language: canonicalLanguage,
+          status: "unavailable" as const,
+          error: "no parser mapping for language"
+        };
+      }
+      const availability = getParserAvailability(parserLanguage);
+      return {
+        language: canonicalLanguage,
+        status: availability.status,
+        ...(availability.error ? { error: availability.error } : {})
+      };
+    });
+}
+
+export function __resetChunkingParserStateForTests(): void {
+  parserAvailabilityCache.clear();
+  parserInitAttempts.clear();
+  parserLanguageLoaderOverrides.clear();
+}
+
+export function __setChunkingParserLanguageLoaderForTests(
+  language: ParserLanguage,
+  loader: (() => Parser.Language) | undefined
+): void {
+  if (loader) {
+    parserLanguageLoaderOverrides.set(language, loader);
+  } else {
+    parserLanguageLoaderOverrides.delete(language);
+  }
+  parserAvailabilityCache.delete(language);
+}
+
+export function __getChunkingParserInitAttemptsForTests(): Record<string, number> {
+  return Object.fromEntries(parserInitAttempts.entries());
+}
+
+function toInclusiveEndRow(node: Parser.SyntaxNode): number {
+  const end = node.endPosition;
+  if (end.column === 0 && end.row > node.startPosition.row) {
+    return end.row - 1;
+  }
+  return end.row;
+}
+
+function trimLineRange(lines: string[], startRow: number, endRow: number): { start: number; end: number } | undefined {
+  let start = startRow;
+  let end = endRow;
+  while (start <= end && (lines[start] ?? "").trim().length === 0) {
+    start += 1;
+  }
+  while (end >= start && (lines[end] ?? "").trim().length === 0) {
+    end -= 1;
+  }
+  if (end < start) {
+    return undefined;
+  }
+  return { start, end };
+}
+
+function splitRangeWithBudget(input: {
+  lines: string[];
+  lineTokenCounts: number[];
+  startRow: number;
+  endRow: number;
+  targetChunkTokens: number;
+  overlapTokens: number;
+  maxChunks: number;
+  preferSafeBoundarySplit?: boolean;
+  softMaxChunkTokens?: number;
+}): Array<{ startRow: number; endRow: number }> {
+  const rangeTokenCount = (startRow: number, endRow: number): number => {
+    let total = 0;
+    for (let row = startRow; row <= endRow; row += 1) {
+      total += input.lineTokenCounts[row] ?? 0;
+    }
+    return total;
+  };
+  const isSafeSplitBoundaryLine = (line: string): boolean => {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      return true;
+    }
+    return trimmed.endsWith(";") || trimmed.endsWith("}") || trimmed.endsWith("{");
+  };
+  const segments: Array<{ startRow: number; endRow: number }> = [];
+  let start = input.startRow;
+
+  while (start <= input.endRow && segments.length < input.maxChunks) {
+    let tokens = 0;
+    let end = start - 1;
+    while (end < input.endRow) {
+      const nextEnd = end + 1;
+      tokens += input.lineTokenCounts[nextEnd] ?? 0;
+      end = nextEnd;
+      if (tokens >= input.targetChunkTokens && end >= start) {
+        break;
+      }
+    }
+
+    let safeEnd = Math.min(Math.max(start, end), input.endRow);
+    if (input.preferSafeBoundarySplit && safeEnd > start) {
+      let adjusted = safeEnd;
+      for (let row = safeEnd; row > start; row -= 1) {
+        if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+          adjusted = row;
+          break;
+        }
+      }
+      if (adjusted === safeEnd && typeof input.softMaxChunkTokens === "number" && input.softMaxChunkTokens > input.targetChunkTokens) {
+        for (let row = safeEnd + 1; row <= input.endRow; row += 1) {
+          if (rangeTokenCount(start, row) > input.softMaxChunkTokens) {
+            break;
+          }
+          if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+            adjusted = row;
+            break;
+          }
+        }
+      }
+      safeEnd = Math.max(start, adjusted);
+    }
+    if (safeEnd >= start) {
+      segments.push({ startRow: start, endRow: safeEnd });
+    }
+
+    if (safeEnd >= input.endRow) {
+      break;
+    }
+
+    let nextStart = safeEnd + 1;
+    if (input.overlapTokens > 0) {
+      let overlap = 0;
+      let cursor = safeEnd;
+      while (cursor >= start && overlap < input.overlapTokens) {
+        overlap += input.lineTokenCounts[cursor] ?? 0;
+        cursor -= 1;
+      }
+      nextStart = Math.max(start + 1, cursor + 1);
+    }
+    start = Math.max(start + 1, nextStart);
+  }
+
+  return segments;
+}
+
+function buildSlidingChunks(input: {
+  lines: string[];
+  tokenize: (text: string) => string[];
+  targetChunkTokens: number;
+  overlapTokens: number;
+  maxChunks: number;
+  lineTokenCounts?: number[];
+}): ChunkingOutput[] {
+  const lineTokenCounts = input.lineTokenCounts ?? computeLineTokenCounts(input.lines, input.tokenize);
+  const rawSegments = splitRangeWithBudget({
+    lines: input.lines,
+    lineTokenCounts,
+    startRow: 0,
+    endRow: Math.max(0, input.lines.length - 1),
+    targetChunkTokens: input.targetChunkTokens,
+    overlapTokens: input.overlapTokens,
+    maxChunks: input.maxChunks
+  });
+  const chunks: ChunkingOutput[] = [];
+  for (const segment of rawSegments) {
+    const trimmed = trimLineRange(input.lines, segment.startRow, segment.endRow);
+    if (!trimmed) {
+      continue;
+    }
+    chunks.push({
+      start_line: trimmed.start + 1,
+      end_line: trimmed.end + 1,
+      snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+    });
+  }
+  return chunks;
+}
+
+function hasBoundaryAncestor(node: Parser.SyntaxNode, boundaryTypes: Set<string>): boolean {
+  let current = node.parent;
+  while (current) {
+    if (boundaryTypes.has(current.type)) {
+      return true;
+    }
+    current = current.parent;
+  }
+  return false;
+}
+
+function getBoundaryTypes(
+  parserLanguage: ParserLanguage,
+  boundaryStrictness: "legacy" | "semantic_js_ts"
+): Set<string> {
+  if (boundaryStrictness === "semantic_js_ts") {
+    return DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS[parserLanguage];
+  }
+  return DEFAULT_BOUNDARY_NODE_TYPES_LEGACY[parserLanguage];
+}
+
+function isExpressionBoundaryLanguage(parserLanguage: ParserLanguage, boundaryStrictness: "legacy" | "semantic_js_ts"): boolean {
+  if (boundaryStrictness === "semantic_js_ts") {
+    return (
+      parserLanguage === "javascript" ||
+      parserLanguage === "jsx" ||
+      parserLanguage === "typescript" ||
+      parserLanguage === "tsx"
+    );
+  }
+  return parserLanguage === "javascript" || parserLanguage === "jsx";
+}
+
+function isLanguageBoundaryCandidate(
+  parserLanguage: ParserLanguage,
+  node: Parser.SyntaxNode,
+  boundaryStrictness: "legacy" | "semantic_js_ts"
+): boolean {
+  if (!isExpressionBoundaryLanguage(parserLanguage, boundaryStrictness)) {
+    return true;
+  }
+  if (!JAVASCRIPT_EXPRESSION_BOUNDARY_NODE_TYPES.has(node.type)) {
+    return true;
+  }
+  let owner = node.parent;
+  while (owner && JAVASCRIPT_EXPRESSION_BOUNDARY_WRAPPER_TYPES.has(owner.type)) {
+    owner = owner.parent;
+  }
+  const ownerType = owner?.type;
+  if (!ownerType) {
+    return false;
+  }
+  return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(ownerType);
+}
+
+export function __isChunkingBoundaryCandidateForTests(input: {
+  parserLanguage: ParserLanguage;
+  nodeType: string;
+  parentType?: string;
+  ancestorTypes?: string[];
+  boundaryStrictness?: "legacy" | "semantic_js_ts";
+}): boolean {
+  const strictness = input.boundaryStrictness ?? "legacy";
+  if (!isExpressionBoundaryLanguage(input.parserLanguage, strictness)) {
+    return true;
+  }
+  if (!JAVASCRIPT_EXPRESSION_BOUNDARY_NODE_TYPES.has(input.nodeType)) {
+    return true;
+  }
+  const ancestorChain: string[] = [];
+  if (input.parentType) {
+    ancestorChain.push(input.parentType);
+  }
+  if (input.ancestorTypes && input.ancestorTypes.length > 0) {
+    ancestorChain.push(...input.ancestorTypes);
+  }
+  const ownerType = ancestorChain.find((type) => !JAVASCRIPT_EXPRESSION_BOUNDARY_WRAPPER_TYPES.has(type));
+  if (!ownerType) {
+    return false;
+  }
+  return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(ownerType);
+}
+
+function computeLineTokenCounts(lines: string[], tokenize: (text: string) => string[]): number[] {
+  return lines.map((line) => tokenize(line ?? "").length);
+}
+
+function rangeTokenCount(lineTokenCounts: number[], startRow: number, endRow: number): number {
+  let total = 0;
+  for (let row = startRow; row <= endRow; row += 1) {
+    total += lineTokenCounts[row] ?? 0;
+  }
+  return total;
+}
+
+function listNamedChildren(node: Parser.SyntaxNode): Parser.SyntaxNode[] {
+  const children: Parser.SyntaxNode[] = [];
+  for (let index = 0; index < node.namedChildCount; index += 1) {
+    const child = node.namedChild(index);
+    if (child) {
+      children.push(child);
+    }
+  }
+  return children;
+}
+
+function normalizeNodeWindow(input: {
+  node: Parser.SyntaxNode;
+  lines: string[];
+  lastRow: number;
+}): { startRow: number; endRow: number } | undefined {
+  const startRow = Math.max(0, Math.min(input.lastRow, input.node.startPosition.row));
+  const endRow = Math.max(startRow, Math.min(input.lastRow, toInclusiveEndRow(input.node)));
+  const trimmed = trimLineRange(input.lines, startRow, endRow);
+  if (!trimmed) {
+    return undefined;
+  }
+  return {
+    startRow: trimmed.start,
+    endRow: trimmed.end
+  };
+}
+
+function buildRecursiveSemanticWindows(input: {
+  root: Parser.SyntaxNode;
+  lines: string[];
+  lineTokenCounts: number[];
+  targetChunkTokens: number;
+  maxChunks: number;
+  boundaryStrictness: "legacy" | "semantic_js_ts";
+}): Array<{ startRow: number; endRow: number }> {
+  const lastRow = Math.max(0, input.lines.length - 1);
+  const windows: Array<{ startRow: number; endRow: number }> = [];
+  const softMaxChunkTokens = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+  const seen = new Set<string>();
+
+  const pushSplitWindows = (startRow: number, endRow: number): void => {
+    if (startRow > endRow || windows.length >= input.maxChunks) {
+      return;
+    }
+    const segments = splitRangeWithBudget({
+      lines: input.lines,
+      lineTokenCounts: input.lineTokenCounts,
+      startRow,
+      endRow,
+      targetChunkTokens: input.targetChunkTokens,
+      overlapTokens: 0,
+      maxChunks: input.maxChunks - windows.length,
+      preferSafeBoundarySplit: input.boundaryStrictness === "semantic_js_ts",
+      softMaxChunkTokens
+    });
+    for (const segment of segments) {
+      const trimmed = trimLineRange(input.lines, segment.startRow, segment.endRow);
+      if (!trimmed) {
+        continue;
+      }
+      const key = `${trimmed.start}:${trimmed.end}`;
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+      windows.push({ startRow: trimmed.start, endRow: trimmed.end });
+      if (windows.length >= input.maxChunks) {
+        return;
+      }
+    }
+  };
+
+  const visitNode = (node: Parser.SyntaxNode): void => {
+    if (windows.length >= input.maxChunks) {
+      return;
+    }
+    const range = normalizeNodeWindow({
+      node,
+      lines: input.lines,
+      lastRow
+    });
+    if (!range) {
+      return;
+    }
+    const tokenCount = rangeTokenCount(input.lineTokenCounts, range.startRow, range.endRow);
+    if (tokenCount <= input.targetChunkTokens) {
+      const key = `${range.startRow}:${range.endRow}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        windows.push(range);
+      }
+      return;
+    }
+
+    const children = listNamedChildren(node)
+      .map((child) => ({
+        node: child,
+        range: normalizeNodeWindow({
+          node: child,
+          lines: input.lines,
+          lastRow
+        })
+      }))
+      .filter((child): child is { node: Parser.SyntaxNode; range: { startRow: number; endRow: number } } =>
+        Boolean(child.range)
+      )
+      .sort((a, b) => a.range.startRow - b.range.startRow || a.range.endRow - b.range.endRow);
+
+    if (children.length === 0) {
+      pushSplitWindows(range.startRow, range.endRow);
+      return;
+    }
+
+    let cursor = range.startRow;
+    for (const child of children) {
+      if (windows.length >= input.maxChunks) {
+        return;
+      }
+      if (child.range.endRow < cursor) {
+        continue;
+      }
+      if (child.range.startRow > cursor) {
+        pushSplitWindows(cursor, child.range.startRow - 1);
+      }
+
+      visitNode(child.node);
+      cursor = Math.max(cursor, child.range.endRow + 1);
+      if (cursor > range.endRow) {
+        return;
+      }
+    }
+    if (cursor <= range.endRow) {
+      pushSplitWindows(cursor, range.endRow);
+    }
+  };
+
+  visitNode(input.root);
+  return windows.sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+}
+
+function mergeSemanticWindows(input: {
+  windows: Array<{ startRow: number; endRow: number }>;
+  lineTokenCounts: number[];
+  targetChunkTokens: number;
+  semanticMergeGapLines: number;
+  semanticMergeMaxSpanLines: number;
+}): Array<{ startRow: number; endRow: number }> {
+  if (input.windows.length <= 1) {
+    return [...input.windows];
+  }
+  const ordered = [...input.windows].sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+  const merged: Array<{ startRow: number; endRow: number }> = [];
+  const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+  for (const window of ordered) {
+    const last = merged[merged.length - 1];
+    if (!last) {
+      merged.push({ ...window });
+      continue;
+    }
+    const gapLines = Math.max(0, window.startRow - last.endRow - 1);
+    const nextStartRow = Math.min(last.startRow, window.startRow);
+    const nextEndRow = Math.max(last.endRow, window.endRow);
+    const nextSpanLines = nextEndRow - nextStartRow + 1;
+    const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, nextStartRow, nextEndRow);
+    const canMerge =
+      gapLines <= input.semanticMergeGapLines &&
+      nextSpanLines <= input.semanticMergeMaxSpanLines &&
+      mergedTokenCount <= mergeTokenBudget;
+    if (!canMerge) {
+      merged.push({ ...window });
+      continue;
+    }
+    last.startRow = nextStartRow;
+    last.endRow = nextEndRow;
+  }
+  return merged;
+}
+
+function isCommentOnlyLine(line: string): boolean {
+  const trimmed = line.trim();
+  if (trimmed.length === 0) {
+    return true;
+  }
+  return (
+    trimmed.startsWith("//") ||
+    trimmed.startsWith("/*") ||
+    trimmed.startsWith("*") ||
+    trimmed.startsWith("*/") ||
+    trimmed.startsWith("#")
+  );
+}
+
+function windowLooksCommentOnly(input: { lines: string[]; startRow: number; endRow: number }): boolean {
+  for (let row = input.startRow; row <= input.endRow; row += 1) {
+    if (!isCommentOnlyLine(input.lines[row] ?? "")) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function absorbForwardCommentWindows(input: {
+  windows: Array<{ startRow: number; endRow: number }>;
+  lines: string[];
+  lineTokenCounts: number[];
+  targetChunkTokens: number;
+  semanticMergeMaxSpanLines: number;
+}): Array<{ startRow: number; endRow: number }> {
+  if (input.windows.length <= 1) {
+    return [...input.windows];
+  }
+  const output: Array<{ startRow: number; endRow: number }> = [];
+  const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+  for (let index = 0; index < input.windows.length; index += 1) {
+    const current = input.windows[index];
+    const next = input.windows[index + 1];
+    if (!current) {
+      continue;
+    }
+    if (!next) {
+      output.push({ ...current });
+      continue;
+    }
+    if (!windowLooksCommentOnly({ lines: input.lines, startRow: current.startRow, endRow: current.endRow })) {
+      output.push({ ...current });
+      continue;
+    }
+    const gapLines = Math.max(0, next.startRow - current.endRow - 1);
+    const nextSpanLines = next.endRow - current.startRow + 1;
+    const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, current.startRow, next.endRow);
+    const canAbsorb =
+      gapLines <= 1 && nextSpanLines <= input.semanticMergeMaxSpanLines && mergedTokenCount <= mergeTokenBudget;
+    if (!canAbsorb) {
+      output.push({ ...current });
+      continue;
+    }
+    output.push({
+      startRow: current.startRow,
+      endRow: next.endRow
+    });
+    index += 1;
+  }
+  return output;
+}
+
+function windowsToChunks(input: {
+  windows: Array<{ startRow: number; endRow: number }>;
+  lines: string[];
+  maxChunks: number;
+}): ChunkingOutput[] {
+  const chunks: ChunkingOutput[] = [];
+  for (const window of input.windows) {
+    if (chunks.length >= input.maxChunks) {
+      break;
+    }
+    const trimmed = trimLineRange(input.lines, window.startRow, window.endRow);
+    if (!trimmed) {
+      continue;
+    }
+    chunks.push({
+      start_line: trimmed.start + 1,
+      end_line: trimmed.end + 1,
+      snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+    });
+  }
+  return chunks;
+}
+
+function buildLanguageAwareChunks(input: {
+  file: ChunkingRawFile;
+  lines: string[];
+  parserLanguage: ParserLanguage;
+  config: ChunkingConfig;
+  tokenize: (text: string) => string[];
+}): ChunkingResult {
+  const languageAwareAttemptStart = Date.now();
+  const lineTokenCounts = computeLineTokenCounts(input.lines, input.tokenize);
+  const parser = getParser(input.parserLanguage);
+  if (!parser) {
+    const fallbackStart = Date.now();
+    const chunks = buildSlidingChunks({
+      lines: input.lines,
+      tokenize: input.tokenize,
+      targetChunkTokens: input.config.target_chunk_tokens,
+      overlapTokens: input.config.chunk_overlap_tokens,
+      maxChunks: input.config.max_chunks_per_file,
+      lineTokenCounts
+    });
+    return {
+      chunks,
+      strategy: "sliding",
+      fallback_reason: "parser_unavailable",
+      language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+      fallback_path_latency_ms: Date.now() - fallbackStart,
+      language: parserLanguageToCanonical(input.parserLanguage)
+    };
+  }
+
+  try {
+    parser.setTimeoutMicros(input.config.parse_timeout_ms * 1_000);
+    const parseStart = Date.now();
+    const tree = parser.parse(input.file.content);
+    const parseLatencyMs = Date.now() - parseStart;
+    if (parseLatencyMs > input.config.parse_timeout_ms) {
+      const fallbackStart = Date.now();
+      const chunks = buildSlidingChunks({
+        lines: input.lines,
+        tokenize: input.tokenize,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        overlapTokens: input.config.chunk_overlap_tokens,
+        maxChunks: input.config.max_chunks_per_file,
+        lineTokenCounts
+      });
+      return {
+        chunks,
+        strategy: "sliding",
+        fallback_reason: "parse_timeout_exceeded",
+        parse_latency_ms: parseLatencyMs,
+        language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+        fallback_path_latency_ms: Date.now() - fallbackStart,
+        language: parserLanguageToCanonical(input.parserLanguage)
+      };
+    }
+
+    const root = (tree as Parser.Tree | null)?.rootNode;
+    if (!root) {
+      const fallbackStart = Date.now();
+      const chunks = buildSlidingChunks({
+        lines: input.lines,
+        tokenize: input.tokenize,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        overlapTokens: input.config.chunk_overlap_tokens,
+        maxChunks: input.config.max_chunks_per_file,
+        lineTokenCounts
+      });
+      return {
+        chunks,
+        strategy: "sliding",
+        fallback_reason: "parse_error",
+        parse_latency_ms: parseLatencyMs,
+        language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+        fallback_path_latency_ms: Date.now() - fallbackStart,
+        language: parserLanguageToCanonical(input.parserLanguage)
+      };
+    }
+
+    let chunks: ChunkingOutput[] = [];
+    let recursiveSemanticChunkingUsed = false;
+
+    if (input.config.recursive_semantic_chunking_enabled) {
+      const semanticMergeGapLines = input.config.semantic_merge_gap_lines ?? 6;
+      const semanticMergeMaxSpanLines = input.config.semantic_merge_max_span_lines ?? 220;
+      const recursiveWindows = buildRecursiveSemanticWindows({
+        root,
+        lines: input.lines,
+        lineTokenCounts,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        maxChunks: input.config.max_chunks_per_file,
+        boundaryStrictness: input.config.boundary_strictness
+      });
+      const mergedWindows = mergeSemanticWindows({
+        windows: recursiveWindows,
+        lineTokenCounts,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        semanticMergeGapLines,
+        semanticMergeMaxSpanLines
+      });
+      const absorbedWindows =
+        input.config.comment_forward_absorb_enabled === false
+          ? mergedWindows
+          : absorbForwardCommentWindows({
+              windows: mergedWindows,
+              lines: input.lines,
+              lineTokenCounts,
+              targetChunkTokens: input.config.target_chunk_tokens,
+              semanticMergeMaxSpanLines
+            });
+      chunks = windowsToChunks({
+        windows: absorbedWindows,
+        lines: input.lines,
+        maxChunks: input.config.max_chunks_per_file
+      });
+      recursiveSemanticChunkingUsed = chunks.length > 0;
+    } else {
+      const boundaryTypes = getBoundaryTypes(input.parserLanguage, input.config.boundary_strictness);
+      const candidates = root.descendantsOfType([...boundaryTypes]);
+      const boundaryNodes = candidates
+        .filter((node) => !hasBoundaryAncestor(node, boundaryTypes))
+        .filter((node) => isLanguageBoundaryCandidate(input.parserLanguage, node, input.config.boundary_strictness))
+        .sort((a, b) => a.startPosition.row - b.startPosition.row || a.startPosition.column - b.startPosition.column);
+
+      if (boundaryNodes.length === 0) {
+        const fallbackStart = Date.now();
+        const fallbackChunks = buildSlidingChunks({
+          lines: input.lines,
+          tokenize: input.tokenize,
+          targetChunkTokens: input.config.target_chunk_tokens,
+          overlapTokens: input.config.chunk_overlap_tokens,
+          maxChunks: input.config.max_chunks_per_file,
+          lineTokenCounts
+        });
+        return {
+          chunks: fallbackChunks,
+          strategy: "sliding",
+          fallback_reason: "empty_language_boundaries",
+          parse_latency_ms: parseLatencyMs,
+          language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+          fallback_path_latency_ms: Date.now() - fallbackStart,
+          language: parserLanguageToCanonical(input.parserLanguage)
+        };
+      }
+
+      const segments: Array<{ startRow: number; endRow: number; boundary: boolean }> = [];
+      let cursor = 0;
+      const lastRow = Math.max(0, input.lines.length - 1);
+      for (const node of boundaryNodes) {
+        const startRow = Math.max(0, Math.min(lastRow, node.startPosition.row));
+        const endRow = Math.max(startRow, Math.min(lastRow, toInclusiveEndRow(node)));
+        if (startRow > cursor) {
+          segments.push({ startRow: cursor, endRow: startRow - 1, boundary: false });
+        }
+        segments.push({ startRow, endRow, boundary: true });
+        cursor = endRow + 1;
+        if (cursor > lastRow) {
+          break;
+        }
+      }
+      if (cursor <= lastRow) {
+        segments.push({ startRow: cursor, endRow: lastRow, boundary: false });
+      }
+
+      for (const segment of segments) {
+        if (segment.endRow < segment.startRow || chunks.length >= input.config.max_chunks_per_file) {
+          continue;
+        }
+        const segmentTokenCount = lineTokenCounts
+          .slice(segment.startRow, segment.endRow + 1)
+          .reduce((sum, value) => sum + value, 0);
+        const enableSemanticBoundarySplits =
+          input.config.boundary_strictness === "semantic_js_ts" &&
+          (input.parserLanguage === "javascript" ||
+            input.parserLanguage === "jsx" ||
+            input.parserLanguage === "typescript" ||
+            input.parserLanguage === "tsx") &&
+          segment.boundary;
+        const softMaxChunkTokens = Math.floor(input.config.target_chunk_tokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+        const pieces =
+          enableSemanticBoundarySplits && segmentTokenCount <= softMaxChunkTokens
+            ? [{ startRow: segment.startRow, endRow: segment.endRow }]
+            : splitRangeWithBudget({
+                lines: input.lines,
+                lineTokenCounts,
+                startRow: segment.startRow,
+                endRow: segment.endRow,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file - chunks.length,
+                preferSafeBoundarySplit: enableSemanticBoundarySplits,
+                softMaxChunkTokens
+              });
+        for (const piece of pieces) {
+          const trimmed = trimLineRange(input.lines, piece.startRow, piece.endRow);
+          if (!trimmed) {
+            continue;
+          }
+          chunks.push({
+            start_line: trimmed.start + 1,
+            end_line: trimmed.end + 1,
+            snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+          });
+          if (chunks.length >= input.config.max_chunks_per_file) {
+            break;
+          }
+        }
+      }
+    }
+
+    if (chunks.length === 0) {
+      const fallbackStart = Date.now();
+      const slidingChunks = buildSlidingChunks({
+        lines: input.lines,
+        tokenize: input.tokenize,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        overlapTokens: input.config.chunk_overlap_tokens,
+        maxChunks: input.config.max_chunks_per_file,
+        lineTokenCounts
+      });
+      return {
+        chunks: slidingChunks,
+        strategy: "sliding",
+        fallback_reason: "empty_language_boundaries",
+        parse_latency_ms: parseLatencyMs,
+        language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+        fallback_path_latency_ms: Date.now() - fallbackStart,
+        language: parserLanguageToCanonical(input.parserLanguage)
+      };
+    }
+
+    return {
+      chunks,
+      strategy: "language_aware",
+      parse_latency_ms: parseLatencyMs,
+      language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+      language: parserLanguageToCanonical(input.parserLanguage),
+      recursive_semantic_chunking_used: recursiveSemanticChunkingUsed
+    };
+  } catch {
+    const fallbackStart = Date.now();
+    const chunks = buildSlidingChunks({
+      lines: input.lines,
+      tokenize: input.tokenize,
+      targetChunkTokens: input.config.target_chunk_tokens,
+      overlapTokens: input.config.chunk_overlap_tokens,
+      maxChunks: input.config.max_chunks_per_file,
+      lineTokenCounts
+    });
+    return {
+      chunks,
+      strategy: "sliding",
+      fallback_reason: "parse_error",
+      language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+      fallback_path_latency_ms: Date.now() - fallbackStart,
+      language: parserLanguageToCanonical(input.parserLanguage)
+    };
+  }
+}
+
+export function buildChunksForFile(input: {
+  file: ChunkingRawFile;
+  config: ChunkingConfig;
+  tokenize: (text: string) => string[];
+}): ChunkingResult {
+  const lines = input.file.content.split("\n");
+  const language = resolveParserLanguage(input.file);
+  const enabledLanguageSet = new Set(input.config.enabled_languages.map((value) => value.trim().toLowerCase()));
+
+  if (input.config.strategy === "sliding") {
+    return {
+      chunks: buildSlidingChunks({
+        lines,
+        tokenize: input.tokenize,
+        targetChunkTokens: input.config.target_chunk_tokens,
+        overlapTokens: input.config.chunk_overlap_tokens,
+        maxChunks: input.config.max_chunks_per_file
+      }),
+      strategy: "sliding",
+      language: language ? parserLanguageToCanonical(language) : undefined
+    };
+  }
+
+  if (!language || !enabledLanguageSet.has(parserLanguageToCanonical(language))) {
+    const fallbackStart = Date.now();
+    const chunks = buildSlidingChunks({
+      lines,
+      tokenize: input.tokenize,
+      targetChunkTokens: input.config.target_chunk_tokens,
+      overlapTokens: input.config.chunk_overlap_tokens,
+      maxChunks: input.config.max_chunks_per_file
+    });
+    return {
+      chunks,
+      strategy: "sliding",
+      fallback_reason: "unsupported_language",
+      fallback_path_latency_ms: Date.now() - fallbackStart
+    };
+  }
+
+  return buildLanguageAwareChunks({
+    file: input.file,
+    lines,
+    parserLanguage: language,
+    config: input.config,
+    tokenize: input.tokenize
+  });
+}