@rce-mcp/retrieval-core 0.1.2 → 0.1.3

package/dist/chunking.js

@@ -3,7 +3,7 @@ import Go from "tree-sitter-go";
 import JavaScriptV023 from "tree-sitter-javascript-v023";
 import PythonV023 from "tree-sitter-python-v023";
 import TypeScript from "tree-sitter-typescript";
-const DEFAULT_BOUNDARY_NODE_TYPES = {
+const DEFAULT_BOUNDARY_NODE_TYPES_LEGACY = {
     typescript: new Set([
         "function_declaration",
         "generator_function_declaration",
@@ -20,11 +20,35 @@ const DEFAULT_BOUNDARY_NODE_TYPES = {
         "enum_declaration",
         "type_alias_declaration"
     ]),
-    javascript: new Set(["function_declaration", "generator_function_declaration", "class_declaration"]),
-    jsx: new Set(["function_declaration", "generator_function_declaration", "class_declaration"]),
+    javascript: new Set([
+        "function_declaration",
+        "generator_function_declaration",
+        "class_declaration",
+        "function_expression",
+        "arrow_function"
+    ]),
+    jsx: new Set([
+        "function_declaration",
+        "generator_function_declaration",
+        "class_declaration",
+        "function_expression",
+        "arrow_function"
+    ]),
     python: new Set(["function_definition", "class_definition"]),
     go: new Set(["function_declaration", "method_declaration", "type_declaration"])
 };
+const DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS = {
+    ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY,
+    typescript: new Set([
+        ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.typescript,
+        "function_expression",
+        "arrow_function",
+        "method_definition"
+    ]),
+    tsx: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.tsx, "function_expression", "arrow_function", "method_definition"]),
+    javascript: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.javascript, "method_definition"]),
+    jsx: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.jsx, "method_definition"])
+};
 const parserAvailabilityCache = new Map();
 const parserInitAttempts = new Map();
 const parserLanguageLoaderOverrides = new Map();
@@ -34,6 +58,15 @@ const CANONICAL_TO_PARSER_LANGUAGE = {
     python: "python",
     go: "go"
 };
+const JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES = new Set([
+    "assignment_expression",
+    "variable_declarator",
+    "pair",
+    "export_statement",
+    "public_field_definition",
+    "property_definition"
+]);
+const SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER = 1.35;
 function parserLanguageToCanonical(language) {
     if (language === "tsx") {
         return "typescript";
@@ -70,7 +103,7 @@ function parserLanguageFromPath(path) {
     if (normalized.endsWith(".tsx")) {
         return "tsx";
     }
-    if (normalized.endsWith(".ts")) {
+    if (normalized.endsWith(".ts") || normalized.endsWith(".mts") || normalized.endsWith(".cts")) {
         return "typescript";
     }
     if (normalized.endsWith(".jsx")) {
@@ -238,36 +271,82 @@ function trimLineRange(lines, startRow, endRow) {
     return { start, end };
 }
 function splitRangeWithBudget(input) {
+    const rangeTokenCount = (startRow, endRow) => {
+        let total = 0;
+        for (let row = startRow; row <= endRow; row += 1) {
+            total += input.lineTokenCounts[row] ?? 0;
+        }
+        return total;
+    };
+    const isSafeSplitBoundaryLine = (line) => {
+        const trimmed = line.trim();
+        if (trimmed.length === 0) {
+            return true;
+        }
+        return trimmed.endsWith(";") || trimmed.endsWith("}") || trimmed.endsWith("{");
+    };
     const segments = [];
     let start = input.startRow;
     while (start <= input.endRow && segments.length < input.maxChunks) {
         let tokens = 0;
-        let end = start;
-        while (end <= input.endRow) {
-            tokens += input.tokenize(input.lines[end] ?? "").length;
-            if (tokens >= input.targetChunkTokens) {
+        let end = start - 1;
+        while (end < input.endRow) {
+            const nextEnd = end + 1;
+            tokens += input.lineTokenCounts[nextEnd] ?? 0;
+            end = nextEnd;
+            if (tokens >= input.targetChunkTokens && end >= start) {
                 break;
             }
-            end += 1;
         }
-        const safeEnd = Math.min(end, input.endRow);
+        let safeEnd = Math.min(Math.max(start, end), input.endRow);
+        if (input.preferSafeBoundarySplit && safeEnd > start) {
+            let adjusted = safeEnd;
+            for (let row = safeEnd; row > start; row -= 1) {
+                if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+                    adjusted = row;
+                    break;
+                }
+            }
+            if (adjusted === safeEnd && typeof input.softMaxChunkTokens === "number" && input.softMaxChunkTokens > input.targetChunkTokens) {
+                for (let row = safeEnd + 1; row <= input.endRow; row += 1) {
+                    if (rangeTokenCount(start, row) > input.softMaxChunkTokens) {
+                        break;
+                    }
+                    if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+                        adjusted = row;
+                        break;
+                    }
+                }
+            }
+            safeEnd = Math.max(start, adjusted);
+        }
         if (safeEnd >= start) {
             segments.push({ startRow: start, endRow: safeEnd });
         }
         if (safeEnd >= input.endRow) {
             break;
         }
-        const rewind = Math.max(1, Math.floor(input.overlapTokens / 4));
-        start = Math.max(start + 1, safeEnd - rewind + 1);
+        let nextStart = safeEnd + 1;
+        if (input.overlapTokens > 0) {
+            let overlap = 0;
+            let cursor = safeEnd;
+            while (cursor >= start && overlap < input.overlapTokens) {
+                overlap += input.lineTokenCounts[cursor] ?? 0;
+                cursor -= 1;
+            }
+            nextStart = Math.max(start + 1, cursor + 1);
+        }
+        start = Math.max(start + 1, nextStart);
     }
     return segments;
 }
 function buildSlidingChunks(input) {
+    const lineTokenCounts = input.lineTokenCounts ?? computeLineTokenCounts(input.lines, input.tokenize);
     const rawSegments = splitRangeWithBudget({
         lines: input.lines,
+        lineTokenCounts,
         startRow: 0,
         endRow: Math.max(0, input.lines.length - 1),
-        tokenize: input.tokenize,
         targetChunkTokens: input.targetChunkTokens,
         overlapTokens: input.overlapTokens,
         maxChunks: input.maxChunks
@@ -296,8 +375,281 @@ function hasBoundaryAncestor(node, boundaryTypes) {
     }
     return false;
 }
+function getBoundaryTypes(parserLanguage, boundaryStrictness) {
+    if (boundaryStrictness === "semantic_js_ts") {
+        return DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS[parserLanguage];
+    }
+    return DEFAULT_BOUNDARY_NODE_TYPES_LEGACY[parserLanguage];
+}
+function isExpressionBoundaryLanguage(parserLanguage, boundaryStrictness) {
+    if (boundaryStrictness === "semantic_js_ts") {
+        return (parserLanguage === "javascript" ||
+            parserLanguage === "jsx" ||
+            parserLanguage === "typescript" ||
+            parserLanguage === "tsx");
+    }
+    return parserLanguage === "javascript" || parserLanguage === "jsx";
+}
+function isLanguageBoundaryCandidate(parserLanguage, node, boundaryStrictness) {
+    if (!isExpressionBoundaryLanguage(parserLanguage, boundaryStrictness)) {
+        return true;
+    }
+    if (node.type !== "function_expression" && node.type !== "arrow_function") {
+        return true;
+    }
+    const parentType = node.parent?.type;
+    if (!parentType) {
+        return false;
+    }
+    return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(parentType);
+}
+export function __isChunkingBoundaryCandidateForTests(input) {
+    const strictness = input.boundaryStrictness ?? "legacy";
+    if (!isExpressionBoundaryLanguage(input.parserLanguage, strictness)) {
+        return true;
+    }
+    if (input.nodeType !== "function_expression" && input.nodeType !== "arrow_function") {
+        return true;
+    }
+    if (!input.parentType) {
+        return false;
+    }
+    return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(input.parentType);
+}
+function computeLineTokenCounts(lines, tokenize) {
+    return lines.map((line) => tokenize(line ?? "").length);
+}
+function rangeTokenCount(lineTokenCounts, startRow, endRow) {
+    let total = 0;
+    for (let row = startRow; row <= endRow; row += 1) {
+        total += lineTokenCounts[row] ?? 0;
+    }
+    return total;
+}
+function listNamedChildren(node) {
+    const children = [];
+    for (let index = 0; index < node.namedChildCount; index += 1) {
+        const child = node.namedChild(index);
+        if (child) {
+            children.push(child);
+        }
+    }
+    return children;
+}
+function normalizeNodeWindow(input) {
+    const startRow = Math.max(0, Math.min(input.lastRow, input.node.startPosition.row));
+    const endRow = Math.max(startRow, Math.min(input.lastRow, toInclusiveEndRow(input.node)));
+    const trimmed = trimLineRange(input.lines, startRow, endRow);
+    if (!trimmed) {
+        return undefined;
+    }
+    return {
+        startRow: trimmed.start,
+        endRow: trimmed.end
+    };
+}
+function buildRecursiveSemanticWindows(input) {
+    const lastRow = Math.max(0, input.lines.length - 1);
+    const windows = [];
+    const softMaxChunkTokens = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+    const seen = new Set();
+    const pushSplitWindows = (startRow, endRow) => {
+        if (startRow > endRow || windows.length >= input.maxChunks) {
+            return;
+        }
+        const segments = splitRangeWithBudget({
+            lines: input.lines,
+            lineTokenCounts: input.lineTokenCounts,
+            startRow,
+            endRow,
+            targetChunkTokens: input.targetChunkTokens,
+            overlapTokens: 0,
+            maxChunks: input.maxChunks - windows.length,
+            preferSafeBoundarySplit: input.boundaryStrictness === "semantic_js_ts",
+            softMaxChunkTokens
+        });
+        for (const segment of segments) {
+            const trimmed = trimLineRange(input.lines, segment.startRow, segment.endRow);
+            if (!trimmed) {
+                continue;
+            }
+            const key = `${trimmed.start}:${trimmed.end}`;
+            if (seen.has(key)) {
+                continue;
+            }
+            seen.add(key);
+            windows.push({ startRow: trimmed.start, endRow: trimmed.end });
+            if (windows.length >= input.maxChunks) {
+                return;
+            }
+        }
+    };
+    const visitNode = (node) => {
+        if (windows.length >= input.maxChunks) {
+            return;
+        }
+        const range = normalizeNodeWindow({
+            node,
+            lines: input.lines,
+            lastRow
+        });
+        if (!range) {
+            return;
+        }
+        const tokenCount = rangeTokenCount(input.lineTokenCounts, range.startRow, range.endRow);
+        if (tokenCount <= input.targetChunkTokens) {
+            const key = `${range.startRow}:${range.endRow}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                windows.push(range);
+            }
+            return;
+        }
+        const children = listNamedChildren(node)
+            .map((child) => ({
+            node: child,
+            range: normalizeNodeWindow({
+                node: child,
+                lines: input.lines,
+                lastRow
+            })
+        }))
+            .filter((child) => Boolean(child.range))
+            .sort((a, b) => a.range.startRow - b.range.startRow || a.range.endRow - b.range.endRow);
+        if (children.length === 0) {
+            pushSplitWindows(range.startRow, range.endRow);
+            return;
+        }
+        let cursor = range.startRow;
+        for (const child of children) {
+            if (windows.length >= input.maxChunks) {
+                return;
+            }
+            if (child.range.endRow < cursor) {
+                continue;
+            }
+            if (child.range.startRow > cursor) {
+                pushSplitWindows(cursor, child.range.startRow - 1);
+            }
+            visitNode(child.node);
+            cursor = Math.max(cursor, child.range.endRow + 1);
+            if (cursor > range.endRow) {
+                return;
+            }
+        }
+        if (cursor <= range.endRow) {
+            pushSplitWindows(cursor, range.endRow);
+        }
+    };
+    visitNode(input.root);
+    return windows.sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+}
+function mergeSemanticWindows(input) {
+    if (input.windows.length <= 1) {
+        return [...input.windows];
+    }
+    const ordered = [...input.windows].sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+    const merged = [];
+    const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+    for (const window of ordered) {
+        const last = merged[merged.length - 1];
+        if (!last) {
+            merged.push({ ...window });
+            continue;
+        }
+        const gapLines = Math.max(0, window.startRow - last.endRow - 1);
+        const nextStartRow = Math.min(last.startRow, window.startRow);
+        const nextEndRow = Math.max(last.endRow, window.endRow);
+        const nextSpanLines = nextEndRow - nextStartRow + 1;
+        const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, nextStartRow, nextEndRow);
+        const canMerge = gapLines <= input.semanticMergeGapLines &&
+            nextSpanLines <= input.semanticMergeMaxSpanLines &&
+            mergedTokenCount <= mergeTokenBudget;
+        if (!canMerge) {
+            merged.push({ ...window });
+            continue;
+        }
+        last.startRow = nextStartRow;
+        last.endRow = nextEndRow;
+    }
+    return merged;
+}
+function isCommentOnlyLine(line) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+        return true;
+    }
+    return (trimmed.startsWith("//") ||
+        trimmed.startsWith("/*") ||
+        trimmed.startsWith("*") ||
+        trimmed.startsWith("*/") ||
+        trimmed.startsWith("#"));
+}
+function windowLooksCommentOnly(input) {
+    for (let row = input.startRow; row <= input.endRow; row += 1) {
+        if (!isCommentOnlyLine(input.lines[row] ?? "")) {
+            return false;
+        }
+    }
+    return true;
+}
+function absorbForwardCommentWindows(input) {
+    if (input.windows.length <= 1) {
+        return [...input.windows];
+    }
+    const output = [];
+    const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+    for (let index = 0; index < input.windows.length; index += 1) {
+        const current = input.windows[index];
+        const next = input.windows[index + 1];
+        if (!current) {
+            continue;
+        }
+        if (!next) {
+            output.push({ ...current });
+            continue;
+        }
+        if (!windowLooksCommentOnly({ lines: input.lines, startRow: current.startRow, endRow: current.endRow })) {
+            output.push({ ...current });
+            continue;
+        }
+        const gapLines = Math.max(0, next.startRow - current.endRow - 1);
+        const nextSpanLines = next.endRow - current.startRow + 1;
+        const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, current.startRow, next.endRow);
+        const canAbsorb = gapLines <= 1 && nextSpanLines <= input.semanticMergeMaxSpanLines && mergedTokenCount <= mergeTokenBudget;
+        if (!canAbsorb) {
+            output.push({ ...current });
+            continue;
+        }
+        output.push({
+            startRow: current.startRow,
+            endRow: next.endRow
+        });
+        index += 1;
+    }
+    return output;
+}
+function windowsToChunks(input) {
+    const chunks = [];
+    for (const window of input.windows) {
+        if (chunks.length >= input.maxChunks) {
+            break;
+        }
+        const trimmed = trimLineRange(input.lines, window.startRow, window.endRow);
+        if (!trimmed) {
+            continue;
+        }
+        chunks.push({
+            start_line: trimmed.start + 1,
+            end_line: trimmed.end + 1,
+            snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+        });
+    }
+    return chunks;
+}
 function buildLanguageAwareChunks(input) {
     const languageAwareAttemptStart = Date.now();
+    const lineTokenCounts = computeLineTokenCounts(input.lines, input.tokenize);
     const parser = getParser(input.parserLanguage);
     if (!parser) {
         const fallbackStart = Date.now();
@@ -306,7 +658,8 @@ function buildLanguageAwareChunks(input) {
             tokenize: input.tokenize,
             targetChunkTokens: input.config.target_chunk_tokens,
             overlapTokens: input.config.chunk_overlap_tokens,
-            maxChunks: input.config.max_chunks_per_file
+            maxChunks: input.config.max_chunks_per_file,
+            lineTokenCounts
         });
         return {
             chunks,
@@ -329,7 +682,8 @@ function buildLanguageAwareChunks(input) {
                 tokenize: input.tokenize,
                 targetChunkTokens: input.config.target_chunk_tokens,
                 overlapTokens: input.config.chunk_overlap_tokens,
-                maxChunks: input.config.max_chunks_per_file
+                maxChunks: input.config.max_chunks_per_file,
+                lineTokenCounts
             });
             return {
                 chunks,
@@ -349,7 +703,8 @@ function buildLanguageAwareChunks(input) {
                 tokenize: input.tokenize,
                 targetChunkTokens: input.config.target_chunk_tokens,
                 overlapTokens: input.config.chunk_overlap_tokens,
-                maxChunks: input.config.max_chunks_per_file
+                maxChunks: input.config.max_chunks_per_file,
+                lineTokenCounts
             });
             return {
                 chunks,
@@ -361,74 +716,127 @@ function buildLanguageAwareChunks(input) {
                 language: parserLanguageToCanonical(input.parserLanguage)
             };
         }
-        const boundaryTypes = DEFAULT_BOUNDARY_NODE_TYPES[input.parserLanguage];
-        const candidates = root.descendantsOfType([...boundaryTypes]);
-        const boundaryNodes = candidates
-            .filter((node) => !hasBoundaryAncestor(node, boundaryTypes))
-            .sort((a, b) => a.startPosition.row - b.startPosition.row || a.startPosition.column - b.startPosition.column);
-        if (boundaryNodes.length === 0) {
-            const fallbackStart = Date.now();
-            const chunks = buildSlidingChunks({
+        let chunks = [];
+        let recursiveSemanticChunkingUsed = false;
+        if (input.config.recursive_semantic_chunking_enabled) {
+            const semanticMergeGapLines = input.config.semantic_merge_gap_lines ?? 6;
+            const semanticMergeMaxSpanLines = input.config.semantic_merge_max_span_lines ?? 220;
+            const recursiveWindows = buildRecursiveSemanticWindows({
+                root,
                 lines: input.lines,
-                tokenize: input.tokenize,
+                lineTokenCounts,
                 targetChunkTokens: input.config.target_chunk_tokens,
-                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file,
+                boundaryStrictness: input.config.boundary_strictness
+            });
+            const mergedWindows = mergeSemanticWindows({
+                windows: recursiveWindows,
+                lineTokenCounts,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                semanticMergeGapLines,
+                semanticMergeMaxSpanLines
+            });
+            const absorbedWindows = input.config.comment_forward_absorb_enabled === false
+                ? mergedWindows
+                : absorbForwardCommentWindows({
+                    windows: mergedWindows,
+                    lines: input.lines,
+                    lineTokenCounts,
+                    targetChunkTokens: input.config.target_chunk_tokens,
+                    semanticMergeMaxSpanLines
+                });
+            chunks = windowsToChunks({
+                windows: absorbedWindows,
+                lines: input.lines,
                 maxChunks: input.config.max_chunks_per_file
             });
-            return {
-                chunks,
-                strategy: "sliding",
-                fallback_reason: "empty_language_boundaries",
-                parse_latency_ms: parseLatencyMs,
-                language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
-                fallback_path_latency_ms: Date.now() - fallbackStart,
-                language: parserLanguageToCanonical(input.parserLanguage)
-            };
+            recursiveSemanticChunkingUsed = chunks.length > 0;
         }
-        const segments = [];
-        let cursor = 0;
-        const lastRow = Math.max(0, input.lines.length - 1);
-        for (const node of boundaryNodes) {
-            const startRow = Math.max(0, Math.min(lastRow, node.startPosition.row));
-            const endRow = Math.max(startRow, Math.min(lastRow, toInclusiveEndRow(node)));
-            if (startRow > cursor) {
-                segments.push({ startRow: cursor, endRow: startRow - 1 });
+        else {
+            const boundaryTypes = getBoundaryTypes(input.parserLanguage, input.config.boundary_strictness);
+            const candidates = root.descendantsOfType([...boundaryTypes]);
+            const boundaryNodes = candidates
+                .filter((node) => !hasBoundaryAncestor(node, boundaryTypes))
+                .filter((node) => isLanguageBoundaryCandidate(input.parserLanguage, node, input.config.boundary_strictness))
+                .sort((a, b) => a.startPosition.row - b.startPosition.row || a.startPosition.column - b.startPosition.column);
+            if (boundaryNodes.length === 0) {
+                const fallbackStart = Date.now();
+                const fallbackChunks = buildSlidingChunks({
+                    lines: input.lines,
+                    tokenize: input.tokenize,
+                    targetChunkTokens: input.config.target_chunk_tokens,
+                    overlapTokens: input.config.chunk_overlap_tokens,
+                    maxChunks: input.config.max_chunks_per_file,
+                    lineTokenCounts
+                });
+                return {
+                    chunks: fallbackChunks,
+                    strategy: "sliding",
+                    fallback_reason: "empty_language_boundaries",
+                    parse_latency_ms: parseLatencyMs,
+                    language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+                    fallback_path_latency_ms: Date.now() - fallbackStart,
+                    language: parserLanguageToCanonical(input.parserLanguage)
+                };
             }
-            segments.push({ startRow, endRow });
-            cursor = endRow + 1;
-            if (cursor > lastRow) {
-                break;
+            const segments = [];
+            let cursor = 0;
+            const lastRow = Math.max(0, input.lines.length - 1);
+            for (const node of boundaryNodes) {
+                const startRow = Math.max(0, Math.min(lastRow, node.startPosition.row));
+                const endRow = Math.max(startRow, Math.min(lastRow, toInclusiveEndRow(node)));
+                if (startRow > cursor) {
+                    segments.push({ startRow: cursor, endRow: startRow - 1, boundary: false });
+                }
+                segments.push({ startRow, endRow, boundary: true });
+                cursor = endRow + 1;
+                if (cursor > lastRow) {
+                    break;
+                }
             }
-        }
-        if (cursor <= lastRow) {
-            segments.push({ startRow: cursor, endRow: lastRow });
-        }
-        const chunks = [];
-        for (const segment of segments) {
-            if (segment.endRow < segment.startRow || chunks.length >= input.config.max_chunks_per_file) {
-                continue;
+            if (cursor <= lastRow) {
+                segments.push({ startRow: cursor, endRow: lastRow, boundary: false });
             }
-            const pieces = splitRangeWithBudget({
-                lines: input.lines,
-                startRow: segment.startRow,
-                endRow: segment.endRow,
-                tokenize: input.tokenize,
-                targetChunkTokens: input.config.target_chunk_tokens,
-                overlapTokens: input.config.chunk_overlap_tokens,
-                maxChunks: input.config.max_chunks_per_file - chunks.length
-            });
-            for (const piece of pieces) {
-                const trimmed = trimLineRange(input.lines, piece.startRow, piece.endRow);
-                if (!trimmed) {
+            for (const segment of segments) {
+                if (segment.endRow < segment.startRow || chunks.length >= input.config.max_chunks_per_file) {
                     continue;
                 }
-                chunks.push({
-                    start_line: trimmed.start + 1,
-                    end_line: trimmed.end + 1,
-                    snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
-                });
-                if (chunks.length >= input.config.max_chunks_per_file) {
-                    break;
+                const segmentTokenCount = lineTokenCounts
+                    .slice(segment.startRow, segment.endRow + 1)
+                    .reduce((sum, value) => sum + value, 0);
+                const enableSemanticBoundarySplits = input.config.boundary_strictness === "semantic_js_ts" &&
+                    (input.parserLanguage === "javascript" ||
+                        input.parserLanguage === "jsx" ||
+                        input.parserLanguage === "typescript" ||
+                        input.parserLanguage === "tsx") &&
+                    segment.boundary;
+                const softMaxChunkTokens = Math.floor(input.config.target_chunk_tokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+                const pieces = enableSemanticBoundarySplits && segmentTokenCount <= softMaxChunkTokens
+                    ? [{ startRow: segment.startRow, endRow: segment.endRow }]
+                    : splitRangeWithBudget({
+                        lines: input.lines,
+                        lineTokenCounts,
+                        startRow: segment.startRow,
+                        endRow: segment.endRow,
+                        targetChunkTokens: input.config.target_chunk_tokens,
+                        overlapTokens: input.config.chunk_overlap_tokens,
+                        maxChunks: input.config.max_chunks_per_file - chunks.length,
+                        preferSafeBoundarySplit: enableSemanticBoundarySplits,
+                        softMaxChunkTokens
+                    });
+                for (const piece of pieces) {
+                    const trimmed = trimLineRange(input.lines, piece.startRow, piece.endRow);
+                    if (!trimmed) {
+                        continue;
+                    }
+                    chunks.push({
+                        start_line: trimmed.start + 1,
+                        end_line: trimmed.end + 1,
+                        snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+                    });
+                    if (chunks.length >= input.config.max_chunks_per_file) {
+                        break;
+                    }
                 }
             }
         }
@@ -439,7 +847,8 @@ function buildLanguageAwareChunks(input) {
                 tokenize: input.tokenize,
                 targetChunkTokens: input.config.target_chunk_tokens,
                 overlapTokens: input.config.chunk_overlap_tokens,
-                maxChunks: input.config.max_chunks_per_file
+                maxChunks: input.config.max_chunks_per_file,
+                lineTokenCounts
             });
             return {
                 chunks: slidingChunks,
@@ -456,7 +865,8 @@ function buildLanguageAwareChunks(input) {
             strategy: "language_aware",
             parse_latency_ms: parseLatencyMs,
             language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
-            language: parserLanguageToCanonical(input.parserLanguage)
+            language: parserLanguageToCanonical(input.parserLanguage),
+            recursive_semantic_chunking_used: recursiveSemanticChunkingUsed
         };
     }
     catch {
@@ -466,7 +876,8 @@ function buildLanguageAwareChunks(input) {
             tokenize: input.tokenize,
             targetChunkTokens: input.config.target_chunk_tokens,
             overlapTokens: input.config.chunk_overlap_tokens,
-            maxChunks: input.config.max_chunks_per_file
+            maxChunks: input.config.max_chunks_per_file,
+            lineTokenCounts
         });
         return {
             chunks,