@cue-dev/retrieval-core 0.1.3

package/dist/chunking.js

@@ -0,0 +1,983 @@
+import Parser from "tree-sitter";
+import Go from "tree-sitter-go";
+import JavaScriptV023 from "tree-sitter-javascript-v023";
+import Java from "tree-sitter-java";
+import PythonV023 from "tree-sitter-python-v023";
+import Rust from "tree-sitter-rust";
+import TypeScript from "tree-sitter-typescript";
+const DEFAULT_BOUNDARY_NODE_TYPES_LEGACY = {
+    typescript: new Set([
+        "function_declaration",
+        "generator_function_declaration",
+        "class_declaration",
+        "interface_declaration",
+        "enum_declaration",
+        "type_alias_declaration"
+    ]),
+    tsx: new Set([
+        "function_declaration",
+        "generator_function_declaration",
+        "class_declaration",
+        "interface_declaration",
+        "enum_declaration",
+        "type_alias_declaration"
+    ]),
+    javascript: new Set([
+        "function_declaration",
+        "generator_function_declaration",
+        "class_declaration",
+        "function_expression",
+        "arrow_function"
+    ]),
+    jsx: new Set([
+        "function_declaration",
+        "generator_function_declaration",
+        "class_declaration",
+        "function_expression",
+        "arrow_function"
+    ]),
+    python: new Set(["function_definition", "class_definition"]),
+    go: new Set(["function_declaration", "method_declaration", "type_declaration"]),
+    rust: new Set(["function_item", "impl_item", "trait_item", "struct_item", "enum_item"]),
+    java: new Set(["class_declaration", "interface_declaration", "enum_declaration", "method_declaration"])
+};
+const DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS = {
+    ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY,
+    typescript: new Set([
+        ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.typescript,
+        "function_expression",
+        "arrow_function",
+        "method_definition",
+        "class"
+    ]),
+    tsx: new Set([
+        ...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.tsx,
+        "function_expression",
+        "arrow_function",
+        "method_definition",
+        "class"
+    ]),
+    javascript: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.javascript, "method_definition", "class"]),
+    jsx: new Set([...DEFAULT_BOUNDARY_NODE_TYPES_LEGACY.jsx, "method_definition", "class"])
+};
+const parserAvailabilityCache = new Map();
+const parserInitAttempts = new Map();
+const parserLanguageLoaderOverrides = new Map();
+const CANONICAL_TO_PARSER_LANGUAGE = {
+    typescript: "typescript",
+    javascript: "javascript",
+    python: "python",
+    go: "go",
+    rust: "rust",
+    java: "java"
+};
+const JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES = new Set([
+    "assignment_expression",
+    "variable_declarator",
+    "pair",
+    "export_statement",
+    "public_field_definition",
+    "property_definition"
+]);
+const JAVASCRIPT_EXPRESSION_BOUNDARY_NODE_TYPES = new Set(["function_expression", "arrow_function", "class"]);
+const JAVASCRIPT_EXPRESSION_BOUNDARY_WRAPPER_TYPES = new Set([
+    "parenthesized_expression",
+    "as_expression",
+    "satisfies_expression",
+    "type_assertion",
+    "non_null_expression"
+]);
+const SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER = 1.35;
+function parserLanguageToCanonical(language) {
+    if (language === "tsx") {
+        return "typescript";
+    }
+    if (language === "jsx") {
+        return "javascript";
+    }
+    return language;
+}
+function normalizeLanguageAlias(language) {
+    const normalized = language.trim().toLowerCase();
+    if (normalized === "typescript" || normalized === "ts") {
+        return "typescript";
+    }
+    if (normalized === "tsx") {
+        return "tsx";
+    }
+    if (normalized === "javascript" || normalized === "js") {
+        return "javascript";
+    }
+    if (normalized === "jsx") {
+        return "jsx";
+    }
+    if (normalized === "python" || normalized === "py") {
+        return "python";
+    }
+    if (normalized === "go" || normalized === "golang") {
+        return "go";
+    }
+    if (normalized === "rust" || normalized === "rs") {
+        return "rust";
+    }
+    if (normalized === "java") {
+        return "java";
+    }
+    return undefined;
+}
+function parserLanguageFromPath(path) {
+    const normalized = path.toLowerCase();
+    if (normalized.endsWith(".tsx")) {
+        return "tsx";
+    }
+    if (normalized.endsWith(".ts") || normalized.endsWith(".mts") || normalized.endsWith(".cts")) {
+        return "typescript";
+    }
+    if (normalized.endsWith(".jsx")) {
+        return "jsx";
+    }
+    if (normalized.endsWith(".js") || normalized.endsWith(".mjs") || normalized.endsWith(".cjs")) {
+        return "javascript";
+    }
+    if (normalized.endsWith(".py")) {
+        return "python";
+    }
+    if (normalized.endsWith(".go")) {
+        return "go";
+    }
+    if (normalized.endsWith(".rs")) {
+        return "rust";
+    }
+    if (normalized.endsWith(".java")) {
+        return "java";
+    }
+    return undefined;
+}
+function resolveParserLanguage(file) {
+    const explicit = file.language ? normalizeLanguageAlias(file.language) : undefined;
+    if (explicit) {
+        return explicit;
+    }
+    return parserLanguageFromPath(file.path);
+}
+function formatErrorMessage(error) {
+    if (error instanceof Error && error.message) {
+        return error.message;
+    }
+    return String(error);
+}
+function resolveTreeSitterLanguageHandle(moduleValue) {
+    if (moduleValue && typeof moduleValue === "object" && "language" in moduleValue) {
+        return moduleValue.language;
+    }
+    return moduleValue;
+}
+function loadParserLanguage(language) {
+    const overrideLoader = parserLanguageLoaderOverrides.get(language);
+    if (overrideLoader) {
+        return overrideLoader();
+    }
+    if (language === "typescript") {
+        return TypeScript.typescript;
+    }
+    if (language === "tsx") {
+        return TypeScript.tsx;
+    }
+    if (language === "javascript" || language === "jsx") {
+        // Bun is currently most reliable with tree-sitter 0.23-compatible JS/Python grammars.
+        return resolveTreeSitterLanguageHandle(JavaScriptV023);
+    }
+    if (language === "python") {
+        return resolveTreeSitterLanguageHandle(PythonV023);
+    }
+    if (language === "rust") {
+        return resolveTreeSitterLanguageHandle(Rust);
+    }
+    if (language === "java") {
+        return resolveTreeSitterLanguageHandle(Java);
+    }
+    return resolveTreeSitterLanguageHandle(Go);
+}
+function getParserAvailability(language) {
+    const cached = parserAvailabilityCache.get(language);
+    if (cached) {
+        return cached;
+    }
+    parserInitAttempts.set(language, (parserInitAttempts.get(language) ?? 0) + 1);
+    try {
+        const parser = new Parser();
+        parser.setLanguage(loadParserLanguage(language));
+        const availability = {
+            status: "available",
+            parser
+        };
+        parserAvailabilityCache.set(language, availability);
+        return availability;
+    }
+    catch (error) {
+        const availability = {
+            status: "unavailable",
+            error: formatErrorMessage(error)
+        };
+        parserAvailabilityCache.set(language, availability);
+        return availability;
+    }
+}
+function getParser(language) {
+    const availability = getParserAvailability(language);
+    if (availability.status !== "available") {
+        return undefined;
+    }
+    return availability.parser;
+}
+function toChunkingParserLanguage(canonicalLanguage) {
+    return CANONICAL_TO_PARSER_LANGUAGE[canonicalLanguage];
+}
+export function getChunkingParserAvailabilitySnapshot(input) {
+    const canonicalLanguages = new Set();
+    if (input?.enabled_languages && input.enabled_languages.length > 0) {
+        for (const language of input.enabled_languages) {
+            const parsed = normalizeLanguageAlias(language);
+            if (!parsed) {
+                continue;
+            }
+            const canonical = parserLanguageToCanonical(parsed);
+            if (toChunkingParserLanguage(canonical)) {
+                canonicalLanguages.add(canonical);
+            }
+        }
+    }
+    if (canonicalLanguages.size === 0) {
+        for (const canonical of Object.keys(CANONICAL_TO_PARSER_LANGUAGE)) {
+            canonicalLanguages.add(canonical);
+        }
+    }
+    return [...canonicalLanguages]
+        .sort((a, b) => a.localeCompare(b))
+        .map((canonicalLanguage) => {
+        const parserLanguage = toChunkingParserLanguage(canonicalLanguage);
+        if (!parserLanguage) {
+            return {
+                language: canonicalLanguage,
+                status: "unavailable",
+                error: "no parser mapping for language"
+            };
+        }
+        const availability = getParserAvailability(parserLanguage);
+        return {
+            language: canonicalLanguage,
+            status: availability.status,
+            ...(availability.error ? { error: availability.error } : {})
+        };
+    });
+}
+export function __resetChunkingParserStateForTests() {
+    parserAvailabilityCache.clear();
+    parserInitAttempts.clear();
+    parserLanguageLoaderOverrides.clear();
+}
+export function __setChunkingParserLanguageLoaderForTests(language, loader) {
+    if (loader) {
+        parserLanguageLoaderOverrides.set(language, loader);
+    }
+    else {
+        parserLanguageLoaderOverrides.delete(language);
+    }
+    parserAvailabilityCache.delete(language);
+}
+export function __getChunkingParserInitAttemptsForTests() {
+    return Object.fromEntries(parserInitAttempts.entries());
+}
+function toInclusiveEndRow(node) {
+    const end = node.endPosition;
+    if (end.column === 0 && end.row > node.startPosition.row) {
+        return end.row - 1;
+    }
+    return end.row;
+}
+function trimLineRange(lines, startRow, endRow) {
+    let start = startRow;
+    let end = endRow;
+    while (start <= end && (lines[start] ?? "").trim().length === 0) {
+        start += 1;
+    }
+    while (end >= start && (lines[end] ?? "").trim().length === 0) {
+        end -= 1;
+    }
+    if (end < start) {
+        return undefined;
+    }
+    return { start, end };
+}
+function splitRangeWithBudget(input) {
+    const rangeTokenCount = (startRow, endRow) => {
+        let total = 0;
+        for (let row = startRow; row <= endRow; row += 1) {
+            total += input.lineTokenCounts[row] ?? 0;
+        }
+        return total;
+    };
+    const isSafeSplitBoundaryLine = (line) => {
+        const trimmed = line.trim();
+        if (trimmed.length === 0) {
+            return true;
+        }
+        return trimmed.endsWith(";") || trimmed.endsWith("}") || trimmed.endsWith("{");
+    };
+    const segments = [];
+    let start = input.startRow;
+    while (start <= input.endRow && segments.length < input.maxChunks) {
+        let tokens = 0;
+        let end = start - 1;
+        while (end < input.endRow) {
+            const nextEnd = end + 1;
+            tokens += input.lineTokenCounts[nextEnd] ?? 0;
+            end = nextEnd;
+            if (tokens >= input.targetChunkTokens && end >= start) {
+                break;
+            }
+        }
+        let safeEnd = Math.min(Math.max(start, end), input.endRow);
+        if (input.preferSafeBoundarySplit && safeEnd > start) {
+            let adjusted = safeEnd;
+            for (let row = safeEnd; row > start; row -= 1) {
+                if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+                    adjusted = row;
+                    break;
+                }
+            }
+            if (adjusted === safeEnd && typeof input.softMaxChunkTokens === "number" && input.softMaxChunkTokens > input.targetChunkTokens) {
+                for (let row = safeEnd + 1; row <= input.endRow; row += 1) {
+                    if (rangeTokenCount(start, row) > input.softMaxChunkTokens) {
+                        break;
+                    }
+                    if (isSafeSplitBoundaryLine(input.lines[row] ?? "")) {
+                        adjusted = row;
+                        break;
+                    }
+                }
+            }
+            safeEnd = Math.max(start, adjusted);
+        }
+        if (safeEnd >= start) {
+            segments.push({ startRow: start, endRow: safeEnd });
+        }
+        if (safeEnd >= input.endRow) {
+            break;
+        }
+        let nextStart = safeEnd + 1;
+        if (input.overlapTokens > 0) {
+            let overlap = 0;
+            let cursor = safeEnd;
+            while (cursor >= start && overlap < input.overlapTokens) {
+                overlap += input.lineTokenCounts[cursor] ?? 0;
+                cursor -= 1;
+            }
+            nextStart = Math.max(start + 1, cursor + 1);
+        }
+        start = Math.max(start + 1, nextStart);
+    }
+    return segments;
+}
+function buildSlidingChunks(input) {
+    const lineTokenCounts = input.lineTokenCounts ?? computeLineTokenCounts(input.lines, input.tokenize);
+    const rawSegments = splitRangeWithBudget({
+        lines: input.lines,
+        lineTokenCounts,
+        startRow: 0,
+        endRow: Math.max(0, input.lines.length - 1),
+        targetChunkTokens: input.targetChunkTokens,
+        overlapTokens: input.overlapTokens,
+        maxChunks: input.maxChunks
+    });
+    const chunks = [];
+    for (const segment of rawSegments) {
+        const trimmed = trimLineRange(input.lines, segment.startRow, segment.endRow);
+        if (!trimmed) {
+            continue;
+        }
+        chunks.push({
+            start_line: trimmed.start + 1,
+            end_line: trimmed.end + 1,
+            snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+        });
+    }
+    return chunks;
+}
+function hasBoundaryAncestor(node, boundaryTypes) {
+    let current = node.parent;
+    while (current) {
+        if (boundaryTypes.has(current.type)) {
+            return true;
+        }
+        current = current.parent;
+    }
+    return false;
+}
+function getBoundaryTypes(parserLanguage, boundaryStrictness) {
+    if (boundaryStrictness === "semantic_js_ts") {
+        return DEFAULT_BOUNDARY_NODE_TYPES_SEMANTIC_JS_TS[parserLanguage];
+    }
+    return DEFAULT_BOUNDARY_NODE_TYPES_LEGACY[parserLanguage];
+}
+function isExpressionBoundaryLanguage(parserLanguage, boundaryStrictness) {
+    if (boundaryStrictness === "semantic_js_ts") {
+        return (parserLanguage === "javascript" ||
+            parserLanguage === "jsx" ||
+            parserLanguage === "typescript" ||
+            parserLanguage === "tsx");
+    }
+    return parserLanguage === "javascript" || parserLanguage === "jsx";
+}
+function isLanguageBoundaryCandidate(parserLanguage, node, boundaryStrictness) {
+    if (!isExpressionBoundaryLanguage(parserLanguage, boundaryStrictness)) {
+        return true;
+    }
+    if (!JAVASCRIPT_EXPRESSION_BOUNDARY_NODE_TYPES.has(node.type)) {
+        return true;
+    }
+    let owner = node.parent;
+    while (owner && JAVASCRIPT_EXPRESSION_BOUNDARY_WRAPPER_TYPES.has(owner.type)) {
+        owner = owner.parent;
+    }
+    const ownerType = owner?.type;
+    if (!ownerType) {
+        return false;
+    }
+    return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(ownerType);
+}
+export function __isChunkingBoundaryCandidateForTests(input) {
+    const strictness = input.boundaryStrictness ?? "legacy";
+    if (!isExpressionBoundaryLanguage(input.parserLanguage, strictness)) {
+        return true;
+    }
+    if (!JAVASCRIPT_EXPRESSION_BOUNDARY_NODE_TYPES.has(input.nodeType)) {
+        return true;
+    }
+    const ancestorChain = [];
+    if (input.parentType) {
+        ancestorChain.push(input.parentType);
+    }
+    if (input.ancestorTypes && input.ancestorTypes.length > 0) {
+        ancestorChain.push(...input.ancestorTypes);
+    }
+    const ownerType = ancestorChain.find((type) => !JAVASCRIPT_EXPRESSION_BOUNDARY_WRAPPER_TYPES.has(type));
+    if (!ownerType) {
+        return false;
+    }
+    return JAVASCRIPT_EXPRESSION_BOUNDARY_PARENT_TYPES.has(ownerType);
+}
+function computeLineTokenCounts(lines, tokenize) {
+    return lines.map((line) => tokenize(line ?? "").length);
+}
+function rangeTokenCount(lineTokenCounts, startRow, endRow) {
+    let total = 0;
+    for (let row = startRow; row <= endRow; row += 1) {
+        total += lineTokenCounts[row] ?? 0;
+    }
+    return total;
+}
+function listNamedChildren(node) {
+    const children = [];
+    for (let index = 0; index < node.namedChildCount; index += 1) {
+        const child = node.namedChild(index);
+        if (child) {
+            children.push(child);
+        }
+    }
+    return children;
+}
+function normalizeNodeWindow(input) {
+    const startRow = Math.max(0, Math.min(input.lastRow, input.node.startPosition.row));
+    const endRow = Math.max(startRow, Math.min(input.lastRow, toInclusiveEndRow(input.node)));
+    const trimmed = trimLineRange(input.lines, startRow, endRow);
+    if (!trimmed) {
+        return undefined;
+    }
+    return {
+        startRow: trimmed.start,
+        endRow: trimmed.end
+    };
+}
+function buildRecursiveSemanticWindows(input) {
+    const lastRow = Math.max(0, input.lines.length - 1);
+    const windows = [];
+    const softMaxChunkTokens = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+    const seen = new Set();
+    const pushSplitWindows = (startRow, endRow) => {
+        if (startRow > endRow || windows.length >= input.maxChunks) {
+            return;
+        }
+        const segments = splitRangeWithBudget({
+            lines: input.lines,
+            lineTokenCounts: input.lineTokenCounts,
+            startRow,
+            endRow,
+            targetChunkTokens: input.targetChunkTokens,
+            overlapTokens: 0,
+            maxChunks: input.maxChunks - windows.length,
+            preferSafeBoundarySplit: input.boundaryStrictness === "semantic_js_ts",
+            softMaxChunkTokens
+        });
+        for (const segment of segments) {
+            const trimmed = trimLineRange(input.lines, segment.startRow, segment.endRow);
+            if (!trimmed) {
+                continue;
+            }
+            const key = `${trimmed.start}:${trimmed.end}`;
+            if (seen.has(key)) {
+                continue;
+            }
+            seen.add(key);
+            windows.push({ startRow: trimmed.start, endRow: trimmed.end });
+            if (windows.length >= input.maxChunks) {
+                return;
+            }
+        }
+    };
+    const visitNode = (node) => {
+        if (windows.length >= input.maxChunks) {
+            return;
+        }
+        const range = normalizeNodeWindow({
+            node,
+            lines: input.lines,
+            lastRow
+        });
+        if (!range) {
+            return;
+        }
+        const tokenCount = rangeTokenCount(input.lineTokenCounts, range.startRow, range.endRow);
+        if (tokenCount <= input.targetChunkTokens) {
+            const key = `${range.startRow}:${range.endRow}`;
+            if (!seen.has(key)) {
+                seen.add(key);
+                windows.push(range);
+            }
+            return;
+        }
+        const children = listNamedChildren(node)
+            .map((child) => ({
+            node: child,
+            range: normalizeNodeWindow({
+                node: child,
+                lines: input.lines,
+                lastRow
+            })
+        }))
+            .filter((child) => Boolean(child.range))
+            .sort((a, b) => a.range.startRow - b.range.startRow || a.range.endRow - b.range.endRow);
+        if (children.length === 0) {
+            pushSplitWindows(range.startRow, range.endRow);
+            return;
+        }
+        let cursor = range.startRow;
+        for (const child of children) {
+            if (windows.length >= input.maxChunks) {
+                return;
+            }
+            if (child.range.endRow < cursor) {
+                continue;
+            }
+            if (child.range.startRow > cursor) {
+                pushSplitWindows(cursor, child.range.startRow - 1);
+            }
+            visitNode(child.node);
+            cursor = Math.max(cursor, child.range.endRow + 1);
+            if (cursor > range.endRow) {
+                return;
+            }
+        }
+        if (cursor <= range.endRow) {
+            pushSplitWindows(cursor, range.endRow);
+        }
+    };
+    visitNode(input.root);
+    return windows.sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+}
+function mergeSemanticWindows(input) {
+    if (input.windows.length <= 1) {
+        return [...input.windows];
+    }
+    const ordered = [...input.windows].sort((a, b) => a.startRow - b.startRow || a.endRow - b.endRow);
+    const merged = [];
+    const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+    for (const window of ordered) {
+        const last = merged[merged.length - 1];
+        if (!last) {
+            merged.push({ ...window });
+            continue;
+        }
+        const gapLines = Math.max(0, window.startRow - last.endRow - 1);
+        const nextStartRow = Math.min(last.startRow, window.startRow);
+        const nextEndRow = Math.max(last.endRow, window.endRow);
+        const nextSpanLines = nextEndRow - nextStartRow + 1;
+        const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, nextStartRow, nextEndRow);
+        const canMerge = gapLines <= input.semanticMergeGapLines &&
+            nextSpanLines <= input.semanticMergeMaxSpanLines &&
+            mergedTokenCount <= mergeTokenBudget;
+        if (!canMerge) {
+            merged.push({ ...window });
+            continue;
+        }
+        last.startRow = nextStartRow;
+        last.endRow = nextEndRow;
+    }
+    return merged;
+}
+function isCommentOnlyLine(line) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+        return true;
+    }
+    return (trimmed.startsWith("//") ||
+        trimmed.startsWith("/*") ||
+        trimmed.startsWith("*") ||
+        trimmed.startsWith("*/") ||
+        trimmed.startsWith("#"));
+}
+function windowLooksCommentOnly(input) {
+    for (let row = input.startRow; row <= input.endRow; row += 1) {
+        if (!isCommentOnlyLine(input.lines[row] ?? "")) {
+            return false;
+        }
+    }
+    return true;
+}
+function absorbForwardCommentWindows(input) {
+    if (input.windows.length <= 1) {
+        return [...input.windows];
+    }
+    const output = [];
+    const mergeTokenBudget = Math.floor(input.targetChunkTokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+    for (let index = 0; index < input.windows.length; index += 1) {
+        const current = input.windows[index];
+        const next = input.windows[index + 1];
+        if (!current) {
+            continue;
+        }
+        if (!next) {
+            output.push({ ...current });
+            continue;
+        }
+        if (!windowLooksCommentOnly({ lines: input.lines, startRow: current.startRow, endRow: current.endRow })) {
+            output.push({ ...current });
+            continue;
+        }
+        const gapLines = Math.max(0, next.startRow - current.endRow - 1);
+        const nextSpanLines = next.endRow - current.startRow + 1;
+        const mergedTokenCount = rangeTokenCount(input.lineTokenCounts, current.startRow, next.endRow);
+        const canAbsorb = gapLines <= 1 && nextSpanLines <= input.semanticMergeMaxSpanLines && mergedTokenCount <= mergeTokenBudget;
+        if (!canAbsorb) {
+            output.push({ ...current });
+            continue;
+        }
+        output.push({
+            startRow: current.startRow,
+            endRow: next.endRow
+        });
+        index += 1;
+    }
+    return output;
+}
+function windowsToChunks(input) {
+    const chunks = [];
+    for (const window of input.windows) {
+        if (chunks.length >= input.maxChunks) {
+            break;
+        }
+        const trimmed = trimLineRange(input.lines, window.startRow, window.endRow);
+        if (!trimmed) {
+            continue;
+        }
+        chunks.push({
+            start_line: trimmed.start + 1,
+            end_line: trimmed.end + 1,
+            snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+        });
+    }
+    return chunks;
+}
+function buildLanguageAwareChunks(input) {
+    const languageAwareAttemptStart = Date.now();
+    const lineTokenCounts = computeLineTokenCounts(input.lines, input.tokenize);
+    const parser = getParser(input.parserLanguage);
+    if (!parser) {
+        const fallbackStart = Date.now();
+        const chunks = buildSlidingChunks({
+            lines: input.lines,
+            tokenize: input.tokenize,
+            targetChunkTokens: input.config.target_chunk_tokens,
+            overlapTokens: input.config.chunk_overlap_tokens,
+            maxChunks: input.config.max_chunks_per_file,
+            lineTokenCounts
+        });
+        return {
+            chunks,
+            strategy: "sliding",
+            fallback_reason: "parser_unavailable",
+            language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+            fallback_path_latency_ms: Date.now() - fallbackStart,
+            language: parserLanguageToCanonical(input.parserLanguage)
+        };
+    }
+    try {
+        parser.setTimeoutMicros(input.config.parse_timeout_ms * 1_000);
+        const parseStart = Date.now();
+        const tree = parser.parse(input.file.content);
+        const parseLatencyMs = Date.now() - parseStart;
+        if (parseLatencyMs > input.config.parse_timeout_ms) {
+            const fallbackStart = Date.now();
+            const chunks = buildSlidingChunks({
+                lines: input.lines,
+                tokenize: input.tokenize,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file,
+                lineTokenCounts
+            });
+            return {
+                chunks,
+                strategy: "sliding",
+                fallback_reason: "parse_timeout_exceeded",
+                parse_latency_ms: parseLatencyMs,
+                language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+                fallback_path_latency_ms: Date.now() - fallbackStart,
+                language: parserLanguageToCanonical(input.parserLanguage)
+            };
+        }
+        const root = tree?.rootNode;
+        if (!root) {
+            const fallbackStart = Date.now();
+            const chunks = buildSlidingChunks({
+                lines: input.lines,
+                tokenize: input.tokenize,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file,
+                lineTokenCounts
+            });
+            return {
+                chunks,
+                strategy: "sliding",
+                fallback_reason: "parse_error",
+                parse_latency_ms: parseLatencyMs,
+                language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+                fallback_path_latency_ms: Date.now() - fallbackStart,
+                language: parserLanguageToCanonical(input.parserLanguage)
+            };
+        }
+        let chunks = [];
+        let recursiveSemanticChunkingUsed = false;
+        if (input.config.recursive_semantic_chunking_enabled) {
+            const semanticMergeGapLines = input.config.semantic_merge_gap_lines ?? 6;
+            const semanticMergeMaxSpanLines = input.config.semantic_merge_max_span_lines ?? 220;
+            const recursiveWindows = buildRecursiveSemanticWindows({
+                root,
+                lines: input.lines,
+                lineTokenCounts,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                maxChunks: input.config.max_chunks_per_file,
+                boundaryStrictness: input.config.boundary_strictness
+            });
+            const mergedWindows = mergeSemanticWindows({
+                windows: recursiveWindows,
+                lineTokenCounts,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                semanticMergeGapLines,
+                semanticMergeMaxSpanLines
+            });
+            const absorbedWindows = input.config.comment_forward_absorb_enabled === false
+                ? mergedWindows
+                : absorbForwardCommentWindows({
+                    windows: mergedWindows,
+                    lines: input.lines,
+                    lineTokenCounts,
+                    targetChunkTokens: input.config.target_chunk_tokens,
+                    semanticMergeMaxSpanLines
+                });
+            chunks = windowsToChunks({
+                windows: absorbedWindows,
+                lines: input.lines,
+                maxChunks: input.config.max_chunks_per_file
+            });
+            recursiveSemanticChunkingUsed = chunks.length > 0;
+        }
+        else {
+            const boundaryTypes = getBoundaryTypes(input.parserLanguage, input.config.boundary_strictness);
+            const candidates = root.descendantsOfType([...boundaryTypes]);
+            const boundaryNodes = candidates
+                .filter((node) => !hasBoundaryAncestor(node, boundaryTypes))
+                .filter((node) => isLanguageBoundaryCandidate(input.parserLanguage, node, input.config.boundary_strictness))
+                .sort((a, b) => a.startPosition.row - b.startPosition.row || a.startPosition.column - b.startPosition.column);
+            if (boundaryNodes.length === 0) {
+                const fallbackStart = Date.now();
+                const fallbackChunks = buildSlidingChunks({
+                    lines: input.lines,
+                    tokenize: input.tokenize,
+                    targetChunkTokens: input.config.target_chunk_tokens,
+                    overlapTokens: input.config.chunk_overlap_tokens,
+                    maxChunks: input.config.max_chunks_per_file,
+                    lineTokenCounts
+                });
+                return {
+                    chunks: fallbackChunks,
+                    strategy: "sliding",
+                    fallback_reason: "empty_language_boundaries",
+                    parse_latency_ms: parseLatencyMs,
+                    language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+                    fallback_path_latency_ms: Date.now() - fallbackStart,
+                    language: parserLanguageToCanonical(input.parserLanguage)
+                };
+            }
+            const segments = [];
+            let cursor = 0;
+            const lastRow = Math.max(0, input.lines.length - 1);
+            for (const node of boundaryNodes) {
+                const startRow = Math.max(0, Math.min(lastRow, node.startPosition.row));
+                const endRow = Math.max(startRow, Math.min(lastRow, toInclusiveEndRow(node)));
+                if (startRow > cursor) {
+                    segments.push({ startRow: cursor, endRow: startRow - 1, boundary: false });
+                }
+                segments.push({ startRow, endRow, boundary: true });
+                cursor = endRow + 1;
+                if (cursor > lastRow) {
+                    break;
+                }
+            }
+            if (cursor <= lastRow) {
+                segments.push({ startRow: cursor, endRow: lastRow, boundary: false });
+            }
+            for (const segment of segments) {
+                if (segment.endRow < segment.startRow || chunks.length >= input.config.max_chunks_per_file) {
+                    continue;
+                }
+                const segmentTokenCount = lineTokenCounts
+                    .slice(segment.startRow, segment.endRow + 1)
+                    .reduce((sum, value) => sum + value, 0);
+                const enableSemanticBoundarySplits = input.config.boundary_strictness === "semantic_js_ts" &&
+                    (input.parserLanguage === "javascript" ||
+                        input.parserLanguage === "jsx" ||
+                        input.parserLanguage === "typescript" ||
+                        input.parserLanguage === "tsx") &&
+                    segment.boundary;
+                const softMaxChunkTokens = Math.floor(input.config.target_chunk_tokens * SEMANTIC_JS_TS_SOFT_MAX_MULTIPLIER);
+                const pieces = enableSemanticBoundarySplits && segmentTokenCount <= softMaxChunkTokens
+                    ? [{ startRow: segment.startRow, endRow: segment.endRow }]
+                    : splitRangeWithBudget({
+                        lines: input.lines,
+                        lineTokenCounts,
+                        startRow: segment.startRow,
+                        endRow: segment.endRow,
+                        targetChunkTokens: input.config.target_chunk_tokens,
+                        overlapTokens: input.config.chunk_overlap_tokens,
+                        maxChunks: input.config.max_chunks_per_file - chunks.length,
+                        preferSafeBoundarySplit: enableSemanticBoundarySplits,
+                        softMaxChunkTokens
+                    });
+                for (const piece of pieces) {
+                    const trimmed = trimLineRange(input.lines, piece.startRow, piece.endRow);
+                    if (!trimmed) {
+                        continue;
+                    }
+                    chunks.push({
+                        start_line: trimmed.start + 1,
+                        end_line: trimmed.end + 1,
+                        snippet: input.lines.slice(trimmed.start, trimmed.end + 1).join("\n")
+                    });
+                    if (chunks.length >= input.config.max_chunks_per_file) {
+                        break;
+                    }
+                }
+            }
+        }
+        if (chunks.length === 0) {
+            const fallbackStart = Date.now();
+            const slidingChunks = buildSlidingChunks({
+                lines: input.lines,
+                tokenize: input.tokenize,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file,
+                lineTokenCounts
+            });
+            return {
+                chunks: slidingChunks,
+                strategy: "sliding",
+                fallback_reason: "empty_language_boundaries",
+                parse_latency_ms: parseLatencyMs,
+                language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+                fallback_path_latency_ms: Date.now() - fallbackStart,
+                language: parserLanguageToCanonical(input.parserLanguage)
+            };
+        }
+        return {
+            chunks,
+            strategy: "language_aware",
+            parse_latency_ms: parseLatencyMs,
+            language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+            language: parserLanguageToCanonical(input.parserLanguage),
+            recursive_semantic_chunking_used: recursiveSemanticChunkingUsed
+        };
+    }
+    catch {
+        const fallbackStart = Date.now();
+        const chunks = buildSlidingChunks({
+            lines: input.lines,
+            tokenize: input.tokenize,
+            targetChunkTokens: input.config.target_chunk_tokens,
+            overlapTokens: input.config.chunk_overlap_tokens,
+            maxChunks: input.config.max_chunks_per_file,
+            lineTokenCounts
+        });
+        return {
+            chunks,
+            strategy: "sliding",
+            fallback_reason: "parse_error",
+            language_aware_attempt_latency_ms: Date.now() - languageAwareAttemptStart,
+            fallback_path_latency_ms: Date.now() - fallbackStart,
+            language: parserLanguageToCanonical(input.parserLanguage)
+        };
+    }
+}
+export function buildChunksForFile(input) {
+    const lines = input.file.content.split("\n");
+    const language = resolveParserLanguage(input.file);
+    const enabledLanguageSet = new Set(input.config.enabled_languages.map((value) => value.trim().toLowerCase()));
+    if (input.config.strategy === "sliding") {
+        return {
+            chunks: buildSlidingChunks({
+                lines,
+                tokenize: input.tokenize,
+                targetChunkTokens: input.config.target_chunk_tokens,
+                overlapTokens: input.config.chunk_overlap_tokens,
+                maxChunks: input.config.max_chunks_per_file
+            }),
+            strategy: "sliding",
+            language: language ? parserLanguageToCanonical(language) : undefined
+        };
+    }
+    if (!language || !enabledLanguageSet.has(parserLanguageToCanonical(language))) {
+        const fallbackStart = Date.now();
+        const chunks = buildSlidingChunks({
+            lines,
+            tokenize: input.tokenize,
+            targetChunkTokens: input.config.target_chunk_tokens,
+            overlapTokens: input.config.chunk_overlap_tokens,
+            maxChunks: input.config.max_chunks_per_file
+        });
+        return {
+            chunks,
+            strategy: "sliding",
+            fallback_reason: "unsupported_language",
+            fallback_path_latency_ms: Date.now() - fallbackStart
+        };
+    }
+    return buildLanguageAwareChunks({
+        file: input.file,
+        lines,
+        parserLanguage: language,
+        config: input.config,
+        tokenize: input.tokenize
+    });
+}