@v1nvn/readability 0.44.0 → 0.45.1

package/src/policy/chunk.js

@@ -0,0 +1,365 @@
+// Token-bounded chunking of extracted markdown for RAG/embedding. Chunk
+// `tokenCount` uses the shared chars/4 estimator (policy/text.ts), keeping it
+// directly comparable to `metadata.tokenEstimate`. This is the server-side
+// chunking path for large documents; transport-level streaming was rejected —
+// MCP tool-result progress isn't universally host-supported, and this option
+// already keeps a multi-MB page from returning as one payload.
+import { hardSplitLines, headingText, parseBlocks } from "./markdown.js";
+import { estimateTokens } from "./text.js";
+const HEADING_FIRST_LINE = /^#{1,6}\s/;
+function splitBlocks(markdown) {
+    const parts = markdown.split(/\n{2,}/);
+    const blocks = [];
+    let heading = '';
+    for (const part of parts) {
+        const text = part.trim();
+        if (!text) {
+            continue;
+        }
+        if (HEADING_FIRST_LINE.test(text)) {
+            heading = headingText(text);
+        }
+        blocks.push({ headingContext: heading, text });
+    }
+    return blocks;
+}
+// Char strategy may break a code block — the semantic strategy avoids that.
+function splitOversizedBlock(block, maxChars) {
+    return hardSplitLines(block.text, maxChars).map(text => ({
+        headingContext: block.headingContext,
+        text,
+    }));
+}
+function toUnits(blocks, maxChars) {
+    const units = [];
+    for (const block of blocks) {
+        if (block.text.length <= maxChars) {
+            units.push(block);
+        }
+        else {
+            units.push(...splitOversizedBlock(block, maxChars));
+        }
+    }
+    return units;
+}
+function chunkMarkdownChar(markdown, options) {
+    if (!markdown) {
+        return [];
+    }
+    const maxTokens = Math.max(1, Math.floor(options.maxTokens));
+    const maxChars = maxTokens * 4;
+    const overlapTokens = Math.max(0, Math.floor(options.overlap));
+    // Overlap must be strictly less than maxChars — otherwise a chunk's tail
+    // could consume the entire budget and we'd loop forever re-emitting it.
+    const overlapChars = Math.min(overlapTokens * 4, maxChars - 1);
+    const blocks = splitBlocks(markdown);
+    if (blocks.length === 0) {
+        return [];
+    }
+    const units = toUnits(blocks, maxChars);
+    if (units.length === 0) {
+        return [];
+    }
+    const chunks = [];
+    let i = 0;
+    let overlapText = '';
+    let overlapHeading = '';
+    while (i < units.length) {
+        const spans = [];
+        let chunkText = '';
+        if (overlapText) {
+            chunkText = overlapText;
+            spans.push({
+                end: overlapText.length,
+                headingContext: overlapHeading,
+                start: 0,
+            });
+        }
+        // Force at least one new unit per chunk so the loop terminates. If carrying
+        // the overlap would push the first new unit past maxChars, drop the overlap
+        // for this chunk (the unit was pre-split to <= maxChars so it always fits alone).
+        const firstUnit = units[i];
+        const sepLen = chunkText ? 2 : 0;
+        if (chunkText &&
+            chunkText.length + sepLen + firstUnit.text.length > maxChars) {
+            chunkText = '';
+            spans.length = 0;
+        }
+        {
+            const sep = chunkText ? '\n\n' : '';
+            const start = chunkText.length + sep.length;
+            chunkText = chunkText + sep + firstUnit.text;
+            spans.push({
+                end: chunkText.length,
+                headingContext: firstUnit.headingContext,
+                start,
+            });
+            i += 1;
+        }
+        while (i < units.length) {
+            const unit = units[i];
+            const candidate = `${chunkText}\n\n${unit.text}`;
+            if (candidate.length > maxChars) {
+                break;
+            }
+            const start = chunkText.length + 2;
+            chunkText = candidate;
+            spans.push({
+                end: chunkText.length,
+                headingContext: unit.headingContext,
+                start,
+            });
+            i += 1;
+        }
+        const text = chunkText.trim();
+        if (text) {
+            chunks.push({
+                headingContext: spans[0]?.headingContext ?? '',
+                index: chunks.length,
+                text,
+                tokenCount: estimateTokens(text).tokenEstimate,
+            });
+        }
+        if (overlapChars > 0 && i < units.length) {
+            const overlapStart = Math.max(0, chunkText.length - overlapChars);
+            const carrier = spans.find(span => overlapStart >= span.start && overlapStart < span.end) ?? spans.find(span => span.start >= overlapStart);
+            overlapHeading = carrier?.headingContext ?? '';
+            overlapText = chunkText.slice(overlapStart).replace(/^\n+/, '');
+        }
+        else {
+            overlapText = '';
+            overlapHeading = '';
+        }
+    }
+    return chunks;
+}
+function parseSemanticUnits(markdown) {
+    const units = [];
+    const stack = [];
+    function context() {
+        return stack.map(h => h.text).join(' > ');
+    }
+    for (const block of parseBlocks(markdown)) {
+        const text = markdown.slice(block.start, block.end);
+        if (block.kind === 'heading') {
+            while (stack.length > 0 && stack[stack.length - 1].level >= block.depth) {
+                stack.pop();
+            }
+            stack.push({ level: block.depth, text: headingText(text) });
+            units.push({ headingContext: context(), kind: 'heading', text });
+            continue;
+        }
+        units.push({
+            headingContext: context(),
+            kind: block.kind === 'code' ? 'code' : 'text',
+            text,
+        });
+    }
+    return units;
+}
+// A section = a heading unit plus its following non-heading units until the
+// next heading. Pre-first-heading content forms its own section with an empty
+// hierarchy path.
+function groupSections(units) {
+    const sections = [];
+    let current = [];
+    let currentContext = '';
+    function flush() {
+        if (current.length > 0) {
+            sections.push({ headingContext: currentContext, units: current });
+            current = [];
+        }
+    }
+    for (const unit of units) {
+        if (unit.kind === 'heading') {
+            flush();
+            current = [unit];
+            currentContext = unit.headingContext;
+        }
+        else {
+            if (current.length === 0) {
+                currentContext = unit.headingContext;
+            }
+            current.push(unit);
+        }
+    }
+    flush();
+    return sections;
+}
+function joinedLength(units) {
+    if (units.length === 0) {
+        return 0;
+    }
+    let total = units[0].text.length;
+    for (let i = 1; i < units.length; i++) {
+        total += 2 + units[i].text.length;
+    }
+    return total;
+}
+// Hard-split a text unit that alone exceeds the budget — line-aware first, then
+// hard-split any oversized line. Only ever applied to paragraphs; code blocks
+// are never passed through here.
+function splitOversizedTextUnit(unit, maxChars) {
+    return hardSplitLines(unit.text, maxChars).map(text => ({
+        headingContext: unit.headingContext,
+        kind: 'text',
+        text,
+    }));
+}
+// Pack sections into chunk groups <= maxChars, preferring heading boundaries.
+// A section that fits is never split; an oversized section is split into
+// sub-chunks by body unit (paragraphs may split; code blocks never do).
+function buildBaseGroups(sections, maxChars) {
+    const groups = [];
+    let current = [];
+    let currentLen = 0;
+    function emit() {
+        if (current.length > 0) {
+            groups.push(current);
+            current = [];
+            currentLen = 0;
+        }
+    }
+    function append(unit) {
+        const sep = current.length > 0 ? 2 : 0;
+        current.push(unit);
+        currentLen += sep + unit.text.length;
+    }
+    for (const section of sections) {
+        const sectionLen = joinedLength(section.units);
+        const sep = current.length > 0 ? 2 : 0;
+        if (currentLen + sep + sectionLen <= maxChars) {
+            for (const unit of section.units) {
+                append(unit);
+            }
+            continue;
+        }
+        // Doesn't fit merged — close the current group and retry the section alone
+        // so it has first dibs on the next group (it may still merge with a later
+        // section that fits behind it).
+        emit();
+        if (sectionLen <= maxChars) {
+            for (const unit of section.units) {
+                append(unit);
+            }
+            continue;
+        }
+        // Oversized section: pack body units under the heading; split oversized
+        // paragraphs, emit oversized code blocks whole.
+        const [headingUnit, ...body] = section.units;
+        current.push(headingUnit);
+        currentLen = headingUnit.text.length;
+        for (const unit of body) {
+            if (unit.kind === 'code' && unit.text.length > maxChars) {
+                // Semantic never splits a code fence: a fence cut mid-way is
+                // unrecoverable, so the block is emitted as its own chunk even though
+                // it exceeds the token budget. Deliberate tradeoff vs. the char cap.
+                emit();
+                groups.push([unit]);
+                continue;
+            }
+            const sepNow = current.length > 0 ? 2 : 0;
+            if (unit.text.length <= maxChars &&
+                currentLen + sepNow + unit.text.length <= maxChars) {
+                append(unit);
+                continue;
+            }
+            const pieces = unit.text.length > maxChars
+                ? splitOversizedTextUnit(unit, maxChars)
+                : [unit];
+            for (const piece of pieces) {
+                const pieceSep = current.length > 0 ? 2 : 0;
+                if (current.length > 0 &&
+                    currentLen + pieceSep + piece.text.length > maxChars) {
+                    emit();
+                }
+                append(piece);
+            }
+        }
+        // Leave the trailing sub-chunk in `current` so a following section can
+        // merge into it if it fits.
+    }
+    emit();
+    return groups;
+}
+// Carry the trailing text (non-code) units from the previous group as the
+// prefix of the next, up to overlapChars. Never carry into or out of a code
+// block: a trailing code run is skipped, then contiguous text units before it
+// are carried (stop at the first heading or code block encountered).
+function applySemanticOverlap(groups, overlapChars) {
+    const result = [];
+    for (let i = 0; i < groups.length; i++) {
+        const group = groups[i];
+        const carrier = [];
+        if (i > 0 && overlapChars > 0) {
+            const prev = groups[i - 1];
+            let end = prev.length;
+            while (end > 0 && prev[end - 1].kind === 'code') {
+                end -= 1;
+            }
+            let carrierLen = 0;
+            for (let j = end - 1; j >= 0; j--) {
+                const unit = prev[j];
+                if (unit.kind !== 'text') {
+                    break;
+                }
+                const sep = carrier.length > 0 ? 2 : 0;
+                if (carrierLen + sep + unit.text.length > overlapChars) {
+                    break;
+                }
+                carrier.unshift(unit);
+                carrierLen += sep + unit.text.length;
+            }
+        }
+        const units = carrier.length === 0 ? group : [...carrier, ...group];
+        result.push({
+            // The chunk's heading context follows its first non-carrier unit, so a
+            // heading-leading chunk keeps its heading's path even with overlap.
+            headingContext: group[0]?.headingContext ?? '',
+            units,
+        });
+    }
+    return result;
+}
+function chunkMarkdownSemantic(markdown, options) {
+    if (!markdown) {
+        return [];
+    }
+    const maxTokens = Math.max(1, Math.floor(options.maxTokens));
+    const maxChars = maxTokens * 4;
+    const overlapTokens = Math.max(0, Math.floor(options.overlap));
+    const overlapChars = Math.min(overlapTokens * 4, maxChars - 1);
+    const units = parseSemanticUnits(markdown);
+    if (units.length === 0) {
+        return [];
+    }
+    const sections = groupSections(units);
+    const baseGroups = buildBaseGroups(sections, maxChars);
+    const groups = applySemanticOverlap(baseGroups, overlapChars);
+    const chunks = [];
+    for (const group of groups) {
+        if (group.units.length === 0) {
+            continue;
+        }
+        const text = group.units
+            .map(unit => unit.text)
+            .join('\n\n')
+            .trim();
+        if (!text) {
+            continue;
+        }
+        chunks.push({
+            headingContext: group.headingContext,
+            index: chunks.length,
+            text,
+            tokenCount: estimateTokens(text).tokenEstimate,
+        });
+    }
+    return chunks;
+}
+export function chunkMarkdown(markdown, options) {
+    if (options.strategy === 'semantic') {
+        return chunkMarkdownSemantic(markdown, options);
+    }
+    return chunkMarkdownChar(markdown, options);
+}