@v1nvn/readability 0.45.0 → 0.45.1

package/src/policy/grid-detector.js

@@ -0,0 +1,207 @@
+import { resolveCellText } from "../pipeline/cell-text.js";
+import { CONTAINER_TAGS, describeSelector, groupChildrenByShape, stripLandmarkChrome, } from "./sibling-scan.js";
+const MIN_ROWS = 3;
+const HIGH_CONF_ROWS = 6;
+const MIN_CELLS_PER_ROW = 2;
+function rowCells(row) {
+    return Array.from(row.children).map(resolveCellText);
+}
+function notDetected(note) {
+    return {
+        confidence: 'low',
+        containerSelector: '',
+        detected: false,
+        rowCount: 0,
+        colCount: 0,
+        rows: [],
+        rowTag: '',
+        note,
+    };
+}
+// Confidence tracks the detected cluster (the data-row count), not any header
+// row inferred on top of it — a recovered header is inference, not evidence, so
+// it must not push a grid past HIGH_CONF_ROWS. Both detection modes gate on
+// minRows before reaching buildResult, so this is only ever called with a
+// data-row count >= minRows; `low` comes solely from notDetected().
+function confidenceFor(dataRowCount) {
+    return dataRowCount >= HIGH_CONF_ROWS ? 'high' : 'medium';
+}
+// Pad ragged rows to the max cell width so the matrix is rectangular — the
+// GFM/CSV/JSON renderers in pipeline/tables.ts assume a dense grid (the
+// delimiter row is derived from row-0 width, so a short row would mis-align
+// every column). Mirrors parseTableMatrix's dense-pad step.
+// `evidence` separates the detected cluster from the emitted matrix: rowCount
+// counts every emitted row (data plus any recovered header), but confidence and
+// the "detected N" note track the data-row cluster alone, since the header is
+// inferred rather than detected. Omitted in selector mode and when auto-detect
+// recovers no header, where the two counts coincide.
+function buildResult(raggedRows, rowTag, containerSelector, selectorHint, evidence) {
+    let maxCols = 0;
+    for (const row of raggedRows) {
+        if (row.length > maxCols) {
+            maxCols = row.length;
+        }
+    }
+    if (maxCols === 0) {
+        return notDetected(selectorHint
+            ? `not a grid: rows matched ${selectorHint} but no cells were found`
+            : 'not a grid: rows matched but no cells were found');
+    }
+    const rowCount = raggedRows.length;
+    const dataRowCount = evidence?.dataRowCount ?? rowCount;
+    const headerCount = evidence?.headerCount ?? 0;
+    const rows = raggedRows.map(row => ({
+        cells: Array.from({ length: maxCols }, (_, i) => row[i] ?? ''),
+    }));
+    const where = containerSelector || selectorHint || 'document';
+    const descriptor = headerCount > 0
+        ? `detected ${dataRowCount} ${rowTag} data rows plus ${headerCount} header row${headerCount > 1 ? 's' : ''} (${maxCols} cols) in ${where}`
+        : `detected ${dataRowCount} ${rowTag} rows (${maxCols} cols) in ${where}`;
+    return {
+        confidence: confidenceFor(dataRowCount),
+        containerSelector,
+        detected: true,
+        rowCount,
+        colCount: maxCols,
+        rows,
+        rowTag,
+        note: descriptor,
+    };
+}
+function detectSelectorMode(document, rowSelector, cellSelector, minRows) {
+    const rowEls = Array.from(document.querySelectorAll(rowSelector));
+    if (rowEls.length < minRows) {
+        return notDetected(`not a grid: rowSelector "${rowSelector}" matched ${rowEls.length} row(s) (min ${minRows})`);
+    }
+    const rows = rowEls.map(row => Array.from(row.querySelectorAll(cellSelector)).map(resolveCellText));
+    return buildResult(rows, rowEls[0].tagName, '', rowSelector);
+}
+function modalWidth(members) {
+    const counts = new Map();
+    for (const member of members) {
+        const width = member.children.length;
+        counts.set(width, (counts.get(width) ?? 0) + 1);
+    }
+    let best = members[0].children.length;
+    let bestCount = 0;
+    for (const [width, count] of counts) {
+        if (count > bestCount) {
+            best = width;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+function classTokens(el) {
+    const cls = el.getAttribute('class');
+    return cls ? new Set(cls.trim().split(/\s+/)) : new Set();
+}
+// A div-grid header has no <th>, so recovery leans on two signals, strongest
+// first. (1) An ARIA grid header — role="row" with a columnheader child — is a
+// semantic role and reliable on its own. (2) Class kinship with the data
+// cluster: a real header is the data rows' own class plus a discriminator
+// ("est-header"), so it carries every data class token. Chrome such as
+// "card-header" / "page-header" shares no class with the data rows and must not
+// be promoted — which is why we match token ownership, never the "header"
+// substring. Substring matching is what split the header into its own shape
+// group to begin with, and it over-fires on any chrome whose class merely
+// contains the word.
+function looksLikeHeader(child, rowTag, dataClass, dataWidth) {
+    if (child.tagName !== rowTag || child.children.length !== dataWidth) {
+        return false;
+    }
+    if (child.getAttribute('role') === 'row' &&
+        Array.from(child.children).some(c => c.getAttribute('role') === 'columnheader')) {
+        return true;
+    }
+    const childClass = classTokens(child);
+    return (dataClass.size > 0 && [...dataClass].every(token => childClass.has(token)));
+}
+// A header row often carries an extra class token (e.g. "est-header") that
+// splits it into its own 1-member shape group, below the minRows cutoff, so the
+// data-row cluster wins on its own and the header is dropped — leaving the
+// first data row mis-read as the header and the real header lost. Recover it:
+// scan the winning container's children that precede the first data row and
+// prepend any sibling that looks like a header for this cluster. Children after
+// the first data row (totals/footer) are left alone.
+function collectHeaderRows(container, members) {
+    const memberSet = new Set(members);
+    const rowTag = members[0].tagName;
+    const dataWidth = modalWidth(members);
+    // shapeKey freezes tag + class signature, so every member shares one class
+    // set and members[0] represents it; child count is not part of shapeKey, so
+    // widths can vary within the group (hence modalWidth above).
+    const dataClass = classTokens(members[0]);
+    const headers = [];
+    for (const child of Array.from(container.children)) {
+        if (memberSet.has(child)) {
+            break;
+        }
+        if (looksLikeHeader(child, rowTag, dataClass, dataWidth)) {
+            headers.push(rowCells(child));
+        }
+    }
+    return headers;
+}
+function detectAuto(document, minRows) {
+    stripLandmarkChrome(document);
+    const candidates = [];
+    for (const container of document.querySelectorAll('*')) {
+        if (!CONTAINER_TAGS.has(container.tagName)) {
+            continue;
+        }
+        const groups = groupChildrenByShape(container);
+        for (const members of groups.values()) {
+            if (members.length < minRows) {
+                continue;
+            }
+            // A row needs at least two cells to be a row; a group of single-child
+            // wrappers is a list of links/labels, not a data grid.
+            if (!members.every(m => m.children.length >= MIN_CELLS_PER_ROW)) {
+                continue;
+            }
+            const rows = members.map(rowCells);
+            const totalCellText = rows.reduce((sum, row) => sum + row.reduce((s, c) => s + c.length, 0), 0);
+            candidates.push({
+                container,
+                containerSelector: describeSelector(container),
+                members,
+                rows,
+                rowTag: members[0].tagName,
+                memberCount: members.length,
+                totalCellText,
+            });
+        }
+    }
+    if (candidates.length === 0) {
+        return notDetected('not a grid: no repeating row structure (≥3 same-shape siblings each with ≥2 direct element-children, outside nav/header/footer/aside)');
+    }
+    // Most rows wins; ties break on total cell text so a sparse 6-row grid beats
+    // a dense 6-row one only when it carries more substance, and a longer
+    // analyst-estimates table beats a stub sidebar mini-grid on both axes.
+    let winner = candidates[0];
+    for (let i = 1; i < candidates.length; i++) {
+        const candidate = candidates[i];
+        if (candidate.memberCount > winner.memberCount ||
+            (candidate.memberCount === winner.memberCount &&
+                candidate.totalCellText > winner.totalCellText)) {
+            winner = candidate;
+        }
+    }
+    const headerRows = collectHeaderRows(winner.container, winner.members);
+    return buildResult([...headerRows, ...winner.rows], winner.rowTag, winner.containerSelector, undefined, { dataRowCount: winner.memberCount, headerCount: headerRows.length });
+}
+// Detect a CSS-grid/div "table" — the div equivalent of extract_tables. In
+// selector mode (both rowSelector and cellSelector given) the rows and cells
+// are whatever the caller names. In auto mode, chrome is stripped first and
+// the container whose direct children form the largest same-shape sibling
+// group (each member a row of ≥2 direct element-children) wins. The resulting
+// rows are padded to a dense rectangular matrix so the shared table renderer
+// can serialize them as gfm/csv/json.
+export function detectGrid(document, opts) {
+    const minRows = opts?.minRows ?? MIN_ROWS;
+    if (opts?.rowSelector && opts.cellSelector) {
+        return detectSelectorMode(document, opts.rowSelector, opts.cellSelector, minRows);
+    }
+    return detectAuto(document, minRows);
+}

package/src/policy/identifiers.js

@@ -0,0 +1,62 @@
+export const GIBBERISH_THRESHOLD = 0.3;
+const PRICE_TABLE = {
+    'Upper:Lower': 0.2,
+    'Lower:Upper': 0.5,
+    'Symbol:Upper': 0.4,
+    'Symbol:Lower': 0.3,
+    'Symbol:Digit': 0.9,
+    'Upper:Digit': 1.4,
+    'Lower:Digit': 1.3,
+    'Digit:Upper': 1.4,
+    'Digit:Lower': 1.5,
+    'Digit:Symbol': 1.2,
+    'Upper:Symbol': 0.2,
+    'Lower:Symbol': 0.2,
+    'Digit:Digit': 1.2,
+    'Symbol:Symbol': 0.3,
+    'Upper:Upper': 0.1,
+    'Lower:Lower': 0.1,
+    'Other:Other': 0.0,
+};
+function classifyChar(ch) {
+    if (/[A-Z]/.test(ch)) {
+        return 'Upper';
+    }
+    if (/[a-z]/.test(ch)) {
+        return 'Lower';
+    }
+    if (/[0-9]/.test(ch)) {
+        return 'Digit';
+    }
+    if (/[-_]/.test(ch)) {
+        return 'Symbol';
+    }
+    return 'Other';
+}
+function transitionPrice(from, to) {
+    return PRICE_TABLE[`${from}:${to}`] ?? 1.0;
+}
+export function gibberishScore(value) {
+    if (value.length < 2) {
+        return 0;
+    }
+    let absolute = 0;
+    let previous = null;
+    for (const ch of value) {
+        const current = classifyChar(ch);
+        if (previous !== null) {
+            absolute += transitionPrice(previous, current);
+        }
+        previous = current;
+    }
+    return absolute / Math.max(1, value.length - 1);
+}
+function exceedsThreshold(value) {
+    return value.length >= 4 && gibberishScore(value) >= GIBBERISH_THRESHOLD;
+}
+export function isGeneratedIdentifier(value) {
+    if (exceedsThreshold(value)) {
+        return true;
+    }
+    return value.split(/[-_]/).some(segment => exceedsThreshold(segment));
+}

package/src/policy/images.js

@@ -0,0 +1,49 @@
+import { isPlaceholderSrc } from "../pipeline/normalize.js";
+import { absolutize } from "../pipeline/urls.js";
+function positiveInt(value) {
+    if (value === null) {
+        return undefined;
+    }
+    const n = Number.parseInt(value, 10);
+    return Number.isInteger(n) && n > 0 ? n : undefined;
+}
+// resolveLazyImages has already mutated placeholder srcs to real ones in the
+// article HTML by the time this runs — reading `src` here gets the resolved
+// value, so this walker does NOT re-run source resolution. isPlaceholderSrc
+// skips any remaining data:/placeholder srcs the lazy resolver left alone.
+export function collectImageInventory(html, window, baseUrl) {
+    if (!html) {
+        return [];
+    }
+    const probe = window.document.createElement('div');
+    probe.innerHTML = html;
+    const entries = [];
+    for (const img of probe.querySelectorAll('img')) {
+        const rawSrc = img.getAttribute('src') ?? '';
+        if (isPlaceholderSrc(rawSrc)) {
+            continue;
+        }
+        const src = absolutize(rawSrc, baseUrl);
+        if (!src) {
+            continue;
+        }
+        const alt = img.getAttribute('alt') ?? '';
+        const width = positiveInt(img.getAttribute('width'));
+        const height = positiveInt(img.getAttribute('height'));
+        // <figcaption> may precede OR follow the <img> inside the <figure>, so
+        // closest('figure') + querySelector('figcaption') covers both orderings.
+        const figure = img.closest('figure');
+        const figcaption = figure?.querySelector('figcaption');
+        const caption = figcaption
+            ? figcaption.textContent.replace(/\s+/g, ' ').trim()
+            : alt;
+        entries.push({
+            src,
+            alt,
+            ...(width !== undefined ? { width } : {}),
+            ...(height !== undefined ? { height } : {}),
+            caption,
+        });
+    }
+    return entries;
+}

package/src/policy/list-detector.js

@@ -0,0 +1,232 @@
+import { absolutize } from "../pipeline/urls.js";
+import { CONTAINER_TAGS, describeSelector, groupChildrenByShape, stripLandmarkChrome, } from "./sibling-scan.js";
+// Three siblings of the same shape (tag + class signature), each carrying a
+// real navigation anchor, is the smallest cluster that is not accidentally a
+// nav menu or a article's <ul> of inline links. Below this the detector
+// refuses to claim the page is a list.
+const MIN_ITEMS = 3;
+// Six-or-more items with non-trivial body text per item is the empirical
+// signature of a true index/search/blog-roll page (HN shows 30, Google ~10,
+// WP indexes ~10). Used to separate `high` from `medium` confidence so a
+// 3-item related-links block doesn't masquerade as a feed.
+const HIGH_CONF_ITEMS = 6;
+const HIGH_CONF_AVG_SCORE = 30;
+const MAX_SNIPPET_CHARS = 200;
+// href schemes that don't point at a list target. Excluding them keeps mailto:
+// /tel:/javascript:/ anchors from satisfying the "every item has a link" bar.
+function isNavigationHref(href) {
+    if (!href || href === '#') {
+        return false;
+    }
+    const lower = href.toLowerCase();
+    return (!lower.startsWith('javascript:') &&
+        !lower.startsWith('mailto:') &&
+        !lower.startsWith('tel:'));
+}
+function anchorText(anchor) {
+    return anchor.textContent.replace(/\s+/g, ' ').trim();
+}
+// Anchors with a navigation-worthy href AND visible text. Lives inside the
+// candidate child so a single <a href="#"> footer link doesn't satisfy the
+// "every item has a link" requirement.
+function navigationAnchors(child) {
+    const anchors = [];
+    for (const el of child.querySelectorAll('a[href]')) {
+        const anchor = el;
+        const href = anchor.getAttribute('href') ?? '';
+        if (!isNavigationHref(href)) {
+            continue;
+        }
+        if (anchorText(anchor).length === 0) {
+            continue;
+        }
+        anchors.push(anchor);
+    }
+    return anchors;
+}
+function pickPrimaryAnchor(anchors) {
+    let primary = anchors[0];
+    let best = anchorText(primary).length;
+    for (let i = 1; i < anchors.length; i++) {
+        const len = anchorText(anchors[i]).length;
+        if (len > best) {
+            primary = anchors[i];
+            best = len;
+        }
+    }
+    return primary;
+}
+function clipSnippet(raw) {
+    const collapsed = raw.replace(/\s+/g, ' ').trim();
+    if (collapsed.length <= MAX_SNIPPET_CHARS) {
+        return collapsed;
+    }
+    return `${collapsed.slice(0, MAX_SNIPPET_CHARS)}…`;
+}
+// Score = primary-anchor text length + non-link body text length. A pure
+// link-density penalty (textLength * (1 - linkDensity)) ranks HN's subtext
+// rows ABOVE the title rows because the title row is anchor-dominated (its
+// title IS the link), so non-link body alone mis-ranks. Adding the primary
+// anchor's text length rewards long titles (real feed items) over short nav
+// labels ("Home", "Sign in"), which is the signal we actually want for both
+// the candidate tiebreak and the high-confidence threshold.
+function scoreItem(textLength, linkTextLength, primaryAnchorTextLength) {
+    if (textLength === 0) {
+        return 0;
+    }
+    const nonLinkBody = Math.max(0, textLength - linkTextLength);
+    return primaryAnchorTextLength + nonLinkBody;
+}
+function extractItem(child, baseUrl) {
+    const anchors = navigationAnchors(child);
+    if (anchors.length === 0) {
+        return null;
+    }
+    const primary = pickPrimaryAnchor(anchors);
+    const title = anchorText(primary);
+    if (!title) {
+        return null;
+    }
+    const href = primary.getAttribute('href') ?? '';
+    const url = absolutize(href, baseUrl);
+    if (!url) {
+        return null;
+    }
+    const fullText = child.textContent.replace(/\s+/g, ' ').trim();
+    // Snippet = body text with the title peeled off the front when present, so
+    // the title link doesn't echo into the excerpt.
+    const snippet = fullText === title
+        ? ''
+        : fullText.startsWith(title)
+            ? clipSnippet(fullText.slice(title.length + 1))
+            : clipSnippet(fullText);
+    const linkTextLength = anchors.reduce((sum, anchor) => sum + anchorText(anchor).length, 0);
+    return {
+        score: scoreItem(fullText.length, linkTextLength, title.length),
+        snippet,
+        title,
+        url,
+    };
+}
+// Count of distinct URL pathnames a candidate's items point at. This is the
+// signal that separates a real feed from a metadata cluster: a feed's items
+// each navigate to a distinct destination (HN title rows link to ~30 different
+// external stories; a blog index links to /post-1../post-N; a search results
+// page links to a heterogeneous set of sites), so distinctPathnames ≈ item
+// count. A metadata cluster's items all navigate to the same internal route —
+// HN's subtext rows are 31 anchors all pointing at news.ycombinator.com/item?id=…,
+// collapsing to pathname=/item (distinctPathnames = 1). Counting items alone
+// lets subtext win on HN (31 vs 30 title rows) because of the trailing "More"
+// row; counting distinct destinations restores the title group as the winner.
+function distinctPathnames(items) {
+    const paths = new Set();
+    for (const item of items) {
+        try {
+            paths.add(new URL(item.url).pathname);
+        }
+        catch {
+            paths.add(item.url);
+        }
+    }
+    return paths.size;
+}
+function collectCandidates(document, baseUrl) {
+    const candidates = [];
+    for (const container of document.querySelectorAll('*')) {
+        if (!CONTAINER_TAGS.has(container.tagName)) {
+            continue;
+        }
+        const groups = groupChildrenByShape(container);
+        for (const children of groups.values()) {
+            if (children.length < MIN_ITEMS) {
+                continue;
+            }
+            // EVERY member must carry a real anchor — one linkless <li> breaks
+            // homogeneity and rejects the group, which is what keeps a <ul> of
+            // plain-text list items (e.g. recipe ingredients) from triggering.
+            if (!children.every(child => navigationAnchors(child).length > 0)) {
+                continue;
+            }
+            const items = [];
+            for (const child of children) {
+                const item = extractItem(child, baseUrl);
+                if (item) {
+                    items.push(item);
+                }
+            }
+            if (items.length < MIN_ITEMS) {
+                continue;
+            }
+            candidates.push({
+                containerSelector: describeSelector(container),
+                distinctPathnames: distinctPathnames(items),
+                itemTag: children[0].tagName,
+                items,
+                totalScore: items.reduce((sum, item) => sum + item.score, 0),
+            });
+        }
+    }
+    return candidates;
+}
+function confidenceFor(items) {
+    if (items.length < MIN_ITEMS) {
+        return 'low';
+    }
+    if (items.length >= HIGH_CONF_ITEMS) {
+        const avg = items.reduce((sum, item) => sum + item.score, 0) / items.length;
+        return avg >= HIGH_CONF_AVG_SCORE ? 'high' : 'medium';
+    }
+    return 'medium';
+}
+function notDetected(note) {
+    return {
+        confidence: 'low',
+        containerSelector: '',
+        detected: false,
+        itemCount: 0,
+        itemTag: '',
+        items: [],
+        note,
+    };
+}
+// Detect a list/feed/index structure on a non-article page. Strips chrome
+// (nav/header/footer/aside + ARIA roles) first so an article's menu doesn't
+// look like a 4-item feed, then walks for containers whose direct children
+// form a homogeneous sibling group of ≥3 elements each carrying a real anchor.
+// Winner selection prefers the candidate whose items navigate to the most
+// distinct destinations — the signature of a real feed — over a same-path
+// metadata cluster (HN's subtext rows all link to /item?id=…). Item count is
+// the second key (a 30-item feed beats a 4-item related-links block) and total
+// per-item score breaks remaining ties.
+export function detectList(document, baseUrl) {
+    stripLandmarkChrome(document);
+    const candidates = collectCandidates(document, baseUrl);
+    if (candidates.length === 0) {
+        return notDetected('not a list: no repeated item structure with links (≥3 same-shape siblings each carrying an anchor, outside nav/header/footer/aside)');
+    }
+    let winner = candidates[0];
+    for (let i = 1; i < candidates.length; i++) {
+        const candidate = candidates[i];
+        if (candidate.distinctPathnames > winner.distinctPathnames ||
+            (candidate.distinctPathnames === winner.distinctPathnames &&
+                candidate.items.length > winner.items.length) ||
+            (candidate.distinctPathnames === winner.distinctPathnames &&
+                candidate.items.length === winner.items.length &&
+                candidate.totalScore > winner.totalScore)) {
+            winner = candidate;
+        }
+    }
+    if (winner.items.length < MIN_ITEMS) {
+        return notDetected('not a list: best candidate had fewer than 3 extracted items');
+    }
+    const items = winner.items;
+    return {
+        confidence: confidenceFor(items),
+        containerSelector: winner.containerSelector,
+        detected: true,
+        itemCount: items.length,
+        itemTag: winner.itemTag,
+        items,
+        note: `detected ${items.length} ${winner.itemTag} items in ${winner.containerSelector}`,
+    };
+}

package/src/policy/lost-signal.js

@@ -0,0 +1,44 @@
+import { countWords } from "./text.js";
+// Below this the extraction cannot be the article a reader came for. The
+// measured corpus floor: the smallest healthy extract is gatwick at 435 words;
+// the smallest known-lost one (WIRED's metered barrier) still carries 180.
+export const NEAR_EMPTY_WORDS = 120;
+// Curated junk signatures over the EXTRACTED text, not the page — gating's
+// METERED_TEXT_RE reads the whole document, which is a different question.
+// Each entry is measured on a real capture: the first two are the Daily Mail
+// player controls that survive inside itemprop="articleBody" (dailymail-a66),
+// the third is the metered barrier WIRED prints when the article itself was
+// truncated server-side.
+const DEBRIS_PROBES = [
+    { label: 'player-controls', pattern: /Loaded:\s*\d+%/ },
+    { label: 'player-controls', pattern: /Duration Time \d+:\d\d/ },
+    {
+        label: 'metered-barrier',
+        pattern: /read your last free article|subscribe to continue reading/i,
+    },
+];
+// The verdict the suggest loop fires on. Gating alone never fires it — vendor
+// SDK classes (paywall-ineligible feed badges, piano offer headers) report on
+// free articles, and the pairing is enforced by construction: `gatedReason` is
+// carried as evidence for the prompt and the report, and only
+// fallback/near-empty/debris count as reasons.
+export function assessLostSignal(input) {
+    const wordCount = countWords(input.contentText);
+    const nearEmpty = wordCount < NEAR_EMPTY_WORDS;
+    const debrisProbes = [
+        ...new Set(DEBRIS_PROBES.filter(probe => probe.pattern.test(input.contentText)).map(probe => probe.label)),
+    ];
+    const reasons = [
+        ...(input.fallbackUsed ? ['fallback-used'] : []),
+        ...(nearEmpty ? ['near-empty'] : []),
+        ...debrisProbes.map(label => `debris:${label}`),
+    ];
+    return {
+        debrisProbes,
+        fallbackUsed: input.fallbackUsed,
+        gatedReason: input.gated?.likely ? input.gated.reason : undefined,
+        nearEmpty,
+        reasons,
+        wordCount,
+    };
+}

package/src/policy/markdown.js

@@ -0,0 +1,58 @@
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+const processor = unified().use(remarkParse).use(remarkGfm);
+const HEADING_MARKERS = /^#{1,6}\s+/;
+export function parseBlocks(source) {
+    const tree = processor.parse(source);
+    const blocks = [];
+    for (const node of tree.children) {
+        const start = node.position?.start.offset;
+        const end = node.position?.end.offset;
+        if (start === undefined || end === undefined) {
+            continue;
+        }
+        const kind = node.type === 'code'
+            ? 'code'
+            : node.type === 'heading'
+                ? 'heading'
+                : 'other';
+        const depth = node.type === 'heading' ? node.depth : 0;
+        blocks.push({ depth, end, kind, start });
+    }
+    return blocks;
+}
+export function headingText(raw) {
+    return raw.replace(HEADING_MARKERS, '').trim();
+}
+// Hard-cap a run to <= maxChars pieces: pack whole lines, then hard-split any
+// single line that alone exceeds the budget.
+export function hardSplitLines(text, maxChars) {
+    const pieces = [];
+    let buffer = '';
+    function flush() {
+        if (buffer) {
+            pieces.push(buffer);
+            buffer = '';
+        }
+    }
+    for (const line of text.split('\n')) {
+        if (line.length > maxChars) {
+            flush();
+            for (let i = 0; i < line.length; i += maxChars) {
+                pieces.push(line.slice(i, i + maxChars));
+            }
+            continue;
+        }
+        const candidate = buffer ? `${buffer}\n${line}` : line;
+        if (candidate.length > maxChars) {
+            flush();
+            buffer = line;
+        }
+        else {
+            buffer = candidate;
+        }
+    }
+    flush();
+    return pieces;
+}