@j0hanz/fetch-url-mcp 1.10.21 → 1.10.22

package/dist/transform/markdown-cleanup.js

@@ -35,6 +35,9 @@ const MAX_LINE_LENGTH = 80;
 const TOC_SCAN_LIMIT = 20;
 const TOC_MAX_NON_EMPTY = 12;
 const TOC_LINK_RATIO_THRESHOLD = 0.8;
+// ── List indentation normalization ───────────────────────────────────
+const SOURCE_INDENT_STEP = 2;
+const TARGET_INDENT_STEP = 4;
 // ── Docs-chrome scan depth ───────────────────────────────────────────
 const CHROME_SCAN_LINE_LIMIT = 12;
 // ── Fence pattern ───────────────────────────────────────────────────
@@ -52,6 +55,9 @@ const REGEX = {
     TOC_HEADING: /^(?:#{1,6}\s+)?(?:table of contents|contents|on this page)\s*$/i,
     COMBINED_LINE_REMOVALS: /^(?:\[Skip to (?:main )?(?:content|navigation)\]\(#[^)]*\)|\[Skip link\]\(#[^)]*\)|Was this page helpful\??|\[Back to top\]\(#[^)]*\)|\[\s*\]\(https?:\/\/[^)]*\))\s*$/gim,
     ZERO_WIDTH_ANCHOR: /\[(?:\s|\u200B)*\]\(#[^)]*\)[ \t]*/g,
+    // ReDoS-safe: {0,30} bounds identifier backtracking, negated char class
+    // [^\u0022\u201C\u201D]* has no overlap with delimiters, and \s+ is anchored
+    // between ':' and a quote. Multi-pass capped at PROPERTY_FIX_MAX_PASSES.
     CONCATENATED_PROPS: /([a-z_][a-z0-9_]{0,30}\??:\s+)([\u0022\u201C][^\u0022\u201C\u201D]*[\u0022\u201D])([a-z_][a-z0-9_]{0,30}\??:)/g,
     DOUBLE_NEWLINE_REDUCER: /\n{3,}/g,
     HEADING_SPACING: /(^#{1,6}\s[^\n]*)\n([^\n])/gm,
@@ -101,13 +107,15 @@ function hasFollowingContent(lines, startIndex) {
     }
     return false;
 }
-function findNextNonBlankLine(lines, startIndex) {
-    for (let i = startIndex + 1; i < Math.min(lines.length, startIndex + HAS_FOLLOWING_LOOKAHEAD); i++) {
-        const line = lines[i];
-        if (!isBlank(line))
-            return line?.trim();
+function findNextNonBlank(lines, startIndex, maxLookahead) {
+    const limit = maxLookahead !== undefined
+        ? Math.min(lines.length, startIndex + maxLookahead)
+        : lines.length;
+    for (let i = startIndex; i < limit; i++) {
+        if (!isBlank(lines[i]))
+            return i;
     }
-    return undefined;
+    return -1;
 }
 function stripAnchorOnlyHeading(line) {
     return line.replace(/^(#{1,6})\s+\[([^\]]+)\]\(#[^)]+\)\s*$/, '$1 $2');
@@ -140,33 +148,37 @@ function isTitleCaseOrKeyword(trimmed) {
     }
     return capitalizedCount >= TITLE_MIN_CAPITALIZED;
 }
+function isMarkdownStructuralLine(trimmed) {
+    const firstChar = trimmed.charCodeAt(0);
+    if (firstChar !== ASCII_MARKERS.HASH &&
+        firstChar !== ASCII_MARKERS.DASH &&
+        firstChar !== ASCII_MARKERS.ASTERISK &&
+        firstChar !== ASCII_MARKERS.PLUS &&
+        firstChar !== ASCII_MARKERS.BRACKET_OPEN &&
+        (firstChar < ASCII_MARKERS.DIGIT_0 || firstChar > ASCII_MARKERS.DIGIT_9)) {
+        return false;
+    }
+    return (REGEX.HEADING_MARKER.test(trimmed) ||
+        REGEX.LIST_MARKER.test(trimmed) ||
+        /^\d+\.\s/.test(trimmed) ||
+        /^\[.*\]\(.*\)$/.test(trimmed));
+}
+function isTerminalPunctuation(charCode) {
+    return (charCode === ASCII_MARKERS.PERIOD ||
+        charCode === ASCII_MARKERS.EXCLAMATION ||
+        charCode === ASCII_MARKERS.QUESTION);
+}
 function getHeadingPrefix(trimmed) {
     if (trimmed.length > MAX_LINE_LENGTH)
         return null;
     if (REPL_PROMPT_LINE.test(trimmed))
         return null;
-    // Fast path: Check common markdown markers first
-    const firstChar = trimmed.charCodeAt(0);
-    if (firstChar === ASCII_MARKERS.HASH ||
-        firstChar === ASCII_MARKERS.DASH ||
-        firstChar === ASCII_MARKERS.ASTERISK ||
-        firstChar === ASCII_MARKERS.PLUS ||
-        firstChar === ASCII_MARKERS.BRACKET_OPEN ||
-        (firstChar >= ASCII_MARKERS.DIGIT_0 && firstChar <= ASCII_MARKERS.DIGIT_9)) {
-        if (REGEX.HEADING_MARKER.test(trimmed) ||
-            REGEX.LIST_MARKER.test(trimmed) ||
-            /^\d+\.\s/.test(trimmed) ||
-            /^\[.*\]\(.*\)$/.test(trimmed)) {
-            return null;
-        }
-    }
+    if (isMarkdownStructuralLine(trimmed))
+        return null;
     if (SPECIAL_PREFIXES.test(trimmed)) {
         return /^example:\s/i.test(trimmed) ? '### ' : '## ';
     }
-    const lastChar = trimmed.charCodeAt(trimmed.length - 1);
-    if (lastChar === ASCII_MARKERS.PERIOD ||
-        lastChar === ASCII_MARKERS.EXCLAMATION ||
-        lastChar === ASCII_MARKERS.QUESTION)
+    if (isTerminalPunctuation(trimmed.charCodeAt(trimmed.length - 1)))
         return null;
     return isTitleCaseOrKeyword(trimmed) ? '## ' : null;
 }
@@ -219,57 +231,59 @@ function isTypeDocArtifactLine(line) {
     }
     return false;
 }
-function tryPromoteOrphan(lines, i, trimmed) {
-    const prevLine = lines[i - 1];
-    const isOrphan = i === 0 || !prevLine || prevLine.trim().length === 0;
+function tryPromoteOrphan(ctx) {
+    const prevLine = ctx.lines[ctx.index - 1];
+    const isOrphan = ctx.index === 0 || !prevLine || prevLine.trim().length === 0;
     if (!isOrphan)
         return null;
-    const prefix = getHeadingPrefix(trimmed);
+    const prefix = getHeadingPrefix(ctx.trimmed);
     if (!prefix)
         return null;
-    const isSpecialPrefix = SPECIAL_PREFIXES.test(trimmed);
-    if (!isSpecialPrefix && !hasFollowingContent(lines, i))
+    const isSpecialPrefix = SPECIAL_PREFIXES.test(ctx.trimmed);
+    if (!isSpecialPrefix && !hasFollowingContent(ctx.lines, ctx.index))
         return null;
     if (!isSpecialPrefix) {
-        const nextLine = findNextNonBlankLine(lines, i);
+        const nextIdx = findNextNonBlank(ctx.lines, ctx.index + 1, HAS_FOLLOWING_LOOKAHEAD);
+        const nextLine = nextIdx >= 0 ? ctx.lines[nextIdx]?.trim() : undefined;
         if (nextLine && REGEX.HEADING_MARKER.test(nextLine))
             return null;
     }
-    return `${prefix}${trimmed}`;
+    return `${prefix}${ctx.trimmed}`;
 }
-function shouldSkipAsToc(lines, i, trimmed, removeToc, options) {
-    if (!removeToc || !REGEX.TOC_HEADING.test(trimmed))
+function shouldSkipAsToc(ctx, removeToc, options) {
+    if (!removeToc || !REGEX.TOC_HEADING.test(ctx.trimmed))
         return null;
-    const { total, linkCount, nonLinkCount } = getTocBlockStats(lines, i);
+    const { total, linkCount, nonLinkCount } = getTocBlockStats(ctx.lines, ctx.index);
     if (total === 0 || nonLinkCount > 0)
         return null;
     const ratio = linkCount / total;
     if (ratio <= TOC_LINK_RATIO_THRESHOLD)
         return null;
     throwIfAborted(options?.signal, options?.url ?? '', 'markdown:cleanup:toc');
-    return skipTocLines(lines, i + 1);
+    return skipTocLines(ctx.lines, ctx.index + 1);
 }
-function normalizePreprocessLine(lines, i, trimmed, line, options) {
-    if (REGEX.EMPTY_HEADING_LINE.test(trimmed))
+function normalizePreprocessLine(ctx, options) {
+    if (REGEX.EMPTY_HEADING_LINE.test(ctx.trimmed))
         return null;
-    if (!REGEX.ANCHOR_ONLY_HEADING.test(trimmed))
-        return line;
-    if (!hasFollowingContent(lines, i)) {
+    if (!REGEX.ANCHOR_ONLY_HEADING.test(ctx.trimmed))
+        return ctx.line;
+    if (!hasFollowingContent(ctx.lines, ctx.index)) {
         return options?.preserveEmptyHeadings
-            ? stripAnchorOnlyHeading(trimmed)
+            ? stripAnchorOnlyHeading(ctx.trimmed)
             : null;
     }
-    return stripAnchorOnlyHeading(trimmed);
+    return stripAnchorOnlyHeading(ctx.trimmed);
 }
-function maybeSkipTocBlock(lines, i, trimmed, options) {
-    return shouldSkipAsToc(lines, i, trimmed, config.markdownCleanup.removeTocBlocks, options);
+function maybeSkipTocBlock(ctx, options) {
+    return shouldSkipAsToc(ctx, config.markdownCleanup.removeTocBlocks, options);
 }
-function maybePromoteOrphanHeading(lines, i, trimmed, checkAbort) {
-    if (!config.markdownCleanup.promoteOrphanHeadings || trimmed.length === 0) {
+function maybePromoteOrphanHeading(ctx, checkAbort) {
+    if (!config.markdownCleanup.promoteOrphanHeadings ||
+        ctx.trimmed.length === 0) {
         return null;
     }
     checkAbort('markdown:cleanup:promote');
-    return tryPromoteOrphan(lines, i, trimmed);
+    return tryPromoteOrphan(ctx);
 }
 function preprocessLines(lines, options) {
     const checkAbort = createAbortChecker(options);
@@ -280,15 +294,16 @@ function preprocessLines(lines, options) {
             continue;
         const currentLine = lines[i] ?? '';
         const trimmed = currentLine.trim();
-        const normalizedLine = normalizePreprocessLine(lines, i, trimmed, currentLine, options);
+        const ctx = { lines, index: i, trimmed, line: currentLine };
+        const normalizedLine = normalizePreprocessLine(ctx, options);
         if (normalizedLine === null)
             continue;
-        const tocSkip = maybeSkipTocBlock(lines, i, trimmed, options);
+        const tocSkip = maybeSkipTocBlock(ctx, options);
         if (tocSkip !== null) {
             skipUntil = tocSkip;
             continue;
         }
-        const promotedLine = maybePromoteOrphanHeading(lines, i, trimmed, checkAbort);
+        const promotedLine = maybePromoteOrphanHeading(ctx, checkAbort);
         result.push(promotedLine ?? normalizedLine);
     }
     return result.join('\n');
@@ -369,13 +384,6 @@ function getHeadingInfo(line) {
         return null;
     return { level: match[1]?.length ?? 0 };
 }
-function findNextNonBlankIndex(lines, startIndex) {
-    let idx = startIndex;
-    while (idx < lines.length && isBlank(lines[idx])) {
-        idx += 1;
-    }
-    return idx;
-}
 function removeEmptyHeadingSections(text) {
     const lines = text.split('\n');
     const kept = [];
@@ -386,8 +394,8 @@ function removeEmptyHeadingSections(text) {
             kept.push(line);
             continue;
         }
-        const nextIndex = findNextNonBlankIndex(lines, i + 1);
-        const nextLine = lines[nextIndex];
+        const nextIndex = findNextNonBlank(lines, i + 1);
+        const nextLine = nextIndex >= 0 ? lines[nextIndex] : undefined;
         if (nextLine === undefined) {
             kept.push(line);
             continue;
@@ -482,9 +490,9 @@ function applyGlobalRegexes(text, options) {
 function normalizeNestedListIndentation(text) {
     return text.replace(REGEX.NESTED_LIST_INDENT, (match, spaces, marker) => {
         const count = spaces.length;
-        if (count < 2 || count % 2 !== 0)
+        if (count < SOURCE_INDENT_STEP || count % SOURCE_INDENT_STEP !== 0)
             return match;
-        const normalized = ' '.repeat((count / 2) * 4);
+        const normalized = ' '.repeat((count / SOURCE_INDENT_STEP) * TARGET_INDENT_STEP);
         return `${normalized}${marker} `;
     });
 }

package/dist/transform/transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAyCA,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,EACrB,MAAM,uBAAuB,CAAC;AAqB/B,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AA+BpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAgJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AA8XD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAyKD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACxC,GACA,MAAM,CAyBR;AA2DD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAKD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AA4bD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAgBzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAgH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,GACrB,CAAC"}
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AA0CA,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,EACrB,MAAM,uBAAuB,CAAC;AAqB/B,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AA+BpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA4LD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AA2aD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAuKD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACxC,GACA,MAAM,CAyBR;AA2DD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAKD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AA8bD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAMzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAuG1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,GACrB,CAAC"}

package/dist/transform/transform.js

@@ -3,7 +3,7 @@ import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
 import { config } from '../lib/core.js';
 import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
-import { evaluateArticleContent, extractNoscriptImages, getVisibleTextLength, normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, surfaceCodeEditorContent, } from '../lib/dom-prep.js';
+import { evaluateArticleContent, extractNoscriptImages, getVisibleTextLength, normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, stripDocsControls, surfaceCodeEditorContent, } from '../lib/dom-prep.js';
 import { isRawTextContentUrl } from '../lib/http.js';
 import { composeAbortSignal, FetchError, getErrorMessage, getUtf8ByteLength, isAsciiOnly, isObject, throwIfAborted, toError, trimDanglingTagFragment, truncateToUtf8Boundary, } from '../lib/utils.js';
 import { extractLanguageFromClassName } from './html-translators.js';
@@ -139,6 +139,36 @@ class StageTracker {
             });
         }
     }
+    runTrackedSync(url, signal, fn) {
+        const totalStage = this.start(url, 'transform:total');
+        try {
+            throwIfAborted(signal, url, 'transform:begin');
+            const result = fn();
+            this.end(totalStage, result.truncated !== undefined
+                ? { truncated: result.truncated }
+                : undefined);
+            return result;
+        }
+        catch (error) {
+            this.end(totalStage);
+            throw error;
+        }
+    }
+    async runTrackedAsync(url, signal, fn) {
+        const totalStage = this.start(url, 'transform:total');
+        try {
+            throwIfAborted(signal, url, 'transform:begin');
+            const result = await fn();
+            this.end(totalStage, result.truncated !== undefined
+                ? { truncated: result.truncated }
+                : undefined);
+            return result;
+        }
+        catch (error) {
+            this.end(totalStage);
+            throw error;
+        }
+    }
 }
 const stageTracker = new StageTracker();
 export function startTransformStage(url, stage, budget) {
@@ -148,7 +178,7 @@ export function endTransformStage(context, options) {
     return stageTracker.end(context, options);
 }
 function truncateHtml(html, inputTruncated = false) {
-    const maxSize = config.constants.maxHtmlSize;
+    const maxSize = config.constants.maxHtmlBytes;
     if (maxSize <= 0)
         return { html, truncated: false };
     if (html.length <= maxSize) {
@@ -182,7 +212,7 @@ function isReadabilityCompatible(doc) {
             'function' &&
         typeof record.querySelector === 'function');
 }
-function resolveCollapsedTextLengthUpTo(text, max) {
+function getNormalizedTextLengthUpTo(text, max) {
     if (max <= 0)
         return 0;
     let length = 0;
@@ -234,6 +264,45 @@ function preserveAlertElements(doc) {
         el.replaceWith(bq);
     }
 }
+function preserveHeadingLayouts(doc) {
+    // Readability aggressively drops elements matching /header/i in their class/id.
+    // Many technical docs use `<div class="layout__header">` to wrap their title and intro text,
+    // causing the ENTIRE intro and H1 to be dropped.
+    for (const heading of doc.querySelectorAll('h1, h2')) {
+        let p = heading.parentNode;
+        while (p && p.tagName !== 'BODY' && p.tagName !== 'HTML') {
+            const cls = p.getAttribute('class');
+            if (cls && /header/i.test(cls)) {
+                p.setAttribute('class', cls.replace(/header/gi, 'hdr-preserved'));
+            }
+            const id = p.getAttribute('id');
+            if (id && /header/i.test(id)) {
+                p.setAttribute('id', id.replace(/header/gi, 'hdr-preserved'));
+            }
+            p = p.parentNode;
+        }
+    }
+    // To prevent Readability from penalizing sibling document sections
+    // (e.g. intro vs reference tables) and picking only one, we unwrap structural wrappers inside main boundaries.
+    for (const main of doc.querySelectorAll('main, [role="main"], article')) {
+        for (const child of Array.from(main.children)) {
+            // Don't unwrap nav, aside, or blockquotes (alerts are already converted to blockquotes here)
+            if (child.tagName === 'DIV' ||
+                child.tagName === 'HEADER' ||
+                child.tagName === 'SECTION') {
+                // preserve specific structural features Readability might want to keep
+                const cls = child.getAttribute('class') ?? '';
+                if (cls.includes('mermaid'))
+                    continue;
+                const frag = doc.createDocumentFragment();
+                while (child.firstChild) {
+                    frag.appendChild(child.firstChild);
+                }
+                child.replaceWith(frag);
+            }
+        }
+    }
+}
 function preserveCodeLanguageAttributes(doc) {
     for (const el of doc.querySelectorAll('pre, code')) {
         if (el.getAttribute('data-language'))
@@ -247,9 +316,11 @@ function prepareReadabilityDocument(readabilityDoc) {
     extractNoscriptImages(readabilityDoc);
     preserveGalleryImages(readabilityDoc);
     preserveAlertElements(readabilityDoc);
+    preserveHeadingLayouts(readabilityDoc);
     preserveCodeLanguageAttributes(readabilityDoc);
     normalizeTabContent(readabilityDoc);
     surfaceCodeEditorContent(readabilityDoc);
+    stripDocsControls(readabilityDoc);
     for (const el of readabilityDoc.querySelectorAll('[class*="breadcrumb"],[class*="pagination"]')) {
         if (el.tagName === 'HTML' || el.tagName === 'BODY')
             continue;
@@ -261,7 +332,7 @@ function validateReaderability(doc, url, signal) {
     const rawText = doc.querySelector('body')?.textContent ??
         doc.documentElement.textContent ??
         '';
-    const textLength = resolveCollapsedTextLengthUpTo(rawText, MIN_READERABLE_TEXT_LENGTH + 1);
+    const textLength = getNormalizedTextLengthUpTo(rawText, MIN_READERABLE_TEXT_LENGTH + 1);
     if (textLength < MIN_SPA_CONTENT_LENGTH) {
         logWarn('Very minimal server-rendered content detected (< 100 chars). ' +
             'This might be a client-side rendered (SPA) application. ' +
@@ -360,7 +431,7 @@ function createEmptyExtractionContext() {
     return { article: null, metadata: {}, document };
 }
 function extractEarlyMetadataIfNeeded(html, url) {
-    const maxSize = config.constants.maxHtmlSize;
+    const maxSize = config.constants.maxHtmlBytes;
     if (maxSize <= 0)
         return null;
     if (html.length <= maxSize &&
@@ -447,27 +518,26 @@ function findBalancedCloseParen(text, start) {
     return -1;
 }
 function findInlineLink(markdown, start) {
-    let searchFrom = start;
-    while (searchFrom < markdown.length) {
-        const openBracket = markdown.indexOf('[', searchFrom);
-        if (openBracket === -1)
-            return null;
+    let openBracket = markdown.indexOf('[', start);
+    while (openBracket !== -1) {
         const closeBracket = markdown.indexOf(']', openBracket + 1);
         if (closeBracket === -1)
             return null;
         if (markdown[closeBracket + 1] !== '(') {
-            searchFrom = closeBracket + 1;
+            openBracket = markdown.indexOf('[', closeBracket + 1);
             continue;
         }
         const closeParen = findBalancedCloseParen(markdown, closeBracket + 2);
         if (closeParen === -1)
             return null;
-        const prefixStart = openBracket > 0 && markdown[openBracket - 1] === '!'
-            ? openBracket - 1
-            : openBracket;
-        const prefix = markdown.slice(prefixStart, closeBracket + 1);
-        const href = markdown.slice(closeBracket + 2, closeParen);
-        return { prefixStart, closeParen, prefix, href };
+        const isImage = openBracket > 0 && markdown[openBracket - 1] === '!';
+        const prefixStart = isImage ? openBracket - 1 : openBracket;
+        return {
+            prefixStart,
+            closeParen,
+            prefix: markdown.slice(prefixStart, closeBracket + 1),
+            href: markdown.slice(closeBracket + 2, closeParen),
+        };
     }
     return null;
 }
@@ -646,7 +716,7 @@ function resolveContentTitle(params) {
             normalizeSyntheticTitleToken(params.primaryHeading),
     };
 }
-const CONTENT_ROOT_SELECTORS = [
+const CONTENT_REGION_SELECTORS = [
     'article',
     'main',
     '[role="main"]',
@@ -662,14 +732,12 @@ const CONTENT_ROOT_SELECTORS = [
     '.post-body',
     '.article-body',
 ];
-const PRIMARY_HEADING_ROOT_SELECTORS = [
-    ...CONTENT_ROOT_SELECTORS,
+const HEADING_REGION_EXTRA_SELECTORS = [
     '.markdown-body',
-    '.entry-content',
     '[itemprop="text"]',
 ];
 function findContentRoot(document) {
-    for (const selector of CONTENT_ROOT_SELECTORS) {
+    for (const selector of CONTENT_REGION_SELECTORS) {
         const element = document.querySelector(selector);
         if (!element)
             continue;
@@ -702,7 +770,10 @@ function findPrimaryHeading(document) {
     const globalHeading = extractHeadingText(document, PRIMARY_HEADING_SELECTORS_GLOBAL);
     if (globalHeading)
         return globalHeading;
-    for (const selector of PRIMARY_HEADING_ROOT_SELECTORS) {
+    for (const selector of [
+        ...CONTENT_REGION_SELECTORS,
+        ...HEADING_REGION_EXTRA_SELECTORS,
+    ]) {
         const root = document.querySelector(selector);
         if (!root)
             continue;
@@ -759,39 +830,44 @@ function buildRawSource(base, params) {
         title: params.extractedMeta.title,
     };
 }
-function buildContentSource(params) {
-    const { html, url, article, extractedMeta, includeMetadata, evaluatedArticleDoc, document, truncated, signal, } = params;
-    const useArticleContent = evaluatedArticleDoc !== null;
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
+function resolveBaseContentSource(input) {
+    const { html, url, article, extractedMeta, includeMetadata, evaluatedArticleDoc, document, truncated, signal, } = input;
+    const metadata = createContentMetadataBlock(url, article, extractedMeta, evaluatedArticleDoc !== null, includeMetadata);
     const preparedDocument = document
         ? prepareContentSourceDocument(document, url, signal)
         : undefined;
-    const primaryHeading = preparedDocument?.primaryHeading;
     const base = {
         favicon: extractedMeta.favicon,
         metadata,
         extractedMetadata: extractedMeta,
         truncated,
-        primaryHeading,
+        primaryHeading: preparedDocument?.primaryHeading,
         originalHtml: html,
     };
-    if (evaluatedArticleDoc && article) {
+    return { base, preparedDocument };
+}
+function buildContentSource(input) {
+    const { base, preparedDocument } = resolveBaseContentSource(input);
+    if (input.evaluatedArticleDoc && input.article) {
         return buildArticleSource(base, {
-            evaluatedArticleDoc,
-            article,
-            extractedMeta,
-            url,
-            signal,
+            evaluatedArticleDoc: input.evaluatedArticleDoc,
+            article: input.article,
+            extractedMeta: input.extractedMeta,
+            url: input.url,
+            signal: input.signal,
         });
     }
     if (preparedDocument) {
         return buildDocumentSource(base, {
             resolvedDocument: preparedDocument.document,
-            html,
-            extractedMeta,
+            html: input.html,
+            extractedMeta: input.extractedMeta,
         });
     }
-    return buildRawSource(base, { html, extractedMeta });
+    return buildRawSource(base, {
+        html: input.html,
+        extractedMeta: input.extractedMeta,
+    });
 }
 function resolveContentSource(params) {
     const { article, metadata: extractedMeta, document, truncated, } = extractContentContext(params.html, params.url, {
@@ -857,10 +933,6 @@ function resolveTransformContentResult(html, url, options, signal) {
     }));
     return buildMarkdownFromContext(context, url, signal);
 }
-function completeTrackedTransform(totalStage, result) {
-    stageTracker.end(totalStage, result.truncated !== undefined ? { truncated: result.truncated } : undefined);
-    return result;
-}
 const REPLACEMENT_CHAR = '\ufffd';
 const BINARY_INDICATOR_THRESHOLD = 0.1;
 function hasBinaryIndicators(content) {
@@ -879,16 +951,10 @@ function hasBinaryIndicators(content) {
 }
 export function transformHtmlToMarkdownInProcess(html, url, options) {
     const signal = buildTransformSignal(options.signal);
-    const totalStage = stageTracker.start(url, 'transform:total');
-    try {
-        throwIfAborted(signal, url, 'transform:begin');
+    return stageTracker.runTrackedSync(url, signal, () => {
         validateBinaryContent(html, url);
-        return completeTrackedTransform(totalStage, resolveTransformContentResult(html, url, options, signal));
-    }
-    catch (error) {
-        stageTracker.end(totalStage);
-        throw error;
-    }
+        return resolveTransformContentResult(html, url, options, signal);
+    });
 }
 function validateBinaryContent(html, url) {
     if (hasBinaryIndicators(html)) {
@@ -957,15 +1023,7 @@ async function runWorkerTransformWithFallback(htmlOrBuffer, url, options) {
     });
 }
 async function transformInputToMarkdown(htmlOrBuffer, url, options) {
-    const totalStage = stageTracker.start(url, 'transform:total');
-    try {
-        throwIfAborted(options.signal, url, 'transform:begin');
-        return completeTrackedTransform(totalStage, await runWorkerTransformWithFallback(htmlOrBuffer, url, options));
-    }
-    catch (error) {
-        stageTracker.end(totalStage);
-        throw error;
-    }
+    return stageTracker.runTrackedAsync(url, options.signal, () => runWorkerTransformWithFallback(htmlOrBuffer, url, options));
 }
 export async function transformHtmlToMarkdown(html, url, options) {
     return transformInputToMarkdown(html, url, options);

package/dist/transform/worker-pool.d.ts

@@ -24,6 +24,8 @@ declare class WorkerPool implements TransformWorkerPool {
     private closed;
     private taskIdSeq;
     private busyCount;
+    private draining;
+    private readonly restartBackoff;
     constructor(size: number, timeoutMs: number);
     transform(html: string, url: string, options: {
         includeMetadata: boolean;

package/dist/transform/worker-pool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EACV,uBAAuB,EAKxB,MAAM,YAAY,CAAC;AAqIpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAmJD,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAErD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAyCnC,aAAa,IAAI,MAAM;IAIvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAoDzB,OAAO,CAAC,aAAa;YA4BP,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IA6BtB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,eAAe;IA+BvB,OAAO,CAAC,mBAAmB;IA8B3B,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAUhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,iBAAiB;IAoCzB,OAAO,CAAC,gBAAgB;IA8CxB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,YAAY;CAGrB;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}
+{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EACV,uBAAuB,EAKxB,MAAM,YAAY,CAAC;AAqIpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAmJD,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAErD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6B;gBAEhD,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAyCnC,aAAa,IAAI,MAAM;IAIvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAoDzB,OAAO,CAAC,aAAa;YA4BP,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IA6BtB,OAAO,CAAC,aAAa;IAyBrB,OAAO,CAAC,eAAe;IAgCvB,OAAO,CAAC,mBAAmB;IA8B3B,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAUhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IAgClB,OAAO,CAAC,iBAAiB;IAoCzB,OAAO,CAAC,gBAAgB;IA8CxB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,YAAY;CAGrB;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}