@j0hanz/fetch-url-mcp 1.9.0 → 1.9.2

package/dist/transform/transform.d.ts

@@ -20,6 +20,14 @@ export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, o
 export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtmlOrDocument: string | Document): boolean;
 export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
 export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
+declare function findContentRoot(document: Document): string | undefined;
+declare function findPrimaryHeading(document: Document): string | undefined;
+declare function isGithubRepositoryRootUrl(url: string): boolean;
+export declare const TransformHeuristics: {
+    readonly findContentRoot: typeof findContentRoot;
+    readonly findPrimaryHeading: typeof findPrimaryHeading;
+    readonly isGithubRepositoryRootUrl: typeof isGithubRepositoryRootUrl;
+};
 export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
 interface TransformPoolStats {
     queueDepth: number;

package/dist/transform/transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAsCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA4ID,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAkUD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAqPD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuKD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AA6DD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAqB3B;AAqPD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CA6CzB;AAED,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAkI1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAuCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAuJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAuVD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA+OD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAwJD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAiED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAsCD,iBAAS,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAc/D;AAED,iBAAS,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAYlE;AAED,iBAAS,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED,eAAO,MAAM,mBAAmB;;;;CAItB,CAAC;AAiQX,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAmBzB;AA+CD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAsH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}

package/dist/transform/transform.js

@@ -11,7 +11,7 @@ import { createAbortError, throwIfAborted } from '../lib/utils.js';
 import { FetchError, getErrorMessage, toError } from '../lib/utils.js';
 import { isObject } from '../lib/utils.js';
 import { translateHtmlFragmentToMarkdown } from './html-translators.js';
-import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
+import { extractMetadata, extractMetadataFromHead, mergeMetadata, normalizeDocumentTitle, } from './metadata.js';
 import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
 function decodeInput(input, encoding) {
     if (typeof input === 'string')
@@ -35,6 +35,16 @@ function asError(value) {
     return value instanceof Error ? value : undefined;
 }
 const abortPolicy = { throwIfAborted, createAbortError };
+function isWhitespaceChar(code) {
+    return code === 9 || code === 10 || code === 12 || code === 13 || code === 32;
+}
+function containsWhitespace(value) {
+    for (let i = 0; i < value.length; i += 1) {
+        if (isWhitespaceChar(value.charCodeAt(i)))
+            return true;
+    }
+    return false;
+}
 function buildTransformSignal(signal) {
     const { timeoutMs } = config.transform;
     if (timeoutMs <= 0)
@@ -232,6 +242,9 @@ function willTruncate(html) {
     const maxSize = config.constants.maxHtmlSize;
     return (maxSize > 0 && (html.length > maxSize || getUtf8ByteLength(html) > maxSize));
 }
+const MIN_SPA_CONTENT_LENGTH = 100;
+const MIN_READERABLE_TEXT_LENGTH = 400;
+const MAX_READABILITY_ELEMENTS = 20_000;
 function isReadabilityCompatible(doc) {
     if (!isObject(doc))
         return false;
@@ -270,38 +283,47 @@ function resolveCollapsedTextLengthUpTo(text, max) {
     }
     return length;
 }
+function preserveAlertElements(doc) {
+    const alerts = doc.querySelectorAll('[role="alert"], .admonition, .callout');
+    for (const el of alerts) {
+        const bq = doc.createElement('blockquote');
+        bq.innerHTML = el.innerHTML;
+        el.replaceWith(bq);
+    }
+}
 function extractArticle(document, url, signal) {
     if (!isReadabilityCompatible(document)) {
         logWarn('Document not compatible with Readability');
         return null;
     }
+    const checkAbort = (stage) => {
+        abortPolicy.throwIfAborted(signal, url, stage);
+    };
     try {
         const doc = document;
-        // F1: Check abort before DOM text content extraction
-        abortPolicy.throwIfAborted(signal, url, 'extract:article:textCheck');
+        checkAbort('extract:article:textCheck');
         const rawText = doc.querySelector('body')?.textContent ??
             doc.documentElement.textContent ??
             '';
-        const textLength = resolveCollapsedTextLengthUpTo(rawText, 401);
-        if (textLength < 100) {
+        const textLength = resolveCollapsedTextLengthUpTo(rawText, MIN_READERABLE_TEXT_LENGTH + 1);
+        if (textLength < MIN_SPA_CONTENT_LENGTH) {
             logWarn('Very minimal server-rendered content detected (< 100 chars). ' +
                 'This might be a client-side rendered (SPA) application. ' +
                 'Content extraction may be incomplete.', { textLength });
         }
-        // F1: Check abort before isProbablyReaderable (DOM traversal)
-        abortPolicy.throwIfAborted(signal, url, 'extract:article:readabilityCheck');
-        if (textLength >= 400 && !isProbablyReaderable(doc)) {
+        checkAbort('extract:article:readabilityCheck');
+        if (textLength >= MIN_READERABLE_TEXT_LENGTH &&
+            !isProbablyReaderable(doc)) {
             return null;
         }
-        // F1: Check abort before cloning document
-        abortPolicy.throwIfAborted(signal, url, 'extract:article:clone');
+        checkAbort('extract:article:clone');
         const readabilityDoc = typeof doc.cloneNode === 'function'
             ? doc.cloneNode(true)
             : doc;
-        // F1: Check abort before heavy Readability parse
-        abortPolicy.throwIfAborted(signal, url, 'extract:article:parse');
+        preserveAlertElements(readabilityDoc);
+        checkAbort('extract:article:parse');
         const reader = new Readability(readabilityDoc, {
-            maxElemsToParse: 20_000,
+            maxElemsToParse: MAX_READABILITY_ELEMENTS,
             classesToPreserve: [
                 'admonition',
                 'callout',
@@ -401,16 +423,8 @@ export function extractContent(html, url, options = {
     const result = extractContentContext(html, url, options);
     return { article: result.article, metadata: result.metadata };
 }
-function isWhitespaceChar(code) {
-    return code === 9 || code === 10 || code === 12 || code === 13 || code === 32;
-}
-function containsWhitespace(value) {
-    for (let i = 0; i < value.length; i += 1) {
-        if (isWhitespaceChar(value.charCodeAt(i)))
-            return true;
-    }
-    return false;
-}
+const ABORT_CHECK_LINE_INTERVAL = 500;
+const CR_CHAR_CODE = 13;
 function resolveRelativeHref(href, baseUrl, origin) {
     const trimmedHref = href.trim();
     if (!trimmedHref || containsWhitespace(trimmedHref))
@@ -503,39 +517,31 @@ function resolveRelativeUrls(markdown, baseUrl, signal) {
     let output = '';
     let buffer = '';
     let fenceMarker = null;
-    const flushBuffer = () => {
-        if (!buffer)
-            return;
-        output += resolveRelativeUrlsInSegment(buffer, baseUrl, origin);
-        buffer = '';
-    };
     const len = markdown.length;
     let lastIndex = 0;
     let lineCount = 0;
     while (lastIndex < len) {
-        if (++lineCount % 500 === 0) {
+        if (++lineCount % ABORT_CHECK_LINE_INTERVAL === 0) {
             abortPolicy.throwIfAborted(signal, baseUrl, 'markdown:resolve-urls');
         }
+        // Extract next line (handling CR+LF)
         let nextIndex = markdown.indexOf('\n', lastIndex);
-        let line;
-        let lineWithNewline;
-        if (nextIndex === -1) {
-            line = markdown.slice(lastIndex);
-            lineWithNewline = line;
+        const isLastLine = nextIndex === -1;
+        if (isLastLine)
             nextIndex = len;
-        }
-        else {
-            if (nextIndex > lastIndex && markdown.charCodeAt(nextIndex - 1) === 13) {
-                line = markdown.slice(lastIndex, nextIndex - 1);
-            }
-            else {
-                line = markdown.slice(lastIndex, nextIndex);
-            }
-            lineWithNewline = markdown.slice(lastIndex, nextIndex + 1);
-            nextIndex++; // Skip \n
-        }
-        const trimmed = line.trimStart();
+        const lineWithNewline = isLastLine
+            ? markdown.slice(lastIndex)
+            : markdown.slice(lastIndex, nextIndex + 1);
+        const lineEnd = !isLastLine &&
+            nextIndex > lastIndex &&
+            markdown.charCodeAt(nextIndex - 1) === CR_CHAR_CODE
+            ? nextIndex - 1
+            : isLastLine
+                ? len
+                : nextIndex;
+        const trimmed = markdown.slice(lastIndex, lineEnd).trimStart();
         if (fenceMarker) {
+            // Inside a code fence — pass through without URL resolution
             output += lineWithNewline;
             if (trimmed.startsWith(fenceMarker) &&
                 trimmed.slice(fenceMarker.length).trim() === '') {
@@ -543,9 +549,13 @@ function resolveRelativeUrls(markdown, baseUrl, signal) {
             }
         }
         else {
-            const fenceMatch = FENCE_LINE_PATTERN.exec(line);
+            const fenceMatch = FENCE_LINE_PATTERN.exec(markdown.slice(lastIndex, lineEnd));
             if (fenceMatch?.[1]) {
-                flushBuffer();
+                // Entering a code fence — flush buffered content first
+                if (buffer) {
+                    output += resolveRelativeUrlsInSegment(buffer, baseUrl, origin);
+                    buffer = '';
+                }
                 output += lineWithNewline;
                 fenceMarker = fenceMatch[1];
             }
@@ -553,9 +563,11 @@ function resolveRelativeUrls(markdown, baseUrl, signal) {
                 buffer += lineWithNewline;
             }
         }
-        lastIndex = nextIndex;
+        lastIndex = isLastLine ? len : nextIndex + 1;
+    }
+    if (buffer) {
+        output += resolveRelativeUrlsInSegment(buffer, baseUrl, origin);
     }
-    flushBuffer();
     return output;
 }
 function translateHtmlToMarkdown(params) {
@@ -667,9 +679,6 @@ function resolveHtmlDocument(htmlOrDocument) {
         return parseHTML('<!DOCTYPE html><html><body></body></html>').document;
     }
 }
-function countDomSelector(htmlOrDocument, selector) {
-    return resolveHtmlDocument(htmlOrDocument).querySelectorAll(selector).length;
-}
 function countTagsInString(html, regex) {
     let count = 0;
     while (regex.exec(html) !== null) {
@@ -677,12 +686,6 @@ function countTagsInString(html, regex) {
     }
     return count;
 }
-function countHeadingsDom(htmlOrDocument) {
-    return countDomSelector(htmlOrDocument, 'h1,h2,h3,h4,h5,h6');
-}
-function countCodeBlocksDom(htmlOrDocument) {
-    return countDomSelector(htmlOrDocument, 'pre');
-}
 function stripNonVisibleNodes(root) {
     for (const el of root.querySelectorAll('script,style,noscript')) {
         el.remove();
@@ -785,6 +788,9 @@ function hasTruncatedSentences(text) {
         return false;
     return incompleteFound / linesFound > MAX_TRUNCATED_LINE_RATIO;
 }
+const MIN_CONTENT_ROOT_LENGTH = 100;
+const HEADING_SCAN_LIMIT = 12;
+const BINARY_SAMPLE_SIZE = 2000;
 export function determineContentExtractionSource(article) {
     return article !== null;
 }
@@ -797,8 +803,9 @@ export function createContentMetadataBlock(url, article, extractedMeta, shouldEx
         fetchedAt: new Date().toISOString(),
     };
     if (shouldExtractFromArticle && article) {
-        if (article.title !== undefined)
-            metadata.title = article.title;
+        if (article.title !== undefined) {
+            metadata.title = normalizeDocumentTitle(article.title, url);
+        }
         if (article.byline !== undefined)
             metadata.author = article.byline;
     }
@@ -828,6 +835,12 @@ const CONTENT_ROOT_SELECTORS = [
     '.post-body',
     '.article-body',
 ];
+const PRIMARY_HEADING_ROOT_SELECTORS = [
+    ...CONTENT_ROOT_SELECTORS,
+    '.markdown-body',
+    '.entry-content',
+    '[itemprop="text"]',
+];
 function findContentRoot(document) {
     for (const selector of CONTENT_ROOT_SELECTORS) {
         const element = document.querySelector(selector);
@@ -836,30 +849,61 @@ function findContentRoot(document) {
         const innerHTML = typeof element.innerHTML === 'string'
             ? element.innerHTML
             : undefined;
-        if (innerHTML && innerHTML.trim().length > 100)
+        if (innerHTML && innerHTML.trim().length > MIN_CONTENT_ROOT_LENGTH)
             return innerHTML;
     }
     return undefined;
 }
-function shouldUseArticleContent(article, originalHtmlOrDocument) {
+function findPrimaryHeading(document) {
+    for (const selector of PRIMARY_HEADING_ROOT_SELECTORS) {
+        const root = document.querySelector(selector);
+        if (!root)
+            continue;
+        const heading = root.querySelector('h1, h2');
+        if (!heading)
+            continue;
+        const text = heading.textContent.trim();
+        if (text)
+            return text;
+    }
+    return undefined;
+}
+function isGithubRepositoryRootUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return false;
+    }
+    const hostname = parsed.hostname.toLowerCase();
+    if (hostname !== 'github.com' && hostname !== 'www.github.com') {
+        return false;
+    }
+    return parsed.pathname.split('/').filter(Boolean).length === 2;
+}
+export const TransformHeuristics = {
+    findContentRoot,
+    findPrimaryHeading,
+    isGithubRepositoryRootUrl,
+};
+function shouldUseArticleContent(article, document) {
     const articleLength = article.textContent.length;
-    const originalLength = getVisibleTextLength(originalHtmlOrDocument);
+    const originalLength = getVisibleTextLength(document);
     if (originalLength >= MIN_HTML_LENGTH_FOR_GATE) {
         const ratio = articleLength / originalLength;
         if (ratio < MIN_CONTENT_RATIO)
             return false;
     }
-    const originalHeadings = countHeadingsDom(originalHtmlOrDocument);
+    const originalHeadings = document.querySelectorAll('h1,h2,h3,h4,h5,h6').length;
     if (originalHeadings > 0) {
-        // Optimization: Use regex on article content string instead of parsing it to DOM
         const articleHeadings = countTagsInString(article.content, /<h[1-6]\b/gi);
         const retentionRatio = articleHeadings / originalHeadings;
         if (retentionRatio < MIN_HEADING_RETENTION_RATIO)
             return false;
     }
-    const originalCodeBlocks = countCodeBlocksDom(originalHtmlOrDocument);
+    const originalCodeBlocks = document.querySelectorAll('pre').length;
     if (originalCodeBlocks > 0) {
-        // Optimization: Use regex on article content string
         const articleCodeBlocks = countTagsInString(article.content, /<pre\b/gi);
         const codeRetentionRatio = articleCodeBlocks / originalCodeBlocks;
         if (codeRetentionRatio < MIN_CODE_BLOCK_RETENTION_RATIO)
@@ -875,20 +919,27 @@ function buildContentSource(params) {
         metadata,
         extractedMetadata: extractedMeta,
         truncated,
+        primaryHeading: document
+            ? TransformHeuristics.findPrimaryHeading(document)
+            : undefined,
     };
     if (useArticleContent && article) {
         const { document: articleDoc } = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`);
         prepareDocumentForMarkdown(articleDoc, url, signal);
+        const preferPrimaryHeading = TransformHeuristics.isGithubRepositoryRootUrl(url);
         return {
             ...base,
             sourceHtml: articleDoc.body.innerHTML,
-            title: article.title,
+            title: (preferPrimaryHeading ? base.primaryHeading : undefined) ??
+                (article.title !== undefined
+                    ? normalizeDocumentTitle(article.title, url)
+                    : undefined),
             skipNoiseRemoval: true,
         };
     }
     if (document) {
         prepareDocumentForMarkdown(document, url, signal);
-        const contentRoot = findContentRoot(document);
+        const contentRoot = TransformHeuristics.findContentRoot(document);
         return {
             ...base,
             sourceHtml: contentRoot ?? serializeDocumentForMarkdown(document, html),
@@ -931,7 +982,11 @@ function buildMarkdownFromContext(context, url, signal) {
         ...(context.document ? { document: context.document } : {}),
         ...(context.skipNoiseRemoval ? { skipNoiseRemoval: true } : {}),
     }));
-    if (context.title && !content.trim().startsWith('# ')) {
+    if (context.primaryHeading &&
+        TransformHeuristics.isGithubRepositoryRootUrl(url)) {
+        content = stripLeadingHeading(content, context.primaryHeading);
+    }
+    if (context.title && !/^(#{1,6})\s/.test(content.trimStart())) {
         const icon = context.favicon;
         let prefix = ' ';
         if (icon) {
@@ -953,6 +1008,34 @@ function buildMarkdownFromContext(context, url, signal) {
         metadata: context.extractedMetadata,
     };
 }
+function normalizeHeadingText(value) {
+    return value.replace(/\s+/g, ' ').trim().toLowerCase();
+}
+function stripLeadingHeading(markdown, headingText) {
+    if (!markdown)
+        return markdown;
+    const lines = markdown.split('\n');
+    const target = normalizeHeadingText(headingText);
+    let nonEmptySeen = 0;
+    for (let i = 0; i < lines.length && nonEmptySeen < HEADING_SCAN_LIMIT; i += 1) {
+        const trimmed = lines[i]?.trim() ?? '';
+        if (!trimmed)
+            continue;
+        nonEmptySeen += 1;
+        const match = /^(#{1,6})\s+(.+?)\s*$/.exec(trimmed);
+        if (!match)
+            continue;
+        const current = normalizeHeadingText(match[2] ?? '');
+        if (current !== target)
+            return markdown;
+        lines.splice(i, 1);
+        if ((lines[i] ?? '').trim() === '') {
+            lines.splice(i, 1);
+        }
+        return lines.join('\n');
+    }
+    return markdown;
+}
 const REPLACEMENT_CHAR = '\ufffd';
 const BINARY_INDICATOR_THRESHOLD = 0.1;
 function hasBinaryIndicators(content) {
@@ -960,7 +1043,7 @@ function hasBinaryIndicators(content) {
         return false;
     if (content.includes('\x00'))
         return true;
-    const sampleSize = Math.min(content.length, 2000);
+    const sampleSize = Math.min(content.length, BINARY_SAMPLE_SIZE);
     let replacementCount = 0;
     let i = -1;
     while ((i = content.indexOf(REPLACEMENT_CHAR, i + 1)) !== -1 &&
@@ -972,37 +1055,42 @@ function hasBinaryIndicators(content) {
 export function transformHtmlToMarkdownInProcess(html, url, options) {
     const signal = buildTransformSignal(options.signal);
     const totalStage = stageTracker.start(url, 'transform:total');
-    let completed = null;
     try {
         abortPolicy.throwIfAborted(signal, url, 'transform:begin');
-        if (hasBinaryIndicators(html)) {
-            throw new FetchError('Content appears to be binary data (high replacement character ratio or null bytes)', url, 415, { reason: 'binary_content_detected', stage: 'transform:validate' });
-        }
-        const raw = stageTracker.run(url, 'transform:raw', () => tryTransformRawContent({
-            html,
-            url,
-            includeMetadata: options.includeMetadata,
-            ...(options.inputTruncated ? { inputTruncated: true } : {}),
-        }));
-        if (raw) {
-            completed = raw;
-            return raw;
-        }
-        const context = stageTracker.run(url, 'transform:extract', () => resolveContentSource({
-            html,
-            url,
-            includeMetadata: options.includeMetadata,
-            ...(signal ? { signal } : {}),
-            ...(options.inputTruncated ? { inputTruncated: true } : {}),
-        }));
-        const result = buildMarkdownFromContext(context, url, signal);
-        completed = result;
+        validateBinaryContent(html, url);
+        const result = tryRawContentPipeline(html, url, options) ??
+            tryHtmlContentPipeline(html, url, options, signal);
+        stageTracker.end(totalStage, { truncated: result.truncated });
         return result;
     }
-    finally {
-        endTotalTransformStage(totalStage, completed);
+    catch (error) {
+        stageTracker.end(totalStage);
+        throw error;
     }
 }
+function validateBinaryContent(html, url) {
+    if (hasBinaryIndicators(html)) {
+        throw new FetchError('Content appears to be binary data (high replacement character ratio or null bytes)', url, 415, { reason: 'binary_content_detected', stage: 'transform:validate' });
+    }
+}
+function tryRawContentPipeline(html, url, options) {
+    return stageTracker.run(url, 'transform:raw', () => tryTransformRawContent({
+        html,
+        url,
+        includeMetadata: options.includeMetadata,
+        ...(options.inputTruncated ? { inputTruncated: true } : {}),
+    }));
+}
+function tryHtmlContentPipeline(html, url, options, signal) {
+    const context = stageTracker.run(url, 'transform:extract', () => resolveContentSource({
+        html,
+        url,
+        includeMetadata: options.includeMetadata,
+        ...(signal ? { signal } : {}),
+        ...(options.inputTruncated ? { inputTruncated: true } : {}),
+    }));
+    return buildMarkdownFromContext(context, url, signal);
+}
 export function getTransformPoolStats() {
     return getWorkerPoolStats();
 }
@@ -1012,13 +1100,6 @@ export async function shutdownTransformWorkerPool() {
 function transformInputInProcess(htmlOrBuffer, url, options) {
     return transformHtmlToMarkdownInProcess(decodeInput(htmlOrBuffer, options.encoding), url, options);
 }
-function endTotalTransformStage(context, result) {
-    if (!result) {
-        stageTracker.end(context);
-        return;
-    }
-    stageTracker.end(context, { truncated: result.truncated });
-}
 function buildWorkerTransformOptions(options) {
     return {
         includeMetadata: options.includeMetadata,
@@ -1073,15 +1154,15 @@ async function runWorkerTransformWithFallback(htmlOrBuffer, url, options) {
 }
 async function transformInputToMarkdown(htmlOrBuffer, url, options) {
     const totalStage = stageTracker.start(url, 'transform:total');
-    let completed = null;
     try {
         abortPolicy.throwIfAborted(options.signal, url, 'transform:begin');
         const result = await runWorkerTransformWithFallback(htmlOrBuffer, url, options);
-        completed = result;
+        stageTracker.end(totalStage, { truncated: result.truncated });
         return result;
     }
-    finally {
-        endTotalTransformStage(totalStage, completed);
+    catch (error) {
+        stageTracker.end(totalStage);
+        throw error;
     }
 }
 export async function transformHtmlToMarkdown(html, url, options) {

package/dist/transform/worker-pool.d.ts

@@ -17,13 +17,13 @@ declare class WorkerPool implements TransformWorkerPool {
     private readonly minCapacity;
     private readonly maxCapacity;
     private readonly queue;
-    private queueHead;
     private readonly inflight;
     private readonly cancelAcks;
     private readonly timeoutMs;
     private readonly queueMax;
     private closed;
     private taskIdSeq;
+    private busyCount;
     constructor(size: number, timeoutMs: number);
     transform(html: string, url: string, options: {
         includeMetadata: boolean;
@@ -44,8 +44,6 @@ declare class WorkerPool implements TransformWorkerPool {
     private ensureOpen;
     private createPendingTask;
     private onAbortSignal;
-    private resolveCancelAck;
-    private waitForCancelAck;
     private abortInflight;
     private clearAbortListener;
     private spawnWorker;
@@ -57,11 +55,10 @@ declare class WorkerPool implements TransformWorkerPool {
     private failTask;
     private maybeScaleUp;
     private drainQueue;
-    private takeNextQueuedTask;
     private dispatchFromQueue;
+    private registerInflight;
+    private sendToWorker;
     private finalizeTask;
-    private findQueuedIndex;
-    private maybeCompactQueue;
 }
 export declare function getOrCreateWorkerPool(): WorkerPool;
 export declare function getWorkerPoolStats(): {

package/dist/transform/worker-pool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AAuBA,OAAO,KAAK,EACV,uBAAuB,EAGxB,MAAM,YAAY,CAAC;AAqJpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAkBD,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAOvB;IAEJ,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAmCnC,aAAa,IAAI,MAAM;IAKvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAkDzB,OAAO,CAAC,aAAa;IAsCrB,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,gBAAgB;YAyBV,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,eAAe;IAsDvB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,iBAAiB;IAiFzB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,iBAAiB;CAW1B;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}
+{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EACV,uBAAuB,EAGxB,MAAM,YAAY,CAAC;AAqJpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAiID,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA0B;IAErD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAmCnC,aAAa,IAAI,MAAM;IAIvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAkDzB,OAAO,CAAC,aAAa;YA4BP,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,eAAe;IAsDvB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAUhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,iBAAiB;IA4BzB,OAAO,CAAC,gBAAgB;IA8CxB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,YAAY;CAOrB;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}