@j0hanz/superfetch 2.7.0 → 2.7.1

package/dist/cache.js

@@ -41,14 +41,6 @@ export function parseCachedPayload(raw) {
 export function resolveCachedPayloadContent(payload) {
     return payload.markdown ?? payload.content ?? null;
 }
-function stableStringify(value) {
-    try {
-        return stableJsonStringify(value);
-    }
-    catch {
-        return null;
-    }
-}
 function createHashFragment(input, length) {
     return sha256Hex(input).substring(0, length);
 }
@@ -63,9 +55,18 @@ export function createCacheKey(namespace, url, vary) {
     const urlHash = createHashFragment(url, CACHE_CONSTANTS.URL_HASH_LENGTH);
     let varyHash;
     if (vary) {
-        const varyString = typeof vary === 'string' ? vary : stableStringify(vary);
-        if (varyString === null)
-            return null;
+        let varyString;
+        if (typeof vary === 'string') {
+            varyString = vary;
+        }
+        else {
+            try {
+                varyString = stableJsonStringify(vary);
+            }
+            catch {
+                return null;
+            }
+        }
         if (varyString) {
             varyHash = createHashFragment(varyString, CACHE_CONSTANTS.VARY_HASH_LENGTH);
         }

package/dist/markdown-cleanup.js

@@ -22,7 +22,6 @@ const REGEX = {
     SPACING_ADJ_COMBINED: /(?:\]\([^)]+\)|`[^`]+`)(?=[A-Za-z0-9])/g,
     SPACING_CODE_DASH: /(`[^`]+`)\s*\\-\s*/g,
     SPACING_ESCAPES: /\\([[\].])/g,
-    SPACING_URL_ENC: /\]\([^)]*%5[Ff][^)]*\)/g,
     SPACING_LIST_NUM_COMBINED: /^((?![-*+] |\d+\. |[ \t]).+)\n((?:[-*+]|\d+\.) )/gm,
     TYPEDOC: /(`+)(?:(?!\1)[\s\S])*?\1|\s?\/\\?\*[\s\S]*?\\?\*\//g,
 };
@@ -203,7 +202,6 @@ function applyGlobalRegexes(text) {
         .replace(REGEX.SPACING_ADJ_COMBINED, '$& ')
         .replace(REGEX.SPACING_CODE_DASH, '$1 - ')
         .replace(REGEX.SPACING_ESCAPES, '$1')
-        .replace(REGEX.SPACING_URL_ENC, (m) => m.replace(/%5[Ff]/g, '_'))
         .replace(REGEX.SPACING_LIST_NUM_COMBINED, '$1\n\n$2')
         .replace(REGEX.DOUBLE_NEWLINE_REDUCER, '\n\n');
     // fixProperties

package/dist/resources.js

@@ -8,30 +8,18 @@ const REDACTED = '<REDACTED>';
 const CONFIG_RESOURCE_NAME = 'config';
 const CONFIG_RESOURCE_URI = 'internal://config';
 const JSON_MIME = 'application/json';
-function redactIfPresent(value) {
-    return value ? REDACTED : undefined;
-}
-function redactArray(values) {
-    return values.map(() => REDACTED);
-}
-function scrubAuth(auth) {
-    return {
-        ...auth,
-        clientSecret: redactIfPresent(auth.clientSecret),
-        staticTokens: redactArray(auth.staticTokens),
-    };
-}
-function scrubSecurity(security) {
-    return {
-        ...security,
-        apiKey: redactIfPresent(security.apiKey),
-    };
-}
 function scrubConfig(source) {
     return {
         ...source,
-        auth: scrubAuth(source.auth),
-        security: scrubSecurity(source.security),
+        auth: {
+            ...source.auth,
+            clientSecret: source.auth.clientSecret ? REDACTED : undefined,
+            staticTokens: source.auth.staticTokens.map(() => REDACTED),
+        },
+        security: {
+            ...source.security,
+            apiKey: source.security.apiKey ? REDACTED : undefined,
+        },
     };
 }
 export function registerConfigResource(server) {

package/dist/transform.js

@@ -1,7 +1,6 @@
 import { AsyncLocalStorage, AsyncResource } from 'node:async_hooks';
 import { Buffer } from 'node:buffer';
 import { fork } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
 import diagnosticsChannel from 'node:diagnostics_channel';
 import { availableParallelism } from 'node:os';
 import { performance } from 'node:perf_hooks';
@@ -202,7 +201,11 @@ function truncateHtml(html) {
     const byteLength = Buffer.byteLength(html, 'utf8');
     if (byteLength <= maxSize)
         return { html, truncated: false };
-    const htmlBuffer = Buffer.from(html, 'utf8');
+    const sliced = html.slice(0, maxSize);
+    if (Buffer.byteLength(sliced, 'utf8') <= maxSize) {
+        return { html: sliced, truncated: true };
+    }
+    const htmlBuffer = Buffer.from(sliced, 'utf8');
     let content = trimUtf8Buffer(htmlBuffer, maxSize).toString('utf8');
     // Avoid truncating inside tags.
     const lastOpen = content.lastIndexOf('<');
@@ -224,8 +227,11 @@ function willTruncate(html) {
 const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
 const MAX_HEAD_SCAN_LENGTH = 50_000;
 function extractHeadSection(html) {
-    const searchLimit = Math.min(html.length, MAX_HEAD_SCAN_LENGTH);
-    const searchText = html.substring(0, searchLimit);
+    if (html.length <= MAX_HEAD_SCAN_LENGTH) {
+        const match = HEAD_END_PATTERN.exec(html);
+        return match ? html.substring(0, match.index) : null;
+    }
+    const searchText = html.substring(0, MAX_HEAD_SCAN_LENGTH);
     const match = HEAD_END_PATTERN.exec(searchText);
     if (!match)
         return null;
@@ -875,9 +881,22 @@ function createCustomTranslators() {
         sup: () => ({
             postprocess: ({ content }) => `^${content}^`,
         }),
-        section: () => ({
-            postprocess: ({ content }) => `\n\n${content}\n\n`,
-        }),
+        section: (ctx) => {
+            if (isObject(ctx) && isObject(ctx.node)) {
+                const { node } = ctx;
+                const getAttribute = hasGetAttribute(node)
+                    ? node.getAttribute.bind(node)
+                    : undefined;
+                if (getAttribute?.('class')?.includes('tsd-member')) {
+                    return {
+                        postprocess: ({ content }) => `\n\n&nbsp;\n\n${content}\n\n`,
+                    };
+                }
+            }
+            return {
+                postprocess: ({ content }) => `\n\n${content}\n\n`,
+            };
+        },
         details: () => ({
             postprocess: ({ content }) => {
                 const trimmed = content.trim();
@@ -948,48 +967,6 @@ function containsWhitespace(value) {
     }
     return false;
 }
-function extractClassAttribute(openTag) {
-    const lower = openTag.toLowerCase();
-    const classIndex = lower.indexOf('class');
-    if (classIndex === -1)
-        return null;
-    let i = classIndex + 5;
-    while (i < lower.length && isWhitespaceChar(lower.charCodeAt(i)))
-        i += 1;
-    if (lower[i] !== '=')
-        return null;
-    i += 1;
-    while (i < lower.length && isWhitespaceChar(lower.charCodeAt(i)))
-        i += 1;
-    const quote = openTag[i];
-    if (quote !== '"' && quote !== "'")
-        return null;
-    i += 1;
-    const end = openTag.indexOf(quote, i);
-    if (end === -1)
-        return null;
-    return openTag.slice(i, end);
-}
-function skipWhitespace(text, start) {
-    let index = start;
-    while (index < text.length && isWhitespaceChar(text.charCodeAt(index))) {
-        index += 1;
-    }
-    return index;
-}
-function isTsdMemberSectionTag(openTag) {
-    const classValue = extractClassAttribute(openTag);
-    return classValue ? classValue.toLowerCase().includes('tsd-member') : false;
-}
-function findTsdMemberSectionStart(html, scan) {
-    if (scan >= html.length || !html.startsWith('<section', scan))
-        return null;
-    const tagEnd = html.indexOf('>', scan);
-    if (tagEnd === -1)
-        return null;
-    const openTag = html.slice(scan, tagEnd + 1);
-    return isTsdMemberSectionTag(openTag) ? scan : null;
-}
 function resolveRelativeHref(href, baseUrl, origin) {
     const trimmedHref = href.trim();
     if (!trimmedHref || containsWhitespace(trimmedHref))
@@ -1045,26 +1022,6 @@ function findInlineLink(markdown, start) {
     }
     return null;
 }
-function preprocessPropertySections(html) {
-    const closeTag = '</section>';
-    let cursor = 0;
-    let output = '';
-    for (let closeIndex = html.indexOf(closeTag, cursor); closeIndex !== -1; closeIndex = html.indexOf(closeTag, cursor)) {
-        const afterClose = closeIndex + closeTag.length;
-        output += html.slice(cursor, afterClose);
-        const scan = skipWhitespace(html, afterClose);
-        const sectionStart = findTsdMemberSectionStart(html, scan);
-        if (sectionStart !== null) {
-            output += '<p>&nbsp;</p>';
-            cursor = sectionStart;
-            continue;
-        }
-        output += html.slice(afterClose, scan);
-        cursor = scan;
-    }
-    output += html.slice(cursor);
-    return output;
-}
 function isAbsoluteOrSpecialUrl(href) {
     const trimmedHref = href.trim();
     if (!trimmedHref)
@@ -1102,8 +1059,7 @@ function translateHtmlToMarkdown(params) {
         ? html
         : stageTracker.run(url, 'markdown:noise', () => removeNoiseFromHtml(html, document, url));
     abortPolicy.throwIfAborted(signal, url, 'markdown:cleaned');
-    const preprocessedHtml = stageTracker.run(url, 'markdown:preprocess', () => preprocessPropertySections(cleanedHtml));
-    const content = stageTracker.run(url, 'markdown:translate', () => translateHtmlFragmentToMarkdown(preprocessedHtml));
+    const content = stageTracker.run(url, 'markdown:translate', () => translateHtmlFragmentToMarkdown(cleanedHtml));
     abortPolicy.throwIfAborted(signal, url, 'markdown:translated');
     const cleaned = cleanupMarkdownArtifacts(content);
     return url ? resolveRelativeUrls(cleaned, url) : cleaned;
@@ -1673,6 +1629,7 @@ class WorkerPool {
     queueMax;
     spawnWorkerImpl;
     closed = false;
+    taskIdSeq = 0;
     constructor(size, timeoutMs, spawnWorker) {
         if (size === 0) {
             this.capacity = 0;
@@ -1752,7 +1709,7 @@ class WorkerPool {
             throw new Error(WorkerPool.CLOSED_MESSAGE);
     }
     createPendingTask(htmlOrBuffer, url, options, resolve, reject) {
-        const id = randomUUID();
+        const id = (this.taskIdSeq++).toString(36);
         // Preserve request context for resolve/reject even when callbacks fire
         // from worker thread events.
         const context = createTaskContext();
@@ -2153,10 +2110,26 @@ async function transformWithWorkerPool(htmlOrBuffer, url, options) {
     });
 }
 function resolveWorkerFallback(error, htmlOrBuffer, url, options) {
+    const isQueueFull = error instanceof FetchError && error.details.reason === 'queue_full';
+    if (isQueueFull) {
+        logWarn('Transform worker queue full; falling back to in-process', {
+            url: redactUrl(url),
+        });
+        return transformHtmlToMarkdownInProcess(decodeUtf8Input(htmlOrBuffer), url, options);
+    }
     if (error instanceof FetchError)
         throw error;
     abortPolicy.throwIfAborted(options.signal, url, 'transform:worker-fallback');
-    return transformHtmlToMarkdownInProcess(decodeUtf8Input(htmlOrBuffer), url, options);
+    const message = getErrorMessage(error);
+    logWarn('Transform worker failed; refusing in-process fallback', {
+        url: redactUrl(url),
+        error: message,
+    });
+    throw new FetchError('Transform worker failed', url, 503, {
+        reason: 'worker_failed',
+        stage: 'transform:worker',
+        error: message,
+    });
 }
 async function transformInputToMarkdown(htmlOrBuffer, url, options) {
     const totalStage = stageTracker.start(url, 'transform:total');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.7.0",
+  "version": "2.7.1",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",