@j0hanz/fetch-url-mcp 1.6.1 → 1.7.0

package/dist/tasks/manager.js

@@ -4,7 +4,7 @@ import { randomUUID } from 'node:crypto';
 import { setInterval } from 'node:timers';
 import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
 import { config } from '../lib/core.js';
-import { RESOURCE_NOT_FOUND_ERROR_CODE, toError } from '../lib/utils.js';
+import { toError } from '../lib/utils.js';
 import { createUnrefTimeout } from '../lib/utils.js';
 const DEFAULT_TTL_MS = 60_000;
 const MIN_TTL_MS = 1_000;
@@ -69,8 +69,6 @@ class TaskManager {
     applyTaskUpdate(task, updates) {
         Object.assign(task, updates);
         task.lastUpdatedAt = new Date().toISOString();
-        // Slide TTL window on every activity so long-running tasks don't expire mid-flight.
-        task._createdAtMs = Date.now();
     }
     cancelActiveTask(task, statusMessage) {
         this.applyTaskUpdate(task, {
@@ -322,7 +320,7 @@ class TaskManager {
                     cleanup();
                     this.removeWaiter(taskId, waiter);
                     this.removeTask(taskId);
-                    rejectInContext(new McpError(RESOURCE_NOT_FOUND_ERROR_CODE, 'Task expired', {
+                    rejectInContext(new McpError(ErrorCode.InvalidParams, 'Task expired', {
                         taskId,
                     }));
                 });

package/dist/tasks/owner.d.ts

@@ -30,3 +30,4 @@ export declare function resolveToolCallContext(extra?: HandlerExtra): ToolCallCo
 export declare function isServerResult(value: unknown): value is ServerResult;
 export declare function tryReadToolStructuredError(value: unknown): string | undefined;
 export {};
+//# sourceMappingURL=owner.d.ts.map

package/dist/tasks/owner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"owner.d.ts","sourceRoot":"","sources":["../../src/tasks/owner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAOhE,UAAU,YAAY;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED;;sDAEsD;AACtD,KAAK,SAAS,CAAC,CAAC,SAAS,MAAM,IAAI;KAChC,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,KAAK,GACnD,KAAK,GACL,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;CAClC,CAAC;AAEF,wBAAgB,OAAO,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAQ9D;AA2BD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAyB1E;AAED,wBAAgB,mBAAmB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,MAAM,CAMhE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,eAAe,CAQ5E;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAIpE;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAkB7E"}

package/dist/tasks/tool-registry.d.ts

@@ -10,3 +10,4 @@ export declare function unregisterTaskCapableTool(name: string): void;
 export declare function getTaskCapableTool(name: string): TaskCapableToolDescriptor | undefined;
 export declare function hasTaskCapableTool(name: string): boolean;
 export declare function hasRegisteredTaskCapableTools(): boolean;
+//# sourceMappingURL=tool-registry.d.ts.map

package/dist/tasks/tool-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-registry.d.ts","sourceRoot":"","sources":["../../src/tasks/tool-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,MAAM,WAAW,yBAAyB,CAAC,KAAK,GAAG,OAAO;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,KAAK,CAAC;IACzC,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,gBAAgB,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3E;AAID,wBAAgB,uBAAuB,CAAC,KAAK,EAC3C,UAAU,EAAE,yBAAyB,CAAC,KAAK,CAAC,GAC3C,IAAI,CAKN;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,GACX,yBAAyB,GAAG,SAAS,CAEvC;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,6BAA6B,IAAI,OAAO,CAEvD"}

package/dist/tools/fetch-url.d.ts

@@ -1,12 +1,9 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { ContentBlock } from '@modelcontextprotocol/sdk/types.js';
+import { z } from 'zod';
 import { type ToolHandlerExtra } from '../lib/mcp-tools.js';
-interface FetchUrlInput {
-    url: string;
-    skipNoiseRemoval?: boolean | undefined;
-    forceRefresh?: boolean | undefined;
-    maxInlineChars?: number | undefined;
-}
+import { fetchUrlInputSchema } from '../schemas/inputs.js';
+type FetchUrlInput = z.infer<typeof fetchUrlInputSchema>;
 type ToolContentBlockUnion = ContentBlock;
 interface ToolResponseBase {
     [key: string]: unknown;
@@ -26,3 +23,4 @@ export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHa
 export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
 export declare function registerTools(server: McpServer): void;
 export {};
+//# sourceMappingURL=fetch-url.d.ts.map

package/dist/tools/fetch-url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,SAAS,EAEV,MAAM,yCAAyC,CAAC;AACjD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAwBxB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAS3D,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEzD,KAAK,qBAAqB,GAAG,YAAY,CAAC;AAE1C,UAAU,gBAAgB;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,eAAO,MAAM,mBAAmB,cAAc,CAAC;AA8T/C,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,EACpB,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAgDD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAC5E,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC7D,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAmBvD;AAwBD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAwCrD"}

package/dist/tools/fetch-url.js

@@ -7,6 +7,7 @@ import { handleToolError } from '../lib/mcp-tools.js';
 import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
 import { createProgressReporter, } from '../lib/mcp-tools.js';
 import { isAbortError, isObject, toError } from '../lib/utils.js';
+import { formatZodError } from '../lib/zod.js';
 import { fetchUrlInputSchema } from '../schemas/inputs.js';
 import { fetchUrlOutputSchema } from '../schemas/outputs.js';
 import { registerTaskCapableTool, unregisterTaskCapableTool, } from '../tasks/tool-registry.js';
@@ -26,7 +27,6 @@ const TOOL_ICON = {
     src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
     mimeType: 'image/svg+xml',
 };
-const JSON_SCHEMA_DRAFT_2020_12_URI = 'https://json-schema.org/draft/2020-12/schema';
 /* -------------------------------------------------------------------------------------------------
  * Tool response builders
  * ------------------------------------------------------------------------------------------------- */
@@ -121,6 +121,42 @@ function buildResponse(pipeline, inlineResult, inputUrl) {
 /* -------------------------------------------------------------------------------------------------
  * fetch-url tool implementation
  * ------------------------------------------------------------------------------------------------- */
+function isCodeHost(host) {
+    return (host === 'github.com' || host === 'gitlab.com' || host === 'bitbucket.org');
+}
+function summarizeCodeHostPath(host, parts) {
+    if (!isCodeHost(host) || parts.length < 2)
+        return null;
+    const p0 = parts[0] ?? '';
+    const p1 = parts[1] ?? '';
+    return `${host}/${p0}/${p1}`;
+}
+function summarizeWikipediaPath(parts) {
+    if (parts[0] !== 'wiki' || parts.length < 2)
+        return null;
+    const p1 = parts[1] ?? '';
+    return `wikipedia.org/${p1}`;
+}
+function truncatePathSegment(segment, max = 20) {
+    if (segment.length <= max)
+        return segment;
+    return `${segment.substring(0, max - 3)}...`;
+}
+function summarizeUrlPath(host, parts) {
+    const codeHostSummary = summarizeCodeHostPath(host, parts);
+    if (codeHostSummary)
+        return codeHostSummary;
+    if (host.endsWith('wikipedia.org')) {
+        const wikipediaSummary = summarizeWikipediaPath(parts);
+        if (wikipediaSummary)
+            return wikipediaSummary;
+    }
+    const basename = truncatePathSegment(parts.at(-1) ?? '');
+    if (parts.length === 1) {
+        return `${host}/${basename}`;
+    }
+    return basename ? `${host}/…/${basename}` : host;
+}
 function getUrlContext(urlStr) {
     try {
         const u = new URL(urlStr);
@@ -131,38 +167,14 @@ function getUrlContext(urlStr) {
         const parts = path.split('/').filter(Boolean);
         if (parts.length === 0)
             return host;
-        // Special case for GitHub/GitLab/Bitbucket
-        if (host === 'github.com' ||
-            host === 'gitlab.com' ||
-            host === 'bitbucket.org') {
-            if (parts.length >= 2) {
-                const p0 = parts[0] ?? '';
-                const p1 = parts[1] ?? '';
-                return `${host}/${p0}/${p1}`;
-            }
-        }
-        // Special case for Wikipedia
-        if (host.endsWith('wikipedia.org') &&
-            parts[0] === 'wiki' &&
-            parts.length >= 2) {
-            const p1 = parts[1] ?? '';
-            return `wikipedia.org/${p1}`;
-        }
-        let basename = parts.pop() ?? '';
-        if (basename && basename.length > 20) {
-            basename = `${basename.substring(0, 17)}...`;
-        }
-        if (parts.length === 0) {
-            return `${host}/${basename}`;
-        }
-        return basename ? `${host}/…/${basename}` : host;
+        return summarizeUrlPath(host, parts);
     }
     catch {
         return 'unknown';
     }
 }
-async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
-    return performSharedFetch({
+function buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
+    return {
         url,
         ...withSignal(signal),
         ...(skipNoiseRemoval ? { cacheVary: { skipNoiseRemoval: true } } : {}),
@@ -174,7 +186,10 @@ async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefre
         },
         serialize: serializeMarkdownResult,
         deserialize: parseCachedMarkdownResult,
-    });
+    };
+}
+async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
+    return performSharedFetch(buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars));
 }
 function formatContentSize(chars) {
     if (chars < 1000)
@@ -216,21 +231,12 @@ export async function fetchUrlToolHandler(input, extra) {
         return handleToolError(error, input.url, 'Failed to fetch URL');
     });
 }
-function withJsonSchema202012(schema) {
-    if (typeof schema['$schema'] === 'string')
-        return schema;
-    return {
-        $schema: JSON_SCHEMA_DRAFT_2020_12_URI,
-        ...schema,
-    };
-}
 const TOOL_DEFINITION = {
     name: FETCH_URL_TOOL_NAME,
     title: 'Fetch URL',
     description: FETCH_URL_TOOL_DESCRIPTION,
     inputSchema: fetchUrlInputSchema,
-    // Explicitly mark JSON Schema dialect for MCP clients and static reviews.
-    outputSchema: withJsonSchema202012(z.toJSONSchema(fetchUrlOutputSchema)),
+    outputSchema: z.toJSONSchema(fetchUrlOutputSchema),
     handler: fetchUrlToolHandler,
     execution: {
         taskSupport: 'optional',
@@ -299,11 +305,7 @@ export function registerTools(server) {
         parseArguments: (args) => {
             const parsed = fetchUrlInputSchema.safeParse(args);
             if (!parsed.success) {
-                const flat = z.flattenError(parsed.error);
-                const details = Object.entries(flat.fieldErrors)
-                    .map(([k, v]) => `${k}: ${v.join(', ')}`)
-                    .join('; ') || flat.formErrors.join('; ');
-                throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${details}`);
+                throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${formatZodError(parsed.error)}`);
             }
             return parsed.data;
         },

package/dist/tools/index.d.ts

@@ -1,2 +1,3 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 export declare function registerAllTools(server: McpServer): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/tools/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAIzE,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAExD"}

package/dist/transform/html-translators.d.ts

@@ -1 +1,2 @@
 export declare function translateHtmlFragmentToMarkdown(html: string): string;
+//# sourceMappingURL=html-translators.d.ts.map

package/dist/transform/html-translators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AAghBA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}

package/dist/transform/html-translators.js

@@ -157,9 +157,9 @@ function resolveImageSrc(getAttribute) {
         if (url)
             return url;
     }
-    // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
+    // If the only available src is a data URI, omit the image URL entirely.
     if (isDataUri(srcRaw))
-        return '[data URI removed]';
+        return '';
     return '';
 }
 function deriveAltFromImageUrl(src) {
@@ -203,6 +203,9 @@ function buildImageTranslator(ctx) {
         : undefined;
     const src = resolveImageSrc(getAttribute);
     const existingAlt = getAttribute?.('alt') ?? '';
+    if (!src) {
+        return { content: existingAlt.trim() };
+    }
     const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
     const markdown = `![${alt}](${src})`;
     return { content: markdown };

package/dist/transform/metadata.d.ts

@@ -2,3 +2,4 @@ import type { ExtractedMetadata } from './types.js';
 export declare function extractMetadata(document: Document, baseUrl?: string): ExtractedMetadata;
 export declare function extractMetadataFromHead(html: string, baseUrl?: string): ExtractedMetadata | null;
 export declare function mergeMetadata(early: ExtractedMetadata | null, late: ExtractedMetadata): ExtractedMetadata;
+//# sourceMappingURL=metadata.d.ts.map

package/dist/transform/metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../src/transform/metadata.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAyKpD,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,iBAAiB,CAenB;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,iBAAiB,GAAG,IAAI,CAY1B;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,iBAAiB,GAAG,IAAI,EAC/B,IAAI,EAAE,iBAAiB,GACtB,iBAAiB,CAmBnB"}

package/dist/transform/metadata.js

@@ -171,6 +171,7 @@ export function mergeMetadata(early, late) {
         'description',
         'author',
         'image',
+        'favicon',
         'publishedAt',
         'modifiedAt',
     ];

package/dist/transform/{workers/shared.d.ts → shared.d.ts}

@@ -1,7 +1,8 @@
-import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from '../types.js';
+import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from './types.js';
 interface WorkerMessageHandlerOptions {
     sendMessage: (message: TransformWorkerOutgoingMessage) => void;
     runTransform: (html: string, url: string, options: TransformOptions) => MarkdownTransformResult;
 }
 export declare function createTransformMessageHandler(options: WorkerMessageHandlerOptions): (raw: unknown) => void;
 export {};
+//# sourceMappingURL=shared.d.ts.map

package/dist/transform/shared.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/transform/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,8BAA8B,EAE/B,MAAM,YAAY,CAAC;AAEpB,UAAU,2BAA2B;IACnC,WAAW,EAAE,CAAC,OAAO,EAAE,8BAA8B,KAAK,IAAI,CAAC;IAC/D,YAAY,EAAE,CACZ,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,KACtB,uBAAuB,CAAC;CAC9B;AAkGD,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,2BAA2B,GACnC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAgFxB"}

package/dist/transform/{workers/shared.js → shared.js}

@@ -1,4 +1,4 @@
-import { FetchError, getErrorMessage } from '../../lib/utils.js';
+import { FetchError, getErrorMessage } from '../lib/utils.js';
 function isTransformMessage(message) {
     if (!message || typeof message !== 'object')
         return false;

package/dist/transform/transform.d.ts

@@ -34,3 +34,4 @@ type TransformExecutionOptions = TransformOptions & {
 export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
 export declare function transformBufferToMarkdown(htmlBuffer: Uint8Array, url: string, options: TransformExecutionOptions): Promise<MarkdownTransformResult>;
 export {};
+//# sourceMappingURL=transform.d.ts.map

package/dist/transform/transform.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAsCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA4ID,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAkUD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAqPD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuKD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AA6DD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAqB3B;AAmQD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CA8CzB;AAED,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAoI1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}

package/dist/transform/transform.js

@@ -3,8 +3,7 @@ import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
 import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
-import { removeNoiseFromHtml } from '../lib/content.js';
-import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/content.js';
+import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/content.js';
 import { config } from '../lib/core.js';
 import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
 import { isRawTextContentUrl } from '../lib/http.js';
@@ -14,7 +13,6 @@ import { isObject } from '../lib/utils.js';
 import { translateHtmlFragmentToMarkdown } from './html-translators.js';
 import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
 import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
-const utf8Decoder = new TextDecoder('utf-8');
 function decodeInput(input, encoding) {
     if (typeof input === 'string')
         return input;
@@ -22,13 +20,15 @@ function decodeInput(input, encoding) {
     if (!normalizedEncoding ||
         normalizedEncoding === 'utf-8' ||
         normalizedEncoding === 'utf8') {
-        return utf8Decoder.decode(input);
+        const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
+        return buf.toString('utf8');
     }
     try {
         return new TextDecoder(normalizedEncoding).decode(input);
     }
     catch {
-        return utf8Decoder.decode(input);
+        const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
+        return buf.toString('utf8');
     }
 }
 function asError(value) {
@@ -888,13 +888,20 @@ function buildContentSource(params) {
         };
     }
     if (document) {
-        const cleanedHtml = skipNoiseRemoval
-            ? html
-            : removeNoiseFromHtml(html, document, url, signal);
+        if (skipNoiseRemoval) {
+            return {
+                ...base,
+                sourceHtml: html,
+                title: extractedMeta.title,
+                skipNoiseRemoval: true,
+                document,
+            };
+        }
+        prepareDocumentForMarkdown(document, url, signal);
         const contentRoot = findContentRoot(document);
         return {
             ...base,
-            sourceHtml: contentRoot ?? cleanedHtml,
+            sourceHtml: contentRoot ?? serializeDocumentForMarkdown(document, html),
             title: extractedMeta.title,
             skipNoiseRemoval: true,
             document,
@@ -912,7 +919,7 @@ function resolveContentSource(params) {
         ...(params.signal ? { signal: params.signal } : {}),
         ...(params.inputTruncated ? { inputTruncated: true } : {}),
     });
-    const useArticleContent = article
+    const useArticleContent = !params.skipNoiseRemoval && article
         ? shouldUseArticleContent(article, document)
         : false;
     return buildContentSource({
@@ -965,11 +972,11 @@ function hasBinaryIndicators(content) {
     if (content.includes('\x00'))
         return true;
     const sampleSize = Math.min(content.length, 2000);
-    const sample = content.slice(0, sampleSize);
     let replacementCount = 0;
-    for (const char of sample) {
-        if (char === REPLACEMENT_CHAR)
-            replacementCount++;
+    let i = -1;
+    while ((i = content.indexOf(REPLACEMENT_CHAR, i + 1)) !== -1 &&
+        i < sampleSize) {
+        replacementCount++;
     }
     return replacementCount > sampleSize * BINARY_INDICATOR_THRESHOLD;
 }

package/dist/transform/types.d.ts

@@ -101,10 +101,6 @@ export interface TransformWorkerTransformMessage {
     skipNoiseRemoval?: boolean | undefined;
     inputTruncated?: boolean | undefined;
 }
-export interface TransformWorkerCancelMessage {
-    type: 'cancel';
-    id: string;
-}
 export interface TransformWorkerCancelledMessage {
     type: 'cancelled';
     id: string;
@@ -127,3 +123,4 @@ export interface TransformWorkerErrorMessage {
 }
 export type TransformWorkerOutgoingMessage = TransformWorkerResultMessage | TransformWorkerErrorMessage | TransformWorkerCancelledMessage;
 export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/transform/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/transform/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,WAAW,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,WAAW,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC;CACH;AAED,MAAM,MAAM,8BAA8B,GACtC,4BAA4B,GAC5B,2BAA2B,GAC3B,+BAA+B,CAAC"}

package/dist/transform/worker-pool.d.ts

@@ -1,19 +1,4 @@
-import { type Transferable as NodeTransferable } from 'node:worker_threads';
-import type { MarkdownTransformResult, TransformWorkerCancelMessage, TransformWorkerTransformMessage } from './types.js';
-type TransformWorkerMessage = TransformWorkerTransformMessage | TransformWorkerCancelMessage;
-interface WorkerHost {
-    kind: 'thread' | 'process';
-    supportsTransferList: boolean;
-    threadId?: number;
-    pid?: number;
-    postMessage: (message: TransformWorkerMessage, transferList?: NodeTransferable[]) => void;
-    terminate: () => Promise<void>;
-    unref: () => void;
-    onMessage: (handler: (raw: unknown) => void) => void;
-    onError: (handler: (error: unknown) => void) => void;
-    onExit: (handler: (code: number | null, signal: NodeJS.Signals | null) => void) => void;
-}
-type WorkerSpawner = (workerIndex: number, name: string) => WorkerHost;
+import type { MarkdownTransformResult } from './types.js';
 interface TransformWorkerPool {
     transform(html: string, url: string, options: {
         includeMetadata: boolean;
@@ -38,10 +23,9 @@ declare class WorkerPool implements TransformWorkerPool {
     private readonly cancelAcks;
     private readonly timeoutMs;
     private readonly queueMax;
-    private readonly spawnWorkerImpl;
     private closed;
     private taskIdSeq;
-    constructor(size: number, timeoutMs: number, spawnWorker: WorkerSpawner);
+    constructor(size: number, timeoutMs: number);
     transform(html: string, url: string, options: {
         includeMetadata: boolean;
         signal?: AbortSignal;
@@ -90,3 +74,4 @@ export declare function getWorkerPoolStats(): {
 } | null;
 export declare function shutdownWorkerPool(): Promise<void>;
 export {};
+//# sourceMappingURL=worker-pool.d.ts.map

package/dist/transform/worker-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EACV,uBAAuB,EAKxB,MAAM,YAAY,CAAC;AAwLpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAkBD,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAOvB;IAEJ,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAoCnC,aAAa,IAAI,MAAM;IAKvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAoDzB,OAAO,CAAC,aAAa;IAsCrB,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,gBAAgB;YAyBV,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,eAAe;IAuDvB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,iBAAiB;IAiFzB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,iBAAiB;CAW1B;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}