@j0hanz/superfetch 1.2.2 → 1.2.4

package/dist/services/fetcher/interceptors.d.ts

@@ -1,3 +1,25 @@
+export type FetchChannelEvent = {
+    v: 1;
+    type: 'start';
+    requestId: string;
+    method: string;
+    url: string;
+} | {
+    v: 1;
+    type: 'end';
+    requestId: string;
+    status: number;
+    duration: number;
+} | {
+    v: 1;
+    type: 'error';
+    requestId: string;
+    url: string;
+    error: string;
+    code?: string;
+    status?: number;
+    duration: number;
+};
 interface FetchTelemetryContext {
     requestId: string;
     startTime: number;

package/dist/services/fetcher/interceptors.js

@@ -70,13 +70,18 @@ function publishFetchEnd(context, status, duration) {
     });
 }
 function buildResponseMeta(response, contentSize, duration) {
-    const contentType = response.headers.get('content-type') ?? undefined;
     const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
-    return {
-        contentType,
+    const meta = {
         duration: `${Math.round(duration)}ms`,
-        size: contentLength,
     };
+    const contentType = response.headers.get('content-type');
+    if (contentType !== null) {
+        meta.contentType = contentType;
+    }
+    if (contentLength !== undefined) {
+        meta.size = contentLength;
+    }
+    return meta;
 }
 function logSlowRequestIfNeeded(context, duration) {
     if (duration <= 5000)
@@ -91,16 +96,21 @@ export function recordFetchError(context, error, status) {
     const duration = performance.now() - context.startTime;
     const err = error instanceof Error ? error : new Error(String(error));
     const code = isSystemError(err) ? err.code : undefined;
-    publishFetchEvent({
+    const event = {
         v: 1,
         type: 'error',
         requestId: context.requestId,
         url: context.url,
         error: err.message,
-        code,
-        status,
         duration,
-    });
+    };
+    if (code !== undefined) {
+        event.code = code;
+    }
+    if (status !== undefined) {
+        event.status = status;
+    }
+    publishFetchEvent(event);
     const log = status === 429 ? logWarn : logError;
     log('HTTP Request Error', {
         requestId: context.requestId,

package/dist/services/fetcher/response.js

@@ -1,3 +1,5 @@
+import { Readable, Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { FetchError } from '../../errors/app-error.js';
 function assertContentLengthWithinLimit(response, url, maxBytes) {
     const contentLengthHeader = response.headers.get('content-length');
@@ -9,37 +11,43 @@ function assertContentLengthWithinLimit(response, url, maxBytes) {
     }
     throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
 }
-function throwIfReadAborted(url, signal) {
-    if (!signal?.aborted)
-        return;
-    throw new FetchError('Request was aborted during response read', url, 499, {
-        reason: 'aborted',
-    });
-}
 async function readStreamWithLimit(stream, url, maxBytes, signal) {
-    const reader = stream.getReader();
     const decoder = new TextDecoder();
     let total = 0;
-    const chunks = [];
-    try {
-        for (;;) {
-            throwIfReadAborted(url, signal);
-            const { value, done } = await reader.read();
-            if (done)
-                break;
-            total += value.byteLength;
+    let text = '';
+    const toBuffer = (chunk) => {
+        if (typeof chunk === 'string') {
+            return Buffer.from(chunk);
+        }
+        return Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    };
+    const sink = new Writable({
+        write(chunk, _encoding, callback) {
+            const buffer = toBuffer(chunk);
+            total += buffer.length;
             if (total > maxBytes) {
-                await reader.cancel();
-                throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+                callback(new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url));
+                return;
             }
-            chunks.push(decoder.decode(value, { stream: true }));
-        }
-        chunks.push(decoder.decode());
-        return { text: chunks.join(''), size: total };
+            text += decoder.decode(buffer, { stream: true });
+            callback();
+        },
+        final(callback) {
+            text += decoder.decode();
+            callback();
+        },
+    });
+    try {
+        const readable = Readable.fromWeb(stream, { signal });
+        await pipeline(readable, sink, { signal });
     }
-    finally {
-        reader.releaseLock();
+    catch (error) {
+        if (signal?.aborted) {
+            throw new FetchError('Request was aborted during response read', url, 499, { reason: 'aborted' });
+        }
+        throw error;
     }
+    return { text, size: total };
 }
 export async function readResponseText(response, url, maxBytes, signal) {
     assertContentLengthWithinLimit(response, url, maxBytes);

package/dist/services/fetcher.d.ts

@@ -1,5 +1,4 @@
 import type { FetchOptions } from '../config/types/runtime.js';
 import { destroyAgents } from './fetcher/agents.js';
 export { destroyAgents };
-export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
 export declare function fetchNormalizedUrlWithRetry(normalizedUrl: string, options?: FetchOptions, maxRetries?: number): Promise<string>;

package/dist/services/fetcher.js

@@ -1,6 +1,5 @@
 import { config } from '../config/index.js';
 import { normalizeHeaderRecord } from '../utils/header-normalizer.js';
-import { validateAndNormalizeUrl } from '../utils/url-validator.js';
 import { destroyAgents, dispatcher } from './fetcher/agents.js';
 import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
 import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
@@ -66,20 +65,19 @@ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
         throw mapped;
     }
 }
-export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
-    const normalizedUrl = await validateAndNormalizeUrl(url);
-    return fetchNormalizedUrlWithRetry(normalizedUrl, options, maxRetries);
-}
 export async function fetchNormalizedUrlWithRetry(normalizedUrl, options, maxRetries = 3) {
     const context = buildRequestContext(options);
     return executeWithRetry(normalizedUrl, maxRetries, async () => runFetch(normalizedUrl, context), context.signal);
 }
 function buildRequestContext(options) {
-    return {
+    const context = {
         timeoutMs: options?.timeout ?? config.fetcher.timeout,
         headers: buildHeaders(options?.customHeaders),
-        signal: options?.signal,
     };
+    if (options?.signal) {
+        context.signal = options.signal;
+    }
+    return context;
 }
 async function runFetch(normalizedUrl, context) {
     const signal = buildRequestSignal(context.timeoutMs, context.signal);

package/dist/services/metadata-collector.d.ts

@@ -0,0 +1,10 @@
+import type { ExtractedMetadata } from '../config/types/content.js';
+export type MetaSource = 'og' | 'twitter' | 'standard';
+export type MetaField = keyof ExtractedMetadata;
+export interface MetaCollectorState {
+    title: Partial<Record<MetaSource, string>>;
+    description: Partial<Record<MetaSource, string>>;
+    author: Partial<Record<MetaSource, string>>;
+}
+export declare function createMetaCollectorState(): MetaCollectorState;
+export declare function resolveMetaField(state: MetaCollectorState, field: MetaField): string | undefined;

package/dist/services/metadata-collector.js

@@ -0,0 +1,11 @@
+export function createMetaCollectorState() {
+    return {
+        title: {},
+        description: {},
+        author: {},
+    };
+}
+export function resolveMetaField(state, field) {
+    const sources = state[field];
+    return sources.og ?? sources.twitter ?? sources.standard;
+}

package/dist/services/parser.js

@@ -6,18 +6,8 @@ import { getErrorMessage } from '../utils/error-utils.js';
 import { truncateHtml } from '../utils/html-truncator.js';
 import { sanitizeText } from '../utils/sanitizer.js';
 import { logWarn } from './logger.js';
+import { createMetaCollectorState, resolveMetaField, } from './metadata-collector.js';
 const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
-function createMetaCollectorState() {
-    return {
-        title: {},
-        description: {},
-        author: {},
-    };
-}
-function resolveMetaField(state, field) {
-    const sources = state[field];
-    return sources.og ?? sources.twitter ?? sources.standard;
-}
 function extractMetadata($) {
     const state = createMetaCollectorState();
     $('meta').each((_, element) => {
@@ -55,11 +45,17 @@ function extractMetadata($) {
             state.title.standard = titleText;
         }
     }
-    return {
-        title: resolveMetaField(state, 'title'),
-        description: resolveMetaField(state, 'description'),
-        author: resolveMetaField(state, 'author'),
-    };
+    const metadata = {};
+    const title = resolveMetaField(state, 'title');
+    const description = resolveMetaField(state, 'description');
+    const author = resolveMetaField(state, 'author');
+    if (title !== undefined)
+        metadata.title = title;
+    if (description !== undefined)
+        metadata.description = description;
+    if (author !== undefined)
+        metadata.author = author;
+    return metadata;
 }
 function parseHeading($, element) {
     const rawText = sanitizeText($(element).text());
@@ -109,11 +105,14 @@ function parseCode($, element) {
     const dataLang = $(element).attr('data-language') ?? '';
     const language = resolveLanguageFromAttributes(className, dataLang) ??
         detectLanguageFromCode(text);
-    return {
+    const block = {
         type: 'code',
-        language,
         text,
     };
+    if (language !== undefined) {
+        block.language = language;
+    }
+    return block;
 }
 function parseTable($, element) {
     const headers = [];
@@ -144,21 +143,23 @@ function parseTable($, element) {
     });
     if (rows.length === 0)
         return null;
-    return {
-        type: 'table',
-        headers: headers.length > 0 ? headers : undefined,
-        rows,
-    };
+    return headers.length > 0
+        ? { type: 'table', headers, rows }
+        : { type: 'table', rows };
 }
 function parseImage($, element) {
     const src = $(element).attr('src');
     if (!src)
         return null;
-    return {
+    const alt = $(element).attr('alt');
+    const image = {
         type: 'image',
         src,
-        alt: $(element).attr('alt') ?? undefined,
     };
+    if (alt !== undefined) {
+        image.alt = alt;
+    }
+    return image;
 }
 function parseBlockquote($, element) {
     const rawText = sanitizeText($(element).text());

package/dist/services/transform-worker-pool.d.ts

@@ -0,0 +1,14 @@
+import type { JsonlTransformResult, MarkdownTransformResult, TransformOptions } from '../config/types/content.js';
+type TransformMode = 'jsonl' | 'markdown' | 'markdown-blocks';
+export interface TransformJob {
+    mode: TransformMode;
+    html: string;
+    url: string;
+    options: TransformOptions & {
+        includeContentBlocks?: boolean;
+    };
+}
+type TransformResult = JsonlTransformResult | MarkdownTransformResult;
+export declare function runTransformInWorker(job: TransformJob): Promise<TransformResult | null>;
+export declare function destroyTransformWorkers(): void;
+export {};

package/dist/services/transform-worker-pool.js

@@ -0,0 +1,167 @@
+import os from 'node:os';
+import { isMainThread, Worker } from 'node:worker_threads';
+import { config } from '../config/index.js';
+import { getErrorMessage } from '../utils/error-utils.js';
+import { logWarn } from './logger.js';
+const MAX_POOL_SIZE = 4;
+function resolvePoolSize() {
+    const available = os.availableParallelism();
+    return Math.max(1, Math.min(available - 1, MAX_POOL_SIZE));
+}
+let pool = null;
+let poolDisabled = false;
+function shouldUseWorkers() {
+    return isMainThread && config.runtime.httpMode && !poolDisabled;
+}
+function getWorkerUrl() {
+    return new URL('../workers/transform-worker.js', import.meta.url);
+}
+export async function runTransformInWorker(job) {
+    if (!shouldUseWorkers())
+        return null;
+    if (!pool) {
+        try {
+            pool = new TransformWorkerPool(getWorkerUrl(), resolvePoolSize());
+        }
+        catch (error) {
+            poolDisabled = true;
+            logWarn('Failed to initialize transform worker pool', {
+                error: getErrorMessage(error),
+            });
+            return null;
+        }
+    }
+    try {
+        return await pool.run(job);
+    }
+    catch (error) {
+        poolDisabled = true;
+        pool.destroy();
+        pool = null;
+        logWarn('Transform worker failed; falling back to main thread', {
+            error: getErrorMessage(error),
+        });
+        return null;
+    }
+}
+export function destroyTransformWorkers() {
+    pool?.destroy();
+    pool = null;
+}
+class TransformWorkerPool {
+    workerUrl;
+    size;
+    workers = [];
+    queue = [];
+    pending = new Map();
+    nextId = 1;
+    destroyed = false;
+    constructor(workerUrl, size) {
+        this.workerUrl = workerUrl;
+        this.size = size;
+        for (let i = 0; i < size; i += 1) {
+            this.workers.push(this.createWorker());
+        }
+    }
+    run(job) {
+        if (this.destroyed) {
+            return Promise.reject(new Error('Transform worker pool is closed'));
+        }
+        const id = this.nextId++;
+        const queuedJob = { ...job, id };
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+            this.queue.push(queuedJob);
+            this.schedule();
+        });
+    }
+    destroy() {
+        if (this.destroyed)
+            return;
+        this.destroyed = true;
+        for (const workerState of this.workers) {
+            void workerState.worker.terminate();
+        }
+        for (const [id, pending] of this.pending.entries()) {
+            pending.reject(new Error('Transform worker pool shut down'));
+            this.pending.delete(id);
+        }
+        this.queue.length = 0;
+    }
+    createWorker() {
+        const worker = new Worker(this.workerUrl);
+        worker.unref();
+        const state = { worker, busy: false, currentJobId: undefined };
+        worker.on('message', (message) => {
+            this.handleMessage(state, message);
+        });
+        worker.on('error', (error) => {
+            this.handleWorkerError(state, error);
+        });
+        worker.on('exit', (code) => {
+            this.handleWorkerExit(state, code);
+        });
+        return state;
+    }
+    handleMessage(state, message) {
+        const pending = this.pending.get(message.id);
+        if (pending) {
+            this.pending.delete(message.id);
+            if (message.ok) {
+                pending.resolve(message.result);
+            }
+            else {
+                pending.reject(new Error(message.error));
+            }
+        }
+        state.busy = false;
+        state.currentJobId = undefined;
+        this.schedule();
+    }
+    handleWorkerError(state, error) {
+        this.failCurrentJob(state, error);
+        this.replaceWorker(state);
+    }
+    handleWorkerExit(state, code) {
+        if (code !== 0) {
+            this.failCurrentJob(state, new Error(`Transform worker exited with code ${code}`));
+        }
+        this.replaceWorker(state);
+    }
+    failCurrentJob(state, error) {
+        if (!state.currentJobId)
+            return;
+        const pending = this.pending.get(state.currentJobId);
+        if (pending) {
+            pending.reject(error);
+            this.pending.delete(state.currentJobId);
+        }
+        state.currentJobId = undefined;
+        state.busy = false;
+    }
+    replaceWorker(state) {
+        if (this.destroyed)
+            return;
+        const index = this.workers.indexOf(state);
+        if (index === -1)
+            return;
+        this.workers[index] = this.createWorker();
+        this.schedule();
+    }
+    schedule() {
+        if (this.destroyed)
+            return;
+        for (const workerState of this.workers) {
+            if (this.queue.length === 0)
+                return;
+            if (workerState.busy)
+                continue;
+            const job = this.queue.shift();
+            if (!job)
+                return;
+            workerState.busy = true;
+            workerState.currentJobId = job.id;
+            workerState.worker.postMessage(job);
+        }
+    }
+}

package/dist/tools/handlers/fetch-markdown.tool.d.ts

@@ -1,4 +1,12 @@
 import type { FetchMarkdownInput, ToolResponseBase } from '../../config/types/tools.js';
+import { transformHtmlToMarkdownAsync } from '../utils/content-transform-async.js';
+import { performSharedFetch } from './fetch-single.shared.js';
 export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
 export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter and content length limits";
-export declare function fetchMarkdownToolHandler(input: FetchMarkdownInput): Promise<ToolResponseBase>;
+interface FetchMarkdownDeps {
+    readonly performSharedFetch?: typeof performSharedFetch;
+    readonly transformHtmlToMarkdown?: typeof transformHtmlToMarkdownAsync;
+}
+export declare function createFetchMarkdownToolHandler(deps?: FetchMarkdownDeps): (input: FetchMarkdownInput) => Promise<ToolResponseBase>;
+export declare const fetchMarkdownToolHandler: (input: FetchMarkdownInput) => Promise<ToolResponseBase>;
+export {};

package/dist/tools/handlers/fetch-markdown.tool.js

@@ -1,7 +1,7 @@
 import { config } from '../../config/index.js';
 import { logDebug, logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
-import { transformHtmlToMarkdown } from '../utils/content-transform.js';
+import { transformHtmlToMarkdownAsync } from '../utils/content-transform-async.js';
 import { applyInlineResultToStructuredContent, buildToolContentBlocks, getFileDownloadInfo, getInlineErrorResponse, performSharedFetch, } from './fetch-single.shared.js';
 export const FETCH_MARKDOWN_TOOL_NAME = 'fetch-markdown';
 export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter and content length limits';
@@ -39,7 +39,15 @@ function resolveMarkdownOptions(input) {
     return {
         extractMainContent: input.extractMainContent ?? config.extraction.extractMainContent,
         includeMetadata: input.includeMetadata ?? config.extraction.includeMetadata,
-        maxContentLength: input.maxContentLength,
+        ...(input.maxContentLength !== undefined && {
+            maxContentLength: input.maxContentLength,
+        }),
+    };
+}
+function buildFetchMarkdownErrorDetails() {
+    return {
+        fetchedAt: new Date().toISOString(),
+        cached: false,
     };
 }
 function buildMarkdownStructuredContent(pipeline, inlineResult, fileDownload) {
@@ -62,25 +70,30 @@ function buildMarkdownStructuredContent(pipeline, inlineResult, fileDownload) {
 function logFetchMarkdownStart(url, options) {
     logDebug('Fetching markdown', { url, ...options });
 }
-function buildMarkdownTransform(options) {
-    return (html, url) => {
-        const markdownResult = transformHtmlToMarkdown(html, url, options);
+function buildMarkdownTransform(options, transform) {
+    return async (html, url) => {
+        const markdownResult = await transform(html, url, options);
         return { ...markdownResult, content: markdownResult.markdown };
     };
 }
-async function fetchMarkdownPipeline(url, input, options, transformOptions) {
-    return performSharedFetch({
+async function fetchMarkdownPipeline(url, input, options, transformOptions, performSharedFetchImpl, transformImpl) {
+    const sharedOptions = {
         url,
         format: 'markdown',
         extractMainContent: options.extractMainContent,
         includeMetadata: options.includeMetadata,
-        maxContentLength: options.maxContentLength,
-        customHeaders: input.customHeaders,
-        retries: input.retries,
-        timeout: input.timeout,
-        transform: buildMarkdownTransform(transformOptions),
+        ...(options.maxContentLength !== undefined && {
+            maxContentLength: options.maxContentLength,
+        }),
+        ...(input.customHeaders !== undefined && {
+            customHeaders: input.customHeaders,
+        }),
+        ...(input.retries !== undefined && { retries: input.retries }),
+        ...(input.timeout !== undefined && { timeout: input.timeout }),
+        transform: buildMarkdownTransform(transformOptions, transformImpl),
         deserialize: deserializeMarkdownPipelineResult,
-    });
+    };
+    return performSharedFetchImpl(sharedOptions);
 }
 function buildMarkdownResponse(pipeline, inlineResult, fileDownload) {
     const structuredContent = buildMarkdownStructuredContent(pipeline, inlineResult, fileDownload);
@@ -89,33 +102,48 @@ function buildMarkdownResponse(pipeline, inlineResult, fileDownload) {
         structuredContent,
     };
 }
-export async function fetchMarkdownToolHandler(input) {
-    try {
-        return await executeFetchMarkdown(input);
-    }
-    catch (error) {
-        logError('fetch-markdown tool error', error instanceof Error ? error : undefined);
-        return handleToolError(error, input.url, 'Failed to fetch markdown');
-    }
+export function createFetchMarkdownToolHandler(deps = {}) {
+    const performSharedFetchImpl = deps.performSharedFetch ?? performSharedFetch;
+    const transformImpl = deps.transformHtmlToMarkdown ?? transformHtmlToMarkdownAsync;
+    return async (input) => {
+        try {
+            return await executeFetchMarkdown(input, performSharedFetchImpl, transformImpl);
+        }
+        catch (error) {
+            logError('fetch-markdown tool error', error instanceof Error ? error : undefined);
+            const errorDetails = buildFetchMarkdownErrorDetails();
+            return handleToolError(error, input.url, 'Failed to fetch markdown', errorDetails);
+        }
+    };
 }
-async function executeFetchMarkdown(input) {
+export const fetchMarkdownToolHandler = createFetchMarkdownToolHandler();
+async function executeFetchMarkdown(input, performSharedFetchImpl, transformImpl) {
     const { url } = input;
     if (!url) {
-        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR');
+        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR', buildFetchMarkdownErrorDetails());
     }
     const options = resolveMarkdownOptions(input);
     const transformOptions = { ...options };
     logFetchMarkdownStart(url, transformOptions);
-    const { pipeline, inlineResult } = await fetchMarkdownPipeline(url, input, options, transformOptions);
-    const inlineError = getInlineErrorResponse(inlineResult, url);
+    const { pipeline, inlineResult } = await fetchMarkdownPipeline(url, input, options, transformOptions, performSharedFetchImpl, transformImpl);
+    const inlineError = getInlineErrorResponse(inlineResult, url, buildFetchMarkdownErrorDetails());
     if (inlineError)
         return inlineError;
-    const fileDownload = inlineResult.resourceUri
-        ? getFileDownloadInfo({
+    let fileDownload = null;
+    if (inlineResult.resourceUri) {
+        const downloadContext = {
             cacheKey: pipeline.cacheKey ?? null,
             url: pipeline.url,
-            title: pipeline.data.title,
-        })
-        : null;
+        };
+        if (pipeline.data.title !== undefined) {
+            fileDownload = getFileDownloadInfo({
+                ...downloadContext,
+                title: pipeline.data.title,
+            });
+        }
+        else {
+            fileDownload = getFileDownloadInfo(downloadContext);
+        }
+    }
     return buildMarkdownResponse(pipeline, inlineResult, fileDownload);
 }