@j0hanz/superfetch 1.2.4 → 2.0.0

package/dist/services/transform-worker-pool.js

@@ -1,167 +1,215 @@
-import os from 'node:os';
-import { isMainThread, Worker } from 'node:worker_threads';
+import { Worker } from 'node:worker_threads';
 import { config } from '../config/index.js';
 import { getErrorMessage } from '../utils/error-utils.js';
 import { logWarn } from './logger.js';
-const MAX_POOL_SIZE = 4;
-function resolvePoolSize() {
-    const available = os.availableParallelism();
-    return Math.max(1, Math.min(available - 1, MAX_POOL_SIZE));
-}
-let pool = null;
-let poolDisabled = false;
-function shouldUseWorkers() {
-    return isMainThread && config.runtime.httpMode && !poolDisabled;
-}
-function getWorkerUrl() {
-    return new URL('../workers/transform-worker.js', import.meta.url);
-}
-export async function runTransformInWorker(job) {
-    if (!shouldUseWorkers())
-        return null;
-    if (!pool) {
-        try {
-            pool = new TransformWorkerPool(getWorkerUrl(), resolvePoolSize());
-        }
-        catch (error) {
-            poolDisabled = true;
-            logWarn('Failed to initialize transform worker pool', {
-                error: getErrorMessage(error),
-            });
-            return null;
-        }
-    }
-    try {
-        return await pool.run(job);
-    }
-    catch (error) {
-        poolDisabled = true;
-        pool.destroy();
-        pool = null;
-        logWarn('Transform worker failed; falling back to main thread', {
-            error: getErrorMessage(error),
-        });
-        return null;
-    }
-}
-export function destroyTransformWorkers() {
-    pool?.destroy();
-    pool = null;
-}
+import { isWorkerResponse } from './transform-worker-types.js';
 class TransformWorkerPool {
     workerUrl;
-    size;
-    workers = [];
+    slots = [];
     queue = [];
-    pending = new Map();
     nextId = 1;
     destroyed = false;
     constructor(workerUrl, size) {
         this.workerUrl = workerUrl;
-        this.size = size;
         for (let i = 0; i < size; i += 1) {
-            this.workers.push(this.createWorker());
+            this.slots.push(this.spawnWorker());
         }
     }
-    run(job) {
+    run(request, signal) {
         if (this.destroyed) {
-            return Promise.reject(new Error('Transform worker pool is closed'));
+            return Promise.reject(new Error('Worker pool is shut down'));
         }
-        const id = this.nextId++;
-        const queuedJob = { ...job, id };
         return new Promise((resolve, reject) => {
-            this.pending.set(id, { resolve, reject });
-            this.queue.push(queuedJob);
-            this.schedule();
+            if (signal?.aborted) {
+                reject(new Error('Aborted'));
+                return;
+            }
+            const task = this.createTask(request, resolve, reject, signal);
+            this.attachAbortHandler(task, signal);
+            this.enqueueTask(task);
         });
     }
-    destroy() {
+    async destroy() {
         if (this.destroyed)
             return;
         this.destroyed = true;
-        for (const workerState of this.workers) {
-            void workerState.worker.terminate();
+        const pending = this.queue.splice(0);
+        for (const task of pending) {
+            this.cleanupTask(task);
+            task.reject(new Error('Worker pool shutting down'));
         }
-        for (const [id, pending] of this.pending.entries()) {
-            pending.reject(new Error('Transform worker pool shut down'));
-            this.pending.delete(id);
+        for (const slot of this.slots) {
+            if (slot.current) {
+                const task = slot.current;
+                slot.current = undefined;
+                slot.busy = false;
+                this.cleanupTask(task);
+                task.reject(new Error('Worker pool shutting down'));
+            }
         }
-        this.queue.length = 0;
-    }
-    createWorker() {
-        const worker = new Worker(this.workerUrl);
-        worker.unref();
-        const state = { worker, busy: false, currentJobId: undefined };
-        worker.on('message', (message) => {
-            this.handleMessage(state, message);
-        });
-        worker.on('error', (error) => {
-            this.handleWorkerError(state, error);
+        await Promise.allSettled(this.slots.map((slot) => slot.worker.terminate()));
+        this.slots = [];
+    }
+    dispatch() {
+        if (this.destroyed)
+            return;
+        const idle = this.slots.find((slot) => !slot.busy);
+        if (!idle)
+            return;
+        const task = this.queue.shift();
+        if (!task)
+            return;
+        task.status = 'running';
+        idle.busy = true;
+        idle.current = task;
+        try {
+            idle.worker.postMessage(task.request);
+        }
+        catch (error) {
+            this.failTask(idle, error);
+        }
+    }
+    createTask(request, resolve, reject, signal) {
+        const id = this.nextId;
+        this.nextId += 1;
+        return {
+            id,
+            request: { ...request, id },
+            resolve,
+            reject,
+            signal,
+            abortHandler: undefined,
+            status: 'queued',
+        };
+    }
+    attachAbortHandler(task, signal) {
+        if (!signal)
+            return;
+        const onAbort = () => {
+            if (task.status === 'queued') {
+                this.removeQueuedTask(task);
+                task.reject(new Error('Aborted'));
+                return;
+            }
+            this.abortRunningTask(task);
+        };
+        task.abortHandler = onAbort;
+        signal.addEventListener('abort', onAbort, { once: true });
+    }
+    enqueueTask(task) {
+        this.queue.push(task);
+        this.dispatch();
+    }
+    attachWorker(slot) {
+        slot.worker.on('message', (message) => {
+            this.handleMessage(slot, message);
         });
-        worker.on('exit', (code) => {
-            this.handleWorkerExit(state, code);
+        slot.worker.on('error', (error) => {
+            this.handleWorkerFailure(slot, error);
         });
-        return state;
-    }
-    handleMessage(state, message) {
-        const pending = this.pending.get(message.id);
-        if (pending) {
-            this.pending.delete(message.id);
-            if (message.ok) {
-                pending.resolve(message.result);
+        slot.worker.on('exit', (code) => {
+            if (code !== 0) {
+                this.handleWorkerFailure(slot, new Error(`Worker exited with code ${code}`));
             }
-            else {
-                pending.reject(new Error(message.error));
-            }
-        }
-        state.busy = false;
-        state.currentJobId = undefined;
-        this.schedule();
+        });
     }
-    handleWorkerError(state, error) {
-        this.failCurrentJob(state, error);
-        this.replaceWorker(state);
+    spawnWorker() {
+        const slot = {
+            worker: new Worker(this.workerUrl),
+            busy: false,
+            current: undefined,
+        };
+        this.attachWorker(slot);
+        return slot;
     }
-    handleWorkerExit(state, code) {
-        if (code !== 0) {
-            this.failCurrentJob(state, new Error(`Transform worker exited with code ${code}`));
+    handleMessage(slot, message) {
+        const task = slot.current;
+        if (!task)
+            return;
+        if (!isWorkerResponse(message) || message.id !== task.id) {
+            this.handleWorkerFailure(slot, new Error('Unexpected worker response'));
+            return;
+        }
+        slot.current = undefined;
+        slot.busy = false;
+        this.cleanupTask(task);
+        if (message.ok) {
+            task.resolve(message.result);
         }
-        this.replaceWorker(state);
+        else {
+            task.reject(new Error(message.error));
+        }
+        this.dispatch();
     }
-    failCurrentJob(state, error) {
-        if (!state.currentJobId)
-            return;
-        const pending = this.pending.get(state.currentJobId);
-        if (pending) {
-            pending.reject(error);
-            this.pending.delete(state.currentJobId);
+    handleWorkerFailure(slot, error) {
+        const task = slot.current;
+        if (task) {
+            slot.current = undefined;
+            slot.busy = false;
+            this.cleanupTask(task);
+            task.reject(error instanceof Error ? error : new Error(getErrorMessage(error)));
         }
-        state.currentJobId = undefined;
-        state.busy = false;
+        logWarn('Worker thread failure', {
+            error: getErrorMessage(error),
+        });
+        this.replaceWorker(slot);
+        this.dispatch();
     }
-    replaceWorker(state) {
-        if (this.destroyed)
-            return;
-        const index = this.workers.indexOf(state);
-        if (index === -1)
+    replaceWorker(slot) {
+        try {
+            void slot.worker.terminate();
+        }
+        catch {
+            // Best-effort cleanup.
+        }
+        slot.worker = new Worker(this.workerUrl);
+        slot.busy = false;
+        slot.current = undefined;
+        this.attachWorker(slot);
+    }
+    failTask(slot, error) {
+        const task = slot.current;
+        if (!task)
             return;
-        this.workers[index] = this.createWorker();
-        this.schedule();
+        slot.current = undefined;
+        slot.busy = false;
+        this.cleanupTask(task);
+        task.reject(error instanceof Error ? error : new Error(String(error)));
+        this.dispatch();
     }
-    schedule() {
-        if (this.destroyed)
+    abortRunningTask(task) {
+        const slot = this.slots.find((s) => s.current?.id === task.id);
+        if (!slot)
             return;
-        for (const workerState of this.workers) {
-            if (this.queue.length === 0)
-                return;
-            if (workerState.busy)
-                continue;
-            const job = this.queue.shift();
-            if (!job)
-                return;
-            workerState.busy = true;
-            workerState.currentJobId = job.id;
-            workerState.worker.postMessage(job);
+        this.handleWorkerFailure(slot, new Error('Aborted'));
+    }
+    removeQueuedTask(task) {
+        const index = this.queue.findIndex((queued) => queued.id === task.id);
+        if (index >= 0) {
+            this.queue.splice(index, 1);
         }
+        this.cleanupTask(task);
     }
+    cleanupTask(task) {
+        if (task.signal && task.abortHandler) {
+            task.signal.removeEventListener('abort', task.abortHandler);
+        }
+    }
+}
+let pool = null;
+function getPool() {
+    if (pool)
+        return pool;
+    pool = new TransformWorkerPool(new URL('../workers/content-transform.worker.js', import.meta.url), config.workers.poolSize);
+    return pool;
+}
+export async function transformInWorker(request, signal) {
+    return getPool().run(request, signal);
+}
+export async function destroyTransformWorkers() {
+    if (!pool)
+        return;
+    const current = pool;
+    pool = null;
+    await current.destroy();
 }

package/dist/services/transform-worker-types.d.ts

@@ -0,0 +1,32 @@
+import type { Worker } from 'node:worker_threads';
+import type { MarkdownTransformResult, TransformOptions } from '../config/types/content.js';
+export interface WorkerTransformRequest {
+    id: number;
+    html: string;
+    url: string;
+    options: TransformOptions;
+}
+export type WorkerTransformResponse = {
+    id: number;
+    ok: true;
+    result: MarkdownTransformResult;
+} | {
+    id: number;
+    ok: false;
+    error: string;
+};
+export interface TransformTask {
+    id: number;
+    request: WorkerTransformRequest;
+    resolve: (result: MarkdownTransformResult) => void;
+    reject: (error: Error) => void;
+    signal: AbortSignal | undefined;
+    abortHandler: (() => void) | undefined;
+    status: 'queued' | 'running';
+}
+export interface WorkerSlot {
+    worker: Worker;
+    busy: boolean;
+    current: TransformTask | undefined;
+}
+export declare function isWorkerResponse(value: unknown): value is WorkerTransformResponse;

package/dist/services/transform-worker-types.js

@@ -0,0 +1,14 @@
+import { isRecord } from '../utils/guards.js';
+export function isWorkerResponse(value) {
+    if (!isRecord(value))
+        return false;
+    if (typeof value.id !== 'number')
+        return false;
+    if (value.ok === true) {
+        return 'result' in value;
+    }
+    if (value.ok === false) {
+        return typeof value.error === 'string';
+    }
+    return false;
+}

package/dist/tools/handlers/fetch-markdown.tool.d.ts

@@ -1,12 +1,11 @@
 import type { FetchMarkdownInput, ToolResponseBase } from '../../config/types/tools.js';
-import { transformHtmlToMarkdownAsync } from '../utils/content-transform-async.js';
+import { transformHtmlToMarkdown } from '../utils/content-transform.js';
 import { performSharedFetch } from './fetch-single.shared.js';
 export declare const FETCH_MARKDOWN_TOOL_NAME = "fetch-markdown";
-export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter and content length limits";
+export declare const FETCH_MARKDOWN_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format with optional frontmatter";
 interface FetchMarkdownDeps {
     readonly performSharedFetch?: typeof performSharedFetch;
-    readonly transformHtmlToMarkdown?: typeof transformHtmlToMarkdownAsync;
+    readonly transformHtmlToMarkdown?: typeof transformHtmlToMarkdown;
 }
 export declare function createFetchMarkdownToolHandler(deps?: FetchMarkdownDeps): (input: FetchMarkdownInput) => Promise<ToolResponseBase>;
-export declare const fetchMarkdownToolHandler: (input: FetchMarkdownInput) => Promise<ToolResponseBase>;
 export {};

package/dist/tools/handlers/fetch-markdown.tool.js

@@ -1,10 +1,10 @@
 import { config } from '../../config/index.js';
 import { logDebug, logError } from '../../services/logger.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
-import { transformHtmlToMarkdownAsync } from '../utils/content-transform-async.js';
-import { applyInlineResultToStructuredContent, buildToolContentBlocks, getFileDownloadInfo, getInlineErrorResponse, performSharedFetch, } from './fetch-single.shared.js';
+import { transformHtmlToMarkdown } from '../utils/content-transform.js';
+import { buildToolContentBlocks, performSharedFetch, } from './fetch-single.shared.js';
 export const FETCH_MARKDOWN_TOOL_NAME = 'fetch-markdown';
-export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter and content length limits';
+export const FETCH_MARKDOWN_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format with optional frontmatter';
 function isRecord(value) {
     return value !== null && typeof value === 'object';
 }
@@ -13,22 +13,18 @@ function deserializeMarkdownPipelineResult(cached) {
         const parsed = JSON.parse(cached);
         if (!isRecord(parsed))
             return undefined;
-        const { content, markdown, title, truncated } = parsed;
+        const { content, markdown, title } = parsed;
         if (typeof content !== 'string')
             return undefined;
         if (typeof markdown !== 'string')
             return undefined;
         if (title !== undefined && typeof title !== 'string')
             return undefined;
-        if (truncated !== undefined && typeof truncated !== 'boolean') {
-            return undefined;
-        }
-        const resolvedTitle = typeof title === 'string' ? title : undefined;
         return {
             content,
             markdown,
-            title: resolvedTitle,
-            truncated: truncated ?? false,
+            title: typeof title === 'string' ? title : undefined,
+            truncated: false,
         };
     }
     catch {
@@ -37,35 +33,15 @@ function deserializeMarkdownPipelineResult(cached) {
 }
 function resolveMarkdownOptions(input) {
     return {
-        extractMainContent: input.extractMainContent ?? config.extraction.extractMainContent,
         includeMetadata: input.includeMetadata ?? config.extraction.includeMetadata,
-        ...(input.maxContentLength !== undefined && {
-            maxContentLength: input.maxContentLength,
-        }),
     };
 }
-function buildFetchMarkdownErrorDetails() {
+function buildMarkdownStructuredContent(pipeline, inlineResult) {
     return {
-        fetchedAt: new Date().toISOString(),
-        cached: false,
-    };
-}
-function buildMarkdownStructuredContent(pipeline, inlineResult, fileDownload) {
-    const structuredContent = {
         url: pipeline.url,
         title: pipeline.data.title,
-        fetchedAt: pipeline.fetchedAt,
-        contentSize: inlineResult.contentSize,
-        cached: pipeline.fromCache,
+        markdown: inlineResult.content,
     };
-    if (pipeline.data.truncated || inlineResult.truncated) {
-        structuredContent.truncated = true;
-    }
-    applyInlineResultToStructuredContent(structuredContent, inlineResult, 'markdown');
-    if (fileDownload) {
-        structuredContent.file = fileDownload;
-    }
-    return structuredContent;
 }
 function logFetchMarkdownStart(url, options) {
     logDebug('Fetching markdown', { url, ...options });
@@ -76,27 +52,18 @@ function buildMarkdownTransform(options, transform) {
         return { ...markdownResult, content: markdownResult.markdown };
     };
 }
-async function fetchMarkdownPipeline(url, input, options, transformOptions, performSharedFetchImpl, transformImpl) {
+async function fetchMarkdownPipeline(url, options, performSharedFetchImpl, transformImpl) {
     const sharedOptions = {
         url,
         format: 'markdown',
-        extractMainContent: options.extractMainContent,
         includeMetadata: options.includeMetadata,
-        ...(options.maxContentLength !== undefined && {
-            maxContentLength: options.maxContentLength,
-        }),
-        ...(input.customHeaders !== undefined && {
-            customHeaders: input.customHeaders,
-        }),
-        ...(input.retries !== undefined && { retries: input.retries }),
-        ...(input.timeout !== undefined && { timeout: input.timeout }),
-        transform: buildMarkdownTransform(transformOptions, transformImpl),
+        transform: buildMarkdownTransform(options, transformImpl),
         deserialize: deserializeMarkdownPipelineResult,
     };
     return performSharedFetchImpl(sharedOptions);
 }
-function buildMarkdownResponse(pipeline, inlineResult, fileDownload) {
-    const structuredContent = buildMarkdownStructuredContent(pipeline, inlineResult, fileDownload);
+function buildMarkdownResponse(pipeline, inlineResult) {
+    const structuredContent = buildMarkdownStructuredContent(pipeline, inlineResult);
     return {
         content: buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched markdown', pipeline.cacheKey, pipeline.data.content, 'markdown', pipeline.url, pipeline.data.title),
         structuredContent,
@@ -104,46 +71,27 @@ function buildMarkdownResponse(pipeline, inlineResult, fileDownload) {
 }
 export function createFetchMarkdownToolHandler(deps = {}) {
     const performSharedFetchImpl = deps.performSharedFetch ?? performSharedFetch;
-    const transformImpl = deps.transformHtmlToMarkdown ?? transformHtmlToMarkdownAsync;
+    const transformImpl = deps.transformHtmlToMarkdown ?? transformHtmlToMarkdown;
     return async (input) => {
         try {
             return await executeFetchMarkdown(input, performSharedFetchImpl, transformImpl);
         }
         catch (error) {
             logError('fetch-markdown tool error', error instanceof Error ? error : undefined);
-            const errorDetails = buildFetchMarkdownErrorDetails();
-            return handleToolError(error, input.url, 'Failed to fetch markdown', errorDetails);
+            return handleToolError(error, input.url, 'Failed to fetch markdown');
         }
     };
 }
-export const fetchMarkdownToolHandler = createFetchMarkdownToolHandler();
 async function executeFetchMarkdown(input, performSharedFetchImpl, transformImpl) {
     const { url } = input;
     if (!url) {
-        return createToolErrorResponse('URL is required', '', 'VALIDATION_ERROR', buildFetchMarkdownErrorDetails());
+        return createToolErrorResponse('URL is required', '');
     }
     const options = resolveMarkdownOptions(input);
-    const transformOptions = { ...options };
-    logFetchMarkdownStart(url, transformOptions);
-    const { pipeline, inlineResult } = await fetchMarkdownPipeline(url, input, options, transformOptions, performSharedFetchImpl, transformImpl);
-    const inlineError = getInlineErrorResponse(inlineResult, url, buildFetchMarkdownErrorDetails());
-    if (inlineError)
-        return inlineError;
-    let fileDownload = null;
-    if (inlineResult.resourceUri) {
-        const downloadContext = {
-            cacheKey: pipeline.cacheKey ?? null,
-            url: pipeline.url,
-        };
-        if (pipeline.data.title !== undefined) {
-            fileDownload = getFileDownloadInfo({
-                ...downloadContext,
-                title: pipeline.data.title,
-            });
-        }
-        else {
-            fileDownload = getFileDownloadInfo(downloadContext);
-        }
+    logFetchMarkdownStart(url, options);
+    const { pipeline, inlineResult } = await fetchMarkdownPipeline(url, options, performSharedFetchImpl, transformImpl);
+    if (inlineResult.error) {
+        return createToolErrorResponse(inlineResult.error, url);
     }
-    return buildMarkdownResponse(pipeline, inlineResult, fileDownload);
+    return buildMarkdownResponse(pipeline, inlineResult);
 }

package/dist/tools/handlers/fetch-single.shared.d.ts

@@ -1,21 +1,10 @@
 import type { PipelineResult, ToolContentBlock } from '../../config/types/runtime.js';
-import type { FileDownloadInfo, ToolResponseBase } from '../../config/types/tools.js';
 import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
 import { applyInlineContentLimit } from '../utils/inline-content.js';
-type SharedFetchFormat = 'jsonl' | 'markdown';
 interface SharedFetchOptions<T extends {
     content: string;
 }> {
     readonly url: string;
-    readonly format: SharedFetchFormat;
-    readonly extractMainContent: boolean;
-    readonly includeMetadata: boolean;
-    readonly maxContentLength?: number;
-    readonly includeContentBlocks?: boolean;
-    readonly cacheVariant?: string;
-    readonly customHeaders?: Record<string, string>;
-    readonly retries?: number;
-    readonly timeout?: number;
     readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
     readonly serialize?: (result: T) => string;
     readonly deserialize?: (cached: string) => T | undefined;
@@ -30,13 +19,5 @@ export declare function performSharedFetch<T extends {
     inlineResult: ReturnType<typeof applyInlineContentLimit>;
 }>;
 export type InlineResult = ReturnType<typeof applyInlineContentLimit>;
-interface DownloadContext {
-    cacheKey: string | null;
-    url: string;
-    title?: string;
-}
-export declare function getFileDownloadInfo(context: DownloadContext): FileDownloadInfo | null;
-export declare function getInlineErrorResponse(inlineResult: InlineResult, url: string, details?: Record<string, unknown>): ToolResponseBase | null;
-export declare function applyInlineResultToStructuredContent(structuredContent: Record<string, unknown>, inlineResult: InlineResult, contentKey: string): void;
-export declare function buildToolContentBlocks(structuredContent: Record<string, unknown>, fromCache: boolean, inlineResult: InlineResult, resourceName: string, cacheKey?: string | null, fullContent?: string, format?: SharedFetchFormat, url?: string, title?: string): ToolContentBlock[];
+export declare function buildToolContentBlocks(structuredContent: Record<string, unknown>, fromCache: boolean, inlineResult: InlineResult, resourceName: string, cacheKey?: string | null, fullContent?: string, url?: string, title?: string): ToolContentBlock[];
 export {};