@j0hanz/superfetch 1.2.4 → 2.0.0

package/dist/services/fetcher/interceptors.js

@@ -27,14 +27,7 @@ function publishFetchEvent(event) {
         // Avoid crashing the publisher if a subscriber throws.
     }
 }
-export function startFetchTelemetry(url, method) {
-    const safeUrl = redactUrl(url);
-    const context = {
-        requestId: randomUUID(),
-        startTime: performance.now(),
-        url: safeUrl,
-        method: method.toUpperCase(),
-    };
+function publishAndLogFetchStart(context) {
     publishFetchEvent({
         v: 1,
         type: 'start',
@@ -47,6 +40,16 @@ export function startFetchTelemetry(url, method) {
         method: context.method,
         url: context.url,
     });
+}
+export function startFetchTelemetry(url, method) {
+    const safeUrl = redactUrl(url);
+    const context = {
+        requestId: randomUUID(),
+        startTime: performance.now(),
+        url: safeUrl,
+        method: method.toUpperCase(),
+    };
+    publishAndLogFetchStart(context);
     return context;
 }
 export function recordFetchResponse(context, response, contentSize) {
@@ -92,10 +95,10 @@ function logSlowRequestIfNeeded(context, duration) {
         duration: `${Math.round(duration)}ms`,
     });
 }
-export function recordFetchError(context, error, status) {
-    const duration = performance.now() - context.startTime;
-    const err = error instanceof Error ? error : new Error(String(error));
-    const code = isSystemError(err) ? err.code : undefined;
+function normalizeError(error) {
+    return error instanceof Error ? error : new Error(String(error));
+}
+function buildFetchErrorEvent(context, err, duration, status) {
     const event = {
         v: 1,
         type: 'error',
@@ -104,14 +107,28 @@ export function recordFetchError(context, error, status) {
         error: err.message,
         duration,
     };
+    addOptionalErrorFields(event, err, status);
+    return event;
+}
+function addOptionalErrorFields(event, err, status) {
+    const code = isSystemError(err) ? err.code : undefined;
     if (code !== undefined) {
         event.code = code;
     }
     if (status !== undefined) {
         event.status = status;
     }
+}
+function selectErrorLogger(status) {
+    return status === 429 ? logWarn : logError;
+}
+export function recordFetchError(context, error, status) {
+    const duration = performance.now() - context.startTime;
+    const err = normalizeError(error);
+    const event = buildFetchErrorEvent(context, err, duration, status);
     publishFetchEvent(event);
-    const log = status === 429 ? logWarn : logError;
+    const log = selectErrorLogger(status);
+    const code = isSystemError(err) ? err.code : undefined;
     log('HTTP Request Error', {
         requestId: context.requestId,
         url: context.url,

package/dist/services/fetcher/redirects.js

@@ -1,5 +1,6 @@
 import { FetchError } from '../../errors/app-error.js';
 import { createErrorWithCode } from '../../utils/error-utils.js';
+import { isRecord } from '../../utils/guards.js';
 import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 function isRedirectStatus(status) {
@@ -15,7 +16,7 @@ async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount)
     void response.body?.cancel();
     return {
         response,
-        nextUrl: await resolveRedirectTarget(currentUrl, location),
+        nextUrl: resolveRedirectTarget(currentUrl, location),
     };
 }
 function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
@@ -32,11 +33,11 @@ function getRedirectLocation(response, currentUrl) {
     throw new FetchError('Redirect response missing Location header', currentUrl);
 }
 function annotateRedirectError(error, url) {
-    if (!error || typeof error !== 'object')
+    if (!isRecord(error))
         return;
     error.requestUrl = url;
 }
-async function resolveRedirectTarget(baseUrl, location) {
+function resolveRedirectTarget(baseUrl, location) {
     if (!URL.canParse(location, baseUrl)) {
         throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
     }

package/dist/services/fetcher/response.js

@@ -1,5 +1,3 @@
-import { Readable, Writable } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
 import { FetchError } from '../../errors/app-error.js';
 function assertContentLengthWithinLimit(response, url, maxBytes) {
     const contentLengthHeader = response.headers.get('content-length');
@@ -9,51 +7,88 @@ function assertContentLengthWithinLimit(response, url, maxBytes) {
     if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
         return;
     }
+    void response.body?.cancel();
     throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
 }
-async function readStreamWithLimit(stream, url, maxBytes, signal) {
-    const decoder = new TextDecoder();
-    let total = 0;
-    let text = '';
-    const toBuffer = (chunk) => {
-        if (typeof chunk === 'string') {
-            return Buffer.from(chunk);
-        }
-        return Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+function createReadState() {
+    return {
+        decoder: new TextDecoder(),
+        parts: [],
+        total: 0,
     };
-    const sink = new Writable({
-        write(chunk, _encoding, callback) {
-            const buffer = toBuffer(chunk);
-            total += buffer.length;
-            if (total > maxBytes) {
-                callback(new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url));
-                return;
-            }
-            text += decoder.decode(buffer, { stream: true });
-            callback();
-        },
-        final(callback) {
-            text += decoder.decode();
-            callback();
-        },
+}
+function appendChunk(state, chunk, maxBytes, url) {
+    state.total += chunk.byteLength;
+    if (state.total > maxBytes) {
+        throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+    }
+    const decoded = state.decoder.decode(chunk, { stream: true });
+    if (decoded)
+        state.parts.push(decoded);
+}
+function finalizeRead(state) {
+    const decoded = state.decoder.decode();
+    if (decoded)
+        state.parts.push(decoded);
+}
+function createAbortError(url) {
+    return new FetchError('Request was aborted during response read', url, 499, {
+        reason: 'aborted',
     });
+}
+async function cancelReaderQuietly(reader) {
     try {
-        const readable = Readable.fromWeb(stream, { signal });
-        await pipeline(readable, sink, { signal });
+        await reader.cancel();
+    }
+    catch {
+        // Ignore cancel errors; we're already failing this read.
+    }
+}
+async function throwIfAborted(signal, url, reader) {
+    if (!signal?.aborted)
+        return;
+    await cancelReaderQuietly(reader);
+    throw createAbortError(url);
+}
+async function readAllChunks(reader, state, url, maxBytes, signal) {
+    await throwIfAborted(signal, url, reader);
+    let result = await reader.read();
+    while (!result.done) {
+        appendChunk(state, result.value, maxBytes, url);
+        await throwIfAborted(signal, url, reader);
+        result = await reader.read();
+    }
+}
+async function readStreamWithLimit(stream, url, maxBytes, signal) {
+    const state = createReadState();
+    const reader = stream.getReader();
+    try {
+        await readAllChunks(reader, state, url, maxBytes, signal);
     }
     catch (error) {
+        if (!signal?.aborted) {
+            await cancelReaderQuietly(reader);
+        }
         if (signal?.aborted) {
-            throw new FetchError('Request was aborted during response read', url, 499, { reason: 'aborted' });
+            throw createAbortError(url);
         }
         throw error;
     }
-    return { text, size: total };
+    finally {
+        reader.releaseLock();
+    }
+    finalizeRead(state);
+    return { text: state.parts.join(''), size: state.total };
 }
 export async function readResponseText(response, url, maxBytes, signal) {
     assertContentLengthWithinLimit(response, url, maxBytes);
     if (!response.body) {
         const text = await response.text();
-        return { text, size: Buffer.byteLength(text) };
+        const size = Buffer.byteLength(text);
+        if (size > maxBytes) {
+            throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+        }
+        return { text, size };
     }
     return readStreamWithLimit(response.body, url, maxBytes, signal);
 }

package/dist/services/fetcher.d.ts

@@ -1,4 +1,2 @@
 import type { FetchOptions } from '../config/types/runtime.js';
-import { destroyAgents } from './fetcher/agents.js';
-export { destroyAgents };
-export declare function fetchNormalizedUrlWithRetry(normalizedUrl: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
+export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;

package/dist/services/fetcher.js

@@ -1,12 +1,9 @@
 import { config } from '../config/index.js';
-import { normalizeHeaderRecord } from '../utils/header-normalizer.js';
-import { destroyAgents, dispatcher } from './fetcher/agents.js';
+import { dispatcher } from './fetcher/agents.js';
 import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
 import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
 import { fetchWithRedirects } from './fetcher/redirects.js';
 import { readResponseText } from './fetcher/response.js';
-import { executeWithRetry } from './fetcher/retry-policy.js';
-export { destroyAgents };
 const DEFAULT_HEADERS = {
     'User-Agent': config.fetcher.userAgent,
     Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
@@ -14,15 +11,8 @@ const DEFAULT_HEADERS = {
     'Accept-Encoding': 'gzip, deflate, br',
     Connection: 'keep-alive',
 };
-function buildHeaders(customHeaders) {
-    const headers = new Headers(DEFAULT_HEADERS);
-    const sanitized = normalizeHeaderRecord(customHeaders, config.security.blockedHeaders);
-    if (sanitized) {
-        for (const [key, value] of Object.entries(sanitized)) {
-            headers.set(key, value);
-        }
-    }
-    return headers;
+function buildHeaders() {
+    return new Headers(DEFAULT_HEADERS);
 }
 function buildRequestSignal(timeoutMs, external) {
     const timeoutSignal = AbortSignal.timeout(timeoutMs);
@@ -54,9 +44,7 @@ async function handleFetchResponse(response, finalUrl, telemetry, signal) {
 async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
     const telemetry = startFetchTelemetry(normalizedUrl, 'GET');
     try {
-        const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
-        telemetry.url = finalUrl;
-        return await handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
+        return await fetchAndHandle(normalizedUrl, requestInit, telemetry);
     }
     catch (error) {
         const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
@@ -65,22 +53,15 @@ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
         throw mapped;
     }
 }
-export async function fetchNormalizedUrlWithRetry(normalizedUrl, options, maxRetries = 3) {
-    const context = buildRequestContext(options);
-    return executeWithRetry(normalizedUrl, maxRetries, async () => runFetch(normalizedUrl, context), context.signal);
-}
-function buildRequestContext(options) {
-    const context = {
-        timeoutMs: options?.timeout ?? config.fetcher.timeout,
-        headers: buildHeaders(options?.customHeaders),
-    };
-    if (options?.signal) {
-        context.signal = options.signal;
-    }
-    return context;
+async function fetchAndHandle(normalizedUrl, requestInit, telemetry) {
+    const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
+    telemetry.url = finalUrl;
+    return handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
 }
-async function runFetch(normalizedUrl, context) {
-    const signal = buildRequestSignal(context.timeoutMs, context.signal);
-    const requestInit = buildRequestInit(context.headers, signal);
-    return fetchWithTelemetry(normalizedUrl, requestInit, context.timeoutMs);
+export async function fetchNormalizedUrl(normalizedUrl, options) {
+    const timeoutMs = config.fetcher.timeout;
+    const headers = buildHeaders();
+    const signal = buildRequestSignal(timeoutMs, options?.signal);
+    const requestInit = buildRequestInit(headers, signal);
+    return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
 }

package/dist/services/fifo-queue.d.ts

@@ -0,0 +1,8 @@
+export declare class FifoQueue<T> {
+    private items;
+    private head;
+    get length(): number;
+    push(item: T): void;
+    shift(): T | undefined;
+    clear(): void;
+}

package/dist/services/fifo-queue.js

@@ -0,0 +1,25 @@
+export class FifoQueue {
+    items = [];
+    head = 0;
+    get length() {
+        return this.items.length - this.head;
+    }
+    push(item) {
+        this.items.push(item);
+    }
+    shift() {
+        if (this.head >= this.items.length)
+            return undefined;
+        const item = this.items[this.head];
+        this.head += 1;
+        if (this.head > 64 && this.head * 2 >= this.items.length) {
+            this.items = this.items.slice(this.head);
+            this.head = 0;
+        }
+        return item;
+    }
+    clear() {
+        this.items.length = 0;
+        this.head = 0;
+    }
+}

package/dist/services/logger.js

@@ -18,10 +18,10 @@ function formatLogEntry(level, message, meta) {
     return `[${createTimestamp()}] ${level.toUpperCase()}: ${message}${formatMetadata(meta)}`;
 }
 function shouldLog(level) {
-    if (!config.logging.enabled)
-        return false;
+    // Debug logs only when LOG_LEVEL=debug
     if (level === 'debug')
         return config.logging.level === 'debug';
+    // All other levels always log
     return true;
 }
 export function logInfo(message, meta) {

package/dist/services/metadata-collector.d.ts

@@ -1,10 +1,2 @@
 import type { ExtractedMetadata } from '../config/types/content.js';
-export type MetaSource = 'og' | 'twitter' | 'standard';
-export type MetaField = keyof ExtractedMetadata;
-export interface MetaCollectorState {
-    title: Partial<Record<MetaSource, string>>;
-    description: Partial<Record<MetaSource, string>>;
-    author: Partial<Record<MetaSource, string>>;
-}
-export declare function createMetaCollectorState(): MetaCollectorState;
-export declare function resolveMetaField(state: MetaCollectorState, field: MetaField): string | undefined;
+export declare function extractMetadata(document: Document): ExtractedMetadata;

package/dist/services/metadata-collector.js

@@ -1,11 +1,80 @@
-export function createMetaCollectorState() {
+function createMetaCollectorState() {
     return {
         title: {},
         description: {},
         author: {},
     };
 }
-export function resolveMetaField(state, field) {
+function resolveMetaField(state, field) {
     const sources = state[field];
     return sources.og ?? sources.twitter ?? sources.standard;
 }
+function parseOpenGraphKey(property) {
+    if (!property?.startsWith('og:'))
+        return null;
+    const key = property.replace('og:', '');
+    return key === 'title' || key === 'description' ? key : null;
+}
+function parseTwitterKey(name) {
+    if (!name?.startsWith('twitter:'))
+        return null;
+    const key = name.replace('twitter:', '');
+    return key === 'title' || key === 'description' ? key : null;
+}
+function parseStandardKey(name) {
+    if (name === 'description')
+        return 'description';
+    if (name === 'author')
+        return 'author';
+    return null;
+}
+function collectMetaTag(state, tag) {
+    const content = tag.getAttribute('content')?.trim();
+    if (!content)
+        return;
+    const ogKey = parseOpenGraphKey(tag.getAttribute('property'));
+    if (ogKey) {
+        state[ogKey].og = content;
+        return;
+    }
+    const name = tag.getAttribute('name');
+    const twitterKey = parseTwitterKey(name);
+    if (twitterKey) {
+        state[twitterKey].twitter = content;
+        return;
+    }
+    const standardKey = parseStandardKey(name);
+    if (standardKey) {
+        state[standardKey].standard = content;
+    }
+}
+function scanMetaTags(document, state) {
+    const metaTags = document.querySelectorAll('meta');
+    for (const tag of metaTags) {
+        collectMetaTag(state, tag);
+    }
+}
+function ensureTitleFallback(document, state) {
+    if (state.title.standard)
+        return;
+    const titleEl = document.querySelector('title');
+    if (titleEl?.textContent) {
+        state.title.standard = titleEl.textContent.trim();
+    }
+}
+export function extractMetadata(document) {
+    const state = createMetaCollectorState();
+    scanMetaTags(document, state);
+    ensureTitleFallback(document, state);
+    const metadata = {};
+    const title = resolveMetaField(state, 'title');
+    const description = resolveMetaField(state, 'description');
+    const author = resolveMetaField(state, 'author');
+    if (title !== undefined)
+        metadata.title = title;
+    if (description !== undefined)
+        metadata.description = description;
+    if (author !== undefined)
+        metadata.author = author;
+    return metadata;
+}

package/dist/services/transform-worker-pool.d.ts

@@ -1,14 +1,4 @@
-import type { JsonlTransformResult, MarkdownTransformResult, TransformOptions } from '../config/types/content.js';
-type TransformMode = 'jsonl' | 'markdown' | 'markdown-blocks';
-export interface TransformJob {
-    mode: TransformMode;
-    html: string;
-    url: string;
-    options: TransformOptions & {
-        includeContentBlocks?: boolean;
-    };
-}
-type TransformResult = JsonlTransformResult | MarkdownTransformResult;
-export declare function runTransformInWorker(job: TransformJob): Promise<TransformResult | null>;
-export declare function destroyTransformWorkers(): void;
-export {};
+import type { MarkdownTransformResult } from '../config/types/content.js';
+import type { WorkerTransformRequest } from './transform-worker-types.js';
+export declare function transformInWorker(request: Omit<WorkerTransformRequest, 'id'>, signal?: AbortSignal): Promise<MarkdownTransformResult>;
+export declare function destroyTransformWorkers(): Promise<void>;