@j0hanz/superfetch 1.2.1 → 1.2.3

package/dist/services/extractor.js

@@ -3,17 +3,7 @@ import { Readability } from '@mozilla/readability';
 import { getErrorMessage } from '../utils/error-utils.js';
 import { truncateHtml } from '../utils/html-truncator.js';
 import { logError, logInfo, logWarn } from './logger.js';
-function resolveMetaField(state, field) {
-    const sources = state[field];
-    return sources.og ?? sources.twitter ?? sources.standard;
-}
-function createMetaCollectorState() {
-    return {
-        title: {},
-        description: {},
-        author: {},
-    };
-}
+import { createMetaCollectorState, resolveMetaField, } from './metadata-collector.js';
 function collectMetaTag(state, tag) {
     const content = getMetaContent(tag);
     if (!content)
@@ -76,11 +66,17 @@ function extractMetadata(document) {
     const state = createMetaCollectorState();
     scanMetaTags(document, state);
     ensureTitleFallback(document, state);
-    return {
-        title: resolveMetaField(state, 'title'),
-        description: resolveMetaField(state, 'description'),
-        author: resolveMetaField(state, 'author'),
-    };
+    const metadata = {};
+    const title = resolveMetaField(state, 'title');
+    const description = resolveMetaField(state, 'description');
+    const author = resolveMetaField(state, 'author');
+    if (title !== undefined)
+        metadata.title = title;
+    if (description !== undefined)
+        metadata.description = description;
+    if (author !== undefined)
+        metadata.author = author;
+    return metadata;
 }
 function isReadabilityCompatible(doc) {
     if (!doc || typeof doc !== 'object')
@@ -113,14 +109,23 @@ function parseReadabilityArticle(document) {
     }
 }
 function mapReadabilityResult(parsed) {
-    return {
-        title: toOptional(parsed.title),
-        byline: toOptional(parsed.byline),
+    const article = {
         content: parsed.content ?? '',
         textContent: parsed.textContent ?? '',
-        excerpt: toOptional(parsed.excerpt),
-        siteName: toOptional(parsed.siteName),
     };
+    const title = toOptional(parsed.title);
+    if (title !== undefined)
+        article.title = title;
+    const byline = toOptional(parsed.byline);
+    if (byline !== undefined)
+        article.byline = byline;
+    const excerpt = toOptional(parsed.excerpt);
+    if (excerpt !== undefined)
+        article.excerpt = excerpt;
+    const siteName = toOptional(parsed.siteName);
+    if (siteName !== undefined)
+        article.siteName = siteName;
+    return article;
 }
 function toOptional(value) {
     return value ?? undefined;

package/dist/services/fetcher/agents.js

@@ -3,9 +3,26 @@ import os from 'node:os';
 import { Agent } from 'undici';
 import { createErrorWithCode } from '../../utils/error-utils.js';
 import { isBlockedIp } from '../../utils/url-validator.js';
+const DNS_LOOKUP_TIMEOUT_MS = 5000;
 function resolveDns(hostname, options, callback) {
     const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
-    dns.lookup(hostname, { ...normalizedOptions, all: true }, createLookupCallback(hostname, resolvedFamily, useAll, callback));
+    const lookupOptions = buildLookupOptions(normalizedOptions);
+    let done = false;
+    const timer = setTimeout(() => {
+        if (done)
+            return;
+        done = true;
+        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
+    }, DNS_LOOKUP_TIMEOUT_MS);
+    timer.unref();
+    const safeCallback = (err, address, family) => {
+        if (done)
+            return;
+        done = true;
+        clearTimeout(timer);
+        callback(err, address, family);
+    };
+    dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
 }
 function normalizeLookupOptions(options) {
     return typeof options === 'number' ? { family: options } : options;
@@ -18,6 +35,29 @@ function buildLookupContext(options) {
         resolvedFamily: resolveFamily(normalizedOptions.family),
     };
 }
+const DEFAULT_DNS_ORDER = 'verbatim';
+function resolveResultOrder(options) {
+    if (options.order)
+        return options.order;
+    const legacyVerbatim = getLegacyVerbatim(options);
+    if (legacyVerbatim !== undefined) {
+        return legacyVerbatim ? 'verbatim' : 'ipv4first';
+    }
+    return DEFAULT_DNS_ORDER;
+}
+function getLegacyVerbatim(options) {
+    const legacy = options.verbatim;
+    return typeof legacy === 'boolean' ? legacy : undefined;
+}
+function buildLookupOptions(normalizedOptions) {
+    const options = {
+        ...normalizedOptions,
+        order: resolveResultOrder(normalizedOptions),
+        all: true,
+    };
+    delete options.verbatim;
+    return options;
+}
 function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
     return (err, addresses) => {
         handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
@@ -42,6 +82,11 @@ function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll,
         return;
     }
     const list = normalizeLookupResults(addresses, resolvedFamily);
+    const invalidFamilyError = findInvalidFamilyError(list, hostname);
+    if (invalidFamilyError) {
+        callback(invalidFamilyError, list);
+        return;
+    }
     const blockedError = findBlockedIpError(list, hostname);
     if (blockedError) {
         callback(blockedError, list);
@@ -89,6 +134,15 @@ function findBlockedIpError(list, hostname) {
     }
     return null;
 }
+function findInvalidFamilyError(list, hostname) {
+    for (const addr of list) {
+        const family = typeof addr === 'string' ? 0 : addr.family;
+        if (family === 4 || family === 6)
+            continue;
+        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
+    }
+    return null;
+}
 function createNoDnsResultsError(hostname) {
     return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
 }

package/dist/services/fetcher/interceptors.d.ts

@@ -1,3 +1,25 @@
+export type FetchChannelEvent = {
+    v: 1;
+    type: 'start';
+    requestId: string;
+    method: string;
+    url: string;
+} | {
+    v: 1;
+    type: 'end';
+    requestId: string;
+    status: number;
+    duration: number;
+} | {
+    v: 1;
+    type: 'error';
+    requestId: string;
+    url: string;
+    error: string;
+    code?: string;
+    status?: number;
+    duration: number;
+};
 interface FetchTelemetryContext {
     requestId: string;
     startTime: number;

package/dist/services/fetcher/interceptors.js

@@ -4,21 +4,44 @@ import { performance } from 'node:perf_hooks';
 import { isSystemError } from '../../utils/error-utils.js';
 import { logDebug, logError, logWarn } from '../logger.js';
 const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
+function redactUrl(rawUrl) {
+    try {
+        const url = new URL(rawUrl);
+        url.username = '';
+        url.password = '';
+        url.hash = '';
+        url.search = '';
+        return url.toString();
+    }
+    catch {
+        return rawUrl;
+    }
+}
+function publishFetchEvent(event) {
+    if (!fetchChannel.hasSubscribers)
+        return;
+    try {
+        fetchChannel.publish(event);
+    }
+    catch {
+        // Avoid crashing the publisher if a subscriber throws.
+    }
+}
 export function startFetchTelemetry(url, method) {
+    const safeUrl = redactUrl(url);
     const context = {
         requestId: randomUUID(),
         startTime: performance.now(),
-        url,
+        url: safeUrl,
         method: method.toUpperCase(),
     };
-    if (fetchChannel.hasSubscribers) {
-        fetchChannel.publish({
-            type: 'start',
-            requestId: context.requestId,
-            method: context.method,
-            url: context.url,
-        });
-    }
+    publishFetchEvent({
+        v: 1,
+        type: 'start',
+        requestId: context.requestId,
+        method: context.method,
+        url: context.url,
+    });
     logDebug('HTTP Request', {
         requestId: context.requestId,
         method: context.method,
@@ -38,9 +61,8 @@ export function recordFetchResponse(context, response, contentSize) {
     logSlowRequestIfNeeded(context, duration);
 }
 function publishFetchEnd(context, status, duration) {
-    if (!fetchChannel.hasSubscribers)
-        return;
-    fetchChannel.publish({
+    publishFetchEvent({
+        v: 1,
         type: 'end',
         requestId: context.requestId,
         status,
@@ -48,13 +70,18 @@ function publishFetchEnd(context, status, duration) {
     });
 }
 function buildResponseMeta(response, contentSize, duration) {
-    const contentType = response.headers.get('content-type') ?? undefined;
     const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
-    return {
-        contentType,
+    const meta = {
         duration: `${Math.round(duration)}ms`,
-        size: contentLength,
     };
+    const contentType = response.headers.get('content-type');
+    if (contentType !== null) {
+        meta.contentType = contentType;
+    }
+    if (contentLength !== undefined) {
+        meta.size = contentLength;
+    }
+    return meta;
 }
 function logSlowRequestIfNeeded(context, duration) {
     if (duration <= 5000)
@@ -69,17 +96,21 @@ export function recordFetchError(context, error, status) {
     const duration = performance.now() - context.startTime;
     const err = error instanceof Error ? error : new Error(String(error));
     const code = isSystemError(err) ? err.code : undefined;
-    if (fetchChannel.hasSubscribers) {
-        fetchChannel.publish({
-            type: 'error',
-            requestId: context.requestId,
-            url: context.url,
-            error: err.message,
-            code,
-            status,
-            duration,
-        });
+    const event = {
+        v: 1,
+        type: 'error',
+        requestId: context.requestId,
+        url: context.url,
+        error: err.message,
+        duration,
+    };
+    if (code !== undefined) {
+        event.code = code;
+    }
+    if (status !== undefined) {
+        event.status = status;
     }
+    publishFetchEvent(event);
     const log = status === 429 ? logWarn : logError;
     log('HTTP Request Error', {
         requestId: context.requestId,

package/dist/services/fetcher/response.d.ts

@@ -1,4 +1,4 @@
-export declare function readResponseText(response: Response, url: string, maxBytes: number): Promise<{
+export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
     text: string;
     size: number;
 }>;

package/dist/services/fetcher/response.js

@@ -1,3 +1,5 @@
+import { Readable, Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { FetchError } from '../../errors/app-error.js';
 function assertContentLengthWithinLimit(response, url, maxBytes) {
     const contentLengthHeader = response.headers.get('content-length');
@@ -9,30 +11,49 @@ function assertContentLengthWithinLimit(response, url, maxBytes) {
     }
     throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
 }
-async function readStreamWithLimit(stream, url, maxBytes) {
-    const reader = stream.getReader();
+async function readStreamWithLimit(stream, url, maxBytes, signal) {
     const decoder = new TextDecoder();
     let total = 0;
-    const chunks = [];
-    for (;;) {
-        const { value, done } = await reader.read();
-        if (done)
-            break;
-        total += value.byteLength;
-        if (total > maxBytes) {
-            await reader.cancel();
-            throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+    let text = '';
+    const toBuffer = (chunk) => {
+        if (typeof chunk === 'string') {
+            return Buffer.from(chunk);
         }
-        chunks.push(decoder.decode(value, { stream: true }));
+        return Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    };
+    const sink = new Writable({
+        write(chunk, _encoding, callback) {
+            const buffer = toBuffer(chunk);
+            total += buffer.length;
+            if (total > maxBytes) {
+                callback(new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url));
+                return;
+            }
+            text += decoder.decode(buffer, { stream: true });
+            callback();
+        },
+        final(callback) {
+            text += decoder.decode();
+            callback();
+        },
+    });
+    try {
+        const readable = Readable.fromWeb(stream, { signal });
+        await pipeline(readable, sink, { signal });
     }
-    chunks.push(decoder.decode());
-    return { text: chunks.join(''), size: total };
+    catch (error) {
+        if (signal?.aborted) {
+            throw new FetchError('Request was aborted during response read', url, 499, { reason: 'aborted' });
+        }
+        throw error;
+    }
+    return { text, size: total };
 }
-export async function readResponseText(response, url, maxBytes) {
+export async function readResponseText(response, url, maxBytes, signal) {
     assertContentLengthWithinLimit(response, url, maxBytes);
     if (!response.body) {
         const text = await response.text();
         return { text, size: Buffer.byteLength(text) };
     }
-    return readStreamWithLimit(response.body, url, maxBytes);
+    return readStreamWithLimit(response.body, url, maxBytes, signal);
 }

package/dist/services/fetcher.d.ts

@@ -1,4 +1,4 @@
 import type { FetchOptions } from '../config/types/runtime.js';
 import { destroyAgents } from './fetcher/agents.js';
 export { destroyAgents };
-export declare function fetchUrlWithRetry(url: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
+export declare function fetchNormalizedUrlWithRetry(normalizedUrl: string, options?: FetchOptions, maxRetries?: number): Promise<string>;

package/dist/services/fetcher.js

@@ -1,6 +1,5 @@
 import { config } from '../config/index.js';
 import { normalizeHeaderRecord } from '../utils/header-normalizer.js';
-import { validateAndNormalizeUrl } from '../utils/url-validator.js';
 import { destroyAgents, dispatcher } from './fetcher/agents.js';
 import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
 import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
@@ -39,7 +38,7 @@ function buildRequestInit(headers, signal) {
         dispatcher,
     };
 }
-async function handleFetchResponse(response, finalUrl, telemetry) {
+async function handleFetchResponse(response, finalUrl, telemetry, signal) {
     if (response.status === 429) {
         void response.body?.cancel();
         throw createRateLimitError(finalUrl, response.headers.get('retry-after'));
@@ -48,7 +47,7 @@ async function handleFetchResponse(response, finalUrl, telemetry) {
         void response.body?.cancel();
         throw createHttpError(finalUrl, response.status, response.statusText);
     }
-    const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength);
+    const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength, signal);
     recordFetchResponse(telemetry, response, size);
     return text;
 }
@@ -57,7 +56,7 @@ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
     try {
         const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
         telemetry.url = finalUrl;
-        return await handleFetchResponse(response, finalUrl, telemetry);
+        return await handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
     }
     catch (error) {
         const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
@@ -66,17 +65,19 @@ async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
         throw mapped;
     }
 }
-export async function fetchUrlWithRetry(url, options, maxRetries = 3) {
-    const normalizedUrl = await validateAndNormalizeUrl(url);
+export async function fetchNormalizedUrlWithRetry(normalizedUrl, options, maxRetries = 3) {
     const context = buildRequestContext(options);
     return executeWithRetry(normalizedUrl, maxRetries, async () => runFetch(normalizedUrl, context), context.signal);
 }
 function buildRequestContext(options) {
-    return {
+    const context = {
         timeoutMs: options?.timeout ?? config.fetcher.timeout,
         headers: buildHeaders(options?.customHeaders),
-        signal: options?.signal,
     };
+    if (options?.signal) {
+        context.signal = options.signal;
+    }
+    return context;
 }
 async function runFetch(normalizedUrl, context) {
     const signal = buildRequestSignal(context.timeoutMs, context.signal);

package/dist/services/metadata-collector.d.ts

@@ -0,0 +1,10 @@
+import type { ExtractedMetadata } from '../config/types/content.js';
+export type MetaSource = 'og' | 'twitter' | 'standard';
+export type MetaField = keyof ExtractedMetadata;
+export interface MetaCollectorState {
+    title: Partial<Record<MetaSource, string>>;
+    description: Partial<Record<MetaSource, string>>;
+    author: Partial<Record<MetaSource, string>>;
+}
+export declare function createMetaCollectorState(): MetaCollectorState;
+export declare function resolveMetaField(state: MetaCollectorState, field: MetaField): string | undefined;

package/dist/services/metadata-collector.js

@@ -0,0 +1,11 @@
+export function createMetaCollectorState() {
+    return {
+        title: {},
+        description: {},
+        author: {},
+    };
+}
+export function resolveMetaField(state, field) {
+    const sources = state[field];
+    return sources.og ?? sources.twitter ?? sources.standard;
+}

package/dist/services/parser.d.ts

@@ -1,2 +1,6 @@
-import type { ContentBlockUnion } from '../config/types/content.js';
+import type { ContentBlockUnion, ExtractedMetadata } from '../config/types/content.js';
 export declare function parseHtml(html: string): ContentBlockUnion[];
+export declare function parseHtmlWithMetadata(html: string): {
+    blocks: ContentBlockUnion[];
+    metadata: ExtractedMetadata;
+};

package/dist/services/parser.js

@@ -6,7 +6,57 @@ import { getErrorMessage } from '../utils/error-utils.js';
 import { truncateHtml } from '../utils/html-truncator.js';
 import { sanitizeText } from '../utils/sanitizer.js';
 import { logWarn } from './logger.js';
+import { createMetaCollectorState, resolveMetaField, } from './metadata-collector.js';
 const CONTENT_SELECTOR = 'h1, h2, h3, h4, h5, h6, p, ul, ol, pre, code:not(pre code), table, img, blockquote';
+function extractMetadata($) {
+    const state = createMetaCollectorState();
+    $('meta').each((_, element) => {
+        const content = $(element).attr('content')?.trim();
+        if (!content)
+            return;
+        const property = $(element).attr('property');
+        if (property?.startsWith('og:')) {
+            const key = property.replace('og:', '');
+            if (key === 'title')
+                state.title.og = content;
+            if (key === 'description')
+                state.description.og = content;
+            return;
+        }
+        const name = $(element).attr('name');
+        if (name?.startsWith('twitter:')) {
+            const key = name.replace('twitter:', '');
+            if (key === 'title')
+                state.title.twitter = content;
+            if (key === 'description')
+                state.description.twitter = content;
+            return;
+        }
+        if (name === 'description') {
+            state.description.standard = content;
+        }
+        if (name === 'author') {
+            state.author.standard = content;
+        }
+    });
+    if (!state.title.standard) {
+        const titleText = $('title').first().text().trim();
+        if (titleText) {
+            state.title.standard = titleText;
+        }
+    }
+    const metadata = {};
+    const title = resolveMetaField(state, 'title');
+    const description = resolveMetaField(state, 'description');
+    const author = resolveMetaField(state, 'author');
+    if (title !== undefined)
+        metadata.title = title;
+    if (description !== undefined)
+        metadata.description = description;
+    if (author !== undefined)
+        metadata.author = author;
+    return metadata;
+}
 function parseHeading($, element) {
     const rawText = sanitizeText($(element).text());
     const text = cleanHeading(rawText);
@@ -55,11 +105,14 @@ function parseCode($, element) {
     const dataLang = $(element).attr('data-language') ?? '';
     const language = resolveLanguageFromAttributes(className, dataLang) ??
         detectLanguageFromCode(text);
-    return {
+    const block = {
         type: 'code',
-        language,
         text,
     };
+    if (language !== undefined) {
+        block.language = language;
+    }
+    return block;
 }
 function parseTable($, element) {
     const headers = [];
@@ -90,21 +143,23 @@ function parseTable($, element) {
     });
     if (rows.length === 0)
         return null;
-    return {
-        type: 'table',
-        headers: headers.length > 0 ? headers : undefined,
-        rows,
-    };
+    return headers.length > 0
+        ? { type: 'table', headers, rows }
+        : { type: 'table', rows };
 }
 function parseImage($, element) {
     const src = $(element).attr('src');
     if (!src)
         return null;
-    return {
+    const alt = $(element).attr('alt');
+    const image = {
         type: 'image',
         src,
-        alt: $(element).attr('alt') ?? undefined,
     };
+    if (alt !== undefined) {
+        image.alt = alt;
+    }
+    return image;
 }
 function parseBlockquote($, element) {
     const rawText = sanitizeText($(element).text());
@@ -173,6 +228,10 @@ function loadHtml(html) {
         return null;
     }
 }
+function prepareCheerio(html) {
+    const processedHtml = truncateHtml(html);
+    return loadHtml(processedHtml);
+}
 function removeNoiseElements($) {
     $('script, style, noscript, iframe, svg').remove();
 }
@@ -198,10 +257,22 @@ function safeParseElement($, element) {
 export function parseHtml(html) {
     if (!html || typeof html !== 'string')
         return [];
-    const processedHtml = truncateHtml(html);
-    const $ = loadHtml(processedHtml);
+    const $ = prepareCheerio(html);
     if (!$)
         return [];
     removeNoiseElements($);
     return filterBlocks(collectBlocks($));
 }
+export function parseHtmlWithMetadata(html) {
+    if (!html || typeof html !== 'string') {
+        return { blocks: [], metadata: {} };
+    }
+    const $ = prepareCheerio(html);
+    if (!$) {
+        return { blocks: [], metadata: {} };
+    }
+    const metadata = extractMetadata($);
+    removeNoiseElements($);
+    const blocks = filterBlocks(collectBlocks($));
+    return { blocks, metadata };
+}

package/dist/services/transform-worker-pool.d.ts

@@ -0,0 +1,14 @@
+import type { JsonlTransformResult, MarkdownTransformResult, TransformOptions } from '../config/types/content.js';
+type TransformMode = 'jsonl' | 'markdown' | 'markdown-blocks';
+export interface TransformJob {
+    mode: TransformMode;
+    html: string;
+    url: string;
+    options: TransformOptions & {
+        includeContentBlocks?: boolean;
+    };
+}
+type TransformResult = JsonlTransformResult | MarkdownTransformResult;
+export declare function runTransformInWorker(job: TransformJob): Promise<TransformResult | null>;
+export declare function destroyTransformWorkers(): void;
+export {};