@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/services/cache.js

@@ -1,10 +1,28 @@
 import { setInterval as setIntervalPromise } from 'node:timers/promises';
 import { config } from '../config/index.js';
-import { getErrorMessage } from '../utils/error-utils.js';
+import { getErrorMessage } from '../utils/error-details.js';
 import { parseCacheKey } from './cache-keys.js';
 import { logWarn } from './logger.js';
 const contentCache = new Map();
 let cleanupController = null;
+const updateListeners = new Set();
+export function onCacheUpdate(listener) {
+    updateListeners.add(listener);
+    return () => {
+        updateListeners.delete(listener);
+    };
+}
+function notifyCacheUpdate(cacheKey) {
+    if (updateListeners.size === 0)
+        return;
+    const parts = parseCacheKey(cacheKey);
+    if (!parts)
+        return;
+    const event = { cacheKey, ...parts };
+    for (const listener of updateListeners) {
+        listener(event);
+    }
+}
 function startCleanupLoop() {
     if (cleanupController)
         return;
@@ -17,15 +35,14 @@ function startCleanupLoop() {
 }
 async function runCleanupLoop(signal) {
     const intervalMs = Math.floor(config.cache.ttl * 1000);
-    for await (const _ of setIntervalPromise(intervalMs, undefined, {
+    for await (const getNow of setIntervalPromise(intervalMs, Date.now, {
         signal,
         ref: false,
     })) {
-        enforceCacheLimits();
+        enforceCacheLimits(getNow());
     }
 }
-function enforceCacheLimits() {
-    const now = Date.now();
+function enforceCacheLimits(now) {
     for (const [key, item] of contentCache.entries()) {
         if (now > item.expiresAt) {
             contentCache.delete(key);
@@ -33,21 +50,6 @@ function enforceCacheLimits() {
     }
     trimCacheToMaxKeys();
 }
-const updateListeners = new Set();
-export function onCacheUpdate(listener) {
-    updateListeners.add(listener);
-    return () => {
-        updateListeners.delete(listener);
-    };
-}
-function emitCacheUpdate(cacheKey) {
-    const parts = parseCacheKey(cacheKey);
-    if (!parts)
-        return;
-    for (const listener of updateListeners) {
-        listener({ cacheKey, ...parts });
-    }
-}
 export function get(cacheKey) {
     if (!isCacheReadable(cacheKey))
         return undefined;
@@ -69,16 +71,17 @@ function runCacheOperation(cacheKey, message, operation) {
     }
 }
 function readCacheEntry(cacheKey) {
-    return readCacheItem(cacheKey)?.entry;
+    const now = Date.now();
+    return readCacheItem(cacheKey, now)?.entry;
 }
-function isExpired(item) {
-    return Date.now() > item.expiresAt;
+function isExpired(item, now) {
+    return now > item.expiresAt;
 }
-function readCacheItem(cacheKey) {
+function readCacheItem(cacheKey, now) {
     const item = contentCache.get(cacheKey);
     if (!item)
         return undefined;
-    if (isExpired(item)) {
+    if (isExpired(item, now)) {
         contentCache.delete(cacheKey);
         return undefined;
     }
@@ -89,8 +92,15 @@ export function set(cacheKey, content, metadata) {
         return;
     runCacheOperation(cacheKey, 'Cache set error', () => {
         startCleanupLoop();
-        const entry = buildCacheEntry(content, metadata);
-        persistCacheEntry(cacheKey, entry);
+        const now = Date.now();
+        const expiresAtMs = now + config.cache.ttl * 1000;
+        const entry = buildCacheEntry({
+            content,
+            metadata,
+            fetchedAtMs: now,
+            expiresAtMs,
+        });
+        persistCacheEntry(cacheKey, entry, expiresAtMs);
     });
 }
 export function keys() {
@@ -99,20 +109,19 @@ export function keys() {
 export function isEnabled() {
     return config.cache.enabled;
 }
-function buildCacheEntry(content, metadata) {
+function buildCacheEntry({ content, metadata, fetchedAtMs, expiresAtMs, }) {
     return {
         url: metadata.url,
         content,
-        fetchedAt: new Date().toISOString(),
-        expiresAt: new Date(resolveExpiryTimestamp()).toISOString(),
+        fetchedAt: new Date(fetchedAtMs).toISOString(),
+        expiresAt: new Date(expiresAtMs).toISOString(),
         ...(metadata.title === undefined ? {} : { title: metadata.title }),
     };
 }
-function persistCacheEntry(cacheKey, entry) {
-    const expiresAt = resolveExpiryTimestamp();
-    contentCache.set(cacheKey, { entry, expiresAt });
+function persistCacheEntry(cacheKey, entry, expiresAtMs) {
+    contentCache.set(cacheKey, { entry, expiresAt: expiresAtMs });
     trimCacheToMaxKeys();
-    emitCacheUpdate(cacheKey);
+    notifyCacheUpdate(cacheKey);
 }
 function trimCacheToMaxKeys() {
     if (contentCache.size <= config.cache.maxKeys)
@@ -120,19 +129,14 @@ function trimCacheToMaxKeys() {
     removeOldestEntries(contentCache.size - config.cache.maxKeys);
 }
 function removeOldestEntries(count) {
-    if (count <= 0)
-        return;
-    let removed = 0;
-    for (const key of contentCache.keys()) {
-        contentCache.delete(key);
-        removed += 1;
-        if (removed >= count)
-            return;
+    const iterator = contentCache.keys();
+    for (let removed = 0; removed < count; removed += 1) {
+        const next = iterator.next();
+        if (next.done)
+            break;
+        contentCache.delete(next.value);
     }
 }
-function resolveExpiryTimestamp() {
-    return Date.now() + config.cache.ttl * 1000;
-}
 function logCacheError(message, cacheKey, error) {
     logWarn(message, {
         key: cacheKey.length > 100 ? cacheKey.slice(0, 100) : cacheKey,

package/dist/services/context.d.ts

@@ -1,8 +1,10 @@
 interface RequestContext {
     readonly requestId: string;
     readonly sessionId?: string;
+    readonly operationId?: string;
 }
 export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
 export declare function getRequestId(): string | undefined;
 export declare function getSessionId(): string | undefined;
+export declare function getOperationId(): string | undefined;
 export {};

package/dist/services/context.js

@@ -9,3 +9,6 @@ export function getRequestId() {
 export function getSessionId() {
     return requestContext.getStore()?.sessionId;
 }
+export function getOperationId() {
+    return requestContext.getStore()?.operationId;
+}

package/dist/services/extractor.d.ts

@@ -1,4 +1,5 @@
 import type { ExtractionResult } from '../config/types/content.js';
 export declare function extractContent(html: string, url: string, options?: {
     extractArticle?: boolean;
+    signal?: AbortSignal;
 }): ExtractionResult;

package/dist/services/extractor.js

@@ -1,28 +1,31 @@
 import { parseHTML } from 'linkedom';
 import { Readability } from '@mozilla/readability';
-import { getErrorMessage } from '../utils/error-utils.js';
+import { FetchError } from '../errors/app-error.js';
+import { throwIfAborted } from '../utils/cancellation.js';
+import { getErrorMessage } from '../utils/error-details.js';
 import { isRecord } from '../utils/guards.js';
 import { truncateHtml } from '../utils/html-truncator.js';
 import { logError, logInfo, logWarn } from './logger.js';
 import { extractMetadata } from './metadata-collector.js';
+import { endTransformStage, startTransformStage } from './telemetry.js';
 function isReadabilityCompatible(doc) {
     if (!isRecord(doc))
         return false;
-    if (!('documentElement' in doc))
-        return false;
-    if (typeof doc.querySelectorAll !== 'function')
-        return false;
-    if (typeof doc.querySelector !== 'function')
-        return false;
-    return true;
+    return hasDocumentElement(doc) && hasQuerySelectors(doc);
+}
+function hasDocumentElement(record) {
+    return 'documentElement' in record;
+}
+function hasQuerySelectors(record) {
+    return (typeof record.querySelectorAll === 'function' &&
+        typeof record.querySelector === 'function');
 }
 function extractArticle(document) {
     if (!isReadabilityCompatible(document)) {
         logWarn('Document not compatible with Readability');
         return null;
     }
-    const parsed = parseReadabilityArticle(document);
-    return parsed ? mapReadabilityResult(parsed) : null;
+    return mapParsedArticle(parseReadabilityArticle(document));
 }
 function parseReadabilityArticle(document) {
     try {
@@ -31,33 +34,42 @@ function parseReadabilityArticle(document) {
         return reader.parse();
     }
     catch (error) {
-        logError('Failed to extract article with Readability', error instanceof Error ? error : undefined);
+        logError('Failed to extract article with Readability', asError(error));
         return null;
     }
 }
+function asError(error) {
+    if (error instanceof Error) {
+        return error;
+    }
+    return undefined;
+}
+function mapParsedArticle(parsed) {
+    return parsed ? mapReadabilityResult(parsed) : null;
+}
 function mapReadabilityResult(parsed) {
-    const article = {
+    return {
         content: parsed.content ?? '',
         textContent: parsed.textContent ?? '',
+        ...buildOptionalArticleFields(parsed),
     };
-    const title = toOptional(parsed.title);
-    if (title !== undefined)
-        article.title = title;
-    const byline = toOptional(parsed.byline);
-    if (byline !== undefined)
-        article.byline = byline;
-    const excerpt = toOptional(parsed.excerpt);
-    if (excerpt !== undefined)
-        article.excerpt = excerpt;
-    const siteName = toOptional(parsed.siteName);
-    if (siteName !== undefined)
-        article.siteName = siteName;
-    return article;
-}
-function toOptional(value) {
-    return value ?? undefined;
-}
-export function extractContent(html, url, options = { extractArticle: true }) {
+}
+function buildOptionalArticleFields(parsed) {
+    const optional = {};
+    addOptionalField(optional, 'title', parsed.title);
+    addOptionalField(optional, 'byline', parsed.byline);
+    addOptionalField(optional, 'excerpt', parsed.excerpt);
+    addOptionalField(optional, 'siteName', parsed.siteName);
+    return optional;
+}
+function addOptionalField(target, key, value) {
+    if (value == null)
+        return;
+    target[key] = value;
+}
+export function extractContent(html, url, options = {
+    extractArticle: true,
+}) {
     if (!isValidInput(html, url)) {
         return { article: null, metadata: {} };
     }
@@ -65,29 +77,54 @@ export function extractContent(html, url, options = { extractArticle: true }) {
 }
 function tryExtractContent(html, url, options) {
     try {
+        throwIfAborted(options.signal, url, 'extract:begin');
+        const parseStage = startTransformStage(url, 'extract:parse');
         const { document } = parseHTML(truncateHtml(html));
+        endTransformStage(parseStage);
+        throwIfAborted(options.signal, url, 'extract:parsed');
         applyBaseUri(document, url);
+        const metadataStage = startTransformStage(url, 'extract:metadata');
         const metadata = extractMetadata(document);
+        endTransformStage(metadataStage);
+        throwIfAborted(options.signal, url, 'extract:metadata');
+        let article;
+        if (options.extractArticle) {
+            const articleStage = startTransformStage(url, 'extract:article');
+            article = resolveArticleExtraction(document, options.extractArticle);
+            endTransformStage(articleStage);
+        }
+        else {
+            article = null;
+        }
+        throwIfAborted(options.signal, url, 'extract:article');
         return {
-            article: options.extractArticle ? extractArticle(document) : null,
+            article,
             metadata,
         };
     }
     catch (error) {
+        if (error instanceof FetchError) {
+            throw error;
+        }
+        throwIfAborted(options.signal, url, 'extract:error');
         logError('Failed to extract content', error instanceof Error ? error : undefined);
         return { article: null, metadata: {} };
     }
 }
 function isValidInput(html, url) {
-    if (!html || typeof html !== 'string') {
-        logWarn('extractContent called with invalid HTML input');
-        return false;
-    }
-    if (!url || typeof url !== 'string') {
-        logWarn('extractContent called with invalid URL');
-        return false;
-    }
-    return true;
+    return (validateRequiredString(html, 'extractContent called with invalid HTML input') && validateRequiredString(url, 'extractContent called with invalid URL'));
+}
+function validateRequiredString(value, message) {
+    if (isNonEmptyString(value))
+        return true;
+    logWarn(message);
+    return false;
+}
+function isNonEmptyString(value) {
+    return typeof value === 'string' && value.length > 0;
+}
+function resolveArticleExtraction(document, shouldExtract) {
+    return shouldExtract ? extractArticle(document) : null;
 }
 function applyBaseUri(document, url) {
     try {

package/dist/services/fetcher/agents.js

@@ -1,7 +1,7 @@
 import dns from 'node:dns';
 import os from 'node:os';
 import { Agent } from 'undici';
-import { createErrorWithCode } from '../../utils/error-utils.js';
+import { createErrorWithCode } from '../../utils/error-details.js';
 import { isRecord } from '../../utils/guards.js';
 import { handleLookupResult } from './dns-selection.js';
 const DNS_LOOKUP_TIMEOUT_MS = 5000;

package/dist/services/fetcher/dns-selection.js

@@ -1,4 +1,4 @@
-import { createErrorWithCode } from '../../utils/error-utils.js';
+import { createErrorWithCode } from '../../utils/error-details.js';
 import { isBlockedIp } from '../../utils/url-validator.js';
 function normalizeLookupResults(addresses, family) {
     if (Array.isArray(addresses)) {

package/dist/services/fetcher/interceptors.js

@@ -1,7 +1,7 @@
 import { randomUUID } from 'node:crypto';
 import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
-import { isSystemError } from '../../utils/error-utils.js';
+import { isSystemError } from '../../utils/error-details.js';
 import { logDebug, logError, logWarn } from '../logger.js';
 const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
 function redactUrl(rawUrl) {
@@ -27,7 +27,14 @@ function publishFetchEvent(event) {
         // Avoid crashing the publisher if a subscriber throws.
     }
 }
-function publishAndLogFetchStart(context) {
+export function startFetchTelemetry(url, method) {
+    const safeUrl = redactUrl(url);
+    const context = {
+        requestId: randomUUID(),
+        startTime: performance.now(),
+        url: safeUrl,
+        method: method.toUpperCase(),
+    };
     publishFetchEvent({
         v: 1,
         type: 'start',
@@ -40,65 +47,40 @@ function publishAndLogFetchStart(context) {
         method: context.method,
         url: context.url,
     });
-}
-export function startFetchTelemetry(url, method) {
-    const safeUrl = redactUrl(url);
-    const context = {
-        requestId: randomUUID(),
-        startTime: performance.now(),
-        url: safeUrl,
-        method: method.toUpperCase(),
-    };
-    publishAndLogFetchStart(context);
     return context;
 }
 export function recordFetchResponse(context, response, contentSize) {
     const duration = performance.now() - context.startTime;
-    publishFetchEnd(context, response.status, duration);
-    logDebug('HTTP Response', {
-        requestId: context.requestId,
-        status: response.status,
-        url: context.url,
-        ...buildResponseMeta(response, contentSize, duration),
-    });
-    logSlowRequestIfNeeded(context, duration);
-}
-function publishFetchEnd(context, status, duration) {
+    const durationLabel = `${Math.round(duration)}ms`;
     publishFetchEvent({
         v: 1,
         type: 'end',
         requestId: context.requestId,
-        status,
+        status: response.status,
         duration,
     });
-}
-function buildResponseMeta(response, contentSize, duration) {
-    const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
-    const meta = {
-        duration: `${Math.round(duration)}ms`,
-    };
     const contentType = response.headers.get('content-type');
-    if (contentType !== null) {
-        meta.contentType = contentType;
-    }
-    if (contentLength !== undefined) {
-        meta.size = contentLength;
-    }
-    return meta;
-}
-function logSlowRequestIfNeeded(context, duration) {
-    if (duration <= 5000)
-        return;
-    logWarn('Slow HTTP request detected', {
+    const contentLength = response.headers.get('content-length') ??
+        (contentSize === undefined ? undefined : String(contentSize));
+    logDebug('HTTP Response', {
         requestId: context.requestId,
+        status: response.status,
         url: context.url,
-        duration: `${Math.round(duration)}ms`,
+        duration: durationLabel,
+        ...(contentType ? { contentType } : {}),
+        ...(contentLength ? { size: contentLength } : {}),
     });
+    if (duration > 5000) {
+        logWarn('Slow HTTP request detected', {
+            requestId: context.requestId,
+            url: context.url,
+            duration: durationLabel,
+        });
+    }
 }
-function normalizeError(error) {
-    return error instanceof Error ? error : new Error(String(error));
-}
-function buildFetchErrorEvent(context, err, duration, status) {
+export function recordFetchError(context, error, status) {
+    const duration = performance.now() - context.startTime;
+    const err = error instanceof Error ? error : new Error(String(error));
     const event = {
         v: 1,
         type: 'error',
@@ -107,10 +89,6 @@ function buildFetchErrorEvent(context, err, duration, status) {
         error: err.message,
         duration,
     };
-    addOptionalErrorFields(event, err, status);
-    return event;
-}
-function addOptionalErrorFields(event, err, status) {
     const code = isSystemError(err) ? err.code : undefined;
     if (code !== undefined) {
         event.code = code;
@@ -118,17 +96,8 @@ function addOptionalErrorFields(event, err, status) {
     if (status !== undefined) {
         event.status = status;
     }
-}
-function selectErrorLogger(status) {
-    return status === 429 ? logWarn : logError;
-}
-export function recordFetchError(context, error, status) {
-    const duration = performance.now() - context.startTime;
-    const err = normalizeError(error);
-    const event = buildFetchErrorEvent(context, err, duration, status);
     publishFetchEvent(event);
-    const log = selectErrorLogger(status);
-    const code = isSystemError(err) ? err.code : undefined;
+    const log = status === 429 ? logWarn : logError;
     log('HTTP Request Error', {
         requestId: context.requestId,
         url: context.url,

package/dist/services/fetcher/redirects.js

@@ -1,11 +1,19 @@
 import { FetchError } from '../../errors/app-error.js';
-import { createErrorWithCode } from '../../utils/error-utils.js';
+import { createErrorWithCode } from '../../utils/error-details.js';
 import { isRecord } from '../../utils/guards.js';
 import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 function isRedirectStatus(status) {
     return REDIRECT_STATUSES.has(status);
 }
+function cancelResponseBody(response) {
+    const cancelPromise = response.body?.cancel();
+    if (cancelPromise) {
+        cancelPromise.catch(() => {
+            // Best-effort cancellation; ignore failures.
+        });
+    }
+}
 async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
     const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
     if (!isRedirectStatus(response.status)) {
@@ -13,7 +21,7 @@ async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount)
     }
     assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
     const location = getRedirectLocation(response, currentUrl);
-    void response.body?.cancel();
+    cancelResponseBody(response);
     return {
         response,
         nextUrl: resolveRedirectTarget(currentUrl, location),
@@ -22,14 +30,14 @@ async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount)
 function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
     if (redirectCount < redirectLimit)
         return;
-    void response.body?.cancel();
+    cancelResponseBody(response);
     throw new FetchError('Too many redirects', currentUrl);
 }
 function getRedirectLocation(response, currentUrl) {
     const location = response.headers.get('location');
     if (location)
         return location;
-    void response.body?.cancel();
+    cancelResponseBody(response);
     throw new FetchError('Redirect response missing Location header', currentUrl);
 }
 function annotateRedirectError(error, url) {

package/dist/services/fetcher/response.js

@@ -7,7 +7,7 @@ function assertContentLengthWithinLimit(response, url, maxBytes) {
     if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
         return;
     }
-    void response.body?.cancel();
+    cancelResponseBody(response);
     throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
 }
 function createReadState() {
@@ -36,6 +36,14 @@ function createAbortError(url) {
         reason: 'aborted',
     });
 }
+function cancelResponseBody(response) {
+    const cancelPromise = response.body?.cancel();
+    if (cancelPromise) {
+        cancelPromise.catch(() => {
+            // Best-effort cancellation; ignore failures.
+        });
+    }
+}
 async function cancelReaderQuietly(reader) {
     try {
         await reader.cancel();
@@ -50,6 +58,14 @@ async function throwIfAborted(signal, url, reader) {
     await cancelReaderQuietly(reader);
     throw createAbortError(url);
 }
+async function handleReadFailure(error, signal, url, reader) {
+    const aborted = signal?.aborted ?? false;
+    await cancelReaderQuietly(reader);
+    if (aborted) {
+        throw createAbortError(url);
+    }
+    throw error;
+}
 async function readAllChunks(reader, state, url, maxBytes, signal) {
     await throwIfAborted(signal, url, reader);
     let result = await reader.read();
@@ -66,13 +82,7 @@ async function readStreamWithLimit(stream, url, maxBytes, signal) {
         await readAllChunks(reader, state, url, maxBytes, signal);
     }
     catch (error) {
-        if (!signal?.aborted) {
-            await cancelReaderQuietly(reader);
-        }
-        if (signal?.aborted) {
-            throw createAbortError(url);
-        }
-        throw error;
+        await handleReadFailure(error, signal, url, reader);
     }
     finally {
         reader.releaseLock();

package/dist/services/fetcher.d.ts

@@ -1,2 +1,25 @@
+import type { Dispatcher } from 'undici';
 import type { FetchOptions } from '../config/types/runtime.js';
+export declare const dispatcher: Dispatcher;
+export declare function destroyAgents(): void;
+interface FetchTelemetryContext {
+    requestId: string;
+    startTime: number;
+    url: string;
+    method: string;
+    contextRequestId?: string;
+    operationId?: string;
+}
+export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
+export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
+export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
+export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
+    response: Response;
+    url: string;
+}>;
+export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
+    text: string;
+    size: number;
+}>;
 export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
+export {};