@j0hanz/superfetch 1.2.4 → 2.0.0

package/dist/services/cache.d.ts

@@ -1,8 +1,5 @@
 import type { CacheEntry } from '../config/types/content.js';
-interface CacheKeyParts {
-    namespace: string;
-    urlHash: string;
-}
+import type { CacheKeyParts } from './cache-keys.js';
 interface CacheUpdateEvent extends CacheKeyParts {
     cacheKey: string;
 }
@@ -11,9 +8,6 @@ interface CacheEntryMetadata {
     title?: string;
 }
 type CacheUpdateListener = (event: CacheUpdateEvent) => void;
-export declare function createCacheKey(namespace: string, url: string, vary?: Record<string, unknown> | string): string | null;
-export declare function parseCacheKey(cacheKey: string): CacheKeyParts | null;
-export declare function toResourceUri(cacheKey: string): string | null;
 export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
 export declare function get(cacheKey: string | null): CacheEntry | undefined;
 export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata): void;

package/dist/services/cache.js

@@ -1,8 +1,7 @@
 import { setInterval as setIntervalPromise } from 'node:timers/promises';
-import { CACHE_HASH } from '../config/constants.js';
 import { config } from '../config/index.js';
-import { sha256Hex } from '../utils/crypto.js';
 import { getErrorMessage } from '../utils/error-utils.js';
+import { parseCacheKey } from './cache-keys.js';
 import { logWarn } from './logger.js';
 const contentCache = new Map();
 let cleanupController = null;
@@ -22,90 +21,19 @@ async function runCleanupLoop(signal) {
         signal,
         ref: false,
     })) {
-        evictEntries();
+        enforceCacheLimits();
     }
 }
-function evictEntries() {
+function enforceCacheLimits() {
     const now = Date.now();
     for (const [key, item] of contentCache.entries()) {
         if (now > item.expiresAt) {
             contentCache.delete(key);
         }
     }
-    if (contentCache.size <= config.cache.maxKeys)
-        return;
-    const keysToRemove = contentCache.size - config.cache.maxKeys;
-    const iterator = contentCache.keys();
-    for (let i = 0; i < keysToRemove; i++) {
-        const { value, done } = iterator.next();
-        if (done)
-            break;
-        contentCache.delete(value);
-    }
+    trimCacheToMaxKeys();
 }
 const updateListeners = new Set();
-function stableStringify(value) {
-    if (value === null || value === undefined) {
-        return '';
-    }
-    if (typeof value !== 'object') {
-        return JSON.stringify(value);
-    }
-    if (Array.isArray(value)) {
-        return `[${value.map((item) => stableStringify(item)).join(',')}]`;
-    }
-    const entries = Object.entries(value)
-        .filter(([, entryValue]) => entryValue !== undefined)
-        .sort(([a], [b]) => a.localeCompare(b))
-        .map(([key, entryValue]) => `${JSON.stringify(key)}:${stableStringify(entryValue)}`);
-    return `{${entries.join(',')}}`;
-}
-function createHashFragment(input, length) {
-    return sha256Hex(input).substring(0, length);
-}
-/**
- * Constructs a cache key from namespace, URL hash, and optional vary hash.
- * Format: "namespace:urlHash" or "namespace:urlHash.varyHash" if vary params exist.
- * @param namespace - Cache namespace (e.g., "fetch-markdown")
- * @param urlHash - SHA-256 hash of the URL (truncated to 16 chars)
- * @param varyHash - Optional hash of vary parameters (e.g., headers, options)
- * @returns Complete cache key string
- */
-function buildCacheKey(namespace, urlHash, varyHash) {
-    return varyHash
-        ? `${namespace}:${urlHash}.${varyHash}`
-        : `${namespace}:${urlHash}`;
-}
-function getVaryHash(vary) {
-    if (!vary)
-        return undefined;
-    const varyString = typeof vary === 'string' ? vary : stableStringify(vary);
-    if (!varyString)
-        return undefined;
-    return createHashFragment(varyString, CACHE_HASH.VARY_HASH_LENGTH);
-}
-export function createCacheKey(namespace, url, vary) {
-    if (!namespace || !url)
-        return null;
-    const urlHash = createHashFragment(url, CACHE_HASH.URL_HASH_LENGTH);
-    const varyHash = getVaryHash(vary);
-    return buildCacheKey(namespace, urlHash, varyHash);
-}
-export function parseCacheKey(cacheKey) {
-    if (!cacheKey)
-        return null;
-    const [namespace, ...rest] = cacheKey.split(':');
-    const urlHash = rest.join(':');
-    if (!namespace || !urlHash)
-        return null;
-    return { namespace, urlHash };
-}
-export function toResourceUri(cacheKey) {
-    const parts = parseCacheKey(cacheKey);
-    if (!parts)
-        return null;
-    return `superfetch://cache/${parts.namespace}/${parts.urlHash}`;
-}
 export function onCacheUpdate(listener) {
     updateListeners.add(listener);
     return () => {
@@ -123,21 +51,30 @@ function emitCacheUpdate(cacheKey) {
 export function get(cacheKey) {
     if (!isCacheReadable(cacheKey))
         return undefined;
+    return runCacheOperation(cacheKey, 'Cache get error', () => readCacheEntry(cacheKey));
+}
+function isCacheReadable(cacheKey) {
+    return config.cache.enabled && Boolean(cacheKey);
+}
+function isCacheWritable(cacheKey, content) {
+    return config.cache.enabled && Boolean(cacheKey) && Boolean(content);
+}
+function runCacheOperation(cacheKey, message, operation) {
     try {
-        return readCacheEntry(cacheKey);
+        return operation();
     }
     catch (error) {
-        logWarn('Cache get error', {
-            key: cacheKey.substring(0, 100),
-            error: getErrorMessage(error),
-        });
+        logCacheError(message, cacheKey, error);
         return undefined;
     }
 }
-function isCacheReadable(cacheKey) {
-    return config.cache.enabled && Boolean(cacheKey);
-}
 function readCacheEntry(cacheKey) {
+    return readCacheItem(cacheKey)?.entry;
+}
+function isExpired(item) {
+    return Date.now() > item.expiresAt;
+}
+function readCacheItem(cacheKey) {
     const item = contentCache.get(cacheKey);
     if (!item)
         return undefined;
@@ -145,29 +82,16 @@ function readCacheEntry(cacheKey) {
         contentCache.delete(cacheKey);
         return undefined;
     }
-    return item.entry;
-}
-function isExpired(item) {
-    return Date.now() > item.expiresAt;
+    return item;
 }
 export function set(cacheKey, content, metadata) {
-    if (!config.cache.enabled)
-        return;
-    if (!cacheKey)
-        return;
-    if (!content)
+    if (!isCacheWritable(cacheKey, content))
         return;
-    try {
+    runCacheOperation(cacheKey, 'Cache set error', () => {
         startCleanupLoop();
-        const entry = buildCacheEntry(cacheKey, content, metadata);
+        const entry = buildCacheEntry(content, metadata);
         persistCacheEntry(cacheKey, entry);
-    }
-    catch (error) {
-        logWarn('Cache set error', {
-            key: cacheKey.substring(0, 100),
-            error: getErrorMessage(error),
-        });
-    }
+    });
 }
 export function keys() {
     return Array.from(contentCache.keys());
@@ -175,33 +99,43 @@ export function keys() {
 export function isEnabled() {
     return config.cache.enabled;
 }
-function buildCacheEntry(cacheKey, content, metadata) {
-    const entry = {
+function buildCacheEntry(content, metadata) {
+    return {
         url: metadata.url,
         content,
         fetchedAt: new Date().toISOString(),
-        expiresAt: new Date(Date.now() + config.cache.ttl * 1000).toISOString(),
+        expiresAt: new Date(resolveExpiryTimestamp()).toISOString(),
+        ...(metadata.title === undefined ? {} : { title: metadata.title }),
     };
-    if (metadata.title !== undefined) {
-        entry.title = metadata.title;
-    }
-    return entry;
 }
 function persistCacheEntry(cacheKey, entry) {
-    const expiresAt = Date.now() + config.cache.ttl * 1000;
+    const expiresAt = resolveExpiryTimestamp();
     contentCache.set(cacheKey, { entry, expiresAt });
-    enforceMaxKeysLimit();
+    trimCacheToMaxKeys();
     emitCacheUpdate(cacheKey);
 }
-function enforceMaxKeysLimit() {
+function trimCacheToMaxKeys() {
     if (contentCache.size <= config.cache.maxKeys)
         return;
-    const keysToRemove = contentCache.size - config.cache.maxKeys;
-    const iterator = contentCache.keys();
-    for (let i = 0; i < keysToRemove; i++) {
-        const { value, done } = iterator.next();
-        if (done)
-            break;
-        contentCache.delete(value);
+    removeOldestEntries(contentCache.size - config.cache.maxKeys);
+}
+function removeOldestEntries(count) {
+    if (count <= 0)
+        return;
+    let removed = 0;
+    for (const key of contentCache.keys()) {
+        contentCache.delete(key);
+        removed += 1;
+        if (removed >= count)
+            return;
     }
 }
+function resolveExpiryTimestamp() {
+    return Date.now() + config.cache.ttl * 1000;
+}
+function logCacheError(message, cacheKey, error) {
+    logWarn(message, {
+        key: cacheKey.length > 100 ? cacheKey.slice(0, 100) : cacheKey,
+        error: getErrorMessage(error),
+    });
+}

package/dist/services/context.d.ts

@@ -3,7 +3,6 @@ interface RequestContext {
     readonly sessionId?: string;
 }
 export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
-export declare function bindToRequestContext<T extends (...args: unknown[]) => unknown>(fn: T): T;
 export declare function getRequestId(): string | undefined;
 export declare function getSessionId(): string | undefined;
 export {};

package/dist/services/context.js

@@ -3,13 +3,6 @@ const requestContext = new AsyncLocalStorage();
 export function runWithRequestContext(context, fn) {
     return requestContext.run(context, fn);
 }
-export function bindToRequestContext(fn) {
-    const store = requestContext.getStore();
-    if (!store) {
-        return fn;
-    }
-    return ((...args) => requestContext.run(store, () => fn(...args)));
-}
 export function getRequestId() {
     return requestContext.getStore()?.requestId;
 }

package/dist/services/extractor.js

@@ -1,91 +1,18 @@
 import { parseHTML } from 'linkedom';
 import { Readability } from '@mozilla/readability';
 import { getErrorMessage } from '../utils/error-utils.js';
+import { isRecord } from '../utils/guards.js';
 import { truncateHtml } from '../utils/html-truncator.js';
 import { logError, logInfo, logWarn } from './logger.js';
-import { createMetaCollectorState, resolveMetaField, } from './metadata-collector.js';
-function collectMetaTag(state, tag) {
-    const content = getMetaContent(tag);
-    if (!content)
-        return;
-    if (collectOpenGraphMeta(state, tag, content))
-        return;
-    if (collectTwitterMeta(state, tag, content))
-        return;
-    collectStandardMeta(state, tag, content);
-}
-function getMetaContent(tag) {
-    return tag.getAttribute('content')?.trim() ?? null;
-}
-function collectOpenGraphMeta(state, tag, content) {
-    const property = tag.getAttribute('property');
-    if (!property?.startsWith('og:'))
-        return false;
-    const key = property.replace('og:', '');
-    if (key === 'title')
-        state.title.og = content;
-    if (key === 'description')
-        state.description.og = content;
-    return true;
-}
-function collectTwitterMeta(state, tag, content) {
-    const name = tag.getAttribute('name');
-    if (!name?.startsWith('twitter:'))
-        return false;
-    const key = name.replace('twitter:', '');
-    if (key === 'title')
-        state.title.twitter = content;
-    if (key === 'description')
-        state.description.twitter = content;
-    return true;
-}
-function collectStandardMeta(state, tag, content) {
-    const name = tag.getAttribute('name');
-    if (name === 'description') {
-        state.description.standard = content;
-    }
-    if (name === 'author') {
-        state.author.standard = content;
-    }
-}
-function scanMetaTags(document, state) {
-    const metaTags = document.querySelectorAll('meta');
-    for (const tag of metaTags) {
-        collectMetaTag(state, tag);
-    }
-}
-function ensureTitleFallback(document, state) {
-    if (state.title.standard)
-        return;
-    const titleEl = document.querySelector('title');
-    if (titleEl?.textContent) {
-        state.title.standard = titleEl.textContent.trim();
-    }
-}
-function extractMetadata(document) {
-    const state = createMetaCollectorState();
-    scanMetaTags(document, state);
-    ensureTitleFallback(document, state);
-    const metadata = {};
-    const title = resolveMetaField(state, 'title');
-    const description = resolveMetaField(state, 'description');
-    const author = resolveMetaField(state, 'author');
-    if (title !== undefined)
-        metadata.title = title;
-    if (description !== undefined)
-        metadata.description = description;
-    if (author !== undefined)
-        metadata.author = author;
-    return metadata;
-}
+import { extractMetadata } from './metadata-collector.js';
 function isReadabilityCompatible(doc) {
-    if (!doc || typeof doc !== 'object')
+    if (!isRecord(doc))
         return false;
     if (!('documentElement' in doc))
         return false;
-    if (!('querySelectorAll' in doc))
+    if (typeof doc.querySelectorAll !== 'function')
         return false;
-    if (!('querySelector' in doc))
+    if (typeof doc.querySelector !== 'function')
         return false;
     return true;
 }
@@ -138,12 +65,13 @@ export function extractContent(html, url, options = { extractArticle: true }) {
 }
 function tryExtractContent(html, url, options) {
     try {
-        const processedHtml = truncateHtml(html);
-        const { document } = parseHTML(processedHtml);
+        const { document } = parseHTML(truncateHtml(html));
         applyBaseUri(document, url);
         const metadata = extractMetadata(document);
-        const article = options.extractArticle ? extractArticle(document) : null;
-        return { article, metadata };
+        return {
+            article: options.extractArticle ? extractArticle(document) : null,
+            metadata,
+        };
     }
     catch (error) {
         logError('Failed to extract content', error instanceof Error ? error : undefined);

package/dist/services/fetcher/agents.d.ts

@@ -1,3 +1,3 @@
-import { Agent } from 'undici';
-export declare const dispatcher: Agent;
+import { type Dispatcher } from 'undici';
+export declare const dispatcher: Dispatcher;
 export declare function destroyAgents(): void;

package/dist/services/fetcher/agents.js

@@ -2,26 +2,14 @@ import dns from 'node:dns';
 import os from 'node:os';
 import { Agent } from 'undici';
 import { createErrorWithCode } from '../../utils/error-utils.js';
-import { isBlockedIp } from '../../utils/url-validator.js';
+import { isRecord } from '../../utils/guards.js';
+import { handleLookupResult } from './dns-selection.js';
 const DNS_LOOKUP_TIMEOUT_MS = 5000;
 function resolveDns(hostname, options, callback) {
     const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
     const lookupOptions = buildLookupOptions(normalizedOptions);
-    let done = false;
-    const timer = setTimeout(() => {
-        if (done)
-            return;
-        done = true;
-        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
-    }, DNS_LOOKUP_TIMEOUT_MS);
-    timer.unref();
-    const safeCallback = (err, address, family) => {
-        if (done)
-            return;
-        done = true;
-        clearTimeout(timer);
-        callback(err, address, family);
-    };
+    const timeout = createLookupTimeout(hostname, callback);
+    const safeCallback = wrapLookupCallback(callback, timeout);
     dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
 }
 function normalizeLookupOptions(options) {
@@ -46,17 +34,19 @@ function resolveResultOrder(options) {
     return DEFAULT_DNS_ORDER;
 }
 function getLegacyVerbatim(options) {
-    const legacy = options.verbatim;
-    return typeof legacy === 'boolean' ? legacy : undefined;
+    if (isRecord(options)) {
+        const { verbatim } = options;
+        return typeof verbatim === 'boolean' ? verbatim : undefined;
+    }
+    return undefined;
 }
 function buildLookupOptions(normalizedOptions) {
-    const options = {
-        ...normalizedOptions,
-        order: resolveResultOrder(normalizedOptions),
+    return {
+        family: normalizedOptions.family,
+        hints: normalizedOptions.hints,
         all: true,
+        order: resolveResultOrder(normalizedOptions),
     };
-    delete options.verbatim;
-    return options;
 }
 function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
     return (err, addresses) => {
@@ -70,81 +60,30 @@ function resolveFamily(family) {
         return 6;
     return family;
 }
-function normalizeLookupResults(addresses, family) {
-    if (Array.isArray(addresses)) {
-        return addresses;
-    }
-    return [{ address: addresses, family: family ?? 4 }];
-}
-function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
-    if (error) {
-        callback(error, addresses);
-        return;
-    }
-    const list = normalizeLookupResults(addresses, resolvedFamily);
-    const invalidFamilyError = findInvalidFamilyError(list, hostname);
-    if (invalidFamilyError) {
-        callback(invalidFamilyError, list);
-        return;
-    }
-    const blockedError = findBlockedIpError(list, hostname);
-    if (blockedError) {
-        callback(blockedError, list);
-        return;
-    }
-    const selection = selectLookupResult(list, useAll, hostname);
-    if (selection.error) {
-        callback(selection.error, selection.fallback);
-        return;
-    }
-    callback(null, selection.address, selection.family);
-}
-function selectLookupResult(list, useAll, hostname) {
-    if (list.length === 0) {
-        return {
-            error: createNoDnsResultsError(hostname),
-            fallback: [],
-            address: [],
-        };
-    }
-    if (useAll) {
-        return { address: list, fallback: list };
-    }
-    const first = list.at(0);
-    if (!first) {
-        return {
-            error: createNoDnsResultsError(hostname),
-            fallback: [],
-            address: [],
-        };
-    }
+function createLookupTimeout(hostname, callback) {
+    let done = false;
+    const timer = setTimeout(() => {
+        if (done)
+            return;
+        done = true;
+        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
+    }, DNS_LOOKUP_TIMEOUT_MS);
+    timer.unref();
     return {
-        address: first.address,
-        family: first.family,
-        fallback: list,
+        isDone: () => done,
+        markDone: () => {
+            done = true;
+            clearTimeout(timer);
+        },
     };
 }
-function findBlockedIpError(list, hostname) {
-    for (const addr of list) {
-        const ip = typeof addr === 'string' ? addr : addr.address;
-        if (!isBlockedIp(ip)) {
-            continue;
-        }
-        return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
-    }
-    return null;
-}
-function findInvalidFamilyError(list, hostname) {
-    for (const addr of list) {
-        const family = typeof addr === 'string' ? 0 : addr.family;
-        if (family === 4 || family === 6)
-            continue;
-        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
-    }
-    return null;
-}
-function createNoDnsResultsError(hostname) {
-    return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
+function wrapLookupCallback(callback, timeout) {
+    return (err, address, family) => {
+        if (timeout.isDone())
+            return;
+        timeout.markDone();
+        callback(err, address, family);
+    };
 }
 function getAgentOptions() {
     const cpuCount = os.availableParallelism();

package/dist/services/fetcher/dns-selection.d.ts

@@ -0,0 +1,2 @@
+import type { LookupAddress } from 'node:dns';
+export declare function handleLookupResult(error: NodeJS.ErrnoException | null, addresses: string | LookupAddress[], hostname: string, resolvedFamily: number | undefined, useAll: boolean, callback: (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void): void;

package/dist/services/fetcher/dns-selection.js

@@ -0,0 +1,72 @@
+import { createErrorWithCode } from '../../utils/error-utils.js';
+import { isBlockedIp } from '../../utils/url-validator.js';
+function normalizeLookupResults(addresses, family) {
+    if (Array.isArray(addresses)) {
+        return addresses;
+    }
+    return [{ address: addresses, family: family ?? 4 }];
+}
+function findBlockedIpError(list, hostname) {
+    for (const addr of list) {
+        const ip = typeof addr === 'string' ? addr : addr.address;
+        if (!isBlockedIp(ip)) {
+            continue;
+        }
+        return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
+    }
+    return null;
+}
+function findInvalidFamilyError(list, hostname) {
+    for (const addr of list) {
+        const family = typeof addr === 'string' ? 0 : addr.family;
+        if (family === 4 || family === 6)
+            continue;
+        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
+    }
+    return null;
+}
+function createNoDnsResultsError(hostname) {
+    return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
+}
+function createEmptySelection(hostname) {
+    return {
+        error: createNoDnsResultsError(hostname),
+        fallback: [],
+        address: [],
+    };
+}
+function selectLookupResult(list, useAll, hostname) {
+    if (list.length === 0)
+        return createEmptySelection(hostname);
+    if (useAll)
+        return { address: list, fallback: list };
+    const first = list.at(0);
+    if (!first)
+        return createEmptySelection(hostname);
+    return {
+        address: first.address,
+        family: first.family,
+        fallback: list,
+    };
+}
+function findLookupError(list, hostname) {
+    return (findInvalidFamilyError(list, hostname) ?? findBlockedIpError(list, hostname));
+}
+export function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
+    if (error) {
+        callback(error, addresses);
+        return;
+    }
+    const list = normalizeLookupResults(addresses, resolvedFamily);
+    const lookupError = findLookupError(list, hostname);
+    if (lookupError) {
+        callback(lookupError, list);
+        return;
+    }
+    const selection = selectLookupResult(list, useAll, hostname);
+    if (selection.error) {
+        callback(selection.error, selection.fallback);
+        return;
+    }
+    callback(null, selection.address, selection.family);
+}

package/dist/services/fetcher/interceptors.d.ts

@@ -1,25 +1,3 @@
-export type FetchChannelEvent = {
-    v: 1;
-    type: 'start';
-    requestId: string;
-    method: string;
-    url: string;
-} | {
-    v: 1;
-    type: 'end';
-    requestId: string;
-    status: number;
-    duration: number;
-} | {
-    v: 1;
-    type: 'error';
-    requestId: string;
-    url: string;
-    error: string;
-    code?: string;
-    status?: number;
-    duration: number;
-};
 interface FetchTelemetryContext {
     requestId: string;
     startTime: number;