@j0hanz/superfetch 1.1.2 → 1.1.3

package/dist/services/extractor.js

@@ -1,105 +1,118 @@
-import { JSDOM } from 'jsdom';
+import { parseHTML } from 'linkedom';
 import { Readability } from '@mozilla/readability';
 import { truncateHtml } from '../utils/html-truncator.js';
 import { logError, logWarn } from './logger.js';
-function extractOpenGraph(document) {
-    const data = {};
-    const ogTags = document.querySelectorAll('meta[property^="og:"]');
-    for (const tag of ogTags) {
-        const property = tag.getAttribute('property');
-        const content = tag.getAttribute('content')?.trim();
-        if (!property || !content)
-            continue;
-        const key = property.replace('og:', '');
-        if (key === 'title')
-            data.title = content;
-        else if (key === 'description')
-            data.description = content;
-    }
-    return data;
+function resolveMetaField(state, field) {
+    const sources = state[field];
+    return sources.og ?? sources.twitter ?? sources.standard;
+}
+function createMetaCollectorState() {
+    return {
+        title: {},
+        description: {},
+        author: {},
+    };
+}
+function collectMetaTag(state, tag) {
+    const content = getMetaContent(tag);
+    if (!content)
+        return;
+    if (collectOpenGraphMeta(state, tag, content))
+        return;
+    if (collectTwitterMeta(state, tag, content))
+        return;
+    collectStandardMeta(state, tag, content);
+}
+function getMetaContent(tag) {
+    return tag.getAttribute('content')?.trim() ?? null;
+}
+function collectOpenGraphMeta(state, tag, content) {
+    const property = tag.getAttribute('property');
+    if (!property?.startsWith('og:'))
+        return false;
+    const key = property.replace('og:', '');
+    if (key === 'title')
+        state.title.og = content;
+    if (key === 'description')
+        state.description.og = content;
+    return true;
 }
-function extractTwitterCard(document) {
-    const data = {};
-    const twitterTags = document.querySelectorAll('meta[name^="twitter:"]');
-    for (const tag of twitterTags) {
-        const name = tag.getAttribute('name');
-        const content = tag.getAttribute('content')?.trim();
-        if (!name || !content)
-            continue;
-        const key = name.replace('twitter:', '');
-        if (key === 'title')
-            data.title = content;
-        else if (key === 'description')
-            data.description = content;
+function collectTwitterMeta(state, tag, content) {
+    const name = tag.getAttribute('name');
+    if (!name?.startsWith('twitter:'))
+        return false;
+    const key = name.replace('twitter:', '');
+    if (key === 'title')
+        state.title.twitter = content;
+    if (key === 'description')
+        state.description.twitter = content;
+    return true;
+}
+function collectStandardMeta(state, tag, content) {
+    const name = tag.getAttribute('name');
+    if (name === 'description') {
+        state.description.standard = content;
+    }
+    if (name === 'author') {
+        state.author.standard = content;
     }
-    return data;
 }
-function extractStandardMeta(document) {
-    const data = {};
-    const metaTags = document.querySelectorAll('meta[name][content]');
+function scanMetaTags(document, state) {
+    const metaTags = document.querySelectorAll('meta');
     for (const tag of metaTags) {
-        const name = tag.getAttribute('name');
-        const content = tag.getAttribute('content')?.trim();
-        if (!name || !content)
-            continue;
-        if (name === 'description')
-            data.description = content;
-        else if (name === 'author')
-            data.author = content;
+        collectMetaTag(state, tag);
     }
-    if (!data.title) {
-        const titleEl = document.querySelector('title');
-        if (titleEl?.textContent)
-            data.title = titleEl.textContent.trim();
+}
+function ensureTitleFallback(document, state) {
+    if (state.title.standard)
+        return;
+    const titleEl = document.querySelector('title');
+    if (titleEl?.textContent) {
+        state.title.standard = titleEl.textContent.trim();
     }
-    return data;
 }
-// Main extraction function
-export function extractContent(html, url, options = { extractArticle: true }) {
-    if (!html || typeof html !== 'string') {
-        logWarn('extractContent called with invalid HTML input');
-        return { article: null, metadata: {} };
+function extractMetadata(document) {
+    const state = createMetaCollectorState();
+    scanMetaTags(document, state);
+    ensureTitleFallback(document, state);
+    return {
+        title: resolveMetaField(state, 'title'),
+        description: resolveMetaField(state, 'description'),
+        author: resolveMetaField(state, 'author'),
+    };
+}
+function extractArticle(document) {
+    try {
+        const reader = new Readability(document);
+        const parsed = reader.parse();
+        if (!parsed)
+            return null;
+        return {
+            title: parsed.title ?? undefined,
+            byline: parsed.byline ?? undefined,
+            content: parsed.content ?? '',
+            textContent: parsed.textContent ?? '',
+            excerpt: parsed.excerpt ?? undefined,
+            siteName: parsed.siteName ?? undefined,
+        };
     }
-    if (!url || typeof url !== 'string') {
-        logWarn('extractContent called with invalid URL');
+    catch (error) {
+        logError('Failed to extract article with Readability', error instanceof Error ? error : undefined);
+        return null;
+    }
+}
+export function extractContent(html, url, options = { extractArticle: true }) {
+    if (!isValidInput(html, url)) {
         return { article: null, metadata: {} };
     }
     try {
-        // Truncate HTML to improve performance
         const processedHtml = truncateHtml(html);
-        // Parse HTML with JSDOM
-        const dom = new JSDOM(processedHtml, { url });
-        const { document } = dom.window;
-        const ogData = extractOpenGraph(document);
-        const twitterData = extractTwitterCard(document);
-        const standardData = extractStandardMeta(document);
-        const metadata = {
-            title: ogData.title ?? twitterData.title ?? standardData.title,
-            description: ogData.description ??
-                twitterData.description ??
-                standardData.description,
-            author: standardData.author,
-        };
-        let article = null;
-        if (options.extractArticle) {
-            try {
-                const reader = new Readability(document);
-                const parsed = reader.parse();
-                if (parsed) {
-                    article = {
-                        title: parsed.title ?? undefined,
-                        byline: parsed.byline ?? undefined,
-                        content: parsed.content ?? '',
-                        textContent: parsed.textContent ?? '',
-                        excerpt: parsed.excerpt ?? undefined,
-                        siteName: parsed.siteName ?? undefined,
-                    };
-                }
-            }
-            catch (error) {
-                logError('Failed to extract article with Readability', error instanceof Error ? error : undefined);
-            }
-        }
+        const { document } = parseHTML(processedHtml);
+        applyBaseUri(document, url);
+        const metadata = extractMetadata(document);
+        const article = options.extractArticle
+            ? extractArticle(document)
+            : null;
         return { article, metadata };
     }
     catch (error) {
@@ -107,4 +120,26 @@ export function extractContent(html, url, options = { extractArticle: true }) {
         return { article: null, metadata: {} };
     }
 }
+function isValidInput(html, url) {
+    if (!html || typeof html !== 'string') {
+        logWarn('extractContent called with invalid HTML input');
+        return false;
+    }
+    if (!url || typeof url !== 'string') {
+        logWarn('extractContent called with invalid URL');
+        return false;
+    }
+    return true;
+}
+function applyBaseUri(document, url) {
+    try {
+        Object.defineProperty(document, 'baseURI', {
+            value: url,
+            writable: true,
+        });
+    }
+    catch {
+        // Ignore errors in setting baseURI
+    }
+}
 //# sourceMappingURL=extractor.js.map

package/dist/services/extractor.js.map

@@ -1 +1 @@
-{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAQnD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEhD,SAAS,gBAAgB,CAAC,QAAkB;IAK1C,MAAM,IAAI,GAA6C,EAAE,CAAC;IAC1D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IAElE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;YAAE,SAAS;QAEpC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,GAAG,KAAK,OAAO;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;aACrC,IAAI,GAAG,KAAK,aAAa;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAkB;IAI5C,MAAM,IAAI,GAA6C,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC;IAExE,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;YAAE,SAAS;QAEhC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,OAAO;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;aACrC,IAAI,GAAG,KAAK,aAAa;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;IAC7D,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAkB;IAK7C,MAAM,IAAI,GAA8D,EAAE,CAAC;IAE3E,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC;IAClE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC;QACpD,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;YAAE,SAAS;QAEhC,IAAI,IAAI,KAAK,aAAa;YAAE,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;aAClD,IAAI,IAAI,KAAK,QAAQ;YAAE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;IACpD,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,OAAO,EAAE,WAAW;YAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACpE,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,GAAW,EACX,UAAwC,EAAE,cAAc,EAAE,IAAI,EAAE;IAEhE,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,+CAA+C,CAAC,CAAC;QACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,uCAAuC;QACvC,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACzC,wBAAwB;QACxB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9C,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAsB;YAClC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,WAAW,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK;YAC9D,WAAW,EACT,MAAM,CAAC,WAAW;gBAClB,WAAW,CAAC,WAAW;gBACvB,YAAY,CAAC,WAAW;YAC1B,MAAM,EAAE,YAAY,CAAC,MAAM;SAC5B,CAAC;QACF,IAAI,OAAO,GAA4B,IAAI,CAAC;QAC5C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACzC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;gBAE9B,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,GAAG;wBACR,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS;wBAChC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;wBAClC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;wBAC7B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;wBACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,SAAS;wBACpC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;qBACvC,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,QAAQ,CACN,4CAA4C,EAC5C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC"}
+{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/services/extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAQnD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAWhD,SAAS,gBAAgB,CACvB,KAAyB,EACzB,KAAgB;IAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB;IAC/B,OAAO;QACL,KAAK,EAAE,EAAE;QACT,WAAW,EAAE,EAAE;QACf,MAAM,EAAE,EAAE;KACX,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB,EAAE,GAAoB;IACrE,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,OAAO;QAAE,OAAO;IAErB,IAAI,oBAAoB,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC;QAAE,OAAO;IACtD,IAAI,kBAAkB,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC;QAAE,OAAO;IACpD,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,cAAc,CAAC,GAAoB;IAC1C,OAAO,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;AACrD,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAyB,EACzB,GAAoB,EACpB,OAAe;IAEf,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IAC9C,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/C,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,GAAG,KAAK,OAAO;QAAE,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC;IAC9C,IAAI,GAAG,KAAK,aAAa;QAAE,KAAK,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAyB,EACzB,GAAoB,EACpB,OAAe;IAEf,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACzC,IAAI,GAAG,KAAK,OAAO;QAAE,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IACnD,IAAI,GAAG,KAAK,aAAa;QAAE,KAAK,CAAC,WAAW,CAAC,OAAO,GAAG,OAAO,CAAC;IAC/D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAyB,EACzB,GAAoB,EACpB,OAAe;IAEf,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,WAAW,CAAC,QAAQ,GAAG,OAAO,CAAC;IACvC,CAAC;IAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;IAClC,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,KAAyB;IACjE,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,cAAc,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,QAAkB,EAClB,KAAyB;IAEzB,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ;QAAE,OAAO;IACjC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;QACzB,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,QAAkB;IACzC,MAAM,KAAK,GAAG,wBAAwB,EAAE,CAAC;IAEzC,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC9B,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAErC,OAAO;QACL,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC;QACvC,WAAW,EAAE,gBAAgB,CAAC,KAAK,EAAE,aAAa,CAAC;QACnD,MAAM,EAAE,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAAkB;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAA+B,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE9B,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;YAClC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,SAAS;YACpC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;SACvC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,4CAA4C,EAC5C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,GAAW,EACX,UAAwC,EAAE,cAAc,EAAE,IAAI,EAAE;IAEhE,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;QAE9C,YAAY,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAE5B,MAAM,QAAQ,GAAG,eAAe,CAAC,QAA+B,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc;YACpC,CAAC,CAAC,cAAc,CAAC,QAA+B,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC;QAET,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CACN,2BAA2B,EAC3B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,GAAW;IAC7C,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,CAAC,+CAA+C,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,GAAW;IACnD,IAAI,CAAC;QACH,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE;YACzC,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,mCAAmC;IACrC,CAAC;AACH,CAAC"}

package/dist/services/fetcher/agents.d.ts

@@ -0,0 +1,4 @@
+import { Agent } from 'undici';
+export declare const dispatcher: Agent;
+export declare function destroyAgents(): void;
+//# sourceMappingURL=agents.d.ts.map

package/dist/services/fetcher/agents.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agents.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/agents.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAkM/B,eAAO,MAAM,UAAU,OAA+B,CAAC;AAEvD,wBAAgB,aAAa,IAAI,IAAI,CAEpC"}

package/dist/services/fetcher/agents.js

@@ -0,0 +1,111 @@
+import dns from 'node:dns';
+import os from 'node:os';
+import { Agent } from 'undici';
+import { isBlockedIp } from '../../utils/url-validator.js';
+function resolveDns(hostname, options, callback) {
+    const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
+    dns.lookup(hostname, { ...normalizedOptions, all: true }, createLookupCallback(hostname, resolvedFamily, useAll, callback));
+}
+function normalizeLookupOptions(options) {
+    return typeof options === 'number' ? { family: options } : options;
+}
+function buildLookupContext(options) {
+    const normalizedOptions = normalizeLookupOptions(options);
+    return {
+        normalizedOptions,
+        useAll: Boolean(normalizedOptions.all),
+        resolvedFamily: resolveFamily(normalizedOptions.family),
+    };
+}
+function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
+    return (err, addresses) => {
+        handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
+    };
+}
+function resolveFamily(family) {
+    if (family === 'IPv4')
+        return 4;
+    if (family === 'IPv6')
+        return 6;
+    return family;
+}
+function normalizeLookupResults(addresses, family) {
+    if (Array.isArray(addresses)) {
+        return addresses;
+    }
+    return [{ address: addresses, family: family ?? 4 }];
+}
+function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
+    if (error) {
+        callback(error, addresses);
+        return;
+    }
+    const list = normalizeLookupResults(addresses, resolvedFamily);
+    const blockedError = findBlockedIpError(list, hostname);
+    if (blockedError) {
+        callback(blockedError, list);
+        return;
+    }
+    const selection = selectLookupResult(list, useAll, hostname);
+    if (selection.error) {
+        callback(selection.error, selection.fallback);
+        return;
+    }
+    callback(null, selection.address, selection.family);
+}
+function selectLookupResult(list, useAll, hostname) {
+    if (list.length === 0) {
+        return {
+            error: createNoDnsResultsError(hostname),
+            fallback: [],
+            address: [],
+        };
+    }
+    if (useAll) {
+        return { address: list, fallback: list };
+    }
+    const first = list.at(0);
+    if (!first) {
+        return {
+            error: createNoDnsResultsError(hostname),
+            fallback: [],
+            address: [],
+        };
+    }
+    return {
+        address: first.address,
+        family: first.family,
+        fallback: list,
+    };
+}
+function findBlockedIpError(list, hostname) {
+    for (const addr of list) {
+        const ip = typeof addr === 'string' ? addr : addr.address;
+        if (!isBlockedIp(ip)) {
+            continue;
+        }
+        const error = new Error(`Blocked IP detected for ${hostname}`);
+        error.code = 'EBLOCKED';
+        return error;
+    }
+    return null;
+}
+function createNoDnsResultsError(hostname) {
+    const error = new Error(`No DNS results returned for ${hostname}`);
+    error.code = 'ENODATA';
+    return error;
+}
+function getAgentOptions() {
+    const cpuCount = os.availableParallelism();
+    return {
+        keepAliveTimeout: 60000,
+        connections: Math.max(cpuCount * 2, 25),
+        pipelining: 1,
+        connect: { lookup: resolveDns },
+    };
+}
+export const dispatcher = new Agent(getAgentOptions());
+export function destroyAgents() {
+    void dispatcher.close();
+}
+//# sourceMappingURL=agents.js.map

package/dist/services/fetcher/agents.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agents.js","sourceRoot":"","sources":["../../../src/services/fetcher/agents.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAE3D,SAAS,UAAU,CACjB,QAAgB,EAChB,OAA0B,EAC1B,QAIS;IAET,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,cAAc,EAAE,GACjD,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC9B,GAAG,CAAC,MAAM,CACR,QAAQ,EACR,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,IAAI,EAAE,EACnC,oBAAoB,CAAC,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CACjE,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAmC;IAEnC,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACrE,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAmC;IAK7D,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC1D,OAAO;QACL,iBAAiB;QACjB,MAAM,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC;QACtC,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM,CAAC;KACxD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,cAAkC,EAClC,MAAe,EACf,QAIS;IAKT,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;QACxB,kBAAkB,CAChB,GAAG,EACH,SAAS,EACT,QAAQ,EACR,cAAc,EACd,MAAM,EACN,QAAQ,CACT,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,MAAmC;IAEnC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,sBAAsB,CAC7B,SAAuC,EACvC,MAA0B;IAE1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAmC,EACnC,SAAuC,EACvC,QAAgB,EAChB,cAAkC,EAClC,MAAe,EACf,QAIS;IAET,IAAI,KAAK,EAAE,CAAC;QACV,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,sBAAsB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxD,IAAI,YAAY,EAAE,CAAC;QACjB,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IAED,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,MAAe,EACf,QAAgB;IAOhB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,CAAC;YACxC,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,uBAAuB,CAAC,QAAQ,CAAC;YACxC,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,IAAI;KACf,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,QAAgB;IAEhB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,2BAA2B,QAAQ,EAAE,CACb,CAAC;QAC3B,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,+BAA+B,QAAQ,EAAE,CACjB,CAAC;IAC3B,KAAK,CAAC,IAAI,GAAG,SAAS,CAAC;IACvB,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,QAAQ,GAAG,EAAE,CAAC,oBAAoB,EAAE,CAAC;IAC3C,OAAO;QACL,gBAAgB,EAAE,KAAK;QACvB,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,EAAE,CAAC;QACvC,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;AAEvD,MAAM,UAAU,aAAa;IAC3B,KAAK,UAAU,CAAC,KAAK,EAAE,CAAC;AAC1B,CAAC"}

package/dist/services/fetcher/errors.d.ts

@@ -0,0 +1,5 @@
+import { FetchError } from '../../errors/app-error.js';
+export declare function createRateLimitError(url: string, headerValue: string | null): FetchError;
+export declare function createHttpError(url: string, status: number, statusText: string): FetchError;
+export declare function mapFetchError(error: unknown, fallbackUrl: string, timeoutMs: number): FetchError;
+//# sourceMappingURL=errors.d.ts.map

package/dist/services/fetcher/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAoBvD,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,UAAU,CAGZ;AAED,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,GACjB,UAAU,CAEZ;AA4CD,wBAAgB,aAAa,CAC3B,KAAK,EAAE,OAAO,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,UAAU,CAiBZ"}

package/dist/services/fetcher/errors.js

@@ -0,0 +1,71 @@
+import { FetchError } from '../../errors/app-error.js';
+function parseRetryAfter(header) {
+    if (!header)
+        return 60;
+    const parsed = parseInt(header, 10);
+    return Number.isNaN(parsed) ? 60 : parsed;
+}
+function createCanceledError(url) {
+    return new FetchError('Request was canceled', url, 499, {
+        reason: 'aborted',
+    });
+}
+function createTimeoutError(url, timeoutMs) {
+    return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
+        timeout: timeoutMs,
+    });
+}
+export function createRateLimitError(url, headerValue) {
+    const retryAfter = parseRetryAfter(headerValue);
+    return new FetchError('Too many requests', url, 429, { retryAfter });
+}
+export function createHttpError(url, status, statusText) {
+    return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
+}
+function createNetworkError(url, message) {
+    const details = message ? { message } : undefined;
+    return new FetchError(`Network error: Could not reach ${url}`, url, undefined, details ?? {});
+}
+function createUnknownError(url, message) {
+    return new FetchError(message, url);
+}
+function isAbortError(error) {
+    return (error instanceof Error &&
+        (error.name === 'AbortError' || error.name === 'TimeoutError'));
+}
+function isTimeoutError(error) {
+    return error instanceof Error && error.name === 'TimeoutError';
+}
+function isRecord(value) {
+    return value !== null && typeof value === 'object';
+}
+function getRequestUrl(record) {
+    const value = record.requestUrl;
+    return typeof value === 'string' ? value : null;
+}
+function resolveErrorUrl(error, fallback) {
+    if (error instanceof FetchError)
+        return error.url;
+    if (!isRecord(error))
+        return fallback;
+    const requestUrl = getRequestUrl(error);
+    if (requestUrl)
+        return requestUrl;
+    return fallback;
+}
+export function mapFetchError(error, fallbackUrl, timeoutMs) {
+    if (error instanceof FetchError)
+        return error;
+    const url = resolveErrorUrl(error, fallbackUrl);
+    if (isAbortError(error)) {
+        if (isTimeoutError(error)) {
+            return createTimeoutError(url, timeoutMs);
+        }
+        return createCanceledError(url);
+    }
+    if (error instanceof Error) {
+        return createNetworkError(url, error.message);
+    }
+    return createUnknownError(url, 'Unexpected error');
+}
+//# sourceMappingURL=errors.js.map

package/dist/services/fetcher/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../src/services/fetcher/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,SAAS,eAAe,CAAC,MAAqB;IAC5C,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO,IAAI,UAAU,CAAC,sBAAsB,EAAE,GAAG,EAAE,GAAG,EAAE;QACtD,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,SAAiB;IACxD,OAAO,IAAI,UAAU,CAAC,yBAAyB,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE;QACtE,OAAO,EAAE,SAAS;KACnB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAW,EACX,WAA0B;IAE1B,MAAM,UAAU,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;IAChD,OAAO,IAAI,UAAU,CAAC,mBAAmB,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,GAAW,EACX,MAAc,EACd,UAAkB;IAElB,OAAO,IAAI,UAAU,CAAC,QAAQ,MAAM,KAAK,UAAU,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,OAAgB;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,OAAO,IAAI,UAAU,CACnB,kCAAkC,GAAG,EAAE,EACvC,GAAG,EACH,SAAS,EACT,OAAO,IAAI,EAAE,CACd,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,OAAe;IACtD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,KAAK,YAAY,KAAK;QACtB,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAC/D,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,MAA+B;IACpD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC;IAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,QAAgB;IACvD,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC;IAClD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IACtC,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAClC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,KAAc,EACd,WAAmB,EACnB,SAAiB;IAEjB,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAE9C,MAAM,GAAG,GAAG,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAEhD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;QAC3B,OAAO,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,kBAAkB,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;AACrD,CAAC"}

package/dist/services/fetcher/headers.d.ts

@@ -0,0 +1,2 @@
+export declare function sanitizeHeaders(headers?: Record<string, string>): Record<string, string> | undefined;
+//# sourceMappingURL=headers.d.ts.map

package/dist/services/fetcher/headers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAEA,wBAAgB,eAAe,CAC7B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAQpC"}

package/dist/services/fetcher/headers.js

@@ -0,0 +1,28 @@
+import { config } from '../../config/index.js';
+export function sanitizeHeaders(headers) {
+    if (!headers || Object.keys(headers).length === 0)
+        return undefined;
+    const normalized = normalizeHeaders(headers, config.security.blockedHeaders);
+    const iterator = normalized.keys();
+    if (iterator.next().done)
+        return undefined;
+    return Object.fromEntries(normalized.entries());
+}
+function normalizeHeaders(headers, blockedHeaders) {
+    const normalized = new Headers();
+    for (const [key, value] of Object.entries(headers)) {
+        if (blockedHeaders.has(key.toLowerCase()))
+            continue;
+        setHeaderSafe(normalized, key, value);
+    }
+    return normalized;
+}
+function setHeaderSafe(headers, key, value) {
+    try {
+        headers.set(key, value);
+    }
+    catch {
+        // Ignore invalid header values
+    }
+}
+//# sourceMappingURL=headers.js.map

package/dist/services/fetcher/headers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/services/fetcher/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,MAAM,UAAU,eAAe,CAC7B,OAAgC;IAEhC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpE,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAE3C,OAAO,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAA+B,EAC/B,cAA2B;IAE3B,MAAM,UAAU,GAAG,IAAI,OAAO,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,IAAI,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACpD,aAAa,CAAC,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,OAAgB,EAAE,GAAW,EAAE,KAAa;IACjE,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;AACH,CAAC"}

package/dist/services/fetcher/interceptors.d.ts

@@ -0,0 +1,10 @@
+export interface FetchTelemetryContext {
+    requestId: string;
+    startTime: number;
+    url: string;
+    method: string;
+}
+export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
+export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
+export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
+//# sourceMappingURL=interceptors.d.ts.map

package/dist/services/fetcher/interceptors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"interceptors.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAwBvB;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CA+BN;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CA4BN"}

package/dist/services/fetcher/interceptors.js

@@ -0,0 +1,82 @@
+import { randomUUID } from 'node:crypto';
+import diagnosticsChannel from 'node:diagnostics_channel';
+import { performance } from 'node:perf_hooks';
+import { logDebug, logError, logWarn } from '../logger.js';
+const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
+export function startFetchTelemetry(url, method) {
+    const context = {
+        requestId: randomUUID(),
+        startTime: performance.now(),
+        url,
+        method: method.toUpperCase(),
+    };
+    if (fetchChannel.hasSubscribers) {
+        fetchChannel.publish({
+            type: 'start',
+            requestId: context.requestId,
+            method: context.method,
+            url: context.url,
+        });
+    }
+    logDebug('HTTP Request', {
+        requestId: context.requestId,
+        method: context.method,
+        url: context.url,
+    });
+    return context;
+}
+export function recordFetchResponse(context, response, contentSize) {
+    const duration = performance.now() - context.startTime;
+    const contentType = response.headers.get('content-type') ?? undefined;
+    const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
+    if (fetchChannel.hasSubscribers) {
+        fetchChannel.publish({
+            type: 'end',
+            requestId: context.requestId,
+            status: response.status,
+            duration,
+        });
+    }
+    logDebug('HTTP Response', {
+        requestId: context.requestId,
+        status: response.status,
+        url: context.url,
+        contentType,
+        duration: `${Math.round(duration)}ms`,
+        size: contentLength,
+    });
+    if (duration > 5000) {
+        logWarn('Slow HTTP request detected', {
+            requestId: context.requestId,
+            url: context.url,
+            duration: `${Math.round(duration)}ms`,
+        });
+    }
+}
+export function recordFetchError(context, error, status) {
+    const duration = performance.now() - context.startTime;
+    const err = error instanceof Error ? error : new Error(String(error));
+    const code = typeof err.code === 'string'
+        ? err.code
+        : undefined;
+    if (fetchChannel.hasSubscribers) {
+        fetchChannel.publish({
+            type: 'error',
+            requestId: context.requestId,
+            url: context.url,
+            error: err.message,
+            code,
+            status,
+            duration,
+        });
+    }
+    const log = status === 429 ? logWarn : logError;
+    log('HTTP Request Error', {
+        requestId: context.requestId,
+        url: context.url,
+        status,
+        code,
+        error: err.message,
+    });
+}
+//# sourceMappingURL=interceptors.js.map

package/dist/services/fetcher/interceptors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interceptors.js","sourceRoot":"","sources":["../../../src/services/fetcher/interceptors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,kBAAkB,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE3D,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AASpE,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,MAAc;IAEd,MAAM,OAAO,GAA0B;QACrC,SAAS,EAAE,UAAU,EAAE;QACvB,SAAS,EAAE,WAAW,CAAC,GAAG,EAAE;QAC5B,GAAG;QACH,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;KAC7B,CAAC;IAEF,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,cAAc,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA8B,EAC9B,QAAkB,EAClB,WAAoB;IAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC;IACtE,MAAM,aAAa,GACjB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,WAAW,EAAE,QAAQ,EAAE,CAAC;IAEpE,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,eAAe,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,WAAW;QACX,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI;QACrC,IAAI,EAAE,aAAa;KACpB,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,IAAI,EAAE,CAAC;QACpB,OAAO,CAAC,4BAA4B,EAAE;YACpC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,QAAQ,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI;SACtC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAA8B,EAC9B,KAAc,EACd,MAAe;IAEf,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;IACvD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,MAAM,IAAI,GACR,OAAQ,GAA6B,CAAC,IAAI,KAAK,QAAQ;QACrD,CAAC,CAAE,GAA6B,CAAC,IAAI;QACrC,CAAC,CAAC,SAAS,CAAC;IAEhB,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QAChC,YAAY,CAAC,OAAO,CAAC;YACnB,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,IAAI;YACJ,MAAM;YACN,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IAChD,GAAG,CAAC,oBAAoB,EAAE;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM;QACN,IAAI;QACJ,KAAK,EAAE,GAAG,CAAC,OAAO;KACnB,CAAC,CAAC;AACL,CAAC"}

package/dist/services/fetcher/redirects.d.ts

@@ -0,0 +1,6 @@
+export declare function resolveRedirectTarget(baseUrl: string, location: string): string;
+export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
+    response: Response;
+    url: string;
+}>;
+//# sourceMappingURL=redirects.d.ts.map

package/dist/services/fetcher/redirects.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redirects.d.ts","sourceRoot":"","sources":["../../../src/services/fetcher/redirects.ts"],"names":[],"mappings":"AAqDA,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,MAAM,CAiBR;AAED,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAwB9C"}