@rudderhq/server 0.6.6-canary.8 → 0.7.0

package/dist/services/website-metadata.js

@@ -1,18 +1,23 @@
 import { resolveKnownWebsiteIcon } from "@rudderhq/shared";
+import ipaddr from "ipaddr.js";
 import { JSDOM } from "jsdom";
 import { createHash } from "node:crypto";
 import { lookup } from "node:dns/promises";
-import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { isIP } from "node:net";
 import path from "node:path";
+import { Agent } from "undici";
 import { resolveRudderInstanceRoot } from "../home-paths.js";
 const MAX_HTML_BYTES = 256 * 1024;
 const MAX_ICON_BYTES = 512 * 1024;
+const MAX_PAGE_TITLE_CHARACTERS = 160;
 const FETCH_TIMEOUT_MS = 5_000;
 const MAX_REDIRECTS = 5;
 const SUCCESS_CACHE_TTL_MS = 30 * 60_000;
 const FAILURE_CACHE_TTL_MS = 5 * 60_000;
+const MAX_METADATA_CACHE_ENTRIES = 128;
 const ICON_DISK_CACHE_TTL_MS = 7 * 24 * 60 * 60_000;
+const MAX_ICON_DISK_CACHE_ENTRIES = 128;
 const FALLBACK_FAVICON_SIZE = "64";
 const IMAGE_CONTENT_TYPES = new Set([
     "image/gif",
@@ -26,9 +31,6 @@ const IMAGE_CONTENT_TYPES = new Set([
 ]);
 function isBenchmarkNetworkIpAddress(value) {
     const normalized = value.replace(/^\[|\]$/gu, "").toLowerCase();
-    const ipv4Mapped = normalized.match(/^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/u);
-    if (ipv4Mapped)
-        return isBenchmarkNetworkIpAddress(ipv4Mapped[1]);
     const parts = normalized.split(".");
     if (parts.length !== 4)
         return false;
@@ -40,36 +42,17 @@ function isBenchmarkNetworkIpAddress(value) {
 }
 function isPrivateIpAddress(value) {
     const normalized = value.replace(/^\[|\]$/gu, "").toLowerCase();
-    const ipv4Mapped = normalized.match(/^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/u);
-    if (ipv4Mapped)
-        return isPrivateIpAddress(ipv4Mapped[1]);
-    if (normalized === "::1" || normalized.startsWith("fe80:") || normalized.startsWith("fc") || normalized.startsWith("fd"))
-        return true;
-    const parts = normalized.split(".");
-    if (parts.length !== 4)
-        return false;
-    const octets = parts.map((part) => Number(part));
-    if (octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255))
+    if (!ipaddr.isValid(normalized))
         return false;
-    const [a, b] = octets;
-    if (a === 10 || a === 127 || a === 0)
-        return true;
-    if (a === 100 && b >= 64 && b <= 127)
-        return true;
-    if (a === 169 && b === 254)
-        return true;
-    if (a === 172 && b >= 16 && b <= 31)
-        return true;
-    if (a === 192 && b === 0)
-        return true;
-    if (a === 192 && b === 168)
-        return true;
-    if (isBenchmarkNetworkIpAddress(normalized))
-        return true;
-    return false;
-}
-function isBlockedResolvedIpAddress(value) {
-    return isPrivateIpAddress(value) && !isBenchmarkNetworkIpAddress(value);
+    const address = ipaddr.process(normalized);
+    if (address.kind() === "ipv4") {
+        return address.range() !== "unicast"
+            || isBenchmarkNetworkIpAddress(address.toString());
+    }
+    const bytes = address.toByteArray();
+    const isDeprecatedSiteLocal = bytes[0] === 0xfe
+        && ((bytes[1] ?? 0) & 0xc0) === 0xc0;
+    return address.range() !== "unicast" || isDeprecatedSiteLocal;
 }
 function isPrivateHostname(hostname) {
     const normalized = hostname.replace(/^\[|\]$/gu, "").toLowerCase();
@@ -101,6 +84,9 @@ export function parsePublicHttpUrl(value, options = {}) {
     if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
         throw new Error("Only http and https URLs can be inspected");
     }
+    if (parsed.username || parsed.password) {
+        throw new Error("Credentialed URLs cannot be inspected");
+    }
     if (!options.allowPrivateHosts && isPrivateHostname(parsed.hostname)) {
         throw new Error("Private network URLs cannot be inspected");
     }
@@ -110,24 +96,120 @@ export function parsePublicHttpUrl(value, options = {}) {
 function resolveWebsiteIconCacheDir() {
     return path.resolve(resolveRudderInstanceRoot(), "data", "website-icons");
 }
-async function assertPublicResolvedHost(url, options) {
-    if (options.allowPrivateHosts || options.fetchImpl || isIP(url.hostname.replace(/^\[|\]$/gu, "")))
-        return;
-    const addresses = await lookup(url.hostname, { all: true, verbatim: true });
-    const allowBenchmarkNetworkResolution = isPublicDnsHostname(url.hostname);
-    if (addresses.some((address) => isPrivateIpAddress(address.address) && (!allowBenchmarkNetworkResolution || !isBenchmarkNetworkIpAddress(address.address)))) {
+async function awaitWithAbort(promise, signal, onLateValue) {
+    const disposeLateValue = (value) => {
+        if (!onLateValue)
+            return;
+        void Promise.resolve(onLateValue(value)).catch(() => undefined);
+    };
+    if (signal.aborted) {
+        void promise.then(disposeLateValue, () => undefined);
+        throw signal.reason;
+    }
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        const onAbort = () => {
+            if (settled)
+                return;
+            settled = true;
+            signal.removeEventListener("abort", onAbort);
+            reject(signal.reason);
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+        void promise.then((value) => {
+            if (settled) {
+                disposeLateValue(value);
+                return;
+            }
+            settled = true;
+            signal.removeEventListener("abort", onAbort);
+            resolve(value);
+        }, (error) => {
+            if (settled)
+                return;
+            settled = true;
+            signal.removeEventListener("abort", onAbort);
+            reject(error);
+        });
+    });
+}
+async function resolveValidatedHostAddress(url, options, signal) {
+    const hostname = url.hostname.replace(/^\[|\]$/gu, "");
+    if (options.fetchImpl || isIP(hostname))
+        return null;
+    const addresses = await awaitWithAbort(lookup(url.hostname, { all: true, verbatim: true }), signal);
+    if (addresses.length === 0)
+        throw new Error("Website hostname did not resolve");
+    if (!options.allowPrivateHosts
+        && addresses.some((address) => isPrivateIpAddress(address.address))) {
         throw new Error("Private network URLs cannot be inspected");
     }
+    return addresses[0] ?? null;
+}
+function createPinnedDispatcher(resolvedHost) {
+    if (!resolvedHost)
+        return null;
+    return new Agent({
+        connect: {
+            autoSelectFamily: false,
+            lookup: (_hostname, _options, callback) => {
+                callback(null, resolvedHost.address, resolvedHost.family);
+            },
+        },
+    });
+}
+async function cancelResponseBody(response) {
+    const body = response?.body;
+    if (!body || body.locked)
+        return;
+    try {
+        await body.cancel();
+    }
+    catch {
+        // Fetch aborts and peer disconnects can close the body before cancellation settles.
+    }
+}
+async function closeDispatcher(dispatcher, aborted) {
+    if (!dispatcher)
+        return;
+    try {
+        if (aborted)
+            await dispatcher.destroy();
+        else
+            await dispatcher.close();
+    }
+    catch {
+        // Cleanup must not replace the request result with a connection teardown error.
+    }
 }
 async function fetchWithTimeout(url, options, init) {
     let currentUrl = parsePublicHttpUrl(url.href, options);
     const fetchImpl = options.fetchImpl ?? fetch;
-    for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {
-        await assertPublicResolvedHost(currentUrl, options);
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
-        try {
-            const response = await fetchImpl(currentUrl.href, {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => {
+        controller.abort(new Error("Website metadata request timed out"));
+    }, FETCH_TIMEOUT_MS);
+    timeout.unref?.();
+    let activeResponse = null;
+    let activeDispatcher = null;
+    let released = false;
+    const release = async () => {
+        if (released)
+            return;
+        released = true;
+        clearTimeout(timeout);
+        await cancelResponseBody(activeResponse);
+        activeResponse = null;
+        await closeDispatcher(activeDispatcher, controller.signal.aborted);
+        activeDispatcher = null;
+    };
+    try {
+        for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {
+            const resolvedHost = await resolveValidatedHostAddress(currentUrl, options, controller.signal);
+            if (controller.signal.aborted)
+                throw controller.signal.reason;
+            activeDispatcher = createPinnedDispatcher(resolvedHost);
+            const requestInit = {
                 redirect: "manual",
                 ...init,
                 signal: controller.signal,
@@ -136,25 +218,59 @@ async function fetchWithTimeout(url, options, init) {
                     "user-agent": "RudderWebsiteMetadata/1.0",
                     ...(init?.headers ?? {}),
                 },
-            });
-            if (response.status < 300 || response.status >= 400)
-                return { response, url: currentUrl };
+                ...(activeDispatcher ? { dispatcher: activeDispatcher } : {}),
+            };
+            const response = await awaitWithAbort(fetchImpl(currentUrl.href, { ...requestInit }), controller.signal, cancelResponseBody);
+            activeResponse = response;
+            if (response.status < 300 || response.status >= 400) {
+                return {
+                    response,
+                    url: currentUrl,
+                    signal: controller.signal,
+                    release,
+                };
+            }
             const location = response.headers.get("location");
-            if (!location)
-                return { response, url: currentUrl };
+            if (!location) {
+                return {
+                    response,
+                    url: currentUrl,
+                    signal: controller.signal,
+                    release,
+                };
+            }
+            await cancelResponseBody(activeResponse);
+            activeResponse = null;
+            await closeDispatcher(activeDispatcher, controller.signal.aborted);
+            activeDispatcher = null;
             currentUrl = parsePublicHttpUrl(new URL(location, currentUrl).href, options);
         }
-        finally {
-            clearTimeout(timeout);
-        }
+        throw new Error("Website metadata redirect limit exceeded");
+    }
+    catch (error) {
+        await release();
+        throw error;
     }
-    throw new Error("Website metadata redirect limit exceeded");
 }
-function metadataCacheKey(url) {
-    return url.href;
+function metadataCacheKey(url, purpose) {
+    return `${purpose}:${url.href}`;
 }
 const metadataCache = new Map();
 const metadataInflight = new Map();
+function pruneMetadataCache(now = Date.now()) {
+    for (const [key, entry] of metadataCache) {
+        if (entry.expiresAt <= now)
+            metadataCache.delete(key);
+    }
+}
+function makeRoomInMetadataCache() {
+    while (metadataCache.size >= MAX_METADATA_CACHE_ENTRIES) {
+        const oldestKey = metadataCache.keys().next().value;
+        if (!oldestKey)
+            break;
+        metadataCache.delete(oldestKey);
+    }
+}
 async function readLimitedBuffer(response, maxBytes, options = {}) {
     const body = response.body;
     if (!body)
@@ -162,9 +278,15 @@ async function readLimitedBuffer(response, maxBytes, options = {}) {
     const reader = body.getReader();
     const chunks = [];
     let total = 0;
+    const onAbort = () => {
+        void reader.cancel(options.signal?.reason).catch(() => undefined);
+    };
+    options.signal?.addEventListener("abort", onAbort, { once: true });
     try {
         while (true) {
             const { value, done } = await reader.read();
+            if (options.signal?.aborted)
+                throw options.signal.reason;
             if (done)
                 break;
             if (!value)
@@ -178,12 +300,14 @@ async function readLimitedBuffer(response, maxBytes, options = {}) {
                     await reader.cancel();
                     break;
                 }
+                await reader.cancel();
                 throw new Error("Response exceeds metadata size limit");
             }
             chunks.push(Buffer.from(value));
         }
     }
     finally {
+        options.signal?.removeEventListener("abort", onAbort);
         reader.releaseLock();
     }
     return Buffer.concat(chunks);
@@ -195,6 +319,7 @@ function isInspectableUrlFailure(error) {
     if (!(error instanceof Error))
         return false;
     return error.message === "Only http and https URLs can be inspected"
+        || error.message === "Credentialed URLs cannot be inspected"
         || error.message === "Private network URLs cannot be inspected"
         || error.message === "Website metadata redirect limit exceeded";
 }
@@ -231,6 +356,32 @@ function readSiteName(document) {
     const title = document.querySelector("title")?.textContent?.trim();
     return title || null;
 }
+function normalizePageTitle(document, value) {
+    if (!value)
+        return null;
+    const template = document.createElement("template");
+    template.innerHTML = value;
+    template.content.querySelectorAll("script, style, template, noscript").forEach((element) => element.remove());
+    const plainText = (template.content.textContent ?? "")
+        .replace(/[\s\p{Cc}\p{Cf}]+/gu, " ")
+        .trim();
+    if (!plainText)
+        return null;
+    return Array.from(plainText).slice(0, MAX_PAGE_TITLE_CHARACTERS).join("");
+}
+function readPageTitle(document) {
+    const candidates = [
+        document.querySelector('meta[property="og:title"]')?.getAttribute("content"),
+        document.querySelector('meta[name="twitter:title"]')?.getAttribute("content"),
+        document.querySelector("title")?.textContent,
+    ];
+    for (const candidate of candidates) {
+        const title = normalizePageTitle(document, candidate);
+        if (title)
+            return title;
+    }
+    return null;
+}
 function iconPriority(element) {
     const rel = new Set(linkRelTokens(element));
     if (rel.has("icon") && rel.has("shortcut"))
@@ -263,19 +414,24 @@ function findDeclaredIcon(document, baseUrl, options) {
     return null;
 }
 async function validateIconUrl(iconHref, options) {
+    let result = null;
     try {
-        const { response, url } = await fetchWithTimeout(parsePublicHttpUrl(iconHref, options), options, { method: "GET" });
+        result = await fetchWithTimeout(parsePublicHttpUrl(iconHref, options), options, { method: "GET" });
+        const { response, signal, url } = result;
         if (!response.ok)
             return null;
         const contentType = contentTypeBase(response.headers.get("content-type"));
         if (!IMAGE_CONTENT_TYPES.has(contentType))
             return null;
-        await readLimitedBuffer(response, MAX_ICON_BYTES);
+        await readLimitedBuffer(response, MAX_ICON_BYTES, { signal });
         return url.href;
     }
     catch {
         return null;
     }
+    finally {
+        await result?.release();
+    }
 }
 async function findImplicitFavicon(baseUrl, options) {
     const faviconUrl = new URL("/favicon.ico", baseUrl);
@@ -298,8 +454,15 @@ async function findProviderFavicon(pageUrl, options) {
 async function resolveWebsiteMetadataUncached(value, options) {
     const pageUrl = parsePublicHttpUrl(value, options);
     const knownIcon = resolveKnownWebsiteIcon(pageUrl);
-    if (knownIcon) {
-        return { url: pageUrl.href, siteName: knownIcon.siteName, iconUrl: knownIcon.iconDataUrl };
+    const emptyMetadata = {
+        url: pageUrl.href,
+        siteName: knownIcon?.siteName ?? null,
+        pageTitle: null,
+        iconUrl: knownIcon?.iconDataUrl ?? null,
+    };
+    // Preview rendering keeps bundled icons zero-fetch; authoring falls through to resolve a title.
+    if (knownIcon && (options.purpose ?? "preview") === "preview") {
+        return emptyMetadata;
     }
     let pageResult;
     try {
@@ -308,25 +471,46 @@ async function resolveWebsiteMetadataUncached(value, options) {
     catch (error) {
         if (isInspectableUrlFailure(error))
             throw error;
-        return { url: pageUrl.href, siteName: null, iconUrl: null };
+        return emptyMetadata;
     }
     const { response, url: finalPageUrl } = pageResult;
-    if (!response.ok) {
-        return { url: finalPageUrl.href, siteName: null, iconUrl: null };
+    let html;
+    try {
+        if (!response.ok)
+            return emptyMetadata;
+        const contentType = contentTypeBase(response.headers.get("content-type"));
+        if (contentType && contentType !== "text/html" && contentType !== "application/xhtml+xml") {
+            return emptyMetadata;
+        }
+        html = (await readLimitedBuffer(response, MAX_HTML_BYTES, {
+            signal: pageResult.signal,
+            truncate: true,
+        })).toString("utf8");
     }
-    const contentType = contentTypeBase(response.headers.get("content-type"));
-    if (contentType && contentType !== "text/html" && contentType !== "application/xhtml+xml") {
-        return { url: finalPageUrl.href, siteName: null, iconUrl: null };
+    catch {
+        return emptyMetadata;
+    }
+    finally {
+        await pageResult.release();
     }
-    const html = (await readLimitedBuffer(response, MAX_HTML_BYTES, { truncate: true })).toString("utf8");
     const dom = new JSDOM(html, { url: finalPageUrl.href });
     try {
         const document = dom.window.document;
+        if ((options.purpose ?? "preview") === "authoring") {
+            return {
+                url: pageUrl.href,
+                siteName: knownIcon?.siteName ?? readSiteName(document),
+                pageTitle: readPageTitle(document),
+                iconUrl: knownIcon?.iconDataUrl ?? null,
+            };
+        }
         const declaredIcon = findDeclaredIcon(document, finalPageUrl, options);
         return {
-            url: finalPageUrl.href,
-            siteName: readSiteName(document),
-            iconUrl: (declaredIcon ? await validateIconUrl(declaredIcon, options) : null)
+            url: pageUrl.href,
+            siteName: knownIcon?.siteName ?? readSiteName(document),
+            pageTitle: readPageTitle(document),
+            iconUrl: knownIcon?.iconDataUrl
+                ?? (declaredIcon ? await validateIconUrl(declaredIcon, options) : null)
                 ?? await findImplicitFavicon(finalPageUrl, options)
                 ?? await findProviderFavicon(finalPageUrl, options),
         };
@@ -337,18 +521,26 @@ async function resolveWebsiteMetadataUncached(value, options) {
 }
 export async function resolveWebsiteMetadata(value, options = {}) {
     const pageUrl = parsePublicHttpUrl(value, options);
-    const key = metadataCacheKey(pageUrl);
+    const key = metadataCacheKey(pageUrl, options.purpose ?? "preview");
     if (!options.allowPrivateHosts && !options.fetchImpl) {
+        pruneMetadataCache();
         const cached = metadataCache.get(key);
-        if (cached && cached.expiresAt > Date.now())
+        if (cached) {
+            metadataCache.delete(key);
+            metadataCache.set(key, cached);
             return cached.value;
+        }
         const inflight = metadataInflight.get(key);
         if (inflight)
             return inflight;
         const request = resolveWebsiteMetadataUncached(pageUrl.href, options)
             .then((metadata) => {
+            pruneMetadataCache();
+            makeRoomInMetadataCache();
             metadataCache.set(key, {
-                expiresAt: Date.now() + (metadata.iconUrl ? SUCCESS_CACHE_TTL_MS : FAILURE_CACHE_TTL_MS),
+                expiresAt: Date.now() + (metadata.iconUrl || metadata.pageTitle || metadata.siteName
+                    ? SUCCESS_CACHE_TTL_MS
+                    : FAILURE_CACHE_TTL_MS),
                 value: metadata,
             });
             return metadata;
@@ -368,20 +560,26 @@ export async function fetchWebsiteIcon(value, options = {}) {
         if (cached)
             return cached;
     }
-    const { response } = await fetchWithTimeout(iconUrl, options);
-    if (!response.ok)
-        return null;
-    const contentType = contentTypeBase(response.headers.get("content-type"));
-    if (!IMAGE_CONTENT_TYPES.has(contentType))
-        return null;
-    const body = await readLimitedBuffer(response, MAX_ICON_BYTES);
-    if (body.length <= 0)
-        return null;
-    const icon = { contentType, body };
-    if (!options.allowPrivateHosts && !options.fetchImpl) {
-        await writeCachedWebsiteIcon(iconUrl.href, icon).catch(() => undefined);
+    const result = await fetchWithTimeout(iconUrl, options);
+    try {
+        const { response, signal } = result;
+        if (!response.ok)
+            return null;
+        const contentType = contentTypeBase(response.headers.get("content-type"));
+        if (!IMAGE_CONTENT_TYPES.has(contentType))
+            return null;
+        const body = await readLimitedBuffer(response, MAX_ICON_BYTES, { signal });
+        if (body.length <= 0)
+            return null;
+        const icon = { contentType, body };
+        if (!options.allowPrivateHosts && !options.fetchImpl) {
+            await writeCachedWebsiteIcon(iconUrl.href, icon).catch(() => undefined);
+        }
+        return icon;
+    }
+    finally {
+        await result.release();
     }
-    return icon;
 }
 function cachedWebsiteIconBasename(iconUrl) {
     return createHash("sha256").update(iconUrl).digest("hex");
@@ -401,8 +599,13 @@ async function readCachedWebsiteIcon(iconUrl) {
             readFile(metadataPath, "utf8"),
             stat(bodyPath),
         ]);
-        if (Date.now() - bodyStats.mtimeMs > ICON_DISK_CACHE_TTL_MS)
+        if (Date.now() - bodyStats.mtimeMs > ICON_DISK_CACHE_TTL_MS) {
+            await Promise.all([
+                rm(metadataPath, { force: true }),
+                rm(bodyPath, { force: true }),
+            ]);
             return null;
+        }
         const metadata = JSON.parse(rawMetadata);
         if (metadata.url !== iconUrl || typeof metadata.contentType !== "string")
             return null;
@@ -417,13 +620,53 @@ async function readCachedWebsiteIcon(iconUrl) {
         return null;
     }
 }
+async function pruneWebsiteIconDiskCache(dir, protectedBasename) {
+    const filenames = await readdir(dir);
+    const basenames = filenames
+        .filter((filename) => /^[a-f0-9]{64}\.json$/u.test(filename))
+        .map((filename) => filename.slice(0, -".json".length));
+    const entries = await Promise.all(basenames.map(async (basename) => {
+        try {
+            const [metadataStats, bodyStats] = await Promise.all([
+                stat(path.join(dir, `${basename}.json`)),
+                stat(path.join(dir, `${basename}.bin`)),
+            ]);
+            return {
+                basename,
+                mtimeMs: Math.min(metadataStats.mtimeMs, bodyStats.mtimeMs),
+            };
+        }
+        catch {
+            return { basename, mtimeMs: 0 };
+        }
+    }));
+    const now = Date.now();
+    const retained = entries
+        .filter((entry) => now - entry.mtimeMs <= ICON_DISK_CACHE_TTL_MS)
+        .sort((left, right) => {
+        if (left.basename === protectedBasename)
+            return -1;
+        if (right.basename === protectedBasename)
+            return 1;
+        return right.mtimeMs - left.mtimeMs;
+    })
+        .slice(0, MAX_ICON_DISK_CACHE_ENTRIES);
+    const retainedBasenames = new Set(retained.map((entry) => entry.basename));
+    const discarded = entries.filter((entry) => !retainedBasenames.has(entry.basename));
+    await Promise.all(discarded.flatMap((entry) => [
+        rm(path.join(dir, `${entry.basename}.json`), { force: true }),
+        rm(path.join(dir, `${entry.basename}.bin`), { force: true }),
+    ]));
+}
 async function writeCachedWebsiteIcon(iconUrl, icon) {
     const { metadataPath, bodyPath } = cachedWebsiteIconPaths(iconUrl);
-    await mkdir(path.dirname(bodyPath), { recursive: true });
+    const dir = path.dirname(bodyPath);
+    await mkdir(dir, { recursive: true });
     await Promise.all([
         writeFile(bodyPath, icon.body),
         writeFile(metadataPath, `${JSON.stringify({ url: iconUrl, contentType: icon.contentType })}\n`, "utf8"),
     ]);
+    await pruneWebsiteIconDiskCache(dir, path.basename(metadataPath, ".json"));
 }
 export function __clearWebsiteMetadataCacheForTests() {
     metadataCache.clear();