birdclaw 0.5.0 → 0.6.0

package/src/lib/link-preview-metadata.ts

@@ -1,5 +1,18 @@
+import { lookup } from "node:dns/promises";
+import http from "node:http";
+import https from "node:https";
+import net from "node:net";
+import { Readable } from "node:stream";
+import { createBrotliDecompress, createGunzip, createInflate } from "node:zlib";
+import { Effect } from "effect";
 import { getNativeDb } from "./db";
+import { runEffectPromise, tryPromise } from "./effect-runtime";
 import type { Database } from "./sqlite";
+import {
+	__test__ as urlSafetyTest,
+	assertSafePreviewUrl,
+	isBlockedAddress,
+} from "./url-safety";
 import {
 	normalizeUrlExpansionForIndex,
 	upsertUrlExpansion,
@@ -7,6 +20,8 @@ import {
 
 const FETCH_TIMEOUT_MS = 12_000;
 const MAX_HTML_CHARS = 2_000_000;
+const MAX_REDIRECTS = 4;
+const NO_BODY_STATUS_CODES = new Set([204, 205, 304]);
 
 export interface LinkPreviewMetadata {
 	url: string;
@@ -21,7 +36,14 @@ export interface GetLinkPreviewOptions {
 	shortUrl?: string | null;
 	refresh?: boolean;
 	fetchImpl?: typeof fetch;
+	resolveHost?: (hostname: string) => Promise<string[]>;
 	timeoutMs?: number;
+	method?: "GET" | "HEAD";
+}
+
+interface ResolvedAddress {
+	address: string;
+	family: 4 | 6;
 }
 
 type UrlExpansionPreviewRow = {
@@ -44,13 +66,21 @@ function cleanText(value: string | null | undefined) {
 	return cleaned.length > 0 ? cleaned : null;
 }
 
+function decodeCodePoint(value: number, fallback: string) {
+	try {
+		return String.fromCodePoint(value);
+	} catch {
+		return fallback;
+	}
+}
+
 function decodeHtmlEntities(value: string) {
 	return value
-		.replace(/&#(\d+);/g, (_, code: string) =>
-			String.fromCodePoint(Number(code)),
+		.replace(/&#(\d+);/g, (entity: string, code: string) =>
+			decodeCodePoint(Number(code), entity),
 		)
-		.replace(/&#x([a-f0-9]+);/gi, (_, code: string) =>
-			String.fromCodePoint(Number.parseInt(code, 16)),
+		.replace(/&#x([a-f0-9]+);/gi, (entity: string, code: string) =>
+			decodeCodePoint(Number.parseInt(code, 16), entity),
 		)
 		.replace(/&amp;/g, "&")
 		.replace(/&lt;/g, "<")
@@ -123,6 +153,367 @@ function hostLabel(url: string) {
 	}
 }
 
+function isInjectedFetchAllowed() {
+	return process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+}
+
+function stripAddressBrackets(value: string) {
+	return value.replace(/^\[|\]$/g, "");
+}
+
+function withTimeout<T>(promise: Promise<T>, ms: number) {
+	let timer: NodeJS.Timeout | null = null;
+	const timeout = new Promise<never>((_resolve, reject) => {
+		timer = setTimeout(
+			() => reject(new Error("Link preview request timed out")),
+			ms,
+		);
+	});
+	return Promise.race([promise, timeout]).finally(() => {
+		if (timer) clearTimeout(timer);
+	});
+}
+
+export async function resolvePublicAddresses(
+	hostname: string,
+): Promise<string[]> {
+	const normalized = stripAddressBrackets(hostname);
+	if (net.isIP(normalized)) return [normalized];
+	const records = await lookup(normalized, { all: true, verbatim: true });
+	return records.map((record) => record.address);
+}
+
+function validateResolvedAddresses(addresses: string[]) {
+	if (addresses.length === 0) {
+		throw new Error("Link preview host did not resolve");
+	}
+	if (addresses.some(isBlockedAddress)) {
+		throw new Error("Link preview URL resolves to a private address");
+	}
+}
+
+async function resolveSafeAddresses(
+	hostname: string,
+	resolveHost: (hostname: string) => Promise<string[]>,
+	timeoutMs: number,
+): Promise<ResolvedAddress[]> {
+	const normalizedHostname = stripAddressBrackets(hostname);
+	const addresses = await withTimeout(
+		resolveHost(normalizedHostname),
+		timeoutMs,
+	);
+	validateResolvedAddresses(addresses);
+	return addresses.map((address) => {
+		const normalized = stripAddressBrackets(address);
+		const family = net.isIP(normalized);
+		if (family !== 4 && family !== 6) {
+			throw new Error("Link preview host resolved to an invalid address");
+		}
+		return { address: normalized, family };
+	});
+}
+
+function headersFromIncoming(headers: http.IncomingHttpHeaders): HeadersInit {
+	const result = new Headers();
+	for (const [key, value] of Object.entries(headers)) {
+		if (Array.isArray(value)) {
+			for (const item of value) result.append(key, item);
+		} else if (typeof value === "string") {
+			result.set(key, value);
+		}
+	}
+	return result;
+}
+
+function decodedResponseBody(response: Response) {
+	const body = response.body;
+	if (!body) return null;
+	const encoding = response.headers
+		.get("content-encoding")
+		?.split(",")
+		.at(-1)
+		?.trim()
+		.toLowerCase();
+	if (!encoding || encoding === "identity") return body;
+	const nodeBody = Readable.fromWeb(
+		body as Parameters<typeof Readable.fromWeb>[0],
+	);
+	if (encoding === "gzip" || encoding === "x-gzip") {
+		return Readable.toWeb(nodeBody.pipe(createGunzip())) as ReadableStream;
+	}
+	if (encoding === "br") {
+		return Readable.toWeb(
+			nodeBody.pipe(createBrotliDecompress()),
+		) as ReadableStream;
+	}
+	if (encoding === "deflate") {
+		return Readable.toWeb(nodeBody.pipe(createInflate())) as ReadableStream;
+	}
+	return body;
+}
+
+function cancelResponseBodyEffect(response: Response) {
+	return tryPromise(() => response.body?.cancel() ?? Promise.resolve()).pipe(
+		Effect.catchAll(() => Effect.void),
+	);
+}
+
+function respondWithResolvedAddress(
+	selected: ResolvedAddress,
+	lookupOptions: { all?: boolean } | number | undefined,
+	callback: (
+		error: Error | null,
+		address: string | ResolvedAddress[],
+		family?: 4 | 6,
+	) => void,
+) {
+	if (typeof lookupOptions === "object" && lookupOptions?.all) {
+		callback(null, [selected]);
+		return;
+	}
+	callback(null, selected.address, selected.family);
+}
+
+function nodeSafeFetch(
+	url: URL,
+	options: {
+		addresses: ResolvedAddress[];
+		headers: Record<string, string>;
+		method: "GET" | "HEAD";
+		timeoutMs: number;
+	},
+) {
+	const deadline = Date.now() + options.timeoutMs;
+
+	function fetchAddress(address: ResolvedAddress, attemptTimeoutMs: number) {
+		return new Promise<Response>((resolve, reject) => {
+			let settled = false;
+			const finish = (callback: () => void) => {
+				if (settled) return;
+				settled = true;
+				try {
+					callback();
+				} catch (error) {
+					reject(error);
+				}
+			};
+
+			const client = url.protocol === "https:" ? https : http;
+			let wallClockTimeout: NodeJS.Timeout | null = null;
+			const clearWallClockTimeout = () => {
+				if (!wallClockTimeout) return;
+				clearTimeout(wallClockTimeout);
+				wallClockTimeout = null;
+			};
+			const setWallClockTimeout = (
+				ms: number,
+				target: { destroy: (error: Error) => void },
+			) => {
+				clearWallClockTimeout();
+				wallClockTimeout = setTimeout(() => {
+					target.destroy(new Error("Link preview request timed out"));
+				}, ms);
+			};
+			const request = client.request(
+				url,
+				{
+					headers: options.headers,
+					method: options.method,
+					lookup: (_hostname, lookupOptions, callback) => {
+						respondWithResolvedAddress(address, lookupOptions, callback);
+					},
+				},
+				(incoming) => {
+					request.setTimeout(0);
+					setWallClockTimeout(Math.max(1, deadline - Date.now()), incoming);
+					incoming.once("end", clearWallClockTimeout);
+					incoming.once("close", clearWallClockTimeout);
+					incoming.once("error", clearWallClockTimeout);
+					finish(() => {
+						const status = incoming.statusCode ?? 200;
+						const response = new Response(
+							NO_BODY_STATUS_CODES.has(status)
+								? null
+								: (Readable.toWeb(incoming) as ReadableStream),
+							{
+								headers: headersFromIncoming(incoming.headers),
+								status,
+								statusText: incoming.statusMessage,
+							},
+						);
+						Object.defineProperty(response, "url", {
+							value: url.toString(),
+						});
+						resolve(response);
+					});
+				},
+			);
+			setWallClockTimeout(attemptTimeoutMs, request);
+			request.setTimeout(attemptTimeoutMs, () => {
+				request.destroy(new Error("Link preview request timed out"));
+			});
+			request.on("error", (error) => {
+				clearWallClockTimeout();
+				finish(() => reject(error));
+			});
+			request.end();
+		});
+	}
+
+	return options.addresses.reduce<Promise<Response>>(
+		(previous, address, index) =>
+			previous.catch((error: unknown) => {
+				const remainingMs = deadline - Date.now();
+				if (remainingMs <= 0) throw error;
+				const remainingAddresses = options.addresses.length - index;
+				const attemptTimeoutMs = Math.max(
+					1,
+					Math.ceil(remainingMs / remainingAddresses),
+				);
+				return fetchAddress(address, attemptTimeoutMs);
+			}),
+		Promise.reject(new Error("Link preview host did not resolve")),
+	);
+}
+
+export function safePreviewFetchEffect(
+	url: string,
+	options: Pick<
+		GetLinkPreviewOptions,
+		"fetchImpl" | "method" | "resolveHost" | "timeoutMs"
+	>,
+) {
+	const resolveHost =
+		options.resolveHost ??
+		(options.fetchImpl
+			? null
+			: (hostname: string) => resolvePublicAddresses(hostname));
+	const timeoutMs = options.timeoutMs ?? FETCH_TIMEOUT_MS;
+	const method = options.method ?? "GET";
+	const deadline = Date.now() + timeoutMs;
+	const remainingTimeoutMs = () => Math.max(1, deadline - Date.now());
+	const headers: Record<string, string> = {
+		"user-agent":
+			"Mozilla/5.0 (Macintosh; Intel Mac OS X 15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36 birdclaw/0.4",
+		accept:
+			"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+		"accept-language": "en-US,en;q=0.9",
+	};
+
+	return Effect.gen(function* () {
+		if (options.fetchImpl && !isInjectedFetchAllowed()) {
+			return yield* Effect.fail(
+				new Error("Custom link preview fetch is only available in tests"),
+			);
+		}
+
+		let currentUrl = url;
+		for (let redirect = 0; redirect <= MAX_REDIRECTS; redirect += 1) {
+			if (Date.now() >= deadline) {
+				return yield* Effect.fail(new Error("Link preview request timed out"));
+			}
+			let remainingMs = remainingTimeoutMs();
+			const parsed = yield* Effect.try({
+				try: () => assertSafePreviewUrl(currentUrl),
+				catch: (error) => error,
+			});
+			if (options.fetchImpl && resolveHost) {
+				yield* tryPromise(() =>
+					resolveSafeAddresses(parsed.hostname, resolveHost, remainingMs),
+				);
+			}
+			if (Date.now() >= deadline) {
+				return yield* Effect.fail(new Error("Link preview request timed out"));
+			}
+			remainingMs = remainingTimeoutMs();
+
+			const response = options.fetchImpl
+				? yield* tryPromise(
+						() =>
+							options.fetchImpl?.(parsed.toString(), {
+								headers,
+								method,
+								redirect: "manual",
+								signal: AbortSignal.timeout(remainingMs),
+							}) as Promise<Response>,
+					)
+				: yield* tryPromise(() =>
+						resolveSafeAddresses(
+							parsed.hostname,
+							options.resolveHost ?? resolvePublicAddresses,
+							remainingMs,
+						).then((addresses) =>
+							Date.now() >= deadline
+								? Promise.reject(new Error("Link preview request timed out"))
+								: nodeSafeFetch(parsed, {
+										addresses,
+										headers,
+										method,
+										timeoutMs: remainingTimeoutMs(),
+									}),
+						),
+					);
+			if (response.status < 300 || response.status >= 400) return response;
+
+			const location = response.headers.get("location");
+			if (!location) return response;
+			yield* cancelResponseBodyEffect(response);
+			if (redirect === MAX_REDIRECTS) {
+				return yield* Effect.fail(
+					new Error("Link preview redirected too many times"),
+				);
+			}
+			const nextUrl = yield* Effect.try({
+				try: () => new URL(location, parsed).toString(),
+				catch: (error) => error,
+			});
+			currentUrl = nextUrl;
+		}
+		return yield* Effect.fail(
+			new Error("Link preview redirected too many times"),
+		);
+	});
+}
+
+function readResponseTextEffect(response: Response) {
+	const contentLength = Number(response.headers.get("content-length") ?? 0);
+	if (Number.isFinite(contentLength) && contentLength > MAX_HTML_CHARS) {
+		return tryPromise(() => response.body?.cancel() ?? Promise.resolve()).pipe(
+			Effect.flatMap(() =>
+				Effect.fail(new Error("Link preview response is too large")),
+			),
+		);
+	}
+	const reader = decodedResponseBody(response)?.getReader();
+	if (!reader) return tryPromise(() => response.text());
+	const decoder = new TextDecoder();
+	let total = 0;
+	let text = "";
+	return Effect.gen(function* () {
+		for (;;) {
+			const { done, value } = yield* tryPromise(() => reader.read());
+			if (done) break;
+			total += value.byteLength;
+			if (total > MAX_HTML_CHARS) {
+				yield* tryPromise(() =>
+					reader.cancel(new Error("Link preview response is too large")),
+				);
+				return yield* Effect.fail(
+					new Error("Link preview response is too large"),
+				);
+			}
+			text += decoder.decode(value, { stream: true });
+		}
+		return text + decoder.decode();
+	}).pipe(
+		Effect.ensuring(
+			Effect.sync(() => {
+				reader.releaseLock();
+			}),
+		),
+	);
+}
+
 function youtubeThumbnail(url: string) {
 	try {
 		const parsed = new URL(url);
@@ -183,52 +574,59 @@ export function extractLinkPreviewMetadata(
 	};
 }
 
-export async function fetchLinkPreviewMetadata(
+export function fetchLinkPreviewMetadataEffect(
 	url: string,
-	options: Pick<GetLinkPreviewOptions, "fetchImpl" | "timeoutMs"> = {},
-): Promise<LinkPreviewMetadata> {
-	const fetchImpl = options.fetchImpl ?? globalThis.fetch;
-	const timeoutMs = options.timeoutMs ?? FETCH_TIMEOUT_MS;
-	const headers = {
-		"user-agent":
-			"Mozilla/5.0 (Macintosh; Intel Mac OS X 15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36 birdclaw/0.4",
-		accept:
-			"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
-		"accept-language": "en-US,en;q=0.9",
-	} satisfies HeadersInit;
-
-	try {
-		const response = await fetchImpl(url, {
-			headers,
-			redirect: "follow",
-			signal: AbortSignal.timeout(timeoutMs),
-		});
+	options: Pick<
+		GetLinkPreviewOptions,
+		"fetchImpl" | "resolveHost" | "timeoutMs"
+	> = {},
+): Effect.Effect<LinkPreviewMetadata> {
+	return Effect.gen(function* () {
+		const response = yield* safePreviewFetchEffect(url, options);
 		const finalUrl = response.url || url;
 		const contentType = response.headers.get("content-type") ?? "";
 		if (!response.ok) {
-			throw new Error(`HTTP ${response.status}`);
+			yield* cancelResponseBodyEffect(response);
+			return yield* Effect.fail(new Error(`HTTP ${response.status}`));
 		}
 		if (contentType.toLowerCase().startsWith("image/")) {
+			yield* cancelResponseBodyEffect(response);
 			return {
 				url: finalUrl,
 				title: hostLabel(finalUrl),
 				description: null,
 				imageUrl: finalUrl,
 				siteName: hostLabel(finalUrl),
-			};
+			} satisfies LinkPreviewMetadata;
 		}
-		const html = (await response.text()).slice(0, MAX_HTML_CHARS);
-		return extractLinkPreviewMetadata(html, finalUrl);
-	} catch (error) {
-		return {
-			url,
-			title: hostLabel(url),
-			description: null,
-			imageUrl: youtubeThumbnail(url),
-			siteName: hostLabel(url),
-			error: error instanceof Error ? error.message : String(error),
-		};
-	}
+		const content = yield* readResponseTextEffect(response);
+		return yield* Effect.try({
+			try: () =>
+				extractLinkPreviewMetadata(content.slice(0, MAX_HTML_CHARS), finalUrl),
+			catch: (error) => error,
+		});
+	}).pipe(
+		Effect.catchAll((error) =>
+			Effect.succeed({
+				url,
+				title: hostLabel(url),
+				description: null,
+				imageUrl: youtubeThumbnail(url),
+				siteName: hostLabel(url),
+				error: error instanceof Error ? error.message : String(error),
+			}),
+		),
+	);
+}
+
+export function fetchLinkPreviewMetadata(
+	url: string,
+	options: Pick<
+		GetLinkPreviewOptions,
+		"fetchImpl" | "resolveHost" | "timeoutMs"
+	> = {},
+): Promise<LinkPreviewMetadata> {
+	return runEffectPromise(fetchLinkPreviewMetadataEffect(url, options));
 }
 
 function readCachedPreview(
@@ -267,18 +665,29 @@ function readCachedPreview(
 }
 
 function hasUsefulPreview(row: UrlExpansionPreviewRow) {
+	if (row.status !== "hit") return false;
+	try {
+		assertSafePreviewUrl(row.final_url || row.expanded_url || row.short_url);
+		if (row.image_url) assertSafePreviewUrl(row.image_url);
+	} catch {
+		return false;
+	}
 	return Boolean(
 		row.title || row.description || row.image_url || row.site_name,
 	);
 }
 
 function rowToPreview(row: UrlExpansionPreviewRow): LinkPreviewMetadata {
-	const url = row.final_url || row.expanded_url || row.short_url;
+	const url = assertSafePreviewUrl(
+		row.final_url || row.expanded_url || row.short_url,
+	).toString();
 	return {
 		url,
 		title: row.title ?? null,
 		description: row.description ?? null,
-		imageUrl: row.image_url ?? null,
+		imageUrl: row.image_url
+			? assertSafePreviewUrl(row.image_url).toString()
+			: null,
 		siteName: row.site_name ?? null,
 		...(row.error ? { error: row.error } : {}),
 	};
@@ -310,25 +719,36 @@ function persistPreview(
 	);
 }
 
-export async function getOrFetchLinkPreview(
+export function getOrFetchLinkPreviewEffect(
 	url: string,
 	options: GetLinkPreviewOptions = {},
-): Promise<LinkPreviewMetadata> {
-	const db = getNativeDb({ seedDemoData: false });
-	const cached = readCachedPreview(db, url, options.shortUrl);
-	if (cached && hasUsefulPreview(cached) && !options.refresh) {
-		return rowToPreview(cached);
-	}
+): Effect.Effect<LinkPreviewMetadata> {
+	return Effect.gen(function* () {
+		const db = getNativeDb({ seedDemoData: false });
+		const cached = readCachedPreview(db, url, options.shortUrl);
+		if (cached && hasUsefulPreview(cached) && !options.refresh) {
+			return rowToPreview(cached);
+		}
 
-	const preview = await fetchLinkPreviewMetadata(
-		cached?.final_url || cached?.expanded_url || url,
-		options,
-	);
-	persistPreview(db, url, options.shortUrl, cached, preview);
-	return preview;
+		const preview = yield* fetchLinkPreviewMetadataEffect(
+			cached?.final_url || cached?.expanded_url || url,
+			options,
+		);
+		persistPreview(db, url, options.shortUrl, cached, preview);
+		return preview;
+	});
+}
+
+export function getOrFetchLinkPreview(
+	url: string,
+	options: GetLinkPreviewOptions = {},
+): Promise<LinkPreviewMetadata> {
+	return runEffectPromise(getOrFetchLinkPreviewEffect(url, options));
 }
 
 export const __test__ = {
+	...urlSafetyTest,
 	decodeHtmlEntities,
+	respondWithResolvedAddress,
 	youtubeThumbnail,
 };