birdclaw 0.5.1 → 0.7.0

package/src/lib/avatar-cache.ts

@@ -1,11 +1,25 @@
 import { createHash } from "node:crypto";
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import path from "node:path";
+import { Effect } from "effect";
 import { getBirdclawPaths } from "./config";
 import { getNativeDb } from "./db";
+import { runEffectPromise, tryPromise } from "./effect-runtime";
+import { assertSafePreviewUrl } from "./url-safety";
 
 const AVATAR_SIZE_SUFFIX =
 	/(?:(?:_normal|_bigger|_mini))(?=\.(?:jpg|jpeg|png|webp|gif)(?:$|\?))/i;
+const MAX_AVATAR_BYTES = 2 * 1024 * 1024;
+const MAX_AVATAR_DATA_URL_CHARS = MAX_AVATAR_BYTES * 4;
+const REMOTE_AVATAR_TIMEOUT_MS = 10_000;
+const DEFAULT_AVATAR_PREFETCH_CONCURRENCY = 8;
+const ALLOWED_REMOTE_AVATAR_HOSTS = new Set(["pbs.twimg.com"]);
+const RASTER_CONTENT_TYPES = new Set([
+	"image/jpeg",
+	"image/png",
+	"image/webp",
+	"image/gif",
+]);
 
 function sanitizeFileToken(value: string) {
 	return value.replace(/[^a-zA-Z0-9_-]+/g, "_");
@@ -23,7 +37,6 @@ function getExtensionFromContentType(contentType: string | null) {
 	if (mime === "image/png") return ".png";
 	if (mime === "image/webp") return ".webp";
 	if (mime === "image/gif") return ".gif";
-	if (mime === "image/svg+xml") return ".svg";
 	return ".jpg";
 }
 
@@ -35,8 +48,6 @@ function getContentTypeFromExtension(extension: string) {
 			return "image/webp";
 		case ".gif":
 			return "image/gif";
-		case ".svg":
-			return "image/svg+xml";
 		default:
 			return "image/jpeg";
 	}
@@ -49,7 +60,7 @@ function getExtensionFromAvatarUrl(avatarUrl: string) {
 		if (extension === ".png" || extension === ".webp" || extension === ".gif") {
 			return extension;
 		}
-		return extension === ".svg" ? ".svg" : ".jpg";
+		return ".jpg";
 	} catch {
 		return ".jpg";
 	}
@@ -59,6 +70,9 @@ function decodeDataUrl(dataUrl: string) {
 	if (!dataUrl.startsWith("data:")) {
 		throw new Error("Invalid avatar data URL");
 	}
+	if (dataUrl.length > MAX_AVATAR_DATA_URL_CHARS) {
+		throw new Error("Avatar data URL is too large");
+	}
 
 	const separatorIndex = dataUrl.indexOf(",");
 	if (separatorIndex < 0) {
@@ -68,12 +82,19 @@ function decodeDataUrl(dataUrl: string) {
 	const metadata = dataUrl.slice(5, separatorIndex);
 	const payload = dataUrl.slice(separatorIndex + 1);
 	const contentType = metadata.split(";")[0] || "application/octet-stream";
+	if (!RASTER_CONTENT_TYPES.has(contentType.toLowerCase())) {
+		throw new Error("Avatar data URL must be a raster image");
+	}
 	const isBase64 = metadata.includes(";base64");
+	const buffer = isBase64
+		? Buffer.from(payload, "base64")
+		: Buffer.from(decodeURIComponent(payload), "utf8");
+	if (buffer.byteLength > MAX_AVATAR_BYTES) {
+		throw new Error("Avatar data URL is too large");
+	}
 	return {
 		contentType,
-		buffer: isBase64
-			? Buffer.from(payload, "base64")
-			: Buffer.from(decodeURIComponent(payload), "utf8"),
+		buffer,
 	};
 }
 
@@ -84,6 +105,70 @@ function getAvatarUrlForProfile(profileId: string) {
 	return row?.avatar_url ?? null;
 }
 
+function toError(error: unknown) {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+function trySync<T>(try_: () => T) {
+	return Effect.try({
+		try: try_,
+		catch: toError,
+	});
+}
+
+function assertSafeRemoteAvatarUrl(avatarUrl: string) {
+	const parsed = assertSafePreviewUrl(avatarUrl);
+	if (parsed.protocol !== "https:") {
+		throw new Error("Remote avatar URL must use https");
+	}
+	if (!ALLOWED_REMOTE_AVATAR_HOSTS.has(parsed.hostname.toLowerCase())) {
+		throw new Error("Remote avatar host is not allowed");
+	}
+	return parsed.toString();
+}
+
+function normalizeContentType(value: string | null) {
+	return value?.split(";")[0]?.trim().toLowerCase() ?? "image/jpeg";
+}
+
+function detectRasterContentType(buffer: Buffer, declared: string) {
+	if (
+		buffer.length >= 3 &&
+		buffer[0] === 0xff &&
+		buffer[1] === 0xd8 &&
+		buffer[2] === 0xff
+	) {
+		return "image/jpeg";
+	}
+	if (
+		buffer.length >= 4 &&
+		buffer[0] === 0x89 &&
+		buffer[1] === 0x50 &&
+		buffer[2] === 0x4e &&
+		buffer[3] === 0x47
+	) {
+		return "image/png";
+	}
+	if (
+		buffer.length >= 12 &&
+		buffer.subarray(0, 4).toString("ascii") === "RIFF" &&
+		buffer.subarray(8, 12).toString("ascii") === "WEBP"
+	) {
+		return "image/webp";
+	}
+	if (
+		buffer.length >= 6 &&
+		(buffer.subarray(0, 6).toString("ascii") === "GIF87a" ||
+			buffer.subarray(0, 6).toString("ascii") === "GIF89a")
+	) {
+		return "image/gif";
+	}
+	if (declared === "image/jpeg") {
+		return "image/jpeg";
+	}
+	throw new Error("Avatar response is not a supported raster image");
+}
+
 export function normalizeAvatarUrl(value: unknown) {
 	if (typeof value !== "string" || value.trim().length === 0) {
 		return null;
@@ -122,61 +207,141 @@ export function getAvatarCachePath(profileId: string, avatarUrl: string) {
 	);
 }
 
-async function fetchRemoteAvatar(avatarUrl: string) {
-	const response = await fetch(avatarUrl, {
-		headers: {
-			"user-agent": "birdclaw/avatar-cache",
-		},
-	});
-	if (!response.ok) {
-		throw new Error(`Avatar fetch failed with ${response.status}`);
-	}
+function fetchRemoteAvatarEffect(avatarUrl: string) {
+	return Effect.gen(function* () {
+		const safeUrl = yield* trySync(() => assertSafeRemoteAvatarUrl(avatarUrl));
+		const response = yield* tryPromise(() =>
+			fetch(safeUrl, {
+				headers: {
+					"user-agent": "birdclaw/avatar-cache",
+				},
+				redirect: "error",
+				signal: AbortSignal.timeout(REMOTE_AVATAR_TIMEOUT_MS),
+			}),
+		);
+		if (!response.ok) {
+			return yield* Effect.fail(
+				new Error(`Avatar fetch failed with ${response.status}`),
+			);
+		}
 
-	const buffer = Buffer.from(await response.arrayBuffer());
-	return {
-		contentType: response.headers.get("content-type") ?? "image/jpeg",
-		buffer,
-	};
+		const buffer = Buffer.from(yield* tryPromise(() => response.arrayBuffer()));
+		if (buffer.byteLength > MAX_AVATAR_BYTES) {
+			return yield* Effect.fail(new Error("Avatar response is too large"));
+		}
+		const contentType = yield* trySync(() =>
+			detectRasterContentType(
+				buffer,
+				normalizeContentType(response.headers.get("content-type")),
+			),
+		);
+		return {
+			contentType,
+			buffer,
+		};
+	});
 }
 
-export async function readCachedAvatar(profileId: string) {
-	const avatarUrl = getAvatarUrlForProfile(profileId);
-	if (!avatarUrl) {
-		return null;
-	}
+export function readCachedAvatarEffect(profileId: string) {
+	return Effect.gen(function* () {
+		const avatarUrl = yield* trySync(() => getAvatarUrlForProfile(profileId));
+		if (!avatarUrl) {
+			return null;
+		}
 
-	const normalizedAvatarUrl = normalizeAvatarUrl(avatarUrl);
-	if (!normalizedAvatarUrl) {
-		return null;
-	}
+		const normalizedAvatarUrl = normalizeAvatarUrl(avatarUrl);
+		if (!normalizedAvatarUrl) {
+			return null;
+		}
 
-	const cachePath = getAvatarCachePath(profileId, normalizedAvatarUrl);
-	const cachedExtension = path.extname(cachePath);
+		const cachePath = yield* trySync(() =>
+			getAvatarCachePath(profileId, normalizedAvatarUrl),
+		);
+		const cachedExtension = path.extname(cachePath);
+
+		const cached = yield* trySync(() => readFileSync(cachePath)).pipe(
+			Effect.map((buffer) => ({ ok: true as const, buffer })),
+			Effect.catchAll(() => Effect.succeed({ ok: false as const })),
+		);
+		if (cached.ok) {
+			return {
+				buffer: cached.buffer,
+				contentType: getContentTypeFromExtension(cachedExtension),
+				cachePath,
+				avatarUrl: normalizedAvatarUrl,
+			};
+		}
 
-	try {
-		return {
-			buffer: readFileSync(cachePath),
-			contentType: getContentTypeFromExtension(cachedExtension),
-			cachePath,
-			avatarUrl: normalizedAvatarUrl,
-		};
-	} catch {
 		const payload = normalizedAvatarUrl.startsWith("data:")
-			? decodeDataUrl(normalizedAvatarUrl)
-			: await fetchRemoteAvatar(normalizedAvatarUrl);
+			? yield* trySync(() => decodeDataUrl(normalizedAvatarUrl))
+			: yield* fetchRemoteAvatarEffect(normalizedAvatarUrl);
 
-		writeFileSync(cachePath, payload.buffer);
+		yield* trySync(() => writeFileSync(cachePath, payload.buffer));
 		return {
 			buffer: payload.buffer,
 			contentType: payload.contentType,
 			cachePath,
 			avatarUrl: normalizedAvatarUrl,
 		};
+	});
+}
+
+export function readCachedAvatar(profileId: string) {
+	return runEffectPromise(readCachedAvatarEffect(profileId));
+}
+
+function normalizePrefetchConcurrency(value: number | undefined) {
+	if (!Number.isFinite(value) || value === undefined) {
+		return DEFAULT_AVATAR_PREFETCH_CONCURRENCY;
 	}
+	return Math.max(1, Math.min(Math.floor(value), 32));
+}
+
+export function prefetchCachedAvatarsForProfileIdsEffect(
+	profileIds: string[],
+	options: { concurrency?: number } = {},
+) {
+	const uniqueProfileIds = Array.from(
+		new Set(
+			profileIds
+				.map((profileId) => profileId.trim())
+				.filter((profileId) => profileId.length > 0),
+		),
+	);
+	const concurrency = normalizePrefetchConcurrency(options.concurrency);
+
+	return Effect.forEach(
+		uniqueProfileIds,
+		(profileId) =>
+			readCachedAvatarEffect(profileId).pipe(
+				Effect.map((avatar) =>
+					avatar ? ("available" as const) : ("missing" as const),
+				),
+				Effect.catchAll(() => Effect.succeed("failed" as const)),
+			),
+		{ concurrency },
+	).pipe(
+		Effect.map((statuses) => ({
+			requested: uniqueProfileIds.length,
+			available: statuses.filter((status) => status === "available").length,
+			missing: statuses.filter((status) => status === "missing").length,
+			failed: statuses.filter((status) => status === "failed").length,
+		})),
+	);
+}
+
+export function prefetchCachedAvatarsForProfileIds(
+	profileIds: string[],
+	options: { concurrency?: number } = {},
+) {
+	return runEffectPromise(
+		prefetchCachedAvatarsForProfileIdsEffect(profileIds, options),
+	);
 }
 
 export const __test__ = {
 	decodeDataUrl,
+	detectRasterContentType,
 	getAvatarCacheDir,
 	getContentTypeFromExtension,
 	getExtensionFromAvatarUrl,