birdclaw 0.5.1 → 0.6.0

package/src/lib/http-effect.ts

@@ -0,0 +1,222 @@
+import { Effect } from "effect";
+import { runEffectPromise, tryPromise } from "./effect-runtime";
+
+const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]);
+
+export function jsonResponse(data: unknown, init?: ResponseInit) {
+	const headers = new Headers(init?.headers);
+	headers.set("content-type", "application/json");
+	return new Response(JSON.stringify(data), {
+		...init,
+		headers,
+	});
+}
+
+export function requestJsonEffect<T = Record<string, unknown>>(
+	request: Request,
+	fallback?: T,
+): Effect.Effect<T, unknown> {
+	return tryPromise(() => request.json() as Promise<T>).pipe(
+		Effect.catchAll((error) =>
+			fallback === undefined ? Effect.fail(error) : Effect.succeed(fallback),
+		),
+	);
+}
+
+export function runRouteEffect<A, E>(effect: Effect.Effect<A, E>): Promise<A> {
+	return runEffectPromise(effect);
+}
+
+function normalizedHost(value: string) {
+	return value.toLowerCase().replace(/^\[|\]$/g, "");
+}
+
+function isTestEnvironment() {
+	return process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+}
+
+function requestCookie(request: Request, name: string) {
+	const cookie = request.headers.get("cookie");
+	if (!cookie) return { found: false, value: null };
+	for (const part of cookie.split(";")) {
+		const [key, ...rest] = part.trim().split("=");
+		if (key === name) {
+			try {
+				return { found: true, value: decodeURIComponent(rest.join("=")) };
+			} catch {
+				return { found: true, value: null };
+			}
+		}
+	}
+	return { found: false, value: null };
+}
+
+function configuredWebToken() {
+	const token = process.env.BIRDCLAW_WEB_TOKEN?.trim();
+	return token || null;
+}
+
+function requestWebTokenStatus(request: Request) {
+	const token = configuredWebToken();
+	if (!token)
+		return {
+			configured: false,
+			valid: false,
+			fromCookie: false,
+			provided: false,
+		};
+	const headerToken = request.headers.get("x-birdclaw-token");
+	const cookieToken = requestCookie(request, "birdclaw_token");
+	const provided = headerToken !== null || cookieToken.found;
+	const cookieValue = cookieToken.value;
+	const valid = headerToken === token || cookieValue === token;
+	return {
+		configured: true,
+		valid,
+		fromCookie: cookieValue === token && headerToken !== token,
+		provided,
+	};
+}
+
+function isLocalWebHost(value: string) {
+	const host = normalizedHost(value);
+	return LOCAL_HOSTS.has(host) || host.endsWith(".localhost");
+}
+
+function allowsUnauthenticatedLocalWeb() {
+	return process.env.BIRDCLAW_LOCAL_WEB === "1";
+}
+
+function hasForwardedRequestHeaders(request: Request) {
+	return (
+		request.headers.has("forwarded") ||
+		request.headers.has("x-forwarded-for") ||
+		request.headers.has("x-forwarded-proto") ||
+		request.headers.has("x-forwarded-host") ||
+		request.headers.has("x-real-ip")
+	);
+}
+
+function firstForwardedValue(value: string | null) {
+	return value?.split(",")[0]?.trim() || null;
+}
+
+function forwardedHeaderPair(request: Request, key: "host" | "proto") {
+	const forwarded = firstForwardedValue(request.headers.get("forwarded"));
+	if (!forwarded) return null;
+	for (const part of forwarded.split(";")) {
+		const [name, ...rest] = part.trim().split("=");
+		if (name?.toLowerCase() !== key) continue;
+		const value = rest.join("=").trim().replace(/^"|"$/g, "");
+		return value || null;
+	}
+	return null;
+}
+
+function forwardedOrigin(request: Request) {
+	const proto =
+		firstForwardedValue(request.headers.get("x-forwarded-proto")) ??
+		forwardedHeaderPair(request, "proto");
+	const host =
+		firstForwardedValue(request.headers.get("x-forwarded-host")) ??
+		forwardedHeaderPair(request, "host");
+	if (!proto || !host) return null;
+	const normalizedProto = proto.toLowerCase();
+	if (normalizedProto !== "http" && normalizedProto !== "https") return null;
+	return `${normalizedProto}://${host}`;
+}
+
+function sameRequestOrigin(request: Request, origin: string, url: URL) {
+	return origin === url.origin || origin === forwardedOrigin(request);
+}
+
+export function sensitiveRequestErrorResponse(request: Request) {
+	const url = new URL(request.url);
+	const token = requestWebTokenStatus(request);
+	const isLocalRequest =
+		allowsUnauthenticatedLocalWeb() &&
+		isLocalWebHost(url.hostname) &&
+		!hasForwardedRequestHeaders(request);
+	const fetchSite = request.headers.get("sec-fetch-site");
+	const origin = request.headers.get("origin");
+	const allowRemoteEnv = process.env.BIRDCLAW_ALLOW_REMOTE_WEB === "1";
+	const allowTrustedRemote = allowRemoteEnv && !token.configured;
+
+	if (isTestEnvironment() && !token.valid) return null;
+
+	if (origin && !sameRequestOrigin(request, origin, url)) {
+		return jsonResponse(
+			{ ok: false, message: "Cross-origin web API access is disabled" },
+			{ status: 403 },
+		);
+	}
+
+	if (fetchSite === "cross-site") {
+		return jsonResponse(
+			{ ok: false, message: "Cross-site web API access is disabled" },
+			{ status: 403 },
+		);
+	}
+
+	if (
+		!isTestEnvironment() &&
+		!token.configured &&
+		!isLocalRequest &&
+		!allowTrustedRemote
+	) {
+		return jsonResponse(
+			{
+				ok: false,
+				message:
+					"Remote API access requires BIRDCLAW_ALLOW_REMOTE_WEB=1 for a trusted private proxy, or BIRDCLAW_WEB_TOKEN for tokened access",
+			},
+			{ status: 403 },
+		);
+	}
+
+	if (token.configured && !token.valid && (token.provided || !isLocalRequest)) {
+		return jsonResponse(
+			{ ok: false, message: "Invalid web token" },
+			{ status: 403 },
+		);
+	}
+
+	const allowRemote = allowRemoteEnv && (token.valid || allowTrustedRemote);
+	if (!allowRemote && !isLocalRequest) {
+		return jsonResponse(
+			{ ok: false, message: "Remote web API access is disabled" },
+			{ status: 403 },
+		);
+	}
+
+	if (
+		token.fromCookie &&
+		fetchSite !== "same-origin" &&
+		fetchSite !== "same-site"
+	) {
+		return jsonResponse(
+			{ ok: false, message: "Same-origin browser request required" },
+			{ status: 403 },
+		);
+	}
+
+	return null;
+}
+
+export function parseBoundedInteger(
+	value: number | string | null | undefined,
+	{
+		defaultValue,
+		min = 1,
+		max,
+	}: { defaultValue?: number; min?: number; max: number },
+) {
+	if (value === null || value === undefined || value === "")
+		return defaultValue;
+	if (typeof value === "string" && !/^\d+$/.test(value.trim())) {
+		return defaultValue;
+	}
+	const parsed = typeof value === "number" ? value : Number(value);
+	if (!Number.isSafeInteger(parsed)) return defaultValue;
+	return Math.min(max, Math.max(min, parsed));
+}

package/src/lib/inbox.ts

@@ -1,5 +1,7 @@
+import { Effect } from "effect";
 import { getNativeDb } from "./db";
-import { scoreInboxItemWithOpenAI } from "./openai";
+import { runEffectPromise } from "./effect-runtime";
+import { scoreInboxItemWithOpenAIEffect } from "./openai";
 import { listDmConversations, listTimelineItems } from "./queries";
 import type { InboxItem, InboxQuery, InboxResponse } from "./types";
 
@@ -61,6 +63,7 @@ function readStoredScores() {
 
 export function listInboxItems({
 	kind = "mixed",
+	account,
 	minScore = 0,
 	hideLowSignal = false,
 	limit = 20,
@@ -71,6 +74,7 @@ export function listInboxItems({
 	if (kind === "mixed" || kind === "mentions") {
 		for (const mention of listTimelineItems({
 			resource: "mentions",
+			account,
 			replyFilter: "unreplied",
 			limit: 50,
 		})) {
@@ -102,8 +106,9 @@ export function listInboxItems({
 
 	if (kind === "mixed" || kind === "dms") {
 		for (const dm of listDmConversations({
+			account,
 			replyFilter: "unreplied",
-			sort: "influence",
+			sort: "followers",
 			limit: 50,
 		})) {
 			const scoreKey = `dm:${dm.id}`;
@@ -151,30 +156,47 @@ export function listInboxItems({
 	};
 }
 
-export async function scoreInbox({
+function toError(error: unknown) {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+function trySync<T>(try_: () => T) {
+	return Effect.try({
+		try: try_,
+		catch: toError,
+	});
+}
+
+export function scoreInboxEffect({
 	kind = "mixed",
+	account,
 	limit = 8,
-}: Pick<InboxQuery, "kind" | "limit"> = {}) {
-	const db = getNativeDb();
-	const items = listInboxItems({ kind, limit }).items;
-	const results = [];
-
-	for (const item of items) {
-		const scored = await scoreInboxItemWithOpenAI({
-			entityKind: item.entityKind,
-			title: item.title,
-			text: item.text,
-			influenceScore: item.influenceScore,
-			participant: {
-				handle: item.participant.handle,
-				displayName: item.participant.displayName,
-				bio: item.participant.bio,
-				followersCount: item.participant.followersCount,
-			},
-		});
+}: Pick<InboxQuery, "kind" | "account" | "limit"> = {}) {
+	return Effect.gen(function* () {
+		const db = yield* trySync(() => getNativeDb());
+		const items = yield* trySync(
+			() => listInboxItems({ kind, account, limit }).items,
+		);
+		const results = [];
 
-		db.prepare(
-			`
+		for (const item of items) {
+			const scored = yield* scoreInboxItemWithOpenAIEffect({
+				entityKind: item.entityKind,
+				title: item.title,
+				text: item.text,
+				influenceScore: item.influenceScore,
+				participant: {
+					handle: item.participant.handle,
+					displayName: item.participant.displayName,
+					bio: item.participant.bio,
+					followersCount: item.participant.followersCount,
+				},
+			});
+
+			yield* trySync(() =>
+				db
+					.prepare(
+						`
       insert into ai_scores (
         entity_kind, entity_id, model, score, summary, reasoning, updated_at
       ) values (?, ?, ?, ?, ?, ?, ?)
@@ -185,26 +207,35 @@ export async function scoreInbox({
         reasoning = excluded.reasoning,
         updated_at = excluded.updated_at
       `,
-		).run(
-			item.entityKind,
-			item.entityId,
-			scored.model,
-			scored.score,
-			scored.summary,
-			scored.reasoning,
-			new Date().toISOString(),
-		);
+					)
+					.run(
+						item.entityKind,
+						item.entityId,
+						scored.model,
+						scored.score,
+						scored.summary,
+						scored.reasoning,
+						new Date().toISOString(),
+					),
+			);
 
-		results.push({
-			id: item.id,
-			score: scored.score,
-			source: "openai",
-		});
-	}
+			results.push({
+				id: item.id,
+				score: scored.score,
+				source: "openai",
+			});
+		}
 
-	return {
-		ok: true,
-		scored: results.length,
-		items: results,
-	};
+		return {
+			ok: true,
+			scored: results.length,
+			items: results,
+		};
+	});
+}
+
+export function scoreInbox(
+	options: Pick<InboxQuery, "kind" | "account" | "limit"> = {},
+) {
+	return runEffectPromise(scoreInboxEffect(options));
 }

package/src/lib/link-index.ts

@@ -1,4 +1,6 @@
+import { Effect } from "effect";
 import { getNativeDb } from "./db";
+import { runEffectPromise, tryPromise } from "./effect-runtime";
 import type { Database } from "./sqlite";
 import {
 	normalizeUrlExpansionForIndex,
@@ -246,11 +248,19 @@ function rebuildOccurrences(
 	return { occurrences, entityExpansions };
 }
 
-async function expandWithConcurrency(
+function expandWithConcurrencyEffect(
 	db: Database,
 	urls: string[],
 	options: LinkBackfillOptions,
-) {
+): Effect.Effect<
+	{
+		networkExpansions: number;
+		cacheExpansions: number;
+		misses: number;
+		errors: number;
+	},
+	unknown
+> {
 	const concurrency = Math.max(
 		1,
 		Math.min(options.concurrency ?? DEFAULT_EXPAND_CONCURRENCY, 64),
@@ -261,62 +271,57 @@ async function expandWithConcurrency(
 		misses: 0,
 		errors: 0,
 	};
-	let nextIndex = 0;
 
-	async function worker() {
-		for (;;) {
-			const index = nextIndex++;
-			const url = urls[index];
-			if (!url) {
-				return;
-			}
-			const result = (
-				await expandUrls([url], {
+	return Effect.forEach(
+		urls,
+		(url) =>
+			tryPromise(() =>
+				expandUrls([url], {
 					refresh: options.refresh,
 					fetchImpl: options.fetchImpl,
 					timeoutMs: options.timeoutMs,
-				})
-			)[0]!;
-			if (result.source === "network") {
-				counts.networkExpansions++;
-			} else {
-				counts.cacheExpansions++;
-			}
-			if (result.status === "miss") {
-				counts.misses++;
-			}
-			if (result.status === "error") {
-				counts.errors++;
-			}
-			upsertUrlExpansion(db, normalizeUrlExpansionForIndex(result));
-		}
-	}
-
-	await Promise.all(
-		Array.from({ length: Math.min(concurrency, urls.length) }, () => worker()),
-	);
-	return counts;
+				}),
+			).pipe(
+				Effect.map((results) => {
+					const result = results[0]!;
+					if (result.source === "network") {
+						counts.networkExpansions++;
+					} else {
+						counts.cacheExpansions++;
+					}
+					if (result.status === "miss") {
+						counts.misses++;
+					}
+					if (result.status === "error") {
+						counts.errors++;
+					}
+					upsertUrlExpansion(db, normalizeUrlExpansionForIndex(result));
+				}),
+			),
+		{ concurrency },
+	).pipe(Effect.as(counts));
 }
 
-export async function backfillLinkIndex(
+export function backfillLinkIndexEffect(
 	options: LinkBackfillOptions = {},
-): Promise<LinkBackfillResult> {
-	const db = getNativeDb({ seedDemoData: false });
-	const { occurrences, entityExpansions } = rebuildOccurrences(
-		db,
-		Boolean(options.includeAllUrls),
-		options.source,
-	);
-	const limit =
-		typeof options.limit === "number" && Number.isFinite(options.limit)
-			? Math.max(0, Math.trunc(options.limit))
-			: undefined;
-	const needsExpansionClause = options.refresh
-		? "1 = 1"
-		: "(e.short_url is null or e.status in ('error', 'miss'))";
-
-	const urlsToExpand = db
-		.prepare(`
+): Effect.Effect<LinkBackfillResult, unknown> {
+	return Effect.gen(function* () {
+		const db = getNativeDb({ seedDemoData: false });
+		const { occurrences, entityExpansions } = rebuildOccurrences(
+			db,
+			Boolean(options.includeAllUrls),
+			options.source,
+		);
+		const limit =
+			typeof options.limit === "number" && Number.isFinite(options.limit)
+				? Math.max(0, Math.trunc(options.limit))
+				: undefined;
+		const needsExpansionClause = options.refresh
+			? "1 = 1"
+			: "(e.short_url is null or e.status in ('error', 'miss'))";
+
+		const urlsToExpand = db
+			.prepare(`
     select distinct o.short_url
     from link_occurrences o
     left join url_expansions e on e.short_url = o.short_url
@@ -325,43 +330,50 @@ export async function backfillLinkIndex(
     order by o.short_url
     ${limit === undefined ? "" : "limit ?"}
   `)
-		.all(
-			...(options.source ? [options.source] : []),
-			...(limit === undefined ? [] : [limit]),
-		) as Array<{
-		short_url: string;
-	}>;
-
-	const expansionCounts = await expandWithConcurrency(
-		db,
-		urlsToExpand.map((row) => row.short_url),
-		options,
-	);
+			.all(
+				...(options.source ? [options.source] : []),
+				...(limit === undefined ? [] : [limit]),
+			) as Array<{
+			short_url: string;
+		}>;
+
+		const expansionCounts = yield* expandWithConcurrencyEffect(
+			db,
+			urlsToExpand.map((row) => row.short_url),
+			options,
+		);
 
-	const uniqueUrls = db
-		.prepare(`
+		const uniqueUrls = db
+			.prepare(`
     select count(distinct short_url) as count
     from link_occurrences
     ${options.source ? "where source_kind = ?" : ""}
   `)
-		.get(...(options.source ? [options.source] : [])) as { count: number };
-	const remaining = db
-		.prepare(`
+			.get(...(options.source ? [options.source] : [])) as { count: number };
+		const remaining = db
+			.prepare(`
     select count(distinct o.short_url) as count
     from link_occurrences o
     left join url_expansions e on e.short_url = o.short_url
     where (e.short_url is null or e.status in ('error', 'miss'))
       ${options.source ? "and o.source_kind = ?" : ""}
   `)
-		.get(...(options.source ? [options.source] : [])) as { count: number };
+			.get(...(options.source ? [options.source] : [])) as { count: number };
+
+		return {
+			occurrences,
+			uniqueUrls: Number(uniqueUrls.count),
+			entityExpansions,
+			...expansionCounts,
+			remainingUnexpanded: Number(remaining.count),
+		};
+	});
+}
 
-	return {
-		occurrences,
-		uniqueUrls: Number(uniqueUrls.count),
-		entityExpansions,
-		...expansionCounts,
-		remainingUnexpanded: Number(remaining.count),
-	};
+export function backfillLinkIndex(
+	options: LinkBackfillOptions = {},
+): Promise<LinkBackfillResult> {
+	return runEffectPromise(backfillLinkIndexEffect(options));
 }
 
 function likePattern(value: string) {

package/src/lib/link-insights.ts

@@ -635,6 +635,30 @@ export function getLinkInsights(
 		conditions.push("o.source_kind = ?");
 		params.push(source);
 	}
+	if (query.account && query.account !== "all") {
+		conditions.push(`(
+	    (o.source_kind = 'dm' and o.account_id = ?)
+	    or (
+	      o.source_kind = 'tweet'
+	      and (
+	        o.account_id = ?
+	        or exists (
+	          select 1
+	          from tweet_account_edges edge
+	          where edge.account_id = ?
+	            and edge.tweet_id = o.source_id
+	        )
+	        or exists (
+	          select 1
+	          from tweet_collections collection
+	          where collection.account_id = ?
+	            and collection.tweet_id = o.source_id
+	        )
+	      )
+	    )
+	  )`);
+		params.push(query.account, query.account, query.account, query.account);
+	}
 	if (kind === "videos") {
 		addVideoUrlPrefilter(conditions);
 	}