birdclaw 0.5.1 → 0.7.0

package/src/lib/geocoding.ts

@@ -0,0 +1,296 @@
+import { z } from "zod";
+import { getNativeDb } from "./db";
+import { coordinatesFromLocationKey, normalizeLocationKey } from "./location";
+import type { Database } from "./sqlite";
+
+const OpenCageGeometrySchema = z.object({ lat: z.number(), lng: z.number() });
+const OpenCageResultSchema = z.object({
+	geometry: OpenCageGeometrySchema,
+	confidence: z.number().int().min(0).max(10).optional(),
+	formatted: z.string().optional(),
+	components: z
+		.object({ country_code: z.string().optional() })
+		.loose()
+		.optional(),
+	bounds: z
+		.object({
+			northeast: OpenCageGeometrySchema.optional(),
+			southwest: OpenCageGeometrySchema.optional(),
+		})
+		.optional(),
+});
+const OpenCageResponseSchema = z.object({
+	results: z.array(OpenCageResultSchema),
+	status: z.object({ code: z.number(), message: z.string() }).loose(),
+});
+
+export interface GeocodeResult {
+	normalizedKey: string;
+	original: string;
+	lat: number;
+	lng: number;
+	formatted?: string;
+	countryCode?: string;
+	confidence?: number;
+	provider: "opencage" | "coords";
+	approxRadiusM?: number;
+	bounds?: Record<string, unknown>;
+	components?: Record<string, unknown>;
+}
+
+export class GeocodeRateLimitError extends Error {
+	constructor() {
+		super("opencage:429");
+		this.name = "GeocodeRateLimitError";
+	}
+}
+
+const SQLITE_IN_CHUNK_SIZE = 400;
+
+export function getOpenCageApiKey() {
+	return process.env.OPENCAGE_API_KEY?.trim() || null;
+}
+
+function approximateRadiusM(bounds: GeocodeResult["bounds"]) {
+	const northeast = bounds?.northeast as
+		| { lat?: unknown; lng?: unknown }
+		| undefined;
+	const southwest = bounds?.southwest as
+		| { lat?: unknown; lng?: unknown }
+		| undefined;
+	if (
+		typeof northeast?.lat !== "number" ||
+		typeof northeast.lng !== "number" ||
+		typeof southwest?.lat !== "number" ||
+		typeof southwest.lng !== "number"
+	) {
+		return undefined;
+	}
+	const latMeters = Math.abs(northeast.lat - southwest.lat) * 111_320;
+	const lngMeters =
+		Math.abs(northeast.lng - southwest.lng) *
+		Math.cos((((northeast.lat + southwest.lat) / 2) * Math.PI) / 180) *
+		111_320;
+	return Math.max(500, Math.min(120_000, Math.hypot(latMeters, lngMeters) / 2));
+}
+
+export function readCachedGeocodes(
+	keys: readonly string[],
+	db = getNativeDb(),
+) {
+	if (keys.length === 0) return new Map<string, GeocodeResult>();
+	const rows: Array<Record<string, unknown>> = [];
+	for (let index = 0; index < keys.length; index += SQLITE_IN_CHUNK_SIZE) {
+		const chunk = keys.slice(index, index + SQLITE_IN_CHUNK_SIZE);
+		const placeholders = chunk.map(() => "?").join(",");
+		rows.push(
+			...(db
+				.prepare(
+					`
+          select normalized_key, original, lat, lng, formatted, country_code,
+            confidence, provider, approx_radius_m, bounds_json, components_json
+          from geocoded_locations
+          where normalized_key in (${placeholders})
+          `,
+				)
+				.all(...chunk) as Array<Record<string, unknown>>),
+		);
+	}
+	const map = new Map<string, GeocodeResult>();
+	for (const row of rows) {
+		const key = String(row.normalized_key);
+		map.set(key, {
+			normalizedKey: key,
+			original: String(row.original),
+			lat: Number(row.lat),
+			lng: Number(row.lng),
+			formatted: typeof row.formatted === "string" ? row.formatted : undefined,
+			countryCode:
+				typeof row.country_code === "string" ? row.country_code : undefined,
+			confidence:
+				typeof row.confidence === "number" ? row.confidence : undefined,
+			provider: row.provider === "coords" ? "coords" : "opencage",
+			approxRadiusM:
+				typeof row.approx_radius_m === "number"
+					? row.approx_radius_m
+					: undefined,
+			bounds: parseJsonRecord(row.bounds_json),
+			components: parseJsonRecord(row.components_json),
+		});
+	}
+	return map;
+}
+
+export function readSuppressedGeocodeKeys(
+	keys: readonly string[],
+	db = getNativeDb(),
+) {
+	if (keys.length === 0) return new Set<string>();
+	const now = new Date().toISOString();
+	const rows: Array<{ normalized_key: string }> = [];
+	for (let index = 0; index < keys.length; index += SQLITE_IN_CHUNK_SIZE) {
+		const chunk = keys.slice(index, index + SQLITE_IN_CHUNK_SIZE);
+		const placeholders = chunk.map(() => "?").join(",");
+		rows.push(
+			...(db
+				.prepare(
+					`
+          select normalized_key
+          from geocoded_locations_unresolved
+          where normalized_key in (${placeholders})
+            and (ttl_until is null or ttl_until > ?)
+          `,
+				)
+				.all(...chunk, now) as Array<{ normalized_key: string }>),
+		);
+	}
+	return new Set(rows.map((row) => row.normalized_key));
+}
+
+export function storeGeocode(result: GeocodeResult, db = getNativeDb()) {
+	const now = new Date().toISOString();
+	db.prepare(
+		`
+    insert into geocoded_locations (
+      normalized_key, original, lat, lng, formatted, country_code, confidence,
+      provider, approx_radius_m, bounds_json, components_json, hits, created_at,
+      last_used_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
+    on conflict(normalized_key) do update set
+      original = excluded.original,
+      lat = excluded.lat,
+      lng = excluded.lng,
+      formatted = coalesce(excluded.formatted, geocoded_locations.formatted),
+      country_code = coalesce(excluded.country_code, geocoded_locations.country_code),
+      confidence = coalesce(excluded.confidence, geocoded_locations.confidence),
+      provider = excluded.provider,
+      approx_radius_m = coalesce(excluded.approx_radius_m, geocoded_locations.approx_radius_m),
+      bounds_json = excluded.bounds_json,
+      components_json = excluded.components_json,
+      hits = geocoded_locations.hits + 1,
+      last_used_at = excluded.last_used_at
+    `,
+	).run(
+		result.normalizedKey,
+		result.original,
+		result.lat,
+		result.lng,
+		result.formatted ?? null,
+		result.countryCode?.toUpperCase() ?? null,
+		result.confidence ?? null,
+		result.provider,
+		result.approxRadiusM ?? null,
+		JSON.stringify(result.bounds ?? {}),
+		JSON.stringify(result.components ?? {}),
+		now,
+		now,
+	);
+	db.prepare(
+		"delete from geocoded_locations_unresolved where normalized_key = ?",
+	).run(result.normalizedKey);
+}
+
+export function storeUnresolvedGeocode(
+	normalizedKey: string,
+	original: string,
+	reason: string,
+	db: Database = getNativeDb(),
+) {
+	const now = new Date().toISOString();
+	const ttl = new Date(Date.now() + 7 * 24 * 60 * 60_000).toISOString();
+	db.prepare(
+		`
+    insert into geocoded_locations_unresolved (
+      normalized_key, original, reason, last_attempted_at, ttl_until
+    ) values (?, ?, ?, ?, ?)
+    on conflict(normalized_key) do update set
+      original = excluded.original,
+      reason = excluded.reason,
+      last_attempted_at = excluded.last_attempted_at,
+      ttl_until = excluded.ttl_until
+    `,
+	).run(normalizedKey, original, reason, now, ttl);
+}
+
+export async function geocodeLocation(
+	original: string,
+	db: Database = getNativeDb(),
+	signal?: AbortSignal,
+): Promise<GeocodeResult | null> {
+	if (signal?.aborted) throw new Error("geocode aborted");
+	const normalizedKey = normalizeLocationKey(original);
+	if (!normalizedKey) return null;
+
+	const coord = coordinatesFromLocationKey(normalizedKey);
+	if (coord) {
+		const result: GeocodeResult = {
+			normalizedKey,
+			original,
+			lat: coord.lat,
+			lng: coord.lng,
+			formatted: original,
+			provider: "coords",
+			approxRadiusM: 500,
+		};
+		storeGeocode(result, db);
+		return result;
+	}
+
+	const key = getOpenCageApiKey();
+	if (!key) return null;
+
+	const url = new URL("https://api.opencagedata.com/geocode/v1/json");
+	url.searchParams.set("q", original);
+	url.searchParams.set("key", key);
+	url.searchParams.set("limit", "1");
+	url.searchParams.set("no_annotations", "1");
+
+	const response = await fetch(url, { signal });
+	if (!response.ok) {
+		if (response.status === 429) {
+			throw new GeocodeRateLimitError();
+		}
+		storeUnresolvedGeocode(
+			normalizedKey,
+			original,
+			`opencage:${String(response.status)}`,
+			db,
+		);
+		return null;
+	}
+	const payload = OpenCageResponseSchema.parse(await response.json());
+	const first = payload.results[0];
+	if (!first) {
+		storeUnresolvedGeocode(normalizedKey, original, "no-result", db);
+		return null;
+	}
+
+	const bounds = first.bounds as Record<string, unknown> | undefined;
+	const result: GeocodeResult = {
+		normalizedKey,
+		original,
+		lat: first.geometry.lat,
+		lng: first.geometry.lng,
+		formatted: first.formatted,
+		countryCode: first.components?.country_code?.toUpperCase(),
+		confidence: first.confidence,
+		provider: "opencage",
+		approxRadiusM: approximateRadiusM(bounds),
+		bounds,
+		components: first.components as Record<string, unknown> | undefined,
+	};
+	storeGeocode(result, db);
+	return result;
+}
+
+function parseJsonRecord(value: unknown) {
+	if (typeof value !== "string" || value.length === 0) return undefined;
+	try {
+		const parsed = JSON.parse(value) as unknown;
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, unknown>)
+			: undefined;
+	} catch {
+		return undefined;
+	}
+}

package/src/lib/http-effect.ts

@@ -0,0 +1,222 @@
+import { Effect } from "effect";
+import { runEffectPromise, tryPromise } from "./effect-runtime";
+
+const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]);
+
+export function jsonResponse(data: unknown, init?: ResponseInit) {
+	const headers = new Headers(init?.headers);
+	headers.set("content-type", "application/json");
+	return new Response(JSON.stringify(data), {
+		...init,
+		headers,
+	});
+}
+
+export function requestJsonEffect<T = Record<string, unknown>>(
+	request: Request,
+	fallback?: T,
+): Effect.Effect<T, unknown> {
+	return tryPromise(() => request.json() as Promise<T>).pipe(
+		Effect.catchAll((error) =>
+			fallback === undefined ? Effect.fail(error) : Effect.succeed(fallback),
+		),
+	);
+}
+
+export function runRouteEffect<A, E>(effect: Effect.Effect<A, E>): Promise<A> {
+	return runEffectPromise(effect);
+}
+
+function normalizedHost(value: string) {
+	return value.toLowerCase().replace(/^\[|\]$/g, "");
+}
+
+function isTestEnvironment() {
+	return process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+}
+
+function requestCookie(request: Request, name: string) {
+	const cookie = request.headers.get("cookie");
+	if (!cookie) return { found: false, value: null };
+	for (const part of cookie.split(";")) {
+		const [key, ...rest] = part.trim().split("=");
+		if (key === name) {
+			try {
+				return { found: true, value: decodeURIComponent(rest.join("=")) };
+			} catch {
+				return { found: true, value: null };
+			}
+		}
+	}
+	return { found: false, value: null };
+}
+
+function configuredWebToken() {
+	const token = process.env.BIRDCLAW_WEB_TOKEN?.trim();
+	return token || null;
+}
+
+function requestWebTokenStatus(request: Request) {
+	const token = configuredWebToken();
+	if (!token)
+		return {
+			configured: false,
+			valid: false,
+			fromCookie: false,
+			provided: false,
+		};
+	const headerToken = request.headers.get("x-birdclaw-token");
+	const cookieToken = requestCookie(request, "birdclaw_token");
+	const provided = headerToken !== null || cookieToken.found;
+	const cookieValue = cookieToken.value;
+	const valid = headerToken === token || cookieValue === token;
+	return {
+		configured: true,
+		valid,
+		fromCookie: cookieValue === token && headerToken !== token,
+		provided,
+	};
+}
+
+function isLocalWebHost(value: string) {
+	const host = normalizedHost(value);
+	return LOCAL_HOSTS.has(host) || host.endsWith(".localhost");
+}
+
+function allowsUnauthenticatedLocalWeb() {
+	return process.env.BIRDCLAW_LOCAL_WEB === "1";
+}
+
+function hasForwardedRequestHeaders(request: Request) {
+	return (
+		request.headers.has("forwarded") ||
+		request.headers.has("x-forwarded-for") ||
+		request.headers.has("x-forwarded-proto") ||
+		request.headers.has("x-forwarded-host") ||
+		request.headers.has("x-real-ip")
+	);
+}
+
+function firstForwardedValue(value: string | null) {
+	return value?.split(",")[0]?.trim() || null;
+}
+
+function forwardedHeaderPair(request: Request, key: "host" | "proto") {
+	const forwarded = firstForwardedValue(request.headers.get("forwarded"));
+	if (!forwarded) return null;
+	for (const part of forwarded.split(";")) {
+		const [name, ...rest] = part.trim().split("=");
+		if (name?.toLowerCase() !== key) continue;
+		const value = rest.join("=").trim().replace(/^"|"$/g, "");
+		return value || null;
+	}
+	return null;
+}
+
+function forwardedOrigin(request: Request) {
+	const proto =
+		firstForwardedValue(request.headers.get("x-forwarded-proto")) ??
+		forwardedHeaderPair(request, "proto");
+	const host =
+		firstForwardedValue(request.headers.get("x-forwarded-host")) ??
+		forwardedHeaderPair(request, "host");
+	if (!proto || !host) return null;
+	const normalizedProto = proto.toLowerCase();
+	if (normalizedProto !== "http" && normalizedProto !== "https") return null;
+	return `${normalizedProto}://${host}`;
+}
+
+function sameRequestOrigin(request: Request, origin: string, url: URL) {
+	return origin === url.origin || origin === forwardedOrigin(request);
+}
+
+export function sensitiveRequestErrorResponse(request: Request) {
+	const url = new URL(request.url);
+	const token = requestWebTokenStatus(request);
+	const isLocalRequest =
+		allowsUnauthenticatedLocalWeb() &&
+		isLocalWebHost(url.hostname) &&
+		!hasForwardedRequestHeaders(request);
+	const fetchSite = request.headers.get("sec-fetch-site");
+	const origin = request.headers.get("origin");
+	const allowRemoteEnv = process.env.BIRDCLAW_ALLOW_REMOTE_WEB === "1";
+	const allowTrustedRemote = allowRemoteEnv && !token.configured;
+
+	if (isTestEnvironment() && !token.valid) return null;
+
+	if (origin && !sameRequestOrigin(request, origin, url)) {
+		return jsonResponse(
+			{ ok: false, message: "Cross-origin web API access is disabled" },
+			{ status: 403 },
+		);
+	}
+
+	if (fetchSite === "cross-site") {
+		return jsonResponse(
+			{ ok: false, message: "Cross-site web API access is disabled" },
+			{ status: 403 },
+		);
+	}
+
+	if (
+		!isTestEnvironment() &&
+		!token.configured &&
+		!isLocalRequest &&
+		!allowTrustedRemote
+	) {
+		return jsonResponse(
+			{
+				ok: false,
+				message:
+					"Remote API access requires BIRDCLAW_ALLOW_REMOTE_WEB=1 for a trusted private proxy, or BIRDCLAW_WEB_TOKEN for tokened access",
+			},
+			{ status: 403 },
+		);
+	}
+
+	if (token.configured && !token.valid && (token.provided || !isLocalRequest)) {
+		return jsonResponse(
+			{ ok: false, message: "Invalid web token" },
+			{ status: 403 },
+		);
+	}
+
+	const allowRemote = allowRemoteEnv && (token.valid || allowTrustedRemote);
+	if (!allowRemote && !isLocalRequest) {
+		return jsonResponse(
+			{ ok: false, message: "Remote web API access is disabled" },
+			{ status: 403 },
+		);
+	}
+
+	if (
+		token.fromCookie &&
+		fetchSite !== "same-origin" &&
+		fetchSite !== "same-site"
+	) {
+		return jsonResponse(
+			{ ok: false, message: "Same-origin browser request required" },
+			{ status: 403 },
+		);
+	}
+
+	return null;
+}
+
+export function parseBoundedInteger(
+	value: number | string | null | undefined,
+	{
+		defaultValue,
+		min = 1,
+		max,
+	}: { defaultValue?: number; min?: number; max: number },
+) {
+	if (value === null || value === undefined || value === "")
+		return defaultValue;
+	if (typeof value === "string" && !/^\d+$/.test(value.trim())) {
+		return defaultValue;
+	}
+	const parsed = typeof value === "number" ? value : Number(value);
+	if (!Number.isSafeInteger(parsed)) return defaultValue;
+	return Math.min(max, Math.max(min, parsed));
+}

package/src/lib/inbox.ts

@@ -1,5 +1,7 @@
+import { Effect } from "effect";
 import { getNativeDb } from "./db";
-import { scoreInboxItemWithOpenAI } from "./openai";
+import { runEffectPromise } from "./effect-runtime";
+import { scoreInboxItemWithOpenAIEffect } from "./openai";
 import { listDmConversations, listTimelineItems } from "./queries";
 import type { InboxItem, InboxQuery, InboxResponse } from "./types";
 
@@ -61,6 +63,7 @@ function readStoredScores() {
 
 export function listInboxItems({
 	kind = "mixed",
+	account,
 	minScore = 0,
 	hideLowSignal = false,
 	limit = 20,
@@ -71,6 +74,7 @@ export function listInboxItems({
 	if (kind === "mixed" || kind === "mentions") {
 		for (const mention of listTimelineItems({
 			resource: "mentions",
+			account,
 			replyFilter: "unreplied",
 			limit: 50,
 		})) {
@@ -102,8 +106,9 @@ export function listInboxItems({
 
 	if (kind === "mixed" || kind === "dms") {
 		for (const dm of listDmConversations({
+			account,
 			replyFilter: "unreplied",
-			sort: "influence",
+			sort: "followers",
 			limit: 50,
 		})) {
 			const scoreKey = `dm:${dm.id}`;
@@ -151,30 +156,47 @@ export function listInboxItems({
 	};
 }
 
-export async function scoreInbox({
+function toError(error: unknown) {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+function trySync<T>(try_: () => T) {
+	return Effect.try({
+		try: try_,
+		catch: toError,
+	});
+}
+
+export function scoreInboxEffect({
 	kind = "mixed",
+	account,
 	limit = 8,
-}: Pick<InboxQuery, "kind" | "limit"> = {}) {
-	const db = getNativeDb();
-	const items = listInboxItems({ kind, limit }).items;
-	const results = [];
-
-	for (const item of items) {
-		const scored = await scoreInboxItemWithOpenAI({
-			entityKind: item.entityKind,
-			title: item.title,
-			text: item.text,
-			influenceScore: item.influenceScore,
-			participant: {
-				handle: item.participant.handle,
-				displayName: item.participant.displayName,
-				bio: item.participant.bio,
-				followersCount: item.participant.followersCount,
-			},
-		});
+}: Pick<InboxQuery, "kind" | "account" | "limit"> = {}) {
+	return Effect.gen(function* () {
+		const db = yield* trySync(() => getNativeDb());
+		const items = yield* trySync(
+			() => listInboxItems({ kind, account, limit }).items,
+		);
+		const results = [];
 
-		db.prepare(
-			`
+		for (const item of items) {
+			const scored = yield* scoreInboxItemWithOpenAIEffect({
+				entityKind: item.entityKind,
+				title: item.title,
+				text: item.text,
+				influenceScore: item.influenceScore,
+				participant: {
+					handle: item.participant.handle,
+					displayName: item.participant.displayName,
+					bio: item.participant.bio,
+					followersCount: item.participant.followersCount,
+				},
+			});
+
+			yield* trySync(() =>
+				db
+					.prepare(
+						`
       insert into ai_scores (
         entity_kind, entity_id, model, score, summary, reasoning, updated_at
       ) values (?, ?, ?, ?, ?, ?, ?)
@@ -185,26 +207,35 @@ export async function scoreInbox({
         reasoning = excluded.reasoning,
         updated_at = excluded.updated_at
       `,
-		).run(
-			item.entityKind,
-			item.entityId,
-			scored.model,
-			scored.score,
-			scored.summary,
-			scored.reasoning,
-			new Date().toISOString(),
-		);
+					)
+					.run(
+						item.entityKind,
+						item.entityId,
+						scored.model,
+						scored.score,
+						scored.summary,
+						scored.reasoning,
+						new Date().toISOString(),
+					),
+			);
 
-		results.push({
-			id: item.id,
-			score: scored.score,
-			source: "openai",
-		});
-	}
+			results.push({
+				id: item.id,
+				score: scored.score,
+				source: "openai",
+			});
+		}
 
-	return {
-		ok: true,
-		scored: results.length,
-		items: results,
-	};
+		return {
+			ok: true,
+			scored: results.length,
+			items: results,
+		};
+	});
+}
+
+export function scoreInbox(
+	options: Pick<InboxQuery, "kind" | "account" | "limit"> = {},
+) {
+	return runEffectPromise(scoreInboxEffect(options));
 }