birdclaw 0.6.0 → 0.7.0

package/src/lib/data-sources.ts

@@ -0,0 +1,219 @@
+import { Effect } from "effect";
+import { getAuthenticatedBirdAccountEffect } from "./bird";
+import { getNativeDb } from "./db";
+import { runEffectPromise } from "./effect-runtime";
+import type {
+	LiveDataSourceAccount,
+	LiveDataSourceCapability,
+	LiveDataSourcesResponse,
+	LiveDataSourceStatus,
+} from "./types";
+import {
+	getTransportStatusEffect,
+	lookupAuthenticatedOAuth2UserEffect,
+	readXurlOAuth2AccountsEffect,
+} from "./xurl";
+
+function errorMessage(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
+function readLocalAccounts(): LiveDataSourceAccount[] {
+	const db = getNativeDb();
+	const rows = db
+		.prepare(
+			`
+      select id, handle, external_user_id, is_default
+      from accounts
+      order by is_default desc, lower(handle) asc
+      `,
+		)
+		.all() as Array<{
+		id: string;
+		handle: string;
+		external_user_id: string | null;
+		is_default: number;
+	}>;
+	return rows.map((row) => ({
+		id: row.external_user_id ?? row.id,
+		handle: row.handle,
+		username: row.handle.replace(/^@/, ""),
+		isDefault: row.is_default === 1,
+	}));
+}
+
+function getBirdclawStatusEffect(): Effect.Effect<LiveDataSourceStatus, never> {
+	return Effect.try({
+		try: () => readLocalAccounts(),
+		catch: (error) => error,
+	}).pipe(
+		Effect.map((accounts) => ({
+			source: "birdclaw" as const,
+			label: "Birdclaw local",
+			works: true,
+			installed: true,
+			status: "ok" as const,
+			detail:
+				accounts.length > 0
+					? `${accounts.length.toString()} local account${accounts.length === 1 ? "" : "s"}`
+					: "local database ready; no accounts imported yet",
+			accounts,
+		})),
+		Effect.catchAll((error) =>
+			Effect.succeed({
+				source: "birdclaw" as const,
+				label: "Birdclaw local",
+				works: false,
+				installed: true,
+				status: "error" as const,
+				detail: errorMessage(error),
+				accounts: [],
+			}),
+		),
+	);
+}
+
+function getBirdStatusEffect(): Effect.Effect<LiveDataSourceStatus, never> {
+	return getAuthenticatedBirdAccountEffect().pipe(
+		Effect.map((account) => ({
+			source: "bird" as const,
+			label: "bird",
+			works: true,
+			installed: true,
+			status: "ok" as const,
+			detail: `authenticated as @${account.username}`,
+			accounts: [
+				{
+					...(account.id ? { id: account.id } : {}),
+					username: account.username,
+					handle: `@${account.username}`,
+				},
+			],
+		})),
+		Effect.catchAll((error) =>
+			Effect.succeed({
+				source: "bird" as const,
+				label: "bird",
+				works: false,
+				status: "error" as const,
+				detail: errorMessage(error),
+				accounts: [],
+			}),
+		),
+	);
+}
+
+function getXurlStatusEffect(): Effect.Effect<LiveDataSourceStatus, never> {
+	return Effect.gen(function* () {
+		const transport = yield* getTransportStatusEffect();
+		const oauth2Accounts = yield* readXurlOAuth2AccountsEffect();
+		const authenticated = yield* lookupAuthenticatedOAuth2UserEffect().pipe(
+			Effect.catchAll(() => Effect.succeed(null)),
+		);
+		const authenticatedAccount =
+			authenticated && typeof authenticated === "object"
+				? ({
+						...(typeof authenticated.id === "string"
+							? { id: authenticated.id }
+							: {}),
+						...(typeof authenticated.username === "string"
+							? {
+									username: authenticated.username,
+									handle: `@${authenticated.username}`,
+								}
+							: {}),
+					} satisfies LiveDataSourceAccount)
+				: undefined;
+		const accounts: LiveDataSourceAccount[] = [
+			...(authenticatedAccount ? [authenticatedAccount] : []),
+			...oauth2Accounts,
+		];
+		const deduped = accounts.filter(
+			(account, index) =>
+				accounts.findIndex(
+					(candidate) =>
+						(candidate.app ?? "") === (account.app ?? "") &&
+						(candidate.username ?? candidate.handle ?? "") ===
+							(account.username ?? account.handle ?? ""),
+				) === index,
+		);
+		const works = transport.availableTransport === "xurl";
+		return {
+			source: "xurl" as const,
+			label: "xurl",
+			works,
+			installed: transport.installed,
+			status: works ? ("ok" as const) : ("warning" as const),
+			detail: transport.statusText,
+			accounts: deduped,
+		};
+	}).pipe(
+		Effect.catchAll((error) =>
+			Effect.succeed({
+				source: "xurl" as const,
+				label: "xurl",
+				works: false,
+				status: "error" as const,
+				detail: errorMessage(error),
+				accounts: [],
+			}),
+		),
+	);
+}
+
+const capabilities: LiveDataSourceCapability[] = [
+	{
+		key: "timeline",
+		label: "Home timeline",
+		primary: "xurl",
+		fallbacks: ["bird"],
+	},
+	{
+		key: "mentions",
+		label: "Mentions",
+		primary: "xurl",
+		fallbacks: ["bird", "birdclaw"],
+		notes: "bird fallback is skipped when a since/start cursor requires xurl.",
+	},
+	{
+		key: "search",
+		label: "Fresh search",
+		primary: "bird",
+		fallbacks: ["xurl", "birdclaw"],
+		notes: "dated searches require xurl.",
+	},
+	{
+		key: "dms",
+		label: "DMs",
+		primary: "xurl",
+		fallbacks: ["bird", "birdclaw"],
+		notes: "message requests require bird.",
+	},
+	{
+		key: "follow-graph",
+		label: "Followers / following",
+		primary: "bird",
+		fallbacks: ["xurl", "birdclaw"],
+	},
+];
+
+export function getLiveDataSourcesEffect(): Effect.Effect<
+	LiveDataSourcesResponse,
+	never
+> {
+	return Effect.gen(function* () {
+		const sources = yield* Effect.all(
+			[getBirdclawStatusEffect(), getBirdStatusEffect(), getXurlStatusEffect()],
+			{ concurrency: "unbounded" },
+		);
+		return {
+			generatedAt: new Date().toISOString(),
+			sources,
+			capabilities,
+		};
+	});
+}
+
+export function getLiveDataSources(): Promise<LiveDataSourcesResponse> {
+	return runEffectPromise(getLiveDataSourcesEffect());
+}

package/src/lib/db.ts

@@ -117,7 +117,13 @@ export interface TweetCollectionsTable {
 export interface TweetAccountEdgesTable {
 	account_id: string;
 	tweet_id: string;
-	kind: "home" | "mention" | "authored" | "thread_context";
+	kind:
+		| "home"
+		| "mention"
+		| "authored"
+		| "search"
+		| "profile"
+		| "thread_context";
 	first_seen_at: string;
 	last_seen_at: string;
 	seen_count: number;
@@ -258,6 +264,31 @@ export interface FollowEventsTable {
 	snapshot_id: string;
 }
 
+export interface GeocodedLocationsTable {
+	normalized_key: string;
+	original: string;
+	lat: number;
+	lng: number;
+	formatted: string | null;
+	country_code: string | null;
+	confidence: number | null;
+	provider: string;
+	approx_radius_m: number | null;
+	bounds_json: string;
+	components_json: string;
+	hits: number;
+	created_at: string;
+	last_used_at: string;
+}
+
+export interface GeocodedLocationsUnresolvedTable {
+	normalized_key: string;
+	original: string;
+	reason: string;
+	last_attempted_at: string;
+	ttl_until: string | null;
+}
+
 export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
@@ -281,6 +312,8 @@ export interface BirdclawDatabase {
 	follow_snapshots: FollowSnapshotsTable;
 	follow_snapshot_members: FollowSnapshotMembersTable;
 	follow_events: FollowEventsTable;
+	geocoded_locations: GeocodedLocationsTable;
+	geocoded_locations_unresolved: GeocodedLocationsUnresolvedTable;
 }
 
 let nativeDb: Database | undefined;
@@ -563,6 +596,31 @@ const BASE_SCHEMA_SQL = `
     snapshot_id text not null
   );
 
+  create table if not exists geocoded_locations (
+    normalized_key text primary key,
+    original text not null,
+    lat real not null,
+    lng real not null,
+    formatted text,
+    country_code text,
+    confidence integer,
+    provider text not null,
+    approx_radius_m real,
+    bounds_json text not null default '{}',
+    components_json text not null default '{}',
+    hits integer not null default 1,
+    created_at text not null,
+    last_used_at text not null
+  );
+
+  create table if not exists geocoded_locations_unresolved (
+    normalized_key text primary key,
+    original text not null,
+    reason text not null,
+    last_attempted_at text not null,
+    ttl_until text
+  );
+
   create virtual table if not exists tweets_fts using fts5(
     tweet_id unindexed,
     text
@@ -611,6 +669,9 @@ const INDEX_SQL = `
   create index if not exists idx_follow_edges_profile on follow_edges(profile_id, current);
   create index if not exists idx_follow_snapshots_account on follow_snapshots(account_id, direction, completed_at desc);
   create index if not exists idx_follow_events_account on follow_events(account_id, direction, kind, event_at desc);
+  create index if not exists idx_geocoded_locations_country on geocoded_locations(country_code);
+  create index if not exists idx_geocoded_locations_last_used on geocoded_locations(last_used_at desc);
+  create index if not exists idx_geocoded_unresolved_ttl on geocoded_locations_unresolved(ttl_until desc);
 `;
 
 function getColumnNames(db: Database, tableName: string): Set<string> {

package/src/lib/geocoding.ts

@@ -0,0 +1,296 @@
+import { z } from "zod";
+import { getNativeDb } from "./db";
+import { coordinatesFromLocationKey, normalizeLocationKey } from "./location";
+import type { Database } from "./sqlite";
+
+const OpenCageGeometrySchema = z.object({ lat: z.number(), lng: z.number() });
+const OpenCageResultSchema = z.object({
+	geometry: OpenCageGeometrySchema,
+	confidence: z.number().int().min(0).max(10).optional(),
+	formatted: z.string().optional(),
+	components: z
+		.object({ country_code: z.string().optional() })
+		.loose()
+		.optional(),
+	bounds: z
+		.object({
+			northeast: OpenCageGeometrySchema.optional(),
+			southwest: OpenCageGeometrySchema.optional(),
+		})
+		.optional(),
+});
+const OpenCageResponseSchema = z.object({
+	results: z.array(OpenCageResultSchema),
+	status: z.object({ code: z.number(), message: z.string() }).loose(),
+});
+
+export interface GeocodeResult {
+	normalizedKey: string;
+	original: string;
+	lat: number;
+	lng: number;
+	formatted?: string;
+	countryCode?: string;
+	confidence?: number;
+	provider: "opencage" | "coords";
+	approxRadiusM?: number;
+	bounds?: Record<string, unknown>;
+	components?: Record<string, unknown>;
+}
+
+export class GeocodeRateLimitError extends Error {
+	constructor() {
+		super("opencage:429");
+		this.name = "GeocodeRateLimitError";
+	}
+}
+
+const SQLITE_IN_CHUNK_SIZE = 400;
+
+export function getOpenCageApiKey() {
+	return process.env.OPENCAGE_API_KEY?.trim() || null;
+}
+
+function approximateRadiusM(bounds: GeocodeResult["bounds"]) {
+	const northeast = bounds?.northeast as
+		| { lat?: unknown; lng?: unknown }
+		| undefined;
+	const southwest = bounds?.southwest as
+		| { lat?: unknown; lng?: unknown }
+		| undefined;
+	if (
+		typeof northeast?.lat !== "number" ||
+		typeof northeast.lng !== "number" ||
+		typeof southwest?.lat !== "number" ||
+		typeof southwest.lng !== "number"
+	) {
+		return undefined;
+	}
+	const latMeters = Math.abs(northeast.lat - southwest.lat) * 111_320;
+	const lngMeters =
+		Math.abs(northeast.lng - southwest.lng) *
+		Math.cos((((northeast.lat + southwest.lat) / 2) * Math.PI) / 180) *
+		111_320;
+	return Math.max(500, Math.min(120_000, Math.hypot(latMeters, lngMeters) / 2));
+}
+
+export function readCachedGeocodes(
+	keys: readonly string[],
+	db = getNativeDb(),
+) {
+	if (keys.length === 0) return new Map<string, GeocodeResult>();
+	const rows: Array<Record<string, unknown>> = [];
+	for (let index = 0; index < keys.length; index += SQLITE_IN_CHUNK_SIZE) {
+		const chunk = keys.slice(index, index + SQLITE_IN_CHUNK_SIZE);
+		const placeholders = chunk.map(() => "?").join(",");
+		rows.push(
+			...(db
+				.prepare(
+					`
+          select normalized_key, original, lat, lng, formatted, country_code,
+            confidence, provider, approx_radius_m, bounds_json, components_json
+          from geocoded_locations
+          where normalized_key in (${placeholders})
+          `,
+				)
+				.all(...chunk) as Array<Record<string, unknown>>),
+		);
+	}
+	const map = new Map<string, GeocodeResult>();
+	for (const row of rows) {
+		const key = String(row.normalized_key);
+		map.set(key, {
+			normalizedKey: key,
+			original: String(row.original),
+			lat: Number(row.lat),
+			lng: Number(row.lng),
+			formatted: typeof row.formatted === "string" ? row.formatted : undefined,
+			countryCode:
+				typeof row.country_code === "string" ? row.country_code : undefined,
+			confidence:
+				typeof row.confidence === "number" ? row.confidence : undefined,
+			provider: row.provider === "coords" ? "coords" : "opencage",
+			approxRadiusM:
+				typeof row.approx_radius_m === "number"
+					? row.approx_radius_m
+					: undefined,
+			bounds: parseJsonRecord(row.bounds_json),
+			components: parseJsonRecord(row.components_json),
+		});
+	}
+	return map;
+}
+
+export function readSuppressedGeocodeKeys(
+	keys: readonly string[],
+	db = getNativeDb(),
+) {
+	if (keys.length === 0) return new Set<string>();
+	const now = new Date().toISOString();
+	const rows: Array<{ normalized_key: string }> = [];
+	for (let index = 0; index < keys.length; index += SQLITE_IN_CHUNK_SIZE) {
+		const chunk = keys.slice(index, index + SQLITE_IN_CHUNK_SIZE);
+		const placeholders = chunk.map(() => "?").join(",");
+		rows.push(
+			...(db
+				.prepare(
+					`
+          select normalized_key
+          from geocoded_locations_unresolved
+          where normalized_key in (${placeholders})
+            and (ttl_until is null or ttl_until > ?)
+          `,
+				)
+				.all(...chunk, now) as Array<{ normalized_key: string }>),
+		);
+	}
+	return new Set(rows.map((row) => row.normalized_key));
+}
+
+export function storeGeocode(result: GeocodeResult, db = getNativeDb()) {
+	const now = new Date().toISOString();
+	db.prepare(
+		`
+    insert into geocoded_locations (
+      normalized_key, original, lat, lng, formatted, country_code, confidence,
+      provider, approx_radius_m, bounds_json, components_json, hits, created_at,
+      last_used_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
+    on conflict(normalized_key) do update set
+      original = excluded.original,
+      lat = excluded.lat,
+      lng = excluded.lng,
+      formatted = coalesce(excluded.formatted, geocoded_locations.formatted),
+      country_code = coalesce(excluded.country_code, geocoded_locations.country_code),
+      confidence = coalesce(excluded.confidence, geocoded_locations.confidence),
+      provider = excluded.provider,
+      approx_radius_m = coalesce(excluded.approx_radius_m, geocoded_locations.approx_radius_m),
+      bounds_json = excluded.bounds_json,
+      components_json = excluded.components_json,
+      hits = geocoded_locations.hits + 1,
+      last_used_at = excluded.last_used_at
+    `,
+	).run(
+		result.normalizedKey,
+		result.original,
+		result.lat,
+		result.lng,
+		result.formatted ?? null,
+		result.countryCode?.toUpperCase() ?? null,
+		result.confidence ?? null,
+		result.provider,
+		result.approxRadiusM ?? null,
+		JSON.stringify(result.bounds ?? {}),
+		JSON.stringify(result.components ?? {}),
+		now,
+		now,
+	);
+	db.prepare(
+		"delete from geocoded_locations_unresolved where normalized_key = ?",
+	).run(result.normalizedKey);
+}
+
+export function storeUnresolvedGeocode(
+	normalizedKey: string,
+	original: string,
+	reason: string,
+	db: Database = getNativeDb(),
+) {
+	const now = new Date().toISOString();
+	const ttl = new Date(Date.now() + 7 * 24 * 60 * 60_000).toISOString();
+	db.prepare(
+		`
+    insert into geocoded_locations_unresolved (
+      normalized_key, original, reason, last_attempted_at, ttl_until
+    ) values (?, ?, ?, ?, ?)
+    on conflict(normalized_key) do update set
+      original = excluded.original,
+      reason = excluded.reason,
+      last_attempted_at = excluded.last_attempted_at,
+      ttl_until = excluded.ttl_until
+    `,
+	).run(normalizedKey, original, reason, now, ttl);
+}
+
+export async function geocodeLocation(
+	original: string,
+	db: Database = getNativeDb(),
+	signal?: AbortSignal,
+): Promise<GeocodeResult | null> {
+	if (signal?.aborted) throw new Error("geocode aborted");
+	const normalizedKey = normalizeLocationKey(original);
+	if (!normalizedKey) return null;
+
+	const coord = coordinatesFromLocationKey(normalizedKey);
+	if (coord) {
+		const result: GeocodeResult = {
+			normalizedKey,
+			original,
+			lat: coord.lat,
+			lng: coord.lng,
+			formatted: original,
+			provider: "coords",
+			approxRadiusM: 500,
+		};
+		storeGeocode(result, db);
+		return result;
+	}
+
+	const key = getOpenCageApiKey();
+	if (!key) return null;
+
+	const url = new URL("https://api.opencagedata.com/geocode/v1/json");
+	url.searchParams.set("q", original);
+	url.searchParams.set("key", key);
+	url.searchParams.set("limit", "1");
+	url.searchParams.set("no_annotations", "1");
+
+	const response = await fetch(url, { signal });
+	if (!response.ok) {
+		if (response.status === 429) {
+			throw new GeocodeRateLimitError();
+		}
+		storeUnresolvedGeocode(
+			normalizedKey,
+			original,
+			`opencage:${String(response.status)}`,
+			db,
+		);
+		return null;
+	}
+	const payload = OpenCageResponseSchema.parse(await response.json());
+	const first = payload.results[0];
+	if (!first) {
+		storeUnresolvedGeocode(normalizedKey, original, "no-result", db);
+		return null;
+	}
+
+	const bounds = first.bounds as Record<string, unknown> | undefined;
+	const result: GeocodeResult = {
+		normalizedKey,
+		original,
+		lat: first.geometry.lat,
+		lng: first.geometry.lng,
+		formatted: first.formatted,
+		countryCode: first.components?.country_code?.toUpperCase(),
+		confidence: first.confidence,
+		provider: "opencage",
+		approxRadiusM: approximateRadiusM(bounds),
+		bounds,
+		components: first.components as Record<string, unknown> | undefined,
+	};
+	storeGeocode(result, db);
+	return result;
+}
+
+function parseJsonRecord(value: unknown) {
+	if (typeof value !== "string" || value.length === 0) return undefined;
+	try {
+		const parsed = JSON.parse(value) as unknown;
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, unknown>)
+			: undefined;
+	} catch {
+		return undefined;
+	}
+}