birdclaw 0.4.1 → 0.5.1

package/src/lib/conversation-surface.ts

@@ -0,0 +1,174 @@
+import { useCallback, useEffect, useSyncExternalStore } from "react";
+import type { ReactNode } from "react";
+import type { EmbeddedTweet } from "#/lib/types";
+
+type ConversationStatus = "idle" | "loading" | "ready" | "error";
+
+interface ConversationRecord {
+	error: string | null;
+	items: EmbeddedTweet[];
+	status: ConversationStatus;
+}
+
+interface ConversationSurfaceSnapshot {
+	expandedTweetId: string | null;
+	records: ReadonlyMap<string, ConversationRecord>;
+}
+
+type Listener = () => void;
+
+const emptyRecord: ConversationRecord = {
+	error: null,
+	items: [],
+	status: "idle",
+};
+
+let snapshot: ConversationSurfaceSnapshot = {
+	expandedTweetId: null,
+	records: new Map(),
+};
+const listeners = new Set<Listener>();
+const inFlight = new Set<string>();
+let activeScopes = 0;
+let generation = 0;
+
+function emit() {
+	for (const listener of listeners) {
+		listener();
+	}
+}
+
+function setSnapshot(next: ConversationSurfaceSnapshot) {
+	snapshot = next;
+	emit();
+}
+
+function updateRecord(tweetId: string, record: ConversationRecord) {
+	const records = new Map(snapshot.records);
+	records.set(tweetId, record);
+	setSnapshot({ ...snapshot, records });
+}
+
+function subscribe(listener: Listener) {
+	listeners.add(listener);
+	return () => {
+		listeners.delete(listener);
+	};
+}
+
+function getSnapshot() {
+	return snapshot;
+}
+
+async function loadConversation(tweetId: string) {
+	const current = snapshot.records.get(tweetId);
+	if (current?.status === "ready" || inFlight.has(tweetId)) {
+		return;
+	}
+
+	const loadGeneration = generation;
+	inFlight.add(tweetId);
+	updateRecord(tweetId, {
+		error: null,
+		items: current?.items ?? [],
+		status: "loading",
+	});
+
+	try {
+		const response = await fetch(
+			`/api/conversation?tweetId=${encodeURIComponent(tweetId)}`,
+		);
+		const data = (await response.json()) as {
+			error?: string;
+			items?: EmbeddedTweet[];
+			ok?: boolean;
+		};
+		if (!response.ok || data.ok === false) {
+			throw new Error(data.error ?? "Conversation unavailable");
+		}
+		if (loadGeneration !== generation) {
+			return;
+		}
+		updateRecord(tweetId, {
+			error: null,
+			items: (data.items ?? []).filter(Boolean),
+			status: "ready",
+		});
+	} catch (error) {
+		if (loadGeneration !== generation) {
+			return;
+		}
+		updateRecord(tweetId, {
+			error:
+				error instanceof Error ? error.message : "Conversation unavailable",
+			items: [],
+			status: "error",
+		});
+	} finally {
+		inFlight.delete(tweetId);
+	}
+}
+
+export function retainConversationSurfaceScope() {
+	activeScopes += 1;
+	return () => {
+		activeScopes = Math.max(0, activeScopes - 1);
+		if (activeScopes === 0) {
+			resetConversationSurface();
+		}
+	};
+}
+
+export function resetConversationSurface() {
+	generation += 1;
+	inFlight.clear();
+	setSnapshot({
+		expandedTweetId: null,
+		records: new Map(),
+	});
+}
+
+export function ConversationSurfaceScope({
+	children,
+}: {
+	children: ReactNode;
+}) {
+	useEffect(() => retainConversationSurfaceScope(), []);
+	return children;
+}
+
+export function useConversationSurface(tweetId: string) {
+	const currentSnapshot = useSyncExternalStore(
+		subscribe,
+		getSnapshot,
+		getSnapshot,
+	);
+	const record = currentSnapshot.records.get(tweetId) ?? emptyRecord;
+	const isOpen = currentSnapshot.expandedTweetId === tweetId;
+
+	const toggle = useCallback(() => {
+		const nextExpanded = snapshot.expandedTweetId === tweetId ? null : tweetId;
+		setSnapshot({ ...snapshot, expandedTweetId: nextExpanded });
+		if (nextExpanded) {
+			void loadConversation(tweetId);
+		}
+	}, [tweetId]);
+
+	const prefetch = useCallback(() => {
+		const current = snapshot.records.get(tweetId);
+		if (current?.status === "ready" || current?.status === "loading") {
+			return;
+		}
+		void loadConversation(tweetId);
+	}, [tweetId]);
+
+	return {
+		error: record.error,
+		isOpen,
+		items: record.items,
+		loading: record.status === "loading",
+		prefetch,
+		status: record.status,
+		toggle,
+	};
+}

package/src/lib/db.ts

@@ -20,6 +20,7 @@ export interface ProfilesTable {
 	bio: string;
 	followers_count: number;
 	following_count: number;
+	public_metrics_json: string;
 	avatar_hue: number;
 	avatar_url: string | null;
 	location: string | null;
@@ -116,7 +117,7 @@ export interface TweetCollectionsTable {
 export interface TweetAccountEdgesTable {
 	account_id: string;
 	tweet_id: string;
-	kind: "home" | "mention";
+	kind: "home" | "mention" | "authored" | "thread_context";
 	first_seen_at: string;
 	last_seen_at: string;
 	seen_count: number;
@@ -194,6 +195,8 @@ export interface UrlExpansionsTable {
 	expanded_handle: string | null;
 	title: string | null;
 	description: string | null;
+	image_url: string | null;
+	site_name: string | null;
 	error: string | null;
 	source: string;
 	updated_at: string;
@@ -210,6 +213,50 @@ export interface LinkOccurrencesTable {
 	created_at: string;
 }
 
+export interface FollowEdgesTable {
+	account_id: string;
+	direction: "followers" | "following";
+	profile_id: string;
+	external_user_id: string;
+	source: string;
+	current: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	ended_at: string | null;
+	updated_at: string;
+}
+
+export interface FollowSnapshotsTable {
+	id: string;
+	account_id: string;
+	direction: "followers" | "following";
+	source: string;
+	status: "complete" | "incomplete";
+	page_count: number;
+	result_count: number;
+	started_at: string;
+	completed_at: string;
+	raw_meta_json: string;
+}
+
+export interface FollowSnapshotMembersTable {
+	snapshot_id: string;
+	profile_id: string;
+	external_user_id: string;
+	position: number;
+}
+
+export interface FollowEventsTable {
+	id: string;
+	account_id: string;
+	direction: "followers" | "following";
+	profile_id: string;
+	external_user_id: string;
+	kind: "started" | "ended";
+	event_at: string;
+	snapshot_id: string;
+}
+
 export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
@@ -229,10 +276,15 @@ export interface BirdclawDatabase {
 	sync_cache: SyncCacheTable;
 	url_expansions: UrlExpansionsTable;
 	link_occurrences: LinkOccurrencesTable;
+	follow_edges: FollowEdgesTable;
+	follow_snapshots: FollowSnapshotsTable;
+	follow_snapshot_members: FollowSnapshotMembersTable;
+	follow_events: FollowEventsTable;
 }
 
 let nativeDb: Database | undefined;
 let kyselyDb: Kysely<BirdclawDatabase> | undefined;
+let demoSeedAttempted = false;
 
 export interface InitDatabaseOptions {
 	seedDemoData?: boolean;
@@ -260,6 +312,7 @@ const BASE_SCHEMA_SQL = `
     bio text not null,
     followers_count integer not null default 0,
     following_count integer not null default 0,
+    public_metrics_json text not null default '{}',
     avatar_hue integer not null default 0,
     avatar_url text,
     location text,
@@ -443,6 +496,8 @@ const BASE_SCHEMA_SQL = `
     expanded_handle text,
     title text,
     description text,
+    image_url text,
+    site_name text,
     error text,
     source text not null,
     updated_at text not null
@@ -460,6 +515,52 @@ const BASE_SCHEMA_SQL = `
     primary key (source_kind, source_id, source_position, short_url)
   );
 
+  create table if not exists follow_snapshots (
+    id text primary key,
+    account_id text not null,
+    direction text not null,
+    source text not null,
+    status text not null,
+    page_count integer not null default 0,
+    result_count integer not null default 0,
+    started_at text not null,
+    completed_at text not null,
+    raw_meta_json text not null default '{}'
+  );
+
+  create table if not exists follow_snapshot_members (
+    snapshot_id text not null,
+    profile_id text not null,
+    external_user_id text not null,
+    position integer not null,
+    primary key (snapshot_id, profile_id)
+  );
+
+  create table if not exists follow_edges (
+    account_id text not null,
+    direction text not null,
+    profile_id text not null,
+    external_user_id text not null,
+    source text not null,
+    current integer not null default 1,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    ended_at text,
+    updated_at text not null,
+    primary key (account_id, direction, profile_id)
+  );
+
+  create table if not exists follow_events (
+    id text primary key,
+    account_id text not null,
+    direction text not null,
+    profile_id text not null,
+    external_user_id text not null,
+    kind text not null,
+    event_at text not null,
+    snapshot_id text not null
+  );
+
   create virtual table if not exists tweets_fts using fts5(
     tweet_id unindexed,
     text
@@ -473,16 +574,19 @@ const BASE_SCHEMA_SQL = `
 
 const INDEX_SQL = `
   create index if not exists idx_tweets_kind_created on tweets(kind, created_at desc);
+  create index if not exists idx_tweets_created on tweets(created_at desc);
   create index if not exists idx_tweets_account_created on tweets(account_id, created_at desc);
   create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
   create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
   create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
   create index if not exists idx_tweet_account_edges_kind_account on tweet_account_edges(kind, account_id, last_seen_at desc, tweet_id);
+  create index if not exists idx_tweet_account_edges_kind_tweet on tweet_account_edges(kind, tweet_id, account_id);
   create index if not exists idx_tweet_account_edges_tweet on tweet_account_edges(tweet_id);
   create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
   create index if not exists idx_profiles_followers on profiles(followers_count desc);
   create index if not exists idx_profiles_following on profiles(following_count desc);
+  create index if not exists idx_profiles_handle on profiles(handle);
   create index if not exists idx_profile_affiliations_subject on profile_affiliations(subject_profile_id, is_active, last_seen_at desc);
   create index if not exists idx_profile_affiliations_org on profile_affiliations(organization_profile_id, is_active, last_seen_at desc);
   create index if not exists idx_profile_snapshots_profile on profile_snapshots(profile_id, last_seen_at desc);
@@ -501,6 +605,10 @@ const INDEX_SQL = `
   create index if not exists idx_link_occurrences_created on link_occurrences(created_at desc);
   create index if not exists idx_link_occurrences_account on link_occurrences(account_id, created_at desc);
   create index if not exists idx_link_occurrences_direction on link_occurrences(direction, created_at desc);
+  create index if not exists idx_follow_edges_current on follow_edges(account_id, direction, current, last_seen_at desc);
+  create index if not exists idx_follow_edges_profile on follow_edges(profile_id, current);
+  create index if not exists idx_follow_snapshots_account on follow_snapshots(account_id, direction, completed_at desc);
+  create index if not exists idx_follow_events_account on follow_events(account_id, direction, kind, event_at desc);
 `;
 
 function getColumnNames(db: Database, tableName: string): Set<string> {
@@ -556,6 +664,11 @@ function ensureProfileAvatarColumns(db: Database) {
 			"alter table profiles add column raw_json text not null default '{}'",
 		);
 	}
+	if (!columnNames.has("public_metrics_json")) {
+		db.exec(
+			"alter table profiles add column public_metrics_json text not null default '{}'",
+		);
+	}
 }
 
 function ensureAccountExternalUserIdColumn(db: Database) {
@@ -683,6 +796,8 @@ function ensureLinkIndexTables(db: Database) {
       expanded_handle text,
       title text,
       description text,
+      image_url text,
+      site_name text,
       error text,
       source text not null,
       updated_at text not null
@@ -700,6 +815,64 @@ function ensureLinkIndexTables(db: Database) {
       primary key (source_kind, source_id, source_position, short_url)
     );
   `);
+
+	const urlExpansionColumns = getColumnNames(db, "url_expansions");
+	if (!urlExpansionColumns.has("image_url")) {
+		db.exec("alter table url_expansions add column image_url text");
+	}
+	if (!urlExpansionColumns.has("site_name")) {
+		db.exec("alter table url_expansions add column site_name text");
+	}
+}
+
+function ensureFollowGraphTables(db: Database) {
+	db.exec(`
+    create table if not exists follow_snapshots (
+      id text primary key,
+      account_id text not null,
+      direction text not null,
+      source text not null,
+      status text not null,
+      page_count integer not null default 0,
+      result_count integer not null default 0,
+      started_at text not null,
+      completed_at text not null,
+      raw_meta_json text not null default '{}'
+    );
+
+    create table if not exists follow_snapshot_members (
+      snapshot_id text not null,
+      profile_id text not null,
+      external_user_id text not null,
+      position integer not null,
+      primary key (snapshot_id, profile_id)
+    );
+
+    create table if not exists follow_edges (
+      account_id text not null,
+      direction text not null,
+      profile_id text not null,
+      external_user_id text not null,
+      source text not null,
+      current integer not null default 1,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      ended_at text,
+      updated_at text not null,
+      primary key (account_id, direction, profile_id)
+    );
+
+    create table if not exists follow_events (
+      id text primary key,
+      account_id text not null,
+      direction text not null,
+      profile_id text not null,
+      external_user_id text not null,
+      kind text not null,
+      event_at text not null,
+      snapshot_id text not null
+    );
+	`);
 }
 
 function backfillTweetCollections(db: Database) {
@@ -749,6 +922,17 @@ function ensureSchemaIndexes(db: Database) {
 	db.exec(INDEX_SQL);
 }
 
+function ensureDemoData(db: Database) {
+	if (demoSeedAttempted) {
+		return;
+	}
+
+	seedDemoData(db);
+	backfillTweetCollections(db);
+	backfillTweetAccountEdges(db);
+	demoSeedAttempted = true;
+}
+
 function initDatabase(options: InitDatabaseOptions = {}) {
 	ensureBirdclawDirs();
 
@@ -766,12 +950,16 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 		ensureProfileBioEntitiesTable(nativeDb);
 		ensureIdentitySearchIndexTable(nativeDb);
 		ensureLinkIndexTables(nativeDb);
+		ensureFollowGraphTables(nativeDb);
 		ensureSchemaIndexes(nativeDb);
 		if (options.seedDemoData !== false) {
-			seedDemoData(nativeDb);
+			ensureDemoData(nativeDb);
+		} else {
+			backfillTweetCollections(nativeDb);
+			backfillTweetAccountEdges(nativeDb);
 		}
-		backfillTweetCollections(nativeDb);
-		backfillTweetAccountEdges(nativeDb);
+	} else if (options.seedDemoData !== false) {
+		ensureDemoData(nativeDb);
 	}
 
 	if (!kyselyDb) {
@@ -799,4 +987,5 @@ export function resetDatabaseForTests() {
 
 	nativeDb?.close();
 	nativeDb = undefined;
+	demoSeedAttempted = false;
 }