birdclaw 0.4.0 → 0.5.0

package/src/lib/db.ts

@@ -20,6 +20,7 @@ export interface ProfilesTable {
 	bio: string;
 	followers_count: number;
 	following_count: number;
+	public_metrics_json: string;
 	avatar_hue: number;
 	avatar_url: string | null;
 	location: string | null;
@@ -116,7 +117,7 @@ export interface TweetCollectionsTable {
 export interface TweetAccountEdgesTable {
 	account_id: string;
 	tweet_id: string;
-	kind: "home" | "mention";
+	kind: "home" | "mention" | "authored" | "thread_context";
 	first_seen_at: string;
 	last_seen_at: string;
 	seen_count: number;
@@ -185,6 +186,77 @@ export interface SyncCacheTable {
 	updated_at: string;
 }
 
+export interface UrlExpansionsTable {
+	short_url: string;
+	expanded_url: string;
+	final_url: string;
+	status: string;
+	expanded_tweet_id: string | null;
+	expanded_handle: string | null;
+	title: string | null;
+	description: string | null;
+	image_url: string | null;
+	site_name: string | null;
+	error: string | null;
+	source: string;
+	updated_at: string;
+}
+
+export interface LinkOccurrencesTable {
+	source_kind: "dm" | "tweet";
+	source_id: string;
+	source_position: number;
+	short_url: string;
+	account_id: string | null;
+	conversation_id: string | null;
+	direction: string | null;
+	created_at: string;
+}
+
+export interface FollowEdgesTable {
+	account_id: string;
+	direction: "followers" | "following";
+	profile_id: string;
+	external_user_id: string;
+	source: string;
+	current: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	ended_at: string | null;
+	updated_at: string;
+}
+
+export interface FollowSnapshotsTable {
+	id: string;
+	account_id: string;
+	direction: "followers" | "following";
+	source: string;
+	status: "complete" | "incomplete";
+	page_count: number;
+	result_count: number;
+	started_at: string;
+	completed_at: string;
+	raw_meta_json: string;
+}
+
+export interface FollowSnapshotMembersTable {
+	snapshot_id: string;
+	profile_id: string;
+	external_user_id: string;
+	position: number;
+}
+
+export interface FollowEventsTable {
+	id: string;
+	account_id: string;
+	direction: "followers" | "following";
+	profile_id: string;
+	external_user_id: string;
+	kind: "started" | "ended";
+	event_at: string;
+	snapshot_id: string;
+}
+
 export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
@@ -202,10 +274,17 @@ export interface BirdclawDatabase {
 	mutes: MutesTable;
 	ai_scores: AiScoresTable;
 	sync_cache: SyncCacheTable;
+	url_expansions: UrlExpansionsTable;
+	link_occurrences: LinkOccurrencesTable;
+	follow_edges: FollowEdgesTable;
+	follow_snapshots: FollowSnapshotsTable;
+	follow_snapshot_members: FollowSnapshotMembersTable;
+	follow_events: FollowEventsTable;
 }
 
 let nativeDb: Database | undefined;
 let kyselyDb: Kysely<BirdclawDatabase> | undefined;
+let demoSeedAttempted = false;
 
 export interface InitDatabaseOptions {
 	seedDemoData?: boolean;
@@ -233,6 +312,7 @@ const BASE_SCHEMA_SQL = `
     bio text not null,
     followers_count integer not null default 0,
     following_count integer not null default 0,
+    public_metrics_json text not null default '{}',
     avatar_hue integer not null default 0,
     avatar_url text,
     location text,
@@ -407,6 +487,80 @@ const BASE_SCHEMA_SQL = `
     updated_at text not null
   );
 
+  create table if not exists url_expansions (
+    short_url text primary key,
+    expanded_url text not null,
+    final_url text not null,
+    status text not null,
+    expanded_tweet_id text,
+    expanded_handle text,
+    title text,
+    description text,
+    image_url text,
+    site_name text,
+    error text,
+    source text not null,
+    updated_at text not null
+  );
+
+  create table if not exists link_occurrences (
+    source_kind text not null,
+    source_id text not null,
+    source_position integer not null,
+    short_url text not null,
+    account_id text,
+    conversation_id text,
+    direction text,
+    created_at text not null,
+    primary key (source_kind, source_id, source_position, short_url)
+  );
+
+  create table if not exists follow_snapshots (
+    id text primary key,
+    account_id text not null,
+    direction text not null,
+    source text not null,
+    status text not null,
+    page_count integer not null default 0,
+    result_count integer not null default 0,
+    started_at text not null,
+    completed_at text not null,
+    raw_meta_json text not null default '{}'
+  );
+
+  create table if not exists follow_snapshot_members (
+    snapshot_id text not null,
+    profile_id text not null,
+    external_user_id text not null,
+    position integer not null,
+    primary key (snapshot_id, profile_id)
+  );
+
+  create table if not exists follow_edges (
+    account_id text not null,
+    direction text not null,
+    profile_id text not null,
+    external_user_id text not null,
+    source text not null,
+    current integer not null default 1,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    ended_at text,
+    updated_at text not null,
+    primary key (account_id, direction, profile_id)
+  );
+
+  create table if not exists follow_events (
+    id text primary key,
+    account_id text not null,
+    direction text not null,
+    profile_id text not null,
+    external_user_id text not null,
+    kind text not null,
+    event_at text not null,
+    snapshot_id text not null
+  );
+
   create virtual table if not exists tweets_fts using fts5(
     tweet_id unindexed,
     text
@@ -430,6 +584,7 @@ const INDEX_SQL = `
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
   create index if not exists idx_profiles_followers on profiles(followers_count desc);
   create index if not exists idx_profiles_following on profiles(following_count desc);
+  create index if not exists idx_profiles_handle on profiles(handle);
   create index if not exists idx_profile_affiliations_subject on profile_affiliations(subject_profile_id, is_active, last_seen_at desc);
   create index if not exists idx_profile_affiliations_org on profile_affiliations(organization_profile_id, is_active, last_seen_at desc);
   create index if not exists idx_profile_snapshots_profile on profile_snapshots(profile_id, last_seen_at desc);
@@ -441,6 +596,17 @@ const INDEX_SQL = `
   create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
   create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
   create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
+  create index if not exists idx_url_expansions_expanded on url_expansions(expanded_url);
+  create index if not exists idx_url_expansions_tweet on url_expansions(expanded_tweet_id);
+  create index if not exists idx_url_expansions_handle on url_expansions(expanded_handle);
+  create index if not exists idx_link_occurrences_url on link_occurrences(short_url);
+  create index if not exists idx_link_occurrences_created on link_occurrences(created_at desc);
+  create index if not exists idx_link_occurrences_account on link_occurrences(account_id, created_at desc);
+  create index if not exists idx_link_occurrences_direction on link_occurrences(direction, created_at desc);
+  create index if not exists idx_follow_edges_current on follow_edges(account_id, direction, current, last_seen_at desc);
+  create index if not exists idx_follow_edges_profile on follow_edges(profile_id, current);
+  create index if not exists idx_follow_snapshots_account on follow_snapshots(account_id, direction, completed_at desc);
+  create index if not exists idx_follow_events_account on follow_events(account_id, direction, kind, event_at desc);
 `;
 
 function getColumnNames(db: Database, tableName: string): Set<string> {
@@ -496,6 +662,11 @@ function ensureProfileAvatarColumns(db: Database) {
 			"alter table profiles add column raw_json text not null default '{}'",
 		);
 	}
+	if (!columnNames.has("public_metrics_json")) {
+		db.exec(
+			"alter table profiles add column public_metrics_json text not null default '{}'",
+		);
+	}
 }
 
 function ensureAccountExternalUserIdColumn(db: Database) {
@@ -612,6 +783,96 @@ function ensureIdentitySearchIndexTable(db: Database) {
   `);
 }
 
+function ensureLinkIndexTables(db: Database) {
+	db.exec(`
+    create table if not exists url_expansions (
+      short_url text primary key,
+      expanded_url text not null,
+      final_url text not null,
+      status text not null,
+      expanded_tweet_id text,
+      expanded_handle text,
+      title text,
+      description text,
+      image_url text,
+      site_name text,
+      error text,
+      source text not null,
+      updated_at text not null
+    );
+
+    create table if not exists link_occurrences (
+      source_kind text not null,
+      source_id text not null,
+      source_position integer not null,
+      short_url text not null,
+      account_id text,
+      conversation_id text,
+      direction text,
+      created_at text not null,
+      primary key (source_kind, source_id, source_position, short_url)
+    );
+  `);
+
+	const urlExpansionColumns = getColumnNames(db, "url_expansions");
+	if (!urlExpansionColumns.has("image_url")) {
+		db.exec("alter table url_expansions add column image_url text");
+	}
+	if (!urlExpansionColumns.has("site_name")) {
+		db.exec("alter table url_expansions add column site_name text");
+	}
+}
+
+function ensureFollowGraphTables(db: Database) {
+	db.exec(`
+    create table if not exists follow_snapshots (
+      id text primary key,
+      account_id text not null,
+      direction text not null,
+      source text not null,
+      status text not null,
+      page_count integer not null default 0,
+      result_count integer not null default 0,
+      started_at text not null,
+      completed_at text not null,
+      raw_meta_json text not null default '{}'
+    );
+
+    create table if not exists follow_snapshot_members (
+      snapshot_id text not null,
+      profile_id text not null,
+      external_user_id text not null,
+      position integer not null,
+      primary key (snapshot_id, profile_id)
+    );
+
+    create table if not exists follow_edges (
+      account_id text not null,
+      direction text not null,
+      profile_id text not null,
+      external_user_id text not null,
+      source text not null,
+      current integer not null default 1,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      ended_at text,
+      updated_at text not null,
+      primary key (account_id, direction, profile_id)
+    );
+
+    create table if not exists follow_events (
+      id text primary key,
+      account_id text not null,
+      direction text not null,
+      profile_id text not null,
+      external_user_id text not null,
+      kind text not null,
+      event_at text not null,
+      snapshot_id text not null
+    );
+	`);
+}
+
 function backfillTweetCollections(db: Database) {
 	const now = new Date().toISOString();
 	const insert = db.prepare(`
@@ -659,6 +920,17 @@ function ensureSchemaIndexes(db: Database) {
 	db.exec(INDEX_SQL);
 }
 
+function ensureDemoData(db: Database) {
+	if (demoSeedAttempted) {
+		return;
+	}
+
+	seedDemoData(db);
+	backfillTweetCollections(db);
+	backfillTweetAccountEdges(db);
+	demoSeedAttempted = true;
+}
+
 function initDatabase(options: InitDatabaseOptions = {}) {
 	ensureBirdclawDirs();
 
@@ -675,12 +947,17 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 		ensureProfileSnapshotsTable(nativeDb);
 		ensureProfileBioEntitiesTable(nativeDb);
 		ensureIdentitySearchIndexTable(nativeDb);
+		ensureLinkIndexTables(nativeDb);
+		ensureFollowGraphTables(nativeDb);
 		ensureSchemaIndexes(nativeDb);
 		if (options.seedDemoData !== false) {
-			seedDemoData(nativeDb);
+			ensureDemoData(nativeDb);
+		} else {
+			backfillTweetCollections(nativeDb);
+			backfillTweetAccountEdges(nativeDb);
 		}
-		backfillTweetCollections(nativeDb);
-		backfillTweetAccountEdges(nativeDb);
+	} else if (options.seedDemoData !== false) {
+		ensureDemoData(nativeDb);
 	}
 
 	if (!kyselyDb) {
@@ -708,4 +985,5 @@ export function resetDatabaseForTests() {
 
 	nativeDb?.close();
 	nativeDb = undefined;
+	demoSeedAttempted = false;
 }