birdclaw 0.3.0 → 0.4.1

package/src/lib/db.ts

@@ -1,4 +1,4 @@
-import BetterSqlite3 from "better-sqlite3";
+import NativeSqliteDatabase, { type Database } from "./sqlite";
 import { Kysely, SqliteDialect } from "kysely";
 import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
 import { seedDemoData } from "./seed";
@@ -22,9 +22,69 @@ export interface ProfilesTable {
 	following_count: number;
 	avatar_hue: number;
 	avatar_url: string | null;
+	location: string | null;
+	url: string | null;
+	verified_type: string | null;
+	entities_json: string;
+	raw_json: string;
 	created_at: string;
 }
 
+export interface ProfileAffiliationsTable {
+	subject_profile_id: string;
+	organization_profile_id: string;
+	organization_name: string | null;
+	organization_handle: string | null;
+	badge_url: string | null;
+	url: string | null;
+	label: string | null;
+	source: string;
+	is_active: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	raw_json: string;
+	updated_at: string;
+}
+
+export interface ProfileSnapshotsTable {
+	profile_id: string;
+	snapshot_hash: string;
+	observed_at: string;
+	last_seen_at: string;
+	source: string;
+	handle: string;
+	display_name: string;
+	bio: string;
+	location: string | null;
+	url: string | null;
+	verified_type: string | null;
+	followers_count: number;
+	following_count: number;
+	affiliations_json: string;
+	raw_json: string;
+}
+
+export interface ProfileBioEntitiesTable {
+	profile_id: string;
+	kind: string;
+	value: string;
+	source: string;
+	is_active: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	raw_json: string;
+}
+
+export interface IdentitySearchIndexTable {
+	profile_id: string;
+	kind: string;
+	value: string;
+	normalized_value: string;
+	source: string;
+	weight: number;
+	updated_at: string;
+}
+
 export interface TweetsTable {
 	id: string;
 	account_id: string;
@@ -53,6 +113,18 @@ export interface TweetCollectionsTable {
 	updated_at: string;
 }
 
+export interface TweetAccountEdgesTable {
+	account_id: string;
+	tweet_id: string;
+	kind: "home" | "mention";
+	first_seen_at: string;
+	last_seen_at: string;
+	seen_count: number;
+	source: string;
+	raw_json: string;
+	updated_at: string;
+}
+
 export interface DmConversationsTable {
 	id: string;
 	account_id: string;
@@ -113,11 +185,41 @@ export interface SyncCacheTable {
 	updated_at: string;
 }
 
+export interface UrlExpansionsTable {
+	short_url: string;
+	expanded_url: string;
+	final_url: string;
+	status: string;
+	expanded_tweet_id: string | null;
+	expanded_handle: string | null;
+	title: string | null;
+	description: string | null;
+	error: string | null;
+	source: string;
+	updated_at: string;
+}
+
+export interface LinkOccurrencesTable {
+	source_kind: "dm" | "tweet";
+	source_id: string;
+	source_position: number;
+	short_url: string;
+	account_id: string | null;
+	conversation_id: string | null;
+	direction: string | null;
+	created_at: string;
+}
+
 export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
+	profile_affiliations: ProfileAffiliationsTable;
+	profile_snapshots: ProfileSnapshotsTable;
+	profile_bio_entities: ProfileBioEntitiesTable;
+	identity_search_index: IdentitySearchIndexTable;
 	tweets: TweetsTable;
 	tweet_collections: TweetCollectionsTable;
+	tweet_account_edges: TweetAccountEdgesTable;
 	dm_conversations: DmConversationsTable;
 	dm_messages: DmMessagesTable;
 	tweet_actions: TweetActionsTable;
@@ -125,9 +227,11 @@ export interface BirdclawDatabase {
 	mutes: MutesTable;
 	ai_scores: AiScoresTable;
 	sync_cache: SyncCacheTable;
+	url_expansions: UrlExpansionsTable;
+	link_occurrences: LinkOccurrencesTable;
 }
 
-let nativeDb: BetterSqlite3.Database | undefined;
+let nativeDb: Database | undefined;
 let kyselyDb: Kysely<BirdclawDatabase> | undefined;
 
 export interface InitDatabaseOptions {
@@ -158,9 +262,73 @@ const BASE_SCHEMA_SQL = `
     following_count integer not null default 0,
     avatar_hue integer not null default 0,
     avatar_url text,
+    location text,
+    url text,
+    verified_type text,
+    entities_json text not null default '{}',
+    raw_json text not null default '{}',
     created_at text not null
   );
 
+  create table if not exists profile_affiliations (
+    subject_profile_id text not null,
+    organization_profile_id text not null,
+    organization_name text,
+    organization_handle text,
+    badge_url text,
+    url text,
+    label text,
+    source text not null,
+    is_active integer not null default 1,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    raw_json text not null default '{}',
+    updated_at text not null,
+    primary key (subject_profile_id, organization_profile_id)
+  );
+
+  create table if not exists profile_snapshots (
+    profile_id text not null,
+    snapshot_hash text not null,
+    observed_at text not null,
+    last_seen_at text not null,
+    source text not null,
+    handle text not null,
+    display_name text not null,
+    bio text not null,
+    location text,
+    url text,
+    verified_type text,
+    followers_count integer not null default 0,
+    following_count integer not null default 0,
+    affiliations_json text not null default '[]',
+    raw_json text not null default '{}',
+    primary key (profile_id, snapshot_hash)
+  );
+
+  create table if not exists profile_bio_entities (
+    profile_id text not null,
+    kind text not null,
+    value text not null,
+    source text not null,
+    is_active integer not null default 1,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    raw_json text not null default '{}',
+    primary key (profile_id, kind, value)
+  );
+
+  create table if not exists identity_search_index (
+    profile_id text not null,
+    kind text not null,
+    value text not null,
+    normalized_value text not null,
+    source text not null,
+    weight integer not null,
+    updated_at text not null,
+    primary key (profile_id, kind, value, source)
+  );
+
   create table if not exists tweets (
     id text primary key,
     account_id text not null,
@@ -190,6 +358,19 @@ const BASE_SCHEMA_SQL = `
     primary key (account_id, tweet_id, kind)
   );
 
+  create table if not exists tweet_account_edges (
+    account_id text not null,
+    tweet_id text not null,
+    kind text not null,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    seen_count integer not null default 1,
+    source text not null,
+    raw_json text not null default '{}',
+    updated_at text not null,
+    primary key (account_id, tweet_id, kind)
+  );
+
   create table if not exists dm_conversations (
     id text primary key,
     account_id text not null,
@@ -253,6 +434,32 @@ const BASE_SCHEMA_SQL = `
     updated_at text not null
   );
 
+  create table if not exists url_expansions (
+    short_url text primary key,
+    expanded_url text not null,
+    final_url text not null,
+    status text not null,
+    expanded_tweet_id text,
+    expanded_handle text,
+    title text,
+    description text,
+    error text,
+    source text not null,
+    updated_at text not null
+  );
+
+  create table if not exists link_occurrences (
+    source_kind text not null,
+    source_id text not null,
+    source_position integer not null,
+    short_url text not null,
+    account_id text,
+    conversation_id text,
+    direction text,
+    created_at text not null,
+    primary key (source_kind, source_id, source_position, short_url)
+  );
+
   create virtual table if not exists tweets_fts using fts5(
     tweet_id unindexed,
     text
@@ -270,27 +477,40 @@ const INDEX_SQL = `
   create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
   create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
   create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
+  create index if not exists idx_tweet_account_edges_kind_account on tweet_account_edges(kind, account_id, last_seen_at desc, tweet_id);
+  create index if not exists idx_tweet_account_edges_tweet on tweet_account_edges(tweet_id);
   create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
   create index if not exists idx_profiles_followers on profiles(followers_count desc);
   create index if not exists idx_profiles_following on profiles(following_count desc);
+  create index if not exists idx_profile_affiliations_subject on profile_affiliations(subject_profile_id, is_active, last_seen_at desc);
+  create index if not exists idx_profile_affiliations_org on profile_affiliations(organization_profile_id, is_active, last_seen_at desc);
+  create index if not exists idx_profile_snapshots_profile on profile_snapshots(profile_id, last_seen_at desc);
+  create index if not exists idx_profile_bio_entities_profile on profile_bio_entities(profile_id, is_active, last_seen_at desc);
+  create index if not exists idx_profile_bio_entities_value on profile_bio_entities(kind, value, is_active);
+  create index if not exists idx_identity_search_index_profile on identity_search_index(profile_id);
+  create index if not exists idx_identity_search_index_value on identity_search_index(normalized_value, kind, weight desc);
   create index if not exists idx_blocks_account_created on blocks(account_id, created_at desc);
   create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
   create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
   create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
+  create index if not exists idx_url_expansions_expanded on url_expansions(expanded_url);
+  create index if not exists idx_url_expansions_tweet on url_expansions(expanded_tweet_id);
+  create index if not exists idx_url_expansions_handle on url_expansions(expanded_handle);
+  create index if not exists idx_link_occurrences_url on link_occurrences(short_url);
+  create index if not exists idx_link_occurrences_created on link_occurrences(created_at desc);
+  create index if not exists idx_link_occurrences_account on link_occurrences(account_id, created_at desc);
+  create index if not exists idx_link_occurrences_direction on link_occurrences(direction, created_at desc);
 `;
 
-function getColumnNames(
-	db: BetterSqlite3.Database,
-	tableName: string,
-): Set<string> {
+function getColumnNames(db: Database, tableName: string): Set<string> {
 	const rows = db.prepare(`pragma table_info(${tableName})`).all() as Array<{
 		name: string;
 	}>;
 	return new Set(rows.map((row) => row.name));
 }
 
-function ensureTweetMetadataColumns(db: BetterSqlite3.Database) {
+function ensureTweetMetadataColumns(db: Database) {
 	const columnNames = getColumnNames(db, "tweets");
 	if (!columnNames.has("entities_json")) {
 		db.exec(
@@ -307,7 +527,7 @@ function ensureTweetMetadataColumns(db: BetterSqlite3.Database) {
 	}
 }
 
-function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
+function ensureProfileAvatarColumns(db: Database) {
 	const columnNames = getColumnNames(db, "profiles");
 	if (!columnNames.has("following_count")) {
 		db.exec(
@@ -317,16 +537,35 @@ function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
 	if (!columnNames.has("avatar_url")) {
 		db.exec("alter table profiles add column avatar_url text");
 	}
+	if (!columnNames.has("location")) {
+		db.exec("alter table profiles add column location text");
+	}
+	if (!columnNames.has("url")) {
+		db.exec("alter table profiles add column url text");
+	}
+	if (!columnNames.has("verified_type")) {
+		db.exec("alter table profiles add column verified_type text");
+	}
+	if (!columnNames.has("entities_json")) {
+		db.exec(
+			"alter table profiles add column entities_json text not null default '{}'",
+		);
+	}
+	if (!columnNames.has("raw_json")) {
+		db.exec(
+			"alter table profiles add column raw_json text not null default '{}'",
+		);
+	}
 }
 
-function ensureAccountExternalUserIdColumn(db: BetterSqlite3.Database) {
+function ensureAccountExternalUserIdColumn(db: Database) {
 	const columnNames = getColumnNames(db, "accounts");
 	if (!columnNames.has("external_user_id")) {
 		db.exec("alter table accounts add column external_user_id text");
 	}
 }
 
-function ensureTweetCollectionsTable(db: BetterSqlite3.Database) {
+function ensureTweetCollectionsTable(db: Database) {
 	db.exec(`
     create table if not exists tweet_collections (
       account_id text not null,
@@ -341,7 +580,129 @@ function ensureTweetCollectionsTable(db: BetterSqlite3.Database) {
   `);
 }
 
-function backfillTweetCollections(db: BetterSqlite3.Database) {
+function ensureTweetAccountEdgesTable(db: Database) {
+	db.exec(`
+    create table if not exists tweet_account_edges (
+      account_id text not null,
+      tweet_id text not null,
+      kind text not null,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      seen_count integer not null default 1,
+      source text not null,
+      raw_json text not null default '{}',
+      updated_at text not null,
+      primary key (account_id, tweet_id, kind)
+    );
+  `);
+}
+
+function ensureProfileAffiliationsTable(db: Database) {
+	db.exec(`
+    create table if not exists profile_affiliations (
+      subject_profile_id text not null,
+      organization_profile_id text not null,
+      organization_name text,
+      organization_handle text,
+      badge_url text,
+      url text,
+      label text,
+      source text not null,
+      is_active integer not null default 1,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      raw_json text not null default '{}',
+      updated_at text not null,
+      primary key (subject_profile_id, organization_profile_id)
+    );
+  `);
+}
+
+function ensureProfileSnapshotsTable(db: Database) {
+	db.exec(`
+    create table if not exists profile_snapshots (
+      profile_id text not null,
+      snapshot_hash text not null,
+      observed_at text not null,
+      last_seen_at text not null,
+      source text not null,
+      handle text not null,
+      display_name text not null,
+      bio text not null,
+      location text,
+      url text,
+      verified_type text,
+      followers_count integer not null default 0,
+      following_count integer not null default 0,
+      affiliations_json text not null default '[]',
+      raw_json text not null default '{}',
+      primary key (profile_id, snapshot_hash)
+    );
+  `);
+}
+
+function ensureProfileBioEntitiesTable(db: Database) {
+	db.exec(`
+    create table if not exists profile_bio_entities (
+      profile_id text not null,
+      kind text not null,
+      value text not null,
+      source text not null,
+      is_active integer not null default 1,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      raw_json text not null default '{}',
+      primary key (profile_id, kind, value)
+    );
+  `);
+}
+
+function ensureIdentitySearchIndexTable(db: Database) {
+	db.exec(`
+    create table if not exists identity_search_index (
+      profile_id text not null,
+      kind text not null,
+      value text not null,
+      normalized_value text not null,
+      source text not null,
+      weight integer not null,
+      updated_at text not null,
+      primary key (profile_id, kind, value, source)
+    );
+  `);
+}
+
+function ensureLinkIndexTables(db: Database) {
+	db.exec(`
+    create table if not exists url_expansions (
+      short_url text primary key,
+      expanded_url text not null,
+      final_url text not null,
+      status text not null,
+      expanded_tweet_id text,
+      expanded_handle text,
+      title text,
+      description text,
+      error text,
+      source text not null,
+      updated_at text not null
+    );
+
+    create table if not exists link_occurrences (
+      source_kind text not null,
+      source_id text not null,
+      source_position integer not null,
+      short_url text not null,
+      account_id text,
+      conversation_id text,
+      direction text,
+      created_at text not null,
+      primary key (source_kind, source_id, source_position, short_url)
+    );
+  `);
+}
+
+function backfillTweetCollections(db: Database) {
 	const now = new Date().toISOString();
 	const insert = db.prepare(`
     insert or ignore into tweet_collections (
@@ -362,7 +723,29 @@ function backfillTweetCollections(db: BetterSqlite3.Database) {
 	})();
 }
 
-function ensureSchemaIndexes(db: BetterSqlite3.Database) {
+function backfillTweetAccountEdges(db: Database) {
+	const now = new Date().toISOString();
+	db.prepare(`
+    insert or ignore into tweet_account_edges (
+      account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
+      source, raw_json, updated_at
+    )
+    select
+      account_id,
+      id,
+      kind,
+      created_at,
+      created_at,
+      1,
+      'legacy',
+      '{}',
+      ?
+    from tweets
+    where kind in ('home', 'mention')
+  `).run(now);
+}
+
+function ensureSchemaIndexes(db: Database) {
 	db.exec(INDEX_SQL);
 }
 
@@ -371,17 +754,24 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 
 	if (!nativeDb) {
 		const { dbPath } = getBirdclawPaths();
-		nativeDb = new BetterSqlite3(dbPath);
+		nativeDb = new NativeSqliteDatabase(dbPath);
 		nativeDb.exec(BASE_SCHEMA_SQL);
 		ensureAccountExternalUserIdColumn(nativeDb);
 		ensureTweetMetadataColumns(nativeDb);
 		ensureProfileAvatarColumns(nativeDb);
 		ensureTweetCollectionsTable(nativeDb);
+		ensureTweetAccountEdgesTable(nativeDb);
+		ensureProfileAffiliationsTable(nativeDb);
+		ensureProfileSnapshotsTable(nativeDb);
+		ensureProfileBioEntitiesTable(nativeDb);
+		ensureIdentitySearchIndexTable(nativeDb);
+		ensureLinkIndexTables(nativeDb);
 		ensureSchemaIndexes(nativeDb);
 		if (options.seedDemoData !== false) {
 			seedDemoData(nativeDb);
 		}
 		backfillTweetCollections(nativeDb);
+		backfillTweetAccountEdges(nativeDb);
 	}
 
 	if (!kyselyDb) {
@@ -395,7 +785,7 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 
 export function getNativeDb(options: InitDatabaseOptions = {}) {
 	initDatabase(options);
-	return nativeDb as BetterSqlite3.Database;
+	return nativeDb as Database;
 }
 
 export function getDb() {

package/src/lib/dms-live.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import {
 	type BirdDmConversation,
 	type BirdDmEvent,
@@ -25,7 +25,7 @@ function assertBirdLimit(limit: number) {
 	}
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db
 				.prepare("select id, handle from accounts where id = ?")
@@ -112,7 +112,7 @@ function getLatestEvent(events: BirdDmEvent[]) {
 }
 
 function mergeDirectMessagesIntoLocalStore(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	accountUsername: string,
 	payload: {