birdclaw 0.3.0 → 0.4.0

package/src/lib/bird.ts

@@ -73,6 +73,31 @@ export interface BirdDmsResponse {
 	events: BirdDmEvent[];
 }
 
+interface BirdUserOverviewPayload {
+	user?: {
+		id?: string;
+		username?: string;
+		name?: string;
+		description?: string;
+		location?: string;
+		url?: string;
+		verified?: boolean;
+		verifiedType?: string;
+		verified_type?: string;
+		followersCount?: number;
+		followingCount?: number;
+		profileImageUrl?: string;
+		createdAt?: string;
+		entities?: Record<string, unknown>;
+		affiliation?: Record<string, unknown>;
+	};
+}
+
+interface BirdProfilesPayload {
+	users?: NonNullable<BirdUserOverviewPayload["user"]>[];
+	errors?: Array<{ target?: string; error?: string }>;
+}
+
 function toIsoTimestamp(value: string) {
 	const parsed = new Date(value);
 	if (Number.isNaN(parsed.getTime())) {
@@ -161,6 +186,18 @@ function formatBirdCommandError(error: unknown, birdCommand: string) {
 	return error;
 }
 
+function isUnsupportedBirdOptionError(error: unknown, option: string) {
+	if (!error || typeof error !== "object") {
+		return false;
+	}
+	const text = [
+		error instanceof Error ? error.message : "",
+		"stderr" in error && typeof error.stderr === "string" ? error.stderr : "",
+		"stdout" in error && typeof error.stdout === "string" ? error.stdout : "",
+	].join("\n");
+	return text.includes(option) && /unknown option|error:/i.test(text);
+}
+
 async function runBirdJsonCommand(args: string[], timeoutMs?: number) {
 	const tempDir = mkdtempSync(join(tmpdir(), "birdclaw-bird-"));
 	const stdoutPath = join(tempDir, "stdout.json");
@@ -446,3 +483,155 @@ export async function listDirectMessagesViaBird({
 
 	return payload as BirdDmsResponse;
 }
+
+export async function lookupProfileViaBird(
+	usernameOrId: string,
+): Promise<XurlMentionUser | null> {
+	const target = usernameOrId.trim().replace(/^@/, "");
+	if (!target) {
+		return null;
+	}
+
+	let stdout: string;
+	try {
+		stdout = await runBirdJsonCommand([
+			"user",
+			target,
+			"--json",
+			"--profile-only",
+		]);
+	} catch (error) {
+		if (!isUnsupportedBirdOptionError(error, "--profile-only")) {
+			throw error;
+		}
+		stdout = await runBirdJsonCommand([
+			"user",
+			target,
+			"--json",
+			"--count",
+			"1",
+		]);
+	}
+	const payload = parseBirdJson(stdout) as BirdUserOverviewPayload;
+	return toXurlMentionUser(payload.user);
+}
+
+function toXurlMentionUser(
+	user: BirdUserOverviewPayload["user"],
+): XurlMentionUser | null {
+	if (!user?.id || !user.username) {
+		return null;
+	}
+
+	return {
+		id: String(user.id),
+		username: String(user.username).replace(/^@/, ""),
+		name: String(user.name ?? user.username),
+		description:
+			typeof user.description === "string" ? user.description : undefined,
+		location: typeof user.location === "string" ? user.location : undefined,
+		url: typeof user.url === "string" ? user.url : undefined,
+		verified: typeof user.verified === "boolean" ? user.verified : undefined,
+		verified_type:
+			typeof user.verifiedType === "string"
+				? user.verifiedType
+				: typeof user.verified_type === "string"
+					? user.verified_type
+					: undefined,
+		profile_image_url:
+			typeof user.profileImageUrl === "string"
+				? user.profileImageUrl
+				: undefined,
+		entities:
+			user.entities && typeof user.entities === "object"
+				? user.entities
+				: undefined,
+		affiliation:
+			user.affiliation && typeof user.affiliation === "object"
+				? user.affiliation
+				: undefined,
+		created_at: typeof user.createdAt === "string" ? user.createdAt : undefined,
+		public_metrics: {
+			followers_count: Number(user.followersCount ?? 0),
+			following_count: Number(user.followingCount ?? 0),
+		},
+	};
+}
+
+export async function lookupProfilesViaBird(
+	usernameOrIds: string[],
+): Promise<
+	Array<{ target: string; user: XurlMentionUser | null; error?: string }>
+> {
+	const targets = Array.from(
+		new Set(
+			usernameOrIds
+				.map((target) => target.trim().replace(/^@/, ""))
+				.filter((target) => target.length > 0),
+		),
+	);
+	if (targets.length === 0) {
+		return [];
+	}
+
+	try {
+		const stdout = await runBirdJsonCommand(["profiles", ...targets, "--json"]);
+		const payload = parseBirdJson(stdout) as BirdProfilesPayload;
+		const users = (payload.users ?? [])
+			.map(toXurlMentionUser)
+			.filter((user): user is XurlMentionUser => Boolean(user));
+		const byTarget = new Map<string, XurlMentionUser>();
+		for (const user of users) {
+			byTarget.set(String(user.id), user);
+			byTarget.set(user.username.toLowerCase(), user);
+		}
+		const errors = new Map(
+			(payload.errors ?? []).map((item) => [
+				String(item.target ?? "")
+					.replace(/^@/, "")
+					.toLowerCase(),
+				item.error ?? "Unknown error",
+			]),
+		);
+		return targets.map((target) => ({
+			target,
+			user: byTarget.get(target.toLowerCase()) ?? null,
+			...(errors.has(target.toLowerCase())
+				? { error: errors.get(target.toLowerCase()) }
+				: {}),
+		}));
+	} catch (error) {
+		if (!isUnsupportedBirdOptionError(error, "profiles")) {
+			throw error;
+		}
+		return Promise.all(
+			targets.map(async (target) => {
+				try {
+					return { target, user: await lookupProfileViaBird(target) };
+				} catch (lookupError) {
+					return {
+						target,
+						user: null,
+						error:
+							lookupError instanceof Error
+								? lookupError.message
+								: String(lookupError),
+					};
+				}
+			}),
+		);
+	}
+}
+
+export const __test__ = {
+	toIsoTimestamp,
+	escapeJsonStringControlChars,
+	parseBirdJson,
+	formatBirdCommandError,
+	isUnsupportedBirdOptionError,
+	getBirdTweetItems,
+	getBirdTweetItem,
+	toMediaEntities,
+	toReferencedTweets,
+	normalizeBirdTweets,
+};

package/src/lib/blocks.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { getNativeDb } from "./db";
 import {
 	getAccountHandle,
@@ -16,7 +16,7 @@ function remoteBlockSyncDisabled() {
 }
 
 function upsertRemoteBlock(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	profileId: string,
 	blockedAt: string,
@@ -33,7 +33,7 @@ function upsertRemoteBlock(
 }
 
 function pruneRemoteBlocks(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	profileIds: string[],
 ) {

package/src/lib/bookmark-sync-job.ts

@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { maybeAutoSyncBackup, type BackupAutoUpdateResult } from "./backup";
 import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
 import { getNativeDb } from "./db";
@@ -31,7 +31,7 @@ export interface BookmarkSyncJobOptions {
 	cacheTtlMs?: number;
 	logPath?: string;
 	lockPath?: string;
-	db?: Database.Database;
+	db?: Database;
 }
 
 export interface BookmarkSyncAuditEntry {
@@ -117,7 +117,7 @@ export function getDefaultBookmarkSyncLockPath() {
 	return path.join(getBirdclawPaths().rootDir, "locks", "bookmarks-sync.lock");
 }
 
-function countBookmarks(db: Database.Database) {
+function countBookmarks(db: Database) {
 	const row = db
 		.prepare("select count(*) as count from tweet_collections where kind = ?")
 		.get("bookmarks") as { count: number };

package/src/lib/db.ts

@@ -1,4 +1,4 @@
-import BetterSqlite3 from "better-sqlite3";
+import NativeSqliteDatabase, { type Database } from "./sqlite";
 import { Kysely, SqliteDialect } from "kysely";
 import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
 import { seedDemoData } from "./seed";
@@ -22,9 +22,69 @@ export interface ProfilesTable {
 	following_count: number;
 	avatar_hue: number;
 	avatar_url: string | null;
+	location: string | null;
+	url: string | null;
+	verified_type: string | null;
+	entities_json: string;
+	raw_json: string;
 	created_at: string;
 }
 
+export interface ProfileAffiliationsTable {
+	subject_profile_id: string;
+	organization_profile_id: string;
+	organization_name: string | null;
+	organization_handle: string | null;
+	badge_url: string | null;
+	url: string | null;
+	label: string | null;
+	source: string;
+	is_active: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	raw_json: string;
+	updated_at: string;
+}
+
+export interface ProfileSnapshotsTable {
+	profile_id: string;
+	snapshot_hash: string;
+	observed_at: string;
+	last_seen_at: string;
+	source: string;
+	handle: string;
+	display_name: string;
+	bio: string;
+	location: string | null;
+	url: string | null;
+	verified_type: string | null;
+	followers_count: number;
+	following_count: number;
+	affiliations_json: string;
+	raw_json: string;
+}
+
+export interface ProfileBioEntitiesTable {
+	profile_id: string;
+	kind: string;
+	value: string;
+	source: string;
+	is_active: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	raw_json: string;
+}
+
+export interface IdentitySearchIndexTable {
+	profile_id: string;
+	kind: string;
+	value: string;
+	normalized_value: string;
+	source: string;
+	weight: number;
+	updated_at: string;
+}
+
 export interface TweetsTable {
 	id: string;
 	account_id: string;
@@ -53,6 +113,18 @@ export interface TweetCollectionsTable {
 	updated_at: string;
 }
 
+export interface TweetAccountEdgesTable {
+	account_id: string;
+	tweet_id: string;
+	kind: "home" | "mention";
+	first_seen_at: string;
+	last_seen_at: string;
+	seen_count: number;
+	source: string;
+	raw_json: string;
+	updated_at: string;
+}
+
 export interface DmConversationsTable {
 	id: string;
 	account_id: string;
@@ -116,8 +188,13 @@ export interface SyncCacheTable {
 export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
+	profile_affiliations: ProfileAffiliationsTable;
+	profile_snapshots: ProfileSnapshotsTable;
+	profile_bio_entities: ProfileBioEntitiesTable;
+	identity_search_index: IdentitySearchIndexTable;
 	tweets: TweetsTable;
 	tweet_collections: TweetCollectionsTable;
+	tweet_account_edges: TweetAccountEdgesTable;
 	dm_conversations: DmConversationsTable;
 	dm_messages: DmMessagesTable;
 	tweet_actions: TweetActionsTable;
@@ -127,7 +204,7 @@ export interface BirdclawDatabase {
 	sync_cache: SyncCacheTable;
 }
 
-let nativeDb: BetterSqlite3.Database | undefined;
+let nativeDb: Database | undefined;
 let kyselyDb: Kysely<BirdclawDatabase> | undefined;
 
 export interface InitDatabaseOptions {
@@ -158,9 +235,73 @@ const BASE_SCHEMA_SQL = `
     following_count integer not null default 0,
     avatar_hue integer not null default 0,
     avatar_url text,
+    location text,
+    url text,
+    verified_type text,
+    entities_json text not null default '{}',
+    raw_json text not null default '{}',
     created_at text not null
   );
 
+  create table if not exists profile_affiliations (
+    subject_profile_id text not null,
+    organization_profile_id text not null,
+    organization_name text,
+    organization_handle text,
+    badge_url text,
+    url text,
+    label text,
+    source text not null,
+    is_active integer not null default 1,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    raw_json text not null default '{}',
+    updated_at text not null,
+    primary key (subject_profile_id, organization_profile_id)
+  );
+
+  create table if not exists profile_snapshots (
+    profile_id text not null,
+    snapshot_hash text not null,
+    observed_at text not null,
+    last_seen_at text not null,
+    source text not null,
+    handle text not null,
+    display_name text not null,
+    bio text not null,
+    location text,
+    url text,
+    verified_type text,
+    followers_count integer not null default 0,
+    following_count integer not null default 0,
+    affiliations_json text not null default '[]',
+    raw_json text not null default '{}',
+    primary key (profile_id, snapshot_hash)
+  );
+
+  create table if not exists profile_bio_entities (
+    profile_id text not null,
+    kind text not null,
+    value text not null,
+    source text not null,
+    is_active integer not null default 1,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    raw_json text not null default '{}',
+    primary key (profile_id, kind, value)
+  );
+
+  create table if not exists identity_search_index (
+    profile_id text not null,
+    kind text not null,
+    value text not null,
+    normalized_value text not null,
+    source text not null,
+    weight integer not null,
+    updated_at text not null,
+    primary key (profile_id, kind, value, source)
+  );
+
   create table if not exists tweets (
     id text primary key,
     account_id text not null,
@@ -190,6 +331,19 @@ const BASE_SCHEMA_SQL = `
     primary key (account_id, tweet_id, kind)
   );
 
+  create table if not exists tweet_account_edges (
+    account_id text not null,
+    tweet_id text not null,
+    kind text not null,
+    first_seen_at text not null,
+    last_seen_at text not null,
+    seen_count integer not null default 1,
+    source text not null,
+    raw_json text not null default '{}',
+    updated_at text not null,
+    primary key (account_id, tweet_id, kind)
+  );
+
   create table if not exists dm_conversations (
     id text primary key,
     account_id text not null,
@@ -270,27 +424,33 @@ const INDEX_SQL = `
   create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
   create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
   create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
+  create index if not exists idx_tweet_account_edges_kind_account on tweet_account_edges(kind, account_id, last_seen_at desc, tweet_id);
+  create index if not exists idx_tweet_account_edges_tweet on tweet_account_edges(tweet_id);
   create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
   create index if not exists idx_profiles_followers on profiles(followers_count desc);
   create index if not exists idx_profiles_following on profiles(following_count desc);
+  create index if not exists idx_profile_affiliations_subject on profile_affiliations(subject_profile_id, is_active, last_seen_at desc);
+  create index if not exists idx_profile_affiliations_org on profile_affiliations(organization_profile_id, is_active, last_seen_at desc);
+  create index if not exists idx_profile_snapshots_profile on profile_snapshots(profile_id, last_seen_at desc);
+  create index if not exists idx_profile_bio_entities_profile on profile_bio_entities(profile_id, is_active, last_seen_at desc);
+  create index if not exists idx_profile_bio_entities_value on profile_bio_entities(kind, value, is_active);
+  create index if not exists idx_identity_search_index_profile on identity_search_index(profile_id);
+  create index if not exists idx_identity_search_index_value on identity_search_index(normalized_value, kind, weight desc);
   create index if not exists idx_blocks_account_created on blocks(account_id, created_at desc);
   create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
   create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
   create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
 `;
 
-function getColumnNames(
-	db: BetterSqlite3.Database,
-	tableName: string,
-): Set<string> {
+function getColumnNames(db: Database, tableName: string): Set<string> {
 	const rows = db.prepare(`pragma table_info(${tableName})`).all() as Array<{
 		name: string;
 	}>;
 	return new Set(rows.map((row) => row.name));
 }
 
-function ensureTweetMetadataColumns(db: BetterSqlite3.Database) {
+function ensureTweetMetadataColumns(db: Database) {
 	const columnNames = getColumnNames(db, "tweets");
 	if (!columnNames.has("entities_json")) {
 		db.exec(
@@ -307,7 +467,7 @@ function ensureTweetMetadataColumns(db: BetterSqlite3.Database) {
 	}
 }
 
-function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
+function ensureProfileAvatarColumns(db: Database) {
 	const columnNames = getColumnNames(db, "profiles");
 	if (!columnNames.has("following_count")) {
 		db.exec(
@@ -317,16 +477,35 @@ function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
 	if (!columnNames.has("avatar_url")) {
 		db.exec("alter table profiles add column avatar_url text");
 	}
+	if (!columnNames.has("location")) {
+		db.exec("alter table profiles add column location text");
+	}
+	if (!columnNames.has("url")) {
+		db.exec("alter table profiles add column url text");
+	}
+	if (!columnNames.has("verified_type")) {
+		db.exec("alter table profiles add column verified_type text");
+	}
+	if (!columnNames.has("entities_json")) {
+		db.exec(
+			"alter table profiles add column entities_json text not null default '{}'",
+		);
+	}
+	if (!columnNames.has("raw_json")) {
+		db.exec(
+			"alter table profiles add column raw_json text not null default '{}'",
+		);
+	}
 }
 
-function ensureAccountExternalUserIdColumn(db: BetterSqlite3.Database) {
+function ensureAccountExternalUserIdColumn(db: Database) {
 	const columnNames = getColumnNames(db, "accounts");
 	if (!columnNames.has("external_user_id")) {
 		db.exec("alter table accounts add column external_user_id text");
 	}
 }
 
-function ensureTweetCollectionsTable(db: BetterSqlite3.Database) {
+function ensureTweetCollectionsTable(db: Database) {
 	db.exec(`
     create table if not exists tweet_collections (
       account_id text not null,
@@ -341,7 +520,99 @@ function ensureTweetCollectionsTable(db: BetterSqlite3.Database) {
   `);
 }
 
-function backfillTweetCollections(db: BetterSqlite3.Database) {
+function ensureTweetAccountEdgesTable(db: Database) {
+	db.exec(`
+    create table if not exists tweet_account_edges (
+      account_id text not null,
+      tweet_id text not null,
+      kind text not null,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      seen_count integer not null default 1,
+      source text not null,
+      raw_json text not null default '{}',
+      updated_at text not null,
+      primary key (account_id, tweet_id, kind)
+    );
+  `);
+}
+
+function ensureProfileAffiliationsTable(db: Database) {
+	db.exec(`
+    create table if not exists profile_affiliations (
+      subject_profile_id text not null,
+      organization_profile_id text not null,
+      organization_name text,
+      organization_handle text,
+      badge_url text,
+      url text,
+      label text,
+      source text not null,
+      is_active integer not null default 1,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      raw_json text not null default '{}',
+      updated_at text not null,
+      primary key (subject_profile_id, organization_profile_id)
+    );
+  `);
+}
+
+function ensureProfileSnapshotsTable(db: Database) {
+	db.exec(`
+    create table if not exists profile_snapshots (
+      profile_id text not null,
+      snapshot_hash text not null,
+      observed_at text not null,
+      last_seen_at text not null,
+      source text not null,
+      handle text not null,
+      display_name text not null,
+      bio text not null,
+      location text,
+      url text,
+      verified_type text,
+      followers_count integer not null default 0,
+      following_count integer not null default 0,
+      affiliations_json text not null default '[]',
+      raw_json text not null default '{}',
+      primary key (profile_id, snapshot_hash)
+    );
+  `);
+}
+
+function ensureProfileBioEntitiesTable(db: Database) {
+	db.exec(`
+    create table if not exists profile_bio_entities (
+      profile_id text not null,
+      kind text not null,
+      value text not null,
+      source text not null,
+      is_active integer not null default 1,
+      first_seen_at text not null,
+      last_seen_at text not null,
+      raw_json text not null default '{}',
+      primary key (profile_id, kind, value)
+    );
+  `);
+}
+
+function ensureIdentitySearchIndexTable(db: Database) {
+	db.exec(`
+    create table if not exists identity_search_index (
+      profile_id text not null,
+      kind text not null,
+      value text not null,
+      normalized_value text not null,
+      source text not null,
+      weight integer not null,
+      updated_at text not null,
+      primary key (profile_id, kind, value, source)
+    );
+  `);
+}
+
+function backfillTweetCollections(db: Database) {
 	const now = new Date().toISOString();
 	const insert = db.prepare(`
     insert or ignore into tweet_collections (
@@ -362,7 +633,29 @@ function backfillTweetCollections(db: BetterSqlite3.Database) {
 	})();
 }
 
-function ensureSchemaIndexes(db: BetterSqlite3.Database) {
+function backfillTweetAccountEdges(db: Database) {
+	const now = new Date().toISOString();
+	db.prepare(`
+    insert or ignore into tweet_account_edges (
+      account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
+      source, raw_json, updated_at
+    )
+    select
+      account_id,
+      id,
+      kind,
+      created_at,
+      created_at,
+      1,
+      'legacy',
+      '{}',
+      ?
+    from tweets
+    where kind in ('home', 'mention')
+  `).run(now);
+}
+
+function ensureSchemaIndexes(db: Database) {
 	db.exec(INDEX_SQL);
 }
 
@@ -371,17 +664,23 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 
 	if (!nativeDb) {
 		const { dbPath } = getBirdclawPaths();
-		nativeDb = new BetterSqlite3(dbPath);
+		nativeDb = new NativeSqliteDatabase(dbPath);
 		nativeDb.exec(BASE_SCHEMA_SQL);
 		ensureAccountExternalUserIdColumn(nativeDb);
 		ensureTweetMetadataColumns(nativeDb);
 		ensureProfileAvatarColumns(nativeDb);
 		ensureTweetCollectionsTable(nativeDb);
+		ensureTweetAccountEdgesTable(nativeDb);
+		ensureProfileAffiliationsTable(nativeDb);
+		ensureProfileSnapshotsTable(nativeDb);
+		ensureProfileBioEntitiesTable(nativeDb);
+		ensureIdentitySearchIndexTable(nativeDb);
 		ensureSchemaIndexes(nativeDb);
 		if (options.seedDemoData !== false) {
 			seedDemoData(nativeDb);
 		}
 		backfillTweetCollections(nativeDb);
+		backfillTweetAccountEdges(nativeDb);
 	}
 
 	if (!kyselyDb) {
@@ -395,7 +694,7 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 
 export function getNativeDb(options: InitDatabaseOptions = {}) {
 	initDatabase(options);
-	return nativeDb as BetterSqlite3.Database;
+	return nativeDb as Database;
 }
 
 export function getDb() {