birdclaw 0.1.0

package/src/lib/db.ts

@@ -0,0 +1,326 @@
+import BetterSqlite3 from "better-sqlite3";
+import { Kysely, SqliteDialect } from "kysely";
+import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
+import { seedDemoData } from "./seed";
+
+export interface AccountsTable {
+	id: string;
+	name: string;
+	handle: string;
+	transport: string;
+	is_default: number;
+	created_at: string;
+}
+
+export interface ProfilesTable {
+	id: string;
+	handle: string;
+	display_name: string;
+	bio: string;
+	followers_count: number;
+	avatar_hue: number;
+	avatar_url: string | null;
+	created_at: string;
+}
+
+export interface TweetsTable {
+	id: string;
+	account_id: string;
+	author_profile_id: string;
+	kind: string;
+	text: string;
+	created_at: string;
+	is_replied: number;
+	reply_to_id: string | null;
+	like_count: number;
+	media_count: number;
+	bookmarked: number;
+	liked: number;
+	entities_json: string;
+	media_json: string;
+	quoted_tweet_id: string | null;
+}
+
+export interface DmConversationsTable {
+	id: string;
+	account_id: string;
+	participant_profile_id: string;
+	title: string;
+	last_message_at: string;
+	unread_count: number;
+	needs_reply: number;
+}
+
+export interface DmMessagesTable {
+	id: string;
+	conversation_id: string;
+	sender_profile_id: string;
+	text: string;
+	created_at: string;
+	direction: "inbound" | "outbound";
+	is_replied: number;
+	media_count: number;
+}
+
+export interface TweetActionsTable {
+	id: string;
+	account_id: string;
+	tweet_id: string | null;
+	kind: string;
+	body: string;
+	created_at: string;
+}
+
+export interface BlocksTable {
+	account_id: string;
+	profile_id: string;
+	source: string;
+	created_at: string;
+}
+
+export interface MutesTable {
+	account_id: string;
+	profile_id: string;
+	source: string;
+	created_at: string;
+}
+
+export interface AiScoresTable {
+	entity_kind: string;
+	entity_id: string;
+	model: string;
+	score: number;
+	summary: string;
+	reasoning: string;
+	updated_at: string;
+}
+
+export interface SyncCacheTable {
+	cache_key: string;
+	value_json: string;
+	updated_at: string;
+}
+
+export interface BirdclawDatabase {
+	accounts: AccountsTable;
+	profiles: ProfilesTable;
+	tweets: TweetsTable;
+	dm_conversations: DmConversationsTable;
+	dm_messages: DmMessagesTable;
+	tweet_actions: TweetActionsTable;
+	blocks: BlocksTable;
+	mutes: MutesTable;
+	ai_scores: AiScoresTable;
+	sync_cache: SyncCacheTable;
+}
+
+let nativeDb: BetterSqlite3.Database | undefined;
+let kyselyDb: Kysely<BirdclawDatabase> | undefined;
+
+const BASE_SCHEMA_SQL = `
+  pragma journal_mode = wal;
+  pragma busy_timeout = 5000;
+  pragma foreign_keys = on;
+
+  create table if not exists accounts (
+    id text primary key,
+    name text not null,
+    handle text not null unique,
+    transport text not null,
+    is_default integer not null default 0,
+    created_at text not null
+  );
+
+  create table if not exists profiles (
+    id text primary key,
+    handle text not null unique,
+    display_name text not null,
+    bio text not null,
+    followers_count integer not null default 0,
+    avatar_hue integer not null default 0,
+    avatar_url text,
+    created_at text not null
+  );
+
+  create table if not exists tweets (
+    id text primary key,
+    account_id text not null,
+    author_profile_id text not null,
+    kind text not null,
+    text text not null,
+    created_at text not null,
+    is_replied integer not null default 0,
+    reply_to_id text,
+    like_count integer not null default 0,
+    media_count integer not null default 0,
+    bookmarked integer not null default 0,
+    liked integer not null default 0,
+    entities_json text not null default '{}',
+    media_json text not null default '[]',
+    quoted_tweet_id text
+  );
+
+  create table if not exists dm_conversations (
+    id text primary key,
+    account_id text not null,
+    participant_profile_id text not null,
+    title text not null,
+    last_message_at text not null,
+    unread_count integer not null default 0,
+    needs_reply integer not null default 0
+  );
+
+  create table if not exists dm_messages (
+    id text primary key,
+    conversation_id text not null,
+    sender_profile_id text not null,
+    text text not null,
+    created_at text not null,
+    direction text not null,
+    is_replied integer not null default 0,
+    media_count integer not null default 0
+  );
+
+  create table if not exists tweet_actions (
+    id text primary key,
+    account_id text not null,
+    tweet_id text,
+    kind text not null,
+    body text not null,
+    created_at text not null
+  );
+
+  create table if not exists blocks (
+    account_id text not null,
+    profile_id text not null,
+    source text not null,
+    created_at text not null,
+    primary key (account_id, profile_id)
+  );
+
+  create table if not exists mutes (
+    account_id text not null,
+    profile_id text not null,
+    source text not null,
+    created_at text not null,
+    primary key (account_id, profile_id)
+  );
+
+  create table if not exists ai_scores (
+    entity_kind text not null,
+    entity_id text not null,
+    model text not null,
+    score integer not null,
+    summary text not null,
+    reasoning text not null,
+    updated_at text not null,
+    primary key (entity_kind, entity_id)
+  );
+
+  create table if not exists sync_cache (
+    cache_key text primary key,
+    value_json text not null,
+    updated_at text not null
+  );
+
+  create virtual table if not exists tweets_fts using fts5(
+    tweet_id unindexed,
+    text
+  );
+
+  create virtual table if not exists dm_fts using fts5(
+    message_id unindexed,
+    text
+  );
+`;
+
+const INDEX_SQL = `
+  create index if not exists idx_tweets_kind_created on tweets(kind, created_at desc);
+  create index if not exists idx_tweets_account_created on tweets(account_id, created_at desc);
+  create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
+  create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
+  create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
+  create index if not exists idx_profiles_followers on profiles(followers_count desc);
+  create index if not exists idx_blocks_account_created on blocks(account_id, created_at desc);
+  create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
+  create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
+  create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
+`;
+
+function getColumnNames(
+	db: BetterSqlite3.Database,
+	tableName: string,
+): Set<string> {
+	const rows = db.prepare(`pragma table_info(${tableName})`).all() as Array<{
+		name: string;
+	}>;
+	return new Set(rows.map((row) => row.name));
+}
+
+function ensureTweetMetadataColumns(db: BetterSqlite3.Database) {
+	const columnNames = getColumnNames(db, "tweets");
+	if (!columnNames.has("entities_json")) {
+		db.exec(
+			"alter table tweets add column entities_json text not null default '{}'",
+		);
+	}
+	if (!columnNames.has("media_json")) {
+		db.exec(
+			"alter table tweets add column media_json text not null default '[]'",
+		);
+	}
+	if (!columnNames.has("quoted_tweet_id")) {
+		db.exec("alter table tweets add column quoted_tweet_id text");
+	}
+}
+
+function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
+	const columnNames = getColumnNames(db, "profiles");
+	if (!columnNames.has("avatar_url")) {
+		db.exec("alter table profiles add column avatar_url text");
+	}
+}
+
+function ensureSchemaIndexes(db: BetterSqlite3.Database) {
+	db.exec(INDEX_SQL);
+}
+
+function initDatabase() {
+	ensureBirdclawDirs();
+
+	if (!nativeDb) {
+		const { dbPath } = getBirdclawPaths();
+		nativeDb = new BetterSqlite3(dbPath);
+		nativeDb.exec(BASE_SCHEMA_SQL);
+		ensureTweetMetadataColumns(nativeDb);
+		ensureProfileAvatarColumns(nativeDb);
+		ensureSchemaIndexes(nativeDb);
+		seedDemoData(nativeDb);
+	}
+
+	if (!kyselyDb) {
+		kyselyDb = new Kysely<BirdclawDatabase>({
+			dialect: new SqliteDialect({
+				database: nativeDb,
+			}),
+		});
+	}
+}
+
+export function getNativeDb() {
+	initDatabase();
+	return nativeDb as BetterSqlite3.Database;
+}
+
+export function getDb() {
+	initDatabase();
+	return kyselyDb as Kysely<BirdclawDatabase>;
+}
+
+export function resetDatabaseForTests() {
+	kyselyDb?.destroy();
+	kyselyDb = undefined;
+
+	nativeDb?.close();
+	nativeDb = undefined;
+}

package/src/lib/dms-live.ts

@@ -0,0 +1,279 @@
+import type Database from "better-sqlite3";
+import {
+	type BirdDmConversation,
+	type BirdDmEvent,
+	type BirdDmUser,
+	listDirectMessagesViaBird,
+} from "./bird";
+import { getNativeDb } from "./db";
+import { readSyncCache, writeSyncCache } from "./sync-cache";
+import type { XurlMentionUser } from "./types";
+import { upsertProfileFromXUser } from "./x-profile";
+
+export const DEFAULT_DMS_CACHE_TTL_MS = 2 * 60_000;
+
+function parseCacheTtlMs(value?: number) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+		return DEFAULT_DMS_CACHE_TTL_MS;
+	}
+	return Math.floor(value);
+}
+
+function assertBirdLimit(limit: number) {
+	if (!Number.isFinite(limit) || limit < 1) {
+		throw new Error("bird DM mode requires --limit of at least 1");
+	}
+}
+
+function resolveAccount(db: Database.Database, accountId?: string) {
+	const row = accountId
+		? (db
+				.prepare("select id, handle from accounts where id = ?")
+				.get(accountId) as { id: string; handle: string } | undefined)
+		: (db
+				.prepare(
+					`
+          select id, handle
+          from accounts
+          order by is_default desc, created_at asc
+          limit 1
+          `,
+				)
+				.get() as { id: string; handle: string } | undefined);
+
+	if (!row) {
+		throw new Error(`Unknown account: ${accountId ?? "default"}`);
+	}
+
+	return {
+		accountId: row.id,
+		username: row.handle.replace(/^@/, ""),
+	};
+}
+
+function toIsoTimestamp(value?: string) {
+	if (!value) {
+		return new Date().toISOString();
+	}
+	const parsed = new Date(value);
+	return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString();
+}
+
+function toXUser(user: BirdDmUser): XurlMentionUser {
+	return {
+		id: user.id,
+		username: user.username ?? `user_${user.id}`,
+		name: user.name ?? user.username ?? `user_${user.id}`,
+		profile_image_url: user.profileImageUrl,
+		public_metrics: { followers_count: 0 },
+	};
+}
+
+function collectUsers(payload: {
+	conversations: BirdDmConversation[];
+	events: BirdDmEvent[];
+}) {
+	const users = new Map<string, BirdDmUser>();
+	const add = (user?: BirdDmUser) => {
+		if (!user?.id) return;
+		users.set(user.id, { ...users.get(user.id), ...user });
+	};
+
+	for (const conversation of payload.conversations) {
+		for (const participant of conversation.participants) {
+			add(participant);
+		}
+	}
+	for (const event of payload.events) {
+		add(event.sender);
+		add(event.recipient);
+	}
+	return users;
+}
+
+function getLocalExternalUserId(
+	users: Map<string, BirdDmUser>,
+	accountUsername: string,
+) {
+	for (const user of users.values()) {
+		if (user.username === accountUsername) {
+			return user.id;
+		}
+	}
+	return undefined;
+}
+
+function getLatestEvent(events: BirdDmEvent[]) {
+	return [...events].sort(
+		(left, right) =>
+			new Date(right.createdAt ?? 0).getTime() -
+			new Date(left.createdAt ?? 0).getTime(),
+	)[0];
+}
+
+function mergeDirectMessagesIntoLocalStore(
+	db: Database.Database,
+	accountId: string,
+	accountUsername: string,
+	payload: {
+		conversations: BirdDmConversation[];
+		events: BirdDmEvent[];
+	},
+) {
+	const users = collectUsers(payload);
+	const localExternalUserId = getLocalExternalUserId(users, accountUsername);
+	const profilesByExternalId = new Map<string, string>();
+	for (const user of users.values()) {
+		const resolved = upsertProfileFromXUser(db, toXUser(user));
+		profilesByExternalId.set(user.id, resolved.profile.id);
+	}
+
+	const eventsByConversation = new Map<string, BirdDmEvent[]>();
+	for (const event of payload.events) {
+		if (!event.conversationId) continue;
+		const events = eventsByConversation.get(event.conversationId) ?? [];
+		events.push(event);
+		eventsByConversation.set(event.conversationId, events);
+	}
+
+	const upsertConversation = db.prepare(`
+    insert into dm_conversations (
+      id, account_id, participant_profile_id, title, last_message_at, unread_count, needs_reply
+    ) values (?, ?, ?, ?, ?, 0, ?)
+    on conflict(id) do update set
+      account_id = excluded.account_id,
+      participant_profile_id = excluded.participant_profile_id,
+      title = excluded.title,
+      last_message_at = excluded.last_message_at,
+      needs_reply = excluded.needs_reply
+  `);
+	const upsertMessage = db.prepare(`
+    insert into dm_messages (
+      id, conversation_id, sender_profile_id, text, created_at, direction, is_replied, media_count
+    ) values (?, ?, ?, ?, ?, ?, 0, 0)
+    on conflict(id) do update set
+      conversation_id = excluded.conversation_id,
+      sender_profile_id = excluded.sender_profile_id,
+      text = excluded.text,
+      created_at = excluded.created_at,
+      direction = excluded.direction,
+      media_count = excluded.media_count
+  `);
+	const replaceFts = db.prepare("delete from dm_fts where message_id = ?");
+	const insertFts = db.prepare(
+		"insert into dm_fts (message_id, text) values (?, ?)",
+	);
+
+	db.transaction(() => {
+		for (const conversation of payload.conversations) {
+			const events = eventsByConversation.get(conversation.id) ?? [];
+			if (events.length === 0) {
+				continue;
+			}
+
+			const participant =
+				conversation.participants.find(
+					(user) =>
+						user.id !== localExternalUserId &&
+						user.username !== accountUsername,
+				) ?? conversation.participants[0];
+			if (!participant) {
+				continue;
+			}
+			const participantProfileId = profilesByExternalId.get(participant.id);
+			if (!participantProfileId) {
+				continue;
+			}
+
+			const latest = getLatestEvent(events);
+			const lastMessageAt = toIsoTimestamp(
+				latest?.createdAt ?? conversation.lastMessageAt,
+			);
+			const latestInbound =
+				latest?.senderId !== localExternalUserId &&
+				latest?.sender?.username !== accountUsername;
+			upsertConversation.run(
+				conversation.id,
+				accountId,
+				participantProfileId,
+				participant.username ?? participant.name ?? participant.id,
+				lastMessageAt,
+				latestInbound ? 1 : 0,
+			);
+
+			for (const event of events) {
+				const senderId = event.senderId ?? event.sender?.id;
+				if (!senderId) {
+					continue;
+				}
+				const senderProfileId = profilesByExternalId.get(senderId);
+				if (!senderProfileId) {
+					continue;
+				}
+				const direction =
+					senderId === localExternalUserId ||
+					event.sender?.username === accountUsername
+						? "outbound"
+						: "inbound";
+				upsertMessage.run(
+					event.id,
+					conversation.id,
+					senderProfileId,
+					event.text,
+					toIsoTimestamp(event.createdAt),
+					direction,
+				);
+				replaceFts.run(event.id);
+				insertFts.run(event.id, event.text);
+			}
+		}
+	})();
+}
+
+export async function syncDirectMessagesViaCachedBird({
+	account,
+	limit = 20,
+	refresh = false,
+	cacheTtlMs,
+}: {
+	account?: string;
+	limit?: number;
+	refresh?: boolean;
+	cacheTtlMs?: number;
+}) {
+	assertBirdLimit(limit);
+	const db = getNativeDb();
+	const resolvedAccount = resolveAccount(db, account);
+	const cacheKey = `dms:bird:${resolvedAccount.accountId}:${String(limit)}`;
+	const ttlMs = parseCacheTtlMs(cacheTtlMs);
+	const cached = readSyncCache<{
+		conversations: BirdDmConversation[];
+		events: BirdDmEvent[];
+	}>(cacheKey, db);
+	const cacheAgeMs = cached
+		? Date.now() - new Date(cached.updatedAt).getTime()
+		: Number.POSITIVE_INFINITY;
+
+	const payload =
+		!refresh && cached && cacheAgeMs <= ttlMs
+			? cached.value
+			: await listDirectMessagesViaBird({ maxResults: limit });
+
+	mergeDirectMessagesIntoLocalStore(
+		db,
+		resolvedAccount.accountId,
+		resolvedAccount.username,
+		payload,
+	);
+	if (!cached || refresh || cacheAgeMs > ttlMs) {
+		writeSyncCache(cacheKey, payload, db);
+	}
+
+	return {
+		ok: true,
+		source: cached && !refresh && cacheAgeMs <= ttlMs ? "cache" : "bird",
+		accountId: resolvedAccount.accountId,
+		conversations: payload.conversations.length,
+		messages: payload.events.length,
+	};
+}