birdclaw 0.8.1 → 0.8.3

package/src/lib/bird.ts

@@ -29,6 +29,12 @@ interface BirdTweetAuthor {
 	name?: string;
 }
 
+interface BirdTweetArticle {
+	title?: string;
+	previewText?: string;
+	coverImageUrl?: string;
+}
+
 interface BirdTweetItem {
 	id: string;
 	text: string;
@@ -45,6 +51,7 @@ interface BirdTweetItem {
 	author?: BirdTweetAuthor;
 	authorId?: string;
 	media?: BirdTweetMedia[];
+	article?: BirdTweetArticle | null;
 }
 
 export interface BirdDmUser {
@@ -385,6 +392,29 @@ function toMediaEntities(media: BirdTweetMedia[] | undefined) {
 	};
 }
 
+function toTweetEntities(item: BirdTweetItem) {
+	const mediaEntities = toMediaEntities(item.media);
+	const title = item.article?.title?.trim();
+	if (!title) return mediaEntities;
+	const handle = item.author?.username?.replace(/^@/, "");
+	const url = handle
+		? `https://x.com/${handle}/status/${item.id}`
+		: `https://x.com/i/status/${item.id}`;
+	return {
+		...mediaEntities,
+		article: {
+			title,
+			url,
+			...(item.article?.previewText?.trim()
+				? { previewText: item.article.previewText.trim() }
+				: {}),
+			...(item.article?.coverImageUrl?.trim()
+				? { coverImageUrl: item.article.coverImageUrl.trim() }
+				: {}),
+		},
+	};
+}
+
 function toReferencedTweets(item: BirdTweetItem) {
 	const references: XurlReferencedTweet[] = [];
 	if (typeof item.inReplyToStatusId === "string" && item.inReplyToStatusId) {
@@ -434,7 +464,7 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse {
 			text: item.text,
 			created_at: toIsoTimestamp(item.createdAt),
 			conversation_id: item.conversationId ?? item.id,
-			entities: toMediaEntities(item.media),
+			entities: toTweetEntities(item),
 			referenced_tweets: toReferencedTweets(item),
 			public_metrics: {
 				reply_count: Number(item.replyCount ?? 0),
@@ -1104,6 +1134,7 @@ export const __test__ = {
 	getBirdTweetItems,
 	getBirdTweetItem,
 	toMediaEntities,
+	toTweetEntities,
 	toReferencedTweets,
 	normalizeBirdTweets,
 };

package/src/lib/blocks-write.ts

@@ -1,5 +1,6 @@
 import { Effect } from "effect";
 import { runModerationAction } from "./actions-transport";
+import { databaseWriteEffect } from "./database-writer";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import {
 	deleteModerationRow,
@@ -8,14 +9,6 @@ import {
 	writeModerationRow,
 } from "./moderation-write";
 
-function trySync<T>(try_: () => T) {
-	return Effect.try({
-		try: try_,
-		catch: (error) =>
-			error instanceof Error ? error : new Error(String(error)),
-	});
-}
-
 export function addBlockEffect(
 	accountId: string,
 	query: string,
@@ -49,14 +42,16 @@ export function addBlockEffect(
 		}
 
 		const blockedAt = new Date().toISOString();
-		yield* trySync(() =>
-			writeModerationRow(
-				db,
-				"blocks",
-				resolvedAccountId,
-				resolved.profile.id,
-				blockedAt,
-			),
+		yield* databaseWriteEffect(
+			(writeDb) =>
+				writeModerationRow(
+					writeDb,
+					"blocks",
+					resolvedAccountId,
+					resolved.profile.id,
+					blockedAt,
+				),
+			db,
 		);
 
 		return {
@@ -88,14 +83,16 @@ export function recordBlockEffect(accountId: string, query: string) {
 			});
 
 		const blockedAt = new Date().toISOString();
-		yield* trySync(() =>
-			writeModerationRow(
-				db,
-				"blocks",
-				resolvedAccountId,
-				resolved.profile.id,
-				blockedAt,
-			),
+		yield* databaseWriteEffect(
+			(writeDb) =>
+				writeModerationRow(
+					writeDb,
+					"blocks",
+					resolvedAccountId,
+					resolved.profile.id,
+					blockedAt,
+				),
+			db,
 		);
 
 		return {
@@ -144,8 +141,15 @@ export function removeBlockEffect(
 			};
 		}
 
-		yield* trySync(() =>
-			deleteModerationRow(db, "blocks", resolvedAccountId, resolved.profile.id),
+		yield* databaseWriteEffect(
+			(writeDb) =>
+				deleteModerationRow(
+					writeDb,
+					"blocks",
+					resolvedAccountId,
+					resolved.profile.id,
+				),
+			db,
 		);
 
 		return {

package/src/lib/blocks.ts

@@ -1,6 +1,7 @@
 import { Effect } from "effect";
 import type { Database } from "./sqlite";
 import { getNativeDb } from "./db";
+import { databaseWriteEffect } from "./database-writer";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import {
 	getAccountHandle,
@@ -322,33 +323,30 @@ export function syncBlocksEffect(accountId: string) {
 				const page = yield* tryPromise(() =>
 					listBlockedUsers(sourceUserId, nextToken ?? undefined),
 				).pipe(Effect.mapError(toError));
-				yield* trySync(() => {
-					const mergeRemotePage = db.transaction(() => {
-						for (const user of page.items) {
-							const resolved = upsertProfileFromXUser(db, user);
-							remoteProfileIds.push(resolved.profile.id);
-							upsertRemoteBlock(
-								db,
-								resolvedAccountId,
-								resolved.profile.id,
-								blockedAt,
-							);
-						}
+				const pageProfileIds = yield* databaseWriteEffect((writeDb) => {
+					return page.items.map((user) => {
+						const resolved = upsertProfileFromXUser(writeDb, user);
+						upsertRemoteBlock(
+							writeDb,
+							resolvedAccountId,
+							resolved.profile.id,
+							blockedAt,
+						);
+						return resolved.profile.id;
 					});
-					mergeRemotePage();
-				});
+				}, db);
+				remoteProfileIds.push(...pageProfileIds);
 				nextToken = page.nextToken;
 				pageCount += 1;
 			} while (nextToken && pageCount < 20);
 
 			completed = !nextToken;
 			if (completed) {
-				yield* trySync(() => {
-					const pruneMergedBlocks = db.transaction(() => {
-						pruneRemoteBlocks(db, resolvedAccountId, remoteProfileIds);
-					});
-					pruneMergedBlocks();
-				});
+				yield* databaseWriteEffect(
+					(writeDb) =>
+						pruneRemoteBlocks(writeDb, resolvedAccountId, remoteProfileIds),
+					db,
+				);
 			}
 
 			return {

package/src/lib/database-metrics.ts

@@ -0,0 +1,88 @@
+export type DatabaseConnectionRole = "reader" | "writer";
+
+export interface SlowDatabaseStatement {
+	durationMs: number;
+	role: DatabaseConnectionRole;
+	sql: string;
+}
+
+const SLOW_STATEMENT_MS = 50;
+const MAX_SLOW_STATEMENTS = 10;
+
+let readStatements = 0;
+let writeStatements = 0;
+let slowStatements: SlowDatabaseStatement[] = [];
+let queuedWrites = 0;
+let activeWrites = 0;
+let completedWrites = 0;
+let failedWrites = 0;
+let maxQueuedWrites = 0;
+let maxWriteWaitMs = 0;
+
+function compactSql(sql: string) {
+	return sql.replace(/\s+/g, " ").trim().slice(0, 240);
+}
+
+export function recordDatabaseStatement(
+	role: DatabaseConnectionRole,
+	sql: string,
+	durationMs: number,
+) {
+	if (role === "reader") {
+		readStatements += 1;
+	} else {
+		writeStatements += 1;
+	}
+	if (durationMs < SLOW_STATEMENT_MS) return;
+	slowStatements = [
+		...slowStatements,
+		{ durationMs, role, sql: compactSql(sql) },
+	].slice(-MAX_SLOW_STATEMENTS);
+}
+
+export function recordDatabaseWriteQueued() {
+	queuedWrites += 1;
+	maxQueuedWrites = Math.max(maxQueuedWrites, queuedWrites);
+}
+
+export function recordDatabaseWriteStarted(waitMs: number) {
+	queuedWrites = Math.max(0, queuedWrites - 1);
+	activeWrites += 1;
+	maxWriteWaitMs = Math.max(maxWriteWaitMs, waitMs);
+}
+
+export function recordDatabaseWriteCompleted(failed: boolean) {
+	activeWrites = Math.max(0, activeWrites - 1);
+	completedWrites += 1;
+	if (failed) failedWrites += 1;
+}
+
+export function getDatabaseRuntimeMetrics() {
+	return {
+		connections: {
+			readStatements,
+			writeStatements,
+			slowStatements: [...slowStatements],
+		},
+		writer: {
+			active: activeWrites,
+			completed: completedWrites,
+			failed: failedWrites,
+			maxQueued: maxQueuedWrites,
+			maxWaitMs: maxWriteWaitMs,
+			queued: queuedWrites,
+		},
+	};
+}
+
+export function resetDatabaseRuntimeMetricsForTests() {
+	readStatements = 0;
+	writeStatements = 0;
+	slowStatements = [];
+	queuedWrites = 0;
+	activeWrites = 0;
+	completedWrites = 0;
+	failedWrites = 0;
+	maxQueuedWrites = 0;
+	maxWriteWaitMs = 0;
+}

package/src/lib/database-migrations.ts

@@ -0,0 +1,34 @@
+import type { Database } from "./sqlite";
+
+export interface DatabaseMigration {
+	version: number;
+	name: string;
+	up: (db: Database) => void;
+}
+
+export function getDatabaseSchemaVersion(db: Database) {
+	return Number(db.pragma("user_version", { simple: true }) ?? 0);
+}
+
+export function runDatabaseMigrations(
+	db: Database,
+	migrations: readonly DatabaseMigration[],
+) {
+	let currentVersion = getDatabaseSchemaVersion(db);
+	const pending = [...migrations]
+		.filter((migration) => migration.version > currentVersion)
+		.sort((left, right) => left.version - right.version);
+
+	for (const migration of pending) {
+		if (migration.version !== currentVersion + 1) {
+			throw new Error(
+				`Missing database migration between versions ${String(currentVersion)} and ${String(migration.version)}`,
+			);
+		}
+		migration.up(db);
+		db.pragma(`user_version = ${String(migration.version)}`);
+		currentVersion = migration.version;
+	}
+
+	return currentVersion;
+}

package/src/lib/database-schema.ts

@@ -0,0 +1,312 @@
+export interface AccountsTable {
+	id: string;
+	name: string;
+	handle: string;
+	external_user_id: string | null;
+	transport: string;
+	is_default: number;
+	created_at: string;
+}
+
+export interface ProfilesTable {
+	id: string;
+	handle: string;
+	display_name: string;
+	bio: string;
+	followers_count: number;
+	following_count: number;
+	public_metrics_json: string;
+	avatar_hue: number;
+	avatar_url: string | null;
+	location: string | null;
+	url: string | null;
+	verified_type: string | null;
+	entities_json: string;
+	raw_json: string;
+	created_at: string;
+}
+
+export interface ProfileAffiliationsTable {
+	subject_profile_id: string;
+	organization_profile_id: string;
+	organization_name: string | null;
+	organization_handle: string | null;
+	badge_url: string | null;
+	url: string | null;
+	label: string | null;
+	source: string;
+	is_active: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	raw_json: string;
+	updated_at: string;
+}
+
+export interface ProfileSnapshotsTable {
+	profile_id: string;
+	snapshot_hash: string;
+	observed_at: string;
+	last_seen_at: string;
+	source: string;
+	handle: string;
+	display_name: string;
+	bio: string;
+	location: string | null;
+	url: string | null;
+	verified_type: string | null;
+	followers_count: number;
+	following_count: number;
+	affiliations_json: string;
+	raw_json: string;
+}
+
+export interface ProfileBioEntitiesTable {
+	profile_id: string;
+	kind: string;
+	value: string;
+	source: string;
+	is_active: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	raw_json: string;
+}
+
+export interface IdentitySearchIndexTable {
+	profile_id: string;
+	kind: string;
+	value: string;
+	normalized_value: string;
+	source: string;
+	weight: number;
+	updated_at: string;
+}
+
+export interface TweetsTable {
+	id: string;
+	account_id: string;
+	author_profile_id: string;
+	kind: string;
+	text: string;
+	created_at: string;
+	is_replied: number;
+	reply_to_id: string | null;
+	like_count: number;
+	media_count: number;
+	bookmarked: number;
+	liked: number;
+	entities_json: string;
+	media_json: string;
+	quoted_tweet_id: string | null;
+}
+
+export interface TweetCollectionsTable {
+	account_id: string;
+	tweet_id: string;
+	kind: "likes" | "bookmarks";
+	collected_at: string | null;
+	source: string;
+	raw_json: string;
+	updated_at: string;
+}
+
+export interface TweetAccountEdgesTable {
+	account_id: string;
+	tweet_id: string;
+	kind:
+		| "home"
+		| "mention"
+		| "authored"
+		| "search"
+		| "profile"
+		| "thread_context";
+	first_seen_at: string;
+	last_seen_at: string;
+	seen_count: number;
+	source: string;
+	raw_json: string;
+	updated_at: string;
+}
+
+export interface DmConversationsTable {
+	id: string;
+	account_id: string;
+	participant_profile_id: string;
+	title: string;
+	inbox_kind: "accepted" | "request";
+	last_message_at: string;
+	unread_count: number;
+	needs_reply: number;
+}
+
+export interface DmMessagesTable {
+	id: string;
+	conversation_id: string;
+	sender_profile_id: string;
+	text: string;
+	created_at: string;
+	direction: "inbound" | "outbound";
+	is_replied: number;
+	media_count: number;
+}
+
+export interface TweetActionsTable {
+	id: string;
+	account_id: string;
+	tweet_id: string | null;
+	kind: string;
+	body: string;
+	created_at: string;
+}
+
+export interface BlocksTable {
+	account_id: string;
+	profile_id: string;
+	source: string;
+	created_at: string;
+}
+
+export interface MutesTable {
+	account_id: string;
+	profile_id: string;
+	source: string;
+	created_at: string;
+}
+
+export interface AiScoresTable {
+	entity_kind: string;
+	entity_id: string;
+	model: string;
+	score: number;
+	summary: string;
+	reasoning: string;
+	updated_at: string;
+}
+
+export interface SyncCacheTable {
+	cache_key: string;
+	value_json: string;
+	updated_at: string;
+}
+
+export interface UrlExpansionsTable {
+	short_url: string;
+	expanded_url: string;
+	final_url: string;
+	status: string;
+	expanded_tweet_id: string | null;
+	expanded_handle: string | null;
+	title: string | null;
+	description: string | null;
+	image_url: string | null;
+	site_name: string | null;
+	error: string | null;
+	source: string;
+	updated_at: string;
+}
+
+export interface LinkOccurrencesTable {
+	source_kind: "dm" | "tweet";
+	source_id: string;
+	source_position: number;
+	short_url: string;
+	account_id: string | null;
+	conversation_id: string | null;
+	direction: string | null;
+	created_at: string;
+}
+
+export interface FollowEdgesTable {
+	account_id: string;
+	direction: "followers" | "following";
+	profile_id: string;
+	external_user_id: string;
+	source: string;
+	current: number;
+	first_seen_at: string;
+	last_seen_at: string;
+	ended_at: string | null;
+	updated_at: string;
+}
+
+export interface FollowSnapshotsTable {
+	id: string;
+	account_id: string;
+	direction: "followers" | "following";
+	source: string;
+	status: "complete" | "incomplete";
+	page_count: number;
+	result_count: number;
+	started_at: string;
+	completed_at: string;
+	raw_meta_json: string;
+}
+
+export interface FollowSnapshotMembersTable {
+	snapshot_id: string;
+	profile_id: string;
+	external_user_id: string;
+	position: number;
+}
+
+export interface FollowEventsTable {
+	id: string;
+	account_id: string;
+	direction: "followers" | "following";
+	profile_id: string;
+	external_user_id: string;
+	kind: "started" | "ended";
+	event_at: string;
+	snapshot_id: string;
+}
+
+export interface GeocodedLocationsTable {
+	normalized_key: string;
+	original: string;
+	lat: number;
+	lng: number;
+	formatted: string | null;
+	country_code: string | null;
+	confidence: number | null;
+	provider: string;
+	approx_radius_m: number | null;
+	bounds_json: string;
+	components_json: string;
+	hits: number;
+	created_at: string;
+	last_used_at: string;
+}
+
+export interface GeocodedLocationsUnresolvedTable {
+	normalized_key: string;
+	original: string;
+	reason: string;
+	last_attempted_at: string;
+	ttl_until: string | null;
+}
+
+export interface BirdclawDatabase {
+	accounts: AccountsTable;
+	profiles: ProfilesTable;
+	profile_affiliations: ProfileAffiliationsTable;
+	profile_snapshots: ProfileSnapshotsTable;
+	profile_bio_entities: ProfileBioEntitiesTable;
+	identity_search_index: IdentitySearchIndexTable;
+	tweets: TweetsTable;
+	tweet_collections: TweetCollectionsTable;
+	tweet_account_edges: TweetAccountEdgesTable;
+	dm_conversations: DmConversationsTable;
+	dm_messages: DmMessagesTable;
+	tweet_actions: TweetActionsTable;
+	blocks: BlocksTable;
+	mutes: MutesTable;
+	ai_scores: AiScoresTable;
+	sync_cache: SyncCacheTable;
+	url_expansions: UrlExpansionsTable;
+	link_occurrences: LinkOccurrencesTable;
+	follow_edges: FollowEdgesTable;
+	follow_snapshots: FollowSnapshotsTable;
+	follow_snapshot_members: FollowSnapshotMembersTable;
+	follow_events: FollowEventsTable;
+	geocoded_locations: GeocodedLocationsTable;
+	geocoded_locations_unresolved: GeocodedLocationsUnresolvedTable;
+}

package/src/lib/database-writer.ts

@@ -0,0 +1,69 @@
+import { Effect } from "effect";
+import type { Database } from "./sqlite";
+import {
+	defaultServerRuntimeServices,
+	type ServerRuntimeServices,
+} from "./server-runtime-services";
+import {
+	recordDatabaseWriteCompleted,
+	recordDatabaseWriteQueued,
+	recordDatabaseWriteStarted,
+} from "./database-metrics";
+
+let writeTails = new Map<string | object, Promise<void>>();
+
+export function enqueueDatabaseWrite<T>(
+	write: (db: Database) => T,
+	providedDb?: Database,
+	runtime: ServerRuntimeServices = defaultServerRuntimeServices,
+): Promise<T> {
+	const db = providedDb ?? runtime.getDatabase({ seedDemoData: false });
+	const writeIdentity = db.writeIdentity;
+	const queuedAt = performance.now();
+	recordDatabaseWriteQueued();
+	const writeTail = writeTails.get(writeIdentity) ?? Promise.resolve();
+	const pending = writeTail.then(() => {
+		recordDatabaseWriteStarted(performance.now() - queuedAt);
+		try {
+			const result = db.transaction(() => write(db))();
+			recordDatabaseWriteCompleted(false);
+			return result;
+		} catch (error) {
+			recordDatabaseWriteCompleted(true);
+			throw error;
+		}
+	});
+	const settled = pending.then(
+		() => undefined,
+		() => undefined,
+	);
+	writeTails.set(writeIdentity, settled);
+	void settled.then(() => {
+		if (writeTails.get(writeIdentity) === settled) {
+			writeTails.delete(writeIdentity);
+		}
+	});
+	return pending;
+}
+
+export function databaseWriteEffect<T>(
+	write: (db: Database) => T,
+	providedDb?: Database,
+	runtime: ServerRuntimeServices = defaultServerRuntimeServices,
+) {
+	return Effect.tryPromise({
+		try: () => enqueueDatabaseWrite(write, providedDb, runtime),
+		catch: (error) =>
+			error instanceof Error ? error : new Error(String(error)),
+	});
+}
+
+export async function drainDatabaseWrites() {
+	while (writeTails.size > 0) {
+		await Promise.all(writeTails.values());
+	}
+}
+
+export function resetDatabaseWriterForTests() {
+	writeTails = new Map();
+}