birdclaw 0.8.1 → 0.8.3

package/src/lib/mentions-live.ts

@@ -1,27 +1,26 @@
 import type { Database } from "./sqlite";
 import { Effect } from "effect";
-import {
-	getAuthenticatedBirdAccountEffect,
-	listMentionsViaBirdEffect,
-} from "./bird";
 import type { MentionsDataSource } from "./config";
+import { databaseWriteEffect } from "./database-writer";
 import { getNativeDb } from "./db";
-import { runEffectPromise, tryPromise } from "./effect-runtime";
-import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
+import { runEffectPromise } from "./effect-runtime";
+import { liveTransportGateway } from "./live-transport-gateway";
+import {
+	assertLiveAccountMatches,
+	resolveLiveSyncAccount,
+	type LiveSyncAccount,
+} from "./live-sync-engine";
 import { serializeMentionItemsAsXurlCompatible } from "./mentions-export";
-import { listTimelineItems } from "./queries";
+import { listTimelineItems } from "./timeline-read-model";
 import { deleteSyncCache, readSyncCache, writeSyncCache } from "./sync-cache";
 import type {
 	ReplyFilter,
-	TweetEntities,
 	XurlMentionData,
 	XurlMentionsResponse,
 	XurlMediaItem,
 	XurlMentionUser,
 } from "./types";
-import { upsertTweetAccountEdge } from "./tweet-account-edges";
-import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
-import { listMentionsViaXurl, lookupUsersByHandles } from "./xurl";
+import { ingestTweetPayload } from "./tweet-repository";
 
 export const DEFAULT_MENTIONS_CACHE_TTL_MS = 2 * 60_000;
 const MIN_XURL_MENTIONS_LIMIT = 5;
@@ -430,71 +429,21 @@ function writeMentionHighWaterId(
 	writeSyncCache(getMentionHighWaterKey({ mode, accountId }), { sinceId }, db);
 }
 
-function resolveAccount(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare(
-					"select id, handle, external_user_id from accounts where id = ?",
-				)
-				.get(accountId) as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined)
-		: (db
-				.prepare(
-					`
-          select id, handle, external_user_id
-          from accounts
-          order by is_default desc, created_at asc
-          limit 1
-          `,
-				)
-				.get() as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined);
-
-	if (!row) {
-		throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	}
-
-	return {
-		accountId: row.id,
-		username: row.handle.replace(/^@/, ""),
-		externalUserId:
-			typeof row.external_user_id === "string" &&
-			row.external_user_id.trim().length > 0
-				? row.external_user_id.trim()
-				: undefined,
-	};
-}
-
-function verifyBirdAccountMatchesEffect({
-	accountId,
-	username,
-	externalUserId,
-}: ReturnType<typeof resolveAccount>) {
+function verifyBirdAccountMatchesEffect(account: LiveSyncAccount) {
 	return Effect.gen(function* () {
-		const authenticated = yield* getAuthenticatedBirdAccountEffect();
-		if (
-			externalUserId &&
-			authenticated.id &&
-			authenticated.id === externalUserId
-		) {
-			return;
-		}
-		if (externalUserId && authenticated.id) {
-			return yield* Effect.fail(
-				new Error(
-					`bird is authenticated as user ${authenticated.id}; refusing to sync into ${accountId} (${externalUserId})`,
-				),
-			);
-		}
-		if (authenticated.username.toLowerCase() !== username.toLowerCase()) {
-			return yield* Effect.fail(
-				new Error(
-					`bird is authenticated as @${authenticated.username}; refusing to sync into ${accountId} (@${username})`,
-				),
-			);
-		}
+		const authenticated =
+			yield* liveTransportGateway.bird.getAuthenticatedAccount();
+		return yield* Effect.try({
+			try: () =>
+				assertLiveAccountMatches({
+					source: "bird",
+					account,
+					liveUsername: authenticated.username,
+					liveExternalUserId: authenticated.id,
+				}),
+			catch: (error) =>
+				error instanceof Error ? error : new Error(String(error)),
+		});
 	});
 }
 
@@ -520,139 +469,20 @@ function findNewestArchiveMentionId(db: Database, accountId: string) {
 	return row?.id;
 }
 
-function toLocalEntities(tweet: XurlMentionData): TweetEntities {
-	const raw = tweet.entities;
-	if (!raw || typeof raw !== "object") {
-		return {};
-	}
-
-	const entities = raw as Record<string, unknown>;
-	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
-	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
-
-	return {
-		...(rawMentions.length
-			? {
-					mentions: rawMentions.map((mention) => {
-						const value =
-							mention && typeof mention === "object"
-								? (mention as Record<string, unknown>)
-								: {};
-						return {
-							username: String(value.username ?? ""),
-							id: typeof value.id === "string" ? String(value.id) : undefined,
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawUrls.length
-			? {
-					urls: rawUrls.map((url) => {
-						const value =
-							url && typeof url === "object"
-								? (url as Record<string, unknown>)
-								: {};
-						return {
-							url: String(value.url ?? ""),
-							expandedUrl: String(value.expanded_url ?? value.url ?? ""),
-							displayUrl: String(
-								value.display_url ?? value.expanded_url ?? value.url ?? "",
-							),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-	};
-}
-
-function replaceTweetFts(db: Database, tweetId: string, text: string) {
-	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
-	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
-		tweetId,
-		text,
-	);
-}
-
 function mergeMentionsIntoLocalStore(
 	db: Database,
 	accountId: string,
 	payload: XurlMentionsResponse,
 	source: MentionsDataSource,
 ) {
-	const usersById = new Map(
-		(payload.includes?.users ?? []).map((user) => [user.id, user]),
-	);
-	const upsertTweet = db.prepare(
-		`
-    insert into tweets (
-      id, account_id, author_profile_id, kind, text, created_at,
-      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
-      entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, 'mention', ?, ?, 0, null, ?, ?, 0, 0, ?, ?, null)
-    on conflict(id) do update set
-      account_id = tweets.account_id,
-      author_profile_id = excluded.author_profile_id,
-      kind = case
-        when tweets.kind in ('authored', 'home', 'mention') then tweets.kind
-        when excluded.kind in ('authored', 'home', 'mention') then excluded.kind
-        else excluded.kind
-      end,
-      text = excluded.text,
-      created_at = excluded.created_at,
-      like_count = excluded.like_count,
-      media_count = max(tweets.media_count, excluded.media_count),
-      entities_json = excluded.entities_json,
-      media_json = case
-        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
-        else tweets.media_json
-      end,
-      is_replied = max(tweets.is_replied, excluded.is_replied),
-      bookmarked = max(tweets.bookmarked, excluded.bookmarked),
-      liked = max(tweets.liked, excluded.liked)
-    `,
-	);
-
-	const writePayload = db.transaction(() => {
-		const seenAt = new Date().toISOString();
-		for (const tweet of payload.data) {
-			const author =
-				usersById.get(tweet.author_id) ??
-				({
-					id: tweet.author_id,
-					username: `user_${tweet.author_id}`,
-					name: `user_${tweet.author_id}`,
-				} as const);
-			const profile = usersById.has(tweet.author_id)
-				? upsertProfileFromXUser(db, author)
-				: ensureStubProfileForXUser(db, tweet.author_id);
-			upsertTweet.run(
-				tweet.id,
-				accountId,
-				profile.profile.id,
-				tweet.text,
-				tweet.created_at,
-				Number(tweet.public_metrics?.like_count ?? 0),
-				countTweetMedia(tweet),
-				JSON.stringify(toLocalEntities(tweet)),
-				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
-			);
-			upsertTweetAccountEdge(db, {
-				accountId,
-				tweetId: tweet.id,
-				kind: "mention",
-				source,
-				seenAt,
-				rawJson: JSON.stringify(tweet),
-			});
-			replaceTweetFts(db, tweet.id, tweet.text);
-		}
+	ingestTweetPayload(db, {
+		accountId,
+		payload,
+		kind: "mention",
+		edgeKind: "mention",
+		replaceSecondaryKind: true,
+		source,
 	});
-
-	writePayload();
 }
 
 function shouldReturnFilteredLocalPayload({
@@ -755,7 +585,7 @@ function fetchMentionsViaXurlEffect({
 	startTime,
 	onProgress,
 }: {
-	resolvedAccount: ReturnType<typeof resolveAccount>;
+	resolvedAccount: LiveSyncAccount;
 	limit: number;
 	all: boolean;
 	parsedMaxPages: number | null;
@@ -767,9 +597,9 @@ function fetchMentionsViaXurlEffect({
 	return Effect.gen(function* () {
 		const accountUserId =
 			resolvedAccount.externalUserId ??
-			(yield* tryPromise(() =>
-				lookupUsersByHandles([resolvedAccount.username]),
-			).pipe(Effect.map((users) => users[0]?.id)));
+			(yield* liveTransportGateway.xurl
+				.lookupUsersByHandles([resolvedAccount.username])
+				.pipe(Effect.map((users) => users[0]?.id)));
 		if (!accountUserId) {
 			return yield* Effect.fail(
 				new Error(
@@ -782,16 +612,14 @@ function fetchMentionsViaXurlEffect({
 		let nextToken: string | undefined = startPaginationToken;
 		let pageCount = 0;
 		do {
-			const payload = yield* tryPromise(() =>
-				listMentionsViaXurl({
-					maxResults: limit,
-					username: resolvedAccount.username,
-					userId: String(accountUserId),
-					paginationToken: nextToken,
-					...(sinceId ? { sinceId } : {}),
-					...(startTime ? { startTime } : {}),
-				}),
-			);
+			const payload = yield* liveTransportGateway.xurl.listMentions({
+				maxResults: limit,
+				username: resolvedAccount.username,
+				userId: String(accountUserId),
+				paginationToken: nextToken,
+				...(sinceId ? { sinceId } : {}),
+				...(startTime ? { startTime } : {}),
+			});
 			pages.push(payload);
 			const metaNextToken =
 				typeof payload.meta?.next_token === "string"
@@ -826,7 +654,7 @@ function fetchMentionsViaXurlEffect({
 }
 
 function fetchMentionsViaBirdEffect({ limit }: { limit: number }) {
-	return listMentionsViaBirdEffect({ maxResults: limit });
+	return liveTransportGateway.bird.listMentions({ maxResults: limit });
 }
 
 function isMaxPagesPartial({
@@ -876,7 +704,9 @@ export function syncMentionsEffect({
 			(parsedMaxPages !== null ||
 				Boolean(explicitSinceId || explicitStartTime));
 		const db = yield* trySync(() => getNativeDb());
-		const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
+		const resolvedAccount = yield* trySync(() =>
+			resolveLiveSyncAccount(db, account),
+		);
 		const cursorShape: MentionScanShape = {
 			endpoint: "mentions",
 			mode: primaryMode,
@@ -966,9 +796,9 @@ export function syncMentionsEffect({
 			cached &&
 			cacheAgeMs <= ttlMs
 		) {
-			yield* trySync(() =>
+			yield* databaseWriteEffect((writeDb) =>
 				mergeMentionsIntoLocalStore(
-					db,
+					writeDb,
 					resolvedAccount.accountId,
 					cached.value,
 					primaryMode,
@@ -1046,9 +876,9 @@ export function syncMentionsEffect({
 				}),
 			);
 		}
-		yield* trySync(() =>
+		yield* databaseWriteEffect((writeDb) =>
 			mergeMentionsIntoLocalStore(
-				db,
+				writeDb,
 				resolvedAccount.accountId,
 				payload,
 				source,
@@ -1157,7 +987,9 @@ function exportMentionsViaCachedLiveSourceEffect({
 		const fetchAll = primaryMode === "xurl" && (all || parsedMaxPages !== null);
 
 		const db = yield* trySync(() => getNativeDb());
-		const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
+		const resolvedAccount = yield* trySync(() =>
+			resolveLiveSyncAccount(db, account),
+		);
 		const cacheKey = getMentionsFetchModeKey({
 			scope: "export",
 			mode,
@@ -1215,14 +1047,14 @@ function exportMentionsViaCachedLiveSourceEffect({
 				);
 			}),
 			Effect.flatMap((payload) =>
-				trySync(() => {
+				databaseWriteEffect((writeDb) => {
 					mergeMentionsIntoLocalStore(
-						db,
+						writeDb,
 						resolvedAccount.accountId,
 						payload,
 						source,
 					);
-					writeSyncCache(cacheKey, payload, db);
+					writeSyncCache(cacheKey, payload, writeDb);
 					return readFilteredOrRaw(payload);
 				}),
 			),

package/src/lib/moderation-target.ts

@@ -2,6 +2,7 @@ import type { Database } from "./sqlite";
 import { Effect } from "effect";
 import { lookupProfileViaBird } from "./bird-actions";
 import { getNativeDb } from "./db";
+import { databaseWriteEffect } from "./database-writer";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import type { ProfileRecord, XurlMentionUser } from "./types";
 import { getExternalUserId, upsertProfileFromXUser } from "./x-profile";
@@ -105,9 +106,10 @@ export function resolveLocalProfile(
 
 export function resolveProfileEffect(
 	query: string,
+	providedDb?: Database,
 ): Effect.Effect<ResolvedModerationProfile, unknown> {
 	return Effect.gen(function* () {
-		const db = yield* trySync(() => getNativeDb());
+		const db = providedDb ?? (yield* trySync(() => getNativeDb()));
 		const normalizedQuery = normalizeProfileQuery(query);
 		if (!normalizedQuery) {
 			return yield* Effect.fail(new Error("Missing profile handle or id"));
@@ -180,7 +182,10 @@ export function resolveProfileEffect(
 		}
 
 		if (local) {
-			return yield* trySync(() => upsertProfileFromXUser(db, user));
+			return yield* databaseWriteEffect(
+				(writeDb) => upsertProfileFromXUser(writeDb, user),
+				db,
+			);
 		}
 
 		const username = String(user.username ?? "").replace(/^@/, "");
@@ -189,11 +194,17 @@ export function resolveProfileEffect(
 				resolveLocalProfile(db, username),
 			);
 			if (localByHandle) {
-				return yield* trySync(() => upsertProfileFromXUser(db, user));
+				return yield* databaseWriteEffect(
+					(writeDb) => upsertProfileFromXUser(writeDb, user),
+					db,
+				);
 			}
 		}
 
-		return yield* trySync(() => upsertProfileFromXUser(db, user));
+		return yield* databaseWriteEffect(
+			(writeDb) => upsertProfileFromXUser(writeDb, user),
+			db,
+		);
 	});
 }
 

package/src/lib/moderation-write.ts

@@ -53,7 +53,7 @@ export function resolveModerationTargetEffect({
 			return yield* Effect.fail(new Error(selfActionError));
 		}
 
-		const resolved = yield* resolveProfileEffect(query);
+		const resolved = yield* resolveProfileEffect(query, db);
 		const account = yield* trySync(
 			() =>
 				db

package/src/lib/mutes-write.ts

@@ -1,5 +1,6 @@
 import { Effect } from "effect";
 import { runModerationAction } from "./actions-transport";
+import { databaseWriteEffect } from "./database-writer";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import {
 	deleteModerationRow,
@@ -8,14 +9,6 @@ import {
 	writeModerationRow,
 } from "./moderation-write";
 
-function trySync<T>(try_: () => T) {
-	return Effect.try({
-		try: try_,
-		catch: (error) =>
-			error instanceof Error ? error : new Error(String(error)),
-	});
-}
-
 export function addMuteEffect(
 	accountId: string,
 	query: string,
@@ -49,14 +42,16 @@ export function addMuteEffect(
 		}
 
 		const mutedAt = new Date().toISOString();
-		yield* trySync(() =>
-			writeModerationRow(
-				db,
-				"mutes",
-				resolvedAccountId,
-				resolved.profile.id,
-				mutedAt,
-			),
+		yield* databaseWriteEffect(
+			(writeDb) =>
+				writeModerationRow(
+					writeDb,
+					"mutes",
+					resolvedAccountId,
+					resolved.profile.id,
+					mutedAt,
+				),
+			db,
 		);
 
 		return {
@@ -88,14 +83,16 @@ export function recordMuteEffect(accountId: string, query: string) {
 			});
 
 		const mutedAt = new Date().toISOString();
-		yield* trySync(() =>
-			writeModerationRow(
-				db,
-				"mutes",
-				resolvedAccountId,
-				resolved.profile.id,
-				mutedAt,
-			),
+		yield* databaseWriteEffect(
+			(writeDb) =>
+				writeModerationRow(
+					writeDb,
+					"mutes",
+					resolvedAccountId,
+					resolved.profile.id,
+					mutedAt,
+				),
+			db,
 		);
 
 		return {
@@ -144,8 +141,15 @@ export function removeMuteEffect(
 			};
 		}
 
-		yield* trySync(() =>
-			deleteModerationRow(db, "mutes", resolvedAccountId, resolved.profile.id),
+		yield* databaseWriteEffect(
+			(writeDb) =>
+				deleteModerationRow(
+					writeDb,
+					"mutes",
+					resolvedAccountId,
+					resolved.profile.id,
+				),
+			db,
 		);
 
 		return {