birdclaw 0.8.2 → 0.8.3

package/src/lib/live-transport-gateway.ts

@@ -0,0 +1,128 @@
+import { Effect } from "effect";
+import {
+	getAuthenticatedBirdAccountEffect,
+	listBookmarkedTweetsViaBirdEffect,
+	listDirectMessagesViaBirdEffect,
+	listFollowUsersViaBirdEffect,
+	listHomeTimelineViaBirdEffect,
+	listLikedTweetsViaBirdEffect,
+	listMentionsViaBirdEffect,
+	listThreadViaBirdEffect,
+	searchTweetsViaBirdEffect,
+} from "./bird";
+import {
+	getTransportStatus,
+	getTweetByIdEffect,
+	listBookmarkedTweetsViaXurl,
+	listDirectMessageEventsViaXurlEffect,
+	listFollowUsersViaXurl,
+	listHomeTimelineViaXurlEffect,
+	listLikedTweetsViaXurl,
+	listMentionsViaXurl,
+	listUserTweets,
+	lookupAuthenticatedOAuth2UserEffect,
+	lookupAuthenticatedUser,
+	lookupUsersByHandles,
+	searchRecentByConversationIdEffect,
+	searchRecentTweetsEffect,
+} from "./xurl";
+
+export interface BirdReadTransport {
+	getAuthenticatedAccount: typeof getAuthenticatedBirdAccountEffect;
+	listBookmarks: typeof listBookmarkedTweetsViaBirdEffect;
+	listDirectMessages: typeof listDirectMessagesViaBirdEffect;
+	listFollowUsers: typeof listFollowUsersViaBirdEffect;
+	listHomeTimeline: typeof listHomeTimelineViaBirdEffect;
+	listLikes: typeof listLikedTweetsViaBirdEffect;
+	listMentions: typeof listMentionsViaBirdEffect;
+	listThread: typeof listThreadViaBirdEffect;
+	searchTweets: typeof searchTweetsViaBirdEffect;
+}
+
+export interface XurlReadTransport {
+	getTransportStatus(
+		...args: Parameters<typeof getTransportStatus>
+	): Effect.Effect<Awaited<ReturnType<typeof getTransportStatus>>, Error>;
+	getTweetById: typeof getTweetByIdEffect;
+	listBookmarks(
+		...args: Parameters<typeof listBookmarkedTweetsViaXurl>
+	): Effect.Effect<
+		Awaited<ReturnType<typeof listBookmarkedTweetsViaXurl>>,
+		Error
+	>;
+	listDirectMessages: typeof listDirectMessageEventsViaXurlEffect;
+	listFollowUsers(
+		...args: Parameters<typeof listFollowUsersViaXurl>
+	): Effect.Effect<Awaited<ReturnType<typeof listFollowUsersViaXurl>>, Error>;
+	listHomeTimeline: typeof listHomeTimelineViaXurlEffect;
+	listLikes(
+		...args: Parameters<typeof listLikedTweetsViaXurl>
+	): Effect.Effect<Awaited<ReturnType<typeof listLikedTweetsViaXurl>>, Error>;
+	listMentions(
+		...args: Parameters<typeof listMentionsViaXurl>
+	): Effect.Effect<Awaited<ReturnType<typeof listMentionsViaXurl>>, Error>;
+	listUserTweets(
+		...args: Parameters<typeof listUserTweets>
+	): Effect.Effect<Awaited<ReturnType<typeof listUserTweets>>, Error>;
+	lookupAuthenticatedOAuth2User: typeof lookupAuthenticatedOAuth2UserEffect;
+	lookupAuthenticatedUser(
+		...args: Parameters<typeof lookupAuthenticatedUser>
+	): Effect.Effect<Awaited<ReturnType<typeof lookupAuthenticatedUser>>, Error>;
+	lookupUsersByHandles(
+		...args: Parameters<typeof lookupUsersByHandles>
+	): Effect.Effect<Awaited<ReturnType<typeof lookupUsersByHandles>>, Error>;
+	searchConversation: typeof searchRecentByConversationIdEffect;
+	searchRecentTweets: typeof searchRecentTweetsEffect;
+}
+
+export interface LiveTransportGateway {
+	bird: BirdReadTransport;
+	xurl: XurlReadTransport;
+}
+
+function fromPromise<T>(run: () => PromiseLike<T>): Effect.Effect<T, Error> {
+	return Effect.tryPromise({
+		try: run,
+		catch: (error) =>
+			error instanceof Error ? error : new Error(String(error)),
+	});
+}
+
+export const liveTransportGateway: LiveTransportGateway = {
+	bird: {
+		getAuthenticatedAccount: () => getAuthenticatedBirdAccountEffect(),
+		listBookmarks: (options) => listBookmarkedTweetsViaBirdEffect(options),
+		listDirectMessages: (options) => listDirectMessagesViaBirdEffect(options),
+		listFollowUsers: (options) => listFollowUsersViaBirdEffect(options),
+		listHomeTimeline: (options) => listHomeTimelineViaBirdEffect(options),
+		listLikes: (options) => listLikedTweetsViaBirdEffect(options),
+		listMentions: (options) => listMentionsViaBirdEffect(options),
+		listThread: (options) => listThreadViaBirdEffect(options),
+		searchTweets: (query, options) => searchTweetsViaBirdEffect(query, options),
+	},
+	xurl: {
+		getTransportStatus: (...args) =>
+			fromPromise(() => getTransportStatus(...args)),
+		getTweetById: (id, options) => getTweetByIdEffect(id, options),
+		listBookmarks: (...args) =>
+			fromPromise(() => listBookmarkedTweetsViaXurl(...args)),
+		listDirectMessages: (options) =>
+			listDirectMessageEventsViaXurlEffect(options),
+		listFollowUsers: (...args) =>
+			fromPromise(() => listFollowUsersViaXurl(...args)),
+		listHomeTimeline: (options) => listHomeTimelineViaXurlEffect(options),
+		listLikes: (...args) => fromPromise(() => listLikedTweetsViaXurl(...args)),
+		listMentions: (...args) => fromPromise(() => listMentionsViaXurl(...args)),
+		listUserTweets: (...args) => fromPromise(() => listUserTweets(...args)),
+		lookupAuthenticatedOAuth2User: (username) =>
+			lookupAuthenticatedOAuth2UserEffect(username),
+		lookupAuthenticatedUser: (...args) =>
+			fromPromise(() => lookupAuthenticatedUser(...args)),
+		lookupUsersByHandles: (...args) =>
+			fromPromise(() => lookupUsersByHandles(...args)),
+		searchConversation: (conversationId, options) =>
+			searchRecentByConversationIdEffect(conversationId, options),
+		searchRecentTweets: (query, options) =>
+			searchRecentTweetsEffect(query, options),
+	},
+};

package/src/lib/mention-threads-live.ts

@@ -1,10 +1,11 @@
 import type { Database } from "./sqlite";
 import { Effect } from "effect";
-import { listThreadViaBirdEffect } from "./bird";
+import { databaseWriteEffect } from "./database-writer";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
-import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
-import { tweetEntitiesFromXurl } from "./tweet-render";
+import { liveTransportGateway } from "./live-transport-gateway";
+import { resolveLiveSyncAccount } from "./live-sync-engine";
+import { runSyncPlanEffect } from "./sync-plan";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
@@ -12,9 +13,7 @@ import type {
 	XurlMediaItem,
 	XurlTweetsResponse,
 } from "./types";
-import { upsertTweetAccountEdge } from "./tweet-account-edges";
-import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
-import { getTweetByIdEffect, searchRecentByConversationIdEffect } from "./xurl";
+import { ingestTweetPayload } from "./tweet-repository";
 
 const DEFAULT_LIMIT = 30;
 const DEFAULT_DELAY_MS = 1500;
@@ -107,40 +106,6 @@ function getRemainingThreadTimeoutMs(
 	}
 	return remainingMs;
 }
-function replaceTweetFts(db: Database, tweetId: string, text: string) {
-	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
-	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
-		tweetId,
-		text,
-	);
-}
-
-function resolveAccount(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare("select id, handle from accounts where id = ?")
-				.get(accountId) as { id: string; handle: string } | undefined)
-		: (db
-				.prepare(
-					`
-          select id, handle
-          from accounts
-          order by is_default desc, created_at asc
-          limit 1
-          `,
-				)
-				.get() as { id: string; handle: string } | undefined);
-
-	if (!row) {
-		throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	}
-
-	return {
-		accountId: row.id,
-		handle: row.handle.replace(/^@/, "").toLowerCase(),
-	};
-}
-
 function getReplyToId(tweet: XurlMentionData) {
 	return tweet.referenced_tweets?.find((entry) => entry.type === "replied_to")
 		?.id;
@@ -354,84 +319,32 @@ function mergeMentionThreadIntoLocalStore({
 	const usersById = new Map(
 		(payload.includes?.users ?? []).map((user) => [user.id, user]),
 	);
-	const upsertTweet = db.prepare(
-		`
-    insert into tweets (
-      id, account_id, author_profile_id, kind, text, created_at,
-      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
-      entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?, null)
-    on conflict(id) do update set
-      account_id = excluded.account_id,
-      author_profile_id = excluded.author_profile_id,
-      kind = case
-        when tweets.kind in ('authored', 'home', 'mention') then tweets.kind
-        when excluded.kind in ('authored', 'home', 'mention') then excluded.kind
-        else coalesce(nullif(tweets.kind, ''), excluded.kind)
-      end,
-      text = excluded.text,
-      created_at = excluded.created_at,
-      is_replied = max(tweets.is_replied, excluded.is_replied),
-      reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
-      like_count = excluded.like_count,
-      media_count = max(tweets.media_count, excluded.media_count),
-      entities_json = excluded.entities_json,
-      media_json = case
-        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
-        else tweets.media_json
-      end,
-      bookmarked = tweets.bookmarked,
-      liked = tweets.liked
-    `,
-	);
-
-	db.transaction(() => {
-		const seenAt = new Date().toISOString();
-		for (const tweet of payload.data) {
-			const author =
-				usersById.get(tweet.author_id) ??
-				({
-					id: tweet.author_id,
-					username: `user_${tweet.author_id}`,
-					name: `user_${tweet.author_id}`,
-				} as const);
-			const profile = usersById.has(tweet.author_id)
-				? upsertProfileFromXUser(db, author)
-				: ensureStubProfileForXUser(db, tweet.author_id);
-			const handle = author.username.toLowerCase();
-			const kind = mentionIds.has(tweet.id)
-				? "mention"
-				: handle === accountHandle
-					? "home"
-					: "thread";
-			const replyToId = getReplyToId(tweet);
-			upsertTweet.run(
-				tweet.id,
-				accountId,
-				profile.profile.id,
-				kind,
-				tweet.text,
-				tweet.created_at,
-				replyToId ? 1 : 0,
-				replyToId ?? null,
-				Number(tweet.public_metrics?.like_count ?? 0),
-				countTweetMedia(tweet),
-				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
-				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
-			);
-			if (writeThreadContextEdges) {
-				upsertTweetAccountEdge(db, {
-					accountId,
-					tweetId: tweet.id,
-					kind: "thread_context",
-					source,
-					seenAt,
-					rawJson: JSON.stringify(tweet),
-				});
-			}
-			replaceTweetFts(db, tweet.id, tweet.text);
-		}
-	})();
+	const groups = new Map<"mention" | "home" | "thread", typeof payload.data>([
+		["mention", []],
+		["home", []],
+		["thread", []],
+	]);
+	for (const tweet of payload.data) {
+		const handle = usersById.get(tweet.author_id)?.username.toLowerCase();
+		const kind = mentionIds.has(tweet.id)
+			? "mention"
+			: handle === accountHandle
+				? "home"
+				: "thread";
+		groups.get(kind)?.push(tweet);
+	}
+	for (const [kind, data] of groups) {
+		if (data.length === 0) continue;
+		ingestTweetPayload(db, {
+			accountId,
+			payload: { ...payload, data },
+			kind,
+			edgeKind: writeThreadContextEdges ? "thread_context" : undefined,
+			markRepliesAsReplied: true,
+			replaceSecondaryKind: true,
+			source,
+		});
+	}
 }
 
 function fetchConversationViaRecentSearchEffect({
@@ -448,39 +361,29 @@ function fetchConversationViaRecentSearchEffect({
 	deadlineMs: number;
 }) {
 	return Effect.gen(function* () {
-		const pages: XurlTweetsResponse[] = [];
-		let nextToken: string | undefined;
-		let pageCount = 0;
-
-		do {
-			const payload = yield* searchRecentByConversationIdEffect(
-				conversationId,
-				{
-					maxResults: MAX_XURL_SEARCH_RESULTS,
-					paginationToken: nextToken,
-					timeoutMs: yield* trySync(() =>
-						getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
+		const result = yield* runSyncPlanEffect({
+			fetchPage: ({ cursor }) =>
+				trySync(() => getRemainingThreadTimeoutMs(deadlineMs, timeoutMs)).pipe(
+					Effect.flatMap((remainingTimeoutMs) =>
+						liveTransportGateway.xurl.searchConversation(conversationId, {
+							maxResults: MAX_XURL_SEARCH_RESULTS,
+							paginationToken: cursor,
+							timeoutMs: remainingTimeoutMs,
+						}),
 					),
-				},
-			);
-			pages.push(payload);
-			nextToken =
-				typeof payload.meta?.next_token === "string"
-					? payload.meta.next_token
-					: undefined;
-			pageCount += 1;
-		} while (
-			(all || maxPages !== undefined) &&
-			nextToken &&
-			(maxPages === undefined || pageCount < maxPages)
-		);
-
-		const payload = mergePayloads(pages);
+				),
+			getNextCursor: (page) =>
+				typeof page.meta?.next_token === "string"
+					? page.meta.next_token
+					: undefined,
+			maxPages: all || maxPages !== undefined ? maxPages : 1,
+		});
+		const payload = mergePayloads(result.pages);
 		const paginationRequested = all || maxPages !== undefined;
 		return {
 			payload,
-			pages: pageCount,
-			truncated: paginationRequested && Boolean(nextToken),
+			pages: result.pages.length,
+			truncated: paginationRequested && Boolean(result.nextCursor),
 			generalReadTweets: payload.data.length,
 		};
 	});
@@ -512,11 +415,14 @@ function fetchParentChainViaXurlEffect({
 		let shouldUseRawAnchor = Boolean(rawAnchorPayload);
 
 		if (!nextParentId) {
-			const anchorPayload = yield* getTweetByIdEffect(mention.id, {
-				timeoutMs: yield* trySync(() =>
-					getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
-				),
-			});
+			const anchorPayload = yield* liveTransportGateway.xurl.getTweetById(
+				mention.id,
+				{
+					timeoutMs: yield* trySync(() =>
+						getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
+					),
+				},
+			);
 			pages.push(anchorPayload);
 			generalReadTweets += anchorPayload.data.length;
 			const anchorTweet = anchorPayload.data[0];
@@ -548,11 +454,14 @@ function fetchParentChainViaXurlEffect({
 			}
 
 			fallbackDepth += 1;
-			const parentPayload = yield* getTweetByIdEffect(nextParentId, {
-				timeoutMs: yield* trySync(() =>
-					getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
-				),
-			});
+			const parentPayload = yield* liveTransportGateway.xurl.getTweetById(
+				nextParentId,
+				{
+					timeoutMs: yield* trySync(() =>
+						getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
+					),
+				},
+			);
 			pages.push(parentPayload);
 			generalReadTweets += parentPayload.data.length;
 			const parentTweet = parentPayload.data[0];
@@ -743,7 +652,9 @@ export function syncMentionThreadsEffect({
 			parseNonNegativeInteger(maxPages, "--max-pages"),
 		);
 		const db = yield* trySync(() => getNativeDb());
-		const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
+		const resolvedAccount = yield* trySync(() =>
+			resolveLiveSyncAccount(db, account),
+		);
 		const mentions = yield* trySync(() =>
 			tweetIds
 				? listMentionsByIds(
@@ -779,22 +690,24 @@ export function syncMentionThreadsEffect({
 			}
 			const fetchEffect: Effect.Effect<ThreadFetchResult, unknown, never> =
 				parsedMode === "bird"
-					? listThreadViaBirdEffect({
-							tweetId: mention.id,
-							all,
-							maxPages: parsedMaxPages,
-							timeoutMs: parsedTimeoutMs,
-						}).pipe(
-							Effect.map((payload) => ({
-								strategy: "bird" as const,
-								payload,
-								pages: undefined,
-								fallbackDepth: undefined,
-								generalReadTweets: 0,
-								truncated: undefined,
-								warnings: [] as string[],
-							})),
-						)
+					? liveTransportGateway.bird
+							.listThread({
+								tweetId: mention.id,
+								all,
+								maxPages: parsedMaxPages,
+								timeoutMs: parsedTimeoutMs,
+							})
+							.pipe(
+								Effect.map((payload) => ({
+									strategy: "bird" as const,
+									payload,
+									pages: undefined,
+									fallbackDepth: undefined,
+									generalReadTweets: 0,
+									truncated: undefined,
+									warnings: [] as string[],
+								})),
+							)
 					: fetchThreadContextViaXurlEffect({
 							mention,
 							all,
@@ -804,11 +717,11 @@ export function syncMentionThreadsEffect({
 						});
 			const fetched = yield* fetchEffect.pipe(
 				Effect.flatMap((fetchResult) =>
-					trySync(() =>
+					databaseWriteEffect((writeDb) =>
 						mergeMentionThreadIntoLocalStore({
-							db,
+							db: writeDb,
 							accountId: resolvedAccount.accountId,
-							accountHandle: resolvedAccount.handle,
+							accountHandle: resolvedAccount.username.toLowerCase(),
 							mentionIds: mentionIdSet,
 							payload: fetchResult.payload,
 							source: parsedMode,

package/src/lib/mentions-export.ts

@@ -1,4 +1,4 @@
-import { listTimelineItems } from "./queries";
+import { listTimelineItems } from "./timeline-read-model";
 import { renderTweetMarkdown, renderTweetPlainText } from "./tweet-render";
 import type {
 	ReplyFilter,