birdclaw 0.8.2 → 0.8.3

package/src/lib/timeline-collections-live.ts

@@ -1,26 +1,21 @@
 import type { Database } from "./sqlite";
 import { Effect } from "effect";
-import {
-	listBookmarkedTweetsViaBirdEffect,
-	listLikedTweetsViaBirdEffect,
-} from "./bird";
 import { getNativeDb } from "./db";
-import { runEffectPromise, tryPromise } from "./effect-runtime";
-import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
-import { readSyncCache, writeSyncCache } from "./sync-cache";
-import { tweetEntitiesFromXurl } from "./tweet-render";
+import { runEffectPromise } from "./effect-runtime";
+import { liveTransportGateway } from "./live-transport-gateway";
+import {
+	createLiveTransportAdapter,
+	normalizeCacheTtlMs,
+	resolveLiveSyncAccount,
+	runCachedLiveSyncEffect,
+} from "./live-sync-engine";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
 	XurlMediaItem,
 	XurlMentionUser,
 } from "./types";
-import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
-import {
-	listBookmarkedTweetsViaXurl,
-	listLikedTweetsViaXurl,
-	lookupUsersByHandles,
-} from "./xurl";
+import { ingestTweetPayload } from "./tweet-repository";
 
 export type TimelineCollectionKind = "likes" | "bookmarks";
 export type TimelineCollectionMode = "auto" | "xurl" | "bird";
@@ -52,13 +47,6 @@ function trySync<T>(try_: () => T) {
 	});
 }
 
-function parseCacheTtlMs(value?: number) {
-	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
-		return DEFAULT_COLLECTION_CACHE_TTL_MS;
-	}
-	return Math.floor(value);
-}
-
 function parseMaxPages(value?: number) {
 	if (value === undefined) {
 		return null;
@@ -81,57 +69,6 @@ function assertXurlLimit(limit: number) {
 	}
 }
 
-function resolveAccount(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare(
-					"select id, handle, external_user_id from accounts where id = ?",
-				)
-				.get(accountId) as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined)
-		: (db
-				.prepare(
-					`
-          select id, handle, external_user_id
-          from accounts
-          order by is_default desc, created_at asc
-          limit 1
-          `,
-				)
-				.get() as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined);
-
-	if (!row) {
-		throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	}
-
-	return {
-		accountId: row.id,
-		username: row.handle.replace(/^@/, ""),
-		externalUserId:
-			typeof row.external_user_id === "string" &&
-			row.external_user_id.length > 0
-				? row.external_user_id
-				: undefined,
-	};
-}
-
-function replaceTweetFts(db: Database, tweetId: string, text: string) {
-	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
-	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
-		tweetId,
-		text,
-	);
-}
-
-function getReferencedTweetId(tweet: XurlMentionData, type: string) {
-	return (
-		tweet.referenced_tweets?.find((item) => item.type === type)?.id ?? null
-	);
-}
-
 function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 	const tweets: XurlMentionData[] = [];
 	const seenTweetIds = new Set<string>();
@@ -237,95 +174,16 @@ function mergeTimelineCollectionIntoLocalStore(
 	payload: XurlMentionsResponse,
 	source: "xurl" | "bird",
 ) {
-	const usersById = new Map(
-		(payload.includes?.users ?? []).map((user) => [user.id, user]),
-	);
 	const tweetKind = kind === "likes" ? "like" : "bookmark";
-	const liked = kind === "likes" ? 1 : 0;
-	const bookmarked = kind === "bookmarks" ? 1 : 0;
-	const upsertTweet = db.prepare(
-		`
-    insert into tweets (
-      id, account_id, author_profile_id, kind, text, created_at,
-      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
-      entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    on conflict(id) do update set
-      account_id = tweets.account_id,
-      author_profile_id = excluded.author_profile_id,
-      kind = case
-        when tweets.kind in ('authored', 'home', 'mention') then tweets.kind
-        else excluded.kind
-      end,
-      text = excluded.text,
-      created_at = excluded.created_at,
-      like_count = excluded.like_count,
-      media_count = max(tweets.media_count, excluded.media_count),
-      entities_json = excluded.entities_json,
-      media_json = case
-        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
-        else tweets.media_json
-      end,
-      is_replied = max(tweets.is_replied, excluded.is_replied),
-      reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
-      quoted_tweet_id = coalesce(excluded.quoted_tweet_id, tweets.quoted_tweet_id),
-      bookmarked = tweets.bookmarked,
-      liked = tweets.liked
-    `,
-	);
-	const upsertCollection = db.prepare(`
-    insert into tweet_collections (
-      account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
-    ) values (?, ?, ?, null, ?, ?, ?)
-    on conflict(account_id, tweet_id, kind) do update set
-      source = excluded.source,
-      raw_json = excluded.raw_json,
-      updated_at = excluded.updated_at
-  `);
-
-	db.transaction(() => {
-		const updatedAt = new Date().toISOString();
-		for (const tweet of payload.data) {
-			const author =
-				usersById.get(tweet.author_id) ??
-				({
-					id: tweet.author_id,
-					username: `user_${tweet.author_id}`,
-					name: `user_${tweet.author_id}`,
-				} as const);
-			const profile = usersById.has(tweet.author_id)
-				? upsertProfileFromXUser(db, author)
-				: ensureStubProfileForXUser(db, tweet.author_id);
-			const replyToId = getReferencedTweetId(tweet, "replied_to");
-			const quotedTweetId = getReferencedTweetId(tweet, "quoted");
-			upsertTweet.run(
-				tweet.id,
-				accountId,
-				profile.profile.id,
-				tweetKind,
-				tweet.text,
-				tweet.created_at,
-				replyToId ? 1 : 0,
-				replyToId,
-				Number(tweet.public_metrics?.like_count ?? 0),
-				countTweetMedia(tweet),
-				bookmarked,
-				liked,
-				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
-				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
-				quotedTweetId,
-			);
-			upsertCollection.run(
-				accountId,
-				tweet.id,
-				kind,
-				source,
-				JSON.stringify(tweet),
-				updatedAt,
-			);
-			replaceTweetFts(db, tweet.id, tweet.text);
-		}
-	})();
+	ingestTweetPayload(db, {
+		accountId,
+		payload,
+		kind: tweetKind,
+		collectionKind: kind,
+		markRepliesAsReplied: true,
+		replaceSecondaryKind: true,
+		source,
+	});
 }
 
 function fetchXurlCollectionEffect({
@@ -352,9 +210,8 @@ function fetchXurlCollectionEffect({
 	return Effect.gen(function* () {
 		let resolvedUserId = userId;
 		if (!resolvedUserId) {
-			const [accountUser] = yield* tryPromise(() =>
-				lookupUsersByHandles([username]),
-			);
+			const [accountUser] =
+				yield* liveTransportGateway.xurl.lookupUsersByHandles([username]);
 			if (!accountUser?.id) {
 				return yield* Effect.fail(
 					new Error(`Could not resolve Twitter user id for @${username}`),
@@ -368,22 +225,20 @@ function fetchXurlCollectionEffect({
 		let pageCount = 0;
 		let saturatedAtPage: number | undefined;
 		do {
-			const payload = yield* tryPromise(() =>
-				kind === "likes"
-					? listLikedTweetsViaXurl({
-							maxResults: limit,
-							username,
-							userId: resolvedUserId,
-							paginationToken: nextToken,
-						})
-					: listBookmarkedTweetsViaXurl({
-							maxResults: limit,
-							username,
-							userId: resolvedUserId,
-							isPaginatedWalk: all,
-							paginationToken: nextToken,
-						}),
-			);
+			const payload = yield* kind === "likes"
+				? liveTransportGateway.xurl.listLikes({
+						maxResults: limit,
+						username,
+						userId: resolvedUserId,
+						paginationToken: nextToken,
+					})
+				: liveTransportGateway.xurl.listBookmarks({
+						maxResults: limit,
+						username,
+						userId: resolvedUserId,
+						isPaginatedWalk: all,
+						paginationToken: nextToken,
+					});
 			pageCount += 1;
 			if (earlyStop) {
 				const tweetIds = payload.data.map((tweet) => tweet.id);
@@ -438,12 +293,12 @@ function fetchBirdCollectionEffect({
 	maxPages: number | null;
 }) {
 	return kind === "likes"
-		? listLikedTweetsViaBirdEffect({
+		? liveTransportGateway.bird.listLikes({
 				maxResults: limit,
 				all,
 				maxPages: maxPages ?? undefined,
 			})
-		: listBookmarkedTweetsViaBirdEffect({
+		: liveTransportGateway.bird.listBookmarks({
 				maxResults: limit,
 				all,
 				maxPages: maxPages ?? undefined,
@@ -474,31 +329,11 @@ export function syncTimelineCollectionEffect({
 		}
 
 		const db = yield* trySync(() => getNativeDb());
-		const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
+		const resolvedAccount = yield* trySync(() =>
+			resolveLiveSyncAccount(db, account),
+		);
 		const cacheMaxPages = mode === "bird" ? parsedMaxPages : xurlMaxPages;
 		const cacheKey = `${kind}:${mode}:${resolvedAccount.accountId}:${String(limit)}:${all ? "all" : "single"}:${cacheMaxPages === null ? "all-pages" : String(cacheMaxPages)}${earlyStop ? ":early-stop" : ""}`;
-		const ttlMs = parseCacheTtlMs(cacheTtlMs);
-		const cached = yield* trySync(() =>
-			readSyncCache<XurlMentionsResponse>(cacheKey, db),
-		);
-		const cacheAgeMs = cached
-			? Date.now() - new Date(cached.updatedAt).getTime()
-			: Number.POSITIVE_INFINITY;
-
-		if (!refresh && cached && cacheAgeMs <= ttlMs) {
-			const saturatedAtPage = readSaturatedAtPage(cached.value);
-			return {
-				ok: true,
-				source: "cache",
-				kind,
-				accountId: resolvedAccount.accountId,
-				count: cached.value.data.length,
-				payload: cached.value,
-				...(saturatedAtPage === undefined
-					? {}
-					: { saturated_at_page: saturatedAtPage }),
-			};
-		}
 
 		if (shouldApplyEarlyStopCap) {
 			console.error(
@@ -506,60 +341,51 @@ export function syncTimelineCollectionEffect({
 			);
 		}
 
-		let source: "xurl" | "bird";
-		let payload: XurlMentionsResponse;
-		if (mode === "bird") {
-			payload = yield* fetchBirdCollectionEffect({
-				kind,
-				limit,
-				all,
-				maxPages: parsedMaxPages,
-			});
-			source = "bird";
-		} else {
-			const xurlPayload = yield* fetchXurlCollectionEffect({
-				db,
-				kind,
-				accountId: resolvedAccount.accountId,
-				username: resolvedAccount.username,
-				userId: resolvedAccount.externalUserId,
-				limit,
-				all,
-				maxPages: xurlMaxPages,
-				earlyStop,
-			}).pipe(
-				Effect.map((value) => ({ ok: true as const, value })),
-				Effect.catchAll((error) => {
-					if (mode === "xurl") {
-						return Effect.fail(error);
-					}
-					return Effect.succeed({ ok: false as const });
-				}),
-			);
-			if (xurlPayload.ok) {
-				payload = xurlPayload.value;
-				source = "xurl";
-			} else {
-				payload = yield* fetchBirdCollectionEffect({
-					kind,
-					limit,
-					all,
-					maxPages: parsedMaxPages,
-				});
-				source = "bird";
-			}
-		}
-
-		yield* trySync(() =>
-			mergeTimelineCollectionIntoLocalStore(
-				db,
-				resolvedAccount.accountId,
-				kind,
-				payload,
-				source,
+		const xurlFetch = fetchXurlCollectionEffect({
+			db,
+			kind,
+			accountId: resolvedAccount.accountId,
+			username: resolvedAccount.username,
+			userId: resolvedAccount.externalUserId,
+			limit,
+			all,
+			maxPages: xurlMaxPages,
+			earlyStop,
+		});
+		const birdFetch = fetchBirdCollectionEffect({
+			kind,
+			limit,
+			all,
+			maxPages: parsedMaxPages,
+		});
+		const transports =
+			mode === "bird"
+				? [createLiveTransportAdapter("bird", birdFetch)]
+				: mode === "xurl"
+					? [createLiveTransportAdapter("xurl", xurlFetch)]
+					: [
+							createLiveTransportAdapter("xurl", xurlFetch),
+							createLiveTransportAdapter("bird", birdFetch),
+						];
+		const syncResult = yield* runCachedLiveSyncEffect({
+			db,
+			cacheKey,
+			refresh,
+			cacheTtlMs: normalizeCacheTtlMs(
+				cacheTtlMs,
+				DEFAULT_COLLECTION_CACHE_TTL_MS,
 			),
-		);
-		yield* trySync(() => writeSyncCache(cacheKey, payload, db));
+			transports,
+			persistLive: (writeDb, livePayload, liveSource) =>
+				mergeTimelineCollectionIntoLocalStore(
+					writeDb,
+					resolvedAccount.accountId,
+					kind,
+					livePayload,
+					liveSource,
+				),
+		});
+		const { source, payload } = syncResult;
 		const saturatedAtPage = readSaturatedAtPage(payload);
 
 		return {