birdclaw 0.8.1 → 0.8.3

package/src/lib/authored-live.ts

@@ -1,8 +1,13 @@
 import type { Database } from "./sqlite";
 import { Effect } from "effect";
+import { databaseWriteEffect } from "./database-writer";
 import { getNativeDb } from "./db";
-import { runEffectPromise, tryPromise } from "./effect-runtime";
+import { runEffectPromise } from "./effect-runtime";
+import { liveTransportGateway } from "./live-transport-gateway";
+import { resolveLiveSyncAccount } from "./live-sync-engine";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { runSyncPlanEffect } from "./sync-plan";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	TweetEntities,
 	TweetMediaItem,
@@ -20,11 +25,6 @@ import {
 	ensureStubProfileForXUser,
 	upsertProfileFromXUser,
 } from "./x-profile";
-import {
-	getTransportStatus,
-	listUserTweets,
-	lookupAuthenticatedUser,
-} from "./xurl";
 
 export type AuthoredSyncMode = "xurl";
 
@@ -359,43 +359,6 @@ function getOldestTweetId(tweets: XurlMentionData[]) {
 	}, null);
 }
 
-function resolveAccount(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare(
-					"select id, handle, external_user_id from accounts where id = ?",
-				)
-				.get(accountId) as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined)
-		: (db
-				.prepare(
-					`
-          select id, handle, external_user_id
-          from accounts
-          order by is_default desc, created_at asc
-          limit 1
-          `,
-				)
-				.get() as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined);
-
-	if (!row) {
-		throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	}
-
-	return {
-		accountId: row.id,
-		username: row.handle.replace(/^@/, ""),
-		externalUserId:
-			typeof row.external_user_id === "string" &&
-			row.external_user_id.length > 0
-				? row.external_user_id
-				: undefined,
-	};
-}
-
 function normalizeUsername(value: string) {
 	return value.replace(/^@/, "").trim().toLowerCase();
 }
@@ -443,12 +406,14 @@ function resolveAuthoredIdentityEffect({
 	db: Database;
 }) {
 	return Effect.gen(function* () {
-		const status = yield* tryPromise(() => getTransportStatus());
+		const status = yield* liveTransportGateway.xurl.getTransportStatus();
 		if (status.availableTransport !== "xurl") {
 			return yield* Effect.fail(new AuthoredSyncError(status.statusText, 4));
 		}
 
-		const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
+		const resolvedAccount = yield* trySync(() =>
+			resolveLiveSyncAccount(db, account),
+		);
 		if (resolvedAccount.externalUserId) {
 			return {
 				...resolvedAccount,
@@ -457,7 +422,8 @@ function resolveAuthoredIdentityEffect({
 			};
 		}
 
-		const authenticated = yield* tryPromise(() => lookupAuthenticatedUser());
+		const authenticated =
+			yield* liveTransportGateway.xurl.lookupAuthenticatedUser();
 		const authenticatedUser = userFromAuthenticatedPayload(authenticated);
 		if (!authenticatedUser?.id) {
 			return yield* Effect.fail(
@@ -523,68 +489,7 @@ function toMentionData(tweet: XurlUserTweet, fallbackAuthorId: string) {
 }
 
 function toLocalEntities(tweet: XurlMentionData): TweetEntities {
-	const raw = tweet.entities;
-	if (!raw || typeof raw !== "object") {
-		return {};
-	}
-
-	const entities = raw as Record<string, unknown>;
-	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
-	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
-	const rawHashtags = Array.isArray(entities.hashtags) ? entities.hashtags : [];
-
-	return {
-		...(rawMentions.length
-			? {
-					mentions: rawMentions.map((mention) => {
-						const value =
-							mention && typeof mention === "object"
-								? (mention as Record<string, unknown>)
-								: {};
-						return {
-							username: String(value.username ?? ""),
-							id: typeof value.id === "string" ? String(value.id) : undefined,
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawUrls.length
-			? {
-					urls: rawUrls.map((url) => {
-						const value =
-							url && typeof url === "object"
-								? (url as Record<string, unknown>)
-								: {};
-						return {
-							url: String(value.url ?? ""),
-							expandedUrl: String(value.expanded_url ?? value.url ?? ""),
-							displayUrl: String(
-								value.display_url ?? value.expanded_url ?? value.url ?? "",
-							),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawHashtags.length
-			? {
-					hashtags: rawHashtags.map((hashtag) => {
-						const value =
-							hashtag && typeof hashtag === "object"
-								? (hashtag as Record<string, unknown>)
-								: {};
-						return {
-							tag: String(value.tag ?? ""),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-	};
+	return tweetEntitiesFromXurl(tweet.entities);
 }
 
 function toMediaType(type: string): TweetMediaItem["type"] {
@@ -971,7 +876,7 @@ export function syncAuthoredTweetsEffect({
 			archiveSinceSeed ??
 			(untilId ? persistedUntilSinceId : cursor.sinceId) ??
 			null;
-		let nextToken = usePersistedForward
+		const initialToken = usePersistedForward
 			? cursor.token
 			: usePersistedUntil
 				? cursor.token
@@ -979,19 +884,20 @@ export function syncAuthoredTweetsEffect({
 		let newestSeenId = usePersistedForward
 			? maxTweetId(cursor.sinceId, cursor.pendingNewestId)
 			: cursor.sinceId;
-		const pages: XurlUserTweetsResponse[] = [];
 		const sourceUser = toFallbackUser({
 			userId: identity.userId,
 			username: identity.username,
 			authenticatedUser: identity.authenticatedUser,
 		});
-		let pageCount = 0;
 
-		while (parsedMaxPages === null || pageCount < parsedMaxPages) {
-			const fetched = yield* tryPromise(() =>
-				listUserTweets(identity.userId, {
+		const planResult = yield* runSyncPlanEffect({
+			allowPartialFailure: true,
+			initialCursor: initialToken,
+			maxPages: parsedMaxPages ?? undefined,
+			fetchPage: ({ cursor: paginationToken }) =>
+				liveTransportGateway.xurl.listUserTweets(identity.userId, {
 					maxResults: pageLimit,
-					paginationToken: nextToken,
+					paginationToken,
 					excludeRetweets: false,
 					sinceId: effectiveSinceId ?? undefined,
 					untilId,
@@ -1002,77 +908,66 @@ export function syncAuthoredTweetsEffect({
 					auth: "oauth2",
 					username: identity.username,
 				}),
-			).pipe(
-				Effect.map((page) => ({ ok: true as const, page })),
-				Effect.catchAll((error) =>
-					Effect.succeed({ ok: false as const, error }),
-				),
-			);
-
-			if (!fetched.ok) {
-				if (pages.length === 0) {
-					return yield* Effect.fail(fetched.error);
-				}
-				const payload = mergePages({
-					pages,
-					userId: identity.userId,
-					nextToken: nextToken ?? null,
-				});
-				return buildResult({
-					accountId: identity.accountId,
+			getNextCursor: (page) => page.nextToken,
+			persistPage: ({ page, nextCursor: pendingToken }) => {
+				const pagePayload = mergePages({
+					pages: [page],
 					userId: identity.userId,
-					effectiveSinceId,
-					nextSinceId: untilId ? cursor.sinceId : effectiveSinceId,
-					nextToken: nextToken ?? null,
-					pageCount,
-					payload,
-					partial: true,
-					error: formatError(fetched.error),
+					nextToken: page.nextToken,
 				});
-			}
-
-			const page = fetched.page;
-			const pagePayload = mergePages({
-				pages: [page],
-				userId: identity.userId,
-				nextToken: page.nextToken,
-			});
-			yield* trySync(() =>
-				mergeAuthoredPayloadIntoLocalStore({
-					db,
-					accountId: identity.accountId,
-					payload: pagePayload,
-					sourceUser,
-				}),
-			);
-			pages.push(page);
-			pageCount += 1;
-			newestSeenId = maxTweetId(newestSeenId, pagePayload.meta.newest_id);
-			nextToken = page.nextToken ?? undefined;
-
-			const pendingToken = nextToken;
-			if (pendingToken && untilId) {
-				yield* trySync(() =>
-					writePendingUntilCursor(db, identity.accountId, {
-						sinceId: cursor.sinceId,
-						token: pendingToken,
-						untilId,
-						requestedSinceId: effectiveSinceId,
+				return databaseWriteEffect((writeDb) =>
+					mergeAuthoredPayloadIntoLocalStore({
+						db: writeDb,
+						accountId: identity.accountId,
+						payload: pagePayload,
+						sourceUser,
 					}),
+				).pipe(
+					Effect.flatMap(() =>
+						trySync(() => {
+							newestSeenId = maxTweetId(
+								newestSeenId,
+								pagePayload.meta.newest_id,
+							);
+							if (pendingToken && untilId) {
+								writePendingUntilCursor(db, identity.accountId, {
+									sinceId: cursor.sinceId,
+									token: pendingToken,
+									untilId,
+									requestedSinceId: effectiveSinceId,
+								});
+							} else if (pendingToken) {
+								writePendingForwardCursor(db, identity.accountId, {
+									sinceId: effectiveSinceId,
+									token: pendingToken,
+									pendingNewestId: newestSeenId,
+								});
+							}
+						}),
+					),
 				);
-			} else if (pendingToken) {
-				yield* trySync(() =>
-					writePendingForwardCursor(db, identity.accountId, {
-						sinceId: effectiveSinceId,
-						token: pendingToken,
-						pendingNewestId: newestSeenId,
-					}),
-				);
-			}
-
-			if (!nextToken) {
-				break;
-			}
+			},
+		});
+		const pages = planResult.pages;
+		const pageCount = pages.length;
+		const nextToken = planResult.nextCursor;
+		if (planResult.stopReason === "error") {
+			const payload = mergePages({
+				pages,
+				userId: identity.userId,
+				nextToken: nextToken ?? null,
+			});
+			return buildResult({
+				accountId: identity.accountId,
+				userId: identity.userId,
+				effectiveSinceId,
+				nextSinceId: untilId ? cursor.sinceId : effectiveSinceId,
+				nextToken: nextToken ?? null,
+				pageCount,
+				payload,
+				partial: true,
+				error: formatError(planResult.error),
+			});
 		}
 
 		const capped = Boolean(nextToken);