birdclaw 0.5.1 → 0.7.0

package/src/lib/web-sync.ts

@@ -1,12 +1,14 @@
 import { existsSync } from "node:fs";
-import { maybeAutoSyncBackup } from "./backup";
+import { Effect } from "effect";
+import { maybeAutoSyncBackupEffect } from "./backup";
 import { getBirdclawPaths } from "./config";
-import { syncDirectMessagesViaCachedBird } from "./dms-live";
-import { syncMentionThreads } from "./mention-threads-live";
-import { syncMentions } from "./mentions-live";
+import { syncDirectMessagesViaCachedBirdEffect } from "./dms-live";
+import { runEffectBackground, runEffectPromise } from "./effect-runtime";
+import { syncMentionThreadsEffect } from "./mention-threads-live";
+import { syncMentionsEffect } from "./mentions-live";
 import NativeSqliteDatabase from "./sqlite";
-import { syncTimelineCollection } from "./timeline-collections-live";
-import { syncHomeTimeline } from "./timeline-live";
+import { syncTimelineCollectionEffect } from "./timeline-collections-live";
+import { syncHomeTimelineEffect } from "./timeline-live";
 
 export type WebSyncKind =
 	| "timeline"
@@ -33,7 +35,7 @@ export interface WebSyncResponse {
 	summary: string;
 	steps: WebSyncStep[];
 	inProgress?: boolean;
-	backup?: Awaited<ReturnType<typeof maybeAutoSyncBackup>>;
+	backup?: Effect.Effect.Success<ReturnType<typeof maybeAutoSyncBackupEffect>>;
 	error?: string;
 }
 
@@ -52,10 +54,22 @@ export interface WebSyncJobSnapshot {
 	error?: string;
 }
 
+export type WebSyncDmInbox = "all" | "accepted" | "requests";
+
+export interface WebSyncOptions {
+	inbox?: WebSyncDmInbox;
+	limit?: number;
+	maxPages?: number;
+	allPages?: boolean;
+}
+
 interface WebSyncPlan {
 	label: string;
 	accountAware: boolean;
-	run: (accountId: string | undefined) => Promise<WebSyncStep[]>;
+	run: (
+		accountId: string | undefined,
+		options: WebSyncOptions,
+	) => Effect.Effect<WebSyncStep[], unknown>;
 }
 
 const runningSyncs = new Map<string, WebSyncJobSnapshot>();
@@ -93,6 +107,10 @@ function readBoolean(value: unknown, key: string) {
 	return typeof raw === "boolean" ? raw : undefined;
 }
 
+function messageFromError(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
 export function parseWebSyncKind(value: unknown): WebSyncKind | null {
 	return value === "timeline" ||
 		value === "mentions" ||
@@ -113,65 +131,72 @@ function summarizeSteps(steps: WebSyncStep[]) {
 const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
 	timeline: {
 		label: "Home timeline",
-		accountAware: false,
-		run: async (account) => {
-			const result = await syncHomeTimeline({
-				account,
-				limit: 100,
-				following: true,
-				refresh: true,
-			});
-			return [
-				{
-					kind: "timeline",
-					label: "Home timeline",
-					count: readNumber(result, "count"),
-					source: readString(result, "source"),
-				},
-			];
-		},
+		accountAware: true,
+		run: (account) =>
+			Effect.gen(function* () {
+				const result = yield* syncHomeTimelineEffect({
+					account,
+					mode:
+						!account || account === resolveDefaultSyncAccountId()
+							? "auto"
+							: "xurl",
+					limit: 100,
+					maxPages: 3,
+					following: true,
+					refresh: true,
+				});
+				return [
+					{
+						kind: "timeline",
+						label: "Home timeline",
+						count: readNumber(result, "count"),
+						source: readString(result, "source"),
+					},
+				];
+			}),
 	},
 	mentions: {
 		label: "Mentions",
 		accountAware: true,
-		run: async (account) => {
-			const mentions = await syncMentions({
-				account,
-				mode: "xurl",
-				limit: 100,
-				maxPages: 3,
-				refresh: true,
-			});
-			const steps: WebSyncStep[] = [
-				{
-					kind: "mentions",
-					label: "Mentions",
-					count: readNumber(mentions, "count"),
-					source: readString(mentions, "source"),
-					partial: readBoolean(mentions, "partial"),
-				},
-			];
-
-			const threads = await syncMentionThreads({
-				account,
-				mode: "xurl",
-				limit: 30,
-				delayMs: 1500,
-				timeoutMs: 15000,
-			});
-			steps.push({
-				kind: "mention-threads",
-				label: "Mention threads",
-				count: readNumber(threads, "mergedTweets"),
-				source: readString(threads, "source"),
-				partial: readBoolean(threads, "partial"),
-				warnings:
-					Array.isArray(threads.warnings) && threads.warnings.length > 0
-						? threads.warnings.map(String)
-						: undefined,
-			});
-			return steps;
-		},
+		run: (account) =>
+			Effect.gen(function* () {
+				const mentions = yield* syncMentionsEffect({
+					account,
+					mode: "auto",
+					limit: 100,
+					maxPages: 3,
+					refresh: true,
+				});
+				const steps: WebSyncStep[] = [
+					{
+						kind: "mentions",
+						label: "Mentions",
+						count: readNumber(mentions, "count"),
+						source: readString(mentions, "source"),
+						partial: readBoolean(mentions, "partial"),
+					},
+				];
+
+				const threads = yield* syncMentionThreadsEffect({
+					account,
+					mode: "xurl",
+					limit: 30,
+					delayMs: 1500,
+					timeoutMs: 15000,
+				});
+				steps.push({
+					kind: "mention-threads",
+					label: "Mention threads",
+					count: readNumber(threads, "mergedTweets"),
+					source: readString(threads, "source"),
+					partial: readBoolean(threads, "partial"),
+					warnings:
+						Array.isArray(threads.warnings) && threads.warnings.length > 0
+							? threads.warnings.map(String)
+							: undefined,
+				});
+				return steps;
+			}),
 	},
 	likes: {
 		label: "Likes",
@@ -186,68 +211,83 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
 	dms: {
 		label: "Direct messages",
 		accountAware: false,
-		run: async (account) => {
-			const result = await syncDirectMessagesViaCachedBird({
-				account,
-				limit: 50,
-				refresh: true,
-			});
-			return [
-				{
-					kind: "dms",
-					label: "Direct messages",
-					count: readNumber(result, "messages"),
-					source: readString(result, "source"),
-				},
-			];
-		},
+		run: (account, options) =>
+			Effect.gen(function* () {
+				const inbox = options.inbox ?? "all";
+				const result = yield* syncDirectMessagesViaCachedBirdEffect({
+					account,
+					inbox,
+					limit: options.limit ?? (inbox === "requests" ? 200 : 50),
+					...(options.maxPages !== undefined
+						? { maxPages: options.maxPages }
+						: {}),
+					...(options.allPages !== undefined
+						? { allPages: options.allPages }
+						: {}),
+					pageDelayMs: inbox === "requests" ? 750 : 0,
+					refresh: true,
+				});
+				return [
+					{
+						kind: "dms",
+						label: "Direct messages",
+						count: readNumber(result, "messages"),
+						source: readString(result, "source"),
+					},
+				];
+			}),
 	},
 };
 
-async function syncSavedCollection(
+function syncSavedCollection(
 	kind: "likes" | "bookmarks",
 	account: string | undefined,
-) {
-	const isNonDefaultAccount =
-		account !== undefined && account !== resolveDefaultSyncAccountId();
-	const result = await syncTimelineCollection({
-		kind,
-		account,
-		mode: isNonDefaultAccount ? "xurl" : "auto",
-		limit: 100,
-		maxPages: 5,
-		refresh: true,
-		earlyStop: true,
-	});
-	return [
-		{
+): Effect.Effect<WebSyncStep[], unknown> {
+	return Effect.gen(function* () {
+		const isNonDefaultAccount =
+			account !== undefined && account !== resolveDefaultSyncAccountId();
+		const result = yield* syncTimelineCollectionEffect({
 			kind,
-			label: kind === "likes" ? "Likes" : "Bookmarks",
-			count: readNumber(result, "count"),
-			source: readString(result, "source"),
-		},
-	];
+			account,
+			mode: isNonDefaultAccount ? "xurl" : "auto",
+			limit: 100,
+			maxPages: 5,
+			refresh: true,
+			earlyStop: true,
+		});
+		return [
+			{
+				kind,
+				label: kind === "likes" ? "Likes" : "Bookmarks",
+				count: readNumber(result, "count"),
+				source: readString(result, "source"),
+			},
+		];
+	});
 }
 
-async function performWebSync(
+export function performWebSyncEffect(
 	kind: WebSyncKind,
 	accountId?: string,
-): Promise<WebSyncResponse> {
-	const startedAt = new Date().toISOString();
-	const steps = await WEB_SYNC_PLANS[kind].run(accountId);
+	options: WebSyncOptions = {},
+) {
+	return Effect.gen(function* () {
+		const startedAt = new Date().toISOString();
+		const steps = yield* WEB_SYNC_PLANS[kind].run(accountId, options);
 
-	const backup = await maybeAutoSyncBackup();
-	const finishedAt = new Date().toISOString();
-	return {
-		ok: true,
-		kind,
-		...(accountId ? { accountId } : {}),
-		startedAt,
-		finishedAt,
-		summary: summarizeSteps(steps),
-		steps,
-		backup,
-	};
+		const backup = yield* maybeAutoSyncBackupEffect();
+		const finishedAt = new Date().toISOString();
+		return {
+			ok: true,
+			kind,
+			...(accountId ? { accountId } : {}),
+			startedAt,
+			finishedAt,
+			summary: summarizeSteps(steps),
+			steps,
+			backup,
+		} satisfies WebSyncResponse;
+	});
 }
 
 function createWebSyncJobId(kind: WebSyncKind) {
@@ -281,9 +321,25 @@ function resolveDefaultSyncAccountId() {
 	}
 }
 
-function getRunningSyncKey(kind: WebSyncKind, accountId: string | undefined) {
+function serializeSyncOptions(kind: WebSyncKind, options: WebSyncOptions) {
+	if (kind !== "dms") return "";
+	const parts = [
+		options.inbox ?? "all",
+		options.limit ?? "",
+		options.maxPages ?? "",
+		options.allPages === undefined ? "" : String(options.allPages),
+	];
+	return parts.join(":");
+}
+
+function getRunningSyncKey(
+	kind: WebSyncKind,
+	accountId: string | undefined,
+	options: WebSyncOptions = {},
+) {
 	if (!WEB_SYNC_PLANS[kind].accountAware) {
-		return kind;
+		const optionKey = serializeSyncOptions(kind, options);
+		return optionKey ? `${kind}:${optionKey}` : kind;
 	}
 	return `${kind}:${accountId ?? resolveDefaultSyncAccountId()}`;
 }
@@ -326,7 +382,7 @@ function toFailedResponse(
 	accountId?: string,
 ): WebSyncResponse {
 	const finishedAt = new Date().toISOString();
-	const message = error instanceof Error ? error.message : "Sync failed";
+	const message = messageFromError(error);
 	return {
 		ok: false,
 		kind,
@@ -342,9 +398,10 @@ function toFailedResponse(
 export function startWebSync(
 	kind: WebSyncKind,
 	accountId?: string,
+	options: WebSyncOptions = {},
 ): WebSyncJobSnapshot {
 	const effectiveAccountId = getEffectiveAccountId(kind, accountId);
-	const syncKey = getRunningSyncKey(kind, effectiveAccountId);
+	const syncKey = getRunningSyncKey(kind, effectiveAccountId, options);
 	const current = runningSyncs.get(syncKey);
 	if (current) {
 		return current;
@@ -363,8 +420,8 @@ export function startWebSync(
 	webSyncJobKeys.set(job.id, syncKey);
 	setJobSnapshot(job);
 
-	void performWebSync(kind, effectiveAccountId)
-		.then((result) => {
+	runEffectBackground(performWebSyncEffect(kind, effectiveAccountId, options), {
+		onSuccess: (result) => {
 			setJobSnapshot({
 				...job,
 				status: "succeeded",
@@ -373,8 +430,8 @@ export function startWebSync(
 				inProgress: false,
 				result,
 			});
-		})
-		.catch((error: unknown) => {
+		},
+		onFailure: (error) => {
 			const result = toFailedResponse(
 				kind,
 				startedAt,
@@ -390,7 +447,8 @@ export function startWebSync(
 				result,
 				error: result.error,
 			});
-		});
+		},
+	});
 
 	return job;
 }
@@ -399,37 +457,52 @@ export function getWebSyncJob(id: string): WebSyncJobSnapshot | null {
 	return webSyncJobs.get(id) ?? null;
 }
 
-export async function runWebSync(
+export function runWebSyncEffect(
 	kind: WebSyncKind,
 	accountId?: string,
-): Promise<WebSyncResponse> {
-	const effectiveAccountId = getEffectiveAccountId(kind, accountId);
-	const current = runningSyncs.get(getRunningSyncKey(kind, effectiveAccountId));
-	const startedAt = new Date().toISOString();
-	if (current) {
-		return {
-			ok: false,
-			kind,
-			...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
-			startedAt,
-			summary: "Sync already running",
-			steps: [],
-			inProgress: true,
-		};
-	}
+	options: WebSyncOptions = {},
+): Effect.Effect<WebSyncResponse, Error> {
+	return Effect.gen(function* () {
+		const effectiveAccountId = getEffectiveAccountId(kind, accountId);
+		const current = runningSyncs.get(
+			getRunningSyncKey(kind, effectiveAccountId, options),
+		);
+		const startedAt = new Date().toISOString();
+		if (current) {
+			return {
+				ok: false,
+				kind,
+				...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
+				startedAt,
+				summary: "Sync already running",
+				steps: [],
+				inProgress: true,
+			} satisfies WebSyncResponse;
+		}
 
-	const job = startWebSync(kind, effectiveAccountId);
-	while (job.inProgress) {
-		await new Promise((resolve) => setTimeout(resolve, 25));
-		const latest = getWebSyncJob(job.id);
-		if (!latest?.inProgress) {
-			if (!latest?.result) throw new Error("Sync job disappeared");
-			return latest.result;
+		const job = startWebSync(kind, effectiveAccountId, options);
+		while (job.inProgress) {
+			yield* Effect.sleep(25);
+			const latest = getWebSyncJob(job.id);
+			if (!latest?.inProgress) {
+				if (!latest?.result)
+					return yield* Effect.fail(new Error("Sync job disappeared"));
+				return latest.result;
+			}
 		}
-	}
 
-	if (!job.result) throw new Error("Sync job did not finish");
-	return job.result;
+		if (!job.result)
+			return yield* Effect.fail(new Error("Sync job did not finish"));
+		return job.result;
+	});
+}
+
+export function runWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+	options: WebSyncOptions = {},
+): Promise<WebSyncResponse> {
+	return runEffectPromise(runWebSyncEffect(kind, accountId, options));
 }
 
 export function clearWebSyncLocksForTests() {