birdclaw 0.8.2 → 0.8.3

package/src/lib/dms-live.ts

@@ -4,17 +4,18 @@ import {
 	type BirdDmConversation,
 	type BirdDmEvent,
 	type BirdDmUser,
-	getAuthenticatedBirdAccountEffect,
-	listDirectMessagesViaBirdEffect,
 } from "./bird";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
+import { liveTransportGateway } from "./live-transport-gateway";
+import {
+	assertLiveAccountMatches,
+	resolveLiveSyncAccount,
+	type LiveSyncAccount,
+} from "./live-sync-engine";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { runSyncPlanEffect } from "./sync-plan";
 import type { XurlDmEventsResponse, XurlMentionUser } from "./types";
-import {
-	listDirectMessageEventsViaXurlEffect,
-	lookupAuthenticatedOAuth2UserEffect,
-} from "./xurl";
 import {
 	buildExternalProfileId,
 	randomAvatarHue,
@@ -65,11 +66,6 @@ function parseSyncMode(mode: DirectMessagesSyncMode | undefined) {
 	throw new Error("--mode must be auto, bird, or xurl");
 }
 
-function normalizeExternalUserId(value: string | null | undefined) {
-	const trimmed = value?.trim();
-	return trimmed ? trimmed : undefined;
-}
-
 function makePreviewMessageId(conversationId: string): string {
 	return `${PREVIEW_MESSAGE_ID_PREFIX}${conversationId}`;
 }
@@ -85,39 +81,6 @@ function deleteDmFtsRows(db: Database, messageIds: string[]) {
 	}
 }
 
-function resolveAccount(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare(
-					"select id, handle, external_user_id from accounts where id = ?",
-				)
-				.get(accountId) as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined)
-		: (db
-				.prepare(
-					`
-          select id, handle, external_user_id
-          from accounts
-          order by is_default desc, created_at asc
-          limit 1
-          `,
-				)
-				.get() as
-				| { id: string; handle: string; external_user_id: string | null }
-				| undefined);
-
-	if (!row) {
-		throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	}
-
-	return {
-		accountId: row.id,
-		username: row.handle.replace(/^@/, ""),
-		externalUserId: normalizeExternalUserId(row.external_user_id),
-	};
-}
-
 function toIsoTimestamp(value?: string) {
 	if (!value) {
 		return new Date().toISOString();
@@ -202,36 +165,21 @@ function getLatestEvent(events: BirdDmEvent[]) {
 
 function assertAuthenticatedBirdAccountMatches({
 	source,
-	accountId,
-	username,
-	externalUserId,
+	account,
 	liveUsername,
 	liveExternalUserId,
 }: {
 	source: "bird" | "xurl";
-	accountId: string;
-	username: string;
-	externalUserId?: string;
+	account: LiveSyncAccount;
 	liveUsername: string;
 	liveExternalUserId?: string;
 }) {
-	if (
-		externalUserId &&
-		liveExternalUserId &&
-		liveExternalUserId === externalUserId
-	) {
-		return;
-	}
-	if (externalUserId && liveExternalUserId) {
-		throw new Error(
-			`${source} is authenticated as user ${liveExternalUserId}; refusing to sync into ${accountId} (${externalUserId})`,
-		);
-	}
-	if (liveUsername.toLowerCase() !== username.toLowerCase()) {
-		throw new Error(
-			`${source} is authenticated as @${liveUsername}; refusing to sync into ${accountId} (@${username})`,
-		);
-	}
+	assertLiveAccountMatches({
+		source,
+		account,
+		liveUsername,
+		liveExternalUserId,
+	});
 }
 
 function getAuthenticatedXurlAccount(payload: Record<string, unknown> | null): {
@@ -707,10 +655,6 @@ function mergeXurlDmPages(pages: XurlDmEventsResponse[]): XurlDmEventsResponse {
 	};
 }
 
-function sleepEffect(ms: number | undefined) {
-	return typeof ms === "number" && ms > 0 ? Effect.sleep(ms) : Effect.void;
-}
-
 function fetchDirectMessagesViaXurlEffect({
 	limit,
 	username,
@@ -725,25 +669,21 @@ function fetchDirectMessagesViaXurlEffect({
 	pageDelayMs?: number;
 }) {
 	return Effect.gen(function* () {
-		const pages: XurlDmEventsResponse[] = [];
-		let paginationToken: string | undefined;
-		let pageIndex = 0;
 		const pageLimit = allPages
 			? Number.POSITIVE_INFINITY
 			: Math.max(1, (maxPages ?? 0) + 1);
-		while (pageIndex < pageLimit) {
-			const page = yield* listDirectMessageEventsViaXurlEffect({
-				maxResults: limit,
-				username,
-				...(paginationToken ? { paginationToken } : {}),
-			});
-			pages.push(page);
-			pageIndex += 1;
-			paginationToken = getMetaNextToken(page.meta);
-			if (!paginationToken) break;
-			yield* sleepEffect(pageDelayMs);
-		}
-		return mergeXurlDmPages(pages);
+		const result = yield* runSyncPlanEffect({
+			fetchPage: ({ cursor }) =>
+				liveTransportGateway.xurl.listDirectMessages({
+					maxResults: limit,
+					username,
+					...(cursor ? { paginationToken: cursor } : {}),
+				}),
+			getNextCursor: (page) => getMetaNextToken(page.meta),
+			maxPages: pageLimit,
+			pageDelayMs,
+		});
+		return mergeXurlDmPages(result.pages);
 	});
 }
 
@@ -780,7 +720,7 @@ export function syncDirectMessagesViaCachedBirdEffect({
 			);
 		}
 		const db = getNativeDb();
-		const resolvedAccount = resolveAccount(db, account);
+		const resolvedAccount = resolveLiveSyncAccount(db, account);
 		const pageKey = allPages
 			? "all-pages"
 			: `max-pages:${String(maxPages ?? 0)}`;
@@ -813,7 +753,7 @@ export function syncDirectMessagesViaCachedBirdEffect({
 			if (tryXurl) {
 				const xurlPayload = yield* Effect.gen(function* () {
 					const authenticated = getAuthenticatedXurlAccount(
-						yield* lookupAuthenticatedOAuth2UserEffect(
+						yield* liveTransportGateway.xurl.lookupAuthenticatedOAuth2User(
 							resolvedAccount.username,
 						),
 					);
@@ -824,9 +764,7 @@ export function syncDirectMessagesViaCachedBirdEffect({
 					}
 					assertAuthenticatedBirdAccountMatches({
 						source: "xurl",
-						accountId: resolvedAccount.accountId,
-						username: resolvedAccount.username,
-						externalUserId: resolvedAccount.externalUserId,
+						account: resolvedAccount,
 						liveUsername: authenticated.username ?? resolvedAccount.username,
 						liveExternalUserId: authenticated.id,
 					});
@@ -874,12 +812,11 @@ export function syncDirectMessagesViaCachedBirdEffect({
 				}
 			}
 			if (!payload) {
-				const authenticated = yield* getAuthenticatedBirdAccountEffect();
+				const authenticated =
+					yield* liveTransportGateway.bird.getAuthenticatedAccount();
 				assertAuthenticatedBirdAccountMatches({
 					source: "bird",
-					accountId: resolvedAccount.accountId,
-					username: resolvedAccount.username,
-					externalUserId: resolvedAccount.externalUserId,
+					account: resolvedAccount,
 					liveUsername: authenticated.username,
 					liveExternalUserId: authenticated.id,
 				});
@@ -891,7 +828,7 @@ export function syncDirectMessagesViaCachedBirdEffect({
 						accountExternalUserId,
 					);
 				}
-				payload = yield* listDirectMessagesViaBirdEffect({
+				payload = yield* liveTransportGateway.bird.listDirectMessages({
 					maxResults: limit,
 					...(inbox !== "all" ? { inbox } : {}),
 					...(typeof maxPages === "number" ? { maxPages } : {}),

package/src/lib/follow-graph.ts

@@ -1,10 +1,11 @@
 import { randomUUID } from "node:crypto";
 import { Effect } from "effect";
 import { getNativeDb } from "./db";
-import { listFollowUsersViaBirdEffect } from "./bird";
-import { runEffectPromise, tryPromise } from "./effect-runtime";
+import { runEffectPromise } from "./effect-runtime";
+import { liveTransportGateway } from "./live-transport-gateway";
 import type { Database } from "./sqlite";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { runSyncPlanEffect } from "./sync-plan";
 import type {
 	FollowEventKind,
 	FollowDirection,
@@ -16,7 +17,6 @@ import type {
 	XurlPublicMetrics,
 } from "./types";
 import { upsertProfileFromXUser } from "./x-profile";
-import { listFollowUsersViaXurl } from "./xurl";
 
 const DEFAULT_FOLLOW_CACHE_TTL_MS = 24 * 60 * 60_000;
 const DEFAULT_FOLLOW_PAGE_LIMIT = 1000;
@@ -267,35 +267,25 @@ function fetchFollowGraphViaXurlEffect({
 	maxResources?: number;
 }): Effect.Effect<MergedFollowPayload, unknown> {
 	return Effect.gen(function* () {
-		const pages: XurlFollowUsersResponse[] = [];
-		let nextToken: string | undefined;
-		let pageCount = 0;
-		let collectedCount = 0;
-
-		do {
-			const payload = yield* tryPromise(() =>
-				listFollowUsersViaXurl({
+		const result = yield* runSyncPlanEffect({
+			fetchPage: ({ cursor }) =>
+				liveTransportGateway.xurl.listFollowUsers({
 					direction,
 					username,
 					userId,
 					maxResults: limit,
-					paginationToken: nextToken,
+					...(cursor ? { paginationToken: cursor } : {}),
 				}),
-			);
-			pages.push(payload);
-			collectedCount += payload.data.length;
-			nextToken =
-				typeof payload.meta?.next_token === "string"
-					? String(payload.meta.next_token)
-					: undefined;
-			pageCount += 1;
-		} while (
-			nextToken &&
-			(maxPages === undefined || pageCount < maxPages) &&
-			(maxResources === undefined || collectedCount < maxResources)
-		);
+			getItemCount: (page) => page.data.length,
+			getNextCursor: (page) =>
+				typeof page.meta?.next_token === "string"
+					? String(page.meta.next_token)
+					: undefined,
+			maxItems: maxResources,
+			maxPages,
+		});
 
-		return mergePages(pages, nextToken, maxResources);
+		return mergePages(result.pages, result.nextCursor, maxResources);
 	});
 }
 
@@ -321,7 +311,7 @@ function fetchFollowGraphViaBirdEffect({
 						maxPages ?? Number.POSITIVE_INFINITY,
 						Math.ceil(maxResources / birdLimit),
 					);
-		const payload = yield* listFollowUsersViaBirdEffect({
+		const payload = yield* liveTransportGateway.bird.listFollowUsers({
 			direction,
 			userId,
 			maxResults: Math.min(birdLimit, maxResources ?? birdLimit),

package/src/lib/import-repository.ts

@@ -0,0 +1,138 @@
+import type { Database } from "./sqlite";
+
+export type ImportRow = Record<string, unknown>;
+export type ImportFtsTable =
+	| { table: "tweets_fts"; idColumn: "tweet_id" }
+	| { table: "dm_fts"; idColumn: "message_id" };
+
+export class ImportRepository {
+	constructor(readonly db: Database) {}
+
+	readRows<T>(sql: string, ...parameters: unknown[]) {
+		return this.db.prepare(sql).all(...parameters) as T[];
+	}
+
+	readRow<T>(sql: string, ...parameters: unknown[]) {
+		return this.db.prepare(sql).get(...parameters) as T | undefined;
+	}
+
+	insertRows(sql: string, rows: readonly ImportRow[], keys: readonly string[]) {
+		const statement = this.db.prepare(sql);
+		for (const row of rows) {
+			statement.run(...keys.map((key) => row[key] ?? null));
+		}
+	}
+
+	readFtsIds({ table, idColumn }: ImportFtsTable) {
+		const rows = this.db
+			.prepare(`select ${idColumn} as id from ${table}`)
+			.all() as { id: string }[];
+		return new Set(rows.map((row) => row.id));
+	}
+
+	insertFtsRows({
+		target,
+		rows,
+		idKey,
+		textKey,
+		existingIds = new Set<string>(),
+	}: {
+		target: ImportFtsTable;
+		rows: readonly ImportRow[];
+		idKey: string;
+		textKey: string;
+		existingIds?: Set<string>;
+	}) {
+		const statement = this.db.prepare(
+			`insert into ${target.table} (${target.idColumn}, text) values (?, ?)`,
+		);
+		for (const row of rows) {
+			const id = row[idKey];
+			if (typeof id !== "string" || existingIds.has(id)) continue;
+			const text = row[textKey];
+			statement.run(id, typeof text === "string" ? text : "");
+			existingIds.add(id);
+		}
+	}
+
+	clearArchiveImport() {
+		this.db.exec(`
+      delete from ai_scores;
+      delete from tweet_actions;
+      delete from tweet_account_edges;
+      delete from tweet_collections;
+      delete from link_occurrences;
+      delete from url_expansions;
+      delete from dm_fts;
+      delete from tweets_fts;
+      delete from dm_messages;
+      delete from dm_conversations;
+      delete from tweets;
+      delete from profiles;
+      delete from accounts;
+    `);
+		this.clearAuthoredSyncCursors();
+	}
+
+	clearAuthoredSyncCursors(accountId?: string) {
+		if (accountId) {
+			this.db
+				.prepare("delete from sync_cache where cache_key = ?")
+				.run(`authored:xurl:${accountId}:cursor`);
+			return;
+		}
+		this.db
+			.prepare(
+				"delete from sync_cache where cache_key like 'authored:xurl:%:cursor'",
+			)
+			.run();
+	}
+
+	clearMentionSyncState() {
+		this.db
+			.prepare("delete from sync_cache where cache_key like 'mentions:sync:%'")
+			.run();
+	}
+
+	clearBackupImport() {
+		this.db.exec(`
+      delete from follow_events;
+      delete from follow_edges;
+      delete from follow_snapshot_members;
+      delete from follow_snapshots;
+      delete from ai_scores;
+      delete from tweet_actions;
+      delete from tweet_account_edges;
+      delete from tweet_collections;
+      delete from link_occurrences;
+      delete from url_expansions;
+      delete from blocks;
+      delete from mutes;
+      delete from dm_fts;
+      delete from tweets_fts;
+      delete from dm_messages;
+      delete from dm_conversations;
+      delete from tweets;
+      delete from profile_bio_entities;
+      delete from profile_snapshots;
+      delete from profile_affiliations;
+      delete from profiles;
+      delete from accounts;
+      delete from sync_cache;
+    `);
+	}
+}
+
+let repositories = new WeakMap<Database, ImportRepository>();
+
+export function getImportRepository(db: Database) {
+	const existing = repositories.get(db);
+	if (existing) return existing;
+	const repository = new ImportRepository(db);
+	repositories.set(db, repository);
+	return repository;
+}
+
+export function resetImportRepositoriesForTests() {
+	repositories = new WeakMap();
+}

package/src/lib/inbox.ts

@@ -2,7 +2,8 @@ import { Effect } from "effect";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { scoreInboxItemWithOpenAIEffect } from "./openai";
-import { listDmConversations, listTimelineItems } from "./queries";
+import { listDmConversations } from "./dm-read-model";
+import { listTimelineItems } from "./timeline-read-model";
 import type { InboxItem, InboxQuery, InboxResponse } from "./types";
 
 function heuristicSummary(kind: InboxItem["entityKind"]) {

package/src/lib/live-sync-engine.ts

@@ -0,0 +1,209 @@
+import { Effect } from "effect";
+import { databaseWriteEffect } from "./database-writer";
+import type { Database } from "./sqlite";
+import { readSyncCache, writeSyncCache } from "./sync-cache";
+
+export interface LiveTransportAdapter<Source extends string, Payload> {
+	source: Source;
+	fetch: Effect.Effect<Payload, Error>;
+}
+
+export interface LiveSyncAccount {
+	accountId: string;
+	username: string;
+	externalUserId?: string;
+	isDefault: boolean;
+}
+
+interface CachedLiveSyncOptions<Source extends string, Payload, Persisted> {
+	db: Database;
+	cacheKey: string;
+	refresh: boolean;
+	cacheTtlMs: number;
+	transports: readonly LiveTransportAdapter<Source, Payload>[];
+	persistLive: (db: Database, payload: Payload, source: Source) => Persisted;
+	persistCached?: (db: Database, payload: Payload) => Persisted;
+}
+
+export interface CachedLiveSyncResult<
+	Source extends string,
+	Payload,
+	Persisted,
+> {
+	source: Source | "cache";
+	payload: Payload;
+	persisted: Persisted | undefined;
+}
+
+function toError(error: unknown) {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+export function resolveLiveSyncAccount(
+	db: Database,
+	accountId?: string,
+): LiveSyncAccount {
+	const row = accountId
+		? (db
+				.prepare(
+					"select id, handle, external_user_id, is_default from accounts where id = ?",
+				)
+				.get(accountId) as
+				| {
+						id: string;
+						handle: string;
+						external_user_id: string | null;
+						is_default: number;
+				  }
+				| undefined)
+		: (db
+				.prepare(
+					`
+          select id, handle, external_user_id, is_default
+          from accounts
+          order by is_default desc, created_at asc
+          limit 1
+          `,
+				)
+				.get() as
+				| {
+						id: string;
+						handle: string;
+						external_user_id: string | null;
+						is_default: number;
+				  }
+				| undefined);
+
+	if (!row) {
+		throw new Error(`Unknown account: ${accountId ?? "default"}`);
+	}
+
+	const externalUserId = row.external_user_id?.trim();
+	return {
+		accountId: row.id,
+		username: row.handle.replace(/^@/, ""),
+		...(externalUserId ? { externalUserId } : {}),
+		isDefault: row.is_default === 1,
+	};
+}
+
+export function createLiveTransportAdapter<Source extends string, Payload>(
+	source: Source,
+	fetch: Effect.Effect<Payload, unknown>,
+): LiveTransportAdapter<Source, Payload> {
+	return {
+		source,
+		fetch: fetch.pipe(Effect.mapError(toError)),
+	};
+}
+
+export function assertLiveAccountMatches({
+	source,
+	account,
+	liveUsername,
+	liveExternalUserId,
+}: {
+	source: string;
+	account: LiveSyncAccount;
+	liveUsername: string;
+	liveExternalUserId?: string;
+}) {
+	if (
+		account.externalUserId &&
+		liveExternalUserId &&
+		account.externalUserId === liveExternalUserId
+	) {
+		return;
+	}
+	if (account.externalUserId && liveExternalUserId) {
+		throw new Error(
+			`${source} is authenticated as user ${liveExternalUserId}; refusing to sync into ${account.accountId} (${account.externalUserId})`,
+		);
+	}
+	if (liveUsername.toLowerCase() !== account.username.toLowerCase()) {
+		throw new Error(
+			`${source} is authenticated as @${liveUsername}; refusing to sync into ${account.accountId} (@${account.username})`,
+		);
+	}
+}
+
+export function normalizeCacheTtlMs(
+	value: number | undefined,
+	defaultValue: number,
+) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+		return defaultValue;
+	}
+	return Math.floor(value);
+}
+
+export function fetchWithTransportFallbackEffect<
+	Source extends string,
+	Payload,
+>(
+	transports: readonly LiveTransportAdapter<Source, Payload>[],
+): Effect.Effect<{ source: Source; payload: Payload }, Error> {
+	const [first, ...rest] = transports;
+	if (!first) {
+		return Effect.fail(new Error("No live transport adapters configured"));
+	}
+	return first.fetch.pipe(
+		Effect.map((payload) => ({ source: first.source, payload })),
+		Effect.catchAll((error) =>
+			rest.length > 0
+				? fetchWithTransportFallbackEffect(rest)
+				: Effect.fail(toError(error)),
+		),
+	);
+}
+
+export function runCachedLiveSyncEffect<
+	Source extends string,
+	Payload,
+	Persisted,
+>({
+	db,
+	cacheKey,
+	refresh,
+	cacheTtlMs,
+	transports,
+	persistLive,
+	persistCached,
+}: CachedLiveSyncOptions<Source, Payload, Persisted>): Effect.Effect<
+	CachedLiveSyncResult<Source, Payload, Persisted>,
+	Error
+> {
+	return Effect.gen(function* () {
+		const cached = yield* Effect.try({
+			try: () => readSyncCache<Payload>(cacheKey, db),
+			catch: toError,
+		});
+		const cacheAgeMs = cached
+			? Date.now() - new Date(cached.updatedAt).getTime()
+			: Number.POSITIVE_INFINITY;
+		if (!refresh && cached && cacheAgeMs <= cacheTtlMs) {
+			const persisted = persistCached
+				? yield* databaseWriteEffect((writeDb) =>
+						persistCached(writeDb, cached.value),
+					)
+				: undefined;
+			return {
+				source: "cache",
+				payload: cached.value,
+				persisted,
+			};
+		}
+
+		const fetched = yield* fetchWithTransportFallbackEffect(transports);
+		const persisted = yield* databaseWriteEffect((writeDb) => {
+			const value = persistLive(writeDb, fetched.payload, fetched.source);
+			writeSyncCache(cacheKey, fetched.payload, writeDb);
+			return value;
+		});
+		return {
+			source: fetched.source,
+			payload: fetched.payload,
+			persisted,
+		};
+	});
+}