birdclaw 0.8.2 → 0.8.3

package/src/lib/archive-import.ts

@@ -12,9 +12,20 @@ import path from "node:path";
 import { pipeline } from "node:stream/promises";
 import { promisify } from "node:util";
 import { Effect } from "effect";
+import {
+	ArchiveImportPlan,
+	type ArchiveMessageRow,
+	type ArchiveProfileRow,
+} from "./archive-import-plan";
 import { getBirdclawPaths } from "./config";
 import { getNativeDb } from "./db";
+import { databaseWriteEffect } from "./database-writer";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
+import { getImportRepository } from "./import-repository";
+import {
+	ingestSourcesInBatchesEffect,
+	streamAssignedJsonArray,
+} from "./streaming-ingestion";
 import { safeHttpUrl } from "./url-safety";
 
 const execFileAsync = promisify(execFile);
@@ -204,6 +215,56 @@ function readArchiveEntryEffect(
 	return runUnzipEffect(archivePath, ["-p", archivePath, entryPath]);
 }
 
+async function* streamArchiveArrayRecords(
+	archivePath: string,
+	entryPath: string,
+) {
+	const child = spawn("unzip", ["-p", archivePath, entryPath], {
+		stdio: ["ignore", "pipe", "pipe"],
+	});
+	let stderr = "";
+	child.stderr.setEncoding("utf8");
+	child.stderr.on("data", (chunk) => {
+		stderr += String(chunk);
+	});
+	const exit = new Promise<number | null>((resolve, reject) => {
+		child.on("error", reject);
+		child.on("close", resolve);
+	});
+
+	try {
+		yield* streamAssignedJsonArray(child.stdout);
+		const exitCode = await exit;
+		if (exitCode !== 0) {
+			throw new Error(
+				`Failed to extract ${entryPath}: ${
+					stderr.trim() || `exit ${String(exitCode)}`
+				}`,
+			);
+		}
+	} finally {
+		if (!child.killed) child.kill();
+	}
+}
+
+function processArchiveEntryRecordsEffect(
+	archivePath: string,
+	entryPath: string,
+	processRecord: (record: ArchiveRecord) => void,
+) {
+	return ingestSourcesInBatchesEffect({
+		sources: [
+			{
+				id: entryPath,
+				stream: () => streamArchiveArrayRecords(archivePath, entryPath),
+			},
+		],
+		processBatch: (batch) => {
+			for (const record of batch) processRecord(record);
+		},
+	});
+}
+
 function getFirstEntry(entries: string[], pattern: RegExp) {
 	return entries.find((entry) => pattern.test(normalizeArchivePath(entry)));
 }
@@ -640,58 +701,14 @@ function extractArchiveMediaFilesEffect(
 	});
 }
 
-function getArchiveFollowRows(content: string, key: ArchiveFollowKey) {
-	const rows: Array<{ profileId: string; externalUserId: string }> = [];
-
-	for (const wrapper of parseArchiveArray(content)) {
-		const item = asRecord(wrapper[key]);
-		const externalUserId = String(item?.accountId ?? "");
-		if (!externalUserId) continue;
-
-		rows.push({
-			profileId: `profile_user_${externalUserId}`,
-			externalUserId,
-		});
-	}
-
-	return rows;
-}
-
-function clearImportedData(db = getNativeDb()) {
-	db.exec(`
-    delete from ai_scores;
-    delete from tweet_actions;
-    delete from tweet_account_edges;
-    delete from tweet_collections;
-    delete from link_occurrences;
-    delete from url_expansions;
-    delete from dm_fts;
-    delete from tweets_fts;
-    delete from dm_messages;
-    delete from dm_conversations;
-    delete from tweets;
-    delete from profiles;
-    delete from accounts;
-  `);
-	clearAuthoredSyncCursors(db);
-}
-
-function clearAuthoredSyncCursors(db = getNativeDb(), accountId?: string) {
-	if (accountId) {
-		db.prepare("delete from sync_cache where cache_key = ?").run(
-			`authored:xurl:${accountId}:cursor`,
-		);
-		return;
-	}
-	db.prepare(
-		"delete from sync_cache where cache_key like 'authored:xurl:%:cursor'",
-	).run();
-}
-
-function clearMentionSyncState(db = getNativeDb()) {
-	db.prepare(
-		"delete from sync_cache where cache_key like 'mentions:sync:%'",
-	).run();
+function getArchiveFollowRow(wrapper: ArchiveRecord, key: ArchiveFollowKey) {
+	const item = asRecord(wrapper[key]);
+	const externalUserId = String(item?.accountId ?? "");
+	if (!externalUserId) return undefined;
+	return {
+		profileId: `profile_user_${externalUserId}`,
+		externalUserId,
+	};
 }
 
 function importArchiveInternalEffect(
@@ -767,47 +784,21 @@ function importArchiveInternalEffect(
 			parseArchiveArray(accountContent)[0] ?? null,
 			parseArchiveArray(profileContent)[0] ?? null,
 		);
-
-		const mentionDirectory = new Map<
-			string,
-			{ handle?: string; displayName?: string }
-		>();
-		const tweetRows: Array<{
-			id: string;
-			kind: "home" | "like" | "bookmark";
-			authorProfileId: string;
-			text: string;
-			createdAt: string;
-			isReplied: number;
-			replyToId: string | null;
-			likeCount: number;
-			mediaCount: number;
-			bookmarked: number;
-			liked: number;
-			entitiesJson: string;
-			mediaJson: string;
-			quotedTweetId: string | null;
-		}> = [];
-		const collectionRows: Array<{
-			tweetId: string;
-			kind: "likes" | "bookmarks";
-			collectedAt: string | null;
-			source: string;
-			rawJson: string;
-		}> = [];
-		const tweetRowsById = new Map<string, (typeof tweetRows)[number]>();
-
-		function addTweetRow(row: (typeof tweetRows)[number]) {
-			const existing = tweetRowsById.get(row.id);
-			if (existing) {
-				existing.bookmarked = Math.max(existing.bookmarked, row.bookmarked);
-				existing.liked = Math.max(existing.liked, row.liked);
-				if (!existing.text && row.text) existing.text = row.text;
-				return;
-			}
-			tweetRows.push(row);
-			tweetRowsById.set(row.id, row);
-		}
+		const db = getNativeDb();
+		const repository = getImportRepository(db);
+		const plan = new ArchiveImportPlan();
+		const {
+			mentionDirectory,
+			tweets: tweetRows,
+			collections: collectionRows,
+			profiles,
+			conversations,
+			dmMessages,
+			followers: followerRows,
+			following: followingRows,
+			followerIds,
+			followingIds,
+		} = plan;
 
 		if (tweetEntries.length > 0) {
 			onProgress({
@@ -817,10 +808,9 @@ function importArchiveInternalEffect(
 			});
 		}
 		for (const [tweetFileIndex, entry] of tweetEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			for (const wrapper of parseArchiveArray(content)) {
+			yield* processArchiveEntryRecordsEffect(archivePath, entry, (wrapper) => {
 				const tweet = asRecord(wrapper.tweet);
-				if (!tweet) continue;
+				if (!tweet) return;
 
 				for (const mention of asArray<Record<string, unknown>>(
 					asRecord(tweet.entities)?.user_mentions,
@@ -846,7 +836,7 @@ function importArchiveInternalEffect(
 					});
 				}
 
-				addTweetRow({
+				plan.addTweet({
 					id: String(tweet.id_str ?? tweet.id),
 					kind: "home",
 					authorProfileId: "profile_me",
@@ -866,7 +856,7 @@ function importArchiveInternalEffect(
 						? String(tweet.quoted_status_id_str)
 						: null,
 				});
-			}
+			});
 			onProgress({
 				kind: "slice-file",
 				slice: "tweets",
@@ -891,12 +881,11 @@ function importArchiveInternalEffect(
 		}
 		const tweetRowsBeforeNotes = tweetRows.length;
 		for (const [noteFileIndex, entry] of noteTweetEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			for (const wrapper of parseArchiveArray(content)) {
+			yield* processArchiveEntryRecordsEffect(archivePath, entry, (wrapper) => {
 				const noteTweet = asRecord(wrapper.noteTweet);
-				if (!noteTweet) continue;
+				if (!noteTweet) return;
 				const core = asRecord(noteTweet.core);
-				addTweetRow({
+				plan.addTweet({
 					id: String(noteTweet.noteTweetId ?? noteTweet.id ?? randomUUID()),
 					kind: "home",
 					authorProfileId: "profile_me",
@@ -912,7 +901,7 @@ function importArchiveInternalEffect(
 					mediaJson: "[]",
 					quotedTweetId: null,
 				});
-			}
+			});
 			onProgress({
 				kind: "slice-file",
 				slice: "noteTweets",
@@ -929,38 +918,7 @@ function importArchiveInternalEffect(
 		}
 		const authoredTweetCount = tweetRows.length;
 
-		type MessageRow = {
-			id: string;
-			conversationId: string;
-			senderProfileId: string;
-			text: string;
-			createdAt: string;
-			direction: "inbound" | "outbound";
-			mediaCount: number;
-		};
-
-		const profiles = new Map<
-			string,
-			{
-				id: string;
-				handle: string;
-				displayName: string;
-				bio: string;
-				followersCount: number;
-				followingCount: number;
-				publicMetricsJson: string;
-				avatarHue: number;
-				avatarUrl: string | null;
-				location: string | null;
-				url: string | null;
-				verifiedType: string | null;
-				entitiesJson: string;
-				rawJson: string;
-				createdAt: string;
-			}
-		>();
-		type ProfileRow =
-			typeof profiles extends Map<string, infer Value> ? Value : never;
+		type ProfileRow = ArchiveProfileRow;
 		const defaultProfileMetadata = {
 			publicMetricsJson: "{}",
 			location: null,
@@ -969,25 +927,6 @@ function importArchiveInternalEffect(
 			entitiesJson: "{}",
 			rawJson: "{}",
 		};
-		const conversations = new Map<
-			string,
-			{
-				id: string;
-				title: string;
-				accountId: string;
-				participantProfileId: string;
-				lastMessageAt: string;
-				unreadCount: number;
-				needsReply: number;
-			}
-		>();
-		const dmMessages: MessageRow[] = [];
-		const followerRows: Array<{ profileId: string; externalUserId: string }> =
-			[];
-		const followingRows: Array<{ profileId: string; externalUserId: string }> =
-			[];
-		const followerIds = new Set<string>();
-		const followingIds = new Set<string>();
 		type ExistingProfileRow = {
 			id: string;
 			handle: string;
@@ -1006,18 +945,14 @@ function importArchiveInternalEffect(
 			created_at: string;
 		};
 		const existingProfiles = new Map(
-			(
-				getNativeDb()
-					.prepare(
-						`
+			repository
+				.readRows<ExistingProfileRow>(`
 	        select id, handle, display_name, bio, followers_count, following_count,
 	          public_metrics_json, avatar_hue, avatar_url, location, url,
 	          verified_type, entities_json, raw_json, created_at
 	        from profiles
-	      `,
-					)
-					.all() as ExistingProfileRow[]
-			).map((profile) => [profile.id, profile]),
+	      `)
+				.map((profile) => [profile.id, profile]),
 		);
 		const existingProfilesByHandle = new Map(
 			[...existingProfiles.values()].map((profile) => [
@@ -1025,11 +960,13 @@ function importArchiveInternalEffect(
 				profile,
 			]),
 		);
-		const existingPrimaryAccount = getNativeDb()
-			.prepare("select handle, external_user_id from accounts where id = ?")
-			.get("acct_primary") as
-			| { handle: string; external_user_id: string | null }
-			| undefined;
+		const existingPrimaryAccount = repository.readRow<{
+			handle: string;
+			external_user_id: string | null;
+		}>(
+			"select handle, external_user_id from accounts where id = ?",
+			"acct_primary",
+		);
 		const profileIdAliases = new Map<string, string>();
 
 		type ArchiveProfileTier =
@@ -1363,13 +1300,12 @@ function importArchiveInternalEffect(
 			});
 		}
 		for (const [dmFileIndex, entry] of dmEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			for (const wrapper of parseArchiveArray(content)) {
+			yield* processArchiveEntryRecordsEffect(archivePath, entry, (wrapper) => {
 				const dmConversation = asRecord(wrapper.dmConversation);
-				if (!dmConversation) continue;
+				if (!dmConversation) return;
 
 				const rawConversationId = String(dmConversation.conversationId ?? "");
-				if (!rawConversationId) continue;
+				if (!rawConversationId) return;
 				const conversationId =
 					resolveArchiveDmConversationId(rawConversationId);
 				const conversationIdChanged = conversationId !== rawConversationId;
@@ -1516,18 +1452,18 @@ function importArchiveInternalEffect(
 							direction:
 								senderId === accountPayload.accountId ? "outbound" : "inbound",
 							mediaCount: asArray(messageCreate.mediaUrls).length,
-						} satisfies MessageRow;
+						} satisfies ArchiveMessageRow;
 					})
 					.sort((left, right) =>
 						compareIsoTimestamp(left.createdAt, right.createdAt),
 					);
 
 				if (messageEvents.length === 0) {
-					continue;
+					return;
 				}
 
 				const lastMessage = messageEvents.at(-1);
-				if (!lastMessage) continue;
+				if (!lastMessage) return;
 
 				dmMessages.push(...messageEvents);
 				const resolvedParticipantProfileId =
@@ -1544,7 +1480,7 @@ function importArchiveInternalEffect(
 					unreadCount: 0,
 					needsReply: lastMessage.direction === "inbound" ? 1 : 0,
 				});
-			}
+			});
 			onProgress({
 				kind: "slice-file",
 				slice: "directMessages",
@@ -1569,11 +1505,10 @@ function importArchiveInternalEffect(
 		}
 		let likeCount = 0;
 		for (const [likeFileIndex, entry] of likeEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			const likes = parseArchiveArray(content);
-			for (const like of likes) {
+			yield* processArchiveEntryRecordsEffect(archivePath, entry, (like) => {
+				likeCount += 1;
 				const tweet = extractCollectionTweet(like, "like");
-				if (!tweet) continue;
+				if (!tweet) return;
 				collectionRows.push({
 					tweetId: tweet.id,
 					kind: "likes",
@@ -1581,7 +1516,7 @@ function importArchiveInternalEffect(
 					source: "archive",
 					rawJson: JSON.stringify(like),
 				});
-				addTweetRow({
+				plan.addTweet({
 					id: tweet.id,
 					kind: "like",
 					authorProfileId: "profile_unknown",
@@ -1597,8 +1532,7 @@ function importArchiveInternalEffect(
 					mediaJson: "[]",
 					quotedTweetId: null,
 				});
-			}
-			likeCount += likes.length;
+			});
 			onProgress({
 				kind: "slice-file",
 				slice: "likes",
@@ -1619,36 +1553,38 @@ function importArchiveInternalEffect(
 		}
 		let bookmarkCount = 0;
 		for (const [bookmarkFileIndex, entry] of bookmarkEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			const bookmarks = parseArchiveArray(content);
-			for (const bookmark of bookmarks) {
-				const tweet = extractCollectionTweet(bookmark, "bookmark");
-				if (!tweet) continue;
-				collectionRows.push({
-					tweetId: tweet.id,
-					kind: "bookmarks",
-					collectedAt: tweet.createdAt,
-					source: "archive",
-					rawJson: JSON.stringify(bookmark),
-				});
-				addTweetRow({
-					id: tweet.id,
-					kind: "bookmark",
-					authorProfileId: "profile_unknown",
-					text: tweet.text,
-					createdAt: tweet.createdAt,
-					isReplied: 0,
-					replyToId: null,
-					likeCount: tweet.likeCount,
-					mediaCount: 0,
-					bookmarked: 1,
-					liked: 0,
-					entitiesJson: "{}",
-					mediaJson: "[]",
-					quotedTweetId: null,
-				});
-			}
-			bookmarkCount += bookmarks.length;
+			yield* processArchiveEntryRecordsEffect(
+				archivePath,
+				entry,
+				(bookmark) => {
+					bookmarkCount += 1;
+					const tweet = extractCollectionTweet(bookmark, "bookmark");
+					if (!tweet) return;
+					collectionRows.push({
+						tweetId: tweet.id,
+						kind: "bookmarks",
+						collectedAt: tweet.createdAt,
+						source: "archive",
+						rawJson: JSON.stringify(bookmark),
+					});
+					plan.addTweet({
+						id: tweet.id,
+						kind: "bookmark",
+						authorProfileId: "profile_unknown",
+						text: tweet.text,
+						createdAt: tweet.createdAt,
+						isReplied: 0,
+						replyToId: null,
+						likeCount: tweet.likeCount,
+						mediaCount: 0,
+						bookmarked: 1,
+						liked: 0,
+						entitiesJson: "{}",
+						mediaJson: "[]",
+						quotedTweetId: null,
+					});
+				},
+			);
 			onProgress({
 				kind: "slice-file",
 				slice: "bookmarks",
@@ -1686,12 +1622,12 @@ function importArchiveInternalEffect(
 			});
 		}
 		for (const [followerFileIndex, entry] of followerEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			for (const row of getArchiveFollowRows(content, "follower")) {
-				if (followerIds.has(row.externalUserId)) continue;
+			yield* processArchiveEntryRecordsEffect(archivePath, entry, (wrapper) => {
+				const row = getArchiveFollowRow(wrapper, "follower");
+				if (!row || followerIds.has(row.externalUserId)) return;
 				followerIds.add(row.externalUserId);
 				followerRows.push(row);
-			}
+			});
 			onProgress({
 				kind: "slice-file",
 				slice: "followers",
@@ -1715,12 +1651,12 @@ function importArchiveInternalEffect(
 			});
 		}
 		for (const [followingFileIndex, entry] of followingEntries.entries()) {
-			const content = yield* readArchiveEntryEffect(archivePath, entry);
-			for (const row of getArchiveFollowRows(content, "following")) {
-				if (followingIds.has(row.externalUserId)) continue;
+			yield* processArchiveEntryRecordsEffect(archivePath, entry, (wrapper) => {
+				const row = getArchiveFollowRow(wrapper, "following");
+				if (!row || followingIds.has(row.externalUserId)) return;
 				followingIds.add(row.externalUserId);
 				followingRows.push(row);
-			}
+			});
 			onProgress({
 				kind: "slice-file",
 				slice: "following",
@@ -1747,25 +1683,21 @@ function importArchiveInternalEffect(
 		if (includeFollowing && followingEntries.length === 0) {
 			clearedFollowDirections.add("following");
 		}
-		const retainedFollowProfiles = getNativeDb()
-			.prepare(
-				`
-      select direction, profile_id, external_user_id, source, null as snapshot_id, null as snapshot_source
-      from follow_edges
-      union
-      select ev.direction, ev.profile_id, ev.external_user_id, null as source, ev.snapshot_id, snap.source as snapshot_source
-      from follow_events ev
-      left join follow_snapshots snap on snap.id = ev.snapshot_id
-      `,
-			)
-			.all() as Array<{
+		const retainedFollowProfiles = repository.readRows<{
 			direction: ArchiveFollowDirection;
 			profile_id: string;
 			external_user_id: string;
 			source: string | null;
 			snapshot_id: string | null;
 			snapshot_source: string | null;
-		}>;
+		}>(`
+      select direction, profile_id, external_user_id, source, null as snapshot_id, null as snapshot_source
+      from follow_edges
+      union
+      select ev.direction, ev.profile_id, ev.external_user_id, null as source, ev.snapshot_id, snap.source as snapshot_source
+      from follow_events ev
+      left join follow_snapshots snap on snap.id = ev.snapshot_id
+      `);
 		for (const row of retainedFollowProfiles) {
 			const isClearedArchiveRow =
 				clearedFollowDirections.has(row.direction) &&
@@ -1803,7 +1735,6 @@ function importArchiveInternalEffect(
 			);
 		}
 
-		const db = getNativeDb();
 		const insertAccount = db.prepare(`
     insert into accounts (id, name, handle, external_user_id, transport, is_default, created_at)
     values (?, ?, ?, ?, ?, 1, ?)
@@ -2451,15 +2382,15 @@ function importArchiveInternalEffect(
 				onProgress({ kind: "write-progress", phase, processed, total });
 			}
 		}
-		db.transaction(() => {
+		yield* databaseWriteEffect(() => {
 			if (!selection) {
-				clearImportedData(db);
-				clearMentionSyncState(db);
+				repository.clearArchiveImport();
+				repository.clearMentionSyncState();
 			}
 
 			if (selection) {
 				if (includeTweets) {
-					clearAuthoredSyncCursors(db, "acct_primary");
+					repository.clearAuthoredSyncCursors("acct_primary");
 					demoteSelectedArchiveTweetsWithCollections.run(
 						"acct_primary",
 						"acct_primary",
@@ -2704,7 +2635,7 @@ function importArchiveInternalEffect(
 			} else if (includeFollowing) {
 				clearArchiveFollowRows("following");
 			}
-		})();
+		}, db);
 		onProgress({ kind: "done" });
 
 		return {