birdclaw 0.6.0 → 0.7.0

package/src/lib/archive-import.ts

@@ -61,8 +61,45 @@ export const ARCHIVE_IMPORT_SLICES = [
 
 export type ArchiveImportSlice = (typeof ARCHIVE_IMPORT_SLICES)[number];
 
+export type ImportProgressSlice =
+	| "tweets"
+	| "noteTweets"
+	| "directMessages"
+	| "likes"
+	| "bookmarks"
+	| "media"
+	| "followers"
+	| "following";
+
+export type ImportWritePhase =
+	| "profiles"
+	| "tweets"
+	| "collections"
+	| "dmMessages";
+
+export type ImportProgressEvent =
+	| { kind: "scanned"; entryCount: number }
+	| { kind: "slice-start"; slice: ImportProgressSlice; files: number }
+	| {
+			kind: "slice-file";
+			slice: ImportProgressSlice;
+			processed: number;
+			files: number;
+	  }
+	| { kind: "slice-done"; slice: ImportProgressSlice; count: number }
+	| { kind: "writing" }
+	| { kind: "write-start"; phase: ImportWritePhase; total: number }
+	| {
+			kind: "write-progress";
+			phase: ImportWritePhase;
+			processed: number;
+			total: number;
+	  }
+	| { kind: "done" };
+
 export interface ImportArchiveOptions {
 	select?: ArchiveImportSlice[];
+	onProgress?: (event: ImportProgressEvent) => void;
 }
 
 type ArchiveRecord = Record<string, unknown>;
@@ -662,7 +699,9 @@ function importArchiveInternalEffect(
 	options: ImportArchiveOptions = {},
 ): Effect.Effect<ImportedArchiveSummary, unknown> {
 	return Effect.gen(function* () {
+		const onProgress = options.onProgress ?? (() => {});
 		const entries = yield* listArchiveEntriesEffect(archivePath);
+		onProgress({ kind: "scanned", entryCount: entries.length });
 		const selection = selectedSlices(options);
 		const includeTweets = includesSlice(selection, "tweets");
 		const includeLikes = includesSlice(selection, "likes");
@@ -770,7 +809,14 @@ function importArchiveInternalEffect(
 			tweetRowsById.set(row.id, row);
 		}
 
-		for (const entry of tweetEntries) {
+		if (tweetEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "tweets",
+				files: tweetEntries.length,
+			});
+		}
+		for (const [tweetFileIndex, entry] of tweetEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			for (const wrapper of parseArchiveArray(content)) {
 				const tweet = asRecord(wrapper.tweet);
@@ -821,9 +867,30 @@ function importArchiveInternalEffect(
 						: null,
 				});
 			}
+			onProgress({
+				kind: "slice-file",
+				slice: "tweets",
+				processed: tweetFileIndex + 1,
+				files: tweetEntries.length,
+			});
+		}
+		if (tweetEntries.length > 0) {
+			onProgress({
+				kind: "slice-done",
+				slice: "tweets",
+				count: tweetRows.length,
+			});
 		}
 
-		for (const entry of noteTweetEntries) {
+		if (noteTweetEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "noteTweets",
+				files: noteTweetEntries.length,
+			});
+		}
+		const tweetRowsBeforeNotes = tweetRows.length;
+		for (const [noteFileIndex, entry] of noteTweetEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			for (const wrapper of parseArchiveArray(content)) {
 				const noteTweet = asRecord(wrapper.noteTweet);
@@ -846,6 +913,19 @@ function importArchiveInternalEffect(
 					quotedTweetId: null,
 				});
 			}
+			onProgress({
+				kind: "slice-file",
+				slice: "noteTweets",
+				processed: noteFileIndex + 1,
+				files: noteTweetEntries.length,
+			});
+		}
+		if (noteTweetEntries.length > 0) {
+			onProgress({
+				kind: "slice-done",
+				slice: "noteTweets",
+				count: tweetRows.length - tweetRowsBeforeNotes,
+			});
 		}
 		const authoredTweetCount = tweetRows.length;
 
@@ -1275,7 +1355,14 @@ function importArchiveInternalEffect(
 			return resolved;
 		}
 
-		for (const entry of dmEntries) {
+		if (dmEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "directMessages",
+				files: dmEntries.length,
+			});
+		}
+		for (const [dmFileIndex, entry] of dmEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			for (const wrapper of parseArchiveArray(content)) {
 				const dmConversation = asRecord(wrapper.dmConversation);
@@ -1458,10 +1545,30 @@ function importArchiveInternalEffect(
 					needsReply: lastMessage.direction === "inbound" ? 1 : 0,
 				});
 			}
+			onProgress({
+				kind: "slice-file",
+				slice: "directMessages",
+				processed: dmFileIndex + 1,
+				files: dmEntries.length,
+			});
+		}
+		if (dmEntries.length > 0) {
+			onProgress({
+				kind: "slice-done",
+				slice: "directMessages",
+				count: dmMessages.length,
+			});
 		}
 
+		if (likeEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "likes",
+				files: likeEntries.length,
+			});
+		}
 		let likeCount = 0;
-		for (const entry of likeEntries) {
+		for (const [likeFileIndex, entry] of likeEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			const likes = parseArchiveArray(content);
 			for (const like of likes) {
@@ -1492,10 +1599,26 @@ function importArchiveInternalEffect(
 				});
 			}
 			likeCount += likes.length;
+			onProgress({
+				kind: "slice-file",
+				slice: "likes",
+				processed: likeFileIndex + 1,
+				files: likeEntries.length,
+			});
+		}
+		if (likeEntries.length > 0) {
+			onProgress({ kind: "slice-done", slice: "likes", count: likeCount });
 		}
 
+		if (bookmarkEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "bookmarks",
+				files: bookmarkEntries.length,
+			});
+		}
 		let bookmarkCount = 0;
-		for (const entry of bookmarkEntries) {
+		for (const [bookmarkFileIndex, entry] of bookmarkEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			const bookmarks = parseArchiveArray(content);
 			for (const bookmark of bookmarks) {
@@ -1526,29 +1649,91 @@ function importArchiveInternalEffect(
 				});
 			}
 			bookmarkCount += bookmarks.length;
+			onProgress({
+				kind: "slice-file",
+				slice: "bookmarks",
+				processed: bookmarkFileIndex + 1,
+				files: bookmarkEntries.length,
+			});
+		}
+		if (bookmarkEntries.length > 0) {
+			onProgress({
+				kind: "slice-done",
+				slice: "bookmarks",
+				count: bookmarkCount,
+			});
 		}
 
+		onProgress({ kind: "slice-start", slice: "media", files: 0 });
 		const mediaFileCounts = yield* extractArchiveMediaFilesEffect(
 			archivePath,
 			selectedArchiveMediaKinds(selection),
 		);
+		onProgress({
+			kind: "slice-done",
+			slice: "media",
+			count: Object.values(mediaFileCounts).reduce(
+				(total, value) => total + value,
+				0,
+			),
+		});
 
-		for (const entry of followerEntries) {
+		if (followerEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "followers",
+				files: followerEntries.length,
+			});
+		}
+		for (const [followerFileIndex, entry] of followerEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			for (const row of getArchiveFollowRows(content, "follower")) {
 				if (followerIds.has(row.externalUserId)) continue;
 				followerIds.add(row.externalUserId);
 				followerRows.push(row);
 			}
+			onProgress({
+				kind: "slice-file",
+				slice: "followers",
+				processed: followerFileIndex + 1,
+				files: followerEntries.length,
+			});
+		}
+		if (followerEntries.length > 0) {
+			onProgress({
+				kind: "slice-done",
+				slice: "followers",
+				count: followerRows.length,
+			});
 		}
 
-		for (const entry of followingEntries) {
+		if (followingEntries.length > 0) {
+			onProgress({
+				kind: "slice-start",
+				slice: "following",
+				files: followingEntries.length,
+			});
+		}
+		for (const [followingFileIndex, entry] of followingEntries.entries()) {
 			const content = yield* readArchiveEntryEffect(archivePath, entry);
 			for (const row of getArchiveFollowRows(content, "following")) {
 				if (followingIds.has(row.externalUserId)) continue;
 				followingIds.add(row.externalUserId);
 				followingRows.push(row);
 			}
+			onProgress({
+				kind: "slice-file",
+				slice: "following",
+				processed: followingFileIndex + 1,
+				files: followingEntries.length,
+			});
+		}
+		if (followingEntries.length > 0) {
+			onProgress({
+				kind: "slice-done",
+				slice: "following",
+				count: followingRows.length,
+			});
 		}
 
 		for (const row of [...followerRows, ...followingRows]) {
@@ -2255,6 +2440,17 @@ function importArchiveInternalEffect(
 			deleteArchiveFollowEdges.run("acct_primary", direction);
 		}
 
+		onProgress({ kind: "writing" });
+		const WRITE_PROGRESS_INTERVAL = 1000;
+		function tickWrite(
+			phase: ImportWritePhase,
+			processed: number,
+			total: number,
+		) {
+			if (processed === total || processed % WRITE_PROGRESS_INTERVAL === 0) {
+				onProgress({ kind: "write-progress", phase, processed, total });
+			}
+		}
 		db.transaction(() => {
 			if (!selection) {
 				clearImportedData(db);
@@ -2339,6 +2535,15 @@ function importArchiveInternalEffect(
 
 			const writeProfile =
 				!selection || includeProfiles ? insertProfile : insertProfileIfMissing;
+			const profilesTotal = profiles.size;
+			if (profilesTotal > 0) {
+				onProgress({
+					kind: "write-start",
+					phase: "profiles",
+					total: profilesTotal,
+				});
+			}
+			let profileIndex = 0;
 			for (const profile of profiles.values()) {
 				writeProfile.run(
 					profile.id,
@@ -2357,8 +2562,18 @@ function importArchiveInternalEffect(
 					profile.rawJson,
 					profile.createdAt,
 				);
+				profileIndex += 1;
+				tickWrite("profiles", profileIndex, profilesTotal);
 			}
 
+			if (tweetRows.length > 0) {
+				onProgress({
+					kind: "write-start",
+					phase: "tweets",
+					total: tweetRows.length,
+				});
+			}
+			let tweetWriteIndex = 0;
 			for (const tweet of tweetRows) {
 				const authorProfileId =
 					tweet.authorProfileId === "profile_me"
@@ -2406,9 +2621,19 @@ function importArchiveInternalEffect(
 					| { text: string }
 					| undefined;
 				insertTweetFts.run(tweet.id, storedTweet?.text ?? tweet.text);
+				tweetWriteIndex += 1;
+				tickWrite("tweets", tweetWriteIndex, tweetRows.length);
 			}
 
 			const importedAt = new Date().toISOString();
+			if (collectionRows.length > 0) {
+				onProgress({
+					kind: "write-start",
+					phase: "collections",
+					total: collectionRows.length,
+				});
+			}
+			let collectionIndex = 0;
 			for (const collection of collectionRows) {
 				insertCollection.run(
 					"acct_primary",
@@ -2419,6 +2644,8 @@ function importArchiveInternalEffect(
 					collection.rawJson,
 					importedAt,
 				);
+				collectionIndex += 1;
+				tickWrite("collections", collectionIndex, collectionRows.length);
 			}
 
 			for (const conversation of conversations.values()) {
@@ -2433,6 +2660,14 @@ function importArchiveInternalEffect(
 				);
 			}
 
+			if (dmMessages.length > 0) {
+				onProgress({
+					kind: "write-start",
+					phase: "dmMessages",
+					total: dmMessages.length,
+				});
+			}
+			let dmWriteIndex = 0;
 			for (const message of dmMessages) {
 				insertMessage.run(
 					message.id,
@@ -2445,6 +2680,8 @@ function importArchiveInternalEffect(
 					message.mediaCount,
 				);
 				insertDmFts.run(message.id, message.text);
+				dmWriteIndex += 1;
+				tickWrite("dmMessages", dmWriteIndex, dmMessages.length);
 			}
 
 			if (includeFollowers && followerEntries.length > 0) {
@@ -2468,6 +2705,7 @@ function importArchiveInternalEffect(
 				clearArchiveFollowRows("following");
 			}
 		})();
+		onProgress({ kind: "done" });
 
 		return {
 			ok: true,

package/src/lib/authored-live.ts

@@ -1000,6 +1000,7 @@ export function syncAuthoredTweetsEffect({
 					userFields: AUTHORED_USER_FIELDS,
 					mediaFields: AUTHORED_MEDIA_FIELDS,
 					auth: "oauth2",
+					username: identity.username,
 				}),
 			).pipe(
 				Effect.map((page) => ({ ok: true as const, page })),

package/src/lib/avatar-cache.ts

@@ -12,6 +12,7 @@ const AVATAR_SIZE_SUFFIX =
 const MAX_AVATAR_BYTES = 2 * 1024 * 1024;
 const MAX_AVATAR_DATA_URL_CHARS = MAX_AVATAR_BYTES * 4;
 const REMOTE_AVATAR_TIMEOUT_MS = 10_000;
+const DEFAULT_AVATAR_PREFETCH_CONCURRENCY = 8;
 const ALLOWED_REMOTE_AVATAR_HOSTS = new Set(["pbs.twimg.com"]);
 const RASTER_CONTENT_TYPES = new Set([
 	"image/jpeg",
@@ -289,6 +290,55 @@ export function readCachedAvatar(profileId: string) {
 	return runEffectPromise(readCachedAvatarEffect(profileId));
 }
 
+function normalizePrefetchConcurrency(value: number | undefined) {
+	if (!Number.isFinite(value) || value === undefined) {
+		return DEFAULT_AVATAR_PREFETCH_CONCURRENCY;
+	}
+	return Math.max(1, Math.min(Math.floor(value), 32));
+}
+
+export function prefetchCachedAvatarsForProfileIdsEffect(
+	profileIds: string[],
+	options: { concurrency?: number } = {},
+) {
+	const uniqueProfileIds = Array.from(
+		new Set(
+			profileIds
+				.map((profileId) => profileId.trim())
+				.filter((profileId) => profileId.length > 0),
+		),
+	);
+	const concurrency = normalizePrefetchConcurrency(options.concurrency);
+
+	return Effect.forEach(
+		uniqueProfileIds,
+		(profileId) =>
+			readCachedAvatarEffect(profileId).pipe(
+				Effect.map((avatar) =>
+					avatar ? ("available" as const) : ("missing" as const),
+				),
+				Effect.catchAll(() => Effect.succeed("failed" as const)),
+			),
+		{ concurrency },
+	).pipe(
+		Effect.map((statuses) => ({
+			requested: uniqueProfileIds.length,
+			available: statuses.filter((status) => status === "available").length,
+			missing: statuses.filter((status) => status === "missing").length,
+			failed: statuses.filter((status) => status === "failed").length,
+		})),
+	);
+}
+
+export function prefetchCachedAvatarsForProfileIds(
+	profileIds: string[],
+	options: { concurrency?: number } = {},
+) {
+	return runEffectPromise(
+		prefetchCachedAvatarsForProfileIdsEffect(profileIds, options),
+	);
+}
+
 export const __test__ = {
 	decodeDataUrl,
 	detectRasterContentType,

package/src/lib/backup.ts

@@ -537,7 +537,8 @@ function buildShards(db: Database) {
 					const kind =
 						row.kind === "home" ||
 						row.kind === "mention" ||
-						row.kind === "authored"
+						row.kind === "authored" ||
+						row.kind === "search"
 							? row.kind
 							: "unknown";
 					addRows(shards, `data/timeline_edges/${kind}.jsonl`, [row]);
@@ -1853,8 +1854,8 @@ export function importBackupEffect({
         account_id = coalesce(nullif(excluded.account_id, ''), tweets.account_id),
         author_profile_id = coalesce(nullif(excluded.author_profile_id, ''), tweets.author_profile_id),
         kind = case
-          when tweets.kind in ('home', 'mention', 'authored') then tweets.kind
-          when excluded.kind in ('home', 'mention', 'authored') then excluded.kind
+          when tweets.kind in ('home', 'mention', 'authored', 'search') then tweets.kind
+          when excluded.kind in ('home', 'mention', 'authored', 'search') then excluded.kind
           else coalesce(nullif(excluded.kind, ''), tweets.kind)
         end,
         text = coalesce(nullif(excluded.text, ''), tweets.text),

package/src/lib/bird.ts

@@ -566,6 +566,39 @@ export function listBookmarkedTweetsViaBird(options: {
 	return runEffectPromise(listBookmarkedTweetsViaBirdEffect(options));
 }
 
+export function searchTweetsViaBirdEffect(
+	query: string,
+	options: {
+		maxResults: number;
+		all?: boolean;
+		maxPages?: number;
+	},
+): Effect.Effect<XurlMentionsResponse, unknown> {
+	return Effect.gen(function* () {
+		const args = ["search", query, "-n", String(options.maxResults), "--json"];
+		if (options.all) {
+			args.push("--all");
+		}
+		if (options.all && options.maxPages !== undefined) {
+			args.push("--max-pages", String(options.maxPages));
+		}
+		const stdout = yield* runBirdJsonCommandEffect(args);
+		const payload = yield* parseBirdJsonEffect(stdout);
+		return yield* normalizeBirdTweetsPayloadEffect(payload, "search");
+	});
+}
+
+export function searchTweetsViaBird(
+	query: string,
+	options: {
+		maxResults: number;
+		all?: boolean;
+		maxPages?: number;
+	},
+): Promise<XurlMentionsResponse> {
+	return runEffectPromise(searchTweetsViaBirdEffect(query, options));
+}
+
 export function lookupTweetsByIdsViaBirdEffect(
 	ids: string[],
 ): Effect.Effect<XurlTweetsResponse, unknown> {

package/src/lib/config.ts

@@ -4,6 +4,7 @@ import {
 	existsSync,
 	mkdirSync,
 	readFileSync,
+	writeFileSync,
 } from "node:fs";
 import os from "node:os";
 import path from "node:path";
@@ -16,7 +17,7 @@ export interface BirdclawPaths {
 	configPath: string;
 }
 
-export type MentionsDataSource = "birdclaw" | "xurl" | "bird";
+export type MentionsDataSource = "birdclaw" | "auto" | "xurl" | "bird";
 export type ActionsTransport = "auto" | "bird" | "xurl";
 
 export interface BirdclawConfig {
@@ -82,11 +83,37 @@ export function getBirdclawConfig(): BirdclawConfig {
 	return cachedConfig;
 }
 
+function getConfigPath() {
+	return process.env.BIRDCLAW_CONFIG?.trim() || getBirdclawPaths().configPath;
+}
+
+export function writeBirdclawConfig(config: BirdclawConfig) {
+	const configPath = getConfigPath();
+	mkdirSync(path.dirname(configPath), { recursive: true });
+	writeFileSync(configPath, `${JSON.stringify(config, null, "\t")}\n`, "utf8");
+	cachedConfig = config;
+	return configPath;
+}
+
+export function setActionsTransport(transport: ActionsTransport) {
+	const config = getBirdclawConfig();
+	const nextConfig: BirdclawConfig = {
+		...config,
+		actions: {
+			...config.actions,
+			transport,
+		},
+	};
+	const configPath = writeBirdclawConfig(nextConfig);
+	return { configPath, transport };
+}
+
 export function resolveMentionsDataSource(
 	requestedMode?: string,
 ): MentionsDataSource {
 	if (
 		requestedMode === "birdclaw" ||
+		requestedMode === "auto" ||
 		requestedMode === "xurl" ||
 		requestedMode === "bird"
 	) {
@@ -94,13 +121,19 @@ export function resolveMentionsDataSource(
 	}
 
 	const envMode = process.env.BIRDCLAW_MENTIONS_DATA_SOURCE?.trim();
-	if (envMode === "birdclaw" || envMode === "xurl" || envMode === "bird") {
+	if (
+		envMode === "birdclaw" ||
+		envMode === "auto" ||
+		envMode === "xurl" ||
+		envMode === "bird"
+	) {
 		return envMode;
 	}
 
 	const configMode = getBirdclawConfig().mentions?.dataSource;
 	if (
 		configMode === "birdclaw" ||
+		configMode === "auto" ||
 		configMode === "xurl" ||
 		configMode === "bird"
 	) {