birdclaw 0.4.0 → 0.5.0

package/src/cli.ts

@@ -7,7 +7,16 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 import { Command } from "commander";
 import { registerModerationCommands } from "#/cli-moderation";
 import { findArchives } from "#/lib/archive-finder";
-import { importArchive } from "#/lib/archive-import";
+import {
+	ARCHIVE_IMPORT_SLICES,
+	type ArchiveImportSlice,
+	importArchive,
+} from "#/lib/archive-import";
+import {
+	AuthoredSyncError,
+	syncAuthoredTweets,
+	type AuthoredSyncMode,
+} from "#/lib/authored-live";
 import {
 	exportBackup,
 	importBackup,
@@ -29,12 +38,24 @@ import {
 } from "#/lib/config";
 import { syncDirectMessagesViaCachedBird } from "#/lib/dms-live";
 import { listInboxItems, scoreInbox } from "#/lib/inbox";
+import { backfillLinkIndex, searchLinks } from "#/lib/link-index";
+import { fetchTweetMedia, formatMediaFetchResult } from "#/lib/media-fetch";
 import { syncMentionThreads } from "#/lib/mention-threads-live";
 import { exportMentionItems } from "#/lib/mentions-export";
 import {
 	exportMentionsViaCachedBird,
 	exportMentionsViaCachedXurl,
+	syncMentions,
 } from "#/lib/mentions-live";
+import {
+	getFollowGraphSummary,
+	listFollowEvents,
+	listMutuals,
+	listNonMutualFollowing,
+	listTopFollowers,
+	listUnfollowedSince,
+	syncFollowGraph,
+} from "#/lib/follow-graph";
 import { hydrateProfilesFromX } from "#/lib/profile-hydration";
 import { resolveProfilesForIds } from "#/lib/profile-resolver";
 import { inspectProfileReplies } from "#/lib/profile-replies";
@@ -73,6 +94,28 @@ function printError(error: string) {
 	console.error(JSON.stringify({ error }));
 }
 
+function errorMessage(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
+function formatLinkSearchItems(items: ReturnType<typeof searchLinks>) {
+	return items
+		.map((item) => {
+			const linked = item.linkedTweet
+				? ` -> @${item.linkedTweet.author.handle}/${item.linkedTweet.id}: ${item.linkedTweet.text}`
+				: ` -> ${item.expansion.finalUrl}`;
+			const source =
+				item.occurrence.sourceKind === "dm"
+					? `dm ${item.occurrence.direction ?? ""}`.trim()
+					: "tweet";
+			const participant = item.participant
+				? ` @${item.participant.handle}`
+				: "";
+			return `${item.occurrence.createdAt} ${source}${participant}: ${item.occurrence.shortUrl}${linked}`;
+		})
+		.join("\n");
+}
+
 function parseNonNegativeIntegerOption(
 	value: string | undefined,
 	option: string,
@@ -98,6 +141,68 @@ function parseNonNegativeIntegerOption(
 	return parsed;
 }
 
+function parsePositiveIntegerOption(value: string | undefined, option: string) {
+	const parsed = parseNonNegativeIntegerOption(value, option);
+	if (parsed === undefined) {
+		return undefined;
+	}
+	if (parsed < 1) {
+		printError(`${option} must be at least 1`);
+		process.exitCode = 1;
+		return undefined;
+	}
+	return parsed;
+}
+
+function parseArchiveImportSelect(value: string | undefined) {
+	if (value === undefined) {
+		return undefined;
+	}
+
+	const aliases: Record<string, ArchiveImportSlice> = Object.assign(
+		Object.create(null) as Record<string, ArchiveImportSlice>,
+		{
+			tweets: "tweets",
+			likes: "likes",
+			bookmarks: "bookmarks",
+			directmessages: "directMessages",
+			"direct-messages": "directMessages",
+			dms: "directMessages",
+			profiles: "profiles",
+			followers: "followers",
+			following: "following",
+		},
+	);
+	const selected: ArchiveImportSlice[] = [];
+	const seen = new Set<ArchiveImportSlice>();
+	for (const rawItem of value.split(",")) {
+		const item = rawItem.trim();
+		if (!item) continue;
+		const slice = aliases[item] ?? aliases[item.toLowerCase()];
+		if (!slice) {
+			printError(
+				`--select must be a comma-separated subset of ${ARCHIVE_IMPORT_SLICES.join(", ")}`,
+			);
+			process.exitCode = 1;
+			return undefined;
+		}
+		if (!seen.has(slice)) {
+			seen.add(slice);
+			selected.push(slice);
+		}
+	}
+
+	if (selected.length === 0) {
+		printError(
+			`--select must include at least one of ${ARCHIVE_IMPORT_SLICES.join(", ")}`,
+		);
+		process.exitCode = 1;
+		return undefined;
+	}
+
+	return selected;
+}
+
 function resolveActionOptions(options: { transport?: string }) {
 	return {
 		transport: options.transport as ActionsTransport | undefined,
@@ -216,7 +321,15 @@ const importCommand = program
 importCommand
 	.command("archive [archivePath]")
 	.description("Import a Twitter archive into the local SQLite store")
-	.action(async (archivePath) => {
+	.option(
+		"--select <kinds>",
+		`Import only selected archive slices: ${ARCHIVE_IMPORT_SLICES.join(", ")}`,
+	)
+	.action(async (archivePath, options: { select?: string }) => {
+		const select = parseArchiveImportSelect(options.select);
+		if (options.select !== undefined && !select) {
+			return;
+		}
 		let resolvedArchivePath = archivePath;
 		if (!resolvedArchivePath) {
 			const [latestArchive] = await findArchives();
@@ -229,7 +342,7 @@ importCommand
 			);
 		}
 
-		const result = await importArchive(resolvedArchivePath);
+		const result = await importArchive(resolvedArchivePath, { select });
 		await autoSyncAfterWrite();
 		print(result, program.opts().json ?? false);
 	});
@@ -249,7 +362,7 @@ const searchCommand = program
 
 searchCommand
 	.command("tweets [query]")
-	.option("--resource <resource>", "home or mentions", "home")
+	.option("--resource <resource>", "home, mentions, or authored", "home")
 	.option("--replied", "Only replied items")
 	.option("--unreplied", "Only unreplied items")
 	.option("--since <date>", "Include tweets created at or after this date")
@@ -280,7 +393,12 @@ searchCommand
 				? "unreplied"
 				: "all";
 		const items = listTimelineItems({
-			resource: options.resource === "mentions" ? "mentions" : "home",
+			resource:
+				options.resource === "mentions"
+					? "mentions"
+					: options.resource === "authored"
+						? "authored"
+						: "home",
 			search: query,
 			replyFilter,
 			since: options.since,
@@ -364,6 +482,167 @@ searchCommand
 		print(items, program.opts().json ?? false);
 	});
 
+searchCommand
+	.command("links <query>")
+	.description("Search indexed short links, expansions, and linked tweets")
+	.option("--account <accountIdOrHandle>", "Account id or handle")
+	.option("--since <date>", "Include links created at or after this date")
+	.option("--until <date>", "Include links created before this date")
+	.option("--source <kind>", "dm or tweet")
+	.option("--direction <direction>", "inbound or outbound")
+	.option("--participant <value>", "DM participant handle or name")
+	.option("--media <type>", "image, video, or gif")
+	.option("--limit <n>", "Limit results", "20")
+	.action(async (query, options) => {
+		await autoUpdateBeforeRead();
+		const items = searchLinks(query, {
+			account: options.account,
+			since: options.since,
+			until: options.until,
+			source:
+				options.source === "tweet"
+					? "tweet"
+					: options.source === "dm"
+						? "dm"
+						: undefined,
+			direction:
+				options.direction === "inbound"
+					? "inbound"
+					: options.direction === "outbound"
+						? "outbound"
+						: undefined,
+			participant: options.participant,
+			mediaType:
+				options.media === "image" ||
+				options.media === "video" ||
+				options.media === "gif"
+					? options.media
+					: undefined,
+			limit: Number(options.limit),
+		});
+		if (program.opts().json) {
+			print(items, true);
+			return;
+		}
+		console.log(formatLinkSearchItems(items));
+	});
+
+const linksCommand = program
+	.command("links")
+	.description("Build and inspect the short-link index");
+
+linksCommand
+	.command("backfill")
+	.description("Backfill indexed URL occurrences and t.co expansions")
+	.option("--all-urls", "Index all URLs, not only t.co")
+	.option("--source <kind>", "dm or tweet")
+	.option("--refresh-url-cache", "Re-expand URLs already in the index")
+	.option("--limit <n>", "Limit network/cache expansions for this run")
+	.option("--concurrency <n>", "Concurrent URL expansion workers", "12")
+	.option("--timeout-ms <n>", "Per-redirect fetch timeout", "15000")
+	.action(async (options) => {
+		const limit = parseNonNegativeIntegerOption(options.limit, "--limit");
+		if (options.limit !== undefined && limit === undefined) {
+			return;
+		}
+		const concurrency = parseNonNegativeIntegerOption(
+			options.concurrency,
+			"--concurrency",
+		);
+		if (concurrency === undefined) {
+			return;
+		}
+		const timeoutMs = parseNonNegativeIntegerOption(
+			options.timeoutMs,
+			"--timeout-ms",
+		);
+		if (timeoutMs === undefined) {
+			return;
+		}
+		const result = await backfillLinkIndex({
+			includeAllUrls: Boolean(options.allUrls),
+			refresh: Boolean(options.refreshUrlCache),
+			source:
+				options.source === "tweet"
+					? "tweet"
+					: options.source === "dm"
+						? "dm"
+						: undefined,
+			limit,
+			concurrency,
+			timeoutMs,
+		});
+		await autoSyncAfterWrite();
+		print(result, program.opts().json ?? false);
+	});
+
+const mediaCommand = program
+	.command("media")
+	.description("Manage the local media cache");
+
+mediaCommand
+	.command("fetch")
+	.description(
+		"Fetch missing pbs.twimg.com image media already stored in tweets",
+	)
+	.option("--account <accountId>", "Account id")
+	.option("--limit <n>", "Stop after N tweets processed")
+	.option(
+		"--kind <kind>",
+		"Tweet or collection kind, e.g. home, like, bookmark",
+	)
+	.option("--since <isoDate>", "Only tweets created at or after this date")
+	.option("--parallel <n>", "Concurrent fetch workers, capped at 5", "1")
+	.option("--pacing-ms <n>", "Delay between request starts", "250")
+	.option("--video-pacing-ms <n>", "Delay between video request starts")
+	.option("--retry-max <n>", "Retries per file after rate limiting", "3")
+	.option("--include-video", "Include video and animated GIF media", true)
+	.option("--no-include-video", "Skip video and animated GIF media")
+	.option("--max-bytes <n>", "Maximum media file size in bytes", "104857600")
+	.option("--dry-run", "List what would be fetched without downloading")
+	.option("--json", "Emit JSON output")
+	.action(async (options) => {
+		const limit = parseNonNegativeIntegerOption(options.limit, "--limit");
+		if (options.limit !== undefined && limit === undefined) {
+			return;
+		}
+		const parallel =
+			parsePositiveIntegerOption(options.parallel, "--parallel") ?? 1;
+		const pacingMs =
+			parseNonNegativeIntegerOption(options.pacingMs, "--pacing-ms") ?? 250;
+		const retryMax =
+			parseNonNegativeIntegerOption(options.retryMax, "--retry-max") ?? 3;
+		const videoPacingMs =
+			options.videoPacingMs === undefined
+				? undefined
+				: parseNonNegativeIntegerOption(
+						options.videoPacingMs,
+						"--video-pacing-ms",
+					);
+		const maxBytes =
+			parseNonNegativeIntegerOption(options.maxBytes, "--max-bytes") ??
+			100 * 1024 * 1024;
+		if (process.exitCode) {
+			return;
+		}
+
+		const result = await fetchTweetMedia({
+			account: options.account,
+			limit,
+			kind: options.kind,
+			since: options.since,
+			parallel,
+			pacingMs,
+			videoPacingMs,
+			retryMax,
+			includeVideo: Boolean(options.includeVideo),
+			maxBytes,
+			dryRun: Boolean(options.dryRun),
+		});
+		const asJson = Boolean(program.opts().json || options.json);
+		print(asJson ? result : formatMediaFetchResult(result), asJson);
+	});
+
 program
 	.command("whois <query>")
 	.description("Identify likely people or orgs from local DMs and tweets")
@@ -550,28 +829,130 @@ syncCommand
 		print(result, true);
 	});
 
+syncCommand
+	.command("mentions")
+	.description("Refresh live mentions through xurl or bird")
+	.option("--account <accountId>", "Account id")
+	.option("--mode <mode>", "bird or xurl", "xurl")
+	.option("--limit <n>", "Result limit per page", "20")
+	.option("--max-pages <n>", "Stop after N pages")
+	.option("--since-id <id>", "Fetch mentions newer than this tweet id")
+	.option("--start-time <iso>", "Fetch mentions created at or after this time")
+	.option("--refresh", "Bypass live-cache freshness window")
+	.option("--cache-ttl <seconds>", "Live-cache freshness window", "120")
+	.action(async (options) => {
+		try {
+			const result = await syncMentions({
+				account: options.account,
+				mode: options.mode,
+				limit: Number(options.limit),
+				maxPages: options.maxPages ? Number(options.maxPages) : undefined,
+				sinceId: options.sinceId,
+				startTime: options.startTime,
+				refresh: Boolean(options.refresh),
+				cacheTtlMs: Number(options.cacheTtl) * 1000,
+			});
+			await autoSyncAfterWrite();
+			print(result, true);
+			if (result.partial) {
+				process.exitCode = 5;
+			}
+		} catch (error) {
+			print(
+				{
+					ok: false,
+					kind: "mentions",
+					mode: options.mode ?? "xurl",
+					error: errorMessage(error),
+				},
+				true,
+			);
+			process.exitCode = 1;
+		}
+	});
+
+syncCommand
+	.command("authored")
+	.description("Refresh authenticated authored tweets through xurl")
+	.option("--account <accountId>", "Account id")
+	.option("--mode <mode>", "xurl", "xurl")
+	.option("--limit <n>", "X API page size", "100")
+	.option("--max-pages <n>", "Stop after N pages and resume later")
+	.option("--since-id <tweetId>", "Override the stored since_id cursor")
+	.option(
+		"--until-id <tweetId>",
+		"Fetch tweets older than this id without moving the cursor",
+	)
+	.action(async (options) => {
+		try {
+			const result = await syncAuthoredTweets({
+				account: options.account,
+				mode: options.mode as AuthoredSyncMode,
+				limit: Number(options.limit),
+				maxPages: options.maxPages ? Number(options.maxPages) : undefined,
+				sinceId: options.sinceId,
+				untilId: options.untilId,
+			});
+			await autoSyncAfterWrite();
+			print(result, true);
+			if (result.partial) {
+				process.exitCode = 5;
+			}
+		} catch (error) {
+			print(
+				{
+					ok: false,
+					kind: "authored",
+					source: "xurl",
+					error: errorMessage(error),
+				},
+				true,
+			);
+			process.exitCode =
+				error instanceof AuthoredSyncError ? error.exitCode : 1;
+		}
+	});
+
 syncCommand
 	.command("mention-threads")
 	.description(
-		"Fetch tweet conversation context for recent mentions through bird",
+		"Fetch tweet conversation context for recent mentions through bird or xurl",
 	)
 	.option("--account <accountId>", "Account id")
+	.option("--mode <mode>", "bird or xurl", "bird")
 	.option("--limit <n>", "Recent mentions to inspect", "30")
 	.option("--delay-ms <n>", "Delay between thread fetches", "1500")
 	.option("--timeout-ms <n>", "Per-thread timeout", "15000")
 	.option("--all", "Fetch all retrievable thread pages")
 	.option("--max-pages <n>", "Stop after N pages")
 	.action(async (options) => {
-		const result = await syncMentionThreads({
-			account: options.account,
-			limit: Number(options.limit),
-			delayMs: Number(options.delayMs),
-			timeoutMs: Number(options.timeoutMs),
-			all: Boolean(options.all),
-			maxPages: options.maxPages ? Number(options.maxPages) : undefined,
-		});
-		await autoSyncAfterWrite();
-		print(result, true);
+		try {
+			const result = await syncMentionThreads({
+				account: options.account,
+				mode: options.mode,
+				limit: Number(options.limit),
+				delayMs: Number(options.delayMs),
+				timeoutMs: Number(options.timeoutMs),
+				all: Boolean(options.all),
+				maxPages: options.maxPages ? Number(options.maxPages) : undefined,
+			});
+			await autoSyncAfterWrite();
+			print(result, true);
+			if (result.partial) {
+				process.exitCode = 5;
+			}
+		} catch (error) {
+			print(
+				{
+					ok: false,
+					kind: "mention-threads",
+					mode: options.mode ?? "bird",
+					error: errorMessage(error),
+				},
+				true,
+			);
+			process.exitCode = 1;
+		}
 	});
 
 for (const kind of ["likes", "bookmarks"] as const) {
@@ -582,7 +963,11 @@ for (const kind of ["likes", "bookmarks"] as const) {
 		.option("--mode <mode>", "auto, xurl, or bird", "auto")
 		.option("--limit <n>", "Per-page/result limit", "20")
 		.option("--all", "Fetch every retrievable page")
-		.option("--max-pages <n>", "Stop after N pages when using --all")
+		.option(
+			"--max-pages <n>",
+			"Stop after N pages when using --all or --early-stop",
+		)
+		.option("--early-stop", "Stop when a fetched page is already fully local")
 		.option("--cache-ttl <seconds>", "Live-cache freshness window", "120")
 		.option("--refresh", "Bypass live-cache freshness window")
 		.action(async (options) => {
@@ -595,12 +980,55 @@ for (const kind of ["likes", "bookmarks"] as const) {
 				maxPages: options.maxPages ? Number(options.maxPages) : undefined,
 				refresh: Boolean(options.refresh),
 				cacheTtlMs: Number(options.cacheTtl) * 1000,
+				earlyStop: Boolean(options.earlyStop),
 			});
 			await autoSyncAfterWrite();
 			print(result, true);
 		});
 }
 
+for (const direction of ["followers", "following"] as const) {
+	syncCommand
+		.command(direction)
+		.description(
+			`Dry-run or refresh live ${direction} into the local follow graph`,
+		)
+		.option("--account <accountId>", "Account id")
+		.option("--mode <mode>", "auto, bird, or xurl", "auto")
+		.option("--limit <n>", "X API users per page", "1000")
+		.option("--max-pages <n>", "Stop after N pages")
+		.option("--max-resources <n>", "Stop after N unique users")
+		.option("--cache-ttl <seconds>", "Live-cache freshness window", "86400")
+		.option("--refresh", "Bypass the live-cache freshness window")
+		.option("--allow-partial", "Acknowledge capped/incomplete snapshot")
+		.option("--yes", "Confirm live sync or fresh-cache merge")
+		.action(async (options) => {
+			try {
+				const result = await syncFollowGraph({
+					direction,
+					account: options.account,
+					mode: options.mode,
+					limit: Number(options.limit),
+					maxPages: options.maxPages ? Number(options.maxPages) : undefined,
+					maxResources: options.maxResources
+						? Number(options.maxResources)
+						: undefined,
+					cacheTtlMs: Number(options.cacheTtl) * 1000,
+					refresh: Boolean(options.refresh),
+					allowPartial: Boolean(options.allowPartial),
+					yes: Boolean(options.yes),
+				});
+				if (!result.dryRun) {
+					await autoSyncAfterWrite();
+				}
+				print(result, true);
+			} catch (error) {
+				print({ ok: false, direction, error: errorMessage(error) }, true);
+				process.exitCode = 1;
+			}
+		});
+}
+
 const jobsCommand = program
 	.command("jobs")
 	.description("Run and install background Birdclaw jobs");
@@ -833,6 +1261,120 @@ program
 		);
 	});
 
+const graphCommand = program
+	.command("graph")
+	.description("Query the local cache-only follow graph");
+
+graphCommand
+	.command("summary")
+	.description("Summarize cached followers, following, mutuals, and snapshots")
+	.option("--account <accountId>", "Account id")
+	.action(async (options) => {
+		await autoUpdateBeforeRead();
+		print(getFollowGraphSummary({ account: options.account }), true);
+	});
+
+graphCommand
+	.command("top-followers")
+	.description("List current followers sorted by their follower count")
+	.option("--account <accountId>", "Account id")
+	.option("--limit <n>", "Limit results", "20")
+	.action(async (options) => {
+		await autoUpdateBeforeRead();
+		print(
+			listTopFollowers({
+				account: options.account,
+				limit: Number(options.limit),
+			}),
+			true,
+		);
+	});
+
+graphCommand
+	.command("unfollowed")
+	.description("List cached ended follow edges since a date")
+	.requiredOption("--date <date>", "YYYY-MM-DD or ISO timestamp")
+	.option("--account <accountId>", "Account id")
+	.option("--direction <direction>", "followers or following", "followers")
+	.option("--limit <n>", "Limit results", "100")
+	.action(async (options) => {
+		await autoUpdateBeforeRead();
+		print(
+			listUnfollowedSince({
+				account: options.account,
+				date: options.date,
+				direction:
+					options.direction === "following" ? "following" : "followers",
+				limit: Number(options.limit),
+			}),
+			true,
+		);
+	});
+
+graphCommand
+	.command("events")
+	.description("List cached append-only follow graph events")
+	.option("--account <accountId>", "Account id")
+	.option("--direction <direction>", "followers or following")
+	.option("--kind <kind>", "started or ended")
+	.option("--since <date>", "YYYY-MM-DD or ISO timestamp")
+	.option("--until <date>", "YYYY-MM-DD or ISO timestamp")
+	.option("--limit <n>", "Limit results", "100")
+	.action(async (options) => {
+		await autoUpdateBeforeRead();
+		print(
+			listFollowEvents({
+				account: options.account,
+				direction:
+					options.direction === "followers" || options.direction === "following"
+						? options.direction
+						: undefined,
+				kind:
+					options.kind === "started" || options.kind === "ended"
+						? options.kind
+						: undefined,
+				since: options.since,
+				until: options.until,
+				limit: Number(options.limit),
+			}),
+			true,
+		);
+	});
+
+graphCommand
+	.command("non-mutual-following")
+	.description("List current following who are not current followers")
+	.option("--account <accountId>", "Account id")
+	.option("--sort <mode>", "followers or handle", "followers")
+	.option("--limit <n>", "Limit results", "100")
+	.action(async (options) => {
+		await autoUpdateBeforeRead();
+		print(
+			listNonMutualFollowing({
+				account: options.account,
+				sort: options.sort === "handle" ? "handle" : "followers",
+				limit: Number(options.limit),
+			}),
+			true,
+		);
+	});
+
+graphCommand
+	.command("mutuals")
+	.description("List profiles that are both followers and following")
+	.option("--account <accountId>", "Account id")
+	.option("--limit <n>", "Limit results", "100")
+	.action(async (options) => {
+		await autoUpdateBeforeRead();
+		print(
+			listMutuals({
+				account: options.account,
+				limit: Number(options.limit),
+			}),
+			true,
+		);
+	});
+
 program
 	.command("db stats")
 	.description("Show local storage and dataset stats")
@@ -947,6 +1489,9 @@ export async function runCli(argv = process.argv) {
 if (process.argv[1]) {
 	const entryUrl = pathToFileURL(process.argv[1]).href;
 	if (import.meta.url === entryUrl) {
-		void runCli();
+		void runCli().catch((error) => {
+			console.error(error instanceof Error ? error.message : String(error));
+			process.exitCode = 1;
+		});
 	}
 }