birdclaw 0.2.1 → 0.4.0

package/src/cli.ts

@@ -29,13 +29,16 @@ import {
 } from "#/lib/config";
 import { syncDirectMessagesViaCachedBird } from "#/lib/dms-live";
 import { listInboxItems, scoreInbox } from "#/lib/inbox";
+import { syncMentionThreads } from "#/lib/mention-threads-live";
 import { exportMentionItems } from "#/lib/mentions-export";
 import {
 	exportMentionsViaCachedBird,
 	exportMentionsViaCachedXurl,
 } from "#/lib/mentions-live";
 import { hydrateProfilesFromX } from "#/lib/profile-hydration";
+import { resolveProfilesForIds } from "#/lib/profile-resolver";
 import { inspectProfileReplies } from "#/lib/profile-replies";
+import { runResearchMode } from "#/lib/research";
 import {
 	createDmReply,
 	createPost,
@@ -48,6 +51,9 @@ import {
 	syncTimelineCollection,
 	type TimelineCollectionMode,
 } from "#/lib/timeline-collections-live";
+import { syncHomeTimeline } from "#/lib/timeline-live";
+import { expandUrlsFromTexts } from "#/lib/url-expansion";
+import { formatWhois, runWhois } from "#/lib/whois";
 
 const program = new Command();
 const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
@@ -63,12 +69,91 @@ function print(data: unknown, asJson: boolean) {
 	console.log(data);
 }
 
+function printError(error: string) {
+	console.error(JSON.stringify({ error }));
+}
+
+function parseNonNegativeIntegerOption(
+	value: string | undefined,
+	option: string,
+) {
+	if (value === undefined) {
+		return undefined;
+	}
+
+	const trimmed = value.trim();
+	if (!/^\d+$/.test(trimmed)) {
+		printError(`${option} must be a non-negative integer`);
+		process.exitCode = 1;
+		return undefined;
+	}
+
+	const parsed = Number.parseInt(trimmed, 10);
+	if (!Number.isSafeInteger(parsed)) {
+		printError(`${option} must be a non-negative integer`);
+		process.exitCode = 1;
+		return undefined;
+	}
+
+	return parsed;
+}
+
 function resolveActionOptions(options: { transport?: string }) {
 	return {
 		transport: options.transport as ActionsTransport | undefined,
 	};
 }
 
+async function enrichDmItems(
+	query: Parameters<typeof listDmConversations>[0],
+	options: {
+		resolveProfiles?: boolean;
+		expandUrls?: boolean;
+		refreshProfileCache?: boolean;
+		refreshUrlCache?: boolean;
+		xurlFallback?: boolean;
+	},
+) {
+	let items = listDmConversations(query);
+	const profileResolution = options.resolveProfiles
+		? await resolveProfilesForIds(
+				items.map((item) => item.participant.id),
+				{
+					refresh: options.refreshProfileCache,
+					xurlFallback: options.xurlFallback ?? true,
+				},
+			)
+		: undefined;
+	if (profileResolution) {
+		items = listDmConversations(query);
+	}
+
+	const urlExpansions = options.expandUrls
+		? await expandUrlsFromTexts(
+				items.flatMap((item) => [
+					item.lastMessagePreview,
+					item.searchSnippet ?? "",
+					...(item.matches ?? []).flatMap((match) => [
+						...match.before.map((message) => message.text),
+						match.message.text,
+						...match.after.map((message) => message.text),
+					]),
+				]),
+				{ refresh: options.refreshUrlCache },
+			)
+		: undefined;
+
+	if (!profileResolution && !urlExpansions) {
+		return items;
+	}
+
+	return {
+		items,
+		...(profileResolution ? { profileResolution } : {}),
+		...(urlExpansions ? { urlExpansions } : {}),
+	};
+}
+
 async function autoUpdateBeforeRead() {
 	const result = await maybeAutoUpdateBackup();
 	if (!result.ok) {
@@ -171,10 +256,23 @@ searchCommand
 	.option("--until <date>", "Include tweets created before this date")
 	.option("--originals-only", "Exclude authored replies that start with @")
 	.option("--hide-low-quality", "Hide RTs, tiny replies, and link-only noise")
+	.option(
+		"--min-likes <n>",
+		"Override the low-quality like threshold (default 50)",
+	)
+	.option("--quality-reason", "Include qualityReason on each row")
 	.option("--liked", "Only liked tweets")
 	.option("--bookmarked", "Only bookmarked tweets")
 	.option("--limit <n>", "Limit results", "20")
 	.action(async (query, options) => {
+		const minLikes = parseNonNegativeIntegerOption(
+			options.minLikes,
+			"--min-likes",
+		);
+		if (options.minLikes !== undefined && minLikes === undefined) {
+			return;
+		}
+
 		await autoUpdateBeforeRead();
 		const replyFilter = options.replied
 			? "replied"
@@ -189,6 +287,8 @@ searchCommand
 			until: options.until,
 			includeReplies: !options.originalsOnly,
 			qualityFilter: options.hideLowQuality ? "summary" : "all",
+			lowQualityThreshold: minLikes,
+			includeQualityReason: Boolean(options.qualityReason),
 			likedOnly: Boolean(options.liked),
 			bookmarkedOnly: Boolean(options.bookmarked),
 			limit: Number(options.limit),
@@ -204,17 +304,37 @@ searchCommand
 	.option("--min-influence-score <n>", "Minimum derived influence score")
 	.option("--max-influence-score <n>", "Maximum derived influence score")
 	.option("--sort <mode>", "recent or influence", "recent")
+	.option(
+		"--context <n>",
+		"Include N messages before and after each match",
+		"0",
+	)
+	.option(
+		"--resolve-profiles",
+		"Resolve placeholder DM profiles through cache/bird/xurl",
+	)
+	.option("--expand-urls", "Expand URLs through the persistent URL cache")
+	.option("--refresh-profile-cache", "Bypass profile lookup cache")
+	.option("--refresh-url-cache", "Bypass URL expansion cache")
+	.option(
+		"--no-xurl-fallback",
+		"Do not fall back to xurl after bird profile lookup",
+	)
 	.option("--replied", "Only replied threads")
 	.option("--unreplied", "Only unreplied threads")
 	.option("--limit <n>", "Limit results", "20")
 	.action(async (query, options) => {
 		await autoUpdateBeforeRead();
+		const context = parseNonNegativeIntegerOption(options.context, "--context");
+		if (context === undefined) {
+			return;
+		}
 		const replyFilter = options.replied
 			? "replied"
 			: options.unreplied
 				? "unreplied"
 				: "all";
-		const items = listDmConversations({
+		const dmQuery = {
 			search: query,
 			participant: options.participant,
 			minFollowers: options.minFollowers
@@ -231,11 +351,96 @@ searchCommand
 				: undefined,
 			sort: options.sort === "influence" ? "influence" : "recent",
 			replyFilter,
+			context,
 			limit: Number(options.limit),
+		} as const;
+		const items = await enrichDmItems(dmQuery, {
+			resolveProfiles: Boolean(options.resolveProfiles),
+			expandUrls: Boolean(options.expandUrls),
+			refreshProfileCache: Boolean(options.refreshProfileCache),
+			refreshUrlCache: Boolean(options.refreshUrlCache),
+			xurlFallback: options.xurlFallback,
 		});
 		print(items, program.opts().json ?? false);
 	});
 
+program
+	.command("whois <query>")
+	.description("Identify likely people or orgs from local DMs and tweets")
+	.option("--account <accountId>", "Account id")
+	.option("--no-dms", "Do not search DMs")
+	.option("--tweets", "Include local tweet search evidence")
+	.option("--no-resolve-profiles", "Do not resolve placeholder profiles")
+	.option("--no-expand-urls", "Do not expand URLs")
+	.option("--refresh-profile-cache", "Bypass profile lookup cache")
+	.option("--refresh-url-cache", "Bypass URL expansion cache")
+	.option(
+		"--no-xurl-fallback",
+		"Do not fall back to xurl after bird profile lookup",
+	)
+	.option(
+		"--affiliation <query>",
+		"Require affiliation, bio, or history evidence",
+	)
+	.option(
+		"--current-affiliation <query>",
+		"Require an active affiliation badge",
+	)
+	.option(
+		"--exclude-domain-only",
+		"Drop candidates that only match domains/URLs",
+	)
+	.option("--context <n>", "DM messages before and after each match", "4")
+	.option("--limit <n>", "Limit candidates", "10")
+	.action(async (query, options) => {
+		await autoUpdateBeforeRead();
+		const context = parseNonNegativeIntegerOption(options.context, "--context");
+		if (context === undefined) {
+			return;
+		}
+		const result = await runWhois(query, {
+			account: options.account,
+			dms: options.dms,
+			tweets: Boolean(options.tweets),
+			resolveProfiles: options.resolveProfiles,
+			expandUrls: options.expandUrls,
+			refreshProfileCache: Boolean(options.refreshProfileCache),
+			refreshUrlCache: Boolean(options.refreshUrlCache),
+			xurlFallback: options.xurlFallback,
+			affiliation: options.affiliation,
+			currentAffiliation: options.currentAffiliation,
+			excludeDomainOnly: Boolean(options.excludeDomainOnly),
+			context,
+			limit: Number(options.limit),
+		});
+		print(
+			program.opts().json ? result : formatWhois(result),
+			program.opts().json ?? false,
+		);
+	});
+
+program
+	.command("research [query]")
+	.description("Build a markdown research brief from bookmarked threads")
+	.option("--account <accountId>", "Account id")
+	.option("--limit <n>", "Seed bookmark limit", "20")
+	.option("--thread-depth <n>", "Maximum ancestor walk depth", "10")
+	.option("--out <path>", "Write the markdown brief to a file")
+	.action(async (query, options) => {
+		await autoUpdateBeforeRead();
+		const report = await runResearchMode({
+			account: options.account,
+			query,
+			limit: Number(options.limit),
+			maxThreadDepth: Number(options.threadDepth),
+			outPath: options.out,
+		});
+		print(
+			program.opts().json ? report : report.markdown,
+			program.opts().json ?? false,
+		);
+	});
+
 const mentionsCommand = program
 	.command("mentions")
 	.description("Export local mention tweets for scripts and agents");
@@ -325,6 +530,50 @@ const syncCommand = program
 	.command("sync")
 	.description("Refresh live Twitter collections into the local store");
 
+syncCommand
+	.command("timeline")
+	.description("Refresh live home timeline through bird")
+	.option("--account <accountId>", "Account id")
+	.option("--limit <n>", "Result limit", "100")
+	.option("--for-you", 'Fetch "For You" instead of chronological Following')
+	.option("--cache-ttl <seconds>", "Live-cache freshness window", "120")
+	.option("--refresh", "Bypass live-cache freshness window")
+	.action(async (options) => {
+		const result = await syncHomeTimeline({
+			account: options.account,
+			limit: Number(options.limit),
+			following: !options.forYou,
+			refresh: Boolean(options.refresh),
+			cacheTtlMs: Number(options.cacheTtl) * 1000,
+		});
+		await autoSyncAfterWrite();
+		print(result, true);
+	});
+
+syncCommand
+	.command("mention-threads")
+	.description(
+		"Fetch tweet conversation context for recent mentions through bird",
+	)
+	.option("--account <accountId>", "Account id")
+	.option("--limit <n>", "Recent mentions to inspect", "30")
+	.option("--delay-ms <n>", "Delay between thread fetches", "1500")
+	.option("--timeout-ms <n>", "Per-thread timeout", "15000")
+	.option("--all", "Fetch all retrievable thread pages")
+	.option("--max-pages <n>", "Stop after N pages")
+	.action(async (options) => {
+		const result = await syncMentionThreads({
+			account: options.account,
+			limit: Number(options.limit),
+			delayMs: Number(options.delayMs),
+			timeoutMs: Number(options.timeoutMs),
+			all: Boolean(options.all),
+			maxPages: options.maxPages ? Number(options.maxPages) : undefined,
+		});
+		await autoSyncAfterWrite();
+		print(result, true);
+	});
+
 for (const kind of ["likes", "bookmarks"] as const) {
 	syncCommand
 		.command(kind)
@@ -434,6 +683,17 @@ dmsCommand
 	.option("--min-influence-score <n>", "Minimum derived influence score")
 	.option("--max-influence-score <n>", "Maximum derived influence score")
 	.option("--sort <mode>", "recent or influence", "recent")
+	.option(
+		"--resolve-profiles",
+		"Resolve placeholder DM profiles through cache/bird/xurl",
+	)
+	.option("--expand-urls", "Expand URLs through the persistent URL cache")
+	.option("--refresh-profile-cache", "Bypass profile lookup cache")
+	.option("--refresh-url-cache", "Bypass URL expansion cache")
+	.option(
+		"--no-xurl-fallback",
+		"Do not fall back to xurl after bird profile lookup",
+	)
 	.option("--replied", "Only replied threads")
 	.option("--unreplied", "Only unreplied threads")
 	.option("--limit <n>", "Limit results", "20")
@@ -454,25 +714,34 @@ dmsCommand
 		} else {
 			await autoUpdateBeforeRead();
 		}
-		const items = listDmConversations({
-			account: options.account,
-			participant: options.participant,
-			minFollowers: options.minFollowers
-				? Number(options.minFollowers)
-				: undefined,
-			maxFollowers: options.maxFollowers
-				? Number(options.maxFollowers)
-				: undefined,
-			minInfluenceScore: options.minInfluenceScore
-				? Number(options.minInfluenceScore)
-				: undefined,
-			maxInfluenceScore: options.maxInfluenceScore
-				? Number(options.maxInfluenceScore)
-				: undefined,
-			sort: options.sort === "influence" ? "influence" : "recent",
-			replyFilter,
-			limit: Number(options.limit),
-		});
+		const items = await enrichDmItems(
+			{
+				account: options.account,
+				participant: options.participant,
+				minFollowers: options.minFollowers
+					? Number(options.minFollowers)
+					: undefined,
+				maxFollowers: options.maxFollowers
+					? Number(options.maxFollowers)
+					: undefined,
+				minInfluenceScore: options.minInfluenceScore
+					? Number(options.minInfluenceScore)
+					: undefined,
+				maxInfluenceScore: options.maxInfluenceScore
+					? Number(options.maxInfluenceScore)
+					: undefined,
+				sort: options.sort === "influence" ? "influence" : "recent",
+				replyFilter,
+				limit: Number(options.limit),
+			},
+			{
+				resolveProfiles: Boolean(options.resolveProfiles),
+				expandUrls: Boolean(options.expandUrls),
+				refreshProfileCache: Boolean(options.refreshProfileCache),
+				refreshUrlCache: Boolean(options.refreshUrlCache),
+				xurlFallback: options.xurlFallback,
+			},
+		);
 		print(items, program.opts().json ?? false);
 	});
 

package/src/lib/actions-transport.ts

@@ -9,7 +9,9 @@ import { type ActionsTransport, resolveActionsTransport } from "./config";
 import type {
 	ModerationAction,
 	ModerationActionTransportResult,
+	ModerationTransportKind,
 } from "./types";
+import { blockUserViaXWeb, unblockUserViaXWeb } from "./x-web";
 import {
 	blockUserViaXurl,
 	lookupAuthenticatedUser,
@@ -27,7 +29,7 @@ interface RunActionParams {
 	transport?: string;
 }
 
-function normalizeFailure(transport: "bird" | "xurl", output: string) {
+function normalizeFailure(transport: ModerationTransportKind, output: string) {
 	return `${transport}: ${output}`;
 }
 
@@ -133,6 +135,37 @@ async function runXurlAction(
 	};
 }
 
+async function runXWebAction(
+	action: ModerationAction,
+	targetUserId?: string,
+): Promise<ActionTransportResult> {
+	if (action !== "block" && action !== "unblock") {
+		return {
+			ok: false,
+			output: `x-web does not support ${action}`,
+			transport: "x-web",
+		};
+	}
+
+	if (!targetUserId) {
+		return {
+			ok: false,
+			output: "missing target user id for x-web transport",
+			transport: "x-web",
+		};
+	}
+
+	const result =
+		action === "block"
+			? await blockUserViaXWeb(targetUserId)
+			: await unblockUserViaXWeb(targetUserId);
+
+	return {
+		...result,
+		transport: "x-web",
+	};
+}
+
 export async function runModerationAction({
 	action,
 	query,
@@ -160,6 +193,30 @@ export async function runModerationAction({
 		};
 	}
 
+	if (action === "block" || action === "unblock") {
+		const xWebResult = await runXWebAction(action, targetUserId);
+		if (xWebResult.ok) {
+			return {
+				...xWebResult,
+				output: [
+					xWebResult.output,
+					`falling back after ${normalizeFailure("bird", birdResult.output)}`,
+					`falling back after ${normalizeFailure("xurl", xurlResult.output)}`,
+				].join("\n"),
+			};
+		}
+
+		return {
+			ok: false,
+			output: [
+				normalizeFailure("bird", birdResult.output),
+				normalizeFailure("xurl", xurlResult.output),
+				normalizeFailure("x-web", xWebResult.output),
+			].join("\n"),
+			transport: xWebResult.transport,
+		};
+	}
+
 	return {
 		ok: false,
 		output: [

package/src/lib/archive-import.ts

@@ -286,6 +286,7 @@ function clearImportedData() {
 	db.exec(`
     delete from ai_scores;
     delete from tweet_actions;
+    delete from tweet_account_edges;
     delete from tweet_collections;
     delete from dm_fts;
     delete from tweets_fts;
@@ -476,6 +477,7 @@ export async function importArchive(
 			displayName: string;
 			bio: string;
 			followersCount: number;
+			followingCount: number;
 			avatarHue: number;
 			createdAt: string;
 		}
@@ -500,6 +502,7 @@ export async function importArchive(
 		displayName: accountPayload.displayName,
 		bio: accountPayload.bio,
 		followersCount: 0,
+		followingCount: 0,
 		avatarHue: 18,
 		createdAt: accountPayload.createdAt,
 	};
@@ -583,6 +586,7 @@ export async function importArchive(
 							conversationName || `Group DM ${externalParticipantIds.length}`,
 						bio: `Group DM with ${externalParticipantIds.length} participants`,
 						followersCount: 0,
+						followingCount: 0,
 						avatarHue: 220,
 						createdAt: accountPayload.createdAt,
 					});
@@ -598,6 +602,7 @@ export async function importArchive(
 						displayName: inferred.displayName,
 						bio: `Imported from archive user ${otherUserId}`,
 						followersCount: 0,
+						followingCount: 0,
 						avatarHue: 210,
 						createdAt: accountPayload.createdAt,
 					});
@@ -626,6 +631,7 @@ export async function importArchive(
 								displayName: inferred.displayName,
 								bio: `Imported from archive user ${senderId}`,
 								followersCount: 0,
+								followingCount: 0,
 								avatarHue: 240,
 								createdAt: accountPayload.createdAt,
 							});
@@ -745,6 +751,7 @@ export async function importArchive(
 			displayName: "Unknown",
 			bio: "Imported from archive collection metadata",
 			followersCount: 0,
+			followingCount: 0,
 			avatarHue: 210,
 			createdAt: accountPayload.createdAt,
 		});
@@ -758,8 +765,8 @@ export async function importArchive(
     values (?, ?, ?, ?, ?, 1, ?)
   `);
 	const insertProfile = db.prepare(`
-    insert into profiles (id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at)
-    values (?, ?, ?, ?, ?, ?, ?, ?)
+    insert into profiles (id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at)
+    values (?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 	const insertTweet = db.prepare(`
     insert into tweets (
@@ -770,6 +777,16 @@ export async function importArchive(
 	const insertTweetFts = db.prepare(
 		"insert into tweets_fts (tweet_id, text) values (?, ?)",
 	);
+	const insertTimelineEdge = db.prepare(`
+    insert into tweet_account_edges (
+      account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count, source,
+      raw_json, updated_at
+    ) values (?, ?, ?, ?, ?, 1, 'archive', '{}', ?)
+    on conflict(account_id, tweet_id, kind) do update set
+      first_seen_at = min(tweet_account_edges.first_seen_at, excluded.first_seen_at),
+      last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
+      updated_at = max(tweet_account_edges.updated_at, excluded.updated_at)
+  `);
 	const insertCollection = db.prepare(`
     insert into tweet_collections (
       account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
@@ -811,6 +828,7 @@ export async function importArchive(
 				profile.displayName,
 				profile.bio,
 				profile.followersCount,
+				profile.followingCount,
 				profile.avatarHue,
 				null,
 				profile.createdAt,
@@ -835,6 +853,16 @@ export async function importArchive(
 				tweet.mediaJson,
 				tweet.quotedTweetId,
 			);
+			if (tweet.kind === "home") {
+				insertTimelineEdge.run(
+					"acct_primary",
+					tweet.id,
+					tweet.kind,
+					tweet.createdAt,
+					tweet.createdAt,
+					new Date().toISOString(),
+				);
+			}
 			insertTweetFts.run(tweet.id, tweet.text);
 		}
 
@@ -898,8 +926,11 @@ export async function importArchive(
 }
 
 export const __test__ = {
+	normalizeArchivePath,
 	extractArchiveJson,
 	parseArchiveArray,
+	getFirstEntry,
+	getMatchingEntries,
 	parseTwitterDate,
 	asRecord,
 	asArray,
@@ -908,6 +939,7 @@ export const __test__ = {
 	getTweetMediaCount,
 	extractTweetEntities,
 	extractTweetMedia,
+	extractCollectionTweet,
 	buildAccountPayload,
 	inferProfileFromDirectory,
 };