birdclaw 0.6.0 → 0.7.0

package/src/cli.ts

@@ -10,6 +10,9 @@ import { findArchives } from "#/lib/archive-finder";
 import {
 	ARCHIVE_IMPORT_SLICES,
 	type ArchiveImportSlice,
+	type ImportProgressEvent,
+	type ImportProgressSlice,
+	type ImportWritePhase,
 	importArchive,
 } from "#/lib/archive-import";
 import {
@@ -41,6 +44,7 @@ import {
 	ensureBirdclawDirs,
 	getBirdclawPaths,
 	resolveMentionsDataSource,
+	setActionsTransport,
 } from "#/lib/config";
 import { closeDatabase } from "#/lib/db";
 import {
@@ -53,6 +57,7 @@ import { fetchTweetMedia, formatMediaFetchResult } from "#/lib/media-fetch";
 import { syncMentionThreads } from "#/lib/mention-threads-live";
 import { exportMentionItems } from "#/lib/mentions-export";
 import {
+	exportMentionsViaCachedAuto,
 	exportMentionsViaCachedBird,
 	exportMentionsViaCachedXurl,
 	syncMentions,
@@ -62,6 +67,10 @@ import {
 	type PeriodDigestOptions,
 	type PeriodDigestPreset,
 } from "#/lib/period-digest";
+import {
+	streamProfileAnalysis,
+	type ProfileAnalysisOptions,
+} from "#/lib/profile-analysis";
 import {
 	getFollowGraphSummary,
 	listFollowEvents,
@@ -75,6 +84,11 @@ import { hydrateProfilesFromX } from "#/lib/profile-hydration";
 import { resolveProfilesForIds } from "#/lib/profile-resolver";
 import { inspectProfileReplies } from "#/lib/profile-replies";
 import { runResearchMode } from "#/lib/research";
+import {
+	streamSearchDiscussion,
+	type SearchDiscussionOptions,
+	type SearchDiscussionSource,
+} from "#/lib/search-discussion";
 import {
 	applyDmRequestMutationToLocalStore,
 	createDmReply,
@@ -114,6 +128,71 @@ function errorMessage(error: unknown) {
 	return error instanceof Error ? error.message : String(error);
 }
 
+const IMPORT_SLICE_LABELS: Record<ImportProgressSlice, string> = {
+	tweets: "tweets",
+	noteTweets: "note tweets",
+	directMessages: "direct messages",
+	likes: "likes",
+	bookmarks: "bookmarks",
+	media: "media files",
+	followers: "followers",
+	following: "following",
+};
+
+const IMPORT_WRITE_LABELS: Record<ImportWritePhase, string> = {
+	profiles: "profiles",
+	tweets: "tweets",
+	collections: "likes+bookmarks",
+	dmMessages: "DM messages",
+};
+
+function logImportProgress(event: ImportProgressEvent) {
+	switch (event.kind) {
+		case "scanned":
+			process.stderr.write(
+				`Scanning archive… ${String(event.entryCount)} entries\n`,
+			);
+			return;
+		case "slice-start":
+			if (event.slice === "media") {
+				process.stderr.write("Indexing media files…\n");
+				return;
+			}
+			process.stderr.write(
+				`Parsing ${IMPORT_SLICE_LABELS[event.slice]}… (${String(event.files)} file${event.files === 1 ? "" : "s"})\n`,
+			);
+			return;
+		case "slice-file":
+			if (event.files > 1) {
+				process.stderr.write(
+					`  ${IMPORT_SLICE_LABELS[event.slice]} ${String(event.processed)}/${String(event.files)}\n`,
+				);
+			}
+			return;
+		case "slice-done":
+			process.stderr.write(
+				`  ${IMPORT_SLICE_LABELS[event.slice]}: ${event.count.toLocaleString()}\n`,
+			);
+			return;
+		case "writing":
+			process.stderr.write("Writing to database…\n");
+			return;
+		case "write-start":
+			process.stderr.write(
+				`Writing ${IMPORT_WRITE_LABELS[event.phase]}… (${event.total.toLocaleString()})\n`,
+			);
+			return;
+		case "write-progress":
+			process.stderr.write(
+				`  ${IMPORT_WRITE_LABELS[event.phase]} ${event.processed.toLocaleString()}/${event.total.toLocaleString()}\n`,
+			);
+			return;
+		case "done":
+			process.stderr.write("Import complete.\n");
+			return;
+	}
+}
+
 function formatLinkSearchItems(items: ReturnType<typeof searchLinks>) {
 	return items
 		.map((item) => {
@@ -201,6 +280,18 @@ function parseDmSyncModeOption(
 	return undefined;
 }
 
+function parseDigestLiveModeOption(
+	value: string | undefined,
+): PeriodDigestOptions["liveSyncMode"] {
+	const normalized = (value ?? "xurl").trim().toLowerCase();
+	if (normalized === "auto" || normalized === "bird" || normalized === "xurl") {
+		return normalized;
+	}
+	printError("--live-mode must be auto, bird, or xurl");
+	process.exitCode = 1;
+	return undefined;
+}
+
 function parseArchiveImportSelect(value: string | undefined) {
 	if (value === undefined) {
 		return undefined;
@@ -256,6 +347,16 @@ function resolveActionOptions(options: { transport?: string }) {
 	};
 }
 
+function parseActionsTransport(value: string | undefined) {
+	const normalized = value?.trim().toLowerCase();
+	if (normalized === "auto" || normalized === "bird" || normalized === "xurl") {
+		return normalized;
+	}
+	printError("transport must be auto, bird, or xurl");
+	process.exitCode = 1;
+	return undefined;
+}
+
 function parseDigestPeriod(value: string | undefined): PeriodDigestPreset {
 	const normalized = value?.trim().toLowerCase();
 	if (normalized === "yesterday") return "yesterday";
@@ -275,6 +376,8 @@ function buildDigestOptions(
 		until?: string;
 		maxTweets?: string;
 		maxLinks?: string;
+		liveSync?: boolean;
+		liveMode?: string;
 	},
 ): PeriodDigestOptions | null {
 	const maxTweets = parseNonNegativeIntegerOption(
@@ -291,6 +394,10 @@ function buildDigestOptions(
 	if (options.maxLinks !== undefined && maxLinks === undefined) {
 		return null;
 	}
+	const liveSyncMode = parseDigestLiveModeOption(options.liveMode);
+	if (liveSyncMode === undefined) {
+		return null;
+	}
 	return {
 		period: parseDigestPeriod(period),
 		since: options.since,
@@ -301,6 +408,8 @@ function buildDigestOptions(
 		model: options.model,
 		maxTweets,
 		maxLinks,
+		liveSync: options.liveSync !== false,
+		liveSyncMode,
 	};
 }
 
@@ -323,6 +432,231 @@ function runDigestCli(options: PeriodDigestOptions) {
 	});
 }
 
+function parseSearchDiscussionSource(
+	value: string | undefined,
+): SearchDiscussionSource | undefined {
+	const normalized = (value ?? "all").trim().toLowerCase();
+	if (
+		normalized === "all" ||
+		normalized === "home" ||
+		normalized === "mentions" ||
+		normalized === "authored" ||
+		normalized === "search" ||
+		normalized === "likes" ||
+		normalized === "bookmarks"
+	) {
+		return normalized;
+	}
+	printError(
+		"--source must be all, search, home, mentions, authored, likes, or bookmarks",
+	);
+	process.exitCode = 1;
+	return undefined;
+}
+
+function parseTweetSearchMode(value: string | undefined) {
+	const normalized = (value ?? "auto").trim().toLowerCase();
+	if (
+		normalized === "auto" ||
+		normalized === "bird" ||
+		normalized === "xurl" ||
+		normalized === "local"
+	) {
+		return normalized;
+	}
+	printError("--mode must be auto, bird, xurl, or local");
+	process.exitCode = 1;
+	return undefined;
+}
+
+function buildSearchDiscussionOptions(
+	query: string,
+	options: {
+		account?: string;
+		source?: string;
+		includeDms?: boolean;
+		since?: string;
+		until?: string;
+		question?: string;
+		originalsOnly?: boolean;
+		hideLowQuality?: boolean;
+		mode?: string;
+		model?: string;
+		refresh?: boolean;
+		limit?: string;
+		maxPages?: string;
+	},
+): SearchDiscussionOptions | null {
+	const source = parseSearchDiscussionSource(options.source);
+	if (!source) return null;
+	const mode = parseTweetSearchMode(options.mode);
+	if (!mode) return null;
+	const limit = parsePositiveIntegerOption(options.limit, "--limit");
+	if (options.limit !== undefined && limit === undefined) {
+		return null;
+	}
+	const maxPages = parsePositiveIntegerOption(options.maxPages, "--max-pages");
+	if (options.maxPages !== undefined && maxPages === undefined) {
+		return null;
+	}
+	return {
+		query,
+		account: options.account,
+		source,
+		includeDms: Boolean(options.includeDms),
+		since: options.since,
+		until: options.until,
+		question: options.question,
+		originalsOnly: Boolean(options.originalsOnly),
+		hideLowQuality: Boolean(options.hideLowQuality),
+		mode,
+		model: options.model,
+		refresh: Boolean(options.refresh),
+		limit,
+		maxPages,
+	};
+}
+
+function runSearchDiscussionCli(options: SearchDiscussionOptions) {
+	const asJson = Boolean(program.opts().json);
+	return streamSearchDiscussion(options, {
+		onDelta: asJson
+			? undefined
+			: (delta) => {
+					process.stdout.write(delta);
+				},
+	}).then((result) => {
+		if (asJson) {
+			print(result, true);
+			return;
+		}
+		if (!result.markdown.endsWith("\n")) {
+			process.stdout.write("\n");
+		}
+	});
+}
+
+function buildProfileAnalysisOptions(
+	handle: string,
+	options: {
+		account?: string;
+		model?: string;
+		refresh?: boolean;
+		maxTweets?: string;
+		maxPages?: string;
+		maxConversations?: string;
+		maxConversationPages?: string;
+		conversationDelayMs?: string;
+		rateLimitRetryMs?: string;
+		rateLimitRetries?: string;
+	},
+): ProfileAnalysisOptions | null {
+	const maxTweets = parsePositiveIntegerOption(
+		options.maxTweets,
+		"--max-tweets",
+	);
+	if (options.maxTweets !== undefined && maxTweets === undefined) {
+		return null;
+	}
+	const maxPages = parsePositiveIntegerOption(options.maxPages, "--max-pages");
+	if (options.maxPages !== undefined && maxPages === undefined) {
+		return null;
+	}
+	const maxConversations = parsePositiveIntegerOption(
+		options.maxConversations,
+		"--max-conversations",
+	);
+	if (
+		options.maxConversations !== undefined &&
+		maxConversations === undefined
+	) {
+		return null;
+	}
+	const maxConversationPages = parsePositiveIntegerOption(
+		options.maxConversationPages,
+		"--max-conversation-pages",
+	);
+	if (
+		options.maxConversationPages !== undefined &&
+		maxConversationPages === undefined
+	) {
+		return null;
+	}
+	const conversationDelayMs = parseNonNegativeIntegerOption(
+		options.conversationDelayMs,
+		"--conversation-delay-ms",
+	);
+	if (
+		options.conversationDelayMs !== undefined &&
+		conversationDelayMs === undefined
+	) {
+		return null;
+	}
+	const rateLimitRetryMs = parseNonNegativeIntegerOption(
+		options.rateLimitRetryMs,
+		"--rate-limit-retry-ms",
+	);
+	if (
+		options.rateLimitRetryMs !== undefined &&
+		rateLimitRetryMs === undefined
+	) {
+		return null;
+	}
+	const rateLimitMaxRetries = parseNonNegativeIntegerOption(
+		options.rateLimitRetries,
+		"--rate-limit-retries",
+	);
+	if (
+		options.rateLimitRetries !== undefined &&
+		rateLimitMaxRetries === undefined
+	) {
+		return null;
+	}
+	return {
+		handle,
+		account: options.account,
+		model: options.model,
+		refresh: Boolean(options.refresh),
+		maxTweets,
+		maxPages,
+		maxConversations,
+		maxConversationPages,
+		conversationDelayMs,
+		rateLimitRetryMs,
+		rateLimitMaxRetries,
+	};
+}
+
+function runProfileAnalysisCli(options: ProfileAnalysisOptions) {
+	const asJson = Boolean(program.opts().json);
+	return streamProfileAnalysis(options, {
+		onDelta: asJson
+			? undefined
+			: (delta) => {
+					process.stdout.write(delta);
+				},
+		onEvent: asJson
+			? undefined
+			: (event) => {
+					if (event.type === "status") {
+						process.stderr.write(
+							event.detail
+								? `${event.label}: ${event.detail}\n`
+								: `${event.label}\n`,
+						);
+					}
+				},
+	}).then((result) => {
+		if (asJson) {
+			print(result, true);
+			return;
+		}
+		if (!result.markdown.endsWith("\n")) {
+			process.stdout.write("\n");
+		}
+	});
+}
+
 async function enrichDmItems(
 	query: Parameters<typeof listDmConversations>[0],
 	options: {
@@ -426,14 +760,27 @@ program
 		);
 	});
 
-program
-	.command("auth status")
+const authCommand = program
+	.command("auth")
+	.description("Manage live transport");
+
+authCommand
+	.command("status")
 	.description("Show transport status")
 	.action(async () => {
 		const meta = await getQueryEnvelope();
 		print(meta.transport, program.opts().json ?? false);
 	});
 
+authCommand
+	.command("use <transport>")
+	.description("Set preferred moderation action transport")
+	.action((transport: string) => {
+		const parsed = parseActionsTransport(transport);
+		if (!parsed) return;
+		print(setActionsTransport(parsed), program.opts().json ?? false);
+	});
+
 program
 	.command("archive find")
 	.description("Find likely Twitter archives on disk")
@@ -470,9 +817,13 @@ importCommand
 			);
 		}
 
-		const result = await importArchive(resolvedArchivePath, { select });
+		const asJson = Boolean(program.opts().json);
+		const result = await importArchive(resolvedArchivePath, {
+			select,
+			onProgress: asJson ? undefined : logImportProgress,
+		});
 		await autoSyncAfterWrite();
-		print(result, program.opts().json ?? false);
+		print(result, asJson);
 	});
 
 importCommand
@@ -857,6 +1208,60 @@ program
 		);
 	});
 
+program
+	.command("discuss <query>")
+	.description("Search live/local tweets and summarize the results with AI")
+	.option("--account <accountId>", "Account id")
+	.option(
+		"--source <source>",
+		"all, search, home, mentions, authored, likes, or bookmarks",
+		"search",
+	)
+	.option("--mode <mode>", "auto, bird, xurl, or local", "xurl")
+	.option("--include-dms", "Include private DM search matches")
+	.option("--since <isoDate>", "Include matches created at or after this date")
+	.option("--until <isoDate>", "Include matches created before this date")
+	.option("--question <prompt>", "Discussion question or angle")
+	.option("--originals-only", "Exclude authored replies that start with @")
+	.option("--hide-low-quality", "Hide RTs, tiny replies, and link-only noise")
+	.option("--model <model>", "OpenAI model id")
+	.option("--refresh", "Bypass the local discussion cache")
+	.option("--limit <n>", "Maximum tweet context", "20000")
+	.option("--max-pages <n>", "Maximum live search pages", "200")
+	.action(async (query, options) => {
+		await autoUpdateBeforeRead();
+		const discussionOptions = buildSearchDiscussionOptions(query, options);
+		if (!discussionOptions) return;
+		await runSearchDiscussionCli(discussionOptions);
+	});
+
+program
+	.command("profile-analyze <handle>")
+	.alias("profile-analyse")
+	.description("Backfill a profile with xurl and summarize it with AI")
+	.option("--account <accountId>", "Account id")
+	.option("--model <model>", "OpenAI model id")
+	.option("--refresh", "Bypass profile fetch and analysis caches")
+	.option("--max-tweets <n>", "Maximum profile tweets", "10000")
+	.option("--max-pages <n>", "Maximum profile timeline pages", "100")
+	.option("--max-conversations <n>", "Maximum conversations to backfill", "80")
+	.option("--max-conversation-pages <n>", "Maximum pages per conversation", "3")
+	.option(
+		"--conversation-delay-ms <n>",
+		"Delay between conversation search calls",
+	)
+	.option(
+		"--rate-limit-retry-ms <n>",
+		"Delay before retrying conversation 429s",
+	)
+	.option("--rate-limit-retries <n>", "Conversation 429 retry count")
+	.action(async (handle, options) => {
+		await autoUpdateBeforeRead();
+		const analysisOptions = buildProfileAnalysisOptions(handle, options);
+		if (!analysisOptions) return;
+		await runProfileAnalysisCli(analysisOptions);
+	});
+
 program
 	.command("today")
 	.description("Stream an AI digest of what happened today")
@@ -864,8 +1269,14 @@ program
 	.option("--include-dms", "Include private DM context")
 	.option("--model <model>", "OpenAI model id")
 	.option("--refresh", "Bypass the local digest cache")
-	.option("--max-tweets <n>", "Maximum tweet context", "300")
+	.option("--max-tweets <n>", "Maximum tweet context", "5000")
 	.option("--max-links <n>", "Maximum linked articles", "12")
+	.option("--no-live-sync", "Use only the local database")
+	.option(
+		"--live-mode <mode>",
+		"Live timeline mode: xurl, bird, or auto",
+		"xurl",
+	)
 	.action(async (options) => {
 		await autoUpdateBeforeRead();
 		const digestOptions = buildDigestOptions("today", options);
@@ -882,8 +1293,14 @@ program
 	.option("--until <isoDate>", "End of explicit window")
 	.option("--model <model>", "OpenAI model id")
 	.option("--refresh", "Bypass the local digest cache")
-	.option("--max-tweets <n>", "Maximum tweet context", "300")
+	.option("--max-tweets <n>", "Maximum tweet context", "5000")
 	.option("--max-links <n>", "Maximum linked articles", "12")
+	.option("--no-live-sync", "Use only the local database")
+	.option(
+		"--live-mode <mode>",
+		"Live timeline mode: xurl, bird, or auto",
+		"xurl",
+	)
 	.action(async (period, options) => {
 		await autoUpdateBeforeRead();
 		const digestOptions = buildDigestOptions(period, options);
@@ -899,7 +1316,7 @@ mentionsCommand
 	.command("export [query]")
 	.description("Return mention tweets with plain-text and markdown renderings")
 	.option("--account <accountId>", "Account id")
-	.option("--mode <mode>", "birdclaw, xurl, or bird")
+	.option("--mode <mode>", "birdclaw, auto, xurl, or bird")
 	.option("--replied", "Only replied items")
 	.option("--unreplied", "Only unreplied items")
 	.option("--refresh", "Refresh the live xurl cache before returning")
@@ -919,23 +1336,14 @@ mentionsCommand
 				: "all";
 		const limit = Number(options.limit);
 		const mode = resolveMentionsDataSource(options.mode);
-		if (mode === "xurl") {
-			const payload = await exportMentionsViaCachedXurl({
-				account: options.account,
-				search: query,
-				replyFilter,
-				limit,
-				all: Boolean(options.all) || options.maxPages !== undefined,
-				maxPages: options.maxPages ? Number(options.maxPages) : undefined,
-				refresh: Boolean(options.refresh),
-				cacheTtlMs: Number(options.cacheTtl) * 1000,
-			});
-			await autoSyncAfterWrite();
-			print(payload, true);
-			return;
-		}
-		if (mode === "bird") {
-			const payload = await exportMentionsViaCachedBird({
+		if (mode === "xurl" || mode === "bird" || mode === "auto") {
+			const exportFn =
+				mode === "xurl"
+					? exportMentionsViaCachedXurl
+					: mode === "bird"
+						? exportMentionsViaCachedBird
+						: exportMentionsViaCachedAuto;
+			const payload = await exportFn({
 				account: options.account,
 				search: query,
 				replyFilter,
@@ -982,16 +1390,20 @@ const syncCommand = program
 
 syncCommand
 	.command("timeline")
-	.description("Refresh live home timeline through bird")
+	.description("Refresh live home timeline through xurl or bird")
 	.option("--account <accountId>", "Account id")
+	.option("--mode <mode>", "auto, xurl, or bird", "auto")
 	.option("--limit <n>", "Result limit", "100")
+	.option("--max-pages <n>", "Stop after N xurl pages", "3")
 	.option("--for-you", 'Fetch "For You" instead of chronological Following')
 	.option("--cache-ttl <seconds>", "Live-cache freshness window", "120")
 	.option("--refresh", "Bypass live-cache freshness window")
 	.action(async (options) => {
 		const result = await syncHomeTimeline({
 			account: options.account,
+			mode: options.mode,
 			limit: Number(options.limit),
+			maxPages: Number(options.maxPages),
 			following: !options.forYou,
 			refresh: Boolean(options.refresh),
 			cacheTtlMs: Number(options.cacheTtl) * 1000,
@@ -1004,7 +1416,7 @@ syncCommand
 	.command("mentions")
 	.description("Refresh live mentions through xurl or bird")
 	.option("--account <accountId>", "Account id")
-	.option("--mode <mode>", "bird or xurl", "xurl")
+	.option("--mode <mode>", "auto, bird, or xurl", "auto")
 	.option("--limit <n>", "Result limit per page", "20")
 	.option("--max-pages <n>", "Stop after N pages")
 	.option("--since-id <id>", "Fetch mentions newer than this tweet id")

package/src/components/AppNav.tsx

@@ -3,12 +3,17 @@ import {
 	Bell,
 	Bookmark,
 	CalendarDays,
+	Database,
+	Gauge,
+	Globe2,
 	Heart,
 	Home,
 	Inbox,
 	Link as LinkIcon,
 	Mail,
+	MessagesSquare,
 	ShieldOff,
+	UserSearch,
 } from "lucide-react";
 import {
 	cx,
@@ -36,11 +41,16 @@ import { ThemeSlider } from "./ThemeSlider";
 const links = [
 	{ to: "/inbox", label: "Inbox", icon: Inbox },
 	{ to: "/today", label: "Today", icon: CalendarDays },
+	{ to: "/discuss", label: "Discuss", icon: MessagesSquare },
+	{ to: "/profile-analyze", label: "Analyse", icon: UserSearch },
+	{ to: "/network-map", label: "Map", icon: Globe2 },
+	{ to: "/data-sources", label: "Sources", icon: Database },
 	{ to: "/", label: "Home", icon: Home },
 	{ to: "/mentions", label: "Mentions", icon: Bell },
 	{ to: "/likes", label: "Likes", icon: Heart },
 	{ to: "/bookmarks", label: "Bookmarks", icon: Bookmark },
 	{ to: "/links", label: "Links", icon: LinkIcon },
+	{ to: "/rate-limits", label: "Rate Limits", icon: Gauge },
 	{ to: "/dms", label: "DMs", icon: Mail },
 	{ to: "/blocks", label: "Blocks", icon: ShieldOff },
 ] as const;