birdclaw 0.2.1 → 0.3.0

package/src/lib/bird.ts

@@ -1,10 +1,15 @@
 import { execFile } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { promisify } from "node:util";
 import { getBirdCommand } from "./config";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
 	XurlMentionUser,
+	XurlReferencedTweet,
+	XurlTweetsResponse,
 } from "./types";
 
 const execFileAsync = promisify(execFile);
@@ -28,7 +33,9 @@ interface BirdTweetItem {
 	retweetCount?: number;
 	likeCount?: number;
 	conversationId?: string;
-	inReplyToStatusId?: string;
+	inReplyToStatusId?: string | null;
+	quotedStatusId?: string | null;
+	quotedTweet?: { id?: string | null } | null;
 	author?: BirdTweetAuthor;
 	authorId?: string;
 	media?: BirdTweetMedia[];
@@ -140,6 +147,39 @@ function parseBirdJson(stdout: string) {
 	}
 }
 
+function formatBirdCommandError(error: unknown, birdCommand: string) {
+	if (
+		error instanceof Error &&
+		"code" in error &&
+		(error as { code?: unknown }).code === "ENOENT"
+	) {
+		return new Error(
+			`bird CLI not found at ${birdCommand}. Install @steipete/bird or set BIRDCLAW_BIRD_COMMAND / mentions.birdCommand to a valid bird binary.`,
+		);
+	}
+
+	return error;
+}
+
+async function runBirdJsonCommand(args: string[], timeoutMs?: number) {
+	const tempDir = mkdtempSync(join(tmpdir(), "birdclaw-bird-"));
+	const stdoutPath = join(tempDir, "stdout.json");
+	const birdCommand = getBirdCommand();
+	try {
+		const shellScript = 'out="$1"; shift; exec "$@" > "$out"';
+		await execFileAsync(
+			"/bin/bash",
+			["-lc", shellScript, "birdclaw-bird", stdoutPath, birdCommand, ...args],
+			{ maxBuffer: BIRD_JSON_MAX_BUFFER_BYTES, timeout: timeoutMs },
+		);
+		return readFileSync(stdoutPath, "utf8");
+	} catch (error) {
+		throw formatBirdCommandError(error, birdCommand);
+	} finally {
+		rmSync(tempDir, { recursive: true, force: true });
+	}
+}
+
 function getBirdTweetItems(payload: unknown, command: string) {
 	if (Array.isArray(payload)) {
 		return payload as BirdTweetItem[];
@@ -156,6 +196,17 @@ function getBirdTweetItems(payload: unknown, command: string) {
 	throw new Error(`bird ${command} returned unexpected JSON`);
 }
 
+function getBirdTweetItem(payload: unknown, command: string) {
+	if (payload && typeof payload === "object") {
+		const record = payload as { id?: unknown };
+		if (typeof record.id === "string" && record.id.length > 0) {
+			return payload as BirdTweetItem;
+		}
+	}
+
+	throw new Error(`bird ${command} returned unexpected JSON`);
+}
+
 function toMediaEntities(media: BirdTweetMedia[] | undefined) {
 	if (!Array.isArray(media) || media.length === 0) {
 		return undefined;
@@ -175,6 +226,25 @@ function toMediaEntities(media: BirdTweetMedia[] | undefined) {
 	};
 }
 
+function toReferencedTweets(item: BirdTweetItem) {
+	const references: XurlReferencedTweet[] = [];
+	if (typeof item.inReplyToStatusId === "string" && item.inReplyToStatusId) {
+		references.push({ type: "replied_to", id: item.inReplyToStatusId });
+	}
+
+	const quotedTweetId =
+		typeof item.quotedStatusId === "string" && item.quotedStatusId
+			? item.quotedStatusId
+			: typeof item.quotedTweet?.id === "string" && item.quotedTweet.id
+				? item.quotedTweet.id
+				: null;
+	if (quotedTweetId) {
+		references.push({ type: "quoted", id: quotedTweetId });
+	}
+
+	return references.length > 0 ? references : undefined;
+}
+
 function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse {
 	const users = new Map<string, XurlMentionUser>();
 	const data = items.map((item): XurlMentionData => {
@@ -196,6 +266,7 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse {
 			created_at: toIsoTimestamp(item.createdAt),
 			conversation_id: item.conversationId ?? item.id,
 			entities: toMediaEntities(item.media),
+			referenced_tweets: toReferencedTweets(item),
 			public_metrics: {
 				reply_count: Number(item.replyCount ?? 0),
 				retweet_count: Number(item.retweetCount ?? 0),
@@ -224,12 +295,12 @@ export async function listMentionsViaBird({
 }: {
 	maxResults: number;
 }): Promise<XurlMentionsResponse> {
-	const birdCommand = getBirdCommand();
-	const { stdout } = await execFileAsync(
-		birdCommand,
-		["mentions", "-n", String(maxResults), "--json"],
-		{ maxBuffer: BIRD_JSON_MAX_BUFFER_BYTES },
-	);
+	const stdout = await runBirdJsonCommand([
+		"mentions",
+		"-n",
+		String(maxResults),
+		"--json",
+	]);
 	const payload = parseBirdJson(stdout);
 
 	return normalizeBirdTweets(getBirdTweetItems(payload, "mentions"));
@@ -246,7 +317,6 @@ async function listTweetsViaBirdCommand({
 	all?: boolean;
 	maxPages?: number;
 }): Promise<XurlMentionsResponse> {
-	const birdCommand = getBirdCommand();
 	const args = [command, "-n", String(maxResults), "--json"];
 	if (all) {
 		args.push("--all");
@@ -254,9 +324,7 @@ async function listTweetsViaBirdCommand({
 	if (maxPages !== undefined) {
 		args.push("--max-pages", String(maxPages));
 	}
-	const { stdout } = await execFileAsync(birdCommand, args, {
-		maxBuffer: BIRD_JSON_MAX_BUFFER_BYTES,
-	});
+	const stdout = await runBirdJsonCommand(args);
 	const payload = parseBirdJson(stdout);
 
 	return normalizeBirdTweets(getBirdTweetItems(payload, command));
@@ -296,17 +364,75 @@ export async function listBookmarkedTweetsViaBird({
 	});
 }
 
+export async function lookupTweetsByIdsViaBird(
+	ids: string[],
+): Promise<XurlTweetsResponse> {
+	if (ids.length === 0) {
+		return { data: [] };
+	}
+
+	const tweets = await Promise.all(
+		ids.map(async (id) => {
+			const stdout = await runBirdJsonCommand(["read", id, "--json"]);
+			return getBirdTweetItem(parseBirdJson(stdout), "read");
+		}),
+	);
+
+	return normalizeBirdTweets(tweets);
+}
+
+export async function listHomeTimelineViaBird({
+	maxResults,
+	following = true,
+}: {
+	maxResults: number;
+	following?: boolean;
+}): Promise<XurlMentionsResponse> {
+	const args = ["home", "-n", String(maxResults), "--json"];
+	if (following) {
+		args.push("--following");
+	}
+	const stdout = await runBirdJsonCommand(args);
+	const payload = parseBirdJson(stdout);
+
+	return normalizeBirdTweets(getBirdTweetItems(payload, "home"));
+}
+
+export async function listThreadViaBird({
+	tweetId,
+	all,
+	maxPages,
+	timeoutMs,
+}: {
+	tweetId: string;
+	all?: boolean;
+	maxPages?: number;
+	timeoutMs?: number;
+}): Promise<XurlMentionsResponse> {
+	const args = ["thread", tweetId, "--json"];
+	if (all) {
+		args.push("--all");
+	}
+	if (maxPages !== undefined) {
+		args.push("--max-pages", String(maxPages));
+	}
+	const stdout = await runBirdJsonCommand(args, timeoutMs);
+	const payload = parseBirdJson(stdout);
+
+	return normalizeBirdTweets(getBirdTweetItems(payload, "thread"));
+}
+
 export async function listDirectMessagesViaBird({
 	maxResults,
 }: {
 	maxResults: number;
 }): Promise<BirdDmsResponse> {
-	const birdCommand = getBirdCommand();
-	const { stdout } = await execFileAsync(
-		birdCommand,
-		["dms", "-n", String(maxResults), "--json"],
-		{ maxBuffer: BIRD_JSON_MAX_BUFFER_BYTES },
-	);
+	const stdout = await runBirdJsonCommand([
+		"dms",
+		"-n",
+		String(maxResults),
+		"--json",
+	]);
 	const payload = parseBirdJson(stdout);
 	if (
 		!payload ||

package/src/lib/config.ts

@@ -1,4 +1,10 @@
-import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import {
+	accessSync,
+	constants,
+	existsSync,
+	mkdirSync,
+	readFileSync,
+} from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
@@ -128,6 +134,28 @@ export function resolveActionsTransport(
 	return "auto";
 }
 
+function findCommandOnPath(command: string) {
+	const pathValue = process.env.PATH;
+	if (!pathValue) {
+		return undefined;
+	}
+
+	for (const directory of pathValue.split(path.delimiter)) {
+		if (!directory) {
+			continue;
+		}
+		const candidate = path.join(directory, command);
+		try {
+			accessSync(candidate, constants.X_OK);
+			return candidate;
+		} catch {
+			continue;
+		}
+	}
+
+	return undefined;
+}
+
 export function getBirdCommand() {
 	const envCommand = process.env.BIRDCLAW_BIRD_COMMAND?.trim();
 	if (envCommand) {
@@ -139,6 +167,11 @@ export function getBirdCommand() {
 		return configuredCommand;
 	}
 
+	const pathCommand = findCommandOnPath("bird");
+	if (pathCommand) {
+		return pathCommand;
+	}
+
 	return path.join(os.homedir(), "Projects", "bird", "bird");
 }
 

package/src/lib/db.ts

@@ -19,6 +19,7 @@ export interface ProfilesTable {
 	display_name: string;
 	bio: string;
 	followers_count: number;
+	following_count: number;
 	avatar_hue: number;
 	avatar_url: string | null;
 	created_at: string;
@@ -154,6 +155,7 @@ const BASE_SCHEMA_SQL = `
     display_name text not null,
     bio text not null,
     followers_count integer not null default 0,
+    following_count integer not null default 0,
     avatar_hue integer not null default 0,
     avatar_url text,
     created_at text not null
@@ -271,6 +273,7 @@ const INDEX_SQL = `
   create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
   create index if not exists idx_profiles_followers on profiles(followers_count desc);
+  create index if not exists idx_profiles_following on profiles(following_count desc);
   create index if not exists idx_blocks_account_created on blocks(account_id, created_at desc);
   create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
   create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
@@ -306,6 +309,11 @@ function ensureTweetMetadataColumns(db: BetterSqlite3.Database) {
 
 function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
 	const columnNames = getColumnNames(db, "profiles");
+	if (!columnNames.has("following_count")) {
+		db.exec(
+			"alter table profiles add column following_count integer not null default 0",
+		);
+	}
 	if (!columnNames.has("avatar_url")) {
 		db.exec("alter table profiles add column avatar_url text");
 	}

package/src/lib/mention-threads-live.ts

@@ -0,0 +1,271 @@
+import type Database from "better-sqlite3";
+import { listThreadViaBird } from "./bird";
+import { getNativeDb } from "./db";
+import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
+
+const DEFAULT_LIMIT = 30;
+const DEFAULT_DELAY_MS = 1500;
+const DEFAULT_TIMEOUT_MS = 15_000;
+
+function assertPositiveInteger(value: number, name: string) {
+	if (!Number.isFinite(value) || value < 1) {
+		throw new Error(`${name} must be at least 1`);
+	}
+	return Math.floor(value);
+}
+
+function parseNonNegativeInteger(value: number | undefined, name: string) {
+	if (value === undefined) {
+		return undefined;
+	}
+	if (!Number.isFinite(value) || value < 0) {
+		throw new Error(`${name} must be non-negative`);
+	}
+	return Math.floor(value);
+}
+
+function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function getMediaCount(tweet: XurlMentionData) {
+	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
+	return urls.filter(
+		(url) =>
+			url &&
+			typeof url === "object" &&
+			typeof (url as Record<string, unknown>).media_key === "string",
+	).length;
+}
+
+function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
+	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
+		tweetId,
+		text,
+	);
+}
+
+function resolveAccount(db: Database.Database, accountId?: string) {
+	const row = accountId
+		? (db
+				.prepare("select id, handle from accounts where id = ?")
+				.get(accountId) as { id: string; handle: string } | undefined)
+		: (db
+				.prepare(
+					`
+          select id, handle
+          from accounts
+          order by is_default desc, created_at asc
+          limit 1
+          `,
+				)
+				.get() as { id: string; handle: string } | undefined);
+
+	if (!row) {
+		throw new Error(`Unknown account: ${accountId ?? "default"}`);
+	}
+
+	return {
+		accountId: row.id,
+		handle: row.handle.replace(/^@/, "").toLowerCase(),
+	};
+}
+
+function listRecentMentionIds(
+	db: Database.Database,
+	accountId: string,
+	limit: number,
+) {
+	return (
+		db
+			.prepare(
+				`
+        select id
+        from tweets
+        where kind = 'mention' and account_id = ?
+        order by created_at desc
+        limit ?
+        `,
+			)
+			.all(accountId, limit) as Array<{ id: string }>
+	).map((row) => row.id);
+}
+
+function getReplyToId(tweet: XurlMentionData) {
+	return tweet.referenced_tweets?.find((entry) => entry.type === "replied_to")
+		?.id;
+}
+
+function mergeMentionThreadIntoLocalStore({
+	db,
+	accountId,
+	accountHandle,
+	mentionIds,
+	payload,
+}: {
+	db: Database.Database;
+	accountId: string;
+	accountHandle: string;
+	mentionIds: Set<string>;
+	payload: XurlMentionsResponse;
+}) {
+	const usersById = new Map(
+		(payload.includes?.users ?? []).map((user) => [user.id, user]),
+	);
+	const upsertTweet = db.prepare(
+		`
+    insert into tweets (
+      id, account_id, author_profile_id, kind, text, created_at,
+      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
+      entities_json, media_json, quoted_tweet_id
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, '[]', null)
+    on conflict(id) do update set
+      account_id = excluded.account_id,
+      author_profile_id = excluded.author_profile_id,
+      kind = case
+        when tweets.kind in ('home', 'mention') then tweets.kind
+        when excluded.kind in ('home', 'mention') then excluded.kind
+        else coalesce(nullif(tweets.kind, ''), excluded.kind)
+      end,
+      text = excluded.text,
+      created_at = excluded.created_at,
+      is_replied = max(tweets.is_replied, excluded.is_replied),
+      reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
+      like_count = excluded.like_count,
+      media_count = excluded.media_count,
+      entities_json = excluded.entities_json,
+      media_json = excluded.media_json,
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked
+    `,
+	);
+
+	db.transaction(() => {
+		for (const tweet of payload.data) {
+			const author =
+				usersById.get(tweet.author_id) ??
+				({
+					id: tweet.author_id,
+					username: `user_${tweet.author_id}`,
+					name: `user_${tweet.author_id}`,
+				} as const);
+			const profile = usersById.has(tweet.author_id)
+				? upsertProfileFromXUser(db, author)
+				: ensureStubProfileForXUser(db, tweet.author_id);
+			const handle = author.username.toLowerCase();
+			const kind = mentionIds.has(tweet.id)
+				? "mention"
+				: handle === accountHandle
+					? "home"
+					: "thread";
+			const replyToId = getReplyToId(tweet);
+			upsertTweet.run(
+				tweet.id,
+				accountId,
+				profile.profile.id,
+				kind,
+				tweet.text,
+				tweet.created_at,
+				replyToId ? 1 : 0,
+				replyToId ?? null,
+				Number(tweet.public_metrics?.like_count ?? 0),
+				getMediaCount(tweet),
+				JSON.stringify(tweet.entities ?? {}),
+			);
+			replaceTweetFts(db, tweet.id, tweet.text);
+		}
+	})();
+}
+
+export async function syncMentionThreads({
+	account,
+	limit = DEFAULT_LIMIT,
+	delayMs = DEFAULT_DELAY_MS,
+	timeoutMs = DEFAULT_TIMEOUT_MS,
+	all = false,
+	maxPages,
+}: {
+	account?: string;
+	limit?: number;
+	delayMs?: number;
+	timeoutMs?: number;
+	all?: boolean;
+	maxPages?: number;
+}) {
+	const parsedLimit = assertPositiveInteger(limit, "--limit");
+	const parsedDelayMs = parseNonNegativeInteger(delayMs, "--delay-ms") ?? 0;
+	const parsedTimeoutMs = assertPositiveInteger(timeoutMs, "--timeout-ms");
+	const parsedMaxPages = parseNonNegativeInteger(maxPages, "--max-pages");
+	const db = getNativeDb();
+	const resolvedAccount = resolveAccount(db, account);
+	const mentionIds = listRecentMentionIds(
+		db,
+		resolvedAccount.accountId,
+		parsedLimit,
+	);
+	const mentionIdSet = new Set(mentionIds);
+	const results: Array<{
+		tweetId: string;
+		ok: boolean;
+		count: number;
+		error?: string;
+	}> = [];
+	let mergedTweets = 0;
+	const uniqueTweetIds = new Set<string>();
+
+	for (const [index, tweetId] of mentionIds.entries()) {
+		if (index > 0 && parsedDelayMs > 0) {
+			await sleep(parsedDelayMs);
+		}
+		try {
+			const payload = await listThreadViaBird({
+				tweetId,
+				all,
+				maxPages: parsedMaxPages,
+				timeoutMs: parsedTimeoutMs,
+			});
+			mergeMentionThreadIntoLocalStore({
+				db,
+				accountId: resolvedAccount.accountId,
+				accountHandle: resolvedAccount.handle,
+				mentionIds: mentionIdSet,
+				payload,
+			});
+			for (const tweet of payload.data) {
+				uniqueTweetIds.add(tweet.id);
+			}
+			mergedTweets += payload.data.length;
+			results.push({ tweetId, ok: true, count: payload.data.length });
+		} catch (error) {
+			results.push({
+				tweetId,
+				ok: false,
+				count: 0,
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+	}
+
+	const failures = results.filter((item) => !item.ok);
+	return {
+		ok: true,
+		accountId: resolvedAccount.accountId,
+		mentions: mentionIds.length,
+		threads: results.length,
+		succeeded: results.length - failures.length,
+		failed: failures.length,
+		mergedTweets,
+		uniqueTweets: uniqueTweetIds.size,
+		options: {
+			limit: parsedLimit,
+			delayMs: parsedDelayMs,
+			timeoutMs: parsedTimeoutMs,
+			all,
+			maxPages: parsedMaxPages ?? null,
+		},
+		results,
+		failures,
+	};
+}

package/src/lib/moderation-target.ts

@@ -15,12 +15,14 @@ export interface ResolvedModerationProfile {
 }
 
 export function toProfile(row: Record<string, unknown>): ProfileRecord {
+	const followingCount = Number(row.following_count ?? 0);
 	return {
 		id: String(row.id),
 		handle: String(row.handle),
 		displayName: String(row.display_name),
 		bio: String(row.bio),
 		followersCount: Number(row.followers_count),
+		...(Number.isFinite(followingCount) ? { followingCount } : {}),
 		avatarHue: Number(row.avatar_hue),
 		avatarUrl:
 			typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
@@ -67,7 +69,7 @@ export function resolveLocalProfile(
 	const row = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
       from profiles
       where id = ? or handle = ?
       limit 1

package/src/lib/profile-hydration.ts

@@ -51,6 +51,7 @@ export async function hydrateProfilesFromX() {
         display_name = ?,
         bio = ?,
         followers_count = ?,
+        following_count = coalesce(?, following_count),
         avatar_url = coalesce(?, avatar_url),
         created_at = coalesce(?, created_at)
     where id = ?
@@ -66,6 +67,7 @@ export async function hydrateProfilesFromX() {
         display_name = ?,
         bio = ?,
         followers_count = ?,
+        following_count = coalesce(?, following_count),
         avatar_url = coalesce(?, avatar_url),
         created_at = coalesce(?, created_at)
     where id = 'profile_me'
@@ -97,6 +99,9 @@ export async function hydrateProfilesFromX() {
 					displayName || username || profileId,
 					String(user.description ?? ""),
 					toInt(metrics?.followers_count),
+					metrics && "following_count" in metrics
+						? toInt(metrics.following_count)
+						: null,
 					normalizeAvatarUrl(user.profile_image_url),
 					typeof user.created_at === "string" ? user.created_at : null,
 					profileId,
@@ -120,6 +125,9 @@ export async function hydrateProfilesFromX() {
 				String(me.name ?? "Peter Steinberger"),
 				String(me.description ?? ""),
 				toInt(metrics?.followers_count),
+				metrics && "following_count" in metrics
+					? toInt(metrics.following_count)
+					: null,
 				normalizeAvatarUrl(me.profile_image_url),
 				typeof me.created_at === "string" ? me.created_at : null,
 			);