birdclaw 0.2.1 → 0.3.0

package/src/lib/timeline-live.ts

@@ -0,0 +1,175 @@
+import type Database from "better-sqlite3";
+import { listHomeTimelineViaBird } from "./bird";
+import { getNativeDb } from "./db";
+import { readSyncCache, writeSyncCache } from "./sync-cache";
+import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
+
+const DEFAULT_TIMELINE_CACHE_TTL_MS = 2 * 60_000;
+
+function parseCacheTtlMs(value?: number) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+		return DEFAULT_TIMELINE_CACHE_TTL_MS;
+	}
+	return Math.floor(value);
+}
+
+function assertLimit(limit: number) {
+	if (!Number.isFinite(limit) || limit < 1) {
+		throw new Error("--limit must be at least 1");
+	}
+}
+
+function resolveAccount(db: Database.Database, accountId?: string) {
+	const row = accountId
+		? (db.prepare("select id from accounts where id = ?").get(accountId) as
+				| { id: string }
+				| undefined)
+		: (db
+				.prepare(
+					`
+          select id
+          from accounts
+          order by is_default desc, created_at asc
+          limit 1
+          `,
+				)
+				.get() as { id: string } | undefined);
+
+	if (!row) {
+		throw new Error(`Unknown account: ${accountId ?? "default"}`);
+	}
+
+	return row.id;
+}
+
+function getMediaCount(tweet: XurlMentionData) {
+	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
+	return urls.filter(
+		(url) =>
+			url &&
+			typeof url === "object" &&
+			typeof (url as Record<string, unknown>).media_key === "string",
+	).length;
+}
+
+function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
+	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
+		tweetId,
+		text,
+	);
+}
+
+function mergeHomeTimelineIntoLocalStore(
+	db: Database.Database,
+	accountId: string,
+	payload: XurlMentionsResponse,
+) {
+	const usersById = new Map(
+		(payload.includes?.users ?? []).map((user) => [user.id, user]),
+	);
+	const upsertTweet = db.prepare(
+		`
+    insert into tweets (
+      id, account_id, author_profile_id, kind, text, created_at,
+      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
+      entities_json, media_json, quoted_tweet_id
+    ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
+    on conflict(id) do update set
+      account_id = excluded.account_id,
+      author_profile_id = excluded.author_profile_id,
+      kind = case
+        when tweets.kind = 'mention' then tweets.kind
+        else 'home'
+      end,
+      text = excluded.text,
+      created_at = excluded.created_at,
+      like_count = excluded.like_count,
+      media_count = excluded.media_count,
+      entities_json = excluded.entities_json,
+      media_json = excluded.media_json,
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked
+    `,
+	);
+
+	db.transaction(() => {
+		for (const tweet of payload.data) {
+			const author =
+				usersById.get(tweet.author_id) ??
+				({
+					id: tweet.author_id,
+					username: `user_${tweet.author_id}`,
+					name: `user_${tweet.author_id}`,
+				} as const);
+			const profile = usersById.has(tweet.author_id)
+				? upsertProfileFromXUser(db, author)
+				: ensureStubProfileForXUser(db, tweet.author_id);
+			upsertTweet.run(
+				tweet.id,
+				accountId,
+				profile.profile.id,
+				tweet.text,
+				tweet.created_at,
+				Number(tweet.public_metrics?.like_count ?? 0),
+				getMediaCount(tweet),
+				JSON.stringify(tweet.entities ?? {}),
+			);
+			replaceTweetFts(db, tweet.id, tweet.text);
+		}
+	})();
+}
+
+export async function syncHomeTimeline({
+	account,
+	limit = 100,
+	following = true,
+	refresh = false,
+	cacheTtlMs,
+}: {
+	account?: string;
+	limit?: number;
+	following?: boolean;
+	refresh?: boolean;
+	cacheTtlMs?: number;
+}) {
+	assertLimit(limit);
+	const db = getNativeDb();
+	const accountId = resolveAccount(db, account);
+	const cacheKey = `timeline:bird:${accountId}:${following ? "following" : "for-you"}:${String(limit)}`;
+	const ttlMs = parseCacheTtlMs(cacheTtlMs);
+	const cached = readSyncCache<XurlMentionsResponse>(cacheKey, db);
+	const cacheAgeMs = cached
+		? Date.now() - new Date(cached.updatedAt).getTime()
+		: Number.POSITIVE_INFINITY;
+
+	if (!refresh && cached && cacheAgeMs <= ttlMs) {
+		return {
+			ok: true,
+			source: "cache",
+			kind: "timeline",
+			accountId,
+			feed: following ? "following" : "for-you",
+			count: cached.value.data.length,
+			payload: cached.value,
+		};
+	}
+
+	const payload = await listHomeTimelineViaBird({
+		maxResults: limit,
+		following,
+	});
+	mergeHomeTimelineIntoLocalStore(db, accountId, payload);
+	writeSyncCache(cacheKey, payload, db);
+
+	return {
+		ok: true,
+		source: "bird",
+		kind: "timeline",
+		accountId,
+		feed: following ? "following" : "for-you",
+		count: payload.data.length,
+		payload,
+	};
+}

package/src/lib/tweet-lookup.ts

@@ -0,0 +1,35 @@
+import { lookupTweetsByIdsViaBird } from "./bird";
+import type { XurlTweetsResponse } from "./types";
+import { lookupTweetsByIds as lookupTweetsByIdsViaXurl } from "./xurl";
+
+export type TweetLookupMode = "auto" | "xurl" | "bird";
+
+function errorMessage(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
+export async function lookupTweetsByIds(
+	ids: string[],
+	mode: TweetLookupMode = "auto",
+): Promise<XurlTweetsResponse> {
+	if (mode === "bird") {
+		return lookupTweetsByIdsViaBird(ids);
+	}
+	if (mode === "xurl") {
+		return lookupTweetsByIdsViaXurl(ids);
+	}
+
+	try {
+		return await lookupTweetsByIdsViaXurl(ids);
+	} catch (xurlError) {
+		try {
+			return await lookupTweetsByIdsViaBird(ids);
+		} catch (birdError) {
+			throw new Error(
+				`Tweet lookup failed via xurl and bird: xurl: ${errorMessage(
+					xurlError,
+				)}; bird: ${errorMessage(birdError)}`,
+			);
+		}
+	}
+}

package/src/lib/types.ts

@@ -20,6 +20,7 @@ export interface ProfileRecord {
 	displayName: string;
 	bio: string;
 	followersCount: number;
+	followingCount?: number;
 	avatarHue: number;
 	avatarUrl?: string;
 	createdAt: string;
@@ -98,6 +99,7 @@ export interface TimelineItem {
 	accountHandle: string;
 	kind: "home" | "mention" | "like" | "bookmark";
 	text: string;
+	searchSnippet?: string;
 	createdAt: string;
 	isReplied: boolean;
 	likeCount: number;
@@ -109,6 +111,7 @@ export interface TimelineItem {
 	media: TweetMediaItem[];
 	replyToTweet?: EmbeddedTweet | null;
 	quotedTweet?: EmbeddedTweet | null;
+	qualityReason?: string | null;
 }
 
 export interface DmMessageItem {
@@ -127,6 +130,7 @@ export interface DmConversationItem {
 	accountId: string;
 	accountHandle: string;
 	title: string;
+	searchSnippet?: string;
 	lastMessageAt: string;
 	lastMessagePreview: string;
 	unreadCount: number;
@@ -145,6 +149,8 @@ export interface TimelineQuery {
 	until?: string;
 	includeReplies?: boolean;
 	qualityFilter?: TimelineQualityFilter;
+	lowQualityThreshold?: number;
+	includeQualityReason?: boolean;
 	likedOnly?: boolean;
 	bookmarkedOnly?: boolean;
 	limit?: number;
@@ -171,7 +177,7 @@ export interface TransportStatus {
 }
 
 export type ModerationAction = "block" | "unblock" | "mute" | "unmute";
-export type ModerationTransportKind = "bird" | "xurl";
+export type ModerationTransportKind = "bird" | "xurl" | "x-web";
 
 export interface ModerationActionTransportResult {
 	ok: boolean;
@@ -252,6 +258,7 @@ export interface XurlPublicMetrics {
 	bookmark_count?: number;
 	impression_count?: number;
 	followers_count?: number;
+	following_count?: number;
 }
 
 export interface XurlMentionUser {
@@ -271,6 +278,7 @@ export interface XurlMentionData {
 	created_at: string;
 	conversation_id?: string;
 	entities?: Record<string, unknown>;
+	referenced_tweets?: XurlReferencedTweet[];
 	public_metrics?: XurlPublicMetrics;
 	edit_history_tweet_ids?: string[];
 }
@@ -290,6 +298,18 @@ export interface XurlUserTweet {
 	edit_history_tweet_ids?: string[];
 }
 
+export interface XurlTweetData {
+	id: string;
+	author_id: string;
+	text: string;
+	created_at: string;
+	conversation_id?: string;
+	entities?: Record<string, unknown>;
+	referenced_tweets?: XurlReferencedTweet[];
+	public_metrics?: XurlPublicMetrics;
+	edit_history_tweet_ids?: string[];
+}
+
 export interface ProfileReplyItem {
 	id: string;
 	text: string;
@@ -322,3 +342,11 @@ export interface XurlMentionsResponse {
 	};
 	meta?: Record<string, unknown>;
 }
+
+export interface XurlTweetsResponse {
+	data: XurlTweetData[];
+	includes?: {
+		users?: XurlMentionUser[];
+	};
+	meta?: Record<string, unknown>;
+}

package/src/lib/x-profile.ts

@@ -27,12 +27,14 @@ export function randomAvatarHue(input: string) {
 }
 
 function toProfile(row: Record<string, unknown>): ProfileRecord {
+	const followingCount = Number(row.following_count ?? 0);
 	return {
 		id: String(row.id),
 		handle: String(row.handle),
 		displayName: String(row.display_name),
 		bio: String(row.bio),
 		followersCount: Number(row.followers_count),
+		...(Number.isFinite(followingCount) ? { followingCount } : {}),
 		avatarHue: Number(row.avatar_hue),
 		avatarUrl:
 			typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
@@ -48,6 +50,11 @@ function updateExistingProfileFromUser(
 	const username = String(user.username ?? "").replace(/^@/, "");
 	const displayName = String(user.name ?? "").trim() || username;
 	const followersCount = Number(user.public_metrics?.followers_count ?? 0);
+	const hasFollowingCount =
+		typeof user.public_metrics?.following_count === "number";
+	const followingCount = hasFollowingCount
+		? (user.public_metrics?.following_count ?? null)
+		: null;
 	const bio = String(user.description ?? "");
 	const avatarUrl = normalizeAvatarUrl(user.profile_image_url);
 
@@ -58,15 +65,24 @@ function updateExistingProfileFromUser(
         display_name = ?,
         bio = ?,
         followers_count = ?,
+        following_count = coalesce(?, following_count),
         avatar_url = coalesce(?, avatar_url)
     where id = ?
     `,
-	).run(username, displayName, bio, followersCount, avatarUrl, profileId);
+	).run(
+		username,
+		displayName,
+		bio,
+		followersCount,
+		followingCount,
+		avatarUrl,
+		profileId,
+	);
 
 	const row = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
       from profiles
       where id = ?
       `,
@@ -111,6 +127,11 @@ export function upsertProfileFromXUser(
 
 	const displayName = String(user.name ?? "").trim() || username;
 	const followersCount = Number(user.public_metrics?.followers_count ?? 0);
+	const hasFollowingCount =
+		typeof user.public_metrics?.following_count === "number";
+	const followingCount = hasFollowingCount
+		? (user.public_metrics?.following_count ?? null)
+		: null;
 	const bio = String(user.description ?? "");
 	const avatarUrl = normalizeAvatarUrl(user.profile_image_url);
 	const createdAt = new Date().toISOString();
@@ -119,14 +140,18 @@ export function upsertProfileFromXUser(
 	db.prepare(
 		`
     insert into profiles (
-      id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
-    ) values (?, ?, ?, ?, ?, ?, ?, ?)
+      id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
-      handle = excluded.handle,
-      display_name = excluded.display_name,
-      bio = excluded.bio,
-      followers_count = excluded.followers_count,
-      avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url)
+	      handle = excluded.handle,
+	      display_name = excluded.display_name,
+	      bio = excluded.bio,
+	      followers_count = excluded.followers_count,
+	      following_count = case
+	        when ? then excluded.following_count
+	        else profiles.following_count
+	      end,
+	      avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url)
     `,
 	).run(
 		profileId,
@@ -134,9 +159,11 @@ export function upsertProfileFromXUser(
 		displayName,
 		bio,
 		followersCount,
+		followingCount ?? 0,
 		avatarHue,
 		avatarUrl,
 		createdAt,
+		hasFollowingCount ? 1 : 0,
 	);
 
 	return {
@@ -146,6 +173,7 @@ export function upsertProfileFromXUser(
 			displayName,
 			bio,
 			followersCount,
+			followingCount: followingCount ?? 0,
 			avatarHue,
 			avatarUrl: avatarUrl ?? undefined,
 			createdAt,
@@ -162,7 +190,7 @@ export function ensureStubProfileForXUser(
 	const existingRow = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
       from profiles
       where id = ?
       limit 1
@@ -183,8 +211,8 @@ export function ensureStubProfileForXUser(
 	db.prepare(
 		`
     insert into profiles (
-      id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
-    ) values (?, ?, ?, '', 0, ?, null, ?)
+      id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
+    ) values (?, ?, ?, '', 0, 0, ?, null, ?)
     `,
 	).run(profileId, handle, handle, avatarHue, createdAt);
 
@@ -195,6 +223,7 @@ export function ensureStubProfileForXUser(
 			displayName: handle,
 			bio: "",
 			followersCount: 0,
+			followingCount: 0,
 			avatarHue,
 			createdAt,
 		},

package/src/lib/xurl.ts

@@ -4,6 +4,7 @@ import type {
 	TransportStatus,
 	XurlMentionsResponse,
 	XurlMentionUser,
+	XurlTweetsResponse,
 	XurlUserTweet,
 } from "./types";
 
@@ -391,7 +392,8 @@ async function listTimelineCollectionViaXurl({
 	const query = new URLSearchParams({
 		max_results: String(maxResults),
 		expansions: "author_id",
-		"tweet.fields": "created_at,conversation_id,entities,public_metrics",
+		"tweet.fields":
+			"created_at,conversation_id,entities,public_metrics,referenced_tweets",
 		"user.fields":
 			"description,public_metrics,profile_image_url,created_at,verified",
 	});
@@ -513,6 +515,38 @@ export async function listUserTweets(
 	};
 }
 
+export async function lookupTweetsByIds(
+	ids: string[],
+): Promise<XurlTweetsResponse> {
+	if (ids.length === 0) {
+		return { data: [] };
+	}
+
+	const query = new URLSearchParams({
+		ids: ids.join(","),
+		expansions: "author_id",
+		"tweet.fields":
+			"created_at,conversation_id,entities,public_metrics,referenced_tweets",
+		"user.fields":
+			"description,public_metrics,profile_image_url,created_at,verified",
+	});
+
+	const payload = await runJsonCommand([`/2/tweets?${query.toString()}`]);
+	return {
+		data: Array.isArray(payload.data)
+			? (payload.data as XurlTweetsResponse["data"])
+			: [],
+		includes:
+			payload.includes && typeof payload.includes === "object"
+				? (payload.includes as XurlTweetsResponse["includes"])
+				: undefined,
+		meta:
+			payload.meta && typeof payload.meta === "object"
+				? (payload.meta as XurlTweetsResponse["meta"])
+				: undefined,
+	};
+}
+
 export async function postViaXurl(text: string) {
 	return runShortcut(["post", text]);
 }