birdclaw 0.2.1 → 0.4.0

package/src/lib/sqlite.ts

@@ -0,0 +1,143 @@
+import { DatabaseSync, type StatementSync } from "node:sqlite";
+
+export type Database = NativeSqliteDatabase;
+
+type DatabaseOptions = {
+	readonly?: boolean;
+	fileMustExist?: boolean;
+};
+
+type PragmaOptions = {
+	simple?: boolean;
+};
+
+type RunResult = {
+	changes: number;
+	lastInsertRowid: number;
+};
+
+function bindArgs(parameters: unknown[]) {
+	if (parameters.length === 1 && Array.isArray(parameters[0])) {
+		return parameters[0];
+	}
+	return parameters;
+}
+
+function normalizeValue(value: unknown): unknown {
+	if (value instanceof Uint8Array && !Buffer.isBuffer(value)) {
+		return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+	}
+	return value;
+}
+
+function normalizeRow(row: unknown): unknown {
+	if (
+		!row ||
+		typeof row !== "object" ||
+		Array.isArray(row) ||
+		Buffer.isBuffer(row)
+	) {
+		return normalizeValue(row);
+	}
+	return Object.fromEntries(
+		Object.entries(row as Record<string, unknown>).map(([key, value]) => [
+			key,
+			normalizeValue(value),
+		]),
+	);
+}
+
+class NativeSqliteStatement {
+	readonly reader: boolean;
+
+	constructor(private readonly statement: StatementSync) {
+		this.reader = statement.columns().length > 0;
+	}
+
+	all(...parameters: unknown[]): unknown[] {
+		return this.statement.all(...bindArgs(parameters)).map(normalizeRow);
+	}
+
+	get(...parameters: unknown[]): unknown {
+		return normalizeRow(this.statement.get(...bindArgs(parameters)));
+	}
+
+	run(...parameters: unknown[]): RunResult {
+		const result = this.statement.run(...bindArgs(parameters));
+		return {
+			changes: Number(result.changes),
+			lastInsertRowid: Number(result.lastInsertRowid),
+		};
+	}
+
+	iterate(...parameters: unknown[]): IterableIterator<unknown> {
+		const rows = this.statement.iterate(...bindArgs(parameters));
+		return (function* () {
+			for (const row of rows) {
+				yield normalizeRow(row);
+			}
+		})();
+	}
+}
+
+export class NativeSqliteDatabase {
+	private transactionDepth = 0;
+	private readonly db: DatabaseSync;
+
+	constructor(path: string, options: DatabaseOptions = {}) {
+		this.db = new DatabaseSync(path, {
+			readOnly: options.readonly,
+		});
+	}
+
+	close(): void {
+		if (!this.db.isOpen) {
+			return;
+		}
+		this.db.close();
+	}
+
+	exec(sql: string): void {
+		this.db.exec(sql);
+	}
+
+	pragma(sql: string, options: PragmaOptions = {}): unknown {
+		const rows = this.prepare(`pragma ${sql}`).all() as Array<
+			Record<string, unknown>
+		>;
+		if (!options.simple) {
+			return rows;
+		}
+		const first = rows[0];
+		return first ? Object.values(first)[0] : undefined;
+	}
+
+	prepare(sql: string): NativeSqliteStatement {
+		return new NativeSqliteStatement(this.db.prepare(sql));
+	}
+
+	transaction<TArgs extends unknown[], TResult>(
+		fn: (...args: TArgs) => TResult,
+	): (...args: TArgs) => TResult {
+		return (...args: TArgs) => {
+			const nested = this.db.isTransaction;
+			const savepoint = `__birdclaw_tx_${++this.transactionDepth}`;
+			this.exec(nested ? `savepoint ${savepoint}` : "begin");
+			try {
+				const result = fn(...args);
+				this.exec(nested ? `release ${savepoint}` : "commit");
+				return result;
+			} catch (error) {
+				if (nested) {
+					this.exec(`rollback to ${savepoint}`);
+					this.exec(`release ${savepoint}`);
+				} else {
+					this.exec("rollback");
+				}
+				throw error;
+			}
+		};
+	}
+}
+
+export default NativeSqliteDatabase;

package/src/lib/timeline-collections-live.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { listBookmarkedTweetsViaBird, listLikedTweetsViaBird } from "./bird";
 import { getNativeDb } from "./db";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
@@ -50,7 +50,7 @@ function assertXurlLimit(limit: number) {
 	}
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db
 				.prepare(
@@ -97,7 +97,7 @@ function getMediaCount(tweet: XurlMentionData) {
 	).length;
 }
 
-function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
 		tweetId,
@@ -105,6 +105,12 @@ function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
 	);
 }
 
+function getReferencedTweetId(tweet: XurlMentionData, type: string) {
+	return (
+		tweet.referenced_tweets?.find((item) => item.type === type)?.id ?? null
+	);
+}
+
 function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 	const tweets: XurlMentionData[] = [];
 	const seenTweetIds = new Set<string>();
@@ -144,7 +150,7 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 }
 
 function mergeTimelineCollectionIntoLocalStore(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	kind: TimelineCollectionKind,
 	payload: XurlMentionsResponse,
@@ -162,9 +168,9 @@ function mergeTimelineCollectionIntoLocalStore(
       id, account_id, author_profile_id, kind, text, created_at,
       is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
       entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, ?, ?, ?, 0, null, ?, ?, ?, ?, ?, '[]', null)
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?)
     on conflict(id) do update set
-      account_id = excluded.account_id,
+      account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
       kind = case
         when tweets.kind in ('home', 'mention') then tweets.kind
@@ -176,8 +182,11 @@ function mergeTimelineCollectionIntoLocalStore(
       media_count = excluded.media_count,
       entities_json = excluded.entities_json,
       media_json = excluded.media_json,
-      bookmarked = max(tweets.bookmarked, excluded.bookmarked),
-      liked = max(tweets.liked, excluded.liked)
+      is_replied = max(tweets.is_replied, excluded.is_replied),
+      reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
+      quoted_tweet_id = coalesce(excluded.quoted_tweet_id, tweets.quoted_tweet_id),
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked
     `,
 	);
 	const upsertCollection = db.prepare(`
@@ -203,6 +212,8 @@ function mergeTimelineCollectionIntoLocalStore(
 			const profile = usersById.has(tweet.author_id)
 				? upsertProfileFromXUser(db, author)
 				: ensureStubProfileForXUser(db, tweet.author_id);
+			const replyToId = getReferencedTweetId(tweet, "replied_to");
+			const quotedTweetId = getReferencedTweetId(tweet, "quoted");
 			upsertTweet.run(
 				tweet.id,
 				accountId,
@@ -210,11 +221,14 @@ function mergeTimelineCollectionIntoLocalStore(
 				tweetKind,
 				tweet.text,
 				tweet.created_at,
+				replyToId ? 1 : 0,
+				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				getMediaCount(tweet),
 				bookmarked,
 				liked,
 				JSON.stringify(tweet.entities ?? {}),
+				quotedTweetId,
 			);
 			upsertCollection.run(
 				accountId,

package/src/lib/timeline-live.ts

@@ -0,0 +1,182 @@
+import type { Database } from "./sqlite";
+import { listHomeTimelineViaBird } from "./bird";
+import { getNativeDb } from "./db";
+import { readSyncCache, writeSyncCache } from "./sync-cache";
+import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import { upsertTweetAccountEdge } from "./tweet-account-edges";
+import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
+
+const DEFAULT_TIMELINE_CACHE_TTL_MS = 2 * 60_000;
+
+function parseCacheTtlMs(value?: number) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+		return DEFAULT_TIMELINE_CACHE_TTL_MS;
+	}
+	return Math.floor(value);
+}
+
+function assertLimit(limit: number) {
+	if (!Number.isFinite(limit) || limit < 1) {
+		throw new Error("--limit must be at least 1");
+	}
+}
+
+function resolveAccount(db: Database, accountId?: string) {
+	const row = accountId
+		? (db.prepare("select id from accounts where id = ?").get(accountId) as
+				| { id: string }
+				| undefined)
+		: (db
+				.prepare(
+					`
+          select id
+          from accounts
+          order by is_default desc, created_at asc
+          limit 1
+          `,
+				)
+				.get() as { id: string } | undefined);
+
+	if (!row) {
+		throw new Error(`Unknown account: ${accountId ?? "default"}`);
+	}
+
+	return row.id;
+}
+
+function getMediaCount(tweet: XurlMentionData) {
+	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
+	return urls.filter(
+		(url) =>
+			url &&
+			typeof url === "object" &&
+			typeof (url as Record<string, unknown>).media_key === "string",
+	).length;
+}
+
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
+	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
+	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
+		tweetId,
+		text,
+	);
+}
+
+function mergeHomeTimelineIntoLocalStore(
+	db: Database,
+	accountId: string,
+	payload: XurlMentionsResponse,
+) {
+	const usersById = new Map(
+		(payload.includes?.users ?? []).map((user) => [user.id, user]),
+	);
+	const upsertTweet = db.prepare(
+		`
+    insert into tweets (
+      id, account_id, author_profile_id, kind, text, created_at,
+      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
+      entities_json, media_json, quoted_tweet_id
+    ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
+    on conflict(id) do update set
+      account_id = tweets.account_id,
+      author_profile_id = excluded.author_profile_id,
+      kind = tweets.kind,
+      text = excluded.text,
+      created_at = excluded.created_at,
+      like_count = excluded.like_count,
+      media_count = excluded.media_count,
+      entities_json = excluded.entities_json,
+      media_json = excluded.media_json,
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked
+    `,
+	);
+
+	db.transaction(() => {
+		const seenAt = new Date().toISOString();
+		for (const tweet of payload.data) {
+			const author =
+				usersById.get(tweet.author_id) ??
+				({
+					id: tweet.author_id,
+					username: `user_${tweet.author_id}`,
+					name: `user_${tweet.author_id}`,
+				} as const);
+			const profile = usersById.has(tweet.author_id)
+				? upsertProfileFromXUser(db, author)
+				: ensureStubProfileForXUser(db, tweet.author_id);
+			upsertTweet.run(
+				tweet.id,
+				accountId,
+				profile.profile.id,
+				tweet.text,
+				tweet.created_at,
+				Number(tweet.public_metrics?.like_count ?? 0),
+				getMediaCount(tweet),
+				JSON.stringify(tweet.entities ?? {}),
+			);
+			upsertTweetAccountEdge(db, {
+				accountId,
+				tweetId: tweet.id,
+				kind: "home",
+				source: "bird",
+				seenAt,
+				rawJson: JSON.stringify(tweet),
+			});
+			replaceTweetFts(db, tweet.id, tweet.text);
+		}
+	})();
+}
+
+export async function syncHomeTimeline({
+	account,
+	limit = 100,
+	following = true,
+	refresh = false,
+	cacheTtlMs,
+}: {
+	account?: string;
+	limit?: number;
+	following?: boolean;
+	refresh?: boolean;
+	cacheTtlMs?: number;
+}) {
+	assertLimit(limit);
+	const db = getNativeDb();
+	const accountId = resolveAccount(db, account);
+	const cacheKey = `timeline:bird:${accountId}:${following ? "following" : "for-you"}:${String(limit)}`;
+	const ttlMs = parseCacheTtlMs(cacheTtlMs);
+	const cached = readSyncCache<XurlMentionsResponse>(cacheKey, db);
+	const cacheAgeMs = cached
+		? Date.now() - new Date(cached.updatedAt).getTime()
+		: Number.POSITIVE_INFINITY;
+
+	if (!refresh && cached && cacheAgeMs <= ttlMs) {
+		return {
+			ok: true,
+			source: "cache",
+			kind: "timeline",
+			accountId,
+			feed: following ? "following" : "for-you",
+			count: cached.value.data.length,
+			payload: cached.value,
+		};
+	}
+
+	const payload = await listHomeTimelineViaBird({
+		maxResults: limit,
+		following,
+	});
+	mergeHomeTimelineIntoLocalStore(db, accountId, payload);
+	writeSyncCache(cacheKey, payload, db);
+
+	return {
+		ok: true,
+		source: "bird",
+		kind: "timeline",
+		accountId,
+		feed: following ? "following" : "for-you",
+		count: payload.data.length,
+		payload,
+	};
+}

package/src/lib/tweet-account-edges.ts

@@ -0,0 +1,39 @@
+import type { Database } from "./sqlite";
+
+export type TweetAccountEdgeKind = "home" | "mention";
+
+export function upsertTweetAccountEdge(
+	db: Database,
+	{
+		accountId,
+		tweetId,
+		kind,
+		source,
+		seenAt,
+		rawJson = "{}",
+	}: {
+		accountId: string;
+		tweetId: string;
+		kind: TweetAccountEdgeKind;
+		source: string;
+		seenAt: string;
+		rawJson?: string;
+	},
+) {
+	db.prepare(`
+    insert into tweet_account_edges (
+      account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
+      source, raw_json, updated_at
+    ) values (?, ?, ?, ?, ?, 1, ?, ?, ?)
+    on conflict(account_id, tweet_id, kind) do update set
+      first_seen_at = min(tweet_account_edges.first_seen_at, excluded.first_seen_at),
+      last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
+      seen_count = tweet_account_edges.seen_count + 1,
+      source = coalesce(nullif(excluded.source, ''), tweet_account_edges.source),
+      raw_json = case
+        when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+        else tweet_account_edges.raw_json
+      end,
+      updated_at = max(tweet_account_edges.updated_at, excluded.updated_at)
+  `).run(accountId, tweetId, kind, seenAt, seenAt, source, rawJson, seenAt);
+}

package/src/lib/tweet-lookup.ts

@@ -0,0 +1,35 @@
+import { lookupTweetsByIdsViaBird } from "./bird";
+import type { XurlTweetsResponse } from "./types";
+import { lookupTweetsByIds as lookupTweetsByIdsViaXurl } from "./xurl";
+
+export type TweetLookupMode = "auto" | "xurl" | "bird";
+
+function errorMessage(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
+export async function lookupTweetsByIds(
+	ids: string[],
+	mode: TweetLookupMode = "auto",
+): Promise<XurlTweetsResponse> {
+	if (mode === "bird") {
+		return lookupTweetsByIdsViaBird(ids);
+	}
+	if (mode === "xurl") {
+		return lookupTweetsByIdsViaXurl(ids);
+	}
+
+	try {
+		return await lookupTweetsByIdsViaXurl(ids);
+	} catch (xurlError) {
+		try {
+			return await lookupTweetsByIdsViaBird(ids);
+		} catch (birdError) {
+			throw new Error(
+				`Tweet lookup failed via xurl and bird: xurl: ${errorMessage(
+					xurlError,
+				)}; bird: ${errorMessage(birdError)}`,
+			);
+		}
+	}
+}

package/src/lib/types.ts

@@ -20,11 +20,58 @@ export interface ProfileRecord {
 	displayName: string;
 	bio: string;
 	followersCount: number;
+	followingCount?: number;
 	avatarHue: number;
 	avatarUrl?: string;
+	location?: string;
+	url?: string;
+	verifiedType?: string;
+	entities?: Record<string, unknown>;
+	affiliations?: ProfileAffiliation[];
+	primaryAffiliation?: ProfileAffiliation;
 	createdAt: string;
 }
 
+export interface ProfileAffiliation {
+	organizationProfileId: string;
+	organizationName?: string;
+	organizationHandle?: string;
+	badgeUrl?: string | null;
+	url?: string | null;
+	label?: string | null;
+	source: string;
+	firstSeenAt: string;
+	lastSeenAt: string;
+	isActive: boolean;
+}
+
+export interface ProfileSnapshot {
+	profileId: string;
+	snapshotHash: string;
+	observedAt: string;
+	lastSeenAt: string;
+	source: string;
+	handle: string;
+	displayName: string;
+	bio: string;
+	location: string | null;
+	url: string | null;
+	verifiedType: string | null;
+	followersCount: number;
+	followingCount: number;
+	affiliations: unknown[];
+}
+
+export interface ProfileBioEntity {
+	profileId: string;
+	kind: "handle" | "domain" | "company_phrase";
+	value: string;
+	source: string;
+	firstSeenAt: string;
+	lastSeenAt: string;
+	isActive: boolean;
+}
+
 export interface TweetMentionEntity {
 	username: string;
 	id?: string;
@@ -98,6 +145,7 @@ export interface TimelineItem {
 	accountHandle: string;
 	kind: "home" | "mention" | "like" | "bookmark";
 	text: string;
+	searchSnippet?: string;
 	createdAt: string;
 	isReplied: boolean;
 	likeCount: number;
@@ -109,6 +157,7 @@ export interface TimelineItem {
 	media: TweetMediaItem[];
 	replyToTweet?: EmbeddedTweet | null;
 	quotedTweet?: EmbeddedTweet | null;
+	qualityReason?: string | null;
 }
 
 export interface DmMessageItem {
@@ -122,11 +171,31 @@ export interface DmMessageItem {
 	sender: ProfileRecord;
 }
 
+export interface UrlExpansionItem {
+	url: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	source: "cache" | "network";
+	title?: string;
+	description?: string | null;
+	error?: string;
+	updatedAt: string;
+}
+
+export interface DmSearchMatchItem {
+	message: DmMessageItem;
+	before: DmMessageItem[];
+	after: DmMessageItem[];
+	urlExpansions?: UrlExpansionItem[];
+}
+
 export interface DmConversationItem {
 	id: string;
 	accountId: string;
 	accountHandle: string;
 	title: string;
+	searchSnippet?: string;
 	lastMessageAt: string;
 	lastMessagePreview: string;
 	unreadCount: number;
@@ -134,6 +203,7 @@ export interface DmConversationItem {
 	influenceScore: number;
 	influenceLabel: string;
 	participant: ProfileRecord;
+	matches?: DmSearchMatchItem[];
 }
 
 export interface TimelineQuery {
@@ -145,6 +215,8 @@ export interface TimelineQuery {
 	until?: string;
 	includeReplies?: boolean;
 	qualityFilter?: TimelineQualityFilter;
+	lowQualityThreshold?: number;
+	includeQualityReason?: boolean;
 	likedOnly?: boolean;
 	bookmarkedOnly?: boolean;
 	limit?: number;
@@ -152,6 +224,7 @@ export interface TimelineQuery {
 
 export interface DmQuery {
 	account?: string;
+	conversationIds?: string[];
 	participant?: string;
 	search?: string;
 	replyFilter?: ReplyFilter;
@@ -160,6 +233,7 @@ export interface DmQuery {
 	minInfluenceScore?: number;
 	maxInfluenceScore?: number;
 	sort?: "recent" | "influence";
+	context?: number;
 	limit?: number;
 }
 
@@ -171,7 +245,7 @@ export interface TransportStatus {
 }
 
 export type ModerationAction = "block" | "unblock" | "mute" | "unmute";
-export type ModerationTransportKind = "bird" | "xurl";
+export type ModerationTransportKind = "bird" | "xurl" | "x-web";
 
 export interface ModerationActionTransportResult {
 	ok: boolean;
@@ -252,6 +326,7 @@ export interface XurlPublicMetrics {
 	bookmark_count?: number;
 	impression_count?: number;
 	followers_count?: number;
+	following_count?: number;
 }
 
 export interface XurlMentionUser {
@@ -259,7 +334,13 @@ export interface XurlMentionUser {
 	name: string;
 	username: string;
 	description?: string;
+	location?: string;
+	url?: string;
+	verified?: boolean;
+	verified_type?: string;
 	profile_image_url?: string;
+	entities?: Record<string, unknown>;
+	affiliation?: Record<string, unknown>;
 	public_metrics?: XurlPublicMetrics;
 	created_at?: string;
 }
@@ -271,6 +352,7 @@ export interface XurlMentionData {
 	created_at: string;
 	conversation_id?: string;
 	entities?: Record<string, unknown>;
+	referenced_tweets?: XurlReferencedTweet[];
 	public_metrics?: XurlPublicMetrics;
 	edit_history_tweet_ids?: string[];
 }
@@ -290,6 +372,18 @@ export interface XurlUserTweet {
 	edit_history_tweet_ids?: string[];
 }
 
+export interface XurlTweetData {
+	id: string;
+	author_id: string;
+	text: string;
+	created_at: string;
+	conversation_id?: string;
+	entities?: Record<string, unknown>;
+	referenced_tweets?: XurlReferencedTweet[];
+	public_metrics?: XurlPublicMetrics;
+	edit_history_tweet_ids?: string[];
+}
+
 export interface ProfileReplyItem {
 	id: string;
 	text: string;
@@ -322,3 +416,11 @@ export interface XurlMentionsResponse {
 	};
 	meta?: Record<string, unknown>;
 }
+
+export interface XurlTweetsResponse {
+	data: XurlTweetData[];
+	includes?: {
+		users?: XurlMentionUser[];
+	};
+	meta?: Record<string, unknown>;
+}