birdclaw 0.2.1 → 0.4.0

package/src/lib/identity-search-index.ts

@@ -0,0 +1,369 @@
+import type { Database } from "./sqlite";
+import { fetchProfileAffiliations } from "./profile-affiliations";
+import { fetchProfileBioEntities } from "./profile-bio-entities";
+import { fetchProfileSnapshots } from "./profile-history";
+import type { ProfileRecord } from "./types";
+
+interface IdentityIndexEntry {
+	profileId: string;
+	kind: string;
+	value: string;
+	source: string;
+	weight: number;
+}
+
+const KIND_WEIGHTS: Record<string, number> = {
+	affiliation: 90,
+	bio_handle: 75,
+	bio_company: 65,
+	profile_history: 55,
+	profile_handle: 45,
+	profile_name: 35,
+	profile_bio: 35,
+	profile_location: 20,
+	profile_verified_type: 15,
+	profile_bio_url: 12,
+	profile_url: 10,
+	bio_domain: 8,
+};
+
+function normalizeIndexValue(value: string) {
+	return value.trim().toLowerCase();
+}
+
+function addEntry(
+	entries: Map<string, IdentityIndexEntry>,
+	entry: Omit<IdentityIndexEntry, "weight"> & { weight?: number },
+) {
+	const value = entry.value.trim();
+	if (!value) {
+		return;
+	}
+	const key = `${entry.profileId}:${entry.kind}:${entry.source}:${value.toLowerCase()}`;
+	if (!entries.has(key)) {
+		entries.set(key, {
+			...entry,
+			value,
+			weight: entry.weight ?? KIND_WEIGHTS[entry.kind] ?? 10,
+		});
+	}
+}
+
+function parseJsonObject(value: unknown) {
+	if (typeof value !== "string" || value.length === 0) {
+		return undefined;
+	}
+	try {
+		const parsed = JSON.parse(value) as unknown;
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, unknown>)
+			: undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function getUrlEntityExpandedUrl(entity: unknown) {
+	if (!entity || typeof entity !== "object") {
+		return undefined;
+	}
+	const record = entity as Record<string, unknown>;
+	const expanded = record.expandedUrl ?? record.expanded_url ?? record.url;
+	return typeof expanded === "string" && expanded.length > 0
+		? expanded
+		: undefined;
+}
+
+function getProfileBioUrls(profile: ProfileRecord) {
+	const description = profile.entities?.description;
+	if (!description || typeof description !== "object") {
+		return [];
+	}
+	const urls = (description as { urls?: unknown }).urls;
+	if (!Array.isArray(urls)) {
+		return [];
+	}
+	return urls
+		.map(getUrlEntityExpandedUrl)
+		.filter((url): url is string => Boolean(url));
+}
+
+function profileFromRow(row: Record<string, unknown>): ProfileRecord {
+	const entities = parseJsonObject(row.entities_json);
+	return {
+		id: String(row.id),
+		handle: String(row.handle),
+		displayName: String(row.display_name),
+		bio: String(row.bio ?? ""),
+		followersCount: Number(row.followers_count ?? 0),
+		followingCount: Number(row.following_count ?? 0),
+		avatarHue: Number(row.avatar_hue ?? 0),
+		...(typeof row.avatar_url === "string" && row.avatar_url.length > 0
+			? { avatarUrl: row.avatar_url }
+			: {}),
+		...(typeof row.location === "string" && row.location.length > 0
+			? { location: row.location }
+			: {}),
+		...(typeof row.url === "string" && row.url.length > 0
+			? { url: row.url }
+			: {}),
+		...(typeof row.verified_type === "string" && row.verified_type.length > 0
+			? { verifiedType: row.verified_type }
+			: {}),
+		...(entities ? { entities } : {}),
+		createdAt: String(row.created_at),
+	};
+}
+
+function collectProfileEntries(
+	profile: ProfileRecord,
+): Map<string, IdentityIndexEntry> {
+	const entries = new Map<string, IdentityIndexEntry>();
+	addEntry(entries, {
+		profileId: profile.id,
+		kind: "profile_handle",
+		value: profile.handle,
+		source: "profile",
+	});
+	addEntry(entries, {
+		profileId: profile.id,
+		kind: "profile_name",
+		value: profile.displayName,
+		source: "profile",
+	});
+	addEntry(entries, {
+		profileId: profile.id,
+		kind: "profile_bio",
+		value: profile.bio,
+		source: "profile",
+	});
+	if (profile.location) {
+		addEntry(entries, {
+			profileId: profile.id,
+			kind: "profile_location",
+			value: profile.location,
+			source: "profile",
+		});
+	}
+	if (profile.url) {
+		addEntry(entries, {
+			profileId: profile.id,
+			kind: "profile_url",
+			value: profile.url,
+			source: "profile",
+		});
+	}
+	if (profile.verifiedType) {
+		addEntry(entries, {
+			profileId: profile.id,
+			kind: "profile_verified_type",
+			value: profile.verifiedType,
+			source: "profile",
+		});
+	}
+	for (const url of getProfileBioUrls(profile)) {
+		addEntry(entries, {
+			profileId: profile.id,
+			kind: "profile_bio_url",
+			value: url,
+			source: "profile",
+		});
+	}
+	return entries;
+}
+
+function addAffiliationEntries(
+	entries: Map<string, IdentityIndexEntry>,
+	profileId: string,
+	values: Array<string | null | undefined>,
+	source = "affiliation",
+) {
+	for (const value of values) {
+		if (!value) {
+			continue;
+		}
+		addEntry(entries, {
+			profileId,
+			kind: "affiliation",
+			value,
+			source,
+		});
+	}
+}
+
+export function syncIdentitySearchIndexForProfileIds(
+	db: Database,
+	profileIds: string[],
+) {
+	const uniqueProfileIds = Array.from(new Set(profileIds)).filter(Boolean);
+	if (uniqueProfileIds.length === 0) {
+		return { profiles: 0, entries: 0 };
+	}
+
+	const placeholders = uniqueProfileIds.map(() => "?").join(",");
+	const rows = db
+		.prepare(
+			`
+      select id, handle, display_name, bio, followers_count, following_count,
+        avatar_hue, avatar_url, location, url, verified_type, entities_json, created_at
+      from profiles
+      where id in (${placeholders})
+      `,
+		)
+		.all(...uniqueProfileIds) as Array<Record<string, unknown>>;
+	const profiles = rows.map(profileFromRow);
+	const affiliationsByProfile = fetchProfileAffiliations(db, uniqueProfileIds);
+	const bioEntitiesByProfile = fetchProfileBioEntities(db, uniqueProfileIds);
+	const snapshotsByProfile = fetchProfileSnapshots(db, uniqueProfileIds);
+	const entries = new Map<string, IdentityIndexEntry>();
+
+	for (const profile of profiles) {
+		for (const [key, entry] of collectProfileEntries(profile)) {
+			entries.set(key, entry);
+		}
+		for (const affiliation of affiliationsByProfile.get(profile.id) ?? []) {
+			addAffiliationEntries(entries, profile.id, [
+				affiliation.organizationName,
+				affiliation.organizationHandle,
+				affiliation.label,
+				affiliation.url,
+				affiliation.organizationProfileId,
+			]);
+		}
+		for (const entity of bioEntitiesByProfile.get(profile.id) ?? []) {
+			addEntry(entries, {
+				profileId: profile.id,
+				kind:
+					entity.kind === "handle"
+						? "bio_handle"
+						: entity.kind === "domain"
+							? "bio_domain"
+							: "bio_company",
+				value: entity.value,
+				source: "bio_entity",
+			});
+		}
+		for (const snapshot of snapshotsByProfile.get(profile.id) ?? []) {
+			addEntry(entries, {
+				profileId: profile.id,
+				kind: "profile_history",
+				value: snapshot.handle,
+				source: "history",
+			});
+			addEntry(entries, {
+				profileId: profile.id,
+				kind: "profile_history",
+				value: snapshot.displayName,
+				source: "history",
+			});
+			addEntry(entries, {
+				profileId: profile.id,
+				kind: "profile_history",
+				value: snapshot.bio,
+				source: "history",
+			});
+			if (snapshot.location) {
+				addEntry(entries, {
+					profileId: profile.id,
+					kind: "profile_history",
+					value: snapshot.location,
+					source: "history",
+				});
+			}
+			if (snapshot.url) {
+				addEntry(entries, {
+					profileId: profile.id,
+					kind: "profile_history",
+					value: snapshot.url,
+					source: "history",
+				});
+			}
+			for (const affiliation of snapshot.affiliations) {
+				if (!affiliation || typeof affiliation !== "object") {
+					continue;
+				}
+				const record = affiliation as Record<string, unknown>;
+				addAffiliationEntries(
+					entries,
+					profile.id,
+					[
+						record.organizationName,
+						record.organizationHandle,
+						record.label,
+						record.url,
+					].filter((value): value is string => typeof value === "string"),
+					"history",
+				);
+			}
+		}
+	}
+
+	const now = new Date().toISOString();
+	const deleteStatement = db.prepare(
+		`delete from identity_search_index where profile_id = ?`,
+	);
+	const insertStatement = db.prepare(
+		`
+    insert into identity_search_index (
+      profile_id, kind, value, normalized_value, source, weight, updated_at
+    ) values (?, ?, ?, ?, ?, ?, ?)
+    on conflict(profile_id, kind, value, source) do update set
+      normalized_value = excluded.normalized_value,
+      weight = excluded.weight,
+      updated_at = excluded.updated_at
+    `,
+	);
+	db.transaction(() => {
+		for (const profileId of uniqueProfileIds) {
+			deleteStatement.run(profileId);
+		}
+		for (const entry of entries.values()) {
+			insertStatement.run(
+				entry.profileId,
+				entry.kind,
+				entry.value,
+				normalizeIndexValue(entry.value),
+				entry.source,
+				entry.weight,
+				now,
+			);
+		}
+	})();
+
+	return { profiles: profiles.length, entries: entries.size };
+}
+
+export function ensureIdentitySearchIndexForDmProfiles(
+	db: Database,
+	account?: string,
+) {
+	const params: string[] = [];
+	let accountClause = "";
+	if (account && account !== "all") {
+		accountClause = "where c.account_id = ?";
+		params.push(account);
+	}
+	const rows = db
+		.prepare(
+			`
+      select distinct c.participant_profile_id as profile_id
+      from dm_conversations c
+      left join identity_search_index isi
+        on isi.profile_id = c.participant_profile_id
+      ${accountClause}
+      group by c.participant_profile_id
+      having count(isi.profile_id) = 0
+      `,
+		)
+		.all(...params) as Array<{ profile_id: string }>;
+
+	return syncIdentitySearchIndexForProfileIds(
+		db,
+		rows.map((row) => row.profile_id),
+	);
+}
+
+export const __test__ = {
+	normalizeIndexValue,
+	getProfileBioUrls,
+};

package/src/lib/mention-threads-live.ts

@@ -0,0 +1,267 @@
+import type { Database } from "./sqlite";
+import { listThreadViaBird } from "./bird";
+import { getNativeDb } from "./db";
+import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
+
+const DEFAULT_LIMIT = 30;
+const DEFAULT_DELAY_MS = 1500;
+const DEFAULT_TIMEOUT_MS = 15_000;
+
+function assertPositiveInteger(value: number, name: string) {
+	if (!Number.isFinite(value) || value < 1) {
+		throw new Error(`${name} must be at least 1`);
+	}
+	return Math.floor(value);
+}
+
+function parseNonNegativeInteger(value: number | undefined, name: string) {
+	if (value === undefined) {
+		return undefined;
+	}
+	if (!Number.isFinite(value) || value < 0) {
+		throw new Error(`${name} must be non-negative`);
+	}
+	return Math.floor(value);
+}
+
+function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function getMediaCount(tweet: XurlMentionData) {
+	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
+	return urls.filter(
+		(url) =>
+			url &&
+			typeof url === "object" &&
+			typeof (url as Record<string, unknown>).media_key === "string",
+	).length;
+}
+
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
+	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
+	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
+		tweetId,
+		text,
+	);
+}
+
+function resolveAccount(db: Database, accountId?: string) {
+	const row = accountId
+		? (db
+				.prepare("select id, handle from accounts where id = ?")
+				.get(accountId) as { id: string; handle: string } | undefined)
+		: (db
+				.prepare(
+					`
+          select id, handle
+          from accounts
+          order by is_default desc, created_at asc
+          limit 1
+          `,
+				)
+				.get() as { id: string; handle: string } | undefined);
+
+	if (!row) {
+		throw new Error(`Unknown account: ${accountId ?? "default"}`);
+	}
+
+	return {
+		accountId: row.id,
+		handle: row.handle.replace(/^@/, "").toLowerCase(),
+	};
+}
+
+function listRecentMentionIds(db: Database, accountId: string, limit: number) {
+	return (
+		db
+			.prepare(
+				`
+        select id
+        from tweets
+        where kind = 'mention' and account_id = ?
+        order by created_at desc
+        limit ?
+        `,
+			)
+			.all(accountId, limit) as Array<{ id: string }>
+	).map((row) => row.id);
+}
+
+function getReplyToId(tweet: XurlMentionData) {
+	return tweet.referenced_tweets?.find((entry) => entry.type === "replied_to")
+		?.id;
+}
+
+function mergeMentionThreadIntoLocalStore({
+	db,
+	accountId,
+	accountHandle,
+	mentionIds,
+	payload,
+}: {
+	db: Database;
+	accountId: string;
+	accountHandle: string;
+	mentionIds: Set<string>;
+	payload: XurlMentionsResponse;
+}) {
+	const usersById = new Map(
+		(payload.includes?.users ?? []).map((user) => [user.id, user]),
+	);
+	const upsertTweet = db.prepare(
+		`
+    insert into tweets (
+      id, account_id, author_profile_id, kind, text, created_at,
+      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
+      entities_json, media_json, quoted_tweet_id
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, '[]', null)
+    on conflict(id) do update set
+      account_id = excluded.account_id,
+      author_profile_id = excluded.author_profile_id,
+      kind = case
+        when tweets.kind in ('home', 'mention') then tweets.kind
+        when excluded.kind in ('home', 'mention') then excluded.kind
+        else coalesce(nullif(tweets.kind, ''), excluded.kind)
+      end,
+      text = excluded.text,
+      created_at = excluded.created_at,
+      is_replied = max(tweets.is_replied, excluded.is_replied),
+      reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
+      like_count = excluded.like_count,
+      media_count = excluded.media_count,
+      entities_json = excluded.entities_json,
+      media_json = excluded.media_json,
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked
+    `,
+	);
+
+	db.transaction(() => {
+		for (const tweet of payload.data) {
+			const author =
+				usersById.get(tweet.author_id) ??
+				({
+					id: tweet.author_id,
+					username: `user_${tweet.author_id}`,
+					name: `user_${tweet.author_id}`,
+				} as const);
+			const profile = usersById.has(tweet.author_id)
+				? upsertProfileFromXUser(db, author)
+				: ensureStubProfileForXUser(db, tweet.author_id);
+			const handle = author.username.toLowerCase();
+			const kind = mentionIds.has(tweet.id)
+				? "mention"
+				: handle === accountHandle
+					? "home"
+					: "thread";
+			const replyToId = getReplyToId(tweet);
+			upsertTweet.run(
+				tweet.id,
+				accountId,
+				profile.profile.id,
+				kind,
+				tweet.text,
+				tweet.created_at,
+				replyToId ? 1 : 0,
+				replyToId ?? null,
+				Number(tweet.public_metrics?.like_count ?? 0),
+				getMediaCount(tweet),
+				JSON.stringify(tweet.entities ?? {}),
+			);
+			replaceTweetFts(db, tweet.id, tweet.text);
+		}
+	})();
+}
+
+export async function syncMentionThreads({
+	account,
+	limit = DEFAULT_LIMIT,
+	delayMs = DEFAULT_DELAY_MS,
+	timeoutMs = DEFAULT_TIMEOUT_MS,
+	all = false,
+	maxPages,
+}: {
+	account?: string;
+	limit?: number;
+	delayMs?: number;
+	timeoutMs?: number;
+	all?: boolean;
+	maxPages?: number;
+}) {
+	const parsedLimit = assertPositiveInteger(limit, "--limit");
+	const parsedDelayMs = parseNonNegativeInteger(delayMs, "--delay-ms") ?? 0;
+	const parsedTimeoutMs = assertPositiveInteger(timeoutMs, "--timeout-ms");
+	const parsedMaxPages = parseNonNegativeInteger(maxPages, "--max-pages");
+	const db = getNativeDb();
+	const resolvedAccount = resolveAccount(db, account);
+	const mentionIds = listRecentMentionIds(
+		db,
+		resolvedAccount.accountId,
+		parsedLimit,
+	);
+	const mentionIdSet = new Set(mentionIds);
+	const results: Array<{
+		tweetId: string;
+		ok: boolean;
+		count: number;
+		error?: string;
+	}> = [];
+	let mergedTweets = 0;
+	const uniqueTweetIds = new Set<string>();
+
+	for (const [index, tweetId] of mentionIds.entries()) {
+		if (index > 0 && parsedDelayMs > 0) {
+			await sleep(parsedDelayMs);
+		}
+		try {
+			const payload = await listThreadViaBird({
+				tweetId,
+				all,
+				maxPages: parsedMaxPages,
+				timeoutMs: parsedTimeoutMs,
+			});
+			mergeMentionThreadIntoLocalStore({
+				db,
+				accountId: resolvedAccount.accountId,
+				accountHandle: resolvedAccount.handle,
+				mentionIds: mentionIdSet,
+				payload,
+			});
+			for (const tweet of payload.data) {
+				uniqueTweetIds.add(tweet.id);
+			}
+			mergedTweets += payload.data.length;
+			results.push({ tweetId, ok: true, count: payload.data.length });
+		} catch (error) {
+			results.push({
+				tweetId,
+				ok: false,
+				count: 0,
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+	}
+
+	const failures = results.filter((item) => !item.ok);
+	return {
+		ok: true,
+		accountId: resolvedAccount.accountId,
+		mentions: mentionIds.length,
+		threads: results.length,
+		succeeded: results.length - failures.length,
+		failed: failures.length,
+		mergedTweets,
+		uniqueTweets: uniqueTweetIds.size,
+		options: {
+			limit: parsedLimit,
+			delayMs: parsedDelayMs,
+			timeoutMs: parsedTimeoutMs,
+			all,
+			maxPages: parsedMaxPages ?? null,
+		},
+		results,
+		failures,
+	};
+}

package/src/lib/mentions-live.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { listMentionsViaBird } from "./bird";
 import type { MentionsDataSource } from "./config";
 import { getNativeDb } from "./db";
@@ -12,6 +12,7 @@ import type {
 	XurlMentionsResponse,
 	XurlMentionUser,
 } from "./types";
+import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
 import { listMentionsViaXurl, lookupUsersByHandles } from "./xurl";
 
@@ -64,7 +65,7 @@ function parseMaxPages(value?: number) {
 	return Math.floor(value);
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db
 				.prepare("select id, handle from accounts where id = ?")
@@ -149,7 +150,7 @@ function getMediaCount(tweet: XurlMentionData) {
 	).length;
 }
 
-function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
 		tweetId,
@@ -158,9 +159,10 @@ function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
 }
 
 function mergeMentionsIntoLocalStore(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	payload: XurlMentionsResponse,
+	source: MentionsDataSource,
 ) {
 	const usersById = new Map(
 		(payload.includes?.users ?? []).map((user) => [user.id, user]),
@@ -173,9 +175,9 @@ function mergeMentionsIntoLocalStore(
       entities_json, media_json, quoted_tweet_id
     ) values (?, ?, ?, 'mention', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
     on conflict(id) do update set
-      account_id = excluded.account_id,
+      account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
-      kind = excluded.kind,
+      kind = case when tweets.kind = 'home' then tweets.kind else excluded.kind end,
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
@@ -189,6 +191,7 @@ function mergeMentionsIntoLocalStore(
 	);
 
 	const writePayload = db.transaction(() => {
+		const seenAt = new Date().toISOString();
 		for (const tweet of payload.data) {
 			const author =
 				usersById.get(tweet.author_id) ??
@@ -210,6 +213,14 @@ function mergeMentionsIntoLocalStore(
 				getMediaCount(tweet),
 				JSON.stringify(toLocalEntities(tweet)),
 			);
+			upsertTweetAccountEdge(db, {
+				accountId,
+				tweetId: tweet.id,
+				kind: "mention",
+				source,
+				seenAt,
+				rawJson: JSON.stringify(tweet),
+			});
 			replaceTweetFts(db, tweet.id, tweet.text);
 		}
 	});
@@ -408,7 +419,7 @@ async function exportMentionsViaCachedLiveSource({
 						all: fetchAll,
 						parsedMaxPages,
 					});
-		mergeMentionsIntoLocalStore(db, resolvedAccount.accountId, payload);
+		mergeMentionsIntoLocalStore(db, resolvedAccount.accountId, payload, mode);
 		writeSyncCache(cacheKey, payload, db);
 
 		if (