birdclaw 0.2.1 → 0.4.0

package/src/lib/x-profile.ts

@@ -1,5 +1,9 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { normalizeAvatarUrl } from "./avatar-cache";
+import { syncIdentitySearchIndexForProfileIds } from "./identity-search-index";
+import { syncProfileBioEntitiesForProfileId } from "./profile-bio-entities";
+import { syncProfileAffiliationsFromUser } from "./profile-affiliations";
+import { recordProfileSnapshot } from "./profile-history";
 import type { ProfileRecord, XurlMentionUser } from "./types";
 
 export interface ResolvedXProfile {
@@ -27,30 +31,117 @@ export function randomAvatarHue(input: string) {
 }
 
 function toProfile(row: Record<string, unknown>): ProfileRecord {
+	const followingCount = Number(row.following_count ?? 0);
+	const entities = parseJsonRecord(row.entities_json);
 	return {
 		id: String(row.id),
 		handle: String(row.handle),
 		displayName: String(row.display_name),
 		bio: String(row.bio),
 		followersCount: Number(row.followers_count),
+		...(Number.isFinite(followingCount) ? { followingCount } : {}),
 		avatarHue: Number(row.avatar_hue),
 		avatarUrl:
 			typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
+		...(typeof row.location === "string" && row.location.length > 0
+			? { location: row.location }
+			: {}),
+		...(typeof row.url === "string" && row.url.length > 0
+			? { url: row.url }
+			: {}),
+		...(typeof row.verified_type === "string" && row.verified_type.length > 0
+			? { verifiedType: row.verified_type }
+			: {}),
+		...(entities ? { entities } : {}),
 		createdAt: String(row.created_at),
 	};
 }
 
+function parseJsonRecord(value: unknown) {
+	if (typeof value !== "string" || value.length === 0) {
+		return undefined;
+	}
+	try {
+		const parsed = JSON.parse(value) as unknown;
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, unknown>)
+			: undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function getString(value: unknown) {
+	return typeof value === "string" && value.trim().length > 0
+		? value.trim()
+		: undefined;
+}
+
+function getExpandedUrlFromEntities(
+	entities: Record<string, unknown> | undefined,
+	key: "url" | "description",
+) {
+	const block = entities?.[key];
+	if (!block || typeof block !== "object") {
+		return undefined;
+	}
+	const urls = (block as { urls?: unknown }).urls;
+	if (!Array.isArray(urls)) {
+		return undefined;
+	}
+	for (const entry of urls) {
+		if (!entry || typeof entry !== "object") {
+			continue;
+		}
+		const record = entry as Record<string, unknown>;
+		const expanded =
+			getString(record.expandedUrl) ?? getString(record.expanded_url);
+		if (expanded) {
+			return expanded;
+		}
+	}
+	return undefined;
+}
+
+function normalizeVerifiedType(user: XurlMentionUser) {
+	const verifiedType = getString(user.verified_type);
+	if (verifiedType) {
+		return verifiedType.toLowerCase();
+	}
+	return user.verified ? "verified" : null;
+}
+
+function buildProfileMetadata(user: XurlMentionUser) {
+	const entities = user.entities;
+	const url =
+		getExpandedUrlFromEntities(entities, "url") ?? getString(user.url);
+	return {
+		location: getString(user.location) ?? null,
+		url: url ?? null,
+		verifiedType: normalizeVerifiedType(user),
+		entitiesJson: JSON.stringify(entities ?? {}),
+		rawJson: JSON.stringify(user),
+	};
+}
+
 function updateExistingProfileFromUser(
-	db: Database.Database,
+	db: Database,
 	profileId: string,
 	user: XurlMentionUser,
 ): ResolvedXProfile {
 	const username = String(user.username ?? "").replace(/^@/, "");
 	const displayName = String(user.name ?? "").trim() || username;
 	const followersCount = Number(user.public_metrics?.followers_count ?? 0);
+	const hasFollowingCount =
+		typeof user.public_metrics?.following_count === "number";
+	const followingCount = hasFollowingCount
+		? (user.public_metrics?.following_count ?? null)
+		: null;
 	const bio = String(user.description ?? "");
 	const avatarUrl = normalizeAvatarUrl(user.profile_image_url);
+	const metadata = buildProfileMetadata(user);
 
+	recordProfileSnapshot(db, profileId, "pre_update");
 	db.prepare(
 		`
     update profiles
@@ -58,15 +149,43 @@ function updateExistingProfileFromUser(
         display_name = ?,
         bio = ?,
         followers_count = ?,
-        avatar_url = coalesce(?, avatar_url)
+        following_count = coalesce(?, following_count),
+        avatar_url = coalesce(?, avatar_url),
+        location = coalesce(?, location),
+        url = coalesce(?, url),
+        verified_type = coalesce(?, verified_type),
+        entities_json = case
+          when ? not in ('', '{}', 'null') then ?
+          else profiles.entities_json
+        end,
+        raw_json = ?
     where id = ?
     `,
-	).run(username, displayName, bio, followersCount, avatarUrl, profileId);
+	).run(
+		username,
+		displayName,
+		bio,
+		followersCount,
+		followingCount,
+		avatarUrl,
+		metadata.location,
+		metadata.url,
+		metadata.verifiedType,
+		metadata.entitiesJson,
+		metadata.entitiesJson,
+		metadata.rawJson,
+		profileId,
+	);
+	syncProfileAffiliationsFromUser(db, profileId, user);
+	recordProfileSnapshot(db, profileId, "x_profile");
+	syncProfileBioEntitiesForProfileId(db, profileId);
+	syncIdentitySearchIndexForProfileIds(db, [profileId]);
 
 	const row = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url,
+        location, url, verified_type, entities_json, created_at
       from profiles
       where id = ?
       `,
@@ -80,7 +199,7 @@ function updateExistingProfileFromUser(
 }
 
 export function upsertProfileFromXUser(
-	db: Database.Database,
+	db: Database,
 	user: XurlMentionUser,
 ): ResolvedXProfile {
 	const externalUserId = String(user.id ?? "");
@@ -111,22 +230,41 @@ export function upsertProfileFromXUser(
 
 	const displayName = String(user.name ?? "").trim() || username;
 	const followersCount = Number(user.public_metrics?.followers_count ?? 0);
+	const hasFollowingCount =
+		typeof user.public_metrics?.following_count === "number";
+	const followingCount = hasFollowingCount
+		? (user.public_metrics?.following_count ?? null)
+		: null;
 	const bio = String(user.description ?? "");
 	const avatarUrl = normalizeAvatarUrl(user.profile_image_url);
+	const metadata = buildProfileMetadata(user);
 	const createdAt = new Date().toISOString();
 	const avatarHue = randomAvatarHue(username);
 
 	db.prepare(
 		`
     insert into profiles (
-      id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
-    ) values (?, ?, ?, ?, ?, ?, ?, ?)
+      id, handle, display_name, bio, followers_count, following_count, avatar_hue,
+      avatar_url, location, url, verified_type, entities_json, raw_json, created_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
-      handle = excluded.handle,
-      display_name = excluded.display_name,
-      bio = excluded.bio,
-      followers_count = excluded.followers_count,
-      avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url)
+	      handle = excluded.handle,
+	      display_name = excluded.display_name,
+	      bio = excluded.bio,
+	      followers_count = excluded.followers_count,
+	      following_count = case
+	        when ? then excluded.following_count
+	        else profiles.following_count
+	      end,
+	      avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url),
+	      location = coalesce(excluded.location, profiles.location),
+	      url = coalesce(excluded.url, profiles.url),
+	      verified_type = coalesce(excluded.verified_type, profiles.verified_type),
+	      entities_json = case
+	        when excluded.entities_json not in ('', '{}', 'null') then excluded.entities_json
+	        else profiles.entities_json
+	      end,
+	      raw_json = excluded.raw_json
     `,
 	).run(
 		profileId,
@@ -134,10 +272,21 @@ export function upsertProfileFromXUser(
 		displayName,
 		bio,
 		followersCount,
+		followingCount ?? 0,
 		avatarHue,
 		avatarUrl,
+		metadata.location,
+		metadata.url,
+		metadata.verifiedType,
+		metadata.entitiesJson,
+		metadata.rawJson,
 		createdAt,
+		hasFollowingCount ? 1 : 0,
 	);
+	syncProfileAffiliationsFromUser(db, profileId, user);
+	recordProfileSnapshot(db, profileId, "x_profile");
+	syncProfileBioEntitiesForProfileId(db, profileId);
+	syncIdentitySearchIndexForProfileIds(db, [profileId]);
 
 	return {
 		profile: {
@@ -146,8 +295,13 @@ export function upsertProfileFromXUser(
 			displayName,
 			bio,
 			followersCount,
+			followingCount: followingCount ?? 0,
 			avatarHue,
 			avatarUrl: avatarUrl ?? undefined,
+			...(metadata.location ? { location: metadata.location } : {}),
+			...(metadata.url ? { url: metadata.url } : {}),
+			...(metadata.verifiedType ? { verifiedType: metadata.verifiedType } : {}),
+			...(user.entities ? { entities: user.entities } : {}),
 			createdAt,
 		},
 		externalUserId,
@@ -155,14 +309,15 @@ export function upsertProfileFromXUser(
 }
 
 export function ensureStubProfileForXUser(
-	db: Database.Database,
+	db: Database,
 	externalUserId: string,
 ): ResolvedXProfile {
 	const profileId = buildExternalProfileId(externalUserId);
 	const existingRow = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url,
+        location, url, verified_type, entities_json, created_at
       from profiles
       where id = ?
       limit 1
@@ -183,8 +338,9 @@ export function ensureStubProfileForXUser(
 	db.prepare(
 		`
     insert into profiles (
-      id, handle, display_name, bio, followers_count, avatar_hue, avatar_url, created_at
-    ) values (?, ?, ?, '', 0, ?, null, ?)
+      id, handle, display_name, bio, followers_count, following_count, avatar_hue,
+      avatar_url, location, url, verified_type, entities_json, raw_json, created_at
+    ) values (?, ?, ?, '', 0, 0, ?, null, null, null, null, '{}', '{}', ?)
     `,
 	).run(profileId, handle, handle, avatarHue, createdAt);
 
@@ -195,6 +351,7 @@ export function ensureStubProfileForXUser(
 			displayName: handle,
 			bio: "",
 			followersCount: 0,
+			followingCount: 0,
 			avatarHue,
 			createdAt,
 		},

package/src/lib/xurl.ts

@@ -4,6 +4,7 @@ import type {
 	TransportStatus,
 	XurlMentionsResponse,
 	XurlMentionUser,
+	XurlTweetsResponse,
 	XurlUserTweet,
 } from "./types";
 
@@ -239,7 +240,7 @@ export async function lookupUsersByIds(ids: string[]) {
 	const query = new URLSearchParams({
 		ids: ids.join(","),
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	const payload = await runJsonCommand([`/2/users?${query.toString()}`]);
 	const data = payload.data;
@@ -254,7 +255,7 @@ export async function lookupUsersByHandles(handles: string[]) {
 	const query = new URLSearchParams({
 		usernames: handles.map((item) => item.replace(/^@/, "")).join(","),
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	const payload = await runJsonCommand([`/2/users/by?${query.toString()}`]);
 	const data = payload.data;
@@ -334,7 +335,7 @@ export async function listMentionsViaXurl({
 		expansions: "author_id",
 		"tweet.fields": "created_at,conversation_id,entities,public_metrics",
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	if (paginationToken) {
 		query.set("pagination_token", paginationToken);
@@ -391,9 +392,10 @@ async function listTimelineCollectionViaXurl({
 	const query = new URLSearchParams({
 		max_results: String(maxResults),
 		expansions: "author_id",
-		"tweet.fields": "created_at,conversation_id,entities,public_metrics",
+		"tweet.fields":
+			"created_at,conversation_id,entities,public_metrics,referenced_tweets",
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	if (paginationToken) {
 		query.set("pagination_token", paginationToken);
@@ -449,7 +451,8 @@ export async function listBlockedUsers(
 ) {
 	const query = new URLSearchParams({
 		max_results: "100",
-		"user.fields": "description,public_metrics,profile_image_url,created_at",
+		"user.fields":
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	if (paginationToken) {
 		query.set("pagination_token", paginationToken);
@@ -513,6 +516,38 @@ export async function listUserTweets(
 	};
 }
 
+export async function lookupTweetsByIds(
+	ids: string[],
+): Promise<XurlTweetsResponse> {
+	if (ids.length === 0) {
+		return { data: [] };
+	}
+
+	const query = new URLSearchParams({
+		ids: ids.join(","),
+		expansions: "author_id",
+		"tweet.fields":
+			"created_at,conversation_id,entities,public_metrics,referenced_tweets",
+		"user.fields":
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
+	});
+
+	const payload = await runJsonCommand([`/2/tweets?${query.toString()}`]);
+	return {
+		data: Array.isArray(payload.data)
+			? (payload.data as XurlTweetsResponse["data"])
+			: [],
+		includes:
+			payload.includes && typeof payload.includes === "object"
+				? (payload.includes as XurlTweetsResponse["includes"])
+				: undefined,
+		meta:
+			payload.meta && typeof payload.meta === "object"
+				? (payload.meta as XurlTweetsResponse["meta"])
+				: undefined,
+	};
+}
+
 export async function postViaXurl(text: string) {
 	return runShortcut(["post", text]);
 }