birdclaw 0.3.0 → 0.4.1

package/src/lib/x-profile.ts

@@ -1,5 +1,9 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { normalizeAvatarUrl } from "./avatar-cache";
+import { syncIdentitySearchIndexForProfileIds } from "./identity-search-index";
+import { syncProfileBioEntitiesForProfileId } from "./profile-bio-entities";
+import { syncProfileAffiliationsFromUser } from "./profile-affiliations";
+import { recordProfileSnapshot } from "./profile-history";
 import type { ProfileRecord, XurlMentionUser } from "./types";
 
 export interface ResolvedXProfile {
@@ -28,6 +32,7 @@ export function randomAvatarHue(input: string) {
 
 function toProfile(row: Record<string, unknown>): ProfileRecord {
 	const followingCount = Number(row.following_count ?? 0);
+	const entities = parseJsonRecord(row.entities_json);
 	return {
 		id: String(row.id),
 		handle: String(row.handle),
@@ -38,12 +43,89 @@ function toProfile(row: Record<string, unknown>): ProfileRecord {
 		avatarHue: Number(row.avatar_hue),
 		avatarUrl:
 			typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
+		...(typeof row.location === "string" && row.location.length > 0
+			? { location: row.location }
+			: {}),
+		...(typeof row.url === "string" && row.url.length > 0
+			? { url: row.url }
+			: {}),
+		...(typeof row.verified_type === "string" && row.verified_type.length > 0
+			? { verifiedType: row.verified_type }
+			: {}),
+		...(entities ? { entities } : {}),
 		createdAt: String(row.created_at),
 	};
 }
 
+function parseJsonRecord(value: unknown) {
+	if (typeof value !== "string" || value.length === 0) {
+		return undefined;
+	}
+	try {
+		const parsed = JSON.parse(value) as unknown;
+		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, unknown>)
+			: undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function getString(value: unknown) {
+	return typeof value === "string" && value.trim().length > 0
+		? value.trim()
+		: undefined;
+}
+
+function getExpandedUrlFromEntities(
+	entities: Record<string, unknown> | undefined,
+	key: "url" | "description",
+) {
+	const block = entities?.[key];
+	if (!block || typeof block !== "object") {
+		return undefined;
+	}
+	const urls = (block as { urls?: unknown }).urls;
+	if (!Array.isArray(urls)) {
+		return undefined;
+	}
+	for (const entry of urls) {
+		if (!entry || typeof entry !== "object") {
+			continue;
+		}
+		const record = entry as Record<string, unknown>;
+		const expanded =
+			getString(record.expandedUrl) ?? getString(record.expanded_url);
+		if (expanded) {
+			return expanded;
+		}
+	}
+	return undefined;
+}
+
+function normalizeVerifiedType(user: XurlMentionUser) {
+	const verifiedType = getString(user.verified_type);
+	if (verifiedType) {
+		return verifiedType.toLowerCase();
+	}
+	return user.verified ? "verified" : null;
+}
+
+function buildProfileMetadata(user: XurlMentionUser) {
+	const entities = user.entities;
+	const url =
+		getExpandedUrlFromEntities(entities, "url") ?? getString(user.url);
+	return {
+		location: getString(user.location) ?? null,
+		url: url ?? null,
+		verifiedType: normalizeVerifiedType(user),
+		entitiesJson: JSON.stringify(entities ?? {}),
+		rawJson: JSON.stringify(user),
+	};
+}
+
 function updateExistingProfileFromUser(
-	db: Database.Database,
+	db: Database,
 	profileId: string,
 	user: XurlMentionUser,
 ): ResolvedXProfile {
@@ -57,7 +139,9 @@ function updateExistingProfileFromUser(
 		: null;
 	const bio = String(user.description ?? "");
 	const avatarUrl = normalizeAvatarUrl(user.profile_image_url);
+	const metadata = buildProfileMetadata(user);
 
+	recordProfileSnapshot(db, profileId, "pre_update");
 	db.prepare(
 		`
     update profiles
@@ -66,7 +150,15 @@ function updateExistingProfileFromUser(
         bio = ?,
         followers_count = ?,
         following_count = coalesce(?, following_count),
-        avatar_url = coalesce(?, avatar_url)
+        avatar_url = coalesce(?, avatar_url),
+        location = coalesce(?, location),
+        url = coalesce(?, url),
+        verified_type = coalesce(?, verified_type),
+        entities_json = case
+          when ? not in ('', '{}', 'null') then ?
+          else profiles.entities_json
+        end,
+        raw_json = ?
     where id = ?
     `,
 	).run(
@@ -76,13 +168,24 @@ function updateExistingProfileFromUser(
 		followersCount,
 		followingCount,
 		avatarUrl,
+		metadata.location,
+		metadata.url,
+		metadata.verifiedType,
+		metadata.entitiesJson,
+		metadata.entitiesJson,
+		metadata.rawJson,
 		profileId,
 	);
+	syncProfileAffiliationsFromUser(db, profileId, user);
+	recordProfileSnapshot(db, profileId, "x_profile");
+	syncProfileBioEntitiesForProfileId(db, profileId);
+	syncIdentitySearchIndexForProfileIds(db, [profileId]);
 
 	const row = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url,
+        location, url, verified_type, entities_json, created_at
       from profiles
       where id = ?
       `,
@@ -96,7 +199,7 @@ function updateExistingProfileFromUser(
 }
 
 export function upsertProfileFromXUser(
-	db: Database.Database,
+	db: Database,
 	user: XurlMentionUser,
 ): ResolvedXProfile {
 	const externalUserId = String(user.id ?? "");
@@ -134,14 +237,16 @@ export function upsertProfileFromXUser(
 		: null;
 	const bio = String(user.description ?? "");
 	const avatarUrl = normalizeAvatarUrl(user.profile_image_url);
+	const metadata = buildProfileMetadata(user);
 	const createdAt = new Date().toISOString();
 	const avatarHue = randomAvatarHue(username);
 
 	db.prepare(
 		`
     insert into profiles (
-      id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      id, handle, display_name, bio, followers_count, following_count, avatar_hue,
+      avatar_url, location, url, verified_type, entities_json, raw_json, created_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
 	      handle = excluded.handle,
 	      display_name = excluded.display_name,
@@ -151,7 +256,15 @@ export function upsertProfileFromXUser(
 	        when ? then excluded.following_count
 	        else profiles.following_count
 	      end,
-	      avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url)
+	      avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url),
+	      location = coalesce(excluded.location, profiles.location),
+	      url = coalesce(excluded.url, profiles.url),
+	      verified_type = coalesce(excluded.verified_type, profiles.verified_type),
+	      entities_json = case
+	        when excluded.entities_json not in ('', '{}', 'null') then excluded.entities_json
+	        else profiles.entities_json
+	      end,
+	      raw_json = excluded.raw_json
     `,
 	).run(
 		profileId,
@@ -162,9 +275,18 @@ export function upsertProfileFromXUser(
 		followingCount ?? 0,
 		avatarHue,
 		avatarUrl,
+		metadata.location,
+		metadata.url,
+		metadata.verifiedType,
+		metadata.entitiesJson,
+		metadata.rawJson,
 		createdAt,
 		hasFollowingCount ? 1 : 0,
 	);
+	syncProfileAffiliationsFromUser(db, profileId, user);
+	recordProfileSnapshot(db, profileId, "x_profile");
+	syncProfileBioEntitiesForProfileId(db, profileId);
+	syncIdentitySearchIndexForProfileIds(db, [profileId]);
 
 	return {
 		profile: {
@@ -176,6 +298,10 @@ export function upsertProfileFromXUser(
 			followingCount: followingCount ?? 0,
 			avatarHue,
 			avatarUrl: avatarUrl ?? undefined,
+			...(metadata.location ? { location: metadata.location } : {}),
+			...(metadata.url ? { url: metadata.url } : {}),
+			...(metadata.verifiedType ? { verifiedType: metadata.verifiedType } : {}),
+			...(user.entities ? { entities: user.entities } : {}),
 			createdAt,
 		},
 		externalUserId,
@@ -183,14 +309,15 @@ export function upsertProfileFromXUser(
 }
 
 export function ensureStubProfileForXUser(
-	db: Database.Database,
+	db: Database,
 	externalUserId: string,
 ): ResolvedXProfile {
 	const profileId = buildExternalProfileId(externalUserId);
 	const existingRow = db
 		.prepare(
 			`
-      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
+      select id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url,
+        location, url, verified_type, entities_json, created_at
       from profiles
       where id = ?
       limit 1
@@ -211,8 +338,9 @@ export function ensureStubProfileForXUser(
 	db.prepare(
 		`
     insert into profiles (
-      id, handle, display_name, bio, followers_count, following_count, avatar_hue, avatar_url, created_at
-    ) values (?, ?, ?, '', 0, 0, ?, null, ?)
+      id, handle, display_name, bio, followers_count, following_count, avatar_hue,
+      avatar_url, location, url, verified_type, entities_json, raw_json, created_at
+    ) values (?, ?, ?, '', 0, 0, ?, null, null, null, null, '{}', '{}', ?)
     `,
 	).run(profileId, handle, handle, avatarHue, createdAt);
 

package/src/lib/xurl.ts

@@ -240,7 +240,7 @@ export async function lookupUsersByIds(ids: string[]) {
 	const query = new URLSearchParams({
 		ids: ids.join(","),
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	const payload = await runJsonCommand([`/2/users?${query.toString()}`]);
 	const data = payload.data;
@@ -255,7 +255,7 @@ export async function lookupUsersByHandles(handles: string[]) {
 	const query = new URLSearchParams({
 		usernames: handles.map((item) => item.replace(/^@/, "")).join(","),
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	const payload = await runJsonCommand([`/2/users/by?${query.toString()}`]);
 	const data = payload.data;
@@ -335,7 +335,7 @@ export async function listMentionsViaXurl({
 		expansions: "author_id",
 		"tweet.fields": "created_at,conversation_id,entities,public_metrics",
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	if (paginationToken) {
 		query.set("pagination_token", paginationToken);
@@ -395,7 +395,7 @@ async function listTimelineCollectionViaXurl({
 		"tweet.fields":
 			"created_at,conversation_id,entities,public_metrics,referenced_tweets",
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	if (paginationToken) {
 		query.set("pagination_token", paginationToken);
@@ -451,7 +451,8 @@ export async function listBlockedUsers(
 ) {
 	const query = new URLSearchParams({
 		max_results: "100",
-		"user.fields": "description,public_metrics,profile_image_url,created_at",
+		"user.fields":
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 	if (paginationToken) {
 		query.set("pagination_token", paginationToken);
@@ -528,7 +529,7 @@ export async function lookupTweetsByIds(
 		"tweet.fields":
 			"created_at,conversation_id,entities,public_metrics,referenced_tweets",
 		"user.fields":
-			"description,public_metrics,profile_image_url,created_at,verified",
+			"description,entities,location,public_metrics,profile_image_url,url,created_at,verified,verified_type",
 	});
 
 	const payload = await runJsonCommand([`/2/tweets?${query.toString()}`]);