birdclaw 0.3.0 → 0.4.1

package/src/lib/mention-threads-live.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { listThreadViaBird } from "./bird";
 import { getNativeDb } from "./db";
 import type { XurlMentionData, XurlMentionsResponse } from "./types";
@@ -39,7 +39,7 @@ function getMediaCount(tweet: XurlMentionData) {
 	).length;
 }
 
-function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
 		tweetId,
@@ -47,7 +47,7 @@ function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
 	);
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db
 				.prepare("select id, handle from accounts where id = ?")
@@ -73,11 +73,7 @@ function resolveAccount(db: Database.Database, accountId?: string) {
 	};
 }
 
-function listRecentMentionIds(
-	db: Database.Database,
-	accountId: string,
-	limit: number,
-) {
+function listRecentMentionIds(db: Database, accountId: string, limit: number) {
 	return (
 		db
 			.prepare(
@@ -105,7 +101,7 @@ function mergeMentionThreadIntoLocalStore({
 	mentionIds,
 	payload,
 }: {
-	db: Database.Database;
+	db: Database;
 	accountId: string;
 	accountHandle: string;
 	mentionIds: Set<string>;

package/src/lib/mentions-live.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { listMentionsViaBird } from "./bird";
 import type { MentionsDataSource } from "./config";
 import { getNativeDb } from "./db";
@@ -12,6 +12,7 @@ import type {
 	XurlMentionsResponse,
 	XurlMentionUser,
 } from "./types";
+import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
 import { listMentionsViaXurl, lookupUsersByHandles } from "./xurl";
 
@@ -64,7 +65,7 @@ function parseMaxPages(value?: number) {
 	return Math.floor(value);
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db
 				.prepare("select id, handle from accounts where id = ?")
@@ -149,7 +150,7 @@ function getMediaCount(tweet: XurlMentionData) {
 	).length;
 }
 
-function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
 		tweetId,
@@ -158,9 +159,10 @@ function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
 }
 
 function mergeMentionsIntoLocalStore(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	payload: XurlMentionsResponse,
+	source: MentionsDataSource,
 ) {
 	const usersById = new Map(
 		(payload.includes?.users ?? []).map((user) => [user.id, user]),
@@ -173,9 +175,9 @@ function mergeMentionsIntoLocalStore(
       entities_json, media_json, quoted_tweet_id
     ) values (?, ?, ?, 'mention', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
     on conflict(id) do update set
-      account_id = excluded.account_id,
+      account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
-      kind = excluded.kind,
+      kind = case when tweets.kind = 'home' then tweets.kind else excluded.kind end,
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
@@ -189,6 +191,7 @@ function mergeMentionsIntoLocalStore(
 	);
 
 	const writePayload = db.transaction(() => {
+		const seenAt = new Date().toISOString();
 		for (const tweet of payload.data) {
 			const author =
 				usersById.get(tweet.author_id) ??
@@ -210,6 +213,14 @@ function mergeMentionsIntoLocalStore(
 				getMediaCount(tweet),
 				JSON.stringify(toLocalEntities(tweet)),
 			);
+			upsertTweetAccountEdge(db, {
+				accountId,
+				tweetId: tweet.id,
+				kind: "mention",
+				source,
+				seenAt,
+				rawJson: JSON.stringify(tweet),
+			});
 			replaceTweetFts(db, tweet.id, tweet.text);
 		}
 	});
@@ -408,7 +419,7 @@ async function exportMentionsViaCachedLiveSource({
 						all: fetchAll,
 						parsedMaxPages,
 					});
-		mergeMentionsIntoLocalStore(db, resolvedAccount.accountId, payload);
+		mergeMentionsIntoLocalStore(db, resolvedAccount.accountId, payload, mode);
 		writeSyncCache(cacheKey, payload, db);
 
 		if (

package/src/lib/moderation-target.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { lookupProfileViaBird } from "./bird-actions";
 import { getNativeDb } from "./db";
 import type { ProfileRecord, XurlMentionUser } from "./types";
@@ -41,7 +41,7 @@ export function normalizeProfileQuery(value: string) {
 	return (urlMatch?.[1] ?? withoutPrefix).replace(/^@/, "").trim();
 }
 
-export function getDefaultAccountId(db: Database.Database) {
+export function getDefaultAccountId(db: Database) {
 	const row = db
 		.prepare(
 			`
@@ -55,7 +55,7 @@ export function getDefaultAccountId(db: Database.Database) {
 	return row?.id ?? "acct_primary";
 }
 
-export function getAccountHandle(db: Database.Database, accountId: string) {
+export function getAccountHandle(db: Database, accountId: string) {
 	const row = db
 		.prepare("select handle from accounts where id = ?")
 		.get(accountId) as { handle: string } | undefined;
@@ -63,7 +63,7 @@ export function getAccountHandle(db: Database.Database, accountId: string) {
 }
 
 export function resolveLocalProfile(
-	db: Database.Database,
+	db: Database,
 	normalizedQuery: string,
 ): ResolvedModerationProfile | null {
 	const row = db

package/src/lib/moderation-write.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import type { ActionsTransport } from "./config";
 import { getNativeDb } from "./db";
 import {
@@ -43,7 +43,7 @@ export async function resolveModerationTarget({
 }
 
 export function writeModerationRow(
-	db: Database.Database,
+	db: Database,
 	table: "blocks" | "mutes",
 	accountId: string,
 	profileId: string,
@@ -61,7 +61,7 @@ export function writeModerationRow(
 }
 
 export function deleteModerationRow(
-	db: Database.Database,
+	db: Database,
 	table: "blocks" | "mutes",
 	accountId: string,
 	profileId: string,

package/src/lib/profile-affiliation-hydration.ts

@@ -0,0 +1,189 @@
+import type { Database } from "./sqlite";
+import { lookupProfileViaBird } from "./bird";
+import { syncIdentitySearchIndexForProfileIds } from "./identity-search-index";
+import { syncProfileBioEntitiesForProfileId } from "./profile-bio-entities";
+import { recordProfileSnapshot } from "./profile-history";
+import { upsertProfileFromXUser } from "./x-profile";
+
+export interface ProfileAffiliationHydrationResult {
+	checked: number;
+	hydrated: number;
+	skipped: number;
+	errors: Array<{ handle: string; error: string }>;
+}
+
+interface SyntheticAffiliationRow {
+	subject_profile_id: string;
+	organization_profile_id: string;
+	organization_name: string | null;
+	organization_handle: string | null;
+	badge_url: string | null;
+	url: string | null;
+	label: string | null;
+	source: string;
+	raw_json: string;
+}
+
+function normalizeHandle(value: string | null) {
+	const handle = value?.trim().replace(/^@/, "");
+	return handle && handle.length > 0 ? handle : null;
+}
+
+function replaceSyntheticAffiliation(
+	db: Database,
+	row: SyntheticAffiliationRow,
+	realProfileId: string,
+) {
+	const existing = db
+		.prepare(
+			`
+      select 1
+      from profile_affiliations
+      where subject_profile_id = ?
+        and organization_profile_id = ?
+      limit 1
+      `,
+		)
+		.get(row.subject_profile_id, realProfileId);
+
+	if (existing) {
+		db.prepare(
+			`
+      update profile_affiliations
+      set organization_name = coalesce(organization_name, ?),
+          organization_handle = coalesce(organization_handle, ?),
+          badge_url = coalesce(badge_url, ?),
+          url = coalesce(url, ?),
+          label = coalesce(label, ?),
+          is_active = 1,
+          last_seen_at = ?,
+          updated_at = ?
+      where subject_profile_id = ?
+        and organization_profile_id = ?
+      `,
+		).run(
+			row.organization_name,
+			row.organization_handle,
+			row.badge_url,
+			row.url,
+			row.label,
+			new Date().toISOString(),
+			new Date().toISOString(),
+			row.subject_profile_id,
+			realProfileId,
+		);
+		db.prepare(
+			`
+      delete from profile_affiliations
+      where subject_profile_id = ?
+        and organization_profile_id = ?
+      `,
+		).run(row.subject_profile_id, row.organization_profile_id);
+		return;
+	}
+
+	db.prepare(
+		`
+    update profile_affiliations
+    set organization_profile_id = ?,
+        updated_at = ?
+    where subject_profile_id = ?
+      and organization_profile_id = ?
+    `,
+	).run(
+		realProfileId,
+		new Date().toISOString(),
+		row.subject_profile_id,
+		row.organization_profile_id,
+	);
+}
+
+function findLocalOrganizationProfileId(db: Database, handle: string) {
+	const row = db
+		.prepare(
+			`
+      select id
+      from profiles
+      where lower(handle) = lower(?)
+      limit 1
+      `,
+		)
+		.get(handle) as { id: string } | undefined;
+	return row?.id ?? null;
+}
+
+export async function hydrateProfileAffiliationOrganizations(
+	db: Database,
+	subjectProfileId: string,
+): Promise<ProfileAffiliationHydrationResult> {
+	const rows = db
+		.prepare(
+			`
+      select subject_profile_id, organization_profile_id, organization_name,
+        organization_handle, badge_url, url, label, source, raw_json
+      from profile_affiliations
+      where subject_profile_id = ?
+        and is_active = 1
+        and organization_profile_id like 'profile_affiliation_%'
+        and organization_handle is not null
+      order by last_seen_at desc
+      `,
+		)
+		.all(subjectProfileId) as SyntheticAffiliationRow[];
+
+	const result: ProfileAffiliationHydrationResult = {
+		checked: rows.length,
+		hydrated: 0,
+		skipped: 0,
+		errors: [],
+	};
+
+	for (const row of rows) {
+		const handle = normalizeHandle(row.organization_handle);
+		if (!handle) {
+			result.skipped += 1;
+			continue;
+		}
+		try {
+			const localOrganizationProfileId = findLocalOrganizationProfileId(
+				db,
+				handle,
+			);
+			if (localOrganizationProfileId) {
+				db.transaction(() => {
+					replaceSyntheticAffiliation(db, row, localOrganizationProfileId);
+				})();
+				result.hydrated += 1;
+				continue;
+			}
+
+			const user = await lookupProfileViaBird(handle);
+			if (!user) {
+				result.skipped += 1;
+				continue;
+			}
+			const resolved = upsertProfileFromXUser(db, user);
+			if (resolved.profile.id === row.organization_profile_id) {
+				result.skipped += 1;
+				continue;
+			}
+			db.transaction(() => {
+				replaceSyntheticAffiliation(db, row, resolved.profile.id);
+			})();
+			result.hydrated += 1;
+		} catch (error) {
+			result.errors.push({
+				handle,
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+	}
+
+	if (result.hydrated > 0) {
+		recordProfileSnapshot(db, subjectProfileId, "affiliation_hydration");
+		syncProfileBioEntitiesForProfileId(db, subjectProfileId);
+		syncIdentitySearchIndexForProfileIds(db, [subjectProfileId]);
+	}
+
+	return result;
+}

package/src/lib/profile-affiliations.ts

@@ -0,0 +1,262 @@
+import { createHash } from "node:crypto";
+import type { Database } from "./sqlite";
+import type { ProfileAffiliation, XurlMentionUser } from "./types";
+
+interface NormalizedAffiliationInput {
+	organizationProfileId: string;
+	organizationName?: string;
+	organizationHandle?: string;
+	badgeUrl?: string | null;
+	url?: string | null;
+	label?: string | null;
+	source: string;
+	raw: Record<string, unknown>;
+}
+
+function getString(value: unknown) {
+	return typeof value === "string" && value.trim().length > 0
+		? value.trim()
+		: undefined;
+}
+
+function getStringArray(value: unknown) {
+	if (Array.isArray(value)) {
+		return value.filter(
+			(item): item is string => typeof item === "string" && item.length > 0,
+		);
+	}
+	const single = getString(value);
+	return single ? [single] : [];
+}
+
+function readFirst(record: Record<string, unknown>, keys: string[]) {
+	for (const key of keys) {
+		const value = getString(record[key]);
+		if (value) {
+			return value;
+		}
+	}
+	return undefined;
+}
+
+function inferHandleFromUrl(value: string | undefined) {
+	if (!value) {
+		return undefined;
+	}
+	try {
+		const url = new URL(value);
+		if (!/(^|\.)x\.com$|(^|\.)twitter\.com$/.test(url.hostname)) {
+			return undefined;
+		}
+		const [handle] = url.pathname.split("/").filter(Boolean);
+		return handle?.replace(/^@/, "");
+	} catch {
+		return undefined;
+	}
+}
+
+function syntheticOrganizationId({
+	label,
+	url,
+	badgeUrl,
+}: {
+	label?: string | null;
+	url?: string | null;
+	badgeUrl?: string | null;
+}) {
+	const hash = createHash("sha1")
+		.update([label, url, badgeUrl].filter(Boolean).join("|"))
+		.digest("hex")
+		.slice(0, 16);
+	return hash ? `profile_affiliation_${hash}` : undefined;
+}
+
+export function normalizeProfileAffiliationsFromUser(
+	user: XurlMentionUser,
+): NormalizedAffiliationInput[] {
+	const raw = user.affiliation;
+	if (!raw || typeof raw !== "object") {
+		return [];
+	}
+
+	const organizationIds = [
+		...getStringArray(raw.organizationIds),
+		...getStringArray(raw.organization_ids),
+		...getStringArray(raw.userId),
+		...getStringArray(raw.user_id),
+	];
+	const label =
+		readFirst(raw, ["label", "description", "organizationName"]) ?? null;
+	const url = readFirst(raw, ["url", "expandedUrl", "expanded_url"]) ?? null;
+	const badgeUrl = readFirst(raw, ["badgeUrl", "badge_url"]) ?? null;
+	if (
+		organizationIds.length === 0 &&
+		label === null &&
+		url === null &&
+		badgeUrl === null
+	) {
+		return [];
+	}
+	const organizationHandle =
+		readFirst(raw, ["organizationHandle", "organization_handle", "username"]) ??
+		inferHandleFromUrl(url ?? undefined);
+	const ids =
+		organizationIds.length > 0
+			? Array.from(new Set(organizationIds))
+			: [
+					syntheticOrganizationId({
+						label,
+						url,
+						badgeUrl,
+					}),
+				].filter((item): item is string => Boolean(item));
+
+	return ids.map((organizationProfileId) => ({
+		organizationProfileId,
+		...(label ? { organizationName: label } : {}),
+		...(organizationHandle ? { organizationHandle } : {}),
+		badgeUrl,
+		url,
+		label,
+		source: "x_profile",
+		raw,
+	}));
+}
+
+export function syncProfileAffiliationsFromUser(
+	db: Database,
+	subjectProfileId: string,
+	user: XurlMentionUser,
+) {
+	const affiliations = normalizeProfileAffiliationsFromUser(user);
+	if (affiliations.length === 0) {
+		return [];
+	}
+
+	const now = new Date().toISOString();
+	const statement = db.prepare(`
+    insert into profile_affiliations (
+      subject_profile_id, organization_profile_id, organization_name,
+      organization_handle, badge_url, url, label, source, is_active,
+      first_seen_at, last_seen_at, raw_json, updated_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)
+    on conflict(subject_profile_id, organization_profile_id) do update set
+      organization_name = coalesce(excluded.organization_name, profile_affiliations.organization_name),
+      organization_handle = coalesce(excluded.organization_handle, profile_affiliations.organization_handle),
+      badge_url = coalesce(excluded.badge_url, profile_affiliations.badge_url),
+      url = coalesce(excluded.url, profile_affiliations.url),
+      label = coalesce(excluded.label, profile_affiliations.label),
+      source = excluded.source,
+      is_active = 1,
+      last_seen_at = excluded.last_seen_at,
+      raw_json = excluded.raw_json,
+      updated_at = excluded.updated_at
+  `);
+
+	for (const affiliation of affiliations) {
+		statement.run(
+			subjectProfileId,
+			affiliation.organizationProfileId,
+			affiliation.organizationName ?? null,
+			affiliation.organizationHandle ?? null,
+			affiliation.badgeUrl ?? null,
+			affiliation.url ?? null,
+			affiliation.label ?? null,
+			affiliation.source,
+			now,
+			now,
+			JSON.stringify(affiliation.raw),
+			now,
+		);
+	}
+
+	return affiliations;
+}
+
+export function fetchProfileAffiliations(db: Database, profileIds: string[]) {
+	if (profileIds.length === 0) {
+		return new Map<string, ProfileAffiliation[]>();
+	}
+
+	const placeholders = profileIds.map(() => "?").join(",");
+	const rows = db
+		.prepare(
+			`
+      select
+        subject_profile_id,
+        organization_profile_id,
+        organization_name,
+        organization_handle,
+        badge_url,
+        url,
+        label,
+        source,
+        is_active,
+        first_seen_at,
+        last_seen_at
+      from profile_affiliations
+      where subject_profile_id in (${placeholders})
+        and is_active = 1
+      order by subject_profile_id, last_seen_at desc, organization_profile_id
+      `,
+		)
+		.all(...profileIds) as Array<Record<string, unknown>>;
+
+	const result = new Map<string, ProfileAffiliation[]>();
+	for (const row of rows) {
+		const subjectProfileId = String(row.subject_profile_id);
+		const affiliation: ProfileAffiliation = {
+			organizationProfileId: String(row.organization_profile_id),
+			...(typeof row.organization_name === "string" &&
+			row.organization_name.length > 0
+				? { organizationName: row.organization_name }
+				: {}),
+			...(typeof row.organization_handle === "string" &&
+			row.organization_handle.length > 0
+				? { organizationHandle: row.organization_handle }
+				: {}),
+			badgeUrl:
+				typeof row.badge_url === "string" ? String(row.badge_url) : null,
+			url: typeof row.url === "string" ? String(row.url) : null,
+			label: typeof row.label === "string" ? String(row.label) : null,
+			source: String(row.source),
+			firstSeenAt: String(row.first_seen_at),
+			lastSeenAt: String(row.last_seen_at),
+			isActive: Boolean(row.is_active),
+		};
+		const existing = result.get(subjectProfileId);
+		if (existing) {
+			existing.push(affiliation);
+		} else {
+			result.set(subjectProfileId, [affiliation]);
+		}
+	}
+
+	return result;
+}
+
+export function attachAffiliationsToProfiles<T extends { id: string }>(
+	db: Database,
+	profiles: T[],
+): Array<
+	T & {
+		affiliations?: ProfileAffiliation[];
+		primaryAffiliation?: ProfileAffiliation;
+	}
+> {
+	const affiliations = fetchProfileAffiliations(
+		db,
+		profiles.map((profile) => profile.id),
+	);
+	return profiles.map((profile) => {
+		const profileAffiliations = affiliations.get(profile.id) ?? [];
+		if (profileAffiliations.length === 0) {
+			return profile;
+		}
+		return {
+			...profile,
+			affiliations: profileAffiliations,
+			primaryAffiliation: profileAffiliations[0],
+		};
+	});
+}