birdclaw 0.8.2 → 0.8.4

package/src/lib/profile-bio-entities.ts

@@ -1,318 +0,0 @@
-import type { Database } from "./sqlite";
-import { fetchProfileAffiliations } from "./profile-affiliations";
-import type { ProfileBioEntity, ProfileRecord } from "./types";
-
-interface ExtractedBioEntity {
-	kind: ProfileBioEntity["kind"];
-	value: string;
-	source: string;
-	raw: Record<string, unknown>;
-}
-
-const HANDLE_RE = /(^|[^\w@./])@([A-Za-z0-9_]{1,15})\b/g;
-const DOMAIN_RE =
-	/\b(?:https?:\/\/)?(?:www\.)?([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+)\b/gi;
-const COMPANY_RE =
-	/\b(?:at|with|for|building|founder at|cofounder at|co-founder at|working on)\s+(@[A-Za-z0-9_]{1,15}|[A-Z][A-Za-z0-9&.+ -]{2,48})/g;
-
-function normalizeDomain(value: string) {
-	try {
-		const url = value.startsWith("http")
-			? new URL(value)
-			: new URL(`https://${value}`);
-		return url.hostname.replace(/^www\./, "").toLowerCase();
-	} catch {
-		return value.replace(/^www\./, "").toLowerCase();
-	}
-}
-
-function addEntity(
-	entities: Map<string, ExtractedBioEntity>,
-	entity: ExtractedBioEntity,
-) {
-	const key = `${entity.kind}:${entity.value.toLowerCase()}`;
-	if (!entities.has(key)) {
-		entities.set(key, entity);
-	}
-}
-
-function addHandleDerivedCompany(
-	entities: Map<string, ExtractedBioEntity>,
-	handle: string,
-	source: string,
-) {
-	const cleaned = handle.replace(/^@/, "").trim();
-	if (cleaned.length < 3) {
-		return;
-	}
-	addEntity(entities, {
-		kind: "company_phrase",
-		value: cleaned,
-		source,
-		raw: { handle },
-	});
-}
-
-function getUrlEntityExpandedUrls(profile: ProfileRecord) {
-	const urls: string[] = [];
-	for (const key of ["url", "description"] as const) {
-		const block = profile.entities?.[key];
-		if (!block || typeof block !== "object") {
-			continue;
-		}
-		const entries = (block as { urls?: unknown }).urls;
-		if (!Array.isArray(entries)) {
-			continue;
-		}
-		for (const entry of entries) {
-			if (!entry || typeof entry !== "object") {
-				continue;
-			}
-			const record = entry as Record<string, unknown>;
-			const expanded = record.expandedUrl ?? record.expanded_url ?? record.url;
-			if (typeof expanded === "string" && expanded.length > 0) {
-				urls.push(expanded);
-			}
-		}
-	}
-	return urls;
-}
-
-export function extractProfileBioEntities(profile: ProfileRecord) {
-	const entities = new Map<string, ExtractedBioEntity>();
-	const bio = profile.bio ?? "";
-
-	for (const match of bio.matchAll(HANDLE_RE)) {
-		const handle = `@${match[2]}`;
-		addEntity(entities, {
-			kind: "handle",
-			value: handle,
-			source: "bio",
-			raw: { match: match[0].trim() },
-		});
-		addHandleDerivedCompany(entities, handle, "bio_handle");
-	}
-
-	for (const match of bio.matchAll(DOMAIN_RE)) {
-		addEntity(entities, {
-			kind: "domain",
-			value: normalizeDomain(match[1] ?? match[0]),
-			source: "bio",
-			raw: { match: match[0] },
-		});
-	}
-
-	for (const match of bio.matchAll(COMPANY_RE)) {
-		const phrase = match[1]?.trim();
-		if (!phrase) {
-			continue;
-		}
-		addEntity(entities, {
-			kind: phrase.startsWith("@") ? "handle" : "company_phrase",
-			value: phrase,
-			source: "bio_phrase",
-			raw: { match: match[0] },
-		});
-		if (phrase.startsWith("@")) {
-			addHandleDerivedCompany(entities, phrase, "bio_phrase");
-		}
-	}
-
-	for (const url of [profile.url, ...getUrlEntityExpandedUrls(profile)]) {
-		if (!url) {
-			continue;
-		}
-		addEntity(entities, {
-			kind: "domain",
-			value: normalizeDomain(url),
-			source: "profile_url",
-			raw: { url },
-		});
-	}
-
-	for (const affiliation of profile.affiliations ?? []) {
-		if (affiliation.organizationName) {
-			addEntity(entities, {
-				kind: "company_phrase",
-				value: affiliation.organizationName,
-				source: "affiliation",
-				raw: { organizationProfileId: affiliation.organizationProfileId },
-			});
-		}
-		if (affiliation.organizationHandle) {
-			const handle = `@${affiliation.organizationHandle.replace(/^@/, "")}`;
-			addEntity(entities, {
-				kind: "handle",
-				value: handle,
-				source: "affiliation",
-				raw: { organizationProfileId: affiliation.organizationProfileId },
-			});
-			addHandleDerivedCompany(entities, handle, "affiliation");
-		}
-		if (affiliation.url) {
-			addEntity(entities, {
-				kind: "domain",
-				value: normalizeDomain(affiliation.url),
-				source: "affiliation",
-				raw: { organizationProfileId: affiliation.organizationProfileId },
-			});
-		}
-	}
-
-	return [...entities.values()];
-}
-
-function getProfileRecord(
-	db: Database,
-	profileId: string,
-): ProfileRecord | null {
-	const row = db
-		.prepare(
-			`
-      select id, handle, display_name, bio, followers_count, following_count,
-        avatar_hue, avatar_url, location, url, verified_type, entities_json, created_at
-      from profiles
-      where id = ?
-      `,
-		)
-		.get(profileId) as Record<string, unknown> | undefined;
-	if (!row) {
-		return null;
-	}
-	let parsedEntities: Record<string, unknown> | undefined;
-	if (typeof row.entities_json === "string" && row.entities_json.length > 0) {
-		try {
-			const parsed = JSON.parse(row.entities_json) as unknown;
-			if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
-				parsedEntities = parsed as Record<string, unknown>;
-			}
-		} catch {
-			parsedEntities = undefined;
-		}
-	}
-	const profile: ProfileRecord = {
-		id: String(row.id),
-		handle: String(row.handle),
-		displayName: String(row.display_name),
-		bio: String(row.bio),
-		followersCount: Number(row.followers_count ?? 0),
-		followingCount: Number(row.following_count ?? 0),
-		avatarHue: Number(row.avatar_hue ?? 0),
-		...(typeof row.avatar_url === "string"
-			? { avatarUrl: String(row.avatar_url) }
-			: {}),
-		...(typeof row.location === "string" && row.location.length > 0
-			? { location: row.location }
-			: {}),
-		...(typeof row.url === "string" && row.url.length > 0
-			? { url: row.url }
-			: {}),
-		...(typeof row.verified_type === "string" && row.verified_type.length > 0
-			? { verifiedType: row.verified_type }
-			: {}),
-		...(parsedEntities ? { entities: parsedEntities } : {}),
-		createdAt: String(row.created_at),
-	};
-	const affiliations = fetchProfileAffiliations(db, [profileId]).get(profileId);
-	return affiliations && affiliations.length > 0
-		? { ...profile, affiliations, primaryAffiliation: affiliations[0] }
-		: profile;
-}
-
-export function syncProfileBioEntitiesForProfileId(
-	db: Database,
-	profileId: string,
-) {
-	const profile = getProfileRecord(db, profileId);
-	if (!profile) {
-		return [];
-	}
-	const entities = extractProfileBioEntities(profile);
-	const now = new Date().toISOString();
-	const seen = new Set(
-		entities.map((entity) => `${entity.kind}:${entity.value}`),
-	);
-	const insert = db.prepare(
-		`
-    insert into profile_bio_entities (
-      profile_id, kind, value, source, is_active, first_seen_at, last_seen_at, raw_json
-    ) values (?, ?, ?, ?, 1, ?, ?, ?)
-    on conflict(profile_id, kind, value) do update set
-      source = excluded.source,
-      is_active = 1,
-      last_seen_at = excluded.last_seen_at,
-      raw_json = excluded.raw_json
-    `,
-	);
-
-	for (const entity of entities) {
-		insert.run(
-			profileId,
-			entity.kind,
-			entity.value,
-			entity.source,
-			now,
-			now,
-			JSON.stringify(entity.raw),
-		);
-	}
-
-	const existingRows = db
-		.prepare(
-			`
-      select kind, value
-      from profile_bio_entities
-      where profile_id = ?
-      `,
-		)
-		.all(profileId) as Array<{ kind: string; value: string }>;
-	const deactivate = db.prepare(
-		`
-    update profile_bio_entities
-    set is_active = 0, last_seen_at = ?
-    where profile_id = ? and kind = ? and value = ?
-    `,
-	);
-	for (const row of existingRows) {
-		if (!seen.has(`${row.kind}:${row.value}`)) {
-			deactivate.run(now, profileId, row.kind, row.value);
-		}
-	}
-
-	return entities;
-}
-
-export function fetchProfileBioEntities(db: Database, profileIds: string[]) {
-	if (profileIds.length === 0) {
-		return new Map<string, ProfileBioEntity[]>();
-	}
-	const placeholders = profileIds.map(() => "?").join(",");
-	const rows = db
-		.prepare(
-			`
-      select profile_id, kind, value, source, is_active, first_seen_at, last_seen_at
-      from profile_bio_entities
-      where profile_id in (${placeholders})
-        and is_active = 1
-      order by profile_id, kind, value
-      `,
-		)
-		.all(...profileIds) as Array<Record<string, unknown>>;
-	const result = new Map<string, ProfileBioEntity[]>();
-	for (const row of rows) {
-		const profileId = String(row.profile_id);
-		const entity: ProfileBioEntity = {
-			profileId,
-			kind: row.kind as ProfileBioEntity["kind"],
-			value: String(row.value),
-			source: String(row.source),
-			firstSeenAt: String(row.first_seen_at),
-			lastSeenAt: String(row.last_seen_at),
-			isActive: Boolean(row.is_active),
-		};
-		const existing = result.get(profileId) ?? [];
-		existing.push(entity);
-		result.set(profileId, existing);
-	}
-	return result;
-}

package/src/lib/profile-history.ts

@@ -1,164 +0,0 @@
-import { createHash } from "node:crypto";
-import type { Database } from "./sqlite";
-import { fetchProfileAffiliations } from "./profile-affiliations";
-import type { ProfileSnapshot } from "./types";
-
-function stableJson(value: unknown): string {
-	if (Array.isArray(value)) {
-		return `[${value.map(stableJson).join(",")}]`;
-	}
-	if (value && typeof value === "object") {
-		const record = value as Record<string, unknown>;
-		return `{${Object.keys(record)
-			.sort()
-			.map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
-			.join(",")}}`;
-	}
-	return JSON.stringify(value);
-}
-
-function snapshotHash(value: unknown) {
-	return createHash("sha1").update(stableJson(value)).digest("hex");
-}
-
-function parseJsonArray(value: unknown) {
-	if (typeof value !== "string" || value.length === 0) {
-		return [];
-	}
-	try {
-		const parsed = JSON.parse(value) as unknown;
-		return Array.isArray(parsed) ? parsed : [];
-	} catch {
-		return [];
-	}
-}
-
-function toSnapshot(row: Record<string, unknown>): ProfileSnapshot {
-	return {
-		profileId: String(row.profile_id),
-		snapshotHash: String(row.snapshot_hash),
-		observedAt: String(row.observed_at),
-		lastSeenAt: String(row.last_seen_at),
-		source: String(row.source),
-		handle: String(row.handle),
-		displayName: String(row.display_name),
-		bio: String(row.bio),
-		location:
-			typeof row.location === "string" && row.location.length > 0
-				? row.location
-				: null,
-		url: typeof row.url === "string" && row.url.length > 0 ? row.url : null,
-		verifiedType:
-			typeof row.verified_type === "string" && row.verified_type.length > 0
-				? row.verified_type
-				: null,
-		followersCount: Number(row.followers_count ?? 0),
-		followingCount: Number(row.following_count ?? 0),
-		affiliations: parseJsonArray(row.affiliations_json),
-	};
-}
-
-export function recordProfileSnapshot(
-	db: Database,
-	profileId: string,
-	source = "x_profile",
-) {
-	const profile = db
-		.prepare(
-			`
-      select id, handle, display_name, bio, location, url, verified_type,
-        followers_count, following_count, raw_json
-      from profiles
-      where id = ?
-      `,
-		)
-		.get(profileId) as Record<string, unknown> | undefined;
-	if (!profile) {
-		return null;
-	}
-
-	const affiliations =
-		fetchProfileAffiliations(db, [profileId]).get(profileId) ?? [];
-	const snapshot = {
-		handle: String(profile.handle),
-		displayName: String(profile.display_name),
-		bio: String(profile.bio),
-		location: profile.location ?? null,
-		url: profile.url ?? null,
-		verifiedType: profile.verified_type ?? null,
-		followersCount: Number(profile.followers_count ?? 0),
-		followingCount: Number(profile.following_count ?? 0),
-		affiliations: affiliations.map((affiliation) => ({
-			organizationProfileId: affiliation.organizationProfileId,
-			organizationName: affiliation.organizationName ?? null,
-			organizationHandle: affiliation.organizationHandle ?? null,
-			url: affiliation.url ?? null,
-			label: affiliation.label ?? null,
-		})),
-	};
-	const hash = snapshotHash(snapshot);
-	const now = new Date().toISOString();
-
-	db.prepare(
-		`
-    insert into profile_snapshots (
-      profile_id, snapshot_hash, observed_at, last_seen_at, source, handle,
-      display_name, bio, location, url, verified_type, followers_count,
-      following_count, affiliations_json, raw_json
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    on conflict(profile_id, snapshot_hash) do update set
-      last_seen_at = excluded.last_seen_at,
-      source = excluded.source,
-      raw_json = excluded.raw_json
-    `,
-	).run(
-		profileId,
-		hash,
-		now,
-		now,
-		source,
-		snapshot.handle,
-		snapshot.displayName,
-		snapshot.bio,
-		snapshot.location,
-		snapshot.url,
-		snapshot.verifiedType,
-		snapshot.followersCount,
-		snapshot.followingCount,
-		JSON.stringify(snapshot.affiliations),
-		typeof profile.raw_json === "string" ? profile.raw_json : "{}",
-	);
-
-	return hash;
-}
-
-export function fetchProfileSnapshots(
-	db: Database,
-	profileIds: string[],
-	limitPerProfile = 5,
-) {
-	if (profileIds.length === 0) {
-		return new Map<string, ProfileSnapshot[]>();
-	}
-	const placeholders = profileIds.map(() => "?").join(",");
-	const rows = db
-		.prepare(
-			`
-      select *
-      from profile_snapshots
-      where profile_id in (${placeholders})
-      order by profile_id, last_seen_at desc
-      `,
-		)
-		.all(...profileIds) as Array<Record<string, unknown>>;
-	const result = new Map<string, ProfileSnapshot[]>();
-	for (const row of rows) {
-		const snapshot = toSnapshot(row);
-		const existing = result.get(snapshot.profileId) ?? [];
-		if (existing.length < limitPerProfile) {
-			existing.push(snapshot);
-			result.set(snapshot.profileId, existing);
-		}
-	}
-	return result;
-}

package/src/lib/profile-hydration.ts

@@ -1,180 +0,0 @@
-import { Effect } from "effect";
-
-import { normalizeAvatarUrl } from "./avatar-cache";
-import { getNativeDb } from "./db";
-import { runEffectPromise, tryPromise } from "./effect-runtime";
-import {
-	getTransportStatus,
-	lookupAuthenticatedUser,
-	lookupUsersByIds,
-} from "./xurl";
-import { upsertProfileFromXUser } from "./x-profile";
-
-export type HydrateProfilesResult = {
-	ok: true;
-	hydratedProfiles: number;
-	hydratedAccount: boolean;
-	reason?: string;
-};
-
-function asRecord(value: unknown): Record<string, unknown> | null {
-	return value && typeof value === "object" && !Array.isArray(value)
-		? (value as Record<string, unknown>)
-		: null;
-}
-
-function toInt(value: unknown) {
-	const parsed = Number(value);
-	return Number.isFinite(parsed) ? Math.trunc(parsed) : 0;
-}
-
-function toError(error: unknown) {
-	return error instanceof Error ? error : new Error(String(error));
-}
-
-function trySync<T>(try_: () => T) {
-	return Effect.try({
-		try: try_,
-		catch: toError,
-	});
-}
-
-export function hydrateProfilesFromXEffect(): Effect.Effect<
-	HydrateProfilesResult,
-	unknown
-> {
-	return Effect.gen(function* () {
-		const transport = yield* tryPromise(() => getTransportStatus());
-		if (transport.availableTransport !== "xurl") {
-			return {
-				ok: true,
-				hydratedProfiles: 0,
-				hydratedAccount: false,
-				reason: transport.statusText,
-			};
-		}
-
-		const {
-			candidateIds,
-			db,
-			updateAccount,
-			updateConversationTitle,
-			updateLocalProfile,
-		} = yield* trySync(() => {
-			const db = getNativeDb();
-			const candidateRows = db
-				.prepare(
-					`
-      select id
-      from profiles
-      where id like 'profile_user_%'
-        and (followers_count = 0 or bio like 'Imported from archive user %' or handle like 'id%')
-      order by id asc
-      `,
-				)
-				.all() as Array<{ id: string }>;
-
-			const candidateIds = candidateRows
-				.map((row) => row.id.replace(/^profile_user_/, ""))
-				.filter((id) => /^\d+$/.test(id));
-
-			const updateConversationTitle = db.prepare(`
-    update dm_conversations
-    set title = ?
-    where participant_profile_id = ?
-  `);
-			const updateLocalProfile = db.prepare(`
-    update profiles
-    set handle = ?,
-        display_name = ?,
-        bio = ?,
-        followers_count = ?,
-        following_count = coalesce(?, following_count),
-        avatar_url = coalesce(?, avatar_url),
-        created_at = coalesce(?, created_at)
-    where id = 'profile_me'
-  `);
-			const updateAccount = db.prepare(`
-    update accounts
-    set name = ?,
-        handle = ?,
-        transport = 'xurl'
-    where id = 'acct_primary'
-  `);
-
-			return {
-				candidateIds,
-				db,
-				updateAccount,
-				updateConversationTitle,
-				updateLocalProfile,
-			};
-		});
-
-		let hydratedProfiles = 0;
-
-		for (let index = 0; index < candidateIds.length; index += 100) {
-			const batch = candidateIds.slice(index, index + 100);
-			const users = yield* tryPromise(() => lookupUsersByIds(batch));
-
-			yield* trySync(() => {
-				db.transaction(() => {
-					for (const user of users) {
-						const profileId = `profile_user_${String(user.id ?? "")}`;
-						if (profileId === "profile_user_") continue;
-
-						const resolved = upsertProfileFromXUser(db, user);
-						updateConversationTitle.run(
-							resolved.profile.displayName || resolved.profile.handle,
-							resolved.profile.id,
-						);
-						hydratedProfiles += 1;
-					}
-				})();
-			});
-		}
-
-		let hydratedAccount = false;
-		const me = yield* tryPromise(() => lookupAuthenticatedUser()).pipe(
-			Effect.catchAll(() => Effect.succeed(null)),
-		);
-		if (me) {
-			const metrics = asRecord(me.public_metrics);
-			yield* trySync(() => {
-				db.transaction(() => {
-					updateLocalProfile.run(
-						String(me.username ?? "steipete").replace(/^@/, ""),
-						String(me.name ?? "Peter Steinberger"),
-						String(me.description ?? ""),
-						toInt(metrics?.followers_count),
-						metrics && "following_count" in metrics
-							? toInt(metrics.following_count)
-							: null,
-						normalizeAvatarUrl(me.profile_image_url),
-						typeof me.created_at === "string" ? me.created_at : null,
-					);
-					updateAccount.run(
-						String(me.name ?? "Peter Steinberger"),
-						`@${String(me.username ?? "steipete").replace(/^@/, "")}`,
-					);
-				})();
-			});
-			hydratedAccount = true;
-		}
-
-		return {
-			ok: true,
-			hydratedProfiles,
-			hydratedAccount,
-		};
-	});
-}
-
-export function hydrateProfilesFromX(): Promise<HydrateProfilesResult> {
-	return runEffectPromise(hydrateProfilesFromXEffect());
-}
-
-export const __test__ = {
-	asRecord,
-	toInt,
-};