birdclaw 0.8.2 → 0.8.4

package/src/lib/identity-search-index.ts

@@ -1,369 +0,0 @@
-import type { Database } from "./sqlite";
-import { fetchProfileAffiliations } from "./profile-affiliations";
-import { fetchProfileBioEntities } from "./profile-bio-entities";
-import { fetchProfileSnapshots } from "./profile-history";
-import type { ProfileRecord } from "./types";
-
-interface IdentityIndexEntry {
-	profileId: string;
-	kind: string;
-	value: string;
-	source: string;
-	weight: number;
-}
-
-const KIND_WEIGHTS: Record<string, number> = {
-	affiliation: 90,
-	bio_handle: 75,
-	bio_company: 65,
-	profile_history: 55,
-	profile_handle: 45,
-	profile_name: 35,
-	profile_bio: 35,
-	profile_location: 20,
-	profile_verified_type: 15,
-	profile_bio_url: 12,
-	profile_url: 10,
-	bio_domain: 8,
-};
-
-function normalizeIndexValue(value: string) {
-	return value.trim().toLowerCase();
-}
-
-function addEntry(
-	entries: Map<string, IdentityIndexEntry>,
-	entry: Omit<IdentityIndexEntry, "weight"> & { weight?: number },
-) {
-	const value = entry.value.trim();
-	if (!value) {
-		return;
-	}
-	const key = `${entry.profileId}:${entry.kind}:${entry.source}:${value.toLowerCase()}`;
-	if (!entries.has(key)) {
-		entries.set(key, {
-			...entry,
-			value,
-			weight: entry.weight ?? KIND_WEIGHTS[entry.kind] ?? 10,
-		});
-	}
-}
-
-function parseJsonObject(value: unknown) {
-	if (typeof value !== "string" || value.length === 0) {
-		return undefined;
-	}
-	try {
-		const parsed = JSON.parse(value) as unknown;
-		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
-			? (parsed as Record<string, unknown>)
-			: undefined;
-	} catch {
-		return undefined;
-	}
-}
-
-function getUrlEntityExpandedUrl(entity: unknown) {
-	if (!entity || typeof entity !== "object") {
-		return undefined;
-	}
-	const record = entity as Record<string, unknown>;
-	const expanded = record.expandedUrl ?? record.expanded_url ?? record.url;
-	return typeof expanded === "string" && expanded.length > 0
-		? expanded
-		: undefined;
-}
-
-function getProfileBioUrls(profile: ProfileRecord) {
-	const description = profile.entities?.description;
-	if (!description || typeof description !== "object") {
-		return [];
-	}
-	const urls = (description as { urls?: unknown }).urls;
-	if (!Array.isArray(urls)) {
-		return [];
-	}
-	return urls
-		.map(getUrlEntityExpandedUrl)
-		.filter((url): url is string => Boolean(url));
-}
-
-function profileFromRow(row: Record<string, unknown>): ProfileRecord {
-	const entities = parseJsonObject(row.entities_json);
-	return {
-		id: String(row.id),
-		handle: String(row.handle),
-		displayName: String(row.display_name),
-		bio: String(row.bio ?? ""),
-		followersCount: Number(row.followers_count ?? 0),
-		followingCount: Number(row.following_count ?? 0),
-		avatarHue: Number(row.avatar_hue ?? 0),
-		...(typeof row.avatar_url === "string" && row.avatar_url.length > 0
-			? { avatarUrl: row.avatar_url }
-			: {}),
-		...(typeof row.location === "string" && row.location.length > 0
-			? { location: row.location }
-			: {}),
-		...(typeof row.url === "string" && row.url.length > 0
-			? { url: row.url }
-			: {}),
-		...(typeof row.verified_type === "string" && row.verified_type.length > 0
-			? { verifiedType: row.verified_type }
-			: {}),
-		...(entities ? { entities } : {}),
-		createdAt: String(row.created_at),
-	};
-}
-
-function collectProfileEntries(
-	profile: ProfileRecord,
-): Map<string, IdentityIndexEntry> {
-	const entries = new Map<string, IdentityIndexEntry>();
-	addEntry(entries, {
-		profileId: profile.id,
-		kind: "profile_handle",
-		value: profile.handle,
-		source: "profile",
-	});
-	addEntry(entries, {
-		profileId: profile.id,
-		kind: "profile_name",
-		value: profile.displayName,
-		source: "profile",
-	});
-	addEntry(entries, {
-		profileId: profile.id,
-		kind: "profile_bio",
-		value: profile.bio,
-		source: "profile",
-	});
-	if (profile.location) {
-		addEntry(entries, {
-			profileId: profile.id,
-			kind: "profile_location",
-			value: profile.location,
-			source: "profile",
-		});
-	}
-	if (profile.url) {
-		addEntry(entries, {
-			profileId: profile.id,
-			kind: "profile_url",
-			value: profile.url,
-			source: "profile",
-		});
-	}
-	if (profile.verifiedType) {
-		addEntry(entries, {
-			profileId: profile.id,
-			kind: "profile_verified_type",
-			value: profile.verifiedType,
-			source: "profile",
-		});
-	}
-	for (const url of getProfileBioUrls(profile)) {
-		addEntry(entries, {
-			profileId: profile.id,
-			kind: "profile_bio_url",
-			value: url,
-			source: "profile",
-		});
-	}
-	return entries;
-}
-
-function addAffiliationEntries(
-	entries: Map<string, IdentityIndexEntry>,
-	profileId: string,
-	values: Array<string | null | undefined>,
-	source = "affiliation",
-) {
-	for (const value of values) {
-		if (!value) {
-			continue;
-		}
-		addEntry(entries, {
-			profileId,
-			kind: "affiliation",
-			value,
-			source,
-		});
-	}
-}
-
-export function syncIdentitySearchIndexForProfileIds(
-	db: Database,
-	profileIds: string[],
-) {
-	const uniqueProfileIds = Array.from(new Set(profileIds)).filter(Boolean);
-	if (uniqueProfileIds.length === 0) {
-		return { profiles: 0, entries: 0 };
-	}
-
-	const placeholders = uniqueProfileIds.map(() => "?").join(",");
-	const rows = db
-		.prepare(
-			`
-      select id, handle, display_name, bio, followers_count, following_count,
-        avatar_hue, avatar_url, location, url, verified_type, entities_json, created_at
-      from profiles
-      where id in (${placeholders})
-      `,
-		)
-		.all(...uniqueProfileIds) as Array<Record<string, unknown>>;
-	const profiles = rows.map(profileFromRow);
-	const affiliationsByProfile = fetchProfileAffiliations(db, uniqueProfileIds);
-	const bioEntitiesByProfile = fetchProfileBioEntities(db, uniqueProfileIds);
-	const snapshotsByProfile = fetchProfileSnapshots(db, uniqueProfileIds);
-	const entries = new Map<string, IdentityIndexEntry>();
-
-	for (const profile of profiles) {
-		for (const [key, entry] of collectProfileEntries(profile)) {
-			entries.set(key, entry);
-		}
-		for (const affiliation of affiliationsByProfile.get(profile.id) ?? []) {
-			addAffiliationEntries(entries, profile.id, [
-				affiliation.organizationName,
-				affiliation.organizationHandle,
-				affiliation.label,
-				affiliation.url,
-				affiliation.organizationProfileId,
-			]);
-		}
-		for (const entity of bioEntitiesByProfile.get(profile.id) ?? []) {
-			addEntry(entries, {
-				profileId: profile.id,
-				kind:
-					entity.kind === "handle"
-						? "bio_handle"
-						: entity.kind === "domain"
-							? "bio_domain"
-							: "bio_company",
-				value: entity.value,
-				source: "bio_entity",
-			});
-		}
-		for (const snapshot of snapshotsByProfile.get(profile.id) ?? []) {
-			addEntry(entries, {
-				profileId: profile.id,
-				kind: "profile_history",
-				value: snapshot.handle,
-				source: "history",
-			});
-			addEntry(entries, {
-				profileId: profile.id,
-				kind: "profile_history",
-				value: snapshot.displayName,
-				source: "history",
-			});
-			addEntry(entries, {
-				profileId: profile.id,
-				kind: "profile_history",
-				value: snapshot.bio,
-				source: "history",
-			});
-			if (snapshot.location) {
-				addEntry(entries, {
-					profileId: profile.id,
-					kind: "profile_history",
-					value: snapshot.location,
-					source: "history",
-				});
-			}
-			if (snapshot.url) {
-				addEntry(entries, {
-					profileId: profile.id,
-					kind: "profile_history",
-					value: snapshot.url,
-					source: "history",
-				});
-			}
-			for (const affiliation of snapshot.affiliations) {
-				if (!affiliation || typeof affiliation !== "object") {
-					continue;
-				}
-				const record = affiliation as Record<string, unknown>;
-				addAffiliationEntries(
-					entries,
-					profile.id,
-					[
-						record.organizationName,
-						record.organizationHandle,
-						record.label,
-						record.url,
-					].filter((value): value is string => typeof value === "string"),
-					"history",
-				);
-			}
-		}
-	}
-
-	const now = new Date().toISOString();
-	const deleteStatement = db.prepare(
-		`delete from identity_search_index where profile_id = ?`,
-	);
-	const insertStatement = db.prepare(
-		`
-    insert into identity_search_index (
-      profile_id, kind, value, normalized_value, source, weight, updated_at
-    ) values (?, ?, ?, ?, ?, ?, ?)
-    on conflict(profile_id, kind, value, source) do update set
-      normalized_value = excluded.normalized_value,
-      weight = excluded.weight,
-      updated_at = excluded.updated_at
-    `,
-	);
-	db.transaction(() => {
-		for (const profileId of uniqueProfileIds) {
-			deleteStatement.run(profileId);
-		}
-		for (const entry of entries.values()) {
-			insertStatement.run(
-				entry.profileId,
-				entry.kind,
-				entry.value,
-				normalizeIndexValue(entry.value),
-				entry.source,
-				entry.weight,
-				now,
-			);
-		}
-	})();
-
-	return { profiles: profiles.length, entries: entries.size };
-}
-
-export function ensureIdentitySearchIndexForDmProfiles(
-	db: Database,
-	account?: string,
-) {
-	const params: string[] = [];
-	let accountClause = "";
-	if (account && account !== "all") {
-		accountClause = "where c.account_id = ?";
-		params.push(account);
-	}
-	const rows = db
-		.prepare(
-			`
-      select distinct c.participant_profile_id as profile_id
-      from dm_conversations c
-      left join identity_search_index isi
-        on isi.profile_id = c.participant_profile_id
-      ${accountClause}
-      group by c.participant_profile_id
-      having count(isi.profile_id) = 0
-      `,
-		)
-		.all(...params) as Array<{ profile_id: string }>;
-
-	return syncIdentitySearchIndexForProfileIds(
-		db,
-		rows.map((row) => row.profile_id),
-	);
-}
-
-export const __test__ = {
-	normalizeIndexValue,
-	getProfileBioUrls,
-};

package/src/lib/inbox.ts

@@ -1,241 +0,0 @@
-import { Effect } from "effect";
-import { getNativeDb } from "./db";
-import { runEffectPromise } from "./effect-runtime";
-import { scoreInboxItemWithOpenAIEffect } from "./openai";
-import { listDmConversations, listTimelineItems } from "./queries";
-import type { InboxItem, InboxQuery, InboxResponse } from "./types";
-
-function heuristicSummary(kind: InboxItem["entityKind"]) {
-	return kind === "dm"
-		? "Ranked from reply pressure, influence, and recency."
-		: "Ranked from mention urgency, influence, and specificity.";
-}
-
-function getHeuristicScoreForMention(
-	item: ReturnType<typeof listTimelineItems>[number],
-) {
-	const influence = Math.min(
-		32,
-		Math.round(Math.log10(item.author.followersCount + 10) * 18),
-	);
-	const specificityBoost = item.text.includes("?") ? 8 : 0;
-	return Math.max(0, Math.min(100, 44 + influence + specificityBoost));
-}
-
-function getHeuristicScoreForDm(
-	item: ReturnType<typeof listDmConversations>[number],
-) {
-	const unreadBoost = Math.min(15, item.unreadCount * 5);
-	const replyBoost = item.needsReply ? 12 : 0;
-	return Math.max(
-		0,
-		Math.min(
-			100,
-			34 + Math.round(item.influenceScore * 0.32) + unreadBoost + replyBoost,
-		),
-	);
-}
-
-function readStoredScores() {
-	const db = getNativeDb();
-	const rows = db
-		.prepare(
-			`
-      select entity_kind, entity_id, model, score, summary, reasoning, updated_at
-      from ai_scores
-      `,
-		)
-		.all() as Array<Record<string, unknown>>;
-
-	return new Map(
-		rows.map((row) => [
-			`${row.entity_kind}:${row.entity_id}`,
-			{
-				model: String(row.model),
-				score: Number(row.score),
-				summary: String(row.summary),
-				reasoning: String(row.reasoning),
-				updatedAt: String(row.updated_at),
-			},
-		]),
-	);
-}
-
-export function listInboxItems({
-	kind = "mixed",
-	account,
-	minScore = 0,
-	hideLowSignal = false,
-	limit = 20,
-}: InboxQuery = {}): InboxResponse {
-	const storedScores = readStoredScores();
-	const items: InboxItem[] = [];
-
-	if (kind === "mixed" || kind === "mentions") {
-		for (const mention of listTimelineItems({
-			resource: "mentions",
-			account,
-			replyFilter: "unreplied",
-			limit: 50,
-		})) {
-			const scoreKey = `mention:${mention.id}`;
-			const stored = storedScores.get(scoreKey);
-			items.push({
-				id: scoreKey,
-				entityId: mention.id,
-				entityKind: "mention",
-				accountId: mention.accountId,
-				accountHandle: mention.accountHandle,
-				title: `Mention from ${mention.author.displayName}`,
-				text: mention.text,
-				createdAt: mention.createdAt,
-				needsReply: !mention.isReplied,
-				influenceScore: Math.round(
-					Math.log10(mention.author.followersCount + 10) * 24,
-				),
-				participant: mention.author,
-				source: stored ? "openai" : "heuristic",
-				score: stored?.score ?? getHeuristicScoreForMention(mention),
-				summary: stored?.summary ?? heuristicSummary("mention"),
-				reasoning:
-					stored?.reasoning ??
-					`@${mention.author.handle} · ${mention.author.followersCount} followers`,
-			});
-		}
-	}
-
-	if (kind === "mixed" || kind === "dms") {
-		for (const dm of listDmConversations({
-			account,
-			replyFilter: "unreplied",
-			sort: "followers",
-			limit: 50,
-		})) {
-			const scoreKey = `dm:${dm.id}`;
-			const stored = storedScores.get(scoreKey);
-			items.push({
-				id: scoreKey,
-				entityId: dm.id,
-				entityKind: "dm",
-				accountId: dm.accountId,
-				accountHandle: dm.accountHandle,
-				title: `DM from ${dm.participant.displayName}`,
-				text: dm.lastMessagePreview,
-				createdAt: dm.lastMessageAt,
-				needsReply: dm.needsReply,
-				influenceScore: dm.influenceScore,
-				participant: dm.participant,
-				source: stored ? "openai" : "heuristic",
-				score: stored?.score ?? getHeuristicScoreForDm(dm),
-				summary: stored?.summary ?? heuristicSummary("dm"),
-				reasoning:
-					stored?.reasoning ??
-					`@${dm.participant.handle} · ${dm.participant.followersCount} followers`,
-			});
-		}
-	}
-
-	const lowSignalFloor = hideLowSignal ? Math.max(40, minScore) : minScore;
-	const filtered = items
-		.filter((item) => item.score >= lowSignalFloor)
-		.sort((left, right) => {
-			if (right.score !== left.score) return right.score - left.score;
-			return (
-				new Date(right.createdAt).getTime() - new Date(left.createdAt).getTime()
-			);
-		})
-		.slice(0, limit);
-
-	return {
-		items: filtered,
-		stats: {
-			total: filtered.length,
-			openai: filtered.filter((item) => item.source === "openai").length,
-			heuristic: filtered.filter((item) => item.source === "heuristic").length,
-		},
-	};
-}
-
-function toError(error: unknown) {
-	return error instanceof Error ? error : new Error(String(error));
-}
-
-function trySync<T>(try_: () => T) {
-	return Effect.try({
-		try: try_,
-		catch: toError,
-	});
-}
-
-export function scoreInboxEffect({
-	kind = "mixed",
-	account,
-	limit = 8,
-}: Pick<InboxQuery, "kind" | "account" | "limit"> = {}) {
-	return Effect.gen(function* () {
-		const db = yield* trySync(() => getNativeDb());
-		const items = yield* trySync(
-			() => listInboxItems({ kind, account, limit }).items,
-		);
-		const results = [];
-
-		for (const item of items) {
-			const scored = yield* scoreInboxItemWithOpenAIEffect({
-				entityKind: item.entityKind,
-				title: item.title,
-				text: item.text,
-				influenceScore: item.influenceScore,
-				participant: {
-					handle: item.participant.handle,
-					displayName: item.participant.displayName,
-					bio: item.participant.bio,
-					followersCount: item.participant.followersCount,
-				},
-			});
-
-			yield* trySync(() =>
-				db
-					.prepare(
-						`
-      insert into ai_scores (
-        entity_kind, entity_id, model, score, summary, reasoning, updated_at
-      ) values (?, ?, ?, ?, ?, ?, ?)
-      on conflict(entity_kind, entity_id) do update set
-        model = excluded.model,
-        score = excluded.score,
-        summary = excluded.summary,
-        reasoning = excluded.reasoning,
-        updated_at = excluded.updated_at
-      `,
-					)
-					.run(
-						item.entityKind,
-						item.entityId,
-						scored.model,
-						scored.score,
-						scored.summary,
-						scored.reasoning,
-						new Date().toISOString(),
-					),
-			);
-
-			results.push({
-				id: item.id,
-				score: scored.score,
-				source: "openai",
-			});
-		}
-
-		return {
-			ok: true,
-			scored: results.length,
-			items: results,
-		};
-	});
-}
-
-export function scoreInbox(
-	options: Pick<InboxQuery, "kind" | "account" | "limit"> = {},
-) {
-	return runEffectPromise(scoreInboxEffect(options));
-}