birdclaw 0.3.0 → 0.4.0

package/src/lib/queries.ts

@@ -1,7 +1,8 @@
 import { randomUUID } from "node:crypto";
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { findArchives } from "./archive-finder";
 import { getDb, getNativeDb } from "./db";
+import { fetchProfileAffiliations } from "./profile-affiliations";
 import type {
 	AccountRecord,
 	DmConversationItem,
@@ -38,6 +39,10 @@ function getInfluenceLabel(score: number) {
 
 function toProfile(row: Record<string, unknown>): ProfileRecord {
 	const followingCount = Number(row.following_count ?? 0);
+	const entities = parseJsonField<Record<string, unknown> | undefined>(
+		row.entities_json,
+		undefined,
+	);
 	return {
 		id: String(row.id),
 		handle: String(row.handle),
@@ -48,6 +53,16 @@ function toProfile(row: Record<string, unknown>): ProfileRecord {
 		avatarHue: Number(row.avatar_hue),
 		avatarUrl:
 			typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
+		...(typeof row.location === "string" && row.location.length > 0
+			? { location: row.location }
+			: {}),
+		...(typeof row.url === "string" && row.url.length > 0
+			? { url: row.url }
+			: {}),
+		...(typeof row.verified_type === "string" && row.verified_type.length > 0
+			? { verifiedType: row.verified_type }
+			: {}),
+		...(entities ? { entities } : {}),
 		createdAt: String(row.created_at),
 	};
 }
@@ -64,6 +79,15 @@ function parseJsonField<T>(value: unknown, fallback: T): T {
 	}
 }
 
+function toFtsSearchQuery(value: string) {
+	const terms = value.match(/[\p{L}\p{N}_]+/gu) ?? [];
+	return terms
+		.map((term) => term.trim())
+		.filter((term) => term.length > 0)
+		.map((term) => `"${term.replaceAll('"', '""')}"`)
+		.join(" ");
+}
+
 function enrichEntities(
 	entities: TweetEntities,
 	profiles: Record<string, ProfileRecord>,
@@ -211,19 +235,42 @@ function getTimelineQualityReason(
 	return "keep:long-text";
 }
 
+function countTimelineEdges(db: Database, kind: "home" | "mention") {
+	const row = db
+		.prepare(
+			`
+      with timeline_edges as (
+        select account_id, tweet_id, kind
+        from tweet_account_edges
+        where kind = ?
+        union all
+        select legacy.account_id, legacy.id as tweet_id, legacy.kind
+        from tweets legacy
+        where legacy.kind = ?
+          and not exists (
+            select 1
+            from tweet_account_edges edge
+            where edge.account_id = legacy.account_id
+              and edge.tweet_id = legacy.id
+              and edge.kind = legacy.kind
+          )
+      )
+      select count(*) as count
+      from timeline_edges e
+      join tweets t on t.id = e.tweet_id
+      where e.kind = ?
+      `,
+		)
+		.get(kind, kind, kind) as { count: number | bigint } | undefined;
+	return Number(row?.count ?? 0);
+}
+
 export async function getQueryEnvelope(): Promise<QueryEnvelope> {
 	const db = getDb();
+	const nativeDb = getNativeDb();
+	const homeCount = countTimelineEdges(nativeDb, "home");
+	const mentionCount = countTimelineEdges(nativeDb, "mention");
 	const counts = await Promise.all([
-		db
-			.selectFrom("tweets")
-			.select((eb) => eb.fn.countAll().as("count"))
-			.where("kind", "=", "home")
-			.executeTakeFirstOrThrow(),
-		db
-			.selectFrom("tweets")
-			.select((eb) => eb.fn.countAll().as("count"))
-			.where("kind", "=", "mention")
-			.executeTakeFirstOrThrow(),
 		db
 			.selectFrom("dm_conversations")
 			.select((eb) => eb.fn.countAll().as("count"))
@@ -245,13 +292,13 @@ export async function getQueryEnvelope(): Promise<QueryEnvelope> {
 
 	return {
 		stats: {
-			home: Number(counts[0].count),
-			mentions: Number(counts[1].count),
-			dms: Number(counts[2].count),
-			needsReply: Number(counts[3].count),
-			inbox: Number(counts[1].count) + Number(counts[3].count),
+			home: homeCount,
+			mentions: mentionCount,
+			dms: Number(counts[0].count),
+			needsReply: Number(counts[1].count),
+			inbox: mentionCount + Number(counts[1].count),
 		},
-		accounts: counts[4].map((row) => ({
+		accounts: counts[2].map((row) => ({
 			id: row.id,
 			name: row.name,
 			handle: row.handle,
@@ -260,8 +307,8 @@ export async function getQueryEnvelope(): Promise<QueryEnvelope> {
 			isDefault: row.is_default,
 			createdAt: row.created_at,
 		})) satisfies AccountRecord[],
-		archives: counts[5],
-		transport: counts[6],
+		archives: counts[3],
+		transport: counts[4],
 	};
 }
 
@@ -282,20 +329,88 @@ export function listTimelineItems({
 }: TimelineQuery): TimelineItem[] {
 	const db = getNativeDb();
 	const kind = resource === "mentions" ? "mention" : resource;
-	const params: Array<string | number> = [kind];
+	const params: Array<string | number> = [];
 	const normalizedLowQualityThreshold =
 		normalizeLowQualityThreshold(lowQualityThreshold);
+	let timelineEdgesCte = `
+      with timeline_edges as (
+        select account_id, tweet_id, kind
+        from tweet_account_edges
+        where kind = ?
+        union all
+        select legacy.account_id, legacy.id as tweet_id, legacy.kind
+        from tweets legacy
+        where legacy.kind = ?
+          and not exists (
+            select 1
+            from tweet_account_edges edge
+            where edge.account_id = legacy.account_id
+              and edge.tweet_id = legacy.id
+              and edge.kind = legacy.kind
+          )
+      )
+    `;
 	let join = "";
 	let where = "where t.kind = ?";
 	let searchSnippetSelect = "";
 
 	if (likedOnly || bookmarkedOnly) {
-		params.length = 0;
+		if (likedOnly && bookmarkedOnly) {
+			timelineEdgesCte = `
+        with timeline_edges as (
+          select likes.account_id, likes.tweet_id, 'home' as kind
+          from tweet_collections likes
+          join tweet_collections bookmarks
+            on bookmarks.account_id = likes.account_id
+            and bookmarks.tweet_id = likes.tweet_id
+            and bookmarks.kind = 'bookmarks'
+          where likes.kind = 'likes'
+          union all
+          select legacy.account_id, legacy.id as tweet_id, 'home' as kind
+          from tweets legacy
+          where legacy.liked = 1
+            and legacy.bookmarked = 1
+            and not exists (
+              select 1
+              from tweet_collections collection
+              where collection.account_id = legacy.account_id
+                and collection.tweet_id = legacy.id
+                and collection.kind in ('likes', 'bookmarks')
+            )
+        )
+      `;
+		} else {
+			const collectionKind = likedOnly ? "likes" : "bookmarks";
+			const legacyColumn = likedOnly ? "liked" : "bookmarked";
+			timelineEdgesCte = `
+        with timeline_edges as (
+          select account_id, tweet_id, 'home' as kind
+          from tweet_collections
+          where kind = ?
+          union all
+          select legacy.account_id, legacy.id as tweet_id, 'home' as kind
+          from tweets legacy
+          where legacy.${legacyColumn} = 1
+            and not exists (
+              select 1
+              from tweet_collections collection
+              where collection.account_id = legacy.account_id
+                and collection.tweet_id = legacy.id
+                and collection.kind = ?
+            )
+        )
+      `;
+			params.push(collectionKind, collectionKind);
+		}
 		where = "where 1 = 1";
+	} else {
+		params.push(kind, kind);
+		where = "where e.kind = ?";
+		params.push(kind);
 	}
 
 	if (account && account !== "all") {
-		where += " and a.id = ?";
+		where += " and e.account_id = ?";
 		params.push(account);
 	}
 
@@ -324,20 +439,13 @@ export function listTimelineItems({
 		params.push(until.trim());
 	}
 
-	if (search?.trim()) {
+	const ftsSearch = search?.trim() ? toFtsSearchQuery(search) : "";
+	if (ftsSearch) {
 		join += " join tweets_fts on tweets_fts.tweet_id = t.id ";
 		where += " and tweets_fts.text match ?";
 		searchSnippetSelect =
 			", snippet(tweets_fts, 1, '<mark>', '</mark>', '...', 16) as search_snippet";
-		params.push(search.trim());
-	}
-
-	if (likedOnly) {
-		where += " and t.liked = 1";
-	}
-
-	if (bookmarkedOnly) {
-		where += " and t.bookmarked = 1";
+		params.push(ftsSearch);
 	}
 
 	params.push(limit);
@@ -345,18 +453,37 @@ export function listTimelineItems({
 	const rows = db
 		.prepare(
 			`
+      ${timelineEdgesCte}
       select
         t.id,
-        t.account_id,
+        e.account_id,
         a.handle as account_handle,
-        t.kind,
+        e.kind,
         t.text,
         t.created_at,
         t.is_replied,
         t.like_count,
         t.media_count,
-        t.bookmarked,
-        t.liked,
+        case
+          when exists (
+            select 1 from tweet_collections collection
+            where collection.account_id = e.account_id
+              and collection.tweet_id = t.id
+              and collection.kind = 'bookmarks'
+          ) then 1
+          when t.account_id = e.account_id and t.bookmarked = 1 then 1
+          else 0
+        end as bookmarked,
+        case
+          when exists (
+            select 1 from tweet_collections collection
+            where collection.account_id = e.account_id
+              and collection.tweet_id = t.id
+              and collection.kind = 'likes'
+          ) then 1
+          when t.account_id = e.account_id and t.liked = 1 then 1
+          else 0
+        end as liked,
         t.entities_json,
         t.media_json,
         t.quoted_tweet_id,
@@ -368,6 +495,10 @@ export function listTimelineItems({
         p.following_count,
         p.avatar_hue,
         p.avatar_url,
+        p.location as profile_location,
+        p.url as profile_url,
+        p.verified_type as profile_verified_type,
+        p.entities_json as profile_entities_json,
         p.created_at as profile_created_at,
         rt.id as reply_id,
         rt.text as reply_text,
@@ -398,8 +529,9 @@ export function listTimelineItems({
         qp.avatar_url as quoted_avatar_url,
         qp.created_at as quoted_profile_created_at
         ${searchSnippetSelect}
-      from tweets t
-      join accounts a on a.id = t.account_id
+      from timeline_edges e
+      join tweets t on t.id = e.tweet_id
+      join accounts a on a.id = e.account_id
       join profiles p on p.id = t.author_profile_id
       left join tweets rt on rt.id = t.reply_to_id
       left join profiles rp on rp.id = rt.author_profile_id
@@ -497,6 +629,7 @@ export function listTimelineItems({
 
 export function listDmConversations({
 	account,
+	conversationIds,
 	participant,
 	search,
 	replyFilter = "all",
@@ -505,6 +638,7 @@ export function listDmConversations({
 	minInfluenceScore,
 	maxInfluenceScore,
 	sort = "recent",
+	context = 0,
 	limit = 20,
 }: DmQuery): DmConversationItem[] {
 	const db = getNativeDb();
@@ -514,12 +648,18 @@ export function listDmConversations({
 	let join = "";
 	let where = "where 1 = 1";
 	let searchSnippetSelect = "";
+	const ftsSearch = search?.trim() ? toFtsSearchQuery(search) : "";
 
 	if (account && account !== "all") {
 		where += " and a.id = ?";
 		params.push(account);
 	}
 
+	if (conversationIds && conversationIds.length > 0) {
+		where += ` and c.id in (${conversationIds.map(() => "?").join(",")})`;
+		params.push(...conversationIds);
+	}
+
 	if (participant?.trim()) {
 		where += " and (p.handle like ? or p.display_name like ?)";
 		params.push(`%${participant.trim()}%`, `%${participant.trim()}%`);
@@ -541,9 +681,9 @@ export function listDmConversations({
 		params.push(maxFollowers);
 	}
 
-	if (search?.trim()) {
+	if (ftsSearch) {
 		searchSnippetCte = `
-      with ranked_dm_search as materialized (
+	      with ranked_dm_search as materialized (
         select
           latest_search.id,
           latest_search.conversation_id,
@@ -564,10 +704,10 @@ export function listDmConversations({
         where ranked_dm_search.match_rank = 1
           and dm_fts.text match ?
       )
-    `;
+	`;
 		join += " join dm_search on dm_search.conversation_id = c.id ";
 		searchSnippetSelect = ", dm_search.search_snippet as search_snippet";
-		joinParams.push(search.trim(), search.trim());
+		joinParams.push(ftsSearch, ftsSearch);
 	}
 
 	params.push(limit);
@@ -592,6 +732,10 @@ export function listDmConversations({
         p.following_count,
         p.avatar_hue,
         p.avatar_url,
+        p.location as profile_location,
+        p.url as profile_url,
+        p.verified_type as profile_verified_type,
+        p.entities_json as profile_entities_json,
         p.created_at as profile_created_at,
         (
           select text
@@ -613,9 +757,29 @@ export function listDmConversations({
 		)
 		.all(...joinParams, ...params) as Array<Record<string, unknown>>;
 
-	const items = rows.map((row) => {
+	const affiliationsByProfile = fetchProfileAffiliations(
+		db,
+		rows.map((row) => String(row.profile_id)),
+	);
+	const items: DmConversationItem[] = rows.map((row) => {
 		const followersCount = Number(row.followers_count);
 		const influenceScore = getInfluenceScore(followersCount);
+		const participant = toProfile({
+			id: row.profile_id,
+			handle: row.handle,
+			display_name: row.display_name,
+			bio: row.bio,
+			followers_count: row.followers_count,
+			following_count: row.following_count,
+			avatar_hue: row.avatar_hue,
+			avatar_url: row.avatar_url,
+			location: row.profile_location,
+			url: row.profile_url,
+			verified_type: row.profile_verified_type,
+			entities_json: row.profile_entities_json,
+			created_at: row.profile_created_at,
+		});
+		const affiliations = affiliationsByProfile.get(participant.id) ?? [];
 		return {
 			id: String(row.id),
 			accountId: String(row.account_id),
@@ -631,18 +795,13 @@ export function listDmConversations({
 			influenceScore,
 			influenceLabel: getInfluenceLabel(influenceScore),
 			participant: {
-				id: String(row.profile_id),
-				handle: String(row.handle),
-				displayName: String(row.display_name),
-				bio: String(row.bio),
-				followersCount,
-				followingCount: Number(row.following_count ?? 0),
-				avatarHue: Number(row.avatar_hue),
-				avatarUrl:
-					typeof row.avatar_url === "string"
-						? String(row.avatar_url)
-						: undefined,
-				createdAt: String(row.profile_created_at),
+				...participant,
+				...(affiliations.length > 0
+					? {
+							affiliations,
+							primaryAffiliation: affiliations[0],
+						}
+					: {}),
 			},
 		};
 	});
@@ -681,7 +840,23 @@ export function listDmConversations({
 		});
 	}
 
-	return filtered.slice(0, limit);
+	const limited = filtered.slice(0, limit);
+	const normalizedContext = normalizeDmContext(context);
+	if (ftsSearch && normalizedContext > 0 && limited.length > 0) {
+		const matches = getDmSearchMatches({
+			search: ftsSearch,
+			conversationIds: limited.map((item) => item.id),
+			context: normalizedContext,
+		});
+		for (const item of limited) {
+			const itemMatches = matches.get(item.id);
+			if (itemMatches && itemMatches.length > 0) {
+				item.matches = itemMatches;
+			}
+		}
+	}
+
+	return limited;
 }
 
 export function getConversationThread(
@@ -749,6 +924,165 @@ export function getConversationThread(
 	};
 }
 
+function normalizeDmContext(value: number | undefined) {
+	if (typeof value !== "number" || !Number.isFinite(value)) {
+		return 0;
+	}
+	return Math.max(0, Math.min(20, Math.trunc(value)));
+}
+
+function mapDmMessageRow(row: Record<string, unknown>): DmMessageItem {
+	return {
+		id: String(row.id),
+		conversationId: String(row.conversation_id),
+		text: String(row.text),
+		createdAt: String(row.created_at),
+		direction: row.direction as DmMessageItem["direction"],
+		isReplied: Boolean(row.is_replied),
+		mediaCount: Number(row.media_count),
+		sender: toProfile({
+			id: row.profile_id,
+			handle: row.handle,
+			display_name: row.display_name,
+			bio: row.bio,
+			followers_count: row.followers_count,
+			following_count: row.following_count,
+			avatar_hue: row.avatar_hue,
+			avatar_url: row.avatar_url,
+			created_at: row.profile_created_at,
+		}),
+	};
+}
+
+function selectDmMessageSql(where: string, orderBy: string) {
+	return `
+    select
+      m.id,
+      m.conversation_id,
+      m.text,
+      m.created_at,
+      m.direction,
+      m.is_replied,
+      m.media_count,
+      p.id as profile_id,
+      p.handle,
+      p.display_name,
+      p.bio,
+      p.followers_count,
+      p.following_count,
+      p.avatar_hue,
+      p.avatar_url,
+      p.created_at as profile_created_at
+    from dm_messages m
+    join profiles p on p.id = m.sender_profile_id
+    ${where}
+    ${orderBy}
+  `;
+}
+
+function getDmSearchMatches({
+	search,
+	conversationIds,
+	context,
+}: {
+	search: string;
+	conversationIds: string[];
+	context: number;
+}) {
+	const db = getNativeDb();
+	if (search.length === 0) {
+		return new Map<string, DmConversationItem["matches"]>();
+	}
+	const conversationPlaceholders = conversationIds.map(() => "?").join(", ");
+	const matchRows = db
+		.prepare(
+			`
+      with ranked_matches as (
+        select
+          m.id,
+          m.conversation_id,
+          m.text,
+          m.created_at,
+          m.direction,
+          m.is_replied,
+          m.media_count,
+          p.id as profile_id,
+          p.handle,
+          p.display_name,
+          p.bio,
+          p.followers_count,
+          p.following_count,
+          p.avatar_hue,
+          p.avatar_url,
+          p.created_at as profile_created_at,
+          row_number() over (
+            partition by m.conversation_id
+            order by m.created_at desc, m.id desc
+          ) as match_rank
+        from dm_messages m
+        join dm_fts on dm_fts.message_id = m.id
+        join profiles p on p.id = m.sender_profile_id
+        where dm_fts.text match ?
+          and m.conversation_id in (${conversationPlaceholders})
+      )
+      select *
+      from ranked_matches
+      where match_rank <= 3
+      order by created_at desc, id desc
+      `,
+		)
+		.all(search, ...conversationIds) as Array<Record<string, unknown>>;
+
+	const beforeStatement = db.prepare(
+		selectDmMessageSql(
+			`
+      where m.conversation_id = ?
+        and (m.created_at < ? or (m.created_at = ? and m.id < ?))
+    `,
+			"order by m.created_at desc, m.id desc limit ?",
+		),
+	);
+	const afterStatement = db.prepare(
+		selectDmMessageSql(
+			`
+      where m.conversation_id = ?
+        and (m.created_at > ? or (m.created_at = ? and m.id > ?))
+    `,
+			"order by m.created_at asc, m.id asc limit ?",
+		),
+	);
+	const grouped = new Map<string, DmConversationItem["matches"]>();
+
+	for (const row of matchRows) {
+		const message = mapDmMessageRow(row);
+		const before = (
+			beforeStatement.all(
+				message.conversationId,
+				message.createdAt,
+				message.createdAt,
+				message.id,
+				context,
+			) as Array<Record<string, unknown>>
+		)
+			.map(mapDmMessageRow)
+			.reverse();
+		const after = (
+			afterStatement.all(
+				message.conversationId,
+				message.createdAt,
+				message.createdAt,
+				message.id,
+				context,
+			) as Array<Record<string, unknown>>
+		).map(mapDmMessageRow);
+		const matches = grouped.get(message.conversationId) ?? [];
+		matches.push({ message, before, after });
+		grouped.set(message.conversationId, matches);
+	}
+
+	return grouped;
+}
+
 export function queryResource(
 	resource: "home" | "mentions" | "dms",
 	filters: (TimelineQuery | DmQuery) & { conversationId?: string },
@@ -779,7 +1113,7 @@ export function queryResource(
 }
 
 function refreshDmConversationState(
-	db: Database.Database,
+	db: Database,
 	conversationId: string,
 	lastMessageAt: string,
 ) {

package/src/lib/research.ts

@@ -593,4 +593,15 @@ export async function runResearchMode(
 	return report;
 }
 
+export const __test__ = {
+	parseJsonField,
+	normalizeTweetEntities,
+	toResearchNode,
+	dedupeNodes,
+	orderThreadNodes,
+	collectExternalLinks,
+	collectHandles,
+	renderReportMarkdown,
+};
+
 export type { ResearchRow };

package/src/lib/seed.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 
 const now = new Date("2026-03-08T12:00:00.000Z");
 
@@ -22,7 +22,7 @@ function svgAvatarDataUrl(label: string, hue: number) {
 	return `data:image/svg+xml;utf8,${encodeURIComponent(svg)}`;
 }
 
-export function seedDemoData(db: Database.Database) {
+export function seedDemoData(db: Database) {
 	const accountCount = db
 		.prepare("select count(*) as count from accounts")
 		.get() as {