birdclaw 0.4.1 → 0.5.1

package/src/lib/moderation-target.ts

@@ -100,6 +100,12 @@ export async function resolveProfile(
 	}
 
 	const local = resolveLocalProfile(db, normalizedQuery);
+	if (process.env.BIRDCLAW_DISABLE_LIVE_PROFILE_LOOKUP === "1") {
+		if (local) {
+			return local;
+		}
+		throw new Error(`Profile not found locally: ${query}`);
+	}
 	if (
 		local &&
 		!local.profile.id.startsWith("profile_group_") &&

package/src/lib/profile-hydration.ts

@@ -44,7 +44,7 @@ export async function hydrateProfilesFromX() {
 
 	const candidateIds = candidateRows
 		.map((row) => row.id.replace(/^profile_user_/, ""))
-		.filter((id) => id.length > 0);
+		.filter((id) => /^\d+$/.test(id));
 
 	const updateConversationTitle = db.prepare(`
     update dm_conversations

package/src/lib/profile-resolver.ts

@@ -7,7 +7,7 @@ import {
 } from "./profile-affiliation-hydration";
 import type { ProfileRecord, XurlMentionUser } from "./types";
 import { getExternalUserId, upsertProfileFromXUser } from "./x-profile";
-import { lookupUsersByIds } from "./xurl";
+import { lookupUsersByHandles, lookupUsersByIds } from "./xurl";
 
 const PROFILE_CACHE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
 const PROFILE_NEGATIVE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
@@ -39,6 +39,14 @@ export interface ProfileResolveResult {
 	error?: string;
 }
 
+export interface HandleProfileResolveResult {
+	handle: string;
+	status: ProfileLookupStatus;
+	source: Exclude<ProfileLookupSource, "local">;
+	profile?: ProfileRecord;
+	error?: string;
+}
+
 function toProfile(row: Record<string, unknown>): ProfileRecord {
 	const followingCount = Number(row.following_count ?? 0);
 	let entities: Record<string, unknown> | undefined;
@@ -139,6 +147,10 @@ async function lookupViaXurl(externalUserId: string) {
 	return user ?? null;
 }
 
+function normalizeHandle(value: string) {
+	return value.trim().replace(/^@/, "").toLowerCase();
+}
+
 async function fetchProfileUser(
 	externalUserId: string,
 	xurlFallback: boolean,
@@ -387,6 +399,108 @@ export async function resolveProfilesForIds(
 	return results;
 }
 
+export async function resolveProfilesForHandles(
+	handles: string[],
+	options: Pick<ResolveProfilesOptions, "xurlFallback"> = {},
+): Promise<HandleProfileResolveResult[]> {
+	const xurlFallback = options.xurlFallback ?? true;
+	const targets = Array.from(
+		new Set(handles.map(normalizeHandle).filter((handle) => handle.length > 0)),
+	);
+	if (targets.length === 0) {
+		return [];
+	}
+
+	const results = new Map<string, HandleProfileResolveResult>();
+	let unresolved = targets;
+
+	try {
+		const birdResults = await lookupProfilesViaBird(targets);
+		for (const item of birdResults) {
+			const handle = normalizeHandle(item.target);
+			if (item.user) {
+				const resolved = upsertProfileFromXUser(getNativeDb(), item.user);
+				updateConversationTitles(resolved.profile);
+				results.set(handle, {
+					handle,
+					status: "hit",
+					source: "bird",
+					profile: resolved.profile,
+				});
+			} else if (item.error && !xurlFallback) {
+				results.set(handle, {
+					handle,
+					status: "error",
+					source: "bird",
+					error: item.error,
+				});
+			}
+		}
+		unresolved = targets.filter((handle) => !results.has(handle));
+	} catch (error) {
+		if (!xurlFallback) {
+			for (const handle of targets) {
+				results.set(handle, {
+					handle,
+					status: "error",
+					source: "bird",
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+			unresolved = [];
+		}
+	}
+
+	if (unresolved.length > 0 && xurlFallback) {
+		try {
+			const users = await lookupUsersByHandles(unresolved);
+			const usersByHandle = new Map(
+				users.map((user) => [
+					normalizeHandle(String(user.username ?? "")),
+					user,
+				]),
+			);
+			for (const handle of unresolved) {
+				const user = usersByHandle.get(handle);
+				if (user) {
+					const resolved = upsertProfileFromXUser(getNativeDb(), user);
+					updateConversationTitles(resolved.profile);
+					results.set(handle, {
+						handle,
+						status: "hit",
+						source: "xurl",
+						profile: resolved.profile,
+					});
+				} else {
+					results.set(handle, {
+						handle,
+						status: "miss",
+						source: "xurl",
+					});
+				}
+			}
+		} catch (error) {
+			for (const handle of unresolved) {
+				results.set(handle, {
+					handle,
+					status: "error",
+					source: "xurl",
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+		}
+	}
+
+	return targets.map(
+		(handle) =>
+			results.get(handle) ?? {
+				handle,
+				status: "miss",
+				source: "bird",
+			},
+	);
+}
+
 export async function resolvePlaceholderProfiles(
 	options: ResolveProfilesOptions & { limit?: number } = {},
 ) {

package/src/lib/queries.ts

@@ -3,6 +3,7 @@ import type { Database } from "./sqlite";
 import { findArchives } from "./archive-finder";
 import { getDb, getNativeDb } from "./db";
 import { fetchProfileAffiliations } from "./profile-affiliations";
+import { displayUrlForLink, enrichFallbackUrlEntities } from "./tweet-render";
 import type {
 	AccountRecord,
 	DmConversationItem,
@@ -17,7 +18,9 @@ import type {
 	TimelineItem,
 	TimelineQuery,
 	TweetEntities,
+	TweetConversationResponse,
 	TweetMediaItem,
+	TweetUrlEntity,
 } from "./types";
 import {
 	dmViaXurl,
@@ -107,7 +110,78 @@ function enrichEntities(
 	};
 }
 
+type UrlExpansionCache = Map<
+	string,
+	| (Pick<TweetUrlEntity, "expandedUrl" | "displayUrl"> &
+			Partial<
+				Pick<TweetUrlEntity, "title" | "description" | "imageUrl" | "siteName">
+			>)
+	| null
+>;
+
+function getUrlExpansion(
+	db: Database,
+	cache: UrlExpansionCache,
+	rawUrl: string,
+) {
+	if (cache.has(rawUrl)) {
+		return cache.get(rawUrl);
+	}
+
+	const row = db
+		.prepare(
+			`
+      select expanded_url, final_url, title, description, image_url, site_name
+      from url_expansions
+      where short_url = ?
+        and status = 'hit'
+      `,
+		)
+		.get(rawUrl) as
+		| {
+				expanded_url: string;
+				final_url: string;
+				title: string | null;
+				description: string | null;
+				image_url: string | null;
+				site_name: string | null;
+		  }
+		| undefined;
+	if (!row) {
+		cache.set(rawUrl, null);
+		return null;
+	}
+
+	const expandedUrl = row.final_url || row.expanded_url || rawUrl;
+	const expansion = {
+		expandedUrl,
+		displayUrl: displayUrlForLink(expandedUrl),
+		...(row.title ? { title: row.title } : {}),
+		...(row.description ? { description: row.description } : {}),
+		...(row.image_url ? { imageUrl: row.image_url } : {}),
+		...(row.site_name ? { siteName: row.site_name } : {}),
+	};
+	cache.set(rawUrl, expansion);
+	return expansion;
+}
+
+function enrichTimelineEntities(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
+	text: string,
+	entities: TweetEntities,
+	profiles: Record<string, ProfileRecord>,
+): TweetEntities {
+	return enrichFallbackUrlEntities(
+		text,
+		enrichEntities(entities, profiles),
+		(rawUrl) => getUrlExpansion(db, urlExpansionCache, rawUrl),
+	);
+}
+
 function buildEmbeddedTweet(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
 	row: Record<string, unknown>,
 	prefix: string,
 ): EmbeddedTweet | null {
@@ -127,12 +201,20 @@ function buildEmbeddedTweet(
 		created_at: row[`${prefix}profile_created_at`],
 	});
 
+	const text = String(row[`${prefix}text`] ?? "");
 	return {
 		id: String(row[`${prefix}id`]),
-		text: String(row[`${prefix}text`] ?? ""),
+		text,
 		createdAt: String(row[`${prefix}created_at`] ?? new Date(0).toISOString()),
+		replyToId:
+			typeof row[`${prefix}reply_to_id`] === "string"
+				? String(row[`${prefix}reply_to_id`])
+				: null,
 		author,
-		entities: enrichEntities(
+		entities: enrichTimelineEntities(
+			db,
+			urlExpansionCache,
+			text,
 			parseJsonField<TweetEntities>(row[`${prefix}entities_json`], {}),
 			{
 				[author.id]: author,
@@ -239,32 +321,39 @@ function countTimelineEdges(db: Database, kind: "home" | "mention") {
 	const row = db
 		.prepare(
 			`
-      with timeline_edges as (
-        select account_id, tweet_id, kind
-        from tweet_account_edges
-        where kind = ?
-        union all
-        select legacy.account_id, legacy.id as tweet_id, legacy.kind
-        from tweets legacy
-        where legacy.kind = ?
-          and not exists (
-            select 1
-            from tweet_account_edges edge
-            where edge.account_id = legacy.account_id
-              and edge.tweet_id = legacy.id
-              and edge.kind = legacy.kind
-          )
-      )
-      select count(*) as count
-      from timeline_edges e
-      join tweets t on t.id = e.tweet_id
-      where e.kind = ?
+      select (
+	        (
+	          select count(*)
+	          from tweet_account_edges edge
+	          where edge.kind = ?
+	            and exists (
+	              select 1
+	              from tweets t
+	              where t.id = edge.tweet_id
+	            )
+	        )
+        +
+        (
+          select count(*)
+          from tweets legacy
+          where legacy.kind = ?
+            and not exists (
+              select 1
+              from tweet_account_edges edge
+              where edge.account_id = legacy.account_id
+                and edge.tweet_id = legacy.id
+                and edge.kind = legacy.kind
+            )
+        )
+      ) as count
       `,
 		)
-		.get(kind, kind, kind) as { count: number | bigint } | undefined;
+		.get(kind, kind) as { count: number | bigint } | undefined;
 	return Number(row?.count ?? 0);
 }
 
+const RECENT_TIMELINE_EDGE_CANDIDATES = 5000;
+
 export async function getQueryEnvelope(): Promise<QueryEnvelope> {
 	const db = getDb();
 	const nativeDb = getNativeDb();
@@ -350,10 +439,23 @@ export function listTimelineItems({
           )
       )
     `;
+	const unwindowedTimelineEdgesCte = timelineEdgesCte;
+	let usedRecentEdgeWindow = false;
 	let join = "";
 	let where = "where t.kind = ?";
 	let searchSnippetSelect = "";
 
+	const canUseRecentEdgeWindow =
+		!likedOnly &&
+		!bookmarkedOnly &&
+		!account &&
+		!search?.trim() &&
+		replyFilter === "all" &&
+		!since?.trim() &&
+		!until?.trim() &&
+		includeReplies &&
+		qualityFilter === "all";
+
 	if (likedOnly || bookmarkedOnly) {
 		if (likedOnly && bookmarkedOnly) {
 			timelineEdgesCte = `
@@ -399,10 +501,49 @@ export function listTimelineItems({
                 and collection.kind = ?
             )
         )
-      `;
+			`;
 			params.push(collectionKind, collectionKind);
 		}
 		where = "where 1 = 1";
+	} else if (canUseRecentEdgeWindow) {
+		usedRecentEdgeWindow = true;
+		timelineEdgesCte = `
+      with timeline_edges as (
+        select account_id, tweet_id, kind
+        from tweet_account_edges
+        where kind = ?
+          and tweet_id in (
+            select id
+            from tweets
+            order by created_at desc
+            limit ?
+          )
+        union all
+        select legacy.account_id, legacy.id as tweet_id, legacy.kind
+        from tweets legacy
+        where legacy.kind = ?
+          and legacy.id in (
+            select id
+            from tweets
+            order by created_at desc
+            limit ?
+          )
+          and not exists (
+            select 1
+            from tweet_account_edges edge
+            where edge.account_id = legacy.account_id
+              and edge.tweet_id = legacy.id
+              and edge.kind = legacy.kind
+          )
+      )
+    `;
+		const candidateLimit = Math.max(
+			RECENT_TIMELINE_EDGE_CANDIDATES,
+			limit * 50,
+		);
+		params.push(kind, candidateLimit, kind, candidateLimit);
+		where = "where e.kind = ?";
+		params.push(kind);
 	} else {
 		params.push(kind, kind);
 		where = "where e.kind = ?";
@@ -450,10 +591,8 @@ export function listTimelineItems({
 
 	params.push(limit);
 
-	const rows = db
-		.prepare(
-			`
-      ${timelineEdgesCte}
+	const buildTimelineSelectSql = (timelineEdgesSql: string) => `
+      ${timelineEdgesSql}
       select
         t.id,
         e.account_id,
@@ -461,6 +600,7 @@ export function listTimelineItems({
         e.kind,
         t.text,
         t.created_at,
+        t.reply_to_id,
         t.is_replied,
         t.like_count,
         t.media_count,
@@ -503,6 +643,7 @@ export function listTimelineItems({
         rt.id as reply_id,
         rt.text as reply_text,
         rt.created_at as reply_created_at,
+        rt.reply_to_id as reply_reply_to_id,
         rt.entities_json as reply_entities_json,
         rt.media_json as reply_media_json,
         rp.id as reply_profile_id,
@@ -517,6 +658,7 @@ export function listTimelineItems({
         qt.id as quoted_id,
         qt.text as quoted_text,
         qt.created_at as quoted_created_at,
+        qt.reply_to_id as quoted_reply_to_id,
         qt.entities_json as quoted_entities_json,
         qt.media_json as quoted_media_json,
         qp.id as quoted_profile_id,
@@ -541,10 +683,19 @@ export function listTimelineItems({
       ${where}
       order by t.created_at desc
       limit ?
-      `,
-		)
+      `;
+
+	let rows = db
+		.prepare(buildTimelineSelectSql(timelineEdgesCte))
 		.all(...params) as Array<Record<string, unknown>>;
 
+	if (usedRecentEdgeWindow && rows.length < limit) {
+		rows = db
+			.prepare(buildTimelineSelectSql(unwindowedTimelineEdgesCte))
+			.all(kind, kind, kind, limit) as Array<Record<string, unknown>>;
+	}
+
+	const urlExpansionCache: UrlExpansionCache = new Map();
 	return rows.map((row) => {
 		const author = {
 			id: String(row.profile_id),
@@ -558,7 +709,11 @@ export function listTimelineItems({
 				typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
 			createdAt: String(row.profile_created_at),
 		};
-		const entities = enrichEntities(
+		const text = String(row.text);
+		const entities = enrichTimelineEntities(
+			db,
+			urlExpansionCache,
+			text,
 			parseJsonField<TweetEntities>(row.entities_json, {}),
 			{
 				[author.id]: author,
@@ -599,11 +754,13 @@ export function listTimelineItems({
 			accountId: String(row.account_id),
 			accountHandle: String(row.account_handle),
 			kind: row.kind as TimelineItem["kind"],
-			text: String(row.text),
+			text,
 			...(typeof row.search_snippet === "string"
 				? { searchSnippet: row.search_snippet }
 				: {}),
 			createdAt: String(row.created_at),
+			replyToId:
+				typeof row.reply_to_id === "string" ? String(row.reply_to_id) : null,
 			isReplied: Boolean(row.is_replied),
 			likeCount: Number(row.like_count),
 			mediaCount: Number(row.media_count),
@@ -612,8 +769,8 @@ export function listTimelineItems({
 			author,
 			entities,
 			media: parseJsonField<TweetMediaItem[]>(row.media_json, []),
-			replyToTweet: buildEmbeddedTweet(row, "reply_"),
-			quotedTweet: buildEmbeddedTweet(row, "quoted_"),
+			replyToTweet: buildEmbeddedTweet(db, urlExpansionCache, row, "reply_"),
+			quotedTweet: buildEmbeddedTweet(db, urlExpansionCache, row, "quoted_"),
 		};
 		return includeQualityReason
 			? {
@@ -627,6 +784,140 @@ export function listTimelineItems({
 	});
 }
 
+const conversationTweetSelect = `
+  select
+    t.id,
+    t.text,
+    t.created_at,
+    t.reply_to_id,
+    t.entities_json,
+    t.media_json,
+    p.id as profile_id,
+    p.handle,
+    p.display_name,
+    p.bio,
+    p.followers_count,
+    p.following_count,
+    p.avatar_hue,
+    p.avatar_url,
+    p.created_at as profile_created_at
+  from tweets t
+  join profiles p on p.id = t.author_profile_id
+`;
+
+function getTweetById(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
+	tweetId: string,
+): EmbeddedTweet | null {
+	const row = db
+		.prepare(`${conversationTweetSelect} where t.id = ?`)
+		.get(tweetId) as Record<string, unknown> | undefined;
+	if (!row) return null;
+	return buildEmbeddedTweet(db, urlExpansionCache, row, "");
+}
+
+function listTweetDescendants(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
+	rootId: string,
+	limit: number,
+) {
+	if (limit <= 0) return [];
+	const rows = db
+		.prepare(
+			`
+      with recursive branch(id, depth) as (
+        select t.id, 0
+        from tweets t
+        where t.id = ?
+        union all
+        select child.id, branch.depth + 1
+        from tweets child
+        join branch on child.reply_to_id = branch.id
+        where branch.depth < 8
+      )
+      ${conversationTweetSelect}
+      join branch on branch.id = t.id
+      where t.id != ?
+      order by t.created_at asc
+      limit ?
+      `,
+		)
+		.all(rootId, rootId, limit) as Array<Record<string, unknown>>;
+
+	return rows
+		.map((row) => buildEmbeddedTweet(db, urlExpansionCache, row, ""))
+		.filter((tweet): tweet is EmbeddedTweet => Boolean(tweet));
+}
+
+function appendConversationTweets(
+	target: EmbeddedTweet[],
+	seen: Set<string>,
+	items: EmbeddedTweet[],
+	remaining: number,
+) {
+	for (const tweet of items) {
+		if (target.length >= remaining || seen.has(tweet.id)) continue;
+		seen.add(tweet.id);
+		target.push(tweet);
+	}
+}
+
+export function getTweetConversation(
+	tweetId: string,
+	limit = 80,
+): TweetConversationResponse | null {
+	const db = getNativeDb();
+	const urlExpansionCache: UrlExpansionCache = new Map();
+	const anchor = getTweetById(db, urlExpansionCache, tweetId);
+	if (!anchor) return null;
+
+	const ancestors: EmbeddedTweet[] = [];
+	let current = anchor;
+	for (let depth = 0; depth < 12 && current.replyToId; depth += 1) {
+		const parent = getTweetById(db, urlExpansionCache, current.replyToId);
+		if (!parent || ancestors.some((tweet) => tweet.id === parent.id)) break;
+		ancestors.push(parent);
+		current = parent;
+	}
+
+	const required = [...ancestors].reverse();
+	required.push(anchor);
+	const root = required[0] ?? anchor;
+	const seen = new Set<string>();
+	const items = required.filter((tweet) => {
+		if (seen.has(tweet.id)) return false;
+		seen.add(tweet.id);
+		return true;
+	});
+	const remainingAfterRequired = Math.max(0, limit - items.length);
+	const focusedDescendants = listTweetDescendants(
+		db,
+		urlExpansionCache,
+		anchor.id,
+		remainingAfterRequired,
+	);
+	appendConversationTweets(items, seen, focusedDescendants, limit);
+
+	if (items.length < limit && root.id !== anchor.id) {
+		const ambientDescendants = listTweetDescendants(
+			db,
+			urlExpansionCache,
+			root.id,
+			limit,
+		);
+		appendConversationTweets(items, seen, ambientDescendants, limit);
+	}
+
+	items.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+
+	return {
+		anchorId: anchor.id,
+		items,
+	};
+}
+
 export function listDmConversations({
 	account,
 	conversationIds,
@@ -1084,7 +1375,7 @@ function getDmSearchMatches({
 }
 
 export function queryResource(
-	resource: "home" | "mentions" | "dms",
+	resource: "home" | "mentions" | "authored" | "dms",
 	filters: (TimelineQuery | DmQuery) & { conversationId?: string },
 ): QueryResponse {
 	if (resource === "dms") {