birdclaw 0.4.1 → 0.5.0

package/src/lib/moderation-target.ts

@@ -100,6 +100,12 @@ export async function resolveProfile(
 	}
 
 	const local = resolveLocalProfile(db, normalizedQuery);
+	if (process.env.BIRDCLAW_DISABLE_LIVE_PROFILE_LOOKUP === "1") {
+		if (local) {
+			return local;
+		}
+		throw new Error(`Profile not found locally: ${query}`);
+	}
 	if (
 		local &&
 		!local.profile.id.startsWith("profile_group_") &&

package/src/lib/profile-hydration.ts

@@ -44,7 +44,7 @@ export async function hydrateProfilesFromX() {
 
 	const candidateIds = candidateRows
 		.map((row) => row.id.replace(/^profile_user_/, ""))
-		.filter((id) => id.length > 0);
+		.filter((id) => /^\d+$/.test(id));
 
 	const updateConversationTitle = db.prepare(`
     update dm_conversations

package/src/lib/profile-resolver.ts

@@ -7,7 +7,7 @@ import {
 } from "./profile-affiliation-hydration";
 import type { ProfileRecord, XurlMentionUser } from "./types";
 import { getExternalUserId, upsertProfileFromXUser } from "./x-profile";
-import { lookupUsersByIds } from "./xurl";
+import { lookupUsersByHandles, lookupUsersByIds } from "./xurl";
 
 const PROFILE_CACHE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
 const PROFILE_NEGATIVE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
@@ -39,6 +39,14 @@ export interface ProfileResolveResult {
 	error?: string;
 }
 
+export interface HandleProfileResolveResult {
+	handle: string;
+	status: ProfileLookupStatus;
+	source: Exclude<ProfileLookupSource, "local">;
+	profile?: ProfileRecord;
+	error?: string;
+}
+
 function toProfile(row: Record<string, unknown>): ProfileRecord {
 	const followingCount = Number(row.following_count ?? 0);
 	let entities: Record<string, unknown> | undefined;
@@ -139,6 +147,10 @@ async function lookupViaXurl(externalUserId: string) {
 	return user ?? null;
 }
 
+function normalizeHandle(value: string) {
+	return value.trim().replace(/^@/, "").toLowerCase();
+}
+
 async function fetchProfileUser(
 	externalUserId: string,
 	xurlFallback: boolean,
@@ -387,6 +399,108 @@ export async function resolveProfilesForIds(
 	return results;
 }
 
+export async function resolveProfilesForHandles(
+	handles: string[],
+	options: Pick<ResolveProfilesOptions, "xurlFallback"> = {},
+): Promise<HandleProfileResolveResult[]> {
+	const xurlFallback = options.xurlFallback ?? true;
+	const targets = Array.from(
+		new Set(handles.map(normalizeHandle).filter((handle) => handle.length > 0)),
+	);
+	if (targets.length === 0) {
+		return [];
+	}
+
+	const results = new Map<string, HandleProfileResolveResult>();
+	let unresolved = targets;
+
+	try {
+		const birdResults = await lookupProfilesViaBird(targets);
+		for (const item of birdResults) {
+			const handle = normalizeHandle(item.target);
+			if (item.user) {
+				const resolved = upsertProfileFromXUser(getNativeDb(), item.user);
+				updateConversationTitles(resolved.profile);
+				results.set(handle, {
+					handle,
+					status: "hit",
+					source: "bird",
+					profile: resolved.profile,
+				});
+			} else if (item.error && !xurlFallback) {
+				results.set(handle, {
+					handle,
+					status: "error",
+					source: "bird",
+					error: item.error,
+				});
+			}
+		}
+		unresolved = targets.filter((handle) => !results.has(handle));
+	} catch (error) {
+		if (!xurlFallback) {
+			for (const handle of targets) {
+				results.set(handle, {
+					handle,
+					status: "error",
+					source: "bird",
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+			unresolved = [];
+		}
+	}
+
+	if (unresolved.length > 0 && xurlFallback) {
+		try {
+			const users = await lookupUsersByHandles(unresolved);
+			const usersByHandle = new Map(
+				users.map((user) => [
+					normalizeHandle(String(user.username ?? "")),
+					user,
+				]),
+			);
+			for (const handle of unresolved) {
+				const user = usersByHandle.get(handle);
+				if (user) {
+					const resolved = upsertProfileFromXUser(getNativeDb(), user);
+					updateConversationTitles(resolved.profile);
+					results.set(handle, {
+						handle,
+						status: "hit",
+						source: "xurl",
+						profile: resolved.profile,
+					});
+				} else {
+					results.set(handle, {
+						handle,
+						status: "miss",
+						source: "xurl",
+					});
+				}
+			}
+		} catch (error) {
+			for (const handle of unresolved) {
+				results.set(handle, {
+					handle,
+					status: "error",
+					source: "xurl",
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+		}
+	}
+
+	return targets.map(
+		(handle) =>
+			results.get(handle) ?? {
+				handle,
+				status: "miss",
+				source: "bird",
+			},
+	);
+}
+
 export async function resolvePlaceholderProfiles(
 	options: ResolveProfilesOptions & { limit?: number } = {},
 ) {

package/src/lib/queries.ts

@@ -3,6 +3,7 @@ import type { Database } from "./sqlite";
 import { findArchives } from "./archive-finder";
 import { getDb, getNativeDb } from "./db";
 import { fetchProfileAffiliations } from "./profile-affiliations";
+import { displayUrlForLink, enrichFallbackUrlEntities } from "./tweet-render";
 import type {
 	AccountRecord,
 	DmConversationItem,
@@ -17,7 +18,9 @@ import type {
 	TimelineItem,
 	TimelineQuery,
 	TweetEntities,
+	TweetConversationResponse,
 	TweetMediaItem,
+	TweetUrlEntity,
 } from "./types";
 import {
 	dmViaXurl,
@@ -107,7 +110,78 @@ function enrichEntities(
 	};
 }
 
+type UrlExpansionCache = Map<
+	string,
+	| (Pick<TweetUrlEntity, "expandedUrl" | "displayUrl"> &
+			Partial<
+				Pick<TweetUrlEntity, "title" | "description" | "imageUrl" | "siteName">
+			>)
+	| null
+>;
+
+function getUrlExpansion(
+	db: Database,
+	cache: UrlExpansionCache,
+	rawUrl: string,
+) {
+	if (cache.has(rawUrl)) {
+		return cache.get(rawUrl);
+	}
+
+	const row = db
+		.prepare(
+			`
+      select expanded_url, final_url, title, description, image_url, site_name
+      from url_expansions
+      where short_url = ?
+        and status = 'hit'
+      `,
+		)
+		.get(rawUrl) as
+		| {
+				expanded_url: string;
+				final_url: string;
+				title: string | null;
+				description: string | null;
+				image_url: string | null;
+				site_name: string | null;
+		  }
+		| undefined;
+	if (!row) {
+		cache.set(rawUrl, null);
+		return null;
+	}
+
+	const expandedUrl = row.final_url || row.expanded_url || rawUrl;
+	const expansion = {
+		expandedUrl,
+		displayUrl: displayUrlForLink(expandedUrl),
+		...(row.title ? { title: row.title } : {}),
+		...(row.description ? { description: row.description } : {}),
+		...(row.image_url ? { imageUrl: row.image_url } : {}),
+		...(row.site_name ? { siteName: row.site_name } : {}),
+	};
+	cache.set(rawUrl, expansion);
+	return expansion;
+}
+
+function enrichTimelineEntities(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
+	text: string,
+	entities: TweetEntities,
+	profiles: Record<string, ProfileRecord>,
+): TweetEntities {
+	return enrichFallbackUrlEntities(
+		text,
+		enrichEntities(entities, profiles),
+		(rawUrl) => getUrlExpansion(db, urlExpansionCache, rawUrl),
+	);
+}
+
 function buildEmbeddedTweet(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
 	row: Record<string, unknown>,
 	prefix: string,
 ): EmbeddedTweet | null {
@@ -127,12 +201,20 @@ function buildEmbeddedTweet(
 		created_at: row[`${prefix}profile_created_at`],
 	});
 
+	const text = String(row[`${prefix}text`] ?? "");
 	return {
 		id: String(row[`${prefix}id`]),
-		text: String(row[`${prefix}text`] ?? ""),
+		text,
 		createdAt: String(row[`${prefix}created_at`] ?? new Date(0).toISOString()),
+		replyToId:
+			typeof row[`${prefix}reply_to_id`] === "string"
+				? String(row[`${prefix}reply_to_id`])
+				: null,
 		author,
-		entities: enrichEntities(
+		entities: enrichTimelineEntities(
+			db,
+			urlExpansionCache,
+			text,
 			parseJsonField<TweetEntities>(row[`${prefix}entities_json`], {}),
 			{
 				[author.id]: author,
@@ -461,6 +543,7 @@ export function listTimelineItems({
         e.kind,
         t.text,
         t.created_at,
+        t.reply_to_id,
         t.is_replied,
         t.like_count,
         t.media_count,
@@ -503,6 +586,7 @@ export function listTimelineItems({
         rt.id as reply_id,
         rt.text as reply_text,
         rt.created_at as reply_created_at,
+        rt.reply_to_id as reply_reply_to_id,
         rt.entities_json as reply_entities_json,
         rt.media_json as reply_media_json,
         rp.id as reply_profile_id,
@@ -517,6 +601,7 @@ export function listTimelineItems({
         qt.id as quoted_id,
         qt.text as quoted_text,
         qt.created_at as quoted_created_at,
+        qt.reply_to_id as quoted_reply_to_id,
         qt.entities_json as quoted_entities_json,
         qt.media_json as quoted_media_json,
         qp.id as quoted_profile_id,
@@ -545,6 +630,7 @@ export function listTimelineItems({
 		)
 		.all(...params) as Array<Record<string, unknown>>;
 
+	const urlExpansionCache: UrlExpansionCache = new Map();
 	return rows.map((row) => {
 		const author = {
 			id: String(row.profile_id),
@@ -558,7 +644,11 @@ export function listTimelineItems({
 				typeof row.avatar_url === "string" ? String(row.avatar_url) : undefined,
 			createdAt: String(row.profile_created_at),
 		};
-		const entities = enrichEntities(
+		const text = String(row.text);
+		const entities = enrichTimelineEntities(
+			db,
+			urlExpansionCache,
+			text,
 			parseJsonField<TweetEntities>(row.entities_json, {}),
 			{
 				[author.id]: author,
@@ -599,11 +689,13 @@ export function listTimelineItems({
 			accountId: String(row.account_id),
 			accountHandle: String(row.account_handle),
 			kind: row.kind as TimelineItem["kind"],
-			text: String(row.text),
+			text,
 			...(typeof row.search_snippet === "string"
 				? { searchSnippet: row.search_snippet }
 				: {}),
 			createdAt: String(row.created_at),
+			replyToId:
+				typeof row.reply_to_id === "string" ? String(row.reply_to_id) : null,
 			isReplied: Boolean(row.is_replied),
 			likeCount: Number(row.like_count),
 			mediaCount: Number(row.media_count),
@@ -612,8 +704,8 @@ export function listTimelineItems({
 			author,
 			entities,
 			media: parseJsonField<TweetMediaItem[]>(row.media_json, []),
-			replyToTweet: buildEmbeddedTweet(row, "reply_"),
-			quotedTweet: buildEmbeddedTweet(row, "quoted_"),
+			replyToTweet: buildEmbeddedTweet(db, urlExpansionCache, row, "reply_"),
+			quotedTweet: buildEmbeddedTweet(db, urlExpansionCache, row, "quoted_"),
 		};
 		return includeQualityReason
 			? {
@@ -627,6 +719,140 @@ export function listTimelineItems({
 	});
 }
 
+const conversationTweetSelect = `
+  select
+    t.id,
+    t.text,
+    t.created_at,
+    t.reply_to_id,
+    t.entities_json,
+    t.media_json,
+    p.id as profile_id,
+    p.handle,
+    p.display_name,
+    p.bio,
+    p.followers_count,
+    p.following_count,
+    p.avatar_hue,
+    p.avatar_url,
+    p.created_at as profile_created_at
+  from tweets t
+  join profiles p on p.id = t.author_profile_id
+`;
+
+function getTweetById(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
+	tweetId: string,
+): EmbeddedTweet | null {
+	const row = db
+		.prepare(`${conversationTweetSelect} where t.id = ?`)
+		.get(tweetId) as Record<string, unknown> | undefined;
+	if (!row) return null;
+	return buildEmbeddedTweet(db, urlExpansionCache, row, "");
+}
+
+function listTweetDescendants(
+	db: Database,
+	urlExpansionCache: UrlExpansionCache,
+	rootId: string,
+	limit: number,
+) {
+	if (limit <= 0) return [];
+	const rows = db
+		.prepare(
+			`
+      with recursive branch(id, depth) as (
+        select t.id, 0
+        from tweets t
+        where t.id = ?
+        union all
+        select child.id, branch.depth + 1
+        from tweets child
+        join branch on child.reply_to_id = branch.id
+        where branch.depth < 8
+      )
+      ${conversationTweetSelect}
+      join branch on branch.id = t.id
+      where t.id != ?
+      order by t.created_at asc
+      limit ?
+      `,
+		)
+		.all(rootId, rootId, limit) as Array<Record<string, unknown>>;
+
+	return rows
+		.map((row) => buildEmbeddedTweet(db, urlExpansionCache, row, ""))
+		.filter((tweet): tweet is EmbeddedTweet => Boolean(tweet));
+}
+
+function appendConversationTweets(
+	target: EmbeddedTweet[],
+	seen: Set<string>,
+	items: EmbeddedTweet[],
+	remaining: number,
+) {
+	for (const tweet of items) {
+		if (target.length >= remaining || seen.has(tweet.id)) continue;
+		seen.add(tweet.id);
+		target.push(tweet);
+	}
+}
+
+export function getTweetConversation(
+	tweetId: string,
+	limit = 80,
+): TweetConversationResponse | null {
+	const db = getNativeDb();
+	const urlExpansionCache: UrlExpansionCache = new Map();
+	const anchor = getTweetById(db, urlExpansionCache, tweetId);
+	if (!anchor) return null;
+
+	const ancestors: EmbeddedTweet[] = [];
+	let current = anchor;
+	for (let depth = 0; depth < 12 && current.replyToId; depth += 1) {
+		const parent = getTweetById(db, urlExpansionCache, current.replyToId);
+		if (!parent || ancestors.some((tweet) => tweet.id === parent.id)) break;
+		ancestors.push(parent);
+		current = parent;
+	}
+
+	const required = [...ancestors].reverse();
+	required.push(anchor);
+	const root = required[0] ?? anchor;
+	const seen = new Set<string>();
+	const items = required.filter((tweet) => {
+		if (seen.has(tweet.id)) return false;
+		seen.add(tweet.id);
+		return true;
+	});
+	const remainingAfterRequired = Math.max(0, limit - items.length);
+	const focusedDescendants = listTweetDescendants(
+		db,
+		urlExpansionCache,
+		anchor.id,
+		remainingAfterRequired,
+	);
+	appendConversationTweets(items, seen, focusedDescendants, limit);
+
+	if (items.length < limit && root.id !== anchor.id) {
+		const ambientDescendants = listTweetDescendants(
+			db,
+			urlExpansionCache,
+			root.id,
+			limit,
+		);
+		appendConversationTweets(items, seen, ambientDescendants, limit);
+	}
+
+	items.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+
+	return {
+		anchorId: anchor.id,
+		items,
+	};
+}
+
 export function listDmConversations({
 	account,
 	conversationIds,
@@ -1084,7 +1310,7 @@ function getDmSearchMatches({
 }
 
 export function queryResource(
-	resource: "home" | "mentions" | "dms",
+	resource: "home" | "mentions" | "authored" | "dms",
 	filters: (TimelineQuery | DmQuery) & { conversationId?: string },
 ): QueryResponse {
 	if (resource === "dms") {

package/src/lib/seed.ts

@@ -32,6 +32,9 @@ export function seedDemoData(db: Database) {
 	if (accountCount.count > 0) {
 		return;
 	}
+	const linkNow = new Date();
+	const linkMinutesAgo = (minutes: number) =>
+		new Date(linkNow.getTime() - minutes * 60_000).toISOString();
 
 	const insertAccount = db.prepare(`
     insert into accounts (id, name, handle, external_user_id, transport, is_default, created_at)
@@ -75,6 +78,24 @@ export function seedDemoData(db: Database) {
 	const insertDmFts = db.prepare(
 		"insert into dm_fts (message_id, text) values (?, ?)",
 	);
+	const insertUrlExpansion = db.prepare(`
+    insert into url_expansions (
+      short_url, expanded_url, final_url, status, expanded_tweet_id,
+      expanded_handle, title, description, image_url, site_name, error, source, updated_at
+    ) values (
+      @shortUrl, @expandedUrl, @finalUrl, @status, @expandedTweetId,
+      @expandedHandle, @title, @description, @imageUrl, @siteName, @error, @source, @updatedAt
+    )
+  `);
+	const insertLinkOccurrence = db.prepare(`
+    insert into link_occurrences (
+      source_kind, source_id, source_position, short_url, account_id,
+      conversation_id, direction, created_at
+    ) values (
+      @sourceKind, @sourceId, @sourcePosition, @shortUrl, @accountId,
+      @conversationId, @direction, @createdAt
+    )
+  `);
 
 	const accounts = [
 		{
@@ -172,8 +193,8 @@ export function seedDemoData(db: Database) {
 			accountId: "acct_primary",
 			authorProfileId: "profile_sam",
 			kind: "home",
-			text: "We need more software that defaults to local-first, legible state, and repairable failure modes.",
-			createdAt: isoMinutesAgo(18),
+			text: "We need more software that defaults to local-first, legible state, and repairable failure modes. https://t.co/local",
+			createdAt: linkMinutesAgo(18),
 			isReplied: 0,
 			replyToId: null,
 			likeCount: 1240,
@@ -186,8 +207,8 @@ export function seedDemoData(db: Database) {
 						url: "https://t.co/local",
 						expandedUrl: "https://birdclaw.dev/local-first-systems",
 						displayUrl: "birdclaw.dev/local-first-systems",
-						start: 85,
-						end: 108,
+						start: 97,
+						end: 115,
 						title: "Local-first systems",
 						description: "Design notes on durable local software.",
 					},
@@ -227,8 +248,8 @@ export function seedDemoData(db: Database) {
 			accountId: "acct_primary",
 			authorProfileId: "profile_ava",
 			kind: "home",
-			text: "New developer-platform pricing survey out today. Early signal: teams want fewer layers, not more.",
-			createdAt: isoMinutesAgo(91),
+			text: "New developer-platform pricing survey out today. Early signal: teams want fewer layers, not more. https://t.co/survey https://t.co/video",
+			createdAt: linkMinutesAgo(91),
 			isReplied: 0,
 			replyToId: null,
 			likeCount: 128,
@@ -241,12 +262,21 @@ export function seedDemoData(db: Database) {
 						url: "https://t.co/survey",
 						expandedUrl: "https://example.com/developer-platform-pricing",
 						displayUrl: "example.com/developer-platform-pricing",
-						start: 78,
-						end: 101,
+						start: 98,
+						end: 117,
 						title: "Developer platform pricing survey",
 						description:
 							"A simple inline link preview card from tweet URL entities.",
 					},
+					{
+						url: "https://t.co/video",
+						expandedUrl: "https://youtu.be/GMIWm5y90xA",
+						displayUrl: "youtu.be/GMIWm5y90xA",
+						start: 118,
+						end: 136,
+						title: "Agent query walkthrough",
+						description: "Short demo video for local query workflows.",
+					},
 				],
 			}),
 			mediaJson: JSON.stringify([
@@ -466,6 +496,87 @@ export function seedDemoData(db: Database) {
 		},
 	];
 
+	const urlExpansions = [
+		{
+			shortUrl: "https://t.co/local",
+			expandedUrl: "https://birdclaw.dev/local-first-systems",
+			finalUrl: "https://birdclaw.dev/local-first-systems",
+			status: "hit",
+			expandedTweetId: null,
+			expandedHandle: null,
+			title: "Local-first systems",
+			description: "Design notes on durable local software.",
+			imageUrl: null,
+			siteName: "birdclaw",
+			error: null,
+			source: "demo",
+			updatedAt: linkNow.toISOString(),
+		},
+		{
+			shortUrl: "https://t.co/survey",
+			expandedUrl: "https://example.com/developer-platform-pricing",
+			finalUrl: "https://example.com/developer-platform-pricing",
+			status: "hit",
+			expandedTweetId: null,
+			expandedHandle: null,
+			title: "Developer platform pricing survey",
+			description: "A simple inline link preview card from tweet URL entities.",
+			imageUrl: null,
+			siteName: "Example",
+			error: null,
+			source: "demo",
+			updatedAt: linkNow.toISOString(),
+		},
+		{
+			shortUrl: "https://t.co/video",
+			expandedUrl: "https://youtu.be/GMIWm5y90xA",
+			finalUrl: "https://youtu.be/GMIWm5y90xA",
+			status: "hit",
+			expandedTweetId: null,
+			expandedHandle: null,
+			title: "Agent query walkthrough",
+			description: "Short demo video for local query workflows.",
+			imageUrl: null,
+			siteName: "YouTube",
+			error: null,
+			source: "demo",
+			updatedAt: linkNow.toISOString(),
+		},
+	];
+
+	const linkOccurrences = [
+		{
+			sourceKind: "tweet",
+			sourceId: "tweet_001",
+			sourcePosition: 0,
+			shortUrl: "https://t.co/local",
+			accountId: "acct_primary",
+			conversationId: null,
+			direction: null,
+			createdAt: linkMinutesAgo(18),
+		},
+		{
+			sourceKind: "tweet",
+			sourceId: "tweet_003",
+			sourcePosition: 0,
+			shortUrl: "https://t.co/survey",
+			accountId: "acct_primary",
+			conversationId: null,
+			direction: null,
+			createdAt: linkMinutesAgo(91),
+		},
+		{
+			sourceKind: "tweet",
+			sourceId: "tweet_003",
+			sourcePosition: 1,
+			shortUrl: "https://t.co/video",
+			accountId: "acct_primary",
+			conversationId: null,
+			direction: null,
+			createdAt: linkMinutesAgo(91),
+		},
+	];
+
 	const transaction = db.transaction(() => {
 		for (const account of accounts) {
 			insertAccount.run(account);
@@ -488,6 +599,14 @@ export function seedDemoData(db: Database) {
 			insertMessage.run(message);
 			insertDmFts.run(message.id, message.text);
 		}
+
+		for (const expansion of urlExpansions) {
+			insertUrlExpansion.run(expansion);
+		}
+
+		for (const occurrence of linkOccurrences) {
+			insertLinkOccurrence.run(occurrence);
+		}
 	});
 
 	transaction();