birdclaw 0.4.1 → 0.5.0

package/src/lib/timeline-collections-live.ts

@@ -1,10 +1,12 @@
 import type { Database } from "./sqlite";
 import { listBookmarkedTweetsViaBird, listLikedTweetsViaBird } from "./bird";
 import { getNativeDb } from "./db";
+import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
+	XurlMediaItem,
 	XurlMentionUser,
 } from "./types";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
@@ -18,6 +20,7 @@ export type TimelineCollectionKind = "likes" | "bookmarks";
 export type TimelineCollectionMode = "auto" | "xurl" | "bird";
 
 const DEFAULT_COLLECTION_CACHE_TTL_MS = 2 * 60_000;
+const DEFAULT_EARLY_STOP_MAX_PAGES = 10;
 const MIN_XURL_LIMIT = 5;
 const MAX_XURL_LIMIT = 100;
 
@@ -87,16 +90,6 @@ function resolveAccount(db: Database, accountId?: string) {
 	};
 }
 
-function getMediaCount(tweet: XurlMentionData) {
-	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
-	return urls.filter(
-		(url) =>
-			url &&
-			typeof url === "object" &&
-			typeof (url as Record<string, unknown>).media_key === "string",
-	).length;
-}
-
 function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
@@ -116,6 +109,8 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 	const seenTweetIds = new Set<string>();
 	const users: XurlMentionUser[] = [];
 	const seenUserIds = new Set<string>();
+	const media: XurlMediaItem[] = [];
+	const seenMediaKeys = new Set<string>();
 
 	for (const page of pages) {
 		for (const tweet of page.data) {
@@ -133,12 +128,24 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 			seenUserIds.add(user.id);
 			users.push(user);
 		}
+
+		for (const item of page.includes?.media ?? []) {
+			if (seenMediaKeys.has(item.media_key)) {
+				continue;
+			}
+			seenMediaKeys.add(item.media_key);
+			media.push(item);
+		}
 	}
 
 	const lastPage = pages.at(-1);
+	const includes = {
+		...(users.length > 0 ? { users } : {}),
+		...(media.length > 0 ? { media } : {}),
+	};
 	return {
 		data: tweets,
-		includes: users.length > 0 ? { users } : undefined,
+		includes: users.length > 0 || media.length > 0 ? includes : undefined,
 		meta: {
 			result_count: tweets.length,
 			page_count: pages.length,
@@ -149,6 +156,52 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 	};
 }
 
+function getCollectionPageDedupe(
+	db: Database,
+	accountId: string,
+	kind: TimelineCollectionKind,
+	tweetIds: string[],
+) {
+	const uniqueTweetIds = [...new Set(tweetIds)];
+	if (uniqueTweetIds.length === 0) {
+		return { existingTweetIds: new Set<string>(), uniqueTweetCount: 0 };
+	}
+
+	const rows = db
+		.prepare(
+			`
+      select tweet_id
+      from tweet_collections
+      where account_id = ?
+        and kind = ?
+        and tweet_id in (${uniqueTweetIds.map(() => "?").join(", ")})
+      `,
+		)
+		.all(accountId, kind, ...uniqueTweetIds) as { tweet_id: string }[];
+	return {
+		existingTweetIds: new Set(rows.map((row) => row.tweet_id)),
+		uniqueTweetCount: uniqueTweetIds.length,
+	};
+}
+
+function filterExistingCollectionTweets(
+	payload: XurlMentionsResponse,
+	existingTweetIds: Set<string>,
+) {
+	if (existingTweetIds.size === 0) {
+		return payload;
+	}
+	return {
+		...payload,
+		data: payload.data.filter((tweet) => !existingTweetIds.has(tweet.id)),
+	};
+}
+
+function readSaturatedAtPage(payload: XurlMentionsResponse) {
+	const value = payload.meta?.saturated_at_page;
+	return typeof value === "number" ? value : undefined;
+}
+
 function mergeTimelineCollectionIntoLocalStore(
 	db: Database,
 	accountId: string,
@@ -168,20 +221,23 @@ function mergeTimelineCollectionIntoLocalStore(
       id, account_id, author_profile_id, kind, text, created_at,
       is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
       entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?)
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
       account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
       kind = case
-        when tweets.kind in ('home', 'mention') then tweets.kind
+        when tweets.kind in ('authored', 'home', 'mention') then tweets.kind
         else excluded.kind
       end,
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
-      media_count = excluded.media_count,
+      media_count = max(tweets.media_count, excluded.media_count),
       entities_json = excluded.entities_json,
-      media_json = excluded.media_json,
+      media_json = case
+        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
+        else tweets.media_json
+      end,
       is_replied = max(tweets.is_replied, excluded.is_replied),
       reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
       quoted_tweet_id = coalesce(excluded.quoted_tweet_id, tweets.quoted_tweet_id),
@@ -224,10 +280,11 @@ function mergeTimelineCollectionIntoLocalStore(
 				replyToId ? 1 : 0,
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
-				getMediaCount(tweet),
+				countTweetMedia(tweet),
 				bookmarked,
 				liked,
 				JSON.stringify(tweet.entities ?? {}),
+				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);
 			upsertCollection.run(
@@ -244,19 +301,25 @@ function mergeTimelineCollectionIntoLocalStore(
 }
 
 async function fetchXurlCollection({
+	db,
 	kind,
+	accountId,
 	username,
 	userId,
 	limit,
 	all,
 	maxPages,
+	earlyStop,
 }: {
+	db: Database;
 	kind: TimelineCollectionKind;
+	accountId: string;
 	username: string;
 	userId?: string;
 	limit: number;
 	all: boolean;
 	maxPages: number | null;
+	earlyStop: boolean;
 }) {
 	let resolvedUserId = userId;
 	if (!resolvedUserId) {
@@ -270,6 +333,7 @@ async function fetchXurlCollection({
 	const pages: XurlMentionsResponse[] = [];
 	let nextToken: string | undefined;
 	let pageCount = 0;
+	let saturatedAtPage: number | undefined;
 	do {
 		const payload =
 			kind === "likes"
@@ -283,17 +347,51 @@ async function fetchXurlCollection({
 						maxResults: limit,
 						username,
 						userId: resolvedUserId,
+						isPaginatedWalk: all,
 						paginationToken: nextToken,
 					});
-		pages.push(payload);
+		pageCount += 1;
+		if (earlyStop) {
+			const tweetIds = payload.data.map((tweet) => tweet.id);
+			const { existingTweetIds, uniqueTweetCount } = getCollectionPageDedupe(
+				db,
+				accountId,
+				kind,
+				tweetIds,
+			);
+			if (tweetIds.length > 0 && existingTweetIds.size === uniqueTweetCount) {
+				saturatedAtPage = pageCount;
+				console.error(
+					`${kind} saturated at page ${pageCount} (100% existing rows)`,
+				);
+				break;
+			}
+			pages.push(filterExistingCollectionTweets(payload, existingTweetIds));
+		} else {
+			pages.push(payload);
+		}
 		nextToken =
 			typeof payload.meta?.next_token === "string"
 				? payload.meta.next_token
 				: undefined;
-		pageCount += 1;
-	} while (all && nextToken && (maxPages === null || pageCount < maxPages));
+	} while (
+		(all || earlyStop) &&
+		nextToken &&
+		(maxPages === null || pageCount < maxPages)
+	);
 
-	return mergePayloads(pages);
+	const merged = mergePayloads(pages);
+	// A saturated page may expose another token, but our walk is complete.
+	const saturationMeta =
+		saturatedAtPage === undefined
+			? {}
+			: { saturated_at_page: saturatedAtPage, next_token: null };
+	merged.meta = {
+		...merged.meta,
+		page_count: pageCount,
+		...saturationMeta,
+	};
+	return merged;
 }
 
 async function fetchBirdCollection({
@@ -329,6 +427,7 @@ export async function syncTimelineCollection({
 	maxPages,
 	refresh = false,
 	cacheTtlMs,
+	earlyStop = false,
 }: {
 	kind: TimelineCollectionKind;
 	account?: string;
@@ -338,16 +437,23 @@ export async function syncTimelineCollection({
 	maxPages?: number;
 	refresh?: boolean;
 	cacheTtlMs?: number;
+	earlyStop?: boolean;
 }) {
 	assertLimit(limit);
 	const parsedMaxPages = parseMaxPages(maxPages);
+	const shouldApplyEarlyStopCap =
+		earlyStop && !all && parsedMaxPages === null && mode !== "bird";
+	const xurlMaxPages = shouldApplyEarlyStopCap
+		? DEFAULT_EARLY_STOP_MAX_PAGES
+		: parsedMaxPages;
 	if (mode === "xurl" || mode === "auto") {
 		assertXurlLimit(limit);
 	}
 
 	const db = getNativeDb();
 	const resolvedAccount = resolveAccount(db, account);
-	const cacheKey = `${kind}:${mode}:${resolvedAccount.accountId}:${String(limit)}:${all ? "all" : "single"}:${parsedMaxPages === null ? "all-pages" : String(parsedMaxPages)}`;
+	const cacheMaxPages = mode === "bird" ? parsedMaxPages : xurlMaxPages;
+	const cacheKey = `${kind}:${mode}:${resolvedAccount.accountId}:${String(limit)}:${all ? "all" : "single"}:${cacheMaxPages === null ? "all-pages" : String(cacheMaxPages)}${earlyStop ? ":early-stop" : ""}`;
 	const ttlMs = parseCacheTtlMs(cacheTtlMs);
 	const cached = readSyncCache<XurlMentionsResponse>(cacheKey, db);
 	const cacheAgeMs = cached
@@ -355,6 +461,7 @@ export async function syncTimelineCollection({
 		: Number.POSITIVE_INFINITY;
 
 	if (!refresh && cached && cacheAgeMs <= ttlMs) {
+		const saturatedAtPage = readSaturatedAtPage(cached.value);
 		return {
 			ok: true,
 			source: "cache",
@@ -362,9 +469,18 @@ export async function syncTimelineCollection({
 			accountId: resolvedAccount.accountId,
 			count: cached.value.data.length,
 			payload: cached.value,
+			...(saturatedAtPage === undefined
+				? {}
+				: { saturated_at_page: saturatedAtPage }),
 		};
 	}
 
+	if (shouldApplyEarlyStopCap) {
+		console.error(
+			`${kind} early-stop capped at ${DEFAULT_EARLY_STOP_MAX_PAGES} pages by default; pass --max-pages or --all to override`,
+		);
+	}
+
 	let source: "xurl" | "bird";
 	let payload: XurlMentionsResponse;
 	if (mode === "bird") {
@@ -378,12 +494,15 @@ export async function syncTimelineCollection({
 	} else {
 		try {
 			payload = await fetchXurlCollection({
+				db,
 				kind,
+				accountId: resolvedAccount.accountId,
 				username: resolvedAccount.username,
 				userId: resolvedAccount.externalUserId,
 				limit,
 				all,
-				maxPages: parsedMaxPages,
+				maxPages: xurlMaxPages,
+				earlyStop,
 			});
 			source = "xurl";
 		} catch (error) {
@@ -408,6 +527,7 @@ export async function syncTimelineCollection({
 		source,
 	);
 	writeSyncCache(cacheKey, payload, db);
+	const saturatedAtPage = readSaturatedAtPage(payload);
 
 	return {
 		ok: true,
@@ -416,5 +536,8 @@ export async function syncTimelineCollection({
 		accountId: resolvedAccount.accountId,
 		count: payload.data.length,
 		payload,
+		...(saturatedAtPage === undefined
+			? {}
+			: { saturated_at_page: saturatedAtPage }),
 	};
 }

package/src/lib/timeline-live.ts

@@ -1,8 +1,9 @@
 import type { Database } from "./sqlite";
 import { listHomeTimelineViaBird } from "./bird";
 import { getNativeDb } from "./db";
+import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
-import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import type { XurlMentionsResponse } from "./types";
 import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
 
@@ -44,16 +45,6 @@ function resolveAccount(db: Database, accountId?: string) {
 	return row.id;
 }
 
-function getMediaCount(tweet: XurlMentionData) {
-	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
-	return urls.filter(
-		(url) =>
-			url &&
-			typeof url === "object" &&
-			typeof (url as Record<string, unknown>).media_key === "string",
-	).length;
-}
-
 function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
@@ -76,7 +67,7 @@ function mergeHomeTimelineIntoLocalStore(
       id, account_id, author_profile_id, kind, text, created_at,
       is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
       entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
+    ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, ?, null)
     on conflict(id) do update set
       account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
@@ -84,9 +75,12 @@ function mergeHomeTimelineIntoLocalStore(
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
-      media_count = excluded.media_count,
+      media_count = max(tweets.media_count, excluded.media_count),
       entities_json = excluded.entities_json,
-      media_json = excluded.media_json,
+      media_json = case
+        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
+        else tweets.media_json
+      end,
       bookmarked = tweets.bookmarked,
       liked = tweets.liked
     `,
@@ -112,8 +106,9 @@ function mergeHomeTimelineIntoLocalStore(
 				tweet.text,
 				tweet.created_at,
 				Number(tweet.public_metrics?.like_count ?? 0),
-				getMediaCount(tweet),
+				countTweetMedia(tweet),
 				JSON.stringify(tweet.entities ?? {}),
+				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 			);
 			upsertTweetAccountEdge(db, {
 				accountId,

package/src/lib/tweet-account-edges.ts

@@ -1,6 +1,10 @@
 import type { Database } from "./sqlite";
 
-export type TweetAccountEdgeKind = "home" | "mention";
+export type TweetAccountEdgeKind =
+	| "home"
+	| "mention"
+	| "authored"
+	| "thread_context";
 
 export function upsertTweetAccountEdge(
 	db: Database,
@@ -29,7 +33,10 @@ export function upsertTweetAccountEdge(
       first_seen_at = min(tweet_account_edges.first_seen_at, excluded.first_seen_at),
       last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
       seen_count = tweet_account_edges.seen_count + 1,
-      source = coalesce(nullif(excluded.source, ''), tweet_account_edges.source),
+	      source = case
+	        when tweet_account_edges.source = 'archive' then 'archive'
+	        else coalesce(nullif(excluded.source, ''), tweet_account_edges.source)
+	      end,
       raw_json = case
         when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
         else tweet_account_edges.raw_json

package/src/lib/tweet-render.ts

@@ -10,6 +10,14 @@ type TweetSegment =
 	| ({ kind: "url" } & TweetUrlEntity)
 	| ({ kind: "hashtag" } & TweetHashtagEntity);
 
+type UrlExpansion = Pick<TweetUrlEntity, "expandedUrl" | "displayUrl"> &
+	Partial<
+		Pick<TweetUrlEntity, "title" | "description" | "imageUrl" | "siteName">
+	>;
+
+const RAW_URL_PATTERN = /https?:\/\/[^\s<>"'`]+/g;
+const TRAILING_URL_PUNCTUATION = /[),.;:!?]+$/;
+
 const MARKDOWN_ESCAPE_CHARACTERS = new Set([
 	"\\",
 	"`",
@@ -38,6 +46,95 @@ function escapeMarkdown(text: string) {
 		.join("");
 }
 
+export function displayUrlForLink(url: string) {
+	try {
+		const parsed = new URL(url);
+		const host = parsed.hostname.replace(/^www\./, "");
+		const suffix = `${parsed.pathname}${parsed.search}${parsed.hash}`;
+		return suffix === "/" ? host : `${host}${suffix}`;
+	} catch {
+		return url;
+	}
+}
+
+function spansOverlap(
+	leftStart: number,
+	leftEnd: number,
+	rightStart: number,
+	rightEnd: number,
+) {
+	return leftStart < rightEnd && rightStart < leftEnd;
+}
+
+export function enrichFallbackUrlEntities(
+	text: string,
+	entities: TweetEntities,
+	resolveExpansion?: (rawUrl: string) => UrlExpansion | null | undefined,
+): TweetEntities {
+	const existingUrls = entities.urls ?? [];
+	const enrichedExistingUrls = existingUrls.map((entry) => {
+		const expansion = resolveExpansion?.(entry.url);
+		if (!expansion) return entry;
+		const expandedUrl = expansion.expandedUrl || entry.expandedUrl;
+		return {
+			...entry,
+			expandedUrl,
+			displayUrl: expansion.displayUrl || displayUrlForLink(expandedUrl),
+			...(expansion.title ? { title: expansion.title } : {}),
+			...(expansion && "description" in expansion
+				? { description: expansion.description ?? null }
+				: {}),
+			...(expansion.imageUrl ? { imageUrl: expansion.imageUrl } : {}),
+			...(expansion.siteName ? { siteName: expansion.siteName } : {}),
+		};
+	});
+	const fallbackUrls: TweetUrlEntity[] = [];
+
+	for (const match of text.matchAll(RAW_URL_PATTERN)) {
+		const rawMatch = match[0];
+		const url = rawMatch.replace(TRAILING_URL_PUNCTUATION, "");
+		const start = match.index ?? 0;
+		const end = start + url.length;
+		if (!url) {
+			continue;
+		}
+		if (
+			enrichedExistingUrls.some((entry) =>
+				spansOverlap(start, end, entry.start, entry.end),
+			)
+		) {
+			continue;
+		}
+
+		const expansion = resolveExpansion?.(url);
+		const expandedUrl = expansion?.expandedUrl || url;
+		fallbackUrls.push({
+			url,
+			expandedUrl,
+			displayUrl: expansion?.displayUrl || displayUrlForLink(expandedUrl),
+			start,
+			end,
+			...(expansion?.title ? { title: expansion.title } : {}),
+			...(expansion && "description" in expansion
+				? { description: expansion.description ?? null }
+				: {}),
+			...(expansion?.imageUrl ? { imageUrl: expansion.imageUrl } : {}),
+			...(expansion?.siteName ? { siteName: expansion.siteName } : {}),
+		});
+	}
+
+	if (fallbackUrls.length === 0) {
+		return { ...entities, urls: enrichedExistingUrls };
+	}
+
+	return {
+		...entities,
+		urls: [...enrichedExistingUrls, ...fallbackUrls].sort(
+			(left, right) => left.start - right.start,
+		),
+	};
+}
+
 export function collectTweetSegments(entities: TweetEntities): TweetSegment[] {
 	return [
 		...(entities.mentions?.map((entry) => ({