birdclaw 0.4.1 → 0.5.1

package/src/lib/seed.ts

@@ -32,6 +32,9 @@ export function seedDemoData(db: Database) {
 	if (accountCount.count > 0) {
 		return;
 	}
+	const linkNow = new Date();
+	const linkMinutesAgo = (minutes: number) =>
+		new Date(linkNow.getTime() - minutes * 60_000).toISOString();
 
 	const insertAccount = db.prepare(`
     insert into accounts (id, name, handle, external_user_id, transport, is_default, created_at)
@@ -75,6 +78,24 @@ export function seedDemoData(db: Database) {
 	const insertDmFts = db.prepare(
 		"insert into dm_fts (message_id, text) values (?, ?)",
 	);
+	const insertUrlExpansion = db.prepare(`
+    insert into url_expansions (
+      short_url, expanded_url, final_url, status, expanded_tweet_id,
+      expanded_handle, title, description, image_url, site_name, error, source, updated_at
+    ) values (
+      @shortUrl, @expandedUrl, @finalUrl, @status, @expandedTweetId,
+      @expandedHandle, @title, @description, @imageUrl, @siteName, @error, @source, @updatedAt
+    )
+  `);
+	const insertLinkOccurrence = db.prepare(`
+    insert into link_occurrences (
+      source_kind, source_id, source_position, short_url, account_id,
+      conversation_id, direction, created_at
+    ) values (
+      @sourceKind, @sourceId, @sourcePosition, @shortUrl, @accountId,
+      @conversationId, @direction, @createdAt
+    )
+  `);
 
 	const accounts = [
 		{
@@ -172,8 +193,8 @@ export function seedDemoData(db: Database) {
 			accountId: "acct_primary",
 			authorProfileId: "profile_sam",
 			kind: "home",
-			text: "We need more software that defaults to local-first, legible state, and repairable failure modes.",
-			createdAt: isoMinutesAgo(18),
+			text: "We need more software that defaults to local-first, legible state, and repairable failure modes. https://t.co/local",
+			createdAt: linkMinutesAgo(18),
 			isReplied: 0,
 			replyToId: null,
 			likeCount: 1240,
@@ -186,8 +207,8 @@ export function seedDemoData(db: Database) {
 						url: "https://t.co/local",
 						expandedUrl: "https://birdclaw.dev/local-first-systems",
 						displayUrl: "birdclaw.dev/local-first-systems",
-						start: 85,
-						end: 108,
+						start: 97,
+						end: 115,
 						title: "Local-first systems",
 						description: "Design notes on durable local software.",
 					},
@@ -227,8 +248,8 @@ export function seedDemoData(db: Database) {
 			accountId: "acct_primary",
 			authorProfileId: "profile_ava",
 			kind: "home",
-			text: "New developer-platform pricing survey out today. Early signal: teams want fewer layers, not more.",
-			createdAt: isoMinutesAgo(91),
+			text: "New developer-platform pricing survey out today. Early signal: teams want fewer layers, not more. https://t.co/survey https://t.co/video",
+			createdAt: linkMinutesAgo(91),
 			isReplied: 0,
 			replyToId: null,
 			likeCount: 128,
@@ -241,12 +262,21 @@ export function seedDemoData(db: Database) {
 						url: "https://t.co/survey",
 						expandedUrl: "https://example.com/developer-platform-pricing",
 						displayUrl: "example.com/developer-platform-pricing",
-						start: 78,
-						end: 101,
+						start: 98,
+						end: 117,
 						title: "Developer platform pricing survey",
 						description:
 							"A simple inline link preview card from tweet URL entities.",
 					},
+					{
+						url: "https://t.co/video",
+						expandedUrl: "https://youtu.be/GMIWm5y90xA",
+						displayUrl: "youtu.be/GMIWm5y90xA",
+						start: 118,
+						end: 136,
+						title: "Agent query walkthrough",
+						description: "Short demo video for local query workflows.",
+					},
 				],
 			}),
 			mediaJson: JSON.stringify([
@@ -466,6 +496,87 @@ export function seedDemoData(db: Database) {
 		},
 	];
 
+	const urlExpansions = [
+		{
+			shortUrl: "https://t.co/local",
+			expandedUrl: "https://birdclaw.dev/local-first-systems",
+			finalUrl: "https://birdclaw.dev/local-first-systems",
+			status: "hit",
+			expandedTweetId: null,
+			expandedHandle: null,
+			title: "Local-first systems",
+			description: "Design notes on durable local software.",
+			imageUrl: null,
+			siteName: "birdclaw",
+			error: null,
+			source: "demo",
+			updatedAt: linkNow.toISOString(),
+		},
+		{
+			shortUrl: "https://t.co/survey",
+			expandedUrl: "https://example.com/developer-platform-pricing",
+			finalUrl: "https://example.com/developer-platform-pricing",
+			status: "hit",
+			expandedTweetId: null,
+			expandedHandle: null,
+			title: "Developer platform pricing survey",
+			description: "A simple inline link preview card from tweet URL entities.",
+			imageUrl: null,
+			siteName: "Example",
+			error: null,
+			source: "demo",
+			updatedAt: linkNow.toISOString(),
+		},
+		{
+			shortUrl: "https://t.co/video",
+			expandedUrl: "https://youtu.be/GMIWm5y90xA",
+			finalUrl: "https://youtu.be/GMIWm5y90xA",
+			status: "hit",
+			expandedTweetId: null,
+			expandedHandle: null,
+			title: "Agent query walkthrough",
+			description: "Short demo video for local query workflows.",
+			imageUrl: null,
+			siteName: "YouTube",
+			error: null,
+			source: "demo",
+			updatedAt: linkNow.toISOString(),
+		},
+	];
+
+	const linkOccurrences = [
+		{
+			sourceKind: "tweet",
+			sourceId: "tweet_001",
+			sourcePosition: 0,
+			shortUrl: "https://t.co/local",
+			accountId: "acct_primary",
+			conversationId: null,
+			direction: null,
+			createdAt: linkMinutesAgo(18),
+		},
+		{
+			sourceKind: "tweet",
+			sourceId: "tweet_003",
+			sourcePosition: 0,
+			shortUrl: "https://t.co/survey",
+			accountId: "acct_primary",
+			conversationId: null,
+			direction: null,
+			createdAt: linkMinutesAgo(91),
+		},
+		{
+			sourceKind: "tweet",
+			sourceId: "tweet_003",
+			sourcePosition: 1,
+			shortUrl: "https://t.co/video",
+			accountId: "acct_primary",
+			conversationId: null,
+			direction: null,
+			createdAt: linkMinutesAgo(91),
+		},
+	];
+
 	const transaction = db.transaction(() => {
 		for (const account of accounts) {
 			insertAccount.run(account);
@@ -488,6 +599,14 @@ export function seedDemoData(db: Database) {
 			insertMessage.run(message);
 			insertDmFts.run(message.id, message.text);
 		}
+
+		for (const expansion of urlExpansions) {
+			insertUrlExpansion.run(expansion);
+		}
+
+		for (const occurrence of linkOccurrences) {
+			insertLinkOccurrence.run(occurrence);
+		}
 	});
 
 	transaction();

package/src/lib/timeline-collections-live.ts

@@ -1,10 +1,12 @@
 import type { Database } from "./sqlite";
 import { listBookmarkedTweetsViaBird, listLikedTweetsViaBird } from "./bird";
 import { getNativeDb } from "./db";
+import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
+	XurlMediaItem,
 	XurlMentionUser,
 } from "./types";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
@@ -18,6 +20,7 @@ export type TimelineCollectionKind = "likes" | "bookmarks";
 export type TimelineCollectionMode = "auto" | "xurl" | "bird";
 
 const DEFAULT_COLLECTION_CACHE_TTL_MS = 2 * 60_000;
+const DEFAULT_EARLY_STOP_MAX_PAGES = 10;
 const MIN_XURL_LIMIT = 5;
 const MAX_XURL_LIMIT = 100;
 
@@ -87,16 +90,6 @@ function resolveAccount(db: Database, accountId?: string) {
 	};
 }
 
-function getMediaCount(tweet: XurlMentionData) {
-	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
-	return urls.filter(
-		(url) =>
-			url &&
-			typeof url === "object" &&
-			typeof (url as Record<string, unknown>).media_key === "string",
-	).length;
-}
-
 function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
@@ -116,6 +109,8 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 	const seenTweetIds = new Set<string>();
 	const users: XurlMentionUser[] = [];
 	const seenUserIds = new Set<string>();
+	const media: XurlMediaItem[] = [];
+	const seenMediaKeys = new Set<string>();
 
 	for (const page of pages) {
 		for (const tweet of page.data) {
@@ -133,12 +128,24 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 			seenUserIds.add(user.id);
 			users.push(user);
 		}
+
+		for (const item of page.includes?.media ?? []) {
+			if (seenMediaKeys.has(item.media_key)) {
+				continue;
+			}
+			seenMediaKeys.add(item.media_key);
+			media.push(item);
+		}
 	}
 
 	const lastPage = pages.at(-1);
+	const includes = {
+		...(users.length > 0 ? { users } : {}),
+		...(media.length > 0 ? { media } : {}),
+	};
 	return {
 		data: tweets,
-		includes: users.length > 0 ? { users } : undefined,
+		includes: users.length > 0 || media.length > 0 ? includes : undefined,
 		meta: {
 			result_count: tweets.length,
 			page_count: pages.length,
@@ -149,6 +156,52 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 	};
 }
 
+function getCollectionPageDedupe(
+	db: Database,
+	accountId: string,
+	kind: TimelineCollectionKind,
+	tweetIds: string[],
+) {
+	const uniqueTweetIds = [...new Set(tweetIds)];
+	if (uniqueTweetIds.length === 0) {
+		return { existingTweetIds: new Set<string>(), uniqueTweetCount: 0 };
+	}
+
+	const rows = db
+		.prepare(
+			`
+      select tweet_id
+      from tweet_collections
+      where account_id = ?
+        and kind = ?
+        and tweet_id in (${uniqueTweetIds.map(() => "?").join(", ")})
+      `,
+		)
+		.all(accountId, kind, ...uniqueTweetIds) as { tweet_id: string }[];
+	return {
+		existingTweetIds: new Set(rows.map((row) => row.tweet_id)),
+		uniqueTweetCount: uniqueTweetIds.length,
+	};
+}
+
+function filterExistingCollectionTweets(
+	payload: XurlMentionsResponse,
+	existingTweetIds: Set<string>,
+) {
+	if (existingTweetIds.size === 0) {
+		return payload;
+	}
+	return {
+		...payload,
+		data: payload.data.filter((tweet) => !existingTweetIds.has(tweet.id)),
+	};
+}
+
+function readSaturatedAtPage(payload: XurlMentionsResponse) {
+	const value = payload.meta?.saturated_at_page;
+	return typeof value === "number" ? value : undefined;
+}
+
 function mergeTimelineCollectionIntoLocalStore(
 	db: Database,
 	accountId: string,
@@ -168,20 +221,23 @@ function mergeTimelineCollectionIntoLocalStore(
       id, account_id, author_profile_id, kind, text, created_at,
       is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
       entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?)
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
       account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
       kind = case
-        when tweets.kind in ('home', 'mention') then tweets.kind
+        when tweets.kind in ('authored', 'home', 'mention') then tweets.kind
         else excluded.kind
       end,
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
-      media_count = excluded.media_count,
+      media_count = max(tweets.media_count, excluded.media_count),
       entities_json = excluded.entities_json,
-      media_json = excluded.media_json,
+      media_json = case
+        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
+        else tweets.media_json
+      end,
       is_replied = max(tweets.is_replied, excluded.is_replied),
       reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
       quoted_tweet_id = coalesce(excluded.quoted_tweet_id, tweets.quoted_tweet_id),
@@ -224,10 +280,11 @@ function mergeTimelineCollectionIntoLocalStore(
 				replyToId ? 1 : 0,
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
-				getMediaCount(tweet),
+				countTweetMedia(tweet),
 				bookmarked,
 				liked,
 				JSON.stringify(tweet.entities ?? {}),
+				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);
 			upsertCollection.run(
@@ -244,19 +301,25 @@ function mergeTimelineCollectionIntoLocalStore(
 }
 
 async function fetchXurlCollection({
+	db,
 	kind,
+	accountId,
 	username,
 	userId,
 	limit,
 	all,
 	maxPages,
+	earlyStop,
 }: {
+	db: Database;
 	kind: TimelineCollectionKind;
+	accountId: string;
 	username: string;
 	userId?: string;
 	limit: number;
 	all: boolean;
 	maxPages: number | null;
+	earlyStop: boolean;
 }) {
 	let resolvedUserId = userId;
 	if (!resolvedUserId) {
@@ -270,6 +333,7 @@ async function fetchXurlCollection({
 	const pages: XurlMentionsResponse[] = [];
 	let nextToken: string | undefined;
 	let pageCount = 0;
+	let saturatedAtPage: number | undefined;
 	do {
 		const payload =
 			kind === "likes"
@@ -283,17 +347,51 @@ async function fetchXurlCollection({
 						maxResults: limit,
 						username,
 						userId: resolvedUserId,
+						isPaginatedWalk: all,
 						paginationToken: nextToken,
 					});
-		pages.push(payload);
+		pageCount += 1;
+		if (earlyStop) {
+			const tweetIds = payload.data.map((tweet) => tweet.id);
+			const { existingTweetIds, uniqueTweetCount } = getCollectionPageDedupe(
+				db,
+				accountId,
+				kind,
+				tweetIds,
+			);
+			if (tweetIds.length > 0 && existingTweetIds.size === uniqueTweetCount) {
+				saturatedAtPage = pageCount;
+				console.error(
+					`${kind} saturated at page ${pageCount} (100% existing rows)`,
+				);
+				break;
+			}
+			pages.push(filterExistingCollectionTweets(payload, existingTweetIds));
+		} else {
+			pages.push(payload);
+		}
 		nextToken =
 			typeof payload.meta?.next_token === "string"
 				? payload.meta.next_token
 				: undefined;
-		pageCount += 1;
-	} while (all && nextToken && (maxPages === null || pageCount < maxPages));
+	} while (
+		(all || earlyStop) &&
+		nextToken &&
+		(maxPages === null || pageCount < maxPages)
+	);
 
-	return mergePayloads(pages);
+	const merged = mergePayloads(pages);
+	// A saturated page may expose another token, but our walk is complete.
+	const saturationMeta =
+		saturatedAtPage === undefined
+			? {}
+			: { saturated_at_page: saturatedAtPage, next_token: null };
+	merged.meta = {
+		...merged.meta,
+		page_count: pageCount,
+		...saturationMeta,
+	};
+	return merged;
 }
 
 async function fetchBirdCollection({
@@ -329,6 +427,7 @@ export async function syncTimelineCollection({
 	maxPages,
 	refresh = false,
 	cacheTtlMs,
+	earlyStop = false,
 }: {
 	kind: TimelineCollectionKind;
 	account?: string;
@@ -338,16 +437,23 @@ export async function syncTimelineCollection({
 	maxPages?: number;
 	refresh?: boolean;
 	cacheTtlMs?: number;
+	earlyStop?: boolean;
 }) {
 	assertLimit(limit);
 	const parsedMaxPages = parseMaxPages(maxPages);
+	const shouldApplyEarlyStopCap =
+		earlyStop && !all && parsedMaxPages === null && mode !== "bird";
+	const xurlMaxPages = shouldApplyEarlyStopCap
+		? DEFAULT_EARLY_STOP_MAX_PAGES
+		: parsedMaxPages;
 	if (mode === "xurl" || mode === "auto") {
 		assertXurlLimit(limit);
 	}
 
 	const db = getNativeDb();
 	const resolvedAccount = resolveAccount(db, account);
-	const cacheKey = `${kind}:${mode}:${resolvedAccount.accountId}:${String(limit)}:${all ? "all" : "single"}:${parsedMaxPages === null ? "all-pages" : String(parsedMaxPages)}`;
+	const cacheMaxPages = mode === "bird" ? parsedMaxPages : xurlMaxPages;
+	const cacheKey = `${kind}:${mode}:${resolvedAccount.accountId}:${String(limit)}:${all ? "all" : "single"}:${cacheMaxPages === null ? "all-pages" : String(cacheMaxPages)}${earlyStop ? ":early-stop" : ""}`;
 	const ttlMs = parseCacheTtlMs(cacheTtlMs);
 	const cached = readSyncCache<XurlMentionsResponse>(cacheKey, db);
 	const cacheAgeMs = cached
@@ -355,6 +461,7 @@ export async function syncTimelineCollection({
 		: Number.POSITIVE_INFINITY;
 
 	if (!refresh && cached && cacheAgeMs <= ttlMs) {
+		const saturatedAtPage = readSaturatedAtPage(cached.value);
 		return {
 			ok: true,
 			source: "cache",
@@ -362,9 +469,18 @@ export async function syncTimelineCollection({
 			accountId: resolvedAccount.accountId,
 			count: cached.value.data.length,
 			payload: cached.value,
+			...(saturatedAtPage === undefined
+				? {}
+				: { saturated_at_page: saturatedAtPage }),
 		};
 	}
 
+	if (shouldApplyEarlyStopCap) {
+		console.error(
+			`${kind} early-stop capped at ${DEFAULT_EARLY_STOP_MAX_PAGES} pages by default; pass --max-pages or --all to override`,
+		);
+	}
+
 	let source: "xurl" | "bird";
 	let payload: XurlMentionsResponse;
 	if (mode === "bird") {
@@ -378,12 +494,15 @@ export async function syncTimelineCollection({
 	} else {
 		try {
 			payload = await fetchXurlCollection({
+				db,
 				kind,
+				accountId: resolvedAccount.accountId,
 				username: resolvedAccount.username,
 				userId: resolvedAccount.externalUserId,
 				limit,
 				all,
-				maxPages: parsedMaxPages,
+				maxPages: xurlMaxPages,
+				earlyStop,
 			});
 			source = "xurl";
 		} catch (error) {
@@ -408,6 +527,7 @@ export async function syncTimelineCollection({
 		source,
 	);
 	writeSyncCache(cacheKey, payload, db);
+	const saturatedAtPage = readSaturatedAtPage(payload);
 
 	return {
 		ok: true,
@@ -416,5 +536,8 @@ export async function syncTimelineCollection({
 		accountId: resolvedAccount.accountId,
 		count: payload.data.length,
 		payload,
+		...(saturatedAtPage === undefined
+			? {}
+			: { saturated_at_page: saturatedAtPage }),
 	};
 }

package/src/lib/timeline-live.ts

@@ -1,8 +1,9 @@
 import type { Database } from "./sqlite";
 import { listHomeTimelineViaBird } from "./bird";
 import { getNativeDb } from "./db";
+import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
-import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import type { XurlMentionsResponse } from "./types";
 import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
 
@@ -44,16 +45,6 @@ function resolveAccount(db: Database, accountId?: string) {
 	return row.id;
 }
 
-function getMediaCount(tweet: XurlMentionData) {
-	const urls = Array.isArray(tweet.entities?.urls) ? tweet.entities.urls : [];
-	return urls.filter(
-		(url) =>
-			url &&
-			typeof url === "object" &&
-			typeof (url as Record<string, unknown>).media_key === "string",
-	).length;
-}
-
 function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
@@ -76,7 +67,7 @@ function mergeHomeTimelineIntoLocalStore(
       id, account_id, author_profile_id, kind, text, created_at,
       is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
       entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
+    ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, ?, null)
     on conflict(id) do update set
       account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
@@ -84,9 +75,12 @@ function mergeHomeTimelineIntoLocalStore(
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
-      media_count = excluded.media_count,
+      media_count = max(tweets.media_count, excluded.media_count),
       entities_json = excluded.entities_json,
-      media_json = excluded.media_json,
+      media_json = case
+        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
+        else tweets.media_json
+      end,
       bookmarked = tweets.bookmarked,
       liked = tweets.liked
     `,
@@ -112,8 +106,9 @@ function mergeHomeTimelineIntoLocalStore(
 				tweet.text,
 				tweet.created_at,
 				Number(tweet.public_metrics?.like_count ?? 0),
-				getMediaCount(tweet),
+				countTweetMedia(tweet),
 				JSON.stringify(tweet.entities ?? {}),
+				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 			);
 			upsertTweetAccountEdge(db, {
 				accountId,

package/src/lib/tweet-account-edges.ts

@@ -1,6 +1,10 @@
 import type { Database } from "./sqlite";
 
-export type TweetAccountEdgeKind = "home" | "mention";
+export type TweetAccountEdgeKind =
+	| "home"
+	| "mention"
+	| "authored"
+	| "thread_context";
 
 export function upsertTweetAccountEdge(
 	db: Database,
@@ -29,7 +33,10 @@ export function upsertTweetAccountEdge(
       first_seen_at = min(tweet_account_edges.first_seen_at, excluded.first_seen_at),
       last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
       seen_count = tweet_account_edges.seen_count + 1,
-      source = coalesce(nullif(excluded.source, ''), tweet_account_edges.source),
+	      source = case
+	        when tweet_account_edges.source = 'archive' then 'archive'
+	        else coalesce(nullif(excluded.source, ''), tweet_account_edges.source)
+	      end,
       raw_json = case
         when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
         else tweet_account_edges.raw_json