birdclaw 0.4.0 → 0.4.1

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+## 0.4.1 - 2026-05-11
+
+### Added
+
+- Add a first-class short-link index for `t.co` URLs, including `links backfill` and `search links` so DM shares can be found through expanded tweet text, authors, dates, and media filters.
+
 ## 0.4.0 - 2026-05-09
 
 ### Added

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "birdclaw",
-	"version": "0.4.0",
+	"version": "0.4.1",
 	"description": "Local Twitter memory in SQLite for archives, DMs, likes, bookmarks, and moderation",
 	"homepage": "https://github.com/steipete/birdclaw#readme",
 	"license": "MIT",

package/src/cli.ts

@@ -29,6 +29,7 @@ import {
 } from "#/lib/config";
 import { syncDirectMessagesViaCachedBird } from "#/lib/dms-live";
 import { listInboxItems, scoreInbox } from "#/lib/inbox";
+import { backfillLinkIndex, searchLinks } from "#/lib/link-index";
 import { syncMentionThreads } from "#/lib/mention-threads-live";
 import { exportMentionItems } from "#/lib/mentions-export";
 import {
@@ -73,6 +74,24 @@ function printError(error: string) {
 	console.error(JSON.stringify({ error }));
 }
 
+function formatLinkSearchItems(items: ReturnType<typeof searchLinks>) {
+	return items
+		.map((item) => {
+			const linked = item.linkedTweet
+				? ` -> @${item.linkedTweet.author.handle}/${item.linkedTweet.id}: ${item.linkedTweet.text}`
+				: ` -> ${item.expansion.finalUrl}`;
+			const source =
+				item.occurrence.sourceKind === "dm"
+					? `dm ${item.occurrence.direction ?? ""}`.trim()
+					: "tweet";
+			const participant = item.participant
+				? ` @${item.participant.handle}`
+				: "";
+			return `${item.occurrence.createdAt} ${source}${participant}: ${item.occurrence.shortUrl}${linked}`;
+		})
+		.join("\n");
+}
+
 function parseNonNegativeIntegerOption(
 	value: string | undefined,
 	option: string,
@@ -364,6 +383,100 @@ searchCommand
 		print(items, program.opts().json ?? false);
 	});
 
+searchCommand
+	.command("links <query>")
+	.description("Search indexed short links, expansions, and linked tweets")
+	.option("--account <accountIdOrHandle>", "Account id or handle")
+	.option("--since <date>", "Include links created at or after this date")
+	.option("--until <date>", "Include links created before this date")
+	.option("--source <kind>", "dm or tweet")
+	.option("--direction <direction>", "inbound or outbound")
+	.option("--participant <value>", "DM participant handle or name")
+	.option("--media <type>", "image, video, or gif")
+	.option("--limit <n>", "Limit results", "20")
+	.action(async (query, options) => {
+		await autoUpdateBeforeRead();
+		const items = searchLinks(query, {
+			account: options.account,
+			since: options.since,
+			until: options.until,
+			source:
+				options.source === "tweet"
+					? "tweet"
+					: options.source === "dm"
+						? "dm"
+						: undefined,
+			direction:
+				options.direction === "inbound"
+					? "inbound"
+					: options.direction === "outbound"
+						? "outbound"
+						: undefined,
+			participant: options.participant,
+			mediaType:
+				options.media === "image" ||
+				options.media === "video" ||
+				options.media === "gif"
+					? options.media
+					: undefined,
+			limit: Number(options.limit),
+		});
+		if (program.opts().json) {
+			print(items, true);
+			return;
+		}
+		console.log(formatLinkSearchItems(items));
+	});
+
+const linksCommand = program
+	.command("links")
+	.description("Build and inspect the short-link index");
+
+linksCommand
+	.command("backfill")
+	.description("Backfill indexed URL occurrences and t.co expansions")
+	.option("--all-urls", "Index all URLs, not only t.co")
+	.option("--source <kind>", "dm or tweet")
+	.option("--refresh-url-cache", "Re-expand URLs already in the index")
+	.option("--limit <n>", "Limit network/cache expansions for this run")
+	.option("--concurrency <n>", "Concurrent URL expansion workers", "12")
+	.option("--timeout-ms <n>", "Per-redirect fetch timeout", "15000")
+	.action(async (options) => {
+		const limit = parseNonNegativeIntegerOption(options.limit, "--limit");
+		if (options.limit !== undefined && limit === undefined) {
+			return;
+		}
+		const concurrency = parseNonNegativeIntegerOption(
+			options.concurrency,
+			"--concurrency",
+		);
+		if (concurrency === undefined) {
+			return;
+		}
+		const timeoutMs = parseNonNegativeIntegerOption(
+			options.timeoutMs,
+			"--timeout-ms",
+		);
+		if (timeoutMs === undefined) {
+			return;
+		}
+		const result = await backfillLinkIndex({
+			includeAllUrls: Boolean(options.allUrls),
+			refresh: Boolean(options.refreshUrlCache),
+			source:
+				options.source === "tweet"
+					? "tweet"
+					: options.source === "dm"
+						? "dm"
+						: undefined,
+			limit,
+			concurrency,
+			timeoutMs,
+		});
+		await autoSyncAfterWrite();
+		print(result, program.opts().json ?? false);
+	});
+
 program
 	.command("whois <query>")
 	.description("Identify likely people or orgs from local DMs and tweets")

package/src/lib/archive-import.ts

@@ -288,6 +288,8 @@ function clearImportedData() {
     delete from tweet_actions;
     delete from tweet_account_edges;
     delete from tweet_collections;
+    delete from link_occurrences;
+    delete from url_expansions;
     delete from dm_fts;
     delete from tweets_fts;
     delete from dm_messages;

package/src/lib/backup.ts

@@ -971,6 +971,8 @@ function clearBackupImportData(db: Database) {
     delete from tweet_actions;
     delete from tweet_account_edges;
     delete from tweet_collections;
+    delete from link_occurrences;
+    delete from url_expansions;
     delete from blocks;
     delete from mutes;
     delete from dm_fts;

package/src/lib/db.ts

@@ -185,6 +185,31 @@ export interface SyncCacheTable {
 	updated_at: string;
 }
 
+export interface UrlExpansionsTable {
+	short_url: string;
+	expanded_url: string;
+	final_url: string;
+	status: string;
+	expanded_tweet_id: string | null;
+	expanded_handle: string | null;
+	title: string | null;
+	description: string | null;
+	error: string | null;
+	source: string;
+	updated_at: string;
+}
+
+export interface LinkOccurrencesTable {
+	source_kind: "dm" | "tweet";
+	source_id: string;
+	source_position: number;
+	short_url: string;
+	account_id: string | null;
+	conversation_id: string | null;
+	direction: string | null;
+	created_at: string;
+}
+
 export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
@@ -202,6 +227,8 @@ export interface BirdclawDatabase {
 	mutes: MutesTable;
 	ai_scores: AiScoresTable;
 	sync_cache: SyncCacheTable;
+	url_expansions: UrlExpansionsTable;
+	link_occurrences: LinkOccurrencesTable;
 }
 
 let nativeDb: Database | undefined;
@@ -407,6 +434,32 @@ const BASE_SCHEMA_SQL = `
     updated_at text not null
   );
 
+  create table if not exists url_expansions (
+    short_url text primary key,
+    expanded_url text not null,
+    final_url text not null,
+    status text not null,
+    expanded_tweet_id text,
+    expanded_handle text,
+    title text,
+    description text,
+    error text,
+    source text not null,
+    updated_at text not null
+  );
+
+  create table if not exists link_occurrences (
+    source_kind text not null,
+    source_id text not null,
+    source_position integer not null,
+    short_url text not null,
+    account_id text,
+    conversation_id text,
+    direction text,
+    created_at text not null,
+    primary key (source_kind, source_id, source_position, short_url)
+  );
+
   create virtual table if not exists tweets_fts using fts5(
     tweet_id unindexed,
     text
@@ -441,6 +494,13 @@ const INDEX_SQL = `
   create index if not exists idx_mutes_account_created on mutes(account_id, created_at desc);
   create index if not exists idx_ai_scores_updated on ai_scores(updated_at desc);
   create index if not exists idx_sync_cache_updated on sync_cache(updated_at desc);
+  create index if not exists idx_url_expansions_expanded on url_expansions(expanded_url);
+  create index if not exists idx_url_expansions_tweet on url_expansions(expanded_tweet_id);
+  create index if not exists idx_url_expansions_handle on url_expansions(expanded_handle);
+  create index if not exists idx_link_occurrences_url on link_occurrences(short_url);
+  create index if not exists idx_link_occurrences_created on link_occurrences(created_at desc);
+  create index if not exists idx_link_occurrences_account on link_occurrences(account_id, created_at desc);
+  create index if not exists idx_link_occurrences_direction on link_occurrences(direction, created_at desc);
 `;
 
 function getColumnNames(db: Database, tableName: string): Set<string> {
@@ -612,6 +672,36 @@ function ensureIdentitySearchIndexTable(db: Database) {
   `);
 }
 
+function ensureLinkIndexTables(db: Database) {
+	db.exec(`
+    create table if not exists url_expansions (
+      short_url text primary key,
+      expanded_url text not null,
+      final_url text not null,
+      status text not null,
+      expanded_tweet_id text,
+      expanded_handle text,
+      title text,
+      description text,
+      error text,
+      source text not null,
+      updated_at text not null
+    );
+
+    create table if not exists link_occurrences (
+      source_kind text not null,
+      source_id text not null,
+      source_position integer not null,
+      short_url text not null,
+      account_id text,
+      conversation_id text,
+      direction text,
+      created_at text not null,
+      primary key (source_kind, source_id, source_position, short_url)
+    );
+  `);
+}
+
 function backfillTweetCollections(db: Database) {
 	const now = new Date().toISOString();
 	const insert = db.prepare(`
@@ -675,6 +765,7 @@ function initDatabase(options: InitDatabaseOptions = {}) {
 		ensureProfileSnapshotsTable(nativeDb);
 		ensureProfileBioEntitiesTable(nativeDb);
 		ensureIdentitySearchIndexTable(nativeDb);
+		ensureLinkIndexTables(nativeDb);
 		ensureSchemaIndexes(nativeDb);
 		if (options.seedDemoData !== false) {
 			seedDemoData(nativeDb);

package/src/lib/link-index.ts

@@ -0,0 +1,716 @@
+import { getNativeDb } from "./db";
+import type { Database } from "./sqlite";
+import {
+	expandUrls,
+	extractUrls,
+	type ExpandUrlsOptions,
+} from "./url-expansion";
+import type {
+	LinkIndexItem,
+	LinkOccurrenceItem,
+	LinkSearchItem,
+	ProfileRecord,
+	TimelineItem,
+	TweetEntities,
+	TweetMediaItem,
+} from "./types";
+
+const DEFAULT_EXPAND_CONCURRENCY = 12;
+
+interface TweetUrlEntityLike {
+	url?: unknown;
+	expandedUrl?: unknown;
+	expanded_url?: unknown;
+	displayUrl?: unknown;
+	display_url?: unknown;
+	title?: unknown;
+	description?: unknown;
+}
+
+interface SourceUrl {
+	url: string;
+	expandedUrl?: string;
+	title?: string;
+	description?: string | null;
+}
+
+export interface LinkBackfillOptions {
+	includeAllUrls?: boolean;
+	refresh?: boolean;
+	source?: "dm" | "tweet";
+	limit?: number;
+	concurrency?: number;
+	fetchImpl?: ExpandUrlsOptions["fetchImpl"];
+	timeoutMs?: number;
+}
+
+export interface LinkBackfillResult {
+	occurrences: number;
+	uniqueUrls: number;
+	entityExpansions: number;
+	networkExpansions: number;
+	cacheExpansions: number;
+	misses: number;
+	errors: number;
+	remainingUnexpanded: number;
+}
+
+export interface LinkSearchOptions {
+	since?: string;
+	until?: string;
+	account?: string;
+	source?: "dm" | "tweet";
+	direction?: "inbound" | "outbound";
+	participant?: string;
+	mediaType?: "image" | "video" | "gif";
+	limit?: number;
+}
+
+function parseJsonField<T>(value: unknown, fallback: T): T {
+	if (typeof value !== "string" || value.length === 0) {
+		return fallback;
+	}
+
+	try {
+		return JSON.parse(value) as T;
+	} catch {
+		return fallback;
+	}
+}
+
+function getString(value: unknown) {
+	return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function isIndexedUrl(url: string, includeAllUrls: boolean) {
+	if (includeAllUrls) {
+		return true;
+	}
+
+	try {
+		const host = new URL(url).hostname.toLowerCase();
+		return host === "t.co" || host.endsWith(".t.co");
+	} catch {
+		return false;
+	}
+}
+
+function getTweetTarget(url: string) {
+	try {
+		const parsed = new URL(url);
+		const host = parsed.hostname.toLowerCase();
+		if (
+			host !== "x.com" &&
+			host !== "twitter.com" &&
+			host !== "mobile.twitter.com" &&
+			host !== "www.x.com" &&
+			host !== "www.twitter.com"
+		) {
+			return {};
+		}
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		const statusIndex = parts.findIndex(
+			(part) => part === "status" || part === "statuses",
+		);
+		if (statusIndex === -1 || !parts[statusIndex + 1]) {
+			return {};
+		}
+
+		const tweetId = parts[statusIndex + 1]?.match(/^\d+/)?.[0];
+		const handle =
+			parts[statusIndex - 1] && parts[statusIndex - 1] !== "i"
+				? parts[statusIndex - 1]
+				: undefined;
+		return {
+			expandedTweetId: tweetId,
+			expandedHandle: handle,
+		};
+	} catch {
+		return {};
+	}
+}
+
+function normalizeExpansion(item: {
+	url: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	title?: string;
+	description?: string | null;
+	error?: string;
+	source: string;
+	updatedAt: string;
+}): LinkIndexItem {
+	const target = getTweetTarget(item.finalUrl);
+	return {
+		shortUrl: item.url,
+		expandedUrl: item.expandedUrl,
+		finalUrl: item.finalUrl,
+		status: item.status,
+		expandedTweetId: target.expandedTweetId ?? null,
+		expandedHandle: target.expandedHandle ?? null,
+		title: item.title ?? null,
+		description: item.description ?? null,
+		error: item.error ?? null,
+		source: item.source,
+		updatedAt: item.updatedAt,
+	};
+}
+
+function upsertExpansion(db: Database, item: LinkIndexItem) {
+	db.prepare(`
+    insert into url_expansions (
+      short_url, expanded_url, final_url, status, expanded_tweet_id,
+      expanded_handle, title, description, error, source, updated_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    on conflict(short_url) do update set
+      expanded_url = excluded.expanded_url,
+      final_url = excluded.final_url,
+      status = excluded.status,
+      expanded_tweet_id = excluded.expanded_tweet_id,
+      expanded_handle = excluded.expanded_handle,
+      title = excluded.title,
+      description = excluded.description,
+      error = excluded.error,
+      source = excluded.source,
+      updated_at = excluded.updated_at
+  `).run(
+		item.shortUrl,
+		item.expandedUrl,
+		item.finalUrl,
+		item.status,
+		item.expandedTweetId,
+		item.expandedHandle,
+		item.title,
+		item.description,
+		item.error,
+		item.source,
+		item.updatedAt,
+	);
+}
+
+function toTweetEntityUrls(entitiesJson: string): SourceUrl[] {
+	const entities = parseJsonField<TweetEntities>(entitiesJson, {});
+	const urls = Array.isArray(entities.urls) ? entities.urls : [];
+	return urls
+		.map((entry: TweetUrlEntityLike) => {
+			const url = getString(entry.url);
+			const expandedUrl =
+				getString(entry.expandedUrl) ?? getString(entry.expanded_url);
+			if (!url) {
+				return undefined;
+			}
+			return {
+				url,
+				...(expandedUrl ? { expandedUrl } : {}),
+				...(getString(entry.title) ? { title: getString(entry.title) } : {}),
+				...(entry.description !== undefined
+					? { description: getString(entry.description) ?? null }
+					: {}),
+			};
+		})
+		.filter((entry): entry is SourceUrl => Boolean(entry));
+}
+
+function uniqueSourceUrls(
+	text: string,
+	entityUrls: SourceUrl[],
+	includeAllUrls: boolean,
+) {
+	const byUrl = new Map<string, SourceUrl>();
+	for (const url of extractUrls(text)) {
+		if (isIndexedUrl(url, includeAllUrls)) {
+			byUrl.set(url, { url });
+		}
+	}
+	for (const entityUrl of entityUrls) {
+		if (!isIndexedUrl(entityUrl.url, includeAllUrls)) {
+			continue;
+		}
+		byUrl.set(entityUrl.url, { ...byUrl.get(entityUrl.url), ...entityUrl });
+	}
+	return Array.from(byUrl.values());
+}
+
+function rebuildOccurrences(
+	db: Database,
+	includeAllUrls: boolean,
+	source: LinkBackfillOptions["source"],
+) {
+	const insert = db.prepare(`
+    insert or replace into link_occurrences (
+      source_kind, source_id, source_position, short_url, account_id,
+      conversation_id, direction, created_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+	let occurrences = 0;
+	let entityExpansions = 0;
+	const now = new Date().toISOString();
+
+	db.transaction(() => {
+		if (source) {
+			db.prepare("delete from link_occurrences where source_kind = ?").run(
+				source,
+			);
+		} else {
+			db.exec("delete from link_occurrences");
+		}
+
+		const dmRows =
+			source === "tweet"
+				? []
+				: (db
+						.prepare(`
+      select m.id, m.conversation_id, c.account_id, m.direction, m.created_at, m.text
+      from dm_messages m
+      join dm_conversations c on c.id = m.conversation_id
+      where m.text like '%://%'
+    `)
+						.all() as Array<Record<string, unknown>>);
+		for (const row of dmRows) {
+			const urls = uniqueSourceUrls(String(row.text), [], includeAllUrls);
+			urls.forEach((entry, index) => {
+				insert.run(
+					"dm",
+					String(row.id),
+					index,
+					entry.url,
+					String(row.account_id),
+					String(row.conversation_id),
+					String(row.direction),
+					String(row.created_at),
+				);
+				occurrences++;
+			});
+		}
+
+		const tweetRows =
+			source === "dm"
+				? []
+				: (db
+						.prepare(`
+      select id, account_id, created_at, text, entities_json
+      from tweets
+      where text like '%://%' or entities_json like '%://%'
+    `)
+						.all() as Array<Record<string, unknown>>);
+		for (const row of tweetRows) {
+			const entityUrls = toTweetEntityUrls(String(row.entities_json));
+			const urls = uniqueSourceUrls(
+				String(row.text),
+				entityUrls,
+				includeAllUrls,
+			);
+			urls.forEach((entry, index) => {
+				insert.run(
+					"tweet",
+					String(row.id),
+					index,
+					entry.url,
+					String(row.account_id),
+					null,
+					null,
+					String(row.created_at),
+				);
+				occurrences++;
+				if (entry.expandedUrl) {
+					upsertExpansion(
+						db,
+						normalizeExpansion({
+							url: entry.url,
+							expandedUrl: entry.expandedUrl,
+							finalUrl: entry.expandedUrl,
+							status: "hit",
+							title: entry.title,
+							description: entry.description,
+							source: "entity",
+							updatedAt: now,
+						}),
+					);
+					entityExpansions++;
+				}
+			});
+		}
+	})();
+
+	return { occurrences, entityExpansions };
+}
+
+async function expandWithConcurrency(
+	db: Database,
+	urls: string[],
+	options: LinkBackfillOptions,
+) {
+	const concurrency = Math.max(
+		1,
+		Math.min(options.concurrency ?? DEFAULT_EXPAND_CONCURRENCY, 64),
+	);
+	const counts = {
+		networkExpansions: 0,
+		cacheExpansions: 0,
+		misses: 0,
+		errors: 0,
+	};
+	let nextIndex = 0;
+
+	async function worker() {
+		for (;;) {
+			const index = nextIndex++;
+			const url = urls[index];
+			if (!url) {
+				return;
+			}
+			const result = (
+				await expandUrls([url], {
+					refresh: options.refresh,
+					fetchImpl: options.fetchImpl,
+					timeoutMs: options.timeoutMs,
+				})
+			)[0]!;
+			if (result.source === "network") {
+				counts.networkExpansions++;
+			} else {
+				counts.cacheExpansions++;
+			}
+			if (result.status === "miss") {
+				counts.misses++;
+			}
+			if (result.status === "error") {
+				counts.errors++;
+			}
+			upsertExpansion(db, normalizeExpansion(result));
+		}
+	}
+
+	await Promise.all(
+		Array.from({ length: Math.min(concurrency, urls.length) }, () => worker()),
+	);
+	return counts;
+}
+
+export async function backfillLinkIndex(
+	options: LinkBackfillOptions = {},
+): Promise<LinkBackfillResult> {
+	const db = getNativeDb({ seedDemoData: false });
+	const { occurrences, entityExpansions } = rebuildOccurrences(
+		db,
+		Boolean(options.includeAllUrls),
+		options.source,
+	);
+	const limit =
+		typeof options.limit === "number" && Number.isFinite(options.limit)
+			? Math.max(0, Math.trunc(options.limit))
+			: undefined;
+	const needsExpansionClause = options.refresh
+		? "1 = 1"
+		: "(e.short_url is null or e.status in ('error', 'miss'))";
+
+	const urlsToExpand = db
+		.prepare(`
+    select distinct o.short_url
+    from link_occurrences o
+    left join url_expansions e on e.short_url = o.short_url
+    where ${needsExpansionClause}
+      ${options.source ? "and o.source_kind = ?" : ""}
+    order by o.short_url
+    ${limit === undefined ? "" : "limit ?"}
+  `)
+		.all(
+			...(options.source ? [options.source] : []),
+			...(limit === undefined ? [] : [limit]),
+		) as Array<{
+		short_url: string;
+	}>;
+
+	const expansionCounts = await expandWithConcurrency(
+		db,
+		urlsToExpand.map((row) => row.short_url),
+		options,
+	);
+
+	const uniqueUrls = db
+		.prepare(`
+    select count(distinct short_url) as count
+    from link_occurrences
+    ${options.source ? "where source_kind = ?" : ""}
+  `)
+		.get(...(options.source ? [options.source] : [])) as { count: number };
+	const remaining = db
+		.prepare(`
+    select count(distinct o.short_url) as count
+    from link_occurrences o
+    left join url_expansions e on e.short_url = o.short_url
+    where (e.short_url is null or e.status in ('error', 'miss'))
+      ${options.source ? "and o.source_kind = ?" : ""}
+  `)
+		.get(...(options.source ? [options.source] : [])) as { count: number };
+
+	return {
+		occurrences,
+		uniqueUrls: Number(uniqueUrls.count),
+		entityExpansions,
+		...expansionCounts,
+		remainingUnexpanded: Number(remaining.count),
+	};
+}
+
+function likePattern(value: string) {
+	return `%${value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_").toLowerCase()}%`;
+}
+
+function toProfile(
+	row: Record<string, unknown>,
+	prefix: string,
+): ProfileRecord | null {
+	if (!row[`${prefix}id`]) {
+		return null;
+	}
+	return {
+		id: String(row[`${prefix}id`]),
+		handle: String(row[`${prefix}handle`]),
+		displayName: String(row[`${prefix}display_name`]),
+		bio: String(row[`${prefix}bio`]),
+		followersCount: Number(row[`${prefix}followers_count`]),
+		followingCount: Number(row[`${prefix}following_count`]),
+		avatarHue: Number(row[`${prefix}avatar_hue`]),
+		avatarUrl: getString(row[`${prefix}avatar_url`]),
+		location: getString(row[`${prefix}location`]),
+		url: getString(row[`${prefix}url`]),
+		verifiedType: getString(row[`${prefix}verified_type`]),
+		entities: parseJsonField<Record<string, unknown>>(
+			row[`${prefix}entities_json`],
+			{},
+		),
+		createdAt: String(row[`${prefix}created_at`]),
+	};
+}
+
+function toLinkedTweet(row: Record<string, unknown>): TimelineItem | null {
+	const author = toProfile(row, "linked_author_");
+	if (!row.linked_id || !author) {
+		return null;
+	}
+
+	return {
+		id: String(row.linked_id),
+		accountId: String(row.linked_account_id),
+		accountHandle: String(row.linked_account_handle ?? ""),
+		kind: String(row.linked_kind) as TimelineItem["kind"],
+		text: String(row.linked_text),
+		createdAt: String(row.linked_created_at),
+		isReplied: Boolean(row.linked_is_replied),
+		likeCount: Number(row.linked_like_count),
+		mediaCount: Number(row.linked_media_count),
+		bookmarked: Boolean(row.linked_bookmarked),
+		liked: Boolean(row.linked_liked),
+		author,
+		entities: parseJsonField<TweetEntities>(row.linked_entities_json, {}),
+		media: parseJsonField<TweetMediaItem[]>(row.linked_media_json, []),
+	};
+}
+
+function toLinkSearchItem(row: Record<string, unknown>): LinkSearchItem {
+	const occurrence: LinkOccurrenceItem = {
+		sourceKind: String(row.source_kind) as LinkOccurrenceItem["sourceKind"],
+		sourceId: String(row.source_id),
+		sourcePosition: Number(row.source_position),
+		shortUrl: String(row.short_url),
+		accountId: getString(row.account_id) ?? null,
+		conversationId: getString(row.conversation_id) ?? null,
+		direction: getString(row.direction) ?? null,
+		createdAt: String(row.created_at),
+	};
+	return {
+		occurrence,
+		expansion: {
+			shortUrl: String(row.short_url),
+			expandedUrl: String(row.expanded_url),
+			finalUrl: String(row.final_url),
+			status: String(row.status) as LinkIndexItem["status"],
+			expandedTweetId: getString(row.expanded_tweet_id) ?? null,
+			expandedHandle: getString(row.expanded_handle) ?? null,
+			title: getString(row.title) ?? null,
+			description: getString(row.description) ?? null,
+			error: getString(row.error) ?? null,
+			source: String(row.expansion_source),
+			updatedAt: String(row.updated_at),
+		},
+		sourceText: String(row.source_text),
+		sourceAuthor: toProfile(row, "source_author_"),
+		participant: toProfile(row, "participant_"),
+		linkedTweet: toLinkedTweet(row),
+	};
+}
+
+export function searchLinks(query: string, options: LinkSearchOptions = {}) {
+	const db = getNativeDb({ seedDemoData: false });
+	const conditions: string[] = [];
+	const params: unknown[] = [];
+	const searchFields = `
+    lower(
+      coalesce(e.short_url, '') || ' ' ||
+      coalesce(e.expanded_url, '') || ' ' ||
+      coalesce(e.final_url, '') || ' ' ||
+      coalesce(e.expanded_handle, '') || ' ' ||
+      coalesce(e.title, '') || ' ' ||
+      coalesce(e.description, '') || ' ' ||
+      coalesce(dm.text, '') || ' ' ||
+      coalesce(source_tweet.text, '') || ' ' ||
+      coalesce(linked.text, '') || ' ' ||
+      coalesce(linked_author.handle, '') || ' ' ||
+      coalesce(linked_author.display_name, '')
+    )
+  `;
+
+	for (const term of query.match(/[\p{L}\p{N}_:.@/-]+/gu) ?? []) {
+		conditions.push(`${searchFields} like ? escape '\\'`);
+		params.push(likePattern(term));
+	}
+	if (options.since) {
+		conditions.push("o.created_at >= ?");
+		params.push(options.since);
+	}
+	if (options.until) {
+		conditions.push("o.created_at < ?");
+		params.push(options.until);
+	}
+	if (options.account) {
+		conditions.push("(o.account_id = ? or account.handle = ?)");
+		params.push(options.account, options.account.replace(/^@/, ""));
+	}
+	if (options.source) {
+		conditions.push("o.source_kind = ?");
+		params.push(options.source);
+	}
+	if (options.direction) {
+		conditions.push("o.direction = ?");
+		params.push(options.direction);
+	}
+	if (options.participant) {
+		conditions.push(
+			"(participant.handle = ? or participant.display_name like ?)",
+		);
+		params.push(
+			options.participant.replace(/^@/, ""),
+			`%${options.participant}%`,
+		);
+	}
+	if (options.mediaType) {
+		conditions.push(
+			"(linked.media_json like ? or source_tweet.media_json like ?)",
+		);
+		params.push(
+			`%"type":"${options.mediaType}"%`,
+			`%"type":"${options.mediaType}"%`,
+		);
+	}
+
+	const limit =
+		options.limit && Number.isFinite(options.limit)
+			? Math.max(1, Math.trunc(options.limit))
+			: 20;
+	params.push(limit);
+
+	const rows = db
+		.prepare(`
+    select
+      o.source_kind,
+      o.source_id,
+      o.source_position,
+      o.short_url,
+      o.account_id,
+      o.conversation_id,
+      o.direction,
+      o.created_at,
+      e.expanded_url,
+      e.final_url,
+      e.status,
+      e.expanded_tweet_id,
+      e.expanded_handle,
+      e.title,
+      e.description,
+      e.error,
+      e.source as expansion_source,
+      e.updated_at,
+      coalesce(dm.text, source_tweet.text, '') as source_text,
+      account.handle as account_handle,
+      source_author.id as source_author_id,
+      source_author.handle as source_author_handle,
+      source_author.display_name as source_author_display_name,
+      source_author.bio as source_author_bio,
+      source_author.followers_count as source_author_followers_count,
+      source_author.following_count as source_author_following_count,
+      source_author.avatar_hue as source_author_avatar_hue,
+      source_author.avatar_url as source_author_avatar_url,
+      source_author.location as source_author_location,
+      source_author.url as source_author_url,
+      source_author.verified_type as source_author_verified_type,
+      source_author.entities_json as source_author_entities_json,
+      source_author.created_at as source_author_created_at,
+      participant.id as participant_id,
+      participant.handle as participant_handle,
+      participant.display_name as participant_display_name,
+      participant.bio as participant_bio,
+      participant.followers_count as participant_followers_count,
+      participant.following_count as participant_following_count,
+      participant.avatar_hue as participant_avatar_hue,
+      participant.avatar_url as participant_avatar_url,
+      participant.location as participant_location,
+      participant.url as participant_url,
+      participant.verified_type as participant_verified_type,
+      participant.entities_json as participant_entities_json,
+      participant.created_at as participant_created_at,
+      linked.id as linked_id,
+      linked.account_id as linked_account_id,
+      linked_account.handle as linked_account_handle,
+      linked.kind as linked_kind,
+      linked.text as linked_text,
+      linked.created_at as linked_created_at,
+      linked.is_replied as linked_is_replied,
+      linked.like_count as linked_like_count,
+      linked.media_count as linked_media_count,
+      linked.bookmarked as linked_bookmarked,
+      linked.liked as linked_liked,
+      linked.entities_json as linked_entities_json,
+      linked.media_json as linked_media_json,
+      linked_author.id as linked_author_id,
+      linked_author.handle as linked_author_handle,
+      linked_author.display_name as linked_author_display_name,
+      linked_author.bio as linked_author_bio,
+      linked_author.followers_count as linked_author_followers_count,
+      linked_author.following_count as linked_author_following_count,
+      linked_author.avatar_hue as linked_author_avatar_hue,
+      linked_author.avatar_url as linked_author_avatar_url,
+      linked_author.location as linked_author_location,
+      linked_author.url as linked_author_url,
+      linked_author.verified_type as linked_author_verified_type,
+      linked_author.entities_json as linked_author_entities_json,
+      linked_author.created_at as linked_author_created_at
+    from link_occurrences o
+    join url_expansions e on e.short_url = o.short_url
+    left join accounts account on account.id = o.account_id
+    left join dm_messages dm
+      on o.source_kind = 'dm' and dm.id = o.source_id
+    left join dm_conversations conversation
+      on conversation.id = o.conversation_id
+    left join profiles participant
+      on participant.id = conversation.participant_profile_id
+    left join tweets source_tweet
+      on o.source_kind = 'tweet' and source_tweet.id = o.source_id
+    left join profiles source_author
+      on source_author.id = source_tweet.author_profile_id
+    left join tweets linked
+      on linked.id = e.expanded_tweet_id
+    left join accounts linked_account
+      on linked_account.id = linked.account_id
+    left join profiles linked_author
+      on linked_author.id = linked.author_profile_id
+    ${conditions.length > 0 ? `where ${conditions.join(" and ")}` : ""}
+    order by o.created_at desc
+    limit ?
+  `)
+		.all(...params) as Array<Record<string, unknown>>;
+
+	return rows.map(toLinkSearchItem);
+}

package/src/lib/types.ts

@@ -183,6 +183,40 @@ export interface UrlExpansionItem {
 	updatedAt: string;
 }
 
+export interface LinkOccurrenceItem {
+	sourceKind: "dm" | "tweet";
+	sourceId: string;
+	sourcePosition: number;
+	shortUrl: string;
+	accountId?: string | null;
+	conversationId?: string | null;
+	direction?: string | null;
+	createdAt: string;
+}
+
+export interface LinkIndexItem {
+	shortUrl: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	expandedTweetId?: string | null;
+	expandedHandle?: string | null;
+	title?: string | null;
+	description?: string | null;
+	error?: string | null;
+	source: string;
+	updatedAt: string;
+}
+
+export interface LinkSearchItem {
+	occurrence: LinkOccurrenceItem;
+	expansion: LinkIndexItem;
+	sourceText: string;
+	sourceAuthor?: ProfileRecord | null;
+	participant?: ProfileRecord | null;
+	linkedTweet?: TimelineItem | null;
+}
+
 export interface DmSearchMatchItem {
 	message: DmMessageItem;
 	before: DmMessageItem[];

package/src/lib/url-expansion.ts

@@ -3,6 +3,7 @@ import type { UrlExpansionItem } from "./types";
 
 const SUCCESS_CACHE_TTL_MS = 365 * 24 * 60 * 60 * 1000;
 const FAILURE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_FETCH_TIMEOUT_MS = 15_000;
 const URL_REGEX = /https?:\/\/[^\s<>"')\]]+/g;
 
 interface CachedUrlExpansion {
@@ -19,6 +20,7 @@ export interface ExpandUrlsOptions {
 	successMaxAgeMs?: number;
 	failureMaxAgeMs?: number;
 	fetchImpl?: typeof fetch;
+	timeoutMs?: number;
 }
 
 function cacheKeyForUrl(url: string) {
@@ -67,19 +69,24 @@ function toExpansionItem(
 async function fetchExpansion(
 	url: string,
 	fetchImpl: typeof fetch,
+	timeoutMs: number,
 ): Promise<CachedUrlExpansion> {
+	const requestInit = {
+		redirect: "follow",
+		headers: { "user-agent": "birdclaw/0.3 url-expander" },
+		signal: AbortSignal.timeout(timeoutMs),
+	} satisfies RequestInit;
+
 	try {
 		let response = await fetchImpl(url, {
+			...requestInit,
 			method: "HEAD",
-			redirect: "follow",
-			headers: { "user-agent": "birdclaw/0.3 url-expander" },
 		});
 
 		if (!response.url || response.url === url || response.status >= 400) {
 			response = await fetchImpl(url, {
+				...requestInit,
 				method: "GET",
-				redirect: "follow",
-				headers: { "user-agent": "birdclaw/0.3 url-expander" },
 			});
 		}
 
@@ -106,6 +113,7 @@ export async function expandUrls(
 ): Promise<UrlExpansionItem[]> {
 	const uniqueUrls = Array.from(new Set(urls));
 	const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+	const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
 	const results: UrlExpansionItem[] = [];
 
 	for (const url of uniqueUrls) {
@@ -123,7 +131,7 @@ export async function expandUrls(
 			}
 		}
 
-		const value = await fetchExpansion(url, fetchImpl);
+		const value = await fetchExpansion(url, fetchImpl, timeoutMs);
 		const updatedAt = writeSyncCache(cacheKeyForUrl(url), value);
 		results.push(toExpansionItem(url, value, "network", updatedAt));
 	}