birdclaw 0.4.0 → 0.5.0

package/src/lib/url-expansion-store.ts

@@ -0,0 +1,110 @@
+import type { Database } from "./sqlite";
+import type { LinkIndexItem } from "./types";
+
+export interface UrlExpansionRecordInput {
+	url: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	title?: string | null;
+	description?: string | null;
+	imageUrl?: string | null;
+	siteName?: string | null;
+	error?: string;
+	source: string;
+	updatedAt: string;
+}
+
+function getTweetTarget(url: string) {
+	try {
+		const parsed = new URL(url);
+		const host = parsed.hostname.toLowerCase();
+		if (
+			host !== "x.com" &&
+			host !== "twitter.com" &&
+			host !== "mobile.twitter.com" &&
+			host !== "www.x.com" &&
+			host !== "www.twitter.com"
+		) {
+			return {};
+		}
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		const statusIndex = parts.findIndex(
+			(part) => part === "status" || part === "statuses",
+		);
+		if (statusIndex === -1 || !parts[statusIndex + 1]) {
+			return {};
+		}
+
+		const tweetId = parts[statusIndex + 1]?.match(/^\d+/)?.[0];
+		const handle =
+			parts[statusIndex - 1] && parts[statusIndex - 1] !== "i"
+				? parts[statusIndex - 1]
+				: undefined;
+		return {
+			expandedTweetId: tweetId,
+			expandedHandle: handle,
+		};
+	} catch {
+		return {};
+	}
+}
+
+export function normalizeUrlExpansionForIndex(
+	item: UrlExpansionRecordInput,
+): LinkIndexItem {
+	const target = getTweetTarget(item.finalUrl);
+	return {
+		shortUrl: item.url,
+		expandedUrl: item.expandedUrl,
+		finalUrl: item.finalUrl,
+		status: item.status,
+		expandedTweetId: target.expandedTweetId ?? null,
+		expandedHandle: target.expandedHandle ?? null,
+		title: item.title ?? null,
+		description: item.description ?? null,
+		imageUrl: item.imageUrl ?? null,
+		siteName: item.siteName ?? null,
+		error: item.error ?? null,
+		source: item.source,
+		updatedAt: item.updatedAt,
+	};
+}
+
+export function upsertUrlExpansion(db: Database, item: LinkIndexItem) {
+	db.prepare(`
+    insert into url_expansions (
+      short_url, expanded_url, final_url, status, expanded_tweet_id,
+      expanded_handle, title, description, image_url, site_name, error, source,
+      updated_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    on conflict(short_url) do update set
+      expanded_url = excluded.expanded_url,
+      final_url = excluded.final_url,
+      status = excluded.status,
+      expanded_tweet_id = excluded.expanded_tweet_id,
+      expanded_handle = excluded.expanded_handle,
+      title = excluded.title,
+      description = excluded.description,
+      image_url = excluded.image_url,
+      site_name = excluded.site_name,
+      error = excluded.error,
+      source = excluded.source,
+      updated_at = excluded.updated_at
+  `).run(
+		item.shortUrl,
+		item.expandedUrl,
+		item.finalUrl,
+		item.status,
+		item.expandedTweetId,
+		item.expandedHandle,
+		item.title,
+		item.description,
+		item.imageUrl,
+		item.siteName,
+		item.error,
+		item.source,
+		item.updatedAt,
+	);
+}

package/src/lib/url-expansion.ts

@@ -1,8 +1,14 @@
+import { getNativeDb } from "./db";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import type { UrlExpansionItem } from "./types";
+import {
+	normalizeUrlExpansionForIndex,
+	upsertUrlExpansion,
+} from "./url-expansion-store";
 
 const SUCCESS_CACHE_TTL_MS = 365 * 24 * 60 * 60 * 1000;
 const FAILURE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_FETCH_TIMEOUT_MS = 15_000;
 const URL_REGEX = /https?:\/\/[^\s<>"')\]]+/g;
 
 interface CachedUrlExpansion {
@@ -19,6 +25,7 @@ export interface ExpandUrlsOptions {
 	successMaxAgeMs?: number;
 	failureMaxAgeMs?: number;
 	fetchImpl?: typeof fetch;
+	timeoutMs?: number;
 }
 
 function cacheKeyForUrl(url: string) {
@@ -64,22 +71,32 @@ function toExpansionItem(
 	};
 }
 
+function persistExpansion(item: UrlExpansionItem) {
+	const db = getNativeDb({ seedDemoData: false });
+	upsertUrlExpansion(db, normalizeUrlExpansionForIndex(item));
+}
+
 async function fetchExpansion(
 	url: string,
 	fetchImpl: typeof fetch,
+	timeoutMs: number,
 ): Promise<CachedUrlExpansion> {
+	const requestInit = {
+		redirect: "follow",
+		headers: { "user-agent": "birdclaw/0.3 url-expander" },
+		signal: AbortSignal.timeout(timeoutMs),
+	} satisfies RequestInit;
+
 	try {
 		let response = await fetchImpl(url, {
+			...requestInit,
 			method: "HEAD",
-			redirect: "follow",
-			headers: { "user-agent": "birdclaw/0.3 url-expander" },
 		});
 
 		if (!response.url || response.url === url || response.status >= 400) {
 			response = await fetchImpl(url, {
+				...requestInit,
 				method: "GET",
-				redirect: "follow",
-				headers: { "user-agent": "birdclaw/0.3 url-expander" },
 			});
 		}
 
@@ -106,6 +123,7 @@ export async function expandUrls(
 ): Promise<UrlExpansionItem[]> {
 	const uniqueUrls = Array.from(new Set(urls));
 	const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+	const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
 	const results: UrlExpansionItem[] = [];
 
 	for (const url of uniqueUrls) {
@@ -116,16 +134,23 @@ export async function expandUrls(
 					? (options.successMaxAgeMs ?? SUCCESS_CACHE_TTL_MS)
 					: (options.failureMaxAgeMs ?? FAILURE_CACHE_TTL_MS);
 			if (isFresh(cached.updatedAt, maxAge)) {
-				results.push(
-					toExpansionItem(url, cached.value, "cache", cached.updatedAt),
+				const item = toExpansionItem(
+					url,
+					cached.value,
+					"cache",
+					cached.updatedAt,
 				);
+				persistExpansion(item);
+				results.push(item);
 				continue;
 			}
 		}
 
-		const value = await fetchExpansion(url, fetchImpl);
+		const value = await fetchExpansion(url, fetchImpl, timeoutMs);
 		const updatedAt = writeSyncCache(cacheKeyForUrl(url), value);
-		results.push(toExpansionItem(url, value, "network", updatedAt));
+		const item = toExpansionItem(url, value, "network", updatedAt);
+		persistExpansion(item);
+		results.push(item);
 	}
 
 	return results;

package/src/lib/x-profile.ts

@@ -150,6 +150,7 @@ function updateExistingProfileFromUser(
         bio = ?,
         followers_count = ?,
         following_count = coalesce(?, following_count),
+        public_metrics_json = ?,
         avatar_url = coalesce(?, avatar_url),
         location = coalesce(?, location),
         url = coalesce(?, url),
@@ -167,6 +168,7 @@ function updateExistingProfileFromUser(
 		bio,
 		followersCount,
 		followingCount,
+		JSON.stringify(user.public_metrics ?? {}),
 		avatarUrl,
 		metadata.location,
 		metadata.url,
@@ -245,13 +247,15 @@ export function upsertProfileFromXUser(
 		`
     insert into profiles (
       id, handle, display_name, bio, followers_count, following_count, avatar_hue,
-      avatar_url, location, url, verified_type, entities_json, raw_json, created_at
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      public_metrics_json, avatar_url, location, url, verified_type, entities_json,
+      raw_json, created_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
 	      handle = excluded.handle,
 	      display_name = excluded.display_name,
 	      bio = excluded.bio,
 	      followers_count = excluded.followers_count,
+	      public_metrics_json = excluded.public_metrics_json,
 	      following_count = case
 	        when ? then excluded.following_count
 	        else profiles.following_count
@@ -274,6 +278,7 @@ export function upsertProfileFromXUser(
 		followersCount,
 		followingCount ?? 0,
 		avatarHue,
+		JSON.stringify(user.public_metrics ?? {}),
 		avatarUrl,
 		metadata.location,
 		metadata.url,