birdclaw 0.4.1 → 0.5.1

package/src/lib/link-index.ts

@@ -1,5 +1,9 @@
 import { getNativeDb } from "./db";
 import type { Database } from "./sqlite";
+import {
+	normalizeUrlExpansionForIndex,
+	upsertUrlExpansion,
+} from "./url-expansion-store";
 import {
 	expandUrls,
 	extractUrls,
@@ -95,101 +99,6 @@ function isIndexedUrl(url: string, includeAllUrls: boolean) {
 	}
 }
 
-function getTweetTarget(url: string) {
-	try {
-		const parsed = new URL(url);
-		const host = parsed.hostname.toLowerCase();
-		if (
-			host !== "x.com" &&
-			host !== "twitter.com" &&
-			host !== "mobile.twitter.com" &&
-			host !== "www.x.com" &&
-			host !== "www.twitter.com"
-		) {
-			return {};
-		}
-
-		const parts = parsed.pathname.split("/").filter(Boolean);
-		const statusIndex = parts.findIndex(
-			(part) => part === "status" || part === "statuses",
-		);
-		if (statusIndex === -1 || !parts[statusIndex + 1]) {
-			return {};
-		}
-
-		const tweetId = parts[statusIndex + 1]?.match(/^\d+/)?.[0];
-		const handle =
-			parts[statusIndex - 1] && parts[statusIndex - 1] !== "i"
-				? parts[statusIndex - 1]
-				: undefined;
-		return {
-			expandedTweetId: tweetId,
-			expandedHandle: handle,
-		};
-	} catch {
-		return {};
-	}
-}
-
-function normalizeExpansion(item: {
-	url: string;
-	expandedUrl: string;
-	finalUrl: string;
-	status: "hit" | "miss" | "error";
-	title?: string;
-	description?: string | null;
-	error?: string;
-	source: string;
-	updatedAt: string;
-}): LinkIndexItem {
-	const target = getTweetTarget(item.finalUrl);
-	return {
-		shortUrl: item.url,
-		expandedUrl: item.expandedUrl,
-		finalUrl: item.finalUrl,
-		status: item.status,
-		expandedTweetId: target.expandedTweetId ?? null,
-		expandedHandle: target.expandedHandle ?? null,
-		title: item.title ?? null,
-		description: item.description ?? null,
-		error: item.error ?? null,
-		source: item.source,
-		updatedAt: item.updatedAt,
-	};
-}
-
-function upsertExpansion(db: Database, item: LinkIndexItem) {
-	db.prepare(`
-    insert into url_expansions (
-      short_url, expanded_url, final_url, status, expanded_tweet_id,
-      expanded_handle, title, description, error, source, updated_at
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    on conflict(short_url) do update set
-      expanded_url = excluded.expanded_url,
-      final_url = excluded.final_url,
-      status = excluded.status,
-      expanded_tweet_id = excluded.expanded_tweet_id,
-      expanded_handle = excluded.expanded_handle,
-      title = excluded.title,
-      description = excluded.description,
-      error = excluded.error,
-      source = excluded.source,
-      updated_at = excluded.updated_at
-  `).run(
-		item.shortUrl,
-		item.expandedUrl,
-		item.finalUrl,
-		item.status,
-		item.expandedTweetId,
-		item.expandedHandle,
-		item.title,
-		item.description,
-		item.error,
-		item.source,
-		item.updatedAt,
-	);
-}
-
 function toTweetEntityUrls(entitiesJson: string): SourceUrl[] {
 	const entities = parseJsonField<TweetEntities>(entitiesJson, {});
 	const urls = Array.isArray(entities.urls) ? entities.urls : [];
@@ -315,9 +224,9 @@ function rebuildOccurrences(
 				);
 				occurrences++;
 				if (entry.expandedUrl) {
-					upsertExpansion(
+					upsertUrlExpansion(
 						db,
-						normalizeExpansion({
+						normalizeUrlExpansionForIndex({
 							url: entry.url,
 							expandedUrl: entry.expandedUrl,
 							finalUrl: entry.expandedUrl,
@@ -379,7 +288,7 @@ async function expandWithConcurrency(
 			if (result.status === "error") {
 				counts.errors++;
 			}
-			upsertExpansion(db, normalizeExpansion(result));
+			upsertUrlExpansion(db, normalizeUrlExpansionForIndex(result));
 		}
 	}
 
@@ -532,6 +441,8 @@ function toLinkSearchItem(row: Record<string, unknown>): LinkSearchItem {
 			expandedHandle: getString(row.expanded_handle) ?? null,
 			title: getString(row.title) ?? null,
 			description: getString(row.description) ?? null,
+			imageUrl: getString(row.image_url) ?? null,
+			siteName: getString(row.site_name) ?? null,
 			error: getString(row.error) ?? null,
 			source: String(row.expansion_source),
 			updatedAt: String(row.updated_at),
@@ -630,6 +541,8 @@ export function searchLinks(query: string, options: LinkSearchOptions = {}) {
       e.expanded_handle,
       e.title,
       e.description,
+      e.image_url,
+      e.site_name,
       e.error,
       e.source as expansion_source,
       e.updated_at,