birdclaw 0.4.1 → 0.5.1

package/src/lib/url-expansion-store.ts

@@ -0,0 +1,110 @@
+import type { Database } from "./sqlite";
+import type { LinkIndexItem } from "./types";
+
+export interface UrlExpansionRecordInput {
+	url: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	title?: string | null;
+	description?: string | null;
+	imageUrl?: string | null;
+	siteName?: string | null;
+	error?: string;
+	source: string;
+	updatedAt: string;
+}
+
+function getTweetTarget(url: string) {
+	try {
+		const parsed = new URL(url);
+		const host = parsed.hostname.toLowerCase();
+		if (
+			host !== "x.com" &&
+			host !== "twitter.com" &&
+			host !== "mobile.twitter.com" &&
+			host !== "www.x.com" &&
+			host !== "www.twitter.com"
+		) {
+			return {};
+		}
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		const statusIndex = parts.findIndex(
+			(part) => part === "status" || part === "statuses",
+		);
+		if (statusIndex === -1 || !parts[statusIndex + 1]) {
+			return {};
+		}
+
+		const tweetId = parts[statusIndex + 1]?.match(/^\d+/)?.[0];
+		const handle =
+			parts[statusIndex - 1] && parts[statusIndex - 1] !== "i"
+				? parts[statusIndex - 1]
+				: undefined;
+		return {
+			expandedTweetId: tweetId,
+			expandedHandle: handle,
+		};
+	} catch {
+		return {};
+	}
+}
+
+export function normalizeUrlExpansionForIndex(
+	item: UrlExpansionRecordInput,
+): LinkIndexItem {
+	const target = getTweetTarget(item.finalUrl);
+	return {
+		shortUrl: item.url,
+		expandedUrl: item.expandedUrl,
+		finalUrl: item.finalUrl,
+		status: item.status,
+		expandedTweetId: target.expandedTweetId ?? null,
+		expandedHandle: target.expandedHandle ?? null,
+		title: item.title ?? null,
+		description: item.description ?? null,
+		imageUrl: item.imageUrl ?? null,
+		siteName: item.siteName ?? null,
+		error: item.error ?? null,
+		source: item.source,
+		updatedAt: item.updatedAt,
+	};
+}
+
+export function upsertUrlExpansion(db: Database, item: LinkIndexItem) {
+	db.prepare(`
+    insert into url_expansions (
+      short_url, expanded_url, final_url, status, expanded_tweet_id,
+      expanded_handle, title, description, image_url, site_name, error, source,
+      updated_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    on conflict(short_url) do update set
+      expanded_url = excluded.expanded_url,
+      final_url = excluded.final_url,
+      status = excluded.status,
+      expanded_tweet_id = excluded.expanded_tweet_id,
+      expanded_handle = excluded.expanded_handle,
+      title = excluded.title,
+      description = excluded.description,
+      image_url = excluded.image_url,
+      site_name = excluded.site_name,
+      error = excluded.error,
+      source = excluded.source,
+      updated_at = excluded.updated_at
+  `).run(
+		item.shortUrl,
+		item.expandedUrl,
+		item.finalUrl,
+		item.status,
+		item.expandedTweetId,
+		item.expandedHandle,
+		item.title,
+		item.description,
+		item.imageUrl,
+		item.siteName,
+		item.error,
+		item.source,
+		item.updatedAt,
+	);
+}

package/src/lib/url-expansion.ts

@@ -1,5 +1,10 @@
+import { getNativeDb } from "./db";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import type { UrlExpansionItem } from "./types";
+import {
+	normalizeUrlExpansionForIndex,
+	upsertUrlExpansion,
+} from "./url-expansion-store";
 
 const SUCCESS_CACHE_TTL_MS = 365 * 24 * 60 * 60 * 1000;
 const FAILURE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
@@ -66,6 +71,11 @@ function toExpansionItem(
 	};
 }
 
+function persistExpansion(item: UrlExpansionItem) {
+	const db = getNativeDb({ seedDemoData: false });
+	upsertUrlExpansion(db, normalizeUrlExpansionForIndex(item));
+}
+
 async function fetchExpansion(
 	url: string,
 	fetchImpl: typeof fetch,
@@ -124,16 +134,23 @@ export async function expandUrls(
 					? (options.successMaxAgeMs ?? SUCCESS_CACHE_TTL_MS)
 					: (options.failureMaxAgeMs ?? FAILURE_CACHE_TTL_MS);
 			if (isFresh(cached.updatedAt, maxAge)) {
-				results.push(
-					toExpansionItem(url, cached.value, "cache", cached.updatedAt),
+				const item = toExpansionItem(
+					url,
+					cached.value,
+					"cache",
+					cached.updatedAt,
 				);
+				persistExpansion(item);
+				results.push(item);
 				continue;
 			}
 		}
 
 		const value = await fetchExpansion(url, fetchImpl, timeoutMs);
 		const updatedAt = writeSyncCache(cacheKeyForUrl(url), value);
-		results.push(toExpansionItem(url, value, "network", updatedAt));
+		const item = toExpansionItem(url, value, "network", updatedAt);
+		persistExpansion(item);
+		results.push(item);
 	}
 
 	return results;

package/src/lib/web-sync.ts

@@ -0,0 +1,443 @@
+import { existsSync } from "node:fs";
+import { maybeAutoSyncBackup } from "./backup";
+import { getBirdclawPaths } from "./config";
+import { syncDirectMessagesViaCachedBird } from "./dms-live";
+import { syncMentionThreads } from "./mention-threads-live";
+import { syncMentions } from "./mentions-live";
+import NativeSqliteDatabase from "./sqlite";
+import { syncTimelineCollection } from "./timeline-collections-live";
+import { syncHomeTimeline } from "./timeline-live";
+
+export type WebSyncKind =
+	| "timeline"
+	| "mentions"
+	| "likes"
+	| "bookmarks"
+	| "dms";
+
+export interface WebSyncStep {
+	kind: WebSyncKind | "mention-threads";
+	label: string;
+	count: number;
+	source?: string;
+	partial?: boolean;
+	warnings?: string[];
+}
+
+export interface WebSyncResponse {
+	ok: boolean;
+	kind: WebSyncKind;
+	accountId?: string;
+	startedAt: string;
+	finishedAt?: string;
+	summary: string;
+	steps: WebSyncStep[];
+	inProgress?: boolean;
+	backup?: Awaited<ReturnType<typeof maybeAutoSyncBackup>>;
+	error?: string;
+}
+
+export type WebSyncJobStatus = "running" | "succeeded" | "failed";
+
+export interface WebSyncJobSnapshot {
+	id: string;
+	kind: WebSyncKind;
+	accountId?: string;
+	status: WebSyncJobStatus;
+	startedAt: string;
+	finishedAt?: string;
+	summary: string;
+	inProgress: boolean;
+	result?: WebSyncResponse;
+	error?: string;
+}
+
+interface WebSyncPlan {
+	label: string;
+	accountAware: boolean;
+	run: (accountId: string | undefined) => Promise<WebSyncStep[]>;
+}
+
+const runningSyncs = new Map<string, WebSyncJobSnapshot>();
+const webSyncJobs = new Map<string, WebSyncJobSnapshot>();
+const webSyncJobKeys = new Map<string, string>();
+const completedJobCleanupTimers = new Map<
+	string,
+	ReturnType<typeof setTimeout>
+>();
+const COMPLETED_JOB_TTL_MS = 5 * 60 * 1000;
+
+function assertRecord(
+	value: unknown,
+): asserts value is Record<string, unknown> {
+	if (!value || typeof value !== "object") {
+		throw new Error("Expected sync result object");
+	}
+}
+
+function readNumber(value: unknown, key: string): number {
+	assertRecord(value);
+	const raw = value[key];
+	return typeof raw === "number" && Number.isFinite(raw) ? raw : 0;
+}
+
+function readString(value: unknown, key: string) {
+	assertRecord(value);
+	const raw = value[key];
+	return typeof raw === "string" ? raw : undefined;
+}
+
+function readBoolean(value: unknown, key: string) {
+	assertRecord(value);
+	const raw = value[key];
+	return typeof raw === "boolean" ? raw : undefined;
+}
+
+export function parseWebSyncKind(value: unknown): WebSyncKind | null {
+	return value === "timeline" ||
+		value === "mentions" ||
+		value === "likes" ||
+		value === "bookmarks" ||
+		value === "dms"
+		? value
+		: null;
+}
+
+function summarizeSteps(steps: WebSyncStep[]) {
+	const total = steps.reduce((sum, step) => sum + step.count, 0);
+	const partial = steps.some((step) => step.partial);
+	const suffix = partial ? " (partial)" : "";
+	return `Synced ${String(total)} items${suffix}`;
+}
+
+const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
+	timeline: {
+		label: "Home timeline",
+		accountAware: false,
+		run: async (account) => {
+			const result = await syncHomeTimeline({
+				account,
+				limit: 100,
+				following: true,
+				refresh: true,
+			});
+			return [
+				{
+					kind: "timeline",
+					label: "Home timeline",
+					count: readNumber(result, "count"),
+					source: readString(result, "source"),
+				},
+			];
+		},
+	},
+	mentions: {
+		label: "Mentions",
+		accountAware: true,
+		run: async (account) => {
+			const mentions = await syncMentions({
+				account,
+				mode: "xurl",
+				limit: 100,
+				maxPages: 3,
+				refresh: true,
+			});
+			const steps: WebSyncStep[] = [
+				{
+					kind: "mentions",
+					label: "Mentions",
+					count: readNumber(mentions, "count"),
+					source: readString(mentions, "source"),
+					partial: readBoolean(mentions, "partial"),
+				},
+			];
+
+			const threads = await syncMentionThreads({
+				account,
+				mode: "xurl",
+				limit: 30,
+				delayMs: 1500,
+				timeoutMs: 15000,
+			});
+			steps.push({
+				kind: "mention-threads",
+				label: "Mention threads",
+				count: readNumber(threads, "mergedTweets"),
+				source: readString(threads, "source"),
+				partial: readBoolean(threads, "partial"),
+				warnings:
+					Array.isArray(threads.warnings) && threads.warnings.length > 0
+						? threads.warnings.map(String)
+						: undefined,
+			});
+			return steps;
+		},
+	},
+	likes: {
+		label: "Likes",
+		accountAware: true,
+		run: (account) => syncSavedCollection("likes", account),
+	},
+	bookmarks: {
+		label: "Bookmarks",
+		accountAware: true,
+		run: (account) => syncSavedCollection("bookmarks", account),
+	},
+	dms: {
+		label: "Direct messages",
+		accountAware: false,
+		run: async (account) => {
+			const result = await syncDirectMessagesViaCachedBird({
+				account,
+				limit: 50,
+				refresh: true,
+			});
+			return [
+				{
+					kind: "dms",
+					label: "Direct messages",
+					count: readNumber(result, "messages"),
+					source: readString(result, "source"),
+				},
+			];
+		},
+	},
+};
+
+async function syncSavedCollection(
+	kind: "likes" | "bookmarks",
+	account: string | undefined,
+) {
+	const isNonDefaultAccount =
+		account !== undefined && account !== resolveDefaultSyncAccountId();
+	const result = await syncTimelineCollection({
+		kind,
+		account,
+		mode: isNonDefaultAccount ? "xurl" : "auto",
+		limit: 100,
+		maxPages: 5,
+		refresh: true,
+		earlyStop: true,
+	});
+	return [
+		{
+			kind,
+			label: kind === "likes" ? "Likes" : "Bookmarks",
+			count: readNumber(result, "count"),
+			source: readString(result, "source"),
+		},
+	];
+}
+
+async function performWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+): Promise<WebSyncResponse> {
+	const startedAt = new Date().toISOString();
+	const steps = await WEB_SYNC_PLANS[kind].run(accountId);
+
+	const backup = await maybeAutoSyncBackup();
+	const finishedAt = new Date().toISOString();
+	return {
+		ok: true,
+		kind,
+		...(accountId ? { accountId } : {}),
+		startedAt,
+		finishedAt,
+		summary: summarizeSteps(steps),
+		steps,
+		backup,
+	};
+}
+
+function createWebSyncJobId(kind: WebSyncKind) {
+	return `sync_${kind}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+
+function resolveDefaultSyncAccountId() {
+	const dbPath = getBirdclawPaths().dbPath;
+	if (!existsSync(dbPath)) {
+		return "acct_primary";
+	}
+
+	let db: NativeSqliteDatabase | undefined;
+	try {
+		db = new NativeSqliteDatabase(dbPath, { readonly: true });
+		const row = db
+			.prepare(
+				`
+        select id
+        from accounts
+        order by is_default desc, created_at asc
+        limit 1
+        `,
+			)
+			.get() as { id: string } | undefined;
+		return row?.id ?? "acct_primary";
+	} catch {
+		return "acct_primary";
+	} finally {
+		db?.close();
+	}
+}
+
+function getRunningSyncKey(kind: WebSyncKind, accountId: string | undefined) {
+	if (!WEB_SYNC_PLANS[kind].accountAware) {
+		return kind;
+	}
+	return `${kind}:${accountId ?? resolveDefaultSyncAccountId()}`;
+}
+
+function getEffectiveAccountId(
+	kind: WebSyncKind,
+	accountId: string | undefined,
+) {
+	return WEB_SYNC_PLANS[kind].accountAware ? accountId : undefined;
+}
+
+function setJobSnapshot(snapshot: WebSyncJobSnapshot) {
+	webSyncJobs.set(snapshot.id, snapshot);
+	const syncKey =
+		webSyncJobKeys.get(snapshot.id) ??
+		getRunningSyncKey(snapshot.kind, snapshot.accountId);
+	const cleanupTimer = completedJobCleanupTimers.get(snapshot.id);
+	if (cleanupTimer) {
+		clearTimeout(cleanupTimer);
+		completedJobCleanupTimers.delete(snapshot.id);
+	}
+	if (snapshot.inProgress) {
+		runningSyncs.set(syncKey, snapshot);
+	} else if (runningSyncs.get(syncKey)?.id === snapshot.id) {
+		runningSyncs.delete(syncKey);
+		const timer = setTimeout(() => {
+			webSyncJobs.delete(snapshot.id);
+			webSyncJobKeys.delete(snapshot.id);
+			completedJobCleanupTimers.delete(snapshot.id);
+		}, COMPLETED_JOB_TTL_MS);
+		timer.unref?.();
+		completedJobCleanupTimers.set(snapshot.id, timer);
+	}
+}
+
+function toFailedResponse(
+	kind: WebSyncKind,
+	startedAt: string,
+	error: unknown,
+	accountId?: string,
+): WebSyncResponse {
+	const finishedAt = new Date().toISOString();
+	const message = error instanceof Error ? error.message : "Sync failed";
+	return {
+		ok: false,
+		kind,
+		...(accountId ? { accountId } : {}),
+		startedAt,
+		finishedAt,
+		summary: message,
+		steps: [],
+		error: message,
+	};
+}
+
+export function startWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+): WebSyncJobSnapshot {
+	const effectiveAccountId = getEffectiveAccountId(kind, accountId);
+	const syncKey = getRunningSyncKey(kind, effectiveAccountId);
+	const current = runningSyncs.get(syncKey);
+	if (current) {
+		return current;
+	}
+
+	const startedAt = new Date().toISOString();
+	const job: WebSyncJobSnapshot = {
+		id: createWebSyncJobId(kind),
+		kind,
+		...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
+		status: "running",
+		startedAt,
+		summary: `Syncing ${WEB_SYNC_PLANS[kind].label}`,
+		inProgress: true,
+	};
+	webSyncJobKeys.set(job.id, syncKey);
+	setJobSnapshot(job);
+
+	void performWebSync(kind, effectiveAccountId)
+		.then((result) => {
+			setJobSnapshot({
+				...job,
+				status: "succeeded",
+				finishedAt: result.finishedAt,
+				summary: result.summary,
+				inProgress: false,
+				result,
+			});
+		})
+		.catch((error: unknown) => {
+			const result = toFailedResponse(
+				kind,
+				startedAt,
+				error,
+				effectiveAccountId,
+			);
+			setJobSnapshot({
+				...job,
+				status: "failed",
+				finishedAt: result.finishedAt,
+				summary: result.summary,
+				inProgress: false,
+				result,
+				error: result.error,
+			});
+		});
+
+	return job;
+}
+
+export function getWebSyncJob(id: string): WebSyncJobSnapshot | null {
+	return webSyncJobs.get(id) ?? null;
+}
+
+export async function runWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+): Promise<WebSyncResponse> {
+	const effectiveAccountId = getEffectiveAccountId(kind, accountId);
+	const current = runningSyncs.get(getRunningSyncKey(kind, effectiveAccountId));
+	const startedAt = new Date().toISOString();
+	if (current) {
+		return {
+			ok: false,
+			kind,
+			...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
+			startedAt,
+			summary: "Sync already running",
+			steps: [],
+			inProgress: true,
+		};
+	}
+
+	const job = startWebSync(kind, effectiveAccountId);
+	while (job.inProgress) {
+		await new Promise((resolve) => setTimeout(resolve, 25));
+		const latest = getWebSyncJob(job.id);
+		if (!latest?.inProgress) {
+			if (!latest?.result) throw new Error("Sync job disappeared");
+			return latest.result;
+		}
+	}
+
+	if (!job.result) throw new Error("Sync job did not finish");
+	return job.result;
+}
+
+export function clearWebSyncLocksForTests() {
+	runningSyncs.clear();
+	webSyncJobs.clear();
+	webSyncJobKeys.clear();
+	for (const timer of completedJobCleanupTimers.values()) {
+		clearTimeout(timer);
+	}
+	completedJobCleanupTimers.clear();
+}

package/src/lib/x-profile.ts

@@ -150,6 +150,7 @@ function updateExistingProfileFromUser(
         bio = ?,
         followers_count = ?,
         following_count = coalesce(?, following_count),
+        public_metrics_json = ?,
         avatar_url = coalesce(?, avatar_url),
         location = coalesce(?, location),
         url = coalesce(?, url),
@@ -167,6 +168,7 @@ function updateExistingProfileFromUser(
 		bio,
 		followersCount,
 		followingCount,
+		JSON.stringify(user.public_metrics ?? {}),
 		avatarUrl,
 		metadata.location,
 		metadata.url,
@@ -245,13 +247,15 @@ export function upsertProfileFromXUser(
 		`
     insert into profiles (
       id, handle, display_name, bio, followers_count, following_count, avatar_hue,
-      avatar_url, location, url, verified_type, entities_json, raw_json, created_at
-    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      public_metrics_json, avatar_url, location, url, verified_type, entities_json,
+      raw_json, created_at
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     on conflict(id) do update set
 	      handle = excluded.handle,
 	      display_name = excluded.display_name,
 	      bio = excluded.bio,
 	      followers_count = excluded.followers_count,
+	      public_metrics_json = excluded.public_metrics_json,
 	      following_count = case
 	        when ? then excluded.following_count
 	        else profiles.following_count
@@ -274,6 +278,7 @@ export function upsertProfileFromXUser(
 		followersCount,
 		followingCount ?? 0,
 		avatarHue,
+		JSON.stringify(user.public_metrics ?? {}),
 		avatarUrl,
 		metadata.location,
 		metadata.url,