birdclaw 0.5.0 → 0.5.1

package/src/lib/queries.ts

@@ -321,32 +321,39 @@ function countTimelineEdges(db: Database, kind: "home" | "mention") {
 	const row = db
 		.prepare(
 			`
-      with timeline_edges as (
-        select account_id, tweet_id, kind
-        from tweet_account_edges
-        where kind = ?
-        union all
-        select legacy.account_id, legacy.id as tweet_id, legacy.kind
-        from tweets legacy
-        where legacy.kind = ?
-          and not exists (
-            select 1
-            from tweet_account_edges edge
-            where edge.account_id = legacy.account_id
-              and edge.tweet_id = legacy.id
-              and edge.kind = legacy.kind
-          )
-      )
-      select count(*) as count
-      from timeline_edges e
-      join tweets t on t.id = e.tweet_id
-      where e.kind = ?
+      select (
+	        (
+	          select count(*)
+	          from tweet_account_edges edge
+	          where edge.kind = ?
+	            and exists (
+	              select 1
+	              from tweets t
+	              where t.id = edge.tweet_id
+	            )
+	        )
+        +
+        (
+          select count(*)
+          from tweets legacy
+          where legacy.kind = ?
+            and not exists (
+              select 1
+              from tweet_account_edges edge
+              where edge.account_id = legacy.account_id
+                and edge.tweet_id = legacy.id
+                and edge.kind = legacy.kind
+            )
+        )
+      ) as count
       `,
 		)
-		.get(kind, kind, kind) as { count: number | bigint } | undefined;
+		.get(kind, kind) as { count: number | bigint } | undefined;
 	return Number(row?.count ?? 0);
 }
 
+const RECENT_TIMELINE_EDGE_CANDIDATES = 5000;
+
 export async function getQueryEnvelope(): Promise<QueryEnvelope> {
 	const db = getDb();
 	const nativeDb = getNativeDb();
@@ -432,10 +439,23 @@ export function listTimelineItems({
           )
       )
     `;
+	const unwindowedTimelineEdgesCte = timelineEdgesCte;
+	let usedRecentEdgeWindow = false;
 	let join = "";
 	let where = "where t.kind = ?";
 	let searchSnippetSelect = "";
 
+	const canUseRecentEdgeWindow =
+		!likedOnly &&
+		!bookmarkedOnly &&
+		!account &&
+		!search?.trim() &&
+		replyFilter === "all" &&
+		!since?.trim() &&
+		!until?.trim() &&
+		includeReplies &&
+		qualityFilter === "all";
+
 	if (likedOnly || bookmarkedOnly) {
 		if (likedOnly && bookmarkedOnly) {
 			timelineEdgesCte = `
@@ -481,10 +501,49 @@ export function listTimelineItems({
                 and collection.kind = ?
             )
         )
-      `;
+			`;
 			params.push(collectionKind, collectionKind);
 		}
 		where = "where 1 = 1";
+	} else if (canUseRecentEdgeWindow) {
+		usedRecentEdgeWindow = true;
+		timelineEdgesCte = `
+      with timeline_edges as (
+        select account_id, tweet_id, kind
+        from tweet_account_edges
+        where kind = ?
+          and tweet_id in (
+            select id
+            from tweets
+            order by created_at desc
+            limit ?
+          )
+        union all
+        select legacy.account_id, legacy.id as tweet_id, legacy.kind
+        from tweets legacy
+        where legacy.kind = ?
+          and legacy.id in (
+            select id
+            from tweets
+            order by created_at desc
+            limit ?
+          )
+          and not exists (
+            select 1
+            from tweet_account_edges edge
+            where edge.account_id = legacy.account_id
+              and edge.tweet_id = legacy.id
+              and edge.kind = legacy.kind
+          )
+      )
+    `;
+		const candidateLimit = Math.max(
+			RECENT_TIMELINE_EDGE_CANDIDATES,
+			limit * 50,
+		);
+		params.push(kind, candidateLimit, kind, candidateLimit);
+		where = "where e.kind = ?";
+		params.push(kind);
 	} else {
 		params.push(kind, kind);
 		where = "where e.kind = ?";
@@ -532,10 +591,8 @@ export function listTimelineItems({
 
 	params.push(limit);
 
-	const rows = db
-		.prepare(
-			`
-      ${timelineEdgesCte}
+	const buildTimelineSelectSql = (timelineEdgesSql: string) => `
+      ${timelineEdgesSql}
       select
         t.id,
         e.account_id,
@@ -626,10 +683,18 @@ export function listTimelineItems({
       ${where}
       order by t.created_at desc
       limit ?
-      `,
-		)
+      `;
+
+	let rows = db
+		.prepare(buildTimelineSelectSql(timelineEdgesCte))
 		.all(...params) as Array<Record<string, unknown>>;
 
+	if (usedRecentEdgeWindow && rows.length < limit) {
+		rows = db
+			.prepare(buildTimelineSelectSql(unwindowedTimelineEdgesCte))
+			.all(kind, kind, kind, limit) as Array<Record<string, unknown>>;
+	}
+
 	const urlExpansionCache: UrlExpansionCache = new Map();
 	return rows.map((row) => {
 		const author = {

package/src/lib/ui.ts

@@ -15,8 +15,7 @@ export const sidebarShellClass =
 export const sidebarBrandClass =
 	"flex items-center gap-2.5 px-2 py-2 text-[var(--ink)] min-[1100px]:px-3";
 
-export const sidebarBrandMarkClass =
-	"grid size-9 place-items-center rounded-full bg-[var(--accent)] text-white";
+export const sidebarBrandMarkClass = "grid size-10 place-items-center";
 
 export const sidebarBrandCopyClass =
 	"hidden flex-col leading-tight min-[1100px]:flex";
@@ -121,6 +120,15 @@ export const feedRowBadgeAlertClass =
 export const feedRowBadgeNeutralClass =
 	"bg-[var(--bg-active)] text-[var(--ink-soft)]";
 
+export const feedRowStatePillClass =
+	"inline-flex shrink-0 items-center gap-1 rounded-full border border-[var(--line)] bg-[var(--bg-active)] px-2 py-0.5 text-[12px] font-semibold text-[var(--ink-soft)]";
+
+export const feedRowStatePillActiveClass =
+	"border-[color:color-mix(in_srgb,var(--accent)_35%,var(--line))] bg-[var(--accent-soft)] text-[var(--accent)]";
+
+export const feedRowStatePillOpenClass =
+	"border-[color:color-mix(in_srgb,var(--ink-soft)_34%,var(--line))] bg-transparent text-[var(--ink-soft)]";
+
 /* Forms / inputs. */
 export const searchFieldShellClass =
 	"flex items-center gap-2 rounded-full border border-transparent bg-[var(--bg-active)] px-4 py-2 transition-colors focus-within:border-[var(--accent)] focus-within:bg-[var(--bg)] focus-within:shadow-[0_0_0_1px_var(--accent)]";
@@ -328,7 +336,7 @@ export function tweetMediaGridClass(count: number) {
 
 export function tweetMediaTileClass(index: number, count: number) {
 	return cx(
-		"tweet-media-tile relative block overflow-hidden bg-[var(--bg-active)]",
+		"tweet-media-tile relative block overflow-hidden border-0 bg-[var(--bg-active)] p-0 text-left",
 		count === 1 && "aspect-[16/10]",
 		count === 2 && "aspect-square",
 		count === 3 && index === 0 && "row-span-2 aspect-[3/4]",

package/src/lib/web-sync.ts

@@ -0,0 +1,443 @@
+import { existsSync } from "node:fs";
+import { maybeAutoSyncBackup } from "./backup";
+import { getBirdclawPaths } from "./config";
+import { syncDirectMessagesViaCachedBird } from "./dms-live";
+import { syncMentionThreads } from "./mention-threads-live";
+import { syncMentions } from "./mentions-live";
+import NativeSqliteDatabase from "./sqlite";
+import { syncTimelineCollection } from "./timeline-collections-live";
+import { syncHomeTimeline } from "./timeline-live";
+
+export type WebSyncKind =
+	| "timeline"
+	| "mentions"
+	| "likes"
+	| "bookmarks"
+	| "dms";
+
+export interface WebSyncStep {
+	kind: WebSyncKind | "mention-threads";
+	label: string;
+	count: number;
+	source?: string;
+	partial?: boolean;
+	warnings?: string[];
+}
+
+export interface WebSyncResponse {
+	ok: boolean;
+	kind: WebSyncKind;
+	accountId?: string;
+	startedAt: string;
+	finishedAt?: string;
+	summary: string;
+	steps: WebSyncStep[];
+	inProgress?: boolean;
+	backup?: Awaited<ReturnType<typeof maybeAutoSyncBackup>>;
+	error?: string;
+}
+
+export type WebSyncJobStatus = "running" | "succeeded" | "failed";
+
+export interface WebSyncJobSnapshot {
+	id: string;
+	kind: WebSyncKind;
+	accountId?: string;
+	status: WebSyncJobStatus;
+	startedAt: string;
+	finishedAt?: string;
+	summary: string;
+	inProgress: boolean;
+	result?: WebSyncResponse;
+	error?: string;
+}
+
+interface WebSyncPlan {
+	label: string;
+	accountAware: boolean;
+	run: (accountId: string | undefined) => Promise<WebSyncStep[]>;
+}
+
+const runningSyncs = new Map<string, WebSyncJobSnapshot>();
+const webSyncJobs = new Map<string, WebSyncJobSnapshot>();
+const webSyncJobKeys = new Map<string, string>();
+const completedJobCleanupTimers = new Map<
+	string,
+	ReturnType<typeof setTimeout>
+>();
+const COMPLETED_JOB_TTL_MS = 5 * 60 * 1000;
+
+function assertRecord(
+	value: unknown,
+): asserts value is Record<string, unknown> {
+	if (!value || typeof value !== "object") {
+		throw new Error("Expected sync result object");
+	}
+}
+
+function readNumber(value: unknown, key: string): number {
+	assertRecord(value);
+	const raw = value[key];
+	return typeof raw === "number" && Number.isFinite(raw) ? raw : 0;
+}
+
+function readString(value: unknown, key: string) {
+	assertRecord(value);
+	const raw = value[key];
+	return typeof raw === "string" ? raw : undefined;
+}
+
+function readBoolean(value: unknown, key: string) {
+	assertRecord(value);
+	const raw = value[key];
+	return typeof raw === "boolean" ? raw : undefined;
+}
+
+export function parseWebSyncKind(value: unknown): WebSyncKind | null {
+	return value === "timeline" ||
+		value === "mentions" ||
+		value === "likes" ||
+		value === "bookmarks" ||
+		value === "dms"
+		? value
+		: null;
+}
+
+function summarizeSteps(steps: WebSyncStep[]) {
+	const total = steps.reduce((sum, step) => sum + step.count, 0);
+	const partial = steps.some((step) => step.partial);
+	const suffix = partial ? " (partial)" : "";
+	return `Synced ${String(total)} items${suffix}`;
+}
+
+const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
+	timeline: {
+		label: "Home timeline",
+		accountAware: false,
+		run: async (account) => {
+			const result = await syncHomeTimeline({
+				account,
+				limit: 100,
+				following: true,
+				refresh: true,
+			});
+			return [
+				{
+					kind: "timeline",
+					label: "Home timeline",
+					count: readNumber(result, "count"),
+					source: readString(result, "source"),
+				},
+			];
+		},
+	},
+	mentions: {
+		label: "Mentions",
+		accountAware: true,
+		run: async (account) => {
+			const mentions = await syncMentions({
+				account,
+				mode: "xurl",
+				limit: 100,
+				maxPages: 3,
+				refresh: true,
+			});
+			const steps: WebSyncStep[] = [
+				{
+					kind: "mentions",
+					label: "Mentions",
+					count: readNumber(mentions, "count"),
+					source: readString(mentions, "source"),
+					partial: readBoolean(mentions, "partial"),
+				},
+			];
+
+			const threads = await syncMentionThreads({
+				account,
+				mode: "xurl",
+				limit: 30,
+				delayMs: 1500,
+				timeoutMs: 15000,
+			});
+			steps.push({
+				kind: "mention-threads",
+				label: "Mention threads",
+				count: readNumber(threads, "mergedTweets"),
+				source: readString(threads, "source"),
+				partial: readBoolean(threads, "partial"),
+				warnings:
+					Array.isArray(threads.warnings) && threads.warnings.length > 0
+						? threads.warnings.map(String)
+						: undefined,
+			});
+			return steps;
+		},
+	},
+	likes: {
+		label: "Likes",
+		accountAware: true,
+		run: (account) => syncSavedCollection("likes", account),
+	},
+	bookmarks: {
+		label: "Bookmarks",
+		accountAware: true,
+		run: (account) => syncSavedCollection("bookmarks", account),
+	},
+	dms: {
+		label: "Direct messages",
+		accountAware: false,
+		run: async (account) => {
+			const result = await syncDirectMessagesViaCachedBird({
+				account,
+				limit: 50,
+				refresh: true,
+			});
+			return [
+				{
+					kind: "dms",
+					label: "Direct messages",
+					count: readNumber(result, "messages"),
+					source: readString(result, "source"),
+				},
+			];
+		},
+	},
+};
+
+async function syncSavedCollection(
+	kind: "likes" | "bookmarks",
+	account: string | undefined,
+) {
+	const isNonDefaultAccount =
+		account !== undefined && account !== resolveDefaultSyncAccountId();
+	const result = await syncTimelineCollection({
+		kind,
+		account,
+		mode: isNonDefaultAccount ? "xurl" : "auto",
+		limit: 100,
+		maxPages: 5,
+		refresh: true,
+		earlyStop: true,
+	});
+	return [
+		{
+			kind,
+			label: kind === "likes" ? "Likes" : "Bookmarks",
+			count: readNumber(result, "count"),
+			source: readString(result, "source"),
+		},
+	];
+}
+
+async function performWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+): Promise<WebSyncResponse> {
+	const startedAt = new Date().toISOString();
+	const steps = await WEB_SYNC_PLANS[kind].run(accountId);
+
+	const backup = await maybeAutoSyncBackup();
+	const finishedAt = new Date().toISOString();
+	return {
+		ok: true,
+		kind,
+		...(accountId ? { accountId } : {}),
+		startedAt,
+		finishedAt,
+		summary: summarizeSteps(steps),
+		steps,
+		backup,
+	};
+}
+
+function createWebSyncJobId(kind: WebSyncKind) {
+	return `sync_${kind}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+
+function resolveDefaultSyncAccountId() {
+	const dbPath = getBirdclawPaths().dbPath;
+	if (!existsSync(dbPath)) {
+		return "acct_primary";
+	}
+
+	let db: NativeSqliteDatabase | undefined;
+	try {
+		db = new NativeSqliteDatabase(dbPath, { readonly: true });
+		const row = db
+			.prepare(
+				`
+        select id
+        from accounts
+        order by is_default desc, created_at asc
+        limit 1
+        `,
+			)
+			.get() as { id: string } | undefined;
+		return row?.id ?? "acct_primary";
+	} catch {
+		return "acct_primary";
+	} finally {
+		db?.close();
+	}
+}
+
+function getRunningSyncKey(kind: WebSyncKind, accountId: string | undefined) {
+	if (!WEB_SYNC_PLANS[kind].accountAware) {
+		return kind;
+	}
+	return `${kind}:${accountId ?? resolveDefaultSyncAccountId()}`;
+}
+
+function getEffectiveAccountId(
+	kind: WebSyncKind,
+	accountId: string | undefined,
+) {
+	return WEB_SYNC_PLANS[kind].accountAware ? accountId : undefined;
+}
+
+function setJobSnapshot(snapshot: WebSyncJobSnapshot) {
+	webSyncJobs.set(snapshot.id, snapshot);
+	const syncKey =
+		webSyncJobKeys.get(snapshot.id) ??
+		getRunningSyncKey(snapshot.kind, snapshot.accountId);
+	const cleanupTimer = completedJobCleanupTimers.get(snapshot.id);
+	if (cleanupTimer) {
+		clearTimeout(cleanupTimer);
+		completedJobCleanupTimers.delete(snapshot.id);
+	}
+	if (snapshot.inProgress) {
+		runningSyncs.set(syncKey, snapshot);
+	} else if (runningSyncs.get(syncKey)?.id === snapshot.id) {
+		runningSyncs.delete(syncKey);
+		const timer = setTimeout(() => {
+			webSyncJobs.delete(snapshot.id);
+			webSyncJobKeys.delete(snapshot.id);
+			completedJobCleanupTimers.delete(snapshot.id);
+		}, COMPLETED_JOB_TTL_MS);
+		timer.unref?.();
+		completedJobCleanupTimers.set(snapshot.id, timer);
+	}
+}
+
+function toFailedResponse(
+	kind: WebSyncKind,
+	startedAt: string,
+	error: unknown,
+	accountId?: string,
+): WebSyncResponse {
+	const finishedAt = new Date().toISOString();
+	const message = error instanceof Error ? error.message : "Sync failed";
+	return {
+		ok: false,
+		kind,
+		...(accountId ? { accountId } : {}),
+		startedAt,
+		finishedAt,
+		summary: message,
+		steps: [],
+		error: message,
+	};
+}
+
+export function startWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+): WebSyncJobSnapshot {
+	const effectiveAccountId = getEffectiveAccountId(kind, accountId);
+	const syncKey = getRunningSyncKey(kind, effectiveAccountId);
+	const current = runningSyncs.get(syncKey);
+	if (current) {
+		return current;
+	}
+
+	const startedAt = new Date().toISOString();
+	const job: WebSyncJobSnapshot = {
+		id: createWebSyncJobId(kind),
+		kind,
+		...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
+		status: "running",
+		startedAt,
+		summary: `Syncing ${WEB_SYNC_PLANS[kind].label}`,
+		inProgress: true,
+	};
+	webSyncJobKeys.set(job.id, syncKey);
+	setJobSnapshot(job);
+
+	void performWebSync(kind, effectiveAccountId)
+		.then((result) => {
+			setJobSnapshot({
+				...job,
+				status: "succeeded",
+				finishedAt: result.finishedAt,
+				summary: result.summary,
+				inProgress: false,
+				result,
+			});
+		})
+		.catch((error: unknown) => {
+			const result = toFailedResponse(
+				kind,
+				startedAt,
+				error,
+				effectiveAccountId,
+			);
+			setJobSnapshot({
+				...job,
+				status: "failed",
+				finishedAt: result.finishedAt,
+				summary: result.summary,
+				inProgress: false,
+				result,
+				error: result.error,
+			});
+		});
+
+	return job;
+}
+
+export function getWebSyncJob(id: string): WebSyncJobSnapshot | null {
+	return webSyncJobs.get(id) ?? null;
+}
+
+export async function runWebSync(
+	kind: WebSyncKind,
+	accountId?: string,
+): Promise<WebSyncResponse> {
+	const effectiveAccountId = getEffectiveAccountId(kind, accountId);
+	const current = runningSyncs.get(getRunningSyncKey(kind, effectiveAccountId));
+	const startedAt = new Date().toISOString();
+	if (current) {
+		return {
+			ok: false,
+			kind,
+			...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
+			startedAt,
+			summary: "Sync already running",
+			steps: [],
+			inProgress: true,
+		};
+	}
+
+	const job = startWebSync(kind, effectiveAccountId);
+	while (job.inProgress) {
+		await new Promise((resolve) => setTimeout(resolve, 25));
+		const latest = getWebSyncJob(job.id);
+		if (!latest?.inProgress) {
+			if (!latest?.result) throw new Error("Sync job disappeared");
+			return latest.result;
+		}
+	}
+
+	if (!job.result) throw new Error("Sync job did not finish");
+	return job.result;
+}
+
+export function clearWebSyncLocksForTests() {
+	runningSyncs.clear();
+	webSyncJobs.clear();
+	webSyncJobKeys.clear();
+	for (const timer of completedJobCleanupTimers.values()) {
+		clearTimeout(timer);
+	}
+	completedJobCleanupTimers.clear();
+}