birdclaw 0.8.1 → 0.8.2

package/src/lib/api-client.ts

@@ -1,5 +1,11 @@
 import { Data, Effect } from "effect";
 import { z } from "zod";
+import {
+	deleteClientCache,
+	deleteClientCacheByPrefix,
+	loadClientCache,
+	readClientCache,
+} from "./client-cache";
 import { runEffectPromise } from "./effect-runtime";
 import type {
 	DmConversationItem,
@@ -105,6 +111,15 @@ const webSyncJobSchema = z
 
 const actionResponseSchema = jsonRecordSchema;
 const SYNC_POLL_INTERVAL_MS = 500;
+const STATUS_CACHE_KEY = "api:/status";
+const QUERY_CACHE_PREFIX = "api:/query:";
+const STATUS_CACHE_MAX_AGE_MS = 60_000;
+const QUERY_CACHE_MAX_AGE_MS = 5 * 60_000;
+
+interface ClientFetchCacheOptions {
+	force?: boolean;
+	maxAgeMs?: number;
+}
 
 export class ApiFetchError extends Data.TaggedError("ApiFetchError")<{
 	readonly message: string;
@@ -148,6 +163,55 @@ function runApiEffect<T, E>(effect: Effect.Effect<T, E>) {
 	return runEffectPromise(effect);
 }
 
+function splitSignal(init?: RequestInit) {
+	if (!init) return { requestInit: undefined, signal: undefined };
+	const { signal, ...requestInit } = init;
+	return { requestInit, signal: signal ?? undefined };
+}
+
+function waitForSignal<T>(promise: Promise<T>, signal?: AbortSignal) {
+	if (!signal) return promise;
+	if (signal.aborted) {
+		return Promise.reject(
+			new DOMException("The operation was aborted.", "AbortError"),
+		);
+	}
+
+	return new Promise<T>((resolve, reject) => {
+		const onAbort = () => {
+			reject(new DOMException("The operation was aborted.", "AbortError"));
+		};
+		signal.addEventListener("abort", onAbort, { once: true });
+		promise.then(
+			(value) => {
+				signal.removeEventListener("abort", onAbort);
+				resolve(value);
+			},
+			(error: unknown) => {
+				signal.removeEventListener("abort", onAbort);
+				reject(error);
+			},
+		);
+	});
+}
+
+function queryCacheKey(input: RequestInfo | URL) {
+	const raw =
+		typeof input === "string"
+			? input
+			: input instanceof URL
+				? input.toString()
+				: input.url;
+	const base =
+		typeof window === "undefined"
+			? "http://birdclaw.local"
+			: window.location.origin;
+	const url = new URL(raw, base);
+	url.searchParams.delete("refresh");
+	url.searchParams.sort();
+	return `${QUERY_CACHE_PREFIX}${url.pathname}?${url.searchParams.toString()}`;
+}
+
 export function fetchJsonEffect<T>(
 	input: RequestInfo | URL,
 	init: RequestInit | undefined,
@@ -191,8 +255,27 @@ export function fetchJson<T>(
 	return runApiEffect(fetchJsonEffect(input, init, schema, fallbackMessage));
 }
 
-export function fetchQueryEnvelope(init?: RequestInit) {
-	return runApiEffect(fetchQueryEnvelopeEffect(init));
+export function readCachedQueryEnvelope() {
+	return readClientCache<QueryEnvelope>(
+		STATUS_CACHE_KEY,
+		STATUS_CACHE_MAX_AGE_MS,
+	);
+}
+
+export function fetchQueryEnvelope(
+	init?: RequestInit,
+	{
+		force = false,
+		maxAgeMs = STATUS_CACHE_MAX_AGE_MS,
+	}: ClientFetchCacheOptions = {},
+) {
+	const { requestInit, signal } = splitSignal(init);
+	const request = loadClientCache(
+		STATUS_CACHE_KEY,
+		() => runApiEffect(fetchQueryEnvelopeEffect(requestInit)),
+		{ force, maxAgeMs },
+	);
+	return waitForSignal(request, signal);
 }
 
 export function fetchQueryEnvelopeEffect(init?: RequestInit) {
@@ -211,6 +294,38 @@ export function fetchQueryResponse(
 	return runApiEffect(fetchQueryResponseEffect(input, init));
 }
 
+export function readCachedQueryResponse(input: RequestInfo | URL) {
+	return readClientCache<QueryResponse>(
+		queryCacheKey(input),
+		QUERY_CACHE_MAX_AGE_MS,
+	);
+}
+
+export function fetchCachedQueryResponse(
+	input: RequestInfo | URL,
+	init?: RequestInit,
+	{
+		force = false,
+		maxAgeMs = QUERY_CACHE_MAX_AGE_MS,
+	}: ClientFetchCacheOptions = {},
+) {
+	const { requestInit, signal } = splitSignal(init);
+	const request = loadClientCache(
+		queryCacheKey(input),
+		() => runApiEffect(fetchQueryResponseEffect(input, requestInit)),
+		{ force, maxAgeMs },
+	);
+	return waitForSignal(request, signal);
+}
+
+export function invalidateCachedQueryResponse(input: RequestInfo | URL) {
+	deleteClientCache(queryCacheKey(input));
+}
+
+export function invalidateCachedQueryResponses() {
+	deleteClientCacheByPrefix(QUERY_CACHE_PREFIX);
+}
+
 export function fetchQueryResponseEffect(
 	input: RequestInfo | URL,
 	init?: RequestInit,

package/src/lib/archive-finder.ts

@@ -8,11 +8,16 @@ import { runEffectPromise, tryPromise } from "./effect-runtime";
 import type { ArchiveCandidate } from "./types";
 
 const execAsync = promisify(exec);
+const ARCHIVE_DISCOVERY_CACHE_MS = 5 * 60_000;
 const ARCHIVE_NAME_PATTERNS = [
 	/^twitter-.*\.zip$/i,
 	/^x-.*\.zip$/i,
 	/archive.*\.zip$/i,
 ];
+let archiveDiscoveryCache:
+	| { value: ArchiveCandidate[]; updatedAt: number }
+	| undefined;
+let archiveDiscoveryInFlight: Promise<ArchiveCandidate[]> | undefined;
 
 function formatFileSize(bytes: number): string {
 	const units = ["B", "KB", "MB", "GB"];
@@ -142,6 +147,39 @@ export function findArchivesEffect(): Effect.Effect<
 	});
 }
 
+export function findArchivesCachedEffect(): Effect.Effect<
+	ArchiveCandidate[],
+	unknown
+> {
+	return Effect.suspend(() => {
+		const cached = archiveDiscoveryCache;
+		if (cached && Date.now() - cached.updatedAt < ARCHIVE_DISCOVERY_CACHE_MS) {
+			return Effect.succeed(cached.value);
+		}
+		if (archiveDiscoveryInFlight) {
+			return Effect.promise(() => archiveDiscoveryInFlight!);
+		}
+
+		const request = runEffectPromise(findArchivesEffect())
+			.then((value) => {
+				archiveDiscoveryCache = { value, updatedAt: Date.now() };
+				return value;
+			})
+			.finally(() => {
+				if (archiveDiscoveryInFlight === request) {
+					archiveDiscoveryInFlight = undefined;
+				}
+			});
+		archiveDiscoveryInFlight = request;
+		return Effect.promise(() => request);
+	});
+}
+
 export function findArchives(): Promise<ArchiveCandidate[]> {
 	return runEffectPromise(findArchivesEffect());
 }
+
+export function clearArchiveFinderCacheForTests() {
+	archiveDiscoveryCache = undefined;
+	archiveDiscoveryInFlight = undefined;
+}

package/src/lib/authored-live.ts

@@ -3,6 +3,7 @@ import { Effect } from "effect";
 import { getNativeDb } from "./db";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	TweetEntities,
 	TweetMediaItem,
@@ -523,68 +524,7 @@ function toMentionData(tweet: XurlUserTweet, fallbackAuthorId: string) {
 }
 
 function toLocalEntities(tweet: XurlMentionData): TweetEntities {
-	const raw = tweet.entities;
-	if (!raw || typeof raw !== "object") {
-		return {};
-	}
-
-	const entities = raw as Record<string, unknown>;
-	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
-	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
-	const rawHashtags = Array.isArray(entities.hashtags) ? entities.hashtags : [];
-
-	return {
-		...(rawMentions.length
-			? {
-					mentions: rawMentions.map((mention) => {
-						const value =
-							mention && typeof mention === "object"
-								? (mention as Record<string, unknown>)
-								: {};
-						return {
-							username: String(value.username ?? ""),
-							id: typeof value.id === "string" ? String(value.id) : undefined,
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawUrls.length
-			? {
-					urls: rawUrls.map((url) => {
-						const value =
-							url && typeof url === "object"
-								? (url as Record<string, unknown>)
-								: {};
-						return {
-							url: String(value.url ?? ""),
-							expandedUrl: String(value.expanded_url ?? value.url ?? ""),
-							displayUrl: String(
-								value.display_url ?? value.expanded_url ?? value.url ?? "",
-							),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawHashtags.length
-			? {
-					hashtags: rawHashtags.map((hashtag) => {
-						const value =
-							hashtag && typeof hashtag === "object"
-								? (hashtag as Record<string, unknown>)
-								: {};
-						return {
-							tag: String(value.tag ?? ""),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-	};
+	return tweetEntitiesFromXurl(tweet.entities);
 }
 
 function toMediaType(type: string): TweetMediaItem["type"] {

package/src/lib/bird.ts

@@ -29,6 +29,12 @@ interface BirdTweetAuthor {
 	name?: string;
 }
 
+interface BirdTweetArticle {
+	title?: string;
+	previewText?: string;
+	coverImageUrl?: string;
+}
+
 interface BirdTweetItem {
 	id: string;
 	text: string;
@@ -45,6 +51,7 @@ interface BirdTweetItem {
 	author?: BirdTweetAuthor;
 	authorId?: string;
 	media?: BirdTweetMedia[];
+	article?: BirdTweetArticle | null;
 }
 
 export interface BirdDmUser {
@@ -385,6 +392,29 @@ function toMediaEntities(media: BirdTweetMedia[] | undefined) {
 	};
 }
 
+function toTweetEntities(item: BirdTweetItem) {
+	const mediaEntities = toMediaEntities(item.media);
+	const title = item.article?.title?.trim();
+	if (!title) return mediaEntities;
+	const handle = item.author?.username?.replace(/^@/, "");
+	const url = handle
+		? `https://x.com/${handle}/status/${item.id}`
+		: `https://x.com/i/status/${item.id}`;
+	return {
+		...mediaEntities,
+		article: {
+			title,
+			url,
+			...(item.article?.previewText?.trim()
+				? { previewText: item.article.previewText.trim() }
+				: {}),
+			...(item.article?.coverImageUrl?.trim()
+				? { coverImageUrl: item.article.coverImageUrl.trim() }
+				: {}),
+		},
+	};
+}
+
 function toReferencedTweets(item: BirdTweetItem) {
 	const references: XurlReferencedTweet[] = [];
 	if (typeof item.inReplyToStatusId === "string" && item.inReplyToStatusId) {
@@ -434,7 +464,7 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse {
 			text: item.text,
 			created_at: toIsoTimestamp(item.createdAt),
 			conversation_id: item.conversationId ?? item.id,
-			entities: toMediaEntities(item.media),
+			entities: toTweetEntities(item),
 			referenced_tweets: toReferencedTweets(item),
 			public_metrics: {
 				reply_count: Number(item.replyCount ?? 0),
@@ -1104,6 +1134,7 @@ export const __test__ = {
 	getBirdTweetItems,
 	getBirdTweetItem,
 	toMediaEntities,
+	toTweetEntities,
 	toReferencedTweets,
 	normalizeBirdTweets,
 };

package/src/lib/client-cache.ts

@@ -0,0 +1,109 @@
+const DEFAULT_MAX_ENTRIES = 100;
+
+interface ClientCacheEntry<T> {
+	value: T;
+	updatedAt: number;
+}
+
+interface LoadClientCacheOptions {
+	force?: boolean;
+	maxAgeMs?: number;
+}
+
+const values = new Map<string, ClientCacheEntry<unknown>>();
+const pending = new Map<string, Promise<unknown>>();
+const revisions = new Map<string, number>();
+let generation = 0;
+
+function cacheToken(key: string) {
+	return `${String(generation)}:${String(revisions.get(key) ?? 0)}`;
+}
+
+function invalidateKey(key: string) {
+	revisions.set(key, (revisions.get(key) ?? 0) + 1);
+}
+
+function pruneCache() {
+	while (values.size > DEFAULT_MAX_ENTRIES) {
+		const oldestKey = values.keys().next().value as string | undefined;
+		if (!oldestKey) return;
+		values.delete(oldestKey);
+	}
+}
+
+export function readClientCache<T>(
+	key: string,
+	maxAgeMs = Number.POSITIVE_INFINITY,
+) {
+	const entry = values.get(key) as ClientCacheEntry<T> | undefined;
+	if (!entry) return undefined;
+	if (Date.now() - entry.updatedAt > maxAgeMs) {
+		values.delete(key);
+		return undefined;
+	}
+	return entry.value;
+}
+
+export function writeClientCache<T>(key: string, value: T) {
+	values.delete(key);
+	values.set(key, { value, updatedAt: Date.now() });
+	pruneCache();
+	return value;
+}
+
+export function loadClientCache<T>(
+	key: string,
+	load: () => Promise<T>,
+	{
+		force = false,
+		maxAgeMs = Number.POSITIVE_INFINITY,
+	}: LoadClientCacheOptions = {},
+) {
+	if (force) {
+		invalidateKey(key);
+		values.delete(key);
+		pending.delete(key);
+	} else {
+		const cached = readClientCache<T>(key, maxAgeMs);
+		if (cached !== undefined) return Promise.resolve(cached);
+	}
+
+	const active = pending.get(key) as Promise<T> | undefined;
+	if (active) return active;
+
+	const token = cacheToken(key);
+	let request: Promise<T>;
+	request = load()
+		.then((value) => {
+			if (cacheToken(key) === token) writeClientCache(key, value);
+			return value;
+		})
+		.finally(() => {
+			if (pending.get(key) === request) pending.delete(key);
+		});
+	pending.set(key, request);
+	return request;
+}
+
+export function deleteClientCache(key: string) {
+	invalidateKey(key);
+	values.delete(key);
+	pending.delete(key);
+}
+
+export function deleteClientCacheByPrefix(prefix: string) {
+	const keys = new Set([...values.keys(), ...pending.keys()]);
+	for (const key of keys) {
+		if (!key.startsWith(prefix)) continue;
+		invalidateKey(key);
+		values.delete(key);
+		pending.delete(key);
+	}
+}
+
+export function clearClientCache() {
+	generation += 1;
+	values.clear();
+	pending.clear();
+	revisions.clear();
+}

package/src/lib/db.ts

@@ -1,4 +1,7 @@
-import NativeSqliteDatabase, { type Database } from "./sqlite";
+import NativeSqliteDatabase, {
+	type Database,
+	SQLITE_BUSY_TIMEOUT_MS,
+} from "./sqlite";
 import { Kysely, SqliteDialect } from "kysely";
 import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
 import { seedDemoData } from "./seed";
@@ -326,7 +329,7 @@ export interface InitDatabaseOptions {
 
 const BASE_SCHEMA_SQL = `
   pragma journal_mode = wal;
-  pragma busy_timeout = 5000;
+  pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
   pragma foreign_keys = on;
 
   create table if not exists accounts (
@@ -948,6 +951,34 @@ function ensureFollowGraphTables(db: Database) {
 }
 
 function backfillTweetCollections(db: Database) {
+	const missingKinds = (
+		[
+			["likes", "liked"],
+			["bookmarks", "bookmarked"],
+		] as const
+	).filter(([, column]) =>
+		db
+			.prepare(
+				`
+        select 1
+        from tweets as tweet
+        where tweet.${column} = 1
+          and not exists (
+            select 1
+            from tweet_collections as collection
+            where collection.account_id = tweet.account_id
+              and collection.tweet_id = tweet.id
+              and collection.kind = ?
+          )
+        limit 1
+      `,
+			)
+			.get(column === "liked" ? "likes" : "bookmarks"),
+	);
+	if (missingKinds.length === 0) {
+		return;
+	}
+
 	const now = new Date().toISOString();
 	const insert = db.prepare(`
     insert or ignore into tweet_collections (
@@ -963,12 +994,34 @@ function backfillTweetCollections(db: Database) {
   `);
 
 	db.transaction(() => {
-		insert.run("likes", now, "likes");
-		insert.run("bookmarks", now, "bookmarks");
+		for (const [kind] of missingKinds) {
+			insert.run(kind, now, kind);
+		}
 	})();
 }
 
 function backfillTweetAccountEdges(db: Database) {
+	const missing = db
+		.prepare(
+			`
+      select 1
+      from tweets as tweet
+      where tweet.kind in ('home', 'mention')
+        and not exists (
+          select 1
+          from tweet_account_edges as edge
+          where edge.account_id = tweet.account_id
+            and edge.tweet_id = tweet.id
+            and edge.kind = tweet.kind
+        )
+      limit 1
+    `,
+		)
+		.get();
+	if (!missing) {
+		return;
+	}
+
 	const now = new Date().toISOString();
 	db.prepare(`
     insert or ignore into tweet_account_edges (

package/src/lib/mention-threads-live.ts

@@ -4,6 +4,7 @@ import { listThreadViaBirdEffect } from "./bird";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
@@ -415,7 +416,7 @@ function mergeMentionThreadIntoLocalStore({
 				replyToId ?? null,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 			);
 			if (writeThreadContextEdges) {

package/src/lib/mentions-live.ts

@@ -11,6 +11,7 @@ import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { serializeMentionItemsAsXurlCompatible } from "./mentions-export";
 import { listTimelineItems } from "./queries";
 import { deleteSyncCache, readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	ReplyFilter,
 	TweetEntities,
@@ -521,52 +522,7 @@ function findNewestArchiveMentionId(db: Database, accountId: string) {
 }
 
 function toLocalEntities(tweet: XurlMentionData): TweetEntities {
-	const raw = tweet.entities;
-	if (!raw || typeof raw !== "object") {
-		return {};
-	}
-
-	const entities = raw as Record<string, unknown>;
-	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
-	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
-
-	return {
-		...(rawMentions.length
-			? {
-					mentions: rawMentions.map((mention) => {
-						const value =
-							mention && typeof mention === "object"
-								? (mention as Record<string, unknown>)
-								: {};
-						return {
-							username: String(value.username ?? ""),
-							id: typeof value.id === "string" ? String(value.id) : undefined,
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawUrls.length
-			? {
-					urls: rawUrls.map((url) => {
-						const value =
-							url && typeof url === "object"
-								? (url as Record<string, unknown>)
-								: {};
-						return {
-							url: String(value.url ?? ""),
-							expandedUrl: String(value.expanded_url ?? value.url ?? ""),
-							displayUrl: String(
-								value.display_url ?? value.expanded_url ?? value.url ?? "",
-							),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-	};
+	return tweetEntitiesFromXurl(tweet.entities);
 }
 
 function replaceTweetFts(db: Database, tweetId: string, text: string) {

package/src/lib/profile-analysis.ts

@@ -407,7 +407,7 @@ function mergeXurlTweetsIntoLocalStore(
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/queries.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { Effect } from "effect";
 import type { Database } from "./sqlite";
-import { findArchivesEffect } from "./archive-finder";
+import { findArchivesCachedEffect } from "./archive-finder";
 import { getDb, getNativeDb } from "./db";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { fetchProfileAffiliations } from "./profile-affiliations";
@@ -701,10 +701,9 @@ function getAccountProfileMeta(
 		| undefined;
 }
 
-export function getQueryEnvelopeEffect(): Effect.Effect<
-	QueryEnvelope,
-	unknown
-> {
+export function getQueryEnvelopeEffect({
+	includeArchives = true,
+}: { includeArchives?: boolean } = {}): Effect.Effect<QueryEnvelope, unknown> {
 	return Effect.gen(function* () {
 		const db = yield* trySync(() => getDb());
 		const nativeDb = yield* trySync(() => getNativeDb());
@@ -736,7 +735,9 @@ export function getQueryEnvelopeEffect(): Effect.Effect<
 					.orderBy("name", "asc")
 					.execute(),
 			),
-			archives: findArchivesEffect(),
+			archives: includeArchives
+				? findArchivesCachedEffect()
+				: Effect.succeed([]),
 			transport: getTransportStatusEffect(),
 		});