birdclaw 0.8.1 → 0.8.2

package/src/lib/sqlite.ts

@@ -5,6 +5,7 @@ export type Database = NativeSqliteDatabase;
 type DatabaseOptions = {
 	readonly?: boolean;
 	fileMustExist?: boolean;
+	timeout?: number;
 };
 
 type PragmaOptions = {
@@ -16,6 +17,8 @@ type RunResult = {
 	lastInsertRowid: number;
 };
 
+export const SQLITE_BUSY_TIMEOUT_MS = 30_000;
+
 function bindArgs(parameters: unknown[]) {
 	if (parameters.length === 1 && Array.isArray(parameters[0])) {
 		return parameters[0];
@@ -87,7 +90,7 @@ export class NativeSqliteDatabase {
 	constructor(path: string, options: DatabaseOptions = {}) {
 		this.db = new DatabaseSync(path, {
 			readOnly: options.readonly,
-			timeout: 5000,
+			timeout: options.timeout ?? SQLITE_BUSY_TIMEOUT_MS,
 		});
 	}
 
@@ -123,7 +126,7 @@ export class NativeSqliteDatabase {
 		return (...args: TArgs) => {
 			const nested = this.db.isTransaction;
 			const savepoint = `__birdclaw_tx_${++this.transactionDepth}`;
-			this.exec(nested ? `savepoint ${savepoint}` : "begin");
+			this.exec(nested ? `savepoint ${savepoint}` : "begin immediate");
 			try {
 				const result = fn(...args);
 				this.exec(nested ? `release ${savepoint}` : "commit");

package/src/lib/timeline-collections-live.ts

@@ -8,6 +8,7 @@ import { getNativeDb } from "./db";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
@@ -310,7 +311,7 @@ function mergeTimelineCollectionIntoLocalStore(
 				countTweetMedia(tweet),
 				bookmarked,
 				liked,
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/timeline-live.ts

@@ -5,6 +5,7 @@ import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	XurlMediaItem,
 	XurlMentionUser,
@@ -244,7 +245,7 @@ function mergeHomeTimelineIntoLocalStore(
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/tweet-render.ts

@@ -1,4 +1,5 @@
 import type {
+	TweetArticle,
 	TweetEntities,
 	TweetHashtagEntity,
 	TweetMentionEntity,
@@ -57,6 +58,34 @@ export function displayUrlForLink(url: string) {
 	}
 }
 
+function comparableUrl(value: string) {
+	try {
+		const parsed = new URL(value);
+		return `${parsed.protocol}//${parsed.hostname.replace(/^www\./, "")}${parsed.pathname}`;
+	} catch {
+		return value.split("?")[0] ?? value;
+	}
+}
+
+export function isTweetArticleUrlEntity(
+	entry: TweetUrlEntity,
+	article: TweetArticle,
+) {
+	if (comparableUrl(entry.expandedUrl) === comparableUrl(article.url)) {
+		return true;
+	}
+	try {
+		const parsed = new URL(entry.expandedUrl);
+		const host = parsed.hostname.replace(/^www\./, "");
+		return (
+			(host === "x.com" || host === "twitter.com") &&
+			parsed.pathname.startsWith("/i/article/")
+		);
+	} catch {
+		return false;
+	}
+}
+
 function asRecord(value: unknown) {
 	return value && typeof value === "object"
 		? (value as Record<string, unknown>)
@@ -68,6 +97,9 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
 	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
 	const rawHashtags = Array.isArray(entities.hashtags) ? entities.hashtags : [];
+	const rawArticle = asRecord(entities.article);
+	const articleTitle = String(rawArticle.title ?? "").trim();
+	const articleUrl = String(rawArticle.url ?? "").trim();
 
 	return {
 		...(rawMentions.length
@@ -102,6 +134,23 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 							),
 							start: Number(value.start ?? 0),
 							end: Number(value.end ?? 0),
+							...(typeof value.title === "string"
+								? { title: value.title }
+								: {}),
+							...(typeof value.description === "string" ||
+							value.description === null
+								? { description: value.description }
+								: {}),
+							...(typeof value.imageUrl === "string"
+								? { imageUrl: value.imageUrl }
+								: typeof value.image_url === "string"
+									? { imageUrl: value.image_url }
+									: {}),
+							...(typeof value.siteName === "string"
+								? { siteName: value.siteName }
+								: typeof value.site_name === "string"
+									? { siteName: value.site_name }
+									: {}),
 						};
 					}),
 				}
@@ -118,6 +167,35 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 					}),
 				}
 			: {}),
+		...(articleTitle && articleUrl
+			? {
+					article: {
+						title: articleTitle,
+						url: articleUrl,
+						...(typeof (rawArticle.previewText ?? rawArticle.preview_text) ===
+							"string" &&
+						String(rawArticle.previewText ?? rawArticle.preview_text).trim()
+							? {
+									previewText: String(
+										rawArticle.previewText ?? rawArticle.preview_text,
+									).trim(),
+								}
+							: {}),
+						...(typeof (
+							rawArticle.coverImageUrl ?? rawArticle.cover_image_url
+						) === "string" &&
+						String(
+							rawArticle.coverImageUrl ?? rawArticle.cover_image_url,
+						).trim()
+							? {
+									coverImageUrl: String(
+										rawArticle.coverImageUrl ?? rawArticle.cover_image_url,
+									).trim(),
+								}
+							: {}),
+					},
+				}
+			: {}),
 	};
 }
 

package/src/lib/tweet-search-live.ts

@@ -5,6 +5,7 @@ import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type { XurlMentionsResponse, XurlTweetsResponse } from "./types";
 import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
@@ -190,7 +191,7 @@ function mergeTweetSearchIntoLocalStore(
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/types.ts

@@ -101,10 +101,18 @@ export interface TweetHashtagEntity {
 	end: number;
 }
 
+export interface TweetArticle {
+	title: string;
+	previewText?: string;
+	url: string;
+	coverImageUrl?: string;
+}
+
 export interface TweetEntities {
 	mentions?: TweetMentionEntity[];
 	urls?: TweetUrlEntity[];
 	hashtags?: TweetHashtagEntity[];
+	article?: TweetArticle;
 }
 
 export interface TweetMediaItem {

package/src/lib/ui.ts

@@ -311,7 +311,7 @@ export const profilePreviewTriggerClass =
 	"profile-preview-trigger inline-flex text-inherit";
 
 export const profilePreviewCardClass =
-	"pointer-events-none absolute left-0 z-30 grid w-[280px] gap-2 rounded-2xl border border-[var(--line)] bg-[var(--bg-elevated)] p-3 opacity-0 shadow-[0_8px_28px_var(--shadow-strong)] transition-all duration-150 group-hover:pointer-events-auto group-hover:opacity-100 group-focus-within:pointer-events-auto group-focus-within:opacity-100";
+	"fixed z-40 w-[280px] overflow-y-auto rounded-2xl border border-[var(--line)] bg-[var(--bg-elevated)] p-3 shadow-[0_8px_28px_var(--shadow-strong)]";
 
 export const profilePreviewHeaderClass = "flex items-center gap-3";
 

package/src/router.tsx

@@ -7,7 +7,7 @@ export function getRouter() {
 
 		scrollRestoration: true,
 		defaultPreload: "intent",
-		defaultPreloadStaleTime: 0,
+		defaultPreloadStaleTime: 60_000,
 	});
 
 	return router;

package/src/routes/__root.tsx

@@ -1,11 +1,9 @@
-import { TanStackDevtools } from "@tanstack/react-devtools";
 import {
 	createRootRoute,
 	HeadContent,
 	Scripts,
 	useRouterState,
 } from "@tanstack/react-router";
-import { TanStackRouterDevtoolsPanel } from "@tanstack/react-router-devtools";
 import type { ReactNode } from "react";
 import { AppNav } from "#/components/AppNav";
 import { ThemeProvider, themeScript } from "#/lib/theme";
@@ -74,17 +72,6 @@ function RootDocument({ children }: { children: ReactNode }) {
 						</main>
 					</div>
 				</ThemeProvider>
-				<TanStackDevtools
-					config={{
-						position: "bottom-right",
-					}}
-					plugins={[
-						{
-							name: "Tanstack Router",
-							render: <TanStackRouterDevtoolsPanel />,
-						},
-					]}
-				/>
 				<Scripts />
 			</body>
 		</html>

package/src/routes/api/status.tsx

@@ -18,7 +18,9 @@ export const Route = createFileRoute("/api/status")({
 						if (denied) return denied;
 
 						yield* maybeAutoUpdateBackupEffect();
-						return jsonResponse(yield* getQueryEnvelopeEffect());
+						return jsonResponse(
+							yield* getQueryEnvelopeEffect({ includeArchives: false }),
+						);
 					}),
 				),
 		},

package/src/routes/dms.tsx

@@ -6,9 +6,11 @@ import { FeedEmpty, FeedError, FeedLoading } from "#/components/FeedState";
 import { SyncNowButton } from "#/components/SyncNowButton";
 import { useSelectedAccountId } from "#/components/account-selection";
 import {
+	fetchCachedQueryResponse,
 	fetchQueryEnvelope,
-	fetchQueryResponse,
+	invalidateCachedQueryResponses,
 	postAction,
+	readCachedQueryResponse,
 } from "#/lib/api-client";
 import type {
 	DmConversationItem,
@@ -16,6 +18,7 @@ import type {
 	QueryEnvelope,
 	ReplyFilter,
 } from "#/lib/types";
+import { useDebouncedValue } from "#/components/useDebouncedValue";
 import {
 	cx,
 	pageHeaderClass,
@@ -83,9 +86,10 @@ function DmsRoute() {
 	const [error, setError] = useState<string | null>(null);
 	const [replyError, setReplyError] = useState<string | null>(null);
 	const selectedAccountId = useSelectedAccountId(meta?.accounts);
+	const debouncedSearch = useDebouncedValue(search, 180);
 
-	async function loadStatus() {
-		setMeta(await fetchQueryEnvelope());
+	async function loadStatus(force = false) {
+		setMeta(await fetchQueryEnvelope(undefined, { force }));
 	}
 
 	useEffect(() => {
@@ -113,29 +117,41 @@ function DmsRoute() {
 		if (selectedConversationId) {
 			url.searchParams.set("conversationId", selectedConversationId);
 		}
-		if (search.trim()) {
-			url.searchParams.set("search", search.trim());
+		if (debouncedSearch.trim()) {
+			url.searchParams.set("search", debouncedSearch.trim());
 		}
 
+		const applyResponse = (
+			data: Awaited<ReturnType<typeof fetchCachedQueryResponse>>,
+		) => {
+			const conversations = data.items as DmConversationItem[];
+			const nextSelected =
+				data.selectedConversation?.conversation.id ?? conversations[0]?.id;
+			setLoadedConversationId(data.selectedConversation?.conversation.id);
+			setItems(conversations);
+			setSelectedConversationId((current) => {
+				if (!current) return nextSelected;
+				return conversations.some((conversation) => conversation.id === current)
+					? current
+					: nextSelected;
+			});
+			setMessages(data.selectedConversation?.messages ?? []);
+		};
 		setError(null);
+		const cached = readCachedQueryResponse(url);
+		if (cached) {
+			applyResponse(cached);
+			setLoading(false);
+			return () => {
+				active = false;
+				controller.abort();
+			};
+		}
 		setLoading(true);
-		fetchQueryResponse(url, { signal: controller.signal })
+		fetchCachedQueryResponse(url, { signal: controller.signal })
 			.then((data) => {
 				if (!active) return;
-				const conversations = data.items as DmConversationItem[];
-				const nextSelected =
-					data.selectedConversation?.conversation.id ?? conversations[0]?.id;
-				setLoadedConversationId(data.selectedConversation?.conversation.id);
-				setItems(conversations);
-				setSelectedConversationId((current) => {
-					if (!current) return nextSelected;
-					return conversations.some(
-						(conversation) => conversation.id === current,
-					)
-						? current
-						: nextSelected;
-				});
-				setMessages(data.selectedConversation?.messages ?? []);
+				applyResponse(data);
 			})
 			.catch((fetchError: unknown) => {
 				if (
@@ -170,7 +186,7 @@ function DmsRoute() {
 		inboxFilter,
 		refreshTick,
 		replyFilter,
-		search,
+		debouncedSearch,
 		selectedConversationId,
 		selectedAccountId,
 		sort,
@@ -247,6 +263,7 @@ function DmsRoute() {
 				text,
 			});
 
+			invalidateCachedQueryResponses();
 			setSelectedConversationId(conversationId);
 			setRefreshTick((value) => value + 1);
 		} catch (error) {
@@ -258,8 +275,9 @@ function DmsRoute() {
 	}
 
 	function refreshLocalView() {
+		invalidateCachedQueryResponses();
 		setRefreshTick((value) => value + 1);
-		void loadStatus();
+		void loadStatus(true);
 	}
 
 	return (

package/src/routes/links.tsx

@@ -19,6 +19,11 @@ import {
 } from "#/components/FeedState";
 import { ProfilePreview } from "#/components/ProfilePreview";
 import { SmartTimestamp } from "#/components/SmartTimestamp";
+import {
+	loadClientCache,
+	readClientCache,
+	writeClientCache,
+} from "#/lib/client-cache";
 import { formatCompactNumber } from "#/lib/present";
 import type {
 	LinkInsightItem,
@@ -56,6 +61,10 @@ const LINK_INSIGHTS_LIMIT = 30;
 const LINK_INSIGHTS_COMMENTS_LIMIT = 30;
 const PROFILE_HYDRATION_LIMIT = 30;
 const PROFILE_HYDRATION_DELAY_MS = 1200;
+const LINK_INSIGHTS_CACHE_PREFIX = "link-insights:";
+const LINK_INSIGHTS_CACHE_MAX_AGE_MS = 5 * 60_000;
+const LINK_PROFILE_HYDRATED_CACHE_PREFIX = "link-profile-hydrated:";
+const hydratingLinkProfileHandles = new Set<string>();
 
 const ranges: Array<{ value: LinkInsightRange; label: string }> = [
 	{ value: "today", label: "Today" },
@@ -87,6 +96,19 @@ function insightCacheKey(
 	return `${kind}:${range}:${sort}:${source}:${refreshTick}`;
 }
 
+function sharedInsightCacheKey(
+	kind: LinkInsightKind,
+	range: LinkInsightRange,
+	sort: LinkInsightSort,
+	source: LinkInsightSource,
+) {
+	return `${LINK_INSIGHTS_CACHE_PREFIX}${kind}:${range}:${sort}:${source}`;
+}
+
+function hydratedProfileCacheKey(handle: string) {
+	return `${LINK_PROFILE_HYDRATED_CACHE_PREFIX}${handle.toLowerCase()}`;
+}
+
 function linkInsightsUrl(
 	kind: LinkInsightKind,
 	range: LinkInsightRange,
@@ -651,8 +673,16 @@ function LinksRoute() {
 	const [sort, setSort] = useState<LinkInsightSort>("rank");
 	const [search, setSearch] = useState("");
 	const [refreshTick, setRefreshTick] = useState(0);
-	const hydratedHandlesRef = useRef(new Set<string>());
-	const cacheRef = useRef(new Map<string, LinkInsightResponse>());
+	const initialCacheKey = insightCacheKey(kind, range, sort, source, 0);
+	const initialSharedData = readClientCache<LinkInsightResponse>(
+		sharedInsightCacheKey(kind, range, sort, source),
+		LINK_INSIGHTS_CACHE_MAX_AGE_MS,
+	);
+	const cacheRef = useRef(
+		new Map<string, LinkInsightResponse>(
+			initialSharedData ? [[initialCacheKey, initialSharedData]] : [],
+		),
+	);
 	const inFlightRef = useRef(new Set<string>());
 	const mountedRef = useRef(true);
 	const [errorByKey, setErrorByKey] = useState<Record<string, string>>({});
@@ -680,14 +710,35 @@ function LinksRoute() {
 			if (cacheRef.current.has(key) || inFlightRef.current.has(key)) {
 				return;
 			}
+			const sharedKey = sharedInsightCacheKey(fetchKind, range, sort, source);
+			if (refreshTick === 0) {
+				const cached = readClientCache<LinkInsightResponse>(
+					sharedKey,
+					LINK_INSIGHTS_CACHE_MAX_AGE_MS,
+				);
+				if (cached) {
+					cacheRef.current.set(key, cached);
+					bumpCacheVersion((value) => value + 1);
+					return;
+				}
+			}
 			inFlightRef.current.add(key);
 			setErrorByKey((current) => {
 				const rest = { ...current };
 				delete rest[key];
 				return rest;
 			});
-			fetch(linkInsightsUrl(fetchKind, range, sort, source, refreshTick))
-				.then((response) => response.json())
+			loadClientCache(
+				sharedKey,
+				() =>
+					fetch(
+						linkInsightsUrl(fetchKind, range, sort, source, refreshTick),
+					).then((response) => response.json() as Promise<LinkInsightResponse>),
+				{
+					force: refreshTick > 0,
+					maxAgeMs: LINK_INSIGHTS_CACHE_MAX_AGE_MS,
+				},
+			)
 				.then((response: LinkInsightResponse) => {
 					cacheRef.current.set(key, response);
 					if (mountedRef.current) {
@@ -732,7 +783,9 @@ function LinksRoute() {
 
 	useEffect(() => {
 		const handles = collectProfilesForHydration(data).filter(
-			(handle) => !hydratedHandlesRef.current.has(handle.toLowerCase()),
+			(handle) =>
+				!readClientCache<boolean>(hydratedProfileCacheKey(handle)) &&
+				!hydratingLinkProfileHandles.has(handle.toLowerCase()),
 		);
 		if (handles.length === 0) {
 			return;
@@ -742,19 +795,30 @@ function LinksRoute() {
 		const url = new URL("/api/profile-hydrate", window.location.origin);
 		url.searchParams.set("handles", handles.join(","));
 		for (const handle of handles) {
-			hydratedHandlesRef.current.add(handle.toLowerCase());
+			hydratingLinkProfileHandles.add(handle.toLowerCase());
 		}
+		const finishHydration = (succeeded: boolean) => {
+			for (const handle of handles) {
+				const normalized = handle.toLowerCase();
+				hydratingLinkProfileHandles.delete(normalized);
+				if (succeeded) {
+					writeClientCache(hydratedProfileCacheKey(normalized), true);
+				}
+			}
+		};
 
 		let idleId: number | null = null;
 		const runHydration = () => {
 			fetch(url, { signal: controller.signal })
 				.then((response) => response.json())
 				.then((response: { hydratedProfiles?: number }) => {
+					finishHydration(true);
 					if ((response.hydratedProfiles ?? 0) > 0) {
 						setRefreshTick((value) => value + 1);
 					}
 				})
 				.catch((error: unknown) => {
+					finishHydration(false);
 					if (error instanceof DOMException && error.name === "AbortError") {
 						return;
 					}
@@ -771,6 +835,7 @@ function LinksRoute() {
 
 		return () => {
 			controller.abort();
+			finishHydration(false);
 			window.clearTimeout(timer);
 			if (idleId !== null && "cancelIdleCallback" in window) {
 				window.cancelIdleCallback(idleId);

package/vite.config.ts

@@ -1,5 +1,4 @@
 import tailwindcss from "@tailwindcss/vite";
-import { devtools } from "@tanstack/devtools-vite";
 import { tanstackStart } from "@tanstack/react-start/plugin/vite";
 import viteReact from "@vitejs/plugin-react";
 import { defineConfig } from "vite";
@@ -11,7 +10,6 @@ const extraAllowedHosts =
 
 const config = defineConfig({
 	plugins: [
-		devtools(),
 		tailwindcss(),
 		tanstackStart({
 			router: {