birdclaw 0.5.0 → 0.5.1

package/src/components/useTimelineRouteData.ts

@@ -0,0 +1,137 @@
+import { useEffect, useState } from "react";
+import type {
+	QueryEnvelope,
+	ReplyFilter,
+	ResourceKind,
+	TimelineItem,
+} from "#/lib/types";
+import {
+	fetchQueryEnvelope,
+	fetchQueryResponse,
+	postAction,
+} from "#/lib/api-client";
+
+interface UseTimelineRouteDataOptions {
+	resource: Exclude<ResourceKind, "dms">;
+	search: string;
+	errorFallback: string;
+	replyFilter?: ReplyFilter;
+	likedOnly?: boolean;
+	bookmarkedOnly?: boolean;
+}
+
+export function useTimelineRouteData({
+	resource,
+	search,
+	errorFallback,
+	replyFilter,
+	likedOnly = false,
+	bookmarkedOnly = false,
+}: UseTimelineRouteDataOptions) {
+	const [meta, setMeta] = useState<QueryEnvelope | null>(null);
+	const [items, setItems] = useState<TimelineItem[]>([]);
+	const [loading, setLoading] = useState(true);
+	const [error, setError] = useState<string | null>(null);
+	const [refreshTick, setRefreshTick] = useState(0);
+
+	async function loadStatus() {
+		setMeta(await fetchQueryEnvelope());
+	}
+
+	useEffect(() => {
+		void loadStatus();
+	}, []);
+
+	useEffect(() => {
+		const url = new URL("/api/query", window.location.origin);
+		url.searchParams.set("resource", resource);
+		url.searchParams.set("refresh", String(refreshTick));
+		if (replyFilter) {
+			url.searchParams.set("replyFilter", replyFilter);
+		}
+		if (likedOnly) {
+			url.searchParams.set("liked", "true");
+		}
+		if (bookmarkedOnly) {
+			url.searchParams.set("bookmarked", "true");
+		}
+		if (search.trim()) {
+			url.searchParams.set("search", search.trim());
+		}
+
+		const controller = new AbortController();
+		let active = true;
+		setError(null);
+		setLoading(true);
+		fetchQueryResponse(url, { signal: controller.signal })
+			.then((data) => {
+				if (active) {
+					setItems(data.items as TimelineItem[]);
+				}
+			})
+			.catch((fetchError: unknown) => {
+				if (
+					fetchError instanceof DOMException &&
+					fetchError.name === "AbortError"
+				) {
+					return;
+				}
+				if (!active) return;
+				setError(
+					fetchError instanceof Error ? fetchError.message : errorFallback,
+				);
+				setItems([]);
+			})
+			.finally(() => {
+				if (active) {
+					setLoading(false);
+				}
+			});
+
+		return () => {
+			active = false;
+			controller.abort();
+		};
+	}, [
+		bookmarkedOnly,
+		errorFallback,
+		likedOnly,
+		refreshTick,
+		replyFilter,
+		resource,
+		search,
+	]);
+
+	function retry() {
+		setRefreshTick((value) => value + 1);
+	}
+
+	function refreshLocalView() {
+		setRefreshTick((value) => value + 1);
+		void loadStatus();
+	}
+
+	async function replyToTweet(tweetId: string) {
+		const text = window.prompt("Reply text");
+		if (!text?.trim()) return;
+
+		await postAction({
+			kind: "replyTweet",
+			accountId: "acct_primary",
+			tweetId,
+			text,
+		});
+
+		retry();
+	}
+
+	return {
+		meta,
+		items,
+		loading,
+		error,
+		retry,
+		refreshLocalView,
+		replyToTweet,
+	};
+}

package/src/lib/api-client.ts

@@ -0,0 +1,229 @@
+import { z } from "zod";
+import type {
+	DmConversationItem,
+	DmMessageItem,
+	QueryEnvelope,
+	QueryResponse,
+	TimelineItem,
+} from "./types";
+import type {
+	WebSyncJobSnapshot,
+	WebSyncKind,
+	WebSyncResponse,
+} from "./web-sync";
+
+const jsonRecordSchema = z.object({}).passthrough();
+const resourceKindSchema = z.enum(["home", "mentions", "authored", "dms"]);
+const webSyncKindSchema = z.enum([
+	"timeline",
+	"mentions",
+	"likes",
+	"bookmarks",
+	"dms",
+]);
+
+const queryEnvelopeSchema = z
+	.object({
+		accounts: z.array(jsonRecordSchema),
+		archives: z.array(jsonRecordSchema),
+		transport: z
+			.object({
+				statusText: z.string(),
+			})
+			.passthrough(),
+		stats: z.object({
+			home: z.number(),
+			mentions: z.number(),
+			dms: z.number(),
+			needsReply: z.number(),
+			inbox: z.number(),
+		}),
+	})
+	.transform((value) => value as unknown as QueryEnvelope);
+
+const queryResponseSchema = z
+	.object({
+		resource: resourceKindSchema,
+		items: z.array(jsonRecordSchema),
+		selectedConversation: z
+			.object({
+				conversation: jsonRecordSchema,
+				messages: z.array(jsonRecordSchema),
+			})
+			.nullish(),
+	})
+	.transform(
+		(value) =>
+			({
+				...value,
+				items: value.items as unknown as Array<
+					TimelineItem | DmConversationItem
+				>,
+				selectedConversation: value.selectedConversation
+					? {
+							conversation: value.selectedConversation
+								.conversation as unknown as DmConversationItem,
+							messages: value.selectedConversation
+								.messages as unknown as DmMessageItem[],
+						}
+					: value.selectedConversation,
+			}) as QueryResponse,
+	);
+
+const webSyncResponseSchema = z
+	.object({
+		ok: z.boolean(),
+		kind: webSyncKindSchema,
+		accountId: z.string().optional(),
+		summary: z.string(),
+		steps: z.array(jsonRecordSchema),
+		startedAt: z.string().optional(),
+		finishedAt: z.string().optional(),
+		inProgress: z.boolean().optional(),
+		backup: z.unknown().optional(),
+		error: z.string().optional(),
+	})
+	.transform((value) => value as unknown as WebSyncResponse);
+
+const webSyncJobSchema = z
+	.object({
+		id: z.string(),
+		kind: webSyncKindSchema,
+		accountId: z.string().optional(),
+		status: z.enum(["running", "succeeded", "failed"]),
+		startedAt: z.string(),
+		finishedAt: z.string().optional(),
+		summary: z.string(),
+		inProgress: z.boolean(),
+		result: webSyncResponseSchema.optional(),
+		error: z.string().optional(),
+	})
+	.transform((value) => value as unknown as WebSyncJobSnapshot);
+
+const actionResponseSchema = jsonRecordSchema;
+const SYNC_POLL_INTERVAL_MS = 500;
+
+export class ApiFetchError extends Error {
+	constructor(
+		message: string,
+		readonly status?: number,
+	) {
+		super(message);
+		this.name = "ApiFetchError";
+	}
+}
+
+function responseMessage(data: unknown, fallback: string) {
+	if (data && typeof data === "object") {
+		const record = data as {
+			message?: unknown;
+			error?: unknown;
+			summary?: unknown;
+		};
+		if (typeof record.message === "string") return record.message;
+		if (typeof record.error === "string") return record.error;
+		if (typeof record.summary === "string") return record.summary;
+	}
+	return fallback;
+}
+
+async function readJson(response: Response) {
+	try {
+		return await response.json();
+	} catch {
+		return null;
+	}
+}
+
+export async function fetchJson<T>(
+	input: RequestInfo | URL,
+	init: RequestInit | undefined,
+	schema: z.ZodType<T>,
+	fallbackMessage: string,
+): Promise<T> {
+	const response = await fetch(input, init);
+	const data = await readJson(response);
+	if (!response.ok) {
+		throw new ApiFetchError(
+			responseMessage(data, fallbackMessage),
+			response.status,
+		);
+	}
+
+	const parsed = schema.safeParse(data);
+	if (!parsed.success) {
+		throw new ApiFetchError(fallbackMessage);
+	}
+	return parsed.data;
+}
+
+export function fetchQueryEnvelope(init?: RequestInit) {
+	return fetchJson(
+		"/api/status",
+		init,
+		queryEnvelopeSchema,
+		"Status unavailable",
+	);
+}
+
+export function fetchQueryResponse(
+	input: RequestInfo | URL,
+	init?: RequestInit,
+) {
+	return fetchJson(input, init, queryResponseSchema, "Query unavailable");
+}
+
+export function postAction(body: Record<string, unknown>) {
+	return fetchJson(
+		"/api/action",
+		{
+			method: "POST",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify(body),
+		},
+		actionResponseSchema,
+		"Action failed",
+	);
+}
+
+export function postSync(kind: WebSyncKind, accountId?: string) {
+	return fetchJson(
+		"/api/sync",
+		{
+			method: "POST",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify({
+				kind,
+				...(accountId ? { accountId } : {}),
+			}),
+		},
+		webSyncJobSchema,
+		"Sync failed",
+	).then(waitForWebSyncJob);
+}
+
+function fetchSyncJob(id: string) {
+	const url = new URL("/api/sync", window.location.origin);
+	url.searchParams.set("id", id);
+	return fetchJson(url, undefined, webSyncJobSchema, "Sync status unavailable");
+}
+
+function delay(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForWebSyncJob(job: WebSyncJobSnapshot) {
+	let current = job;
+	while (current.inProgress) {
+		await delay(SYNC_POLL_INTERVAL_MS);
+		current = await fetchSyncJob(current.id);
+	}
+
+	if (!current.result) {
+		throw new ApiFetchError(current.error ?? current.summary);
+	}
+	if (!current.result.ok) {
+		throw new ApiFetchError(current.result.error ?? current.result.summary);
+	}
+	return current.result;
+}

package/src/lib/archive-finder.ts

@@ -96,27 +96,31 @@ export async function findArchives(): Promise<ArchiveCandidate[]> {
 		'kMDItemDisplayName == "*archive*.zip" && kMDItemKind == "Zip archive"',
 	];
 
-	for (const query of queries) {
-		try {
-			const { stdout } = await execAsync(`mdfind -onlyin ~ '${query}'`, {
-				timeout: 5000,
-			});
-
-			const paths = stdout
-				.split("\n")
-				.map((item) => item.trim())
-				.filter((item) => item.length > 0 && item.endsWith(".zip"));
-
-			const candidates = await Promise.all(
-				paths.map((filePath) => getCandidate(filePath)),
-			);
-			for (const candidate of candidates) {
-				if (candidate) {
-					found.set(candidate.path, candidate);
-				}
+	const spotlightCandidates = await Promise.all(
+		queries.map(async (query) => {
+			try {
+				const { stdout } = await execAsync(`mdfind -onlyin ~ '${query}'`, {
+					timeout: 5000,
+				});
+
+				const paths = stdout
+					.split("\n")
+					.map((item) => item.trim())
+					.filter((item) => item.length > 0 && item.endsWith(".zip"));
+
+				return Promise.all(paths.map((filePath) => getCandidate(filePath)));
+			} catch {
+				// Best-effort only.
+				return [];
+			}
+		}),
+	);
+
+	for (const candidates of spotlightCandidates) {
+		for (const candidate of candidates) {
+			if (candidate) {
+				found.set(candidate.path, candidate);
 			}
-		} catch {
-			// Best-effort only.
 		}
 	}
 

package/src/lib/archive-import.ts

@@ -218,7 +218,12 @@ function toFiniteNumber(value: unknown) {
 
 function extractTweetEntities(tweet: Record<string, unknown>) {
 	const entities = asRecord(tweet.entities);
-	const urls = asArray<Record<string, unknown>>(entities?.urls)
+	const urlEntries = [
+		...asArray<Record<string, unknown>>(entities?.urls),
+		...asArray<Record<string, unknown>>(entities?.media),
+	];
+	const seenUrls = new Set<string>();
+	const urls = urlEntries
 		.map((entry) => ({
 			url: String(entry.url ?? ""),
 			expandedUrl: String(
@@ -243,7 +248,11 @@ function extractTweetEntities(tweet: Record<string, unknown>) {
 						? entry.imageUrl
 						: typeof entry.thumbnail_url === "string"
 							? entry.thumbnail_url
-							: undefined,
+							: typeof entry.media_url_https === "string"
+								? entry.media_url_https
+								: typeof entry.media_url === "string"
+									? entry.media_url
+									: undefined,
 			siteName:
 				typeof entry.site_name === "string"
 					? entry.site_name
@@ -251,7 +260,13 @@ function extractTweetEntities(tweet: Record<string, unknown>) {
 						? entry.siteName
 						: undefined,
 		}))
-		.filter((entry) => entry.url.length > 0 || entry.expandedUrl.length > 0);
+		.filter((entry) => entry.url.length > 0 || entry.expandedUrl.length > 0)
+		.filter((entry) => {
+			const key = `${entry.start}:${entry.end}:${entry.url}:${entry.expandedUrl}`;
+			if (seenUrls.has(key)) return false;
+			seenUrls.add(key);
+			return true;
+		});
 	const mentions = asArray<Record<string, unknown>>(entities?.user_mentions)
 		.map((entry) => ({
 			username: String(entry.screen_name ?? ""),

package/src/lib/conversation-surface.ts

@@ -0,0 +1,174 @@
+import { useCallback, useEffect, useSyncExternalStore } from "react";
+import type { ReactNode } from "react";
+import type { EmbeddedTweet } from "#/lib/types";
+
+type ConversationStatus = "idle" | "loading" | "ready" | "error";
+
+interface ConversationRecord {
+	error: string | null;
+	items: EmbeddedTweet[];
+	status: ConversationStatus;
+}
+
+interface ConversationSurfaceSnapshot {
+	expandedTweetId: string | null;
+	records: ReadonlyMap<string, ConversationRecord>;
+}
+
+type Listener = () => void;
+
+const emptyRecord: ConversationRecord = {
+	error: null,
+	items: [],
+	status: "idle",
+};
+
+let snapshot: ConversationSurfaceSnapshot = {
+	expandedTweetId: null,
+	records: new Map(),
+};
+const listeners = new Set<Listener>();
+const inFlight = new Set<string>();
+let activeScopes = 0;
+let generation = 0;
+
+function emit() {
+	for (const listener of listeners) {
+		listener();
+	}
+}
+
+function setSnapshot(next: ConversationSurfaceSnapshot) {
+	snapshot = next;
+	emit();
+}
+
+function updateRecord(tweetId: string, record: ConversationRecord) {
+	const records = new Map(snapshot.records);
+	records.set(tweetId, record);
+	setSnapshot({ ...snapshot, records });
+}
+
+function subscribe(listener: Listener) {
+	listeners.add(listener);
+	return () => {
+		listeners.delete(listener);
+	};
+}
+
+function getSnapshot() {
+	return snapshot;
+}
+
+async function loadConversation(tweetId: string) {
+	const current = snapshot.records.get(tweetId);
+	if (current?.status === "ready" || inFlight.has(tweetId)) {
+		return;
+	}
+
+	const loadGeneration = generation;
+	inFlight.add(tweetId);
+	updateRecord(tweetId, {
+		error: null,
+		items: current?.items ?? [],
+		status: "loading",
+	});
+
+	try {
+		const response = await fetch(
+			`/api/conversation?tweetId=${encodeURIComponent(tweetId)}`,
+		);
+		const data = (await response.json()) as {
+			error?: string;
+			items?: EmbeddedTweet[];
+			ok?: boolean;
+		};
+		if (!response.ok || data.ok === false) {
+			throw new Error(data.error ?? "Conversation unavailable");
+		}
+		if (loadGeneration !== generation) {
+			return;
+		}
+		updateRecord(tweetId, {
+			error: null,
+			items: (data.items ?? []).filter(Boolean),
+			status: "ready",
+		});
+	} catch (error) {
+		if (loadGeneration !== generation) {
+			return;
+		}
+		updateRecord(tweetId, {
+			error:
+				error instanceof Error ? error.message : "Conversation unavailable",
+			items: [],
+			status: "error",
+		});
+	} finally {
+		inFlight.delete(tweetId);
+	}
+}
+
+export function retainConversationSurfaceScope() {
+	activeScopes += 1;
+	return () => {
+		activeScopes = Math.max(0, activeScopes - 1);
+		if (activeScopes === 0) {
+			resetConversationSurface();
+		}
+	};
+}
+
+export function resetConversationSurface() {
+	generation += 1;
+	inFlight.clear();
+	setSnapshot({
+		expandedTweetId: null,
+		records: new Map(),
+	});
+}
+
+export function ConversationSurfaceScope({
+	children,
+}: {
+	children: ReactNode;
+}) {
+	useEffect(() => retainConversationSurfaceScope(), []);
+	return children;
+}
+
+export function useConversationSurface(tweetId: string) {
+	const currentSnapshot = useSyncExternalStore(
+		subscribe,
+		getSnapshot,
+		getSnapshot,
+	);
+	const record = currentSnapshot.records.get(tweetId) ?? emptyRecord;
+	const isOpen = currentSnapshot.expandedTweetId === tweetId;
+
+	const toggle = useCallback(() => {
+		const nextExpanded = snapshot.expandedTweetId === tweetId ? null : tweetId;
+		setSnapshot({ ...snapshot, expandedTweetId: nextExpanded });
+		if (nextExpanded) {
+			void loadConversation(tweetId);
+		}
+	}, [tweetId]);
+
+	const prefetch = useCallback(() => {
+		const current = snapshot.records.get(tweetId);
+		if (current?.status === "ready" || current?.status === "loading") {
+			return;
+		}
+		void loadConversation(tweetId);
+	}, [tweetId]);
+
+	return {
+		error: record.error,
+		isOpen,
+		items: record.items,
+		loading: record.status === "loading",
+		prefetch,
+		status: record.status,
+		toggle,
+	};
+}

package/src/lib/db.ts

@@ -574,11 +574,13 @@ const BASE_SCHEMA_SQL = `
 
 const INDEX_SQL = `
   create index if not exists idx_tweets_kind_created on tweets(kind, created_at desc);
+  create index if not exists idx_tweets_created on tweets(created_at desc);
   create index if not exists idx_tweets_account_created on tweets(account_id, created_at desc);
   create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
   create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
   create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
   create index if not exists idx_tweet_account_edges_kind_account on tweet_account_edges(kind, account_id, last_seen_at desc, tweet_id);
+  create index if not exists idx_tweet_account_edges_kind_tweet on tweet_account_edges(kind, tweet_id, account_id);
   create index if not exists idx_tweet_account_edges_tweet on tweet_account_edges(tweet_id);
   create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);