birdclaw 0.5.1 → 0.7.0

package/src/routes/api/search-discussion.tsx

@@ -0,0 +1,169 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { Effect } from "effect";
+import { maybeAutoUpdateBackupEffect } from "#/lib/backup";
+import { runEffectBackground } from "#/lib/effect-runtime";
+import {
+	parseBoundedInteger,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import {
+	streamSearchDiscussionEffect,
+	type SearchDiscussionOptions,
+	type SearchDiscussionSource,
+	type SearchDiscussionStreamEvent,
+} from "#/lib/search-discussion";
+import type { TweetSearchMode } from "#/lib/tweet-search-live";
+
+const encoder = new TextEncoder();
+const MAX_DISCUSSION_SEARCH_LIMIT = 20_000;
+const MAX_DISCUSSION_SEARCH_PAGES = 200;
+
+function parseBoolean(value: string | null) {
+	return value === "true" || value === "1" || value === "yes";
+}
+
+function parseSource(value: string | null): SearchDiscussionSource {
+	if (
+		value === "all" ||
+		value === "home" ||
+		value === "mentions" ||
+		value === "authored" ||
+		value === "search" ||
+		value === "likes" ||
+		value === "bookmarks"
+	) {
+		return value;
+	}
+	return "search";
+}
+
+function parseMode(value: string | null): TweetSearchMode {
+	if (
+		value === "auto" ||
+		value === "bird" ||
+		value === "xurl" ||
+		value === "local"
+	) {
+		return value;
+	}
+	return "auto";
+}
+
+function parseOptions(url: URL): SearchDiscussionOptions {
+	return {
+		query: url.searchParams.get("query") ?? "",
+		account: url.searchParams.get("account") ?? undefined,
+		source: parseSource(url.searchParams.get("source")),
+		mode: parseMode(url.searchParams.get("mode")),
+		includeDms: parseBoolean(url.searchParams.get("includeDms")),
+		since: url.searchParams.get("since") ?? undefined,
+		until: url.searchParams.get("until") ?? undefined,
+		question: url.searchParams.get("question") ?? undefined,
+		originalsOnly: parseBoolean(url.searchParams.get("originalsOnly")),
+		hideLowQuality: parseBoolean(url.searchParams.get("hideLowQuality")),
+		refresh: parseBoolean(url.searchParams.get("refresh")),
+		model: url.searchParams.get("model") === "gpt-5.5" ? "gpt-5.5" : undefined,
+		limit: parseBoundedInteger(url.searchParams.get("limit"), {
+			max: MAX_DISCUSSION_SEARCH_LIMIT,
+		}),
+		maxPages: parseBoundedInteger(url.searchParams.get("maxPages"), {
+			max: MAX_DISCUSSION_SEARCH_PAGES,
+		}),
+	};
+}
+
+function encodeEvent(event: SearchDiscussionStreamEvent) {
+	return encoder.encode(`${JSON.stringify(event)}\n`);
+}
+
+export const Route = createFileRoute("/api/search-discussion")({
+	server: {
+		handlers: {
+			GET: ({ request }) =>
+				runRouteEffect(
+					Effect.sync(() => {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
+
+						const url = new URL(request.url);
+						const options = parseOptions(url);
+						let abortDiscussion: (() => void) | undefined;
+
+						return new Response(
+							new ReadableStream({
+								cancel() {
+									abortDiscussion?.();
+								},
+								start(controller) {
+									const abortController = new AbortController();
+									let closed = false;
+									const close = () => {
+										closed = true;
+										abortController.abort();
+									};
+									const closeController = () => {
+										request.signal.removeEventListener("abort", onAbort);
+										if (!closed) {
+											closed = true;
+											controller.close();
+										}
+									};
+									const onAbort = () => close();
+									request.signal.addEventListener("abort", onAbort, {
+										once: true,
+									});
+									abortDiscussion = close;
+									const enqueue = (event: SearchDiscussionStreamEvent) => {
+										if (closed) return;
+										try {
+											controller.enqueue(encodeEvent(event));
+										} catch {
+											close();
+										}
+									};
+
+									runEffectBackground(
+										maybeAutoUpdateBackupEffect().pipe(
+											Effect.flatMap(() => {
+												if (closed || abortController.signal.aborted) {
+													return Effect.succeed(undefined);
+												}
+												return streamSearchDiscussionEffect(
+													{
+														...options,
+														signal: abortController.signal,
+														prefetchAvatars: true,
+													},
+													{ onEvent: enqueue },
+												);
+											}),
+										),
+										{
+											onSuccess: closeController,
+											onFailure: (error) => {
+												enqueue({
+													type: "error",
+													error:
+														error instanceof Error
+															? error.message
+															: "Discussion failed",
+												});
+												closeController();
+											},
+										},
+									);
+								},
+							}),
+							{
+								headers: {
+									"cache-control": "no-store",
+									"content-type": "application/x-ndjson; charset=utf-8",
+								},
+							},
+						);
+					}),
+				),
+		},
+	},
+});

package/src/routes/api/status.tsx

@@ -1,18 +1,26 @@
 import { createFileRoute } from "@tanstack/react-router";
-import { maybeAutoUpdateBackup } from "#/lib/backup";
-import { getQueryEnvelope } from "#/lib/queries";
+import { Effect } from "effect";
+import { maybeAutoUpdateBackupEffect } from "#/lib/backup";
+import {
+	jsonResponse,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import { getQueryEnvelopeEffect } from "#/lib/queries";
 
 export const Route = createFileRoute("/api/status")({
 	server: {
 		handlers: {
-			GET: async () => {
-				await maybeAutoUpdateBackup();
-				return new Response(JSON.stringify(await getQueryEnvelope()), {
-					headers: {
-						"content-type": "application/json",
-					},
-				});
-			},
+			GET: ({ request }) =>
+				runRouteEffect(
+					Effect.gen(function* () {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
+
+						yield* maybeAutoUpdateBackupEffect();
+						return jsonResponse(yield* getQueryEnvelopeEffect());
+					}),
+				),
 		},
 	},
 });

package/src/routes/api/sync.tsx

@@ -1,28 +1,64 @@
 import { createFileRoute } from "@tanstack/react-router";
-import { getWebSyncJob, parseWebSyncKind, startWebSync } from "#/lib/web-sync";
-
-function json(data: unknown, init?: ResponseInit) {
-	return new Response(JSON.stringify(data), {
-		...init,
-		headers: {
-			"content-type": "application/json",
-			...init?.headers,
-		},
-	});
-}
+import { Effect } from "effect";
+import {
+	jsonResponse,
+	requestJsonEffect,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import {
+	getWebSyncJob,
+	parseWebSyncKind,
+	startWebSync,
+	type WebSyncDmInbox,
+	type WebSyncOptions,
+} from "#/lib/web-sync";
 
 function parseAccountId(value: unknown) {
 	return typeof value === "string" && value.trim() ? value.trim() : undefined;
 }
 
+function parseDmInbox(value: unknown): WebSyncDmInbox | undefined {
+	return value === "all" || value === "accepted" || value === "requests"
+		? value
+		: undefined;
+}
+
+function parsePositiveInteger(value: unknown, max: number) {
+	const parsed =
+		typeof value === "number"
+			? value
+			: typeof value === "string"
+				? Number(value)
+				: Number.NaN;
+	if (!Number.isInteger(parsed) || parsed <= 0) return undefined;
+	return Math.min(parsed, max);
+}
+
+function parseSyncOptions(kind: string, body: Record<string, unknown>) {
+	if (kind !== "dms") return {};
+	const options: WebSyncOptions = {};
+	const inbox = parseDmInbox(body.inbox);
+	const limit = parsePositiveInteger(body.limit, 1000);
+	const maxPages = parsePositiveInteger(body.maxPages, 250);
+	if (inbox) options.inbox = inbox;
+	if (limit) options.limit = limit;
+	if (maxPages) options.maxPages = maxPages;
+	if (typeof body.allPages === "boolean") options.allPages = body.allPages;
+	return options;
+}
+
 export const Route = createFileRoute("/api/sync")({
 	server: {
 		handlers: {
 			GET: ({ request }) => {
+				const denied = sensitiveRequestErrorResponse(request);
+				if (denied) return denied;
+
 				const url = new URL(request.url);
 				const id = url.searchParams.get("id");
 				if (!id) {
-					return json(
+					return jsonResponse(
 						{ ok: false, message: "Missing sync job id" },
 						{ status: 400 },
 					);
@@ -30,30 +66,40 @@ export const Route = createFileRoute("/api/sync")({
 
 				const job = getWebSyncJob(id);
 				if (!job) {
-					return json(
+					return jsonResponse(
 						{ ok: false, message: "Sync job not found" },
 						{ status: 404 },
 					);
 				}
 
-				return json(job);
+				return jsonResponse(job);
 			},
-			POST: async ({ request }) => {
-				const body = (await request.json().catch(() => ({}))) as Record<
-					string,
-					unknown
-				>;
-				const kind = parseWebSyncKind(body.kind);
-				if (!kind) {
-					return json(
-						{ ok: false, message: "Unknown sync kind" },
-						{ status: 400 },
-					);
-				}
+			POST: ({ request }) =>
+				runRouteEffect(
+					Effect.gen(function* () {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
 
-				const job = startWebSync(kind, parseAccountId(body.accountId));
-				return json(job, { status: job.inProgress ? 202 : 200 });
-			},
+						const body = yield* requestJsonEffect<Record<string, unknown>>(
+							request,
+							{},
+						);
+						const kind = parseWebSyncKind(body.kind);
+						if (!kind) {
+							return jsonResponse(
+								{ ok: false, message: "Unknown sync kind" },
+								{ status: 400 },
+							);
+						}
+
+						const job = startWebSync(
+							kind,
+							parseAccountId(body.accountId),
+							parseSyncOptions(kind, body),
+						);
+						return jsonResponse(job, { status: job.inProgress ? 202 : 200 });
+					}),
+				),
 		},
 	},
 });

package/src/routes/api/xurl-rate-limits.tsx

@@ -0,0 +1,24 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { Effect } from "effect";
+import {
+	jsonResponse,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import { getXurlRateLimitSnapshot } from "#/lib/xurl-rate-limits";
+
+export const Route = createFileRoute("/api/xurl-rate-limits")({
+	server: {
+		handlers: {
+			GET: ({ request }) =>
+				runRouteEffect(
+					Effect.sync(() => {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
+
+						return jsonResponse(getXurlRateLimitSnapshot());
+					}),
+				),
+		},
+	},
+});

package/src/routes/data-sources.tsx

@@ -0,0 +1,255 @@
+import { createFileRoute } from "@tanstack/react-router";
+import {
+	AlertTriangle,
+	CheckCircle2,
+	Database,
+	RefreshCw,
+	Route as RouteIcon,
+	TerminalSquare,
+	XCircle,
+} from "lucide-react";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import type {
+	LiveDataSourceCapability,
+	LiveDataSourceKind,
+	LiveDataSourcesResponse,
+	LiveDataSourceStatus,
+} from "#/lib/types";
+import {
+	cx,
+	errorCopyClass,
+	pageHeaderActionsClass,
+	pageHeaderClass,
+	pageHeaderRowClass,
+	pageSubtitleClass,
+	pageTitleClass,
+	secondaryButtonClass,
+	statusCopyClass,
+} from "#/lib/ui";
+
+export const Route = createFileRoute("/data-sources")({
+	component: DataSourcesRoute,
+});
+
+async function fetchDataSources() {
+	const response = await fetch("/api/data-sources");
+	if (!response.ok) {
+		throw new Error(`Data source status failed (${String(response.status)})`);
+	}
+	return (await response.json()) as LiveDataSourcesResponse;
+}
+
+function sourceIcon(source: LiveDataSourceKind) {
+	if (source === "birdclaw") return Database;
+	if (source === "bird") return TerminalSquare;
+	return RouteIcon;
+}
+
+function statusTone(status: LiveDataSourceStatus["status"]) {
+	if (status === "ok") {
+		return "border-[color:color-mix(in_srgb,#22c55e_45%,var(--line))] bg-[color:color-mix(in_srgb,#22c55e_10%,var(--bg))] text-[color:color-mix(in_srgb,#22c55e_82%,var(--ink))]";
+	}
+	if (status === "warning") {
+		return "border-[color:color-mix(in_srgb,#f59e0b_50%,var(--line))] bg-[color:color-mix(in_srgb,#f59e0b_12%,var(--bg))] text-[color:color-mix(in_srgb,#f59e0b_82%,var(--ink))]";
+	}
+	return "border-[color:color-mix(in_srgb,var(--alert)_55%,var(--line))] bg-[var(--alert-soft)] text-[var(--alert)]";
+}
+
+function statusIcon(status: LiveDataSourceStatus["status"]) {
+	if (status === "ok") return CheckCircle2;
+	if (status === "warning") return AlertTriangle;
+	return XCircle;
+}
+
+function sourceLabel(source: LiveDataSourceKind) {
+	if (source === "birdclaw") return "local";
+	return source;
+}
+
+function SourceCard({ source }: { source: LiveDataSourceStatus }) {
+	const Icon = sourceIcon(source.source);
+	const StatusIcon = statusIcon(source.status);
+	return (
+		<section className="min-w-0 border-b border-[var(--line)] px-4 py-4">
+			<div className="flex items-start justify-between gap-3">
+				<div className="flex min-w-0 items-center gap-3">
+					<div className="grid size-10 shrink-0 place-items-center rounded-md border border-[var(--line)] bg-[var(--bg-active)] text-[var(--ink)]">
+						<Icon className="size-5" strokeWidth={1.9} />
+					</div>
+					<div className="min-w-0">
+						<h2 className="truncate text-[16px] font-bold text-[var(--ink)]">
+							{source.label}
+						</h2>
+						<p className="truncate text-[13px] text-[var(--ink-soft)]">
+							{source.installed === false ? "not installed" : source.detail}
+						</p>
+					</div>
+				</div>
+				<span
+					className={cx(
+						"inline-flex shrink-0 items-center gap-1 rounded-full border px-2 py-0.5 text-[12px] font-bold",
+						statusTone(source.status),
+					)}
+				>
+					<StatusIcon className="size-3.5" strokeWidth={2} />
+					{source.works ? "works" : source.status}
+				</span>
+			</div>
+			<div className="mt-4 flex flex-wrap gap-2">
+				{source.accounts.length > 0 ? (
+					source.accounts.map((account, index) => (
+						<span
+							key={`${account.app ?? ""}:${account.username ?? account.handle ?? account.id ?? String(index)}`}
+							className="inline-flex max-w-full items-center gap-1.5 rounded-full border border-[var(--line)] bg-[var(--bg)] px-2.5 py-1 text-[12px] font-semibold text-[var(--ink-soft)]"
+						>
+							{account.app ? (
+								<span className="text-[var(--ink-faint)]">{account.app}</span>
+							) : null}
+							<span className="truncate">
+								{account.handle ??
+									(account.username ? `@${account.username}` : account.id)}
+							</span>
+							{account.isDefault ? (
+								<span className="text-[var(--accent)]">default</span>
+							) : null}
+						</span>
+					))
+				) : (
+					<span className="text-[13px] text-[var(--ink-soft)]">
+						no authenticated account detected
+					</span>
+				)}
+			</div>
+		</section>
+	);
+}
+
+function CapabilityRow({
+	capability,
+	sourcesByKind,
+}: {
+	capability: LiveDataSourceCapability;
+	sourcesByKind: Map<LiveDataSourceKind, LiveDataSourceStatus>;
+}) {
+	const chain = [capability.primary, ...capability.fallbacks];
+	return (
+		<div className="grid gap-3 border-b border-[var(--line)] px-4 py-4 min-[840px]:grid-cols-[220px_minmax(0,1fr)]">
+			<div className="min-w-0">
+				<div className="font-bold text-[var(--ink)]">{capability.label}</div>
+				{capability.notes ? (
+					<div className="mt-1 text-[12px] text-[var(--ink-soft)]">
+						{capability.notes}
+					</div>
+				) : null}
+			</div>
+			<div className="flex min-w-0 flex-wrap items-center gap-2">
+				{chain.map((source, index) => {
+					const status = sourcesByKind.get(source);
+					const ok = Boolean(status?.works);
+					return (
+						<div
+							key={`${capability.key}:${source}:${String(index)}`}
+							className={cx(
+								"inline-flex items-center gap-2 rounded-full border px-3 py-1 text-[13px] font-semibold",
+								ok
+									? "border-[color:color-mix(in_srgb,#22c55e_38%,var(--line))] text-[var(--ink)]"
+									: "border-[var(--line)] text-[var(--ink-soft)]",
+							)}
+						>
+							<span className="text-[12px] text-[var(--ink-faint)]">
+								{index === 0 ? "primary" : `fallback ${String(index)}`}
+							</span>
+							<span>{sourceLabel(source)}</span>
+						</div>
+					);
+				})}
+			</div>
+		</div>
+	);
+}
+
+function DataSourcesRoute() {
+	const [snapshot, setSnapshot] = useState<LiveDataSourcesResponse | null>(null);
+	const [error, setError] = useState<string | null>(null);
+	const [loading, setLoading] = useState(true);
+	const sourcesByKind = useMemo(
+		() =>
+			new Map(
+				(snapshot?.sources ?? []).map((source) => [source.source, source]),
+			),
+		[snapshot],
+	);
+
+	const load = useCallback(() => {
+		setLoading(true);
+		setError(null);
+		fetchDataSources()
+			.then(setSnapshot)
+			.catch((cause: unknown) => {
+				setError(
+					cause instanceof Error ? cause.message : "Data sources unavailable",
+				);
+			})
+			.finally(() => setLoading(false));
+	}, []);
+
+	useEffect(() => {
+		load();
+	}, [load]);
+
+	return (
+		<section className="flex min-h-screen flex-col">
+			<header className={pageHeaderClass}>
+				<div className={pageHeaderRowClass}>
+					<div>
+						<h1 className={pageTitleClass}>Data Sources</h1>
+						<p className={pageSubtitleClass}>
+							Live auth, local archive, and automatic fallback order
+						</p>
+					</div>
+					<div className={pageHeaderActionsClass}>
+						<button
+							className={secondaryButtonClass}
+							type="button"
+							onClick={load}
+							disabled={loading}
+						>
+							<RefreshCw className={cx("size-4", loading && "animate-spin")} />
+							Refresh
+						</button>
+					</div>
+				</div>
+			</header>
+			{error ? <div className={errorCopyClass}>{error}</div> : null}
+			{snapshot ? (
+				<>
+					<div className="grid border-t border-[var(--line)]">
+						{snapshot.sources.map((source) => (
+							<SourceCard key={source.source} source={source} />
+						))}
+					</div>
+					<section className="min-h-0 flex-1">
+						<header className="border-b border-[var(--line)] px-4 py-3">
+							<h2 className="text-[16px] font-bold text-[var(--ink)]">
+								Fallbacks
+							</h2>
+							<p className="text-[13px] text-[var(--ink-soft)]">
+								Calls try the next source when the current source cannot serve
+								the request.
+							</p>
+						</header>
+						{snapshot.capabilities.map((capability) => (
+							<CapabilityRow
+								key={capability.key}
+								capability={capability}
+								sourcesByKind={sourcesByKind}
+							/>
+						))}
+					</section>
+				</>
+			) : loading ? (
+				<div className={statusCopyClass}>Checking data sources...</div>
+			) : null}
+		</section>
+	);
+}