birdclaw 0.6.0 → 0.7.0

package/src/routes/api/profile-analysis.tsx

@@ -0,0 +1,152 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { Effect } from "effect";
+import { maybeAutoUpdateBackupEffect } from "#/lib/backup";
+import { runEffectBackground } from "#/lib/effect-runtime";
+import {
+	parseBoundedInteger,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import {
+	streamProfileAnalysisEffect,
+	type ProfileAnalysisOptions,
+	type ProfileAnalysisStreamEvent,
+} from "#/lib/profile-analysis";
+
+const encoder = new TextEncoder();
+
+function parseBoolean(value: string | null) {
+	return value === "true" || value === "1" || value === "yes";
+}
+
+function parseOptions(url: URL): ProfileAnalysisOptions {
+	const conversationDelayMs = parseBoundedInteger(
+		url.searchParams.get("conversationDelayMs"),
+		{ min: 0, max: 60_000 },
+	);
+	const rateLimitRetryMs = parseBoundedInteger(
+		url.searchParams.get("rateLimitRetryMs"),
+		{ min: 0, max: 900_000 },
+	);
+	const rateLimitMaxRetries = parseBoundedInteger(
+		url.searchParams.get("rateLimitRetries"),
+		{ min: 0, max: 10 },
+	);
+	return {
+		handle: url.searchParams.get("handle") ?? "",
+		account: url.searchParams.get("account") ?? undefined,
+		refresh: parseBoolean(url.searchParams.get("refresh")),
+		model: url.searchParams.get("model") === "gpt-5.5" ? "gpt-5.5" : undefined,
+		maxTweets: parseBoundedInteger(url.searchParams.get("maxTweets"), {
+			max: 20_000,
+		}),
+		maxPages: parseBoundedInteger(url.searchParams.get("maxPages"), {
+			max: 500,
+		}),
+		maxConversations: parseBoundedInteger(
+			url.searchParams.get("maxConversations"),
+			{ max: 500 },
+		),
+		maxConversationPages: parseBoundedInteger(
+			url.searchParams.get("maxConversationPages"),
+			{ max: 50 },
+		),
+		...(conversationDelayMs !== undefined ? { conversationDelayMs } : {}),
+		...(rateLimitRetryMs !== undefined ? { rateLimitRetryMs } : {}),
+		...(rateLimitMaxRetries !== undefined ? { rateLimitMaxRetries } : {}),
+	};
+}
+
+function encodeEvent(event: ProfileAnalysisStreamEvent) {
+	return encoder.encode(`${JSON.stringify(event)}\n`);
+}
+
+export const Route = createFileRoute("/api/profile-analysis")({
+	server: {
+		handlers: {
+			GET: ({ request }) =>
+				runRouteEffect(
+					Effect.sync(() => {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
+
+						const url = new URL(request.url);
+						const options = parseOptions(url);
+						let abortAnalysis: (() => void) | undefined;
+
+						return new Response(
+							new ReadableStream({
+								cancel() {
+									abortAnalysis?.();
+								},
+								start(controller) {
+									const abortController = new AbortController();
+									let closed = false;
+									const close = () => {
+										closed = true;
+										abortController.abort();
+									};
+									const closeController = () => {
+										request.signal.removeEventListener("abort", onAbort);
+										if (!closed) {
+											closed = true;
+											controller.close();
+										}
+									};
+									const onAbort = () => close();
+									request.signal.addEventListener("abort", onAbort, {
+										once: true,
+									});
+									abortAnalysis = close;
+									const enqueue = (event: ProfileAnalysisStreamEvent) => {
+										if (closed) return;
+										try {
+											controller.enqueue(encodeEvent(event));
+										} catch {
+											close();
+										}
+									};
+									enqueue({
+										type: "status",
+										label: "Starting profile analysis",
+									});
+
+									runEffectBackground(
+										Effect.gen(function* () {
+											yield* maybeAutoUpdateBackupEffect();
+											return yield* streamProfileAnalysisEffect(
+												{
+													...options,
+													signal: abortController.signal,
+												},
+												{ onEvent: enqueue },
+											);
+										}),
+										{
+											onSuccess: closeController,
+											onFailure: (error) => {
+												enqueue({
+													type: "error",
+													error:
+														error instanceof Error
+															? error.message
+															: "Profile analysis failed",
+												});
+												closeController();
+											},
+										},
+									);
+								},
+							}),
+							{
+								headers: {
+									"cache-control": "no-store",
+									"content-type": "application/x-ndjson; charset=utf-8",
+								},
+							},
+						);
+					}),
+				),
+		},
+	},
+});

package/src/routes/api/query.tsx

@@ -105,6 +105,7 @@ export const Route = createFileRoute("/api/query")({
 							queryResource(resource, {
 								...baseFilters,
 								resource,
+								untilId: url.searchParams.get("untilId") ?? undefined,
 							}),
 						);
 					}),

package/src/routes/api/search-discussion.tsx

@@ -0,0 +1,169 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { Effect } from "effect";
+import { maybeAutoUpdateBackupEffect } from "#/lib/backup";
+import { runEffectBackground } from "#/lib/effect-runtime";
+import {
+	parseBoundedInteger,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import {
+	streamSearchDiscussionEffect,
+	type SearchDiscussionOptions,
+	type SearchDiscussionSource,
+	type SearchDiscussionStreamEvent,
+} from "#/lib/search-discussion";
+import type { TweetSearchMode } from "#/lib/tweet-search-live";
+
+const encoder = new TextEncoder();
+const MAX_DISCUSSION_SEARCH_LIMIT = 20_000;
+const MAX_DISCUSSION_SEARCH_PAGES = 200;
+
+function parseBoolean(value: string | null) {
+	return value === "true" || value === "1" || value === "yes";
+}
+
+function parseSource(value: string | null): SearchDiscussionSource {
+	if (
+		value === "all" ||
+		value === "home" ||
+		value === "mentions" ||
+		value === "authored" ||
+		value === "search" ||
+		value === "likes" ||
+		value === "bookmarks"
+	) {
+		return value;
+	}
+	return "search";
+}
+
+function parseMode(value: string | null): TweetSearchMode {
+	if (
+		value === "auto" ||
+		value === "bird" ||
+		value === "xurl" ||
+		value === "local"
+	) {
+		return value;
+	}
+	return "auto";
+}
+
+function parseOptions(url: URL): SearchDiscussionOptions {
+	return {
+		query: url.searchParams.get("query") ?? "",
+		account: url.searchParams.get("account") ?? undefined,
+		source: parseSource(url.searchParams.get("source")),
+		mode: parseMode(url.searchParams.get("mode")),
+		includeDms: parseBoolean(url.searchParams.get("includeDms")),
+		since: url.searchParams.get("since") ?? undefined,
+		until: url.searchParams.get("until") ?? undefined,
+		question: url.searchParams.get("question") ?? undefined,
+		originalsOnly: parseBoolean(url.searchParams.get("originalsOnly")),
+		hideLowQuality: parseBoolean(url.searchParams.get("hideLowQuality")),
+		refresh: parseBoolean(url.searchParams.get("refresh")),
+		model: url.searchParams.get("model") === "gpt-5.5" ? "gpt-5.5" : undefined,
+		limit: parseBoundedInteger(url.searchParams.get("limit"), {
+			max: MAX_DISCUSSION_SEARCH_LIMIT,
+		}),
+		maxPages: parseBoundedInteger(url.searchParams.get("maxPages"), {
+			max: MAX_DISCUSSION_SEARCH_PAGES,
+		}),
+	};
+}
+
+function encodeEvent(event: SearchDiscussionStreamEvent) {
+	return encoder.encode(`${JSON.stringify(event)}\n`);
+}
+
+export const Route = createFileRoute("/api/search-discussion")({
+	server: {
+		handlers: {
+			GET: ({ request }) =>
+				runRouteEffect(
+					Effect.sync(() => {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
+
+						const url = new URL(request.url);
+						const options = parseOptions(url);
+						let abortDiscussion: (() => void) | undefined;
+
+						return new Response(
+							new ReadableStream({
+								cancel() {
+									abortDiscussion?.();
+								},
+								start(controller) {
+									const abortController = new AbortController();
+									let closed = false;
+									const close = () => {
+										closed = true;
+										abortController.abort();
+									};
+									const closeController = () => {
+										request.signal.removeEventListener("abort", onAbort);
+										if (!closed) {
+											closed = true;
+											controller.close();
+										}
+									};
+									const onAbort = () => close();
+									request.signal.addEventListener("abort", onAbort, {
+										once: true,
+									});
+									abortDiscussion = close;
+									const enqueue = (event: SearchDiscussionStreamEvent) => {
+										if (closed) return;
+										try {
+											controller.enqueue(encodeEvent(event));
+										} catch {
+											close();
+										}
+									};
+
+									runEffectBackground(
+										maybeAutoUpdateBackupEffect().pipe(
+											Effect.flatMap(() => {
+												if (closed || abortController.signal.aborted) {
+													return Effect.succeed(undefined);
+												}
+												return streamSearchDiscussionEffect(
+													{
+														...options,
+														signal: abortController.signal,
+														prefetchAvatars: true,
+													},
+													{ onEvent: enqueue },
+												);
+											}),
+										),
+										{
+											onSuccess: closeController,
+											onFailure: (error) => {
+												enqueue({
+													type: "error",
+													error:
+														error instanceof Error
+															? error.message
+															: "Discussion failed",
+												});
+												closeController();
+											},
+										},
+									);
+								},
+							}),
+							{
+								headers: {
+									"cache-control": "no-store",
+									"content-type": "application/x-ndjson; charset=utf-8",
+								},
+							},
+						);
+					}),
+				),
+		},
+	},
+});

package/src/routes/api/xurl-rate-limits.tsx

@@ -0,0 +1,24 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { Effect } from "effect";
+import {
+	jsonResponse,
+	runRouteEffect,
+	sensitiveRequestErrorResponse,
+} from "#/lib/http-effect";
+import { getXurlRateLimitSnapshot } from "#/lib/xurl-rate-limits";
+
+export const Route = createFileRoute("/api/xurl-rate-limits")({
+	server: {
+		handlers: {
+			GET: ({ request }) =>
+				runRouteEffect(
+					Effect.sync(() => {
+						const denied = sensitiveRequestErrorResponse(request);
+						if (denied) return denied;
+
+						return jsonResponse(getXurlRateLimitSnapshot());
+					}),
+				),
+		},
+	},
+});

package/src/routes/data-sources.tsx

@@ -0,0 +1,255 @@
+import { createFileRoute } from "@tanstack/react-router";
+import {
+	AlertTriangle,
+	CheckCircle2,
+	Database,
+	RefreshCw,
+	Route as RouteIcon,
+	TerminalSquare,
+	XCircle,
+} from "lucide-react";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import type {
+	LiveDataSourceCapability,
+	LiveDataSourceKind,
+	LiveDataSourcesResponse,
+	LiveDataSourceStatus,
+} from "#/lib/types";
+import {
+	cx,
+	errorCopyClass,
+	pageHeaderActionsClass,
+	pageHeaderClass,
+	pageHeaderRowClass,
+	pageSubtitleClass,
+	pageTitleClass,
+	secondaryButtonClass,
+	statusCopyClass,
+} from "#/lib/ui";
+
+export const Route = createFileRoute("/data-sources")({
+	component: DataSourcesRoute,
+});
+
+async function fetchDataSources() {
+	const response = await fetch("/api/data-sources");
+	if (!response.ok) {
+		throw new Error(`Data source status failed (${String(response.status)})`);
+	}
+	return (await response.json()) as LiveDataSourcesResponse;
+}
+
+function sourceIcon(source: LiveDataSourceKind) {
+	if (source === "birdclaw") return Database;
+	if (source === "bird") return TerminalSquare;
+	return RouteIcon;
+}
+
+function statusTone(status: LiveDataSourceStatus["status"]) {
+	if (status === "ok") {
+		return "border-[color:color-mix(in_srgb,#22c55e_45%,var(--line))] bg-[color:color-mix(in_srgb,#22c55e_10%,var(--bg))] text-[color:color-mix(in_srgb,#22c55e_82%,var(--ink))]";
+	}
+	if (status === "warning") {
+		return "border-[color:color-mix(in_srgb,#f59e0b_50%,var(--line))] bg-[color:color-mix(in_srgb,#f59e0b_12%,var(--bg))] text-[color:color-mix(in_srgb,#f59e0b_82%,var(--ink))]";
+	}
+	return "border-[color:color-mix(in_srgb,var(--alert)_55%,var(--line))] bg-[var(--alert-soft)] text-[var(--alert)]";
+}
+
+function statusIcon(status: LiveDataSourceStatus["status"]) {
+	if (status === "ok") return CheckCircle2;
+	if (status === "warning") return AlertTriangle;
+	return XCircle;
+}
+
+function sourceLabel(source: LiveDataSourceKind) {
+	if (source === "birdclaw") return "local";
+	return source;
+}
+
+function SourceCard({ source }: { source: LiveDataSourceStatus }) {
+	const Icon = sourceIcon(source.source);
+	const StatusIcon = statusIcon(source.status);
+	return (
+		<section className="min-w-0 border-b border-[var(--line)] px-4 py-4">
+			<div className="flex items-start justify-between gap-3">
+				<div className="flex min-w-0 items-center gap-3">
+					<div className="grid size-10 shrink-0 place-items-center rounded-md border border-[var(--line)] bg-[var(--bg-active)] text-[var(--ink)]">
+						<Icon className="size-5" strokeWidth={1.9} />
+					</div>
+					<div className="min-w-0">
+						<h2 className="truncate text-[16px] font-bold text-[var(--ink)]">
+							{source.label}
+						</h2>
+						<p className="truncate text-[13px] text-[var(--ink-soft)]">
+							{source.installed === false ? "not installed" : source.detail}
+						</p>
+					</div>
+				</div>
+				<span
+					className={cx(
+						"inline-flex shrink-0 items-center gap-1 rounded-full border px-2 py-0.5 text-[12px] font-bold",
+						statusTone(source.status),
+					)}
+				>
+					<StatusIcon className="size-3.5" strokeWidth={2} />
+					{source.works ? "works" : source.status}
+				</span>
+			</div>
+			<div className="mt-4 flex flex-wrap gap-2">
+				{source.accounts.length > 0 ? (
+					source.accounts.map((account, index) => (
+						<span
+							key={`${account.app ?? ""}:${account.username ?? account.handle ?? account.id ?? String(index)}`}
+							className="inline-flex max-w-full items-center gap-1.5 rounded-full border border-[var(--line)] bg-[var(--bg)] px-2.5 py-1 text-[12px] font-semibold text-[var(--ink-soft)]"
+						>
+							{account.app ? (
+								<span className="text-[var(--ink-faint)]">{account.app}</span>
+							) : null}
+							<span className="truncate">
+								{account.handle ??
+									(account.username ? `@${account.username}` : account.id)}
+							</span>
+							{account.isDefault ? (
+								<span className="text-[var(--accent)]">default</span>
+							) : null}
+						</span>
+					))
+				) : (
+					<span className="text-[13px] text-[var(--ink-soft)]">
+						no authenticated account detected
+					</span>
+				)}
+			</div>
+		</section>
+	);
+}
+
+function CapabilityRow({
+	capability,
+	sourcesByKind,
+}: {
+	capability: LiveDataSourceCapability;
+	sourcesByKind: Map<LiveDataSourceKind, LiveDataSourceStatus>;
+}) {
+	const chain = [capability.primary, ...capability.fallbacks];
+	return (
+		<div className="grid gap-3 border-b border-[var(--line)] px-4 py-4 min-[840px]:grid-cols-[220px_minmax(0,1fr)]">
+			<div className="min-w-0">
+				<div className="font-bold text-[var(--ink)]">{capability.label}</div>
+				{capability.notes ? (
+					<div className="mt-1 text-[12px] text-[var(--ink-soft)]">
+						{capability.notes}
+					</div>
+				) : null}
+			</div>
+			<div className="flex min-w-0 flex-wrap items-center gap-2">
+				{chain.map((source, index) => {
+					const status = sourcesByKind.get(source);
+					const ok = Boolean(status?.works);
+					return (
+						<div
+							key={`${capability.key}:${source}:${String(index)}`}
+							className={cx(
+								"inline-flex items-center gap-2 rounded-full border px-3 py-1 text-[13px] font-semibold",
+								ok
+									? "border-[color:color-mix(in_srgb,#22c55e_38%,var(--line))] text-[var(--ink)]"
+									: "border-[var(--line)] text-[var(--ink-soft)]",
+							)}
+						>
+							<span className="text-[12px] text-[var(--ink-faint)]">
+								{index === 0 ? "primary" : `fallback ${String(index)}`}
+							</span>
+							<span>{sourceLabel(source)}</span>
+						</div>
+					);
+				})}
+			</div>
+		</div>
+	);
+}
+
+function DataSourcesRoute() {
+	const [snapshot, setSnapshot] = useState<LiveDataSourcesResponse | null>(null);
+	const [error, setError] = useState<string | null>(null);
+	const [loading, setLoading] = useState(true);
+	const sourcesByKind = useMemo(
+		() =>
+			new Map(
+				(snapshot?.sources ?? []).map((source) => [source.source, source]),
+			),
+		[snapshot],
+	);
+
+	const load = useCallback(() => {
+		setLoading(true);
+		setError(null);
+		fetchDataSources()
+			.then(setSnapshot)
+			.catch((cause: unknown) => {
+				setError(
+					cause instanceof Error ? cause.message : "Data sources unavailable",
+				);
+			})
+			.finally(() => setLoading(false));
+	}, []);
+
+	useEffect(() => {
+		load();
+	}, [load]);
+
+	return (
+		<section className="flex min-h-screen flex-col">
+			<header className={pageHeaderClass}>
+				<div className={pageHeaderRowClass}>
+					<div>
+						<h1 className={pageTitleClass}>Data Sources</h1>
+						<p className={pageSubtitleClass}>
+							Live auth, local archive, and automatic fallback order
+						</p>
+					</div>
+					<div className={pageHeaderActionsClass}>
+						<button
+							className={secondaryButtonClass}
+							type="button"
+							onClick={load}
+							disabled={loading}
+						>
+							<RefreshCw className={cx("size-4", loading && "animate-spin")} />
+							Refresh
+						</button>
+					</div>
+				</div>
+			</header>
+			{error ? <div className={errorCopyClass}>{error}</div> : null}
+			{snapshot ? (
+				<>
+					<div className="grid border-t border-[var(--line)]">
+						{snapshot.sources.map((source) => (
+							<SourceCard key={source.source} source={source} />
+						))}
+					</div>
+					<section className="min-h-0 flex-1">
+						<header className="border-b border-[var(--line)] px-4 py-3">
+							<h2 className="text-[16px] font-bold text-[var(--ink)]">
+								Fallbacks
+							</h2>
+							<p className="text-[13px] text-[var(--ink-soft)]">
+								Calls try the next source when the current source cannot serve
+								the request.
+							</p>
+						</header>
+						{snapshot.capabilities.map((capability) => (
+							<CapabilityRow
+								key={capability.key}
+								capability={capability}
+								sourcesByKind={sourcesByKind}
+							/>
+						))}
+					</section>
+				</>
+			) : loading ? (
+				<div className={statusCopyClass}>Checking data sources...</div>
+			) : null}
+		</section>
+	);
+}