birdclaw 0.8.0 → 0.8.2

package/src/lib/present.ts

@@ -2,13 +2,111 @@ export function formatCompactNumber(value: number) {
 	return new Intl.NumberFormat("en", { notation: "compact" }).format(value);
 }
 
+const SECOND_MS = 1_000;
+const MINUTE_MS = 60 * SECOND_MS;
+const HOUR_MS = 60 * MINUTE_MS;
+const DAY_MS = 24 * HOUR_MS;
+
+const timeFormatter = new Intl.DateTimeFormat("en", {
+	hour: "numeric",
+	minute: "2-digit",
+});
+
+const weekdayFormatter = new Intl.DateTimeFormat("en", {
+	weekday: "long",
+});
+
+const sameYearDateFormatter = new Intl.DateTimeFormat("en", {
+	month: "short",
+	day: "numeric",
+});
+
+const otherYearDateFormatter = new Intl.DateTimeFormat("en", {
+	year: "numeric",
+	month: "short",
+	day: "numeric",
+});
+
+const exactTimestampFormatter = new Intl.DateTimeFormat("en", {
+	year: "numeric",
+	month: "long",
+	day: "numeric",
+	hour: "numeric",
+	minute: "2-digit",
+	second: "2-digit",
+	timeZoneName: "short",
+});
+
+function parseTimestamp(value: string) {
+	const date = new Date(value);
+	return Number.isNaN(date.getTime()) ? null : date;
+}
+
+function localCalendarDay(date: Date) {
+	return Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()) / DAY_MS;
+}
+
+function formatCalendarTimestamp(date: Date, now: Date) {
+	const time = timeFormatter.format(date);
+	const dayDifference = localCalendarDay(now) - localCalendarDay(date);
+
+	if (dayDifference === 1) {
+		return `Yesterday at ${time}`;
+	}
+
+	if (dayDifference >= 2 && dayDifference <= 6) {
+		return `${weekdayFormatter.format(date)} at ${time}`;
+	}
+
+	const dateFormatter =
+		date.getFullYear() === now.getFullYear()
+			? sameYearDateFormatter
+			: otherYearDateFormatter;
+	return `${dateFormatter.format(date)} at ${time}`;
+}
+
 export function formatShortTimestamp(value: string) {
+	const date = parseTimestamp(value);
+	if (!date) return value;
+
 	return new Intl.DateTimeFormat("en", {
 		hour: "numeric",
 		minute: "2-digit",
 		month: "short",
 		day: "numeric",
-	}).format(new Date(value));
+	}).format(date);
+}
+
+export function formatExactTimestamp(value: string) {
+	const date = parseTimestamp(value);
+	return date ? exactTimestampFormatter.format(date) : value;
+}
+
+export function formatSmartTimestamp(value: string, nowValue = Date.now()) {
+	const date = parseTimestamp(value);
+	const now = new Date(nowValue);
+	if (!date || Number.isNaN(now.getTime())) return value;
+
+	const elapsed = now.getTime() - date.getTime();
+	if (elapsed >= -45 * SECOND_MS && elapsed < 45 * SECOND_MS) {
+		return "just now";
+	}
+
+	if (elapsed < 0) {
+		return formatCalendarTimestamp(date, now);
+	}
+
+	if (elapsed < HOUR_MS) {
+		const minutes = Math.max(1, Math.floor(elapsed / MINUTE_MS));
+		return `${minutes} min ago`;
+	}
+
+	if (elapsed < DAY_MS) {
+		const hours = Math.max(1, Math.floor(elapsed / HOUR_MS));
+		return `${hours} ${hours === 1 ? "hour" : "hours"} ago`;
+	}
+
+	return formatCalendarTimestamp(date, now);
 }
 
 export function getInitials(value: string) {

package/src/lib/profile-analysis.ts

@@ -407,7 +407,7 @@ function mergeXurlTweetsIntoLocalStore(
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/queries.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { Effect } from "effect";
 import type { Database } from "./sqlite";
-import { findArchivesEffect } from "./archive-finder";
+import { findArchivesCachedEffect } from "./archive-finder";
 import { getDb, getNativeDb } from "./db";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { fetchProfileAffiliations } from "./profile-affiliations";
@@ -701,10 +701,9 @@ function getAccountProfileMeta(
 		| undefined;
 }
 
-export function getQueryEnvelopeEffect(): Effect.Effect<
-	QueryEnvelope,
-	unknown
-> {
+export function getQueryEnvelopeEffect({
+	includeArchives = true,
+}: { includeArchives?: boolean } = {}): Effect.Effect<QueryEnvelope, unknown> {
 	return Effect.gen(function* () {
 		const db = yield* trySync(() => getDb());
 		const nativeDb = yield* trySync(() => getNativeDb());
@@ -736,7 +735,9 @@ export function getQueryEnvelopeEffect(): Effect.Effect<
 					.orderBy("name", "asc")
 					.execute(),
 			),
-			archives: findArchivesEffect(),
+			archives: includeArchives
+				? findArchivesCachedEffect()
+				: Effect.succeed([]),
 			transport: getTransportStatusEffect(),
 		});
 
@@ -1286,6 +1287,67 @@ function getTweetById(
 	);
 }
 
+export function getTweetsByIds(
+	tweetIds: string[],
+	accountId?: string,
+): EmbeddedTweet[] {
+	const db = getNativeDb();
+	const scopedAccountId =
+		accountId && accountId !== "all" ? accountId : undefined;
+	const urlExpansionCache: UrlExpansionCache = new Map();
+	const profileByHandleCache: ProfileByHandleCache = new Map();
+	const resolveProfileByHandle = (handle: string) =>
+		getProfileByHandle(db, profileByHandleCache, handle);
+	const seen = new Set<string>();
+	const tweets: EmbeddedTweet[] = [];
+
+	for (const tweetId of tweetIds) {
+		const normalized = tweetId.trim().replace(/^tweet_/, "");
+		if (!normalized || seen.has(normalized)) continue;
+		seen.add(normalized);
+		if (
+			scopedAccountId &&
+			!db
+				.prepare(
+					`
+					select 1
+					from tweets tweet
+					where tweet.id = ?
+						and (
+							tweet.account_id = ?
+							or exists (
+								select 1
+								from tweet_account_edges edge
+								where edge.account_id = ?
+									and edge.tweet_id = tweet.id
+							)
+							or exists (
+								select 1
+								from tweet_collections collection
+								where collection.account_id = ?
+									and collection.tweet_id = tweet.id
+							)
+						)
+					limit 1
+					`,
+				)
+				.get(normalized, scopedAccountId, scopedAccountId, scopedAccountId)
+		) {
+			continue;
+		}
+		const tweet = getTweetById(
+			db,
+			urlExpansionCache,
+			normalized,
+			resolveProfileByHandle,
+			scopedAccountId,
+		);
+		if (tweet) tweets.push(tweet);
+	}
+
+	return tweets;
+}
+
 function listTweetDescendants(
 	db: Database,
 	urlExpansionCache: UrlExpansionCache,

package/src/lib/sqlite.ts

@@ -5,6 +5,7 @@ export type Database = NativeSqliteDatabase;
 type DatabaseOptions = {
 	readonly?: boolean;
 	fileMustExist?: boolean;
+	timeout?: number;
 };
 
 type PragmaOptions = {
@@ -16,6 +17,8 @@ type RunResult = {
 	lastInsertRowid: number;
 };
 
+export const SQLITE_BUSY_TIMEOUT_MS = 30_000;
+
 function bindArgs(parameters: unknown[]) {
 	if (parameters.length === 1 && Array.isArray(parameters[0])) {
 		return parameters[0];
@@ -87,7 +90,7 @@ export class NativeSqliteDatabase {
 	constructor(path: string, options: DatabaseOptions = {}) {
 		this.db = new DatabaseSync(path, {
 			readOnly: options.readonly,
-			timeout: 5000,
+			timeout: options.timeout ?? SQLITE_BUSY_TIMEOUT_MS,
 		});
 	}
 
@@ -123,7 +126,7 @@ export class NativeSqliteDatabase {
 		return (...args: TArgs) => {
 			const nested = this.db.isTransaction;
 			const savepoint = `__birdclaw_tx_${++this.transactionDepth}`;
-			this.exec(nested ? `savepoint ${savepoint}` : "begin");
+			this.exec(nested ? `savepoint ${savepoint}` : "begin immediate");
 			try {
 				const result = fn(...args);
 				this.exec(nested ? `release ${savepoint}` : "commit");

package/src/lib/timeline-collections-live.ts

@@ -8,6 +8,7 @@ import { getNativeDb } from "./db";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
@@ -310,7 +311,7 @@ function mergeTimelineCollectionIntoLocalStore(
 				countTweetMedia(tweet),
 				bookmarked,
 				liked,
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/timeline-live.ts

@@ -5,6 +5,7 @@ import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	XurlMediaItem,
 	XurlMentionUser,
@@ -244,7 +245,7 @@ function mergeHomeTimelineIntoLocalStore(
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/tweet-render.ts

@@ -1,4 +1,5 @@
 import type {
+	TweetArticle,
 	TweetEntities,
 	TweetHashtagEntity,
 	TweetMentionEntity,
@@ -57,6 +58,34 @@ export function displayUrlForLink(url: string) {
 	}
 }
 
+function comparableUrl(value: string) {
+	try {
+		const parsed = new URL(value);
+		return `${parsed.protocol}//${parsed.hostname.replace(/^www\./, "")}${parsed.pathname}`;
+	} catch {
+		return value.split("?")[0] ?? value;
+	}
+}
+
+export function isTweetArticleUrlEntity(
+	entry: TweetUrlEntity,
+	article: TweetArticle,
+) {
+	if (comparableUrl(entry.expandedUrl) === comparableUrl(article.url)) {
+		return true;
+	}
+	try {
+		const parsed = new URL(entry.expandedUrl);
+		const host = parsed.hostname.replace(/^www\./, "");
+		return (
+			(host === "x.com" || host === "twitter.com") &&
+			parsed.pathname.startsWith("/i/article/")
+		);
+	} catch {
+		return false;
+	}
+}
+
 function asRecord(value: unknown) {
 	return value && typeof value === "object"
 		? (value as Record<string, unknown>)
@@ -68,6 +97,9 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
 	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
 	const rawHashtags = Array.isArray(entities.hashtags) ? entities.hashtags : [];
+	const rawArticle = asRecord(entities.article);
+	const articleTitle = String(rawArticle.title ?? "").trim();
+	const articleUrl = String(rawArticle.url ?? "").trim();
 
 	return {
 		...(rawMentions.length
@@ -102,6 +134,23 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 							),
 							start: Number(value.start ?? 0),
 							end: Number(value.end ?? 0),
+							...(typeof value.title === "string"
+								? { title: value.title }
+								: {}),
+							...(typeof value.description === "string" ||
+							value.description === null
+								? { description: value.description }
+								: {}),
+							...(typeof value.imageUrl === "string"
+								? { imageUrl: value.imageUrl }
+								: typeof value.image_url === "string"
+									? { imageUrl: value.image_url }
+									: {}),
+							...(typeof value.siteName === "string"
+								? { siteName: value.siteName }
+								: typeof value.site_name === "string"
+									? { siteName: value.site_name }
+									: {}),
 						};
 					}),
 				}
@@ -118,6 +167,35 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 					}),
 				}
 			: {}),
+		...(articleTitle && articleUrl
+			? {
+					article: {
+						title: articleTitle,
+						url: articleUrl,
+						...(typeof (rawArticle.previewText ?? rawArticle.preview_text) ===
+							"string" &&
+						String(rawArticle.previewText ?? rawArticle.preview_text).trim()
+							? {
+									previewText: String(
+										rawArticle.previewText ?? rawArticle.preview_text,
+									).trim(),
+								}
+							: {}),
+						...(typeof (
+							rawArticle.coverImageUrl ?? rawArticle.cover_image_url
+						) === "string" &&
+						String(
+							rawArticle.coverImageUrl ?? rawArticle.cover_image_url,
+						).trim()
+							? {
+									coverImageUrl: String(
+										rawArticle.coverImageUrl ?? rawArticle.cover_image_url,
+									).trim(),
+								}
+							: {}),
+					},
+				}
+			: {}),
 	};
 }
 

package/src/lib/tweet-search-live.ts

@@ -5,6 +5,7 @@ import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type { XurlMentionsResponse, XurlTweetsResponse } from "./types";
 import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
@@ -190,7 +191,7 @@ function mergeTweetSearchIntoLocalStore(
 				replyToId,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 				quotedTweetId,
 			);

package/src/lib/types.ts

@@ -101,10 +101,18 @@ export interface TweetHashtagEntity {
 	end: number;
 }
 
+export interface TweetArticle {
+	title: string;
+	previewText?: string;
+	url: string;
+	coverImageUrl?: string;
+}
+
 export interface TweetEntities {
 	mentions?: TweetMentionEntity[];
 	urls?: TweetUrlEntity[];
 	hashtags?: TweetHashtagEntity[];
+	article?: TweetArticle;
 }
 
 export interface TweetMediaItem {

package/src/lib/ui.ts

@@ -311,7 +311,7 @@ export const profilePreviewTriggerClass =
 	"profile-preview-trigger inline-flex text-inherit";
 
 export const profilePreviewCardClass =
-	"pointer-events-none absolute left-0 z-30 grid w-[280px] gap-2 rounded-2xl border border-[var(--line)] bg-[var(--bg-elevated)] p-3 opacity-0 shadow-[0_8px_28px_var(--shadow-strong)] transition-all duration-150 group-hover:pointer-events-auto group-hover:opacity-100 group-focus-within:pointer-events-auto group-focus-within:opacity-100";
+	"fixed z-40 w-[280px] overflow-y-auto rounded-2xl border border-[var(--line)] bg-[var(--bg-elevated)] p-3 shadow-[0_8px_28px_var(--shadow-strong)]";
 
 export const profilePreviewHeaderClass = "flex items-center gap-3";
 

package/src/router.tsx

@@ -7,7 +7,7 @@ export function getRouter() {
 
 		scrollRestoration: true,
 		defaultPreload: "intent",
-		defaultPreloadStaleTime: 0,
+		defaultPreloadStaleTime: 60_000,
 	});
 
 	return router;

package/src/routes/__root.tsx

@@ -1,11 +1,9 @@
-import { TanStackDevtools } from "@tanstack/react-devtools";
 import {
 	createRootRoute,
 	HeadContent,
 	Scripts,
 	useRouterState,
 } from "@tanstack/react-router";
-import { TanStackRouterDevtoolsPanel } from "@tanstack/react-router-devtools";
 import type { ReactNode } from "react";
 import { AppNav } from "#/components/AppNav";
 import { ThemeProvider, themeScript } from "#/lib/theme";
@@ -74,17 +72,6 @@ function RootDocument({ children }: { children: ReactNode }) {
 						</main>
 					</div>
 				</ThemeProvider>
-				<TanStackDevtools
-					config={{
-						position: "bottom-right",
-					}}
-					plugins={[
-						{
-							name: "Tanstack Router",
-							render: <TanStackRouterDevtoolsPanel />,
-						},
-					]}
-				/>
 				<Scripts />
 			</body>
 		</html>

package/src/routes/api/period-digest.tsx

@@ -1,13 +1,13 @@
 import { createFileRoute } from "@tanstack/react-router";
 import { Effect } from "effect";
 import { maybeAutoUpdateBackupEffect } from "#/lib/backup";
-import { runEffectBackground } from "#/lib/effect-runtime";
 import {
 	jsonResponse,
 	parseBoundedInteger,
 	runRouteEffect,
 	sensitiveRequestErrorResponse,
 } from "#/lib/http-effect";
+import { createEffectNdjsonResponse } from "#/lib/ndjson-stream";
 import {
 	normalizeDigestLanguage,
 	streamPeriodDigestEffect,
@@ -15,8 +15,6 @@ import {
 	type PeriodDigestStreamEvent,
 } from "#/lib/period-digest";
 
-const encoder = new TextEncoder();
-
 function parseBoolean(value: string | null) {
 	return value === "true" || value === "1" || value === "yes";
 }
@@ -52,16 +50,12 @@ function parseOptions(url: URL): PeriodDigestOptions {
 	};
 }
 
-function encodeEvent(event: PeriodDigestStreamEvent) {
-	return encoder.encode(`${JSON.stringify(event)}\n`);
-}
-
 export const Route = createFileRoute("/api/period-digest")({
 	server: {
 		handlers: {
 			GET: ({ request }) =>
 				runRouteEffect(
-					Effect.gen(function* () {
+					Effect.sync(() => {
 						const denied = sensitiveRequestErrorResponse(request);
 						if (denied) return denied;
 
@@ -78,70 +72,28 @@ export const Route = createFileRoute("/api/period-digest")({
 								{ status: 400 },
 							);
 						}
-						yield* maybeAutoUpdateBackupEffect();
-						let abortDigest: (() => void) | undefined;
-
-						return new Response(
-							new ReadableStream({
-								cancel() {
-									abortDigest?.();
+						return createEffectNdjsonResponse<PeriodDigestStreamEvent>({
+							request,
+							initialEvents: [
+								{
+									type: "status",
+									label: "Preparing local archive",
+									detail: "Checking for backup updates.",
 								},
-								start(controller) {
-									const abortController = new AbortController();
-									let closed = false;
-									const close = () => {
-										closed = true;
-										abortController.abort();
-									};
-									const closeController = () => {
-										request.signal.removeEventListener("abort", onAbort);
-										if (!closed) {
-											closed = true;
-											controller.close();
-										}
-									};
-									const onAbort = () => close();
-									request.signal.addEventListener("abort", onAbort, {
-										once: true,
-									});
-									abortDigest = close;
-									const enqueue = (event: PeriodDigestStreamEvent) => {
-										if (closed) return;
-										try {
-											controller.enqueue(encodeEvent(event));
-										} catch {
-											close();
-										}
-									};
-
-									runEffectBackground(
-										streamPeriodDigestEffect(
-											{ ...options, signal: abortController.signal },
-											{ onEvent: enqueue },
-										),
-										{
-											onSuccess: closeController,
-											onFailure: (error) => {
-												enqueue({
-													type: "error",
-													error:
-														error instanceof Error
-															? error.message
-															: "Digest failed",
-												});
-												closeController();
-											},
-										},
+							],
+							run: ({ signal, emit }) =>
+								Effect.gen(function* () {
+									yield* maybeAutoUpdateBackupEffect();
+									return yield* streamPeriodDigestEffect(
+										{ ...options, signal },
+										{ onEvent: emit },
 									);
-								},
+								}),
+							errorEvent: (error) => ({
+								type: "error",
+								error: error instanceof Error ? error.message : "Digest failed",
 							}),
-							{
-								headers: {
-									"cache-control": "no-store",
-									"content-type": "application/x-ndjson; charset=utf-8",
-								},
-							},
-						);
+						});
 					}),
 				),
 		},