birdclaw 0.8.0 → 0.8.1

package/src/lib/period-digest.ts

@@ -6,10 +6,14 @@ import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { getLinkInsights } from "./link-insights";
 import { syncMentionThreadsEffect } from "./mention-threads-live";
 import { syncMentionsEffect } from "./mentions-live";
-import { listDmConversations, listTimelineItems } from "./queries";
+import {
+	getTweetsByIds,
+	listDmConversations,
+	listTimelineItems,
+} from "./queries";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
-import type { ProfileRecord, TweetEntities } from "./types";
+import type { EmbeddedTweet, ProfileRecord, TweetEntities } from "./types";
 
 export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
 export type PeriodDigestSourceKind =
@@ -222,6 +226,7 @@ const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
 const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
 const DEFAULT_LIVE_THREAD_LIMIT = 12;
 const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
+const DEFAULT_DIGEST_FRESHNESS_MS = 5 * 60_000;
 const MAX_PROMPT_DATA_CHARS = 1_200_000;
 const DELIMITER_PATTERN = /\n---\s*\n/;
 const VISIBLE_DELIMITER_HOLD = 8;
@@ -363,6 +368,26 @@ function compactTweet(
 	};
 }
 
+function compactEmbeddedTweet(item: EmbeddedTweet): CompactTweet {
+	return {
+		id: item.id,
+		url: tweetUrl(item.author.handle, item.id),
+		source: "home",
+		author: item.author.handle,
+		name: item.author.displayName,
+		authorProfile: item.author,
+		createdAt: item.createdAt,
+		text: item.text,
+		entities: item.entities,
+		likeCount: item.likeCount ?? 0,
+		liked: Boolean(item.liked),
+		bookmarked: Boolean(item.bookmarked),
+		needsReply: !item.isReplied,
+		replyToId: item.replyToId ?? null,
+		replyToTweet: null,
+	};
+}
+
 function dedupeTweets(tweets: CompactTweet[]) {
 	const seen = new Set<string>();
 	const items: CompactTweet[] = [];
@@ -879,6 +904,110 @@ function digestCacheKey(
 	return parts.join(":");
 }
 
+function latestDigestCacheKey(options: PeriodDigestOptions) {
+	const period = normalizePeriod(options.period);
+	const window = resolvePeriodDigestWindow(options);
+	const identity = {
+		period,
+		day:
+			period === "today" || period === "yesterday"
+				? window.since
+				: localDateStart(new Date()).toISOString(),
+		since: options.since?.trim() || null,
+		until: options.until?.trim() || null,
+		account: options.account?.trim() || null,
+		includeDms: Boolean(options.includeDms),
+		maxTweets: Math.max(
+			20,
+			Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
+		),
+		maxLinks: Math.max(3, Math.trunc(options.maxLinks ?? DEFAULT_MAX_LINKS)),
+		model: modelFromOptions(options),
+		language: languageFromOptions(options) ?? null,
+		reasoningEffort: reasoningEffortFromOptions(options),
+		serviceTier: serviceTierFromOptions(options),
+	};
+	return `period-digest-latest:v1:${createHash("sha1")
+		.update(JSON.stringify(identity))
+		.digest("hex")}`;
+}
+
+function collectDigestTweetIds(digest: PeriodDigest) {
+	const tweetIds = new Set(digest.sourceTweetIds);
+	for (const topic of digest.keyTopics) {
+		for (const tweetId of topic.tweetIds) tweetIds.add(tweetId);
+	}
+	for (const link of digest.notableLinks) {
+		for (const tweetId of link.sourceTweetIds) tweetIds.add(tweetId);
+	}
+	for (const action of digest.actionItems) {
+		if (action.tweetId) tweetIds.add(action.tweetId);
+	}
+	return [...tweetIds];
+}
+
+function enrichContextWithCitedTweets(
+	context: PeriodDigestContext,
+	digest: PeriodDigest,
+) {
+	const existingIds = new Set(context.tweets.map((tweet) => tweet.id));
+	const missingIds = collectDigestTweetIds(digest).filter(
+		(tweetId) => !existingIds.has(tweetId.replace(/^tweet_/, "")),
+	);
+	if (missingIds.length === 0) return context;
+	const citedTweets = getTweetsByIds(missingIds, context.account).map(
+		compactEmbeddedTweet,
+	);
+	return citedTweets.length > 0
+		? { ...context, tweets: [...context.tweets, ...citedTweets] }
+		: context;
+}
+
+interface CachedPeriodDigestValue {
+	context?: PeriodDigestContext;
+	digest: PeriodDigest;
+	markdown: string;
+	model: string;
+	reasoningEffort: string;
+	serviceTier: string;
+	updatedAt?: string;
+}
+
+function cachedDigestResult(
+	cached: { value: CachedPeriodDigestValue; updatedAt: string },
+	context: PeriodDigestContext,
+): PeriodDigestRunResult {
+	const digest = PeriodDigestSchema.parse(cached.value.digest);
+	return {
+		context: enrichContextWithCitedTweets(context, digest),
+		digest,
+		markdown: cached.value.markdown,
+		model: cached.value.model,
+		reasoningEffort: cached.value.reasoningEffort,
+		serviceTier: cached.value.serviceTier,
+		cached: true,
+		updatedAt: cached.value.updatedAt ?? cached.updatedAt,
+	};
+}
+
+function isFreshDigestCache(updatedAt: string) {
+	const timestamp = Date.parse(updatedAt);
+	return (
+		Number.isFinite(timestamp) &&
+		Date.now() - timestamp <= DEFAULT_DIGEST_FRESHNESS_MS
+	);
+}
+
+function emitCachedDigest(
+	result: PeriodDigestRunResult,
+	handlers: PeriodDigestStreamHandlers,
+) {
+	handlers.onEvent?.({ type: "start", context: result.context, cached: true });
+	handlers.onDelta?.(result.markdown);
+	handlers.onEvent?.({ type: "delta", delta: result.markdown });
+	handlers.onEvent?.({ type: "done", result });
+}
+
 function buildPrompt(
 	context: PeriodDigestContext,
 	options?: { language?: string },
@@ -1235,6 +1364,9 @@ function readOpenAIStreamEffect(
 					languageFromOptions(options),
 				),
 			);
+			const enrichedContext = yield* tryDigestSync(() =>
+				enrichContextWithCitedTweets(context, parsed.digest),
+			);
 			const cacheKey = digestCacheKey(context, options);
 			const updatedAt = yield* tryDigestSync(() =>
 				writeSyncCache(cacheKey, {
@@ -1248,7 +1380,7 @@ function readOpenAIStreamEffect(
 				}),
 			);
 			const result: PeriodDigestRunResult = {
-				context,
+				context: enrichedContext,
 				digest: parsed.digest,
 				markdown: parsed.markdown,
 				model: modelFromOptions(options),
@@ -1257,6 +1389,17 @@ function readOpenAIStreamEffect(
 				cached: false,
 				updatedAt,
 			};
+			yield* tryDigestSync(() =>
+				writeSyncCache(latestDigestCacheKey(options), {
+					context: result.context,
+					digest: result.digest,
+					markdown: result.markdown,
+					model: result.model,
+					reasoningEffort: result.reasoningEffort,
+					serviceTier: result.serviceTier,
+					updatedAt: result.updatedAt,
+				}),
+			);
 			handlers.onEvent?.({ type: "done", result });
 			return result;
 		}
@@ -1278,6 +1421,28 @@ export function streamPeriodDigestEffect(
 			...options,
 			language: yield* tryDigestSync(() => languageFromOptions(options)),
 		};
+		const latestCached = resolvedOptions.refresh
+			? null
+			: !resolvedOptions.liveSync
+				? yield* tryDigestSync(() =>
+						readSyncCache<CachedPeriodDigestValue>(
+							latestDigestCacheKey(resolvedOptions),
+						),
+					)
+				: null;
+		const latestContext = latestCached?.value.context;
+		if (
+			latestCached &&
+			latestContext &&
+			isFreshDigestCache(latestCached.value.updatedAt ?? latestCached.updatedAt)
+		) {
+			const result = yield* tryDigestSync(() =>
+				cachedDigestResult(latestCached, latestContext),
+			);
+			emitCachedDigest(result, handlers);
+			return result;
+		}
+
 		yield* refreshPeriodDigestInputsEffect(
 			resolvedOptions,
 			{ threads: false },
@@ -1290,30 +1455,25 @@ export function streamPeriodDigestEffect(
 		const cached = resolvedOptions.refresh
 			? null
 			: yield* tryDigestSync(() =>
-					readSyncCache<{
-						digest: PeriodDigest;
-						markdown: string;
-						model: string;
-						reasoningEffort: string;
-						serviceTier: string;
-					}>(cacheKey),
+					readSyncCache<CachedPeriodDigestValue>(cacheKey),
 				);
 
 		if (cached) {
-			const result: PeriodDigestRunResult = yield* tryDigestSync(() => ({
-				context,
-				digest: PeriodDigestSchema.parse(cached.value.digest),
-				markdown: cached.value.markdown,
-				model: cached.value.model,
-				reasoningEffort: cached.value.reasoningEffort,
-				serviceTier: cached.value.serviceTier,
-				cached: true,
-				updatedAt: cached.updatedAt,
-			}));
-			handlers.onEvent?.({ type: "start", context, cached: true });
-			handlers.onDelta?.(result.markdown);
-			handlers.onEvent?.({ type: "delta", delta: result.markdown });
-			handlers.onEvent?.({ type: "done", result });
+			const result = yield* tryDigestSync(() =>
+				cachedDigestResult(cached, context),
+			);
+			yield* tryDigestSync(() =>
+				writeSyncCache(latestDigestCacheKey(resolvedOptions), {
+					context: result.context,
+					digest: result.digest,
+					markdown: result.markdown,
+					model: result.model,
+					reasoningEffort: result.reasoningEffort,
+					serviceTier: result.serviceTier,
+					updatedAt: result.updatedAt,
+				}),
+			);
+			emitCachedDigest(result, handlers);
 			return result;
 		}
 

package/src/lib/present.ts

@@ -2,13 +2,111 @@ export function formatCompactNumber(value: number) {
 	return new Intl.NumberFormat("en", { notation: "compact" }).format(value);
 }
 
+const SECOND_MS = 1_000;
+const MINUTE_MS = 60 * SECOND_MS;
+const HOUR_MS = 60 * MINUTE_MS;
+const DAY_MS = 24 * HOUR_MS;
+
+const timeFormatter = new Intl.DateTimeFormat("en", {
+	hour: "numeric",
+	minute: "2-digit",
+});
+
+const weekdayFormatter = new Intl.DateTimeFormat("en", {
+	weekday: "long",
+});
+
+const sameYearDateFormatter = new Intl.DateTimeFormat("en", {
+	month: "short",
+	day: "numeric",
+});
+
+const otherYearDateFormatter = new Intl.DateTimeFormat("en", {
+	year: "numeric",
+	month: "short",
+	day: "numeric",
+});
+
+const exactTimestampFormatter = new Intl.DateTimeFormat("en", {
+	year: "numeric",
+	month: "long",
+	day: "numeric",
+	hour: "numeric",
+	minute: "2-digit",
+	second: "2-digit",
+	timeZoneName: "short",
+});
+
+function parseTimestamp(value: string) {
+	const date = new Date(value);
+	return Number.isNaN(date.getTime()) ? null : date;
+}
+
+function localCalendarDay(date: Date) {
+	return Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()) / DAY_MS;
+}
+
+function formatCalendarTimestamp(date: Date, now: Date) {
+	const time = timeFormatter.format(date);
+	const dayDifference = localCalendarDay(now) - localCalendarDay(date);
+
+	if (dayDifference === 1) {
+		return `Yesterday at ${time}`;
+	}
+
+	if (dayDifference >= 2 && dayDifference <= 6) {
+		return `${weekdayFormatter.format(date)} at ${time}`;
+	}
+
+	const dateFormatter =
+		date.getFullYear() === now.getFullYear()
+			? sameYearDateFormatter
+			: otherYearDateFormatter;
+	return `${dateFormatter.format(date)} at ${time}`;
+}
+
 export function formatShortTimestamp(value: string) {
+	const date = parseTimestamp(value);
+	if (!date) return value;
+
 	return new Intl.DateTimeFormat("en", {
 		hour: "numeric",
 		minute: "2-digit",
 		month: "short",
 		day: "numeric",
-	}).format(new Date(value));
+	}).format(date);
+}
+
+export function formatExactTimestamp(value: string) {
+	const date = parseTimestamp(value);
+	return date ? exactTimestampFormatter.format(date) : value;
+}
+
+export function formatSmartTimestamp(value: string, nowValue = Date.now()) {
+	const date = parseTimestamp(value);
+	const now = new Date(nowValue);
+	if (!date || Number.isNaN(now.getTime())) return value;
+
+	const elapsed = now.getTime() - date.getTime();
+	if (elapsed >= -45 * SECOND_MS && elapsed < 45 * SECOND_MS) {
+		return "just now";
+	}
+
+	if (elapsed < 0) {
+		return formatCalendarTimestamp(date, now);
+	}
+
+	if (elapsed < HOUR_MS) {
+		const minutes = Math.max(1, Math.floor(elapsed / MINUTE_MS));
+		return `${minutes} min ago`;
+	}
+
+	if (elapsed < DAY_MS) {
+		const hours = Math.max(1, Math.floor(elapsed / HOUR_MS));
+		return `${hours} ${hours === 1 ? "hour" : "hours"} ago`;
+	}
+
+	return formatCalendarTimestamp(date, now);
 }
 
 export function getInitials(value: string) {

package/src/lib/queries.ts

@@ -1286,6 +1286,67 @@ function getTweetById(
 	);
 }
 
+export function getTweetsByIds(
+	tweetIds: string[],
+	accountId?: string,
+): EmbeddedTweet[] {
+	const db = getNativeDb();
+	const scopedAccountId =
+		accountId && accountId !== "all" ? accountId : undefined;
+	const urlExpansionCache: UrlExpansionCache = new Map();
+	const profileByHandleCache: ProfileByHandleCache = new Map();
+	const resolveProfileByHandle = (handle: string) =>
+		getProfileByHandle(db, profileByHandleCache, handle);
+	const seen = new Set<string>();
+	const tweets: EmbeddedTweet[] = [];
+
+	for (const tweetId of tweetIds) {
+		const normalized = tweetId.trim().replace(/^tweet_/, "");
+		if (!normalized || seen.has(normalized)) continue;
+		seen.add(normalized);
+		if (
+			scopedAccountId &&
+			!db
+				.prepare(
+					`
+					select 1
+					from tweets tweet
+					where tweet.id = ?
+						and (
+							tweet.account_id = ?
+							or exists (
+								select 1
+								from tweet_account_edges edge
+								where edge.account_id = ?
+									and edge.tweet_id = tweet.id
+							)
+							or exists (
+								select 1
+								from tweet_collections collection
+								where collection.account_id = ?
+									and collection.tweet_id = tweet.id
+							)
+						)
+					limit 1
+					`,
+				)
+				.get(normalized, scopedAccountId, scopedAccountId, scopedAccountId)
+		) {
+			continue;
+		}
+		const tweet = getTweetById(
+			db,
+			urlExpansionCache,
+			normalized,
+			resolveProfileByHandle,
+			scopedAccountId,
+		);
+		if (tweet) tweets.push(tweet);
+	}
+
+	return tweets;
+}
+
 function listTweetDescendants(
 	db: Database,
 	urlExpansionCache: UrlExpansionCache,

package/src/routes/api/period-digest.tsx

@@ -1,13 +1,13 @@
 import { createFileRoute } from "@tanstack/react-router";
 import { Effect } from "effect";
 import { maybeAutoUpdateBackupEffect } from "#/lib/backup";
-import { runEffectBackground } from "#/lib/effect-runtime";
 import {
 	jsonResponse,
 	parseBoundedInteger,
 	runRouteEffect,
 	sensitiveRequestErrorResponse,
 } from "#/lib/http-effect";
+import { createEffectNdjsonResponse } from "#/lib/ndjson-stream";
 import {
 	normalizeDigestLanguage,
 	streamPeriodDigestEffect,
@@ -15,8 +15,6 @@ import {
 	type PeriodDigestStreamEvent,
 } from "#/lib/period-digest";
 
-const encoder = new TextEncoder();
-
 function parseBoolean(value: string | null) {
 	return value === "true" || value === "1" || value === "yes";
 }
@@ -52,16 +50,12 @@ function parseOptions(url: URL): PeriodDigestOptions {
 	};
 }
 
-function encodeEvent(event: PeriodDigestStreamEvent) {
-	return encoder.encode(`${JSON.stringify(event)}\n`);
-}
-
 export const Route = createFileRoute("/api/period-digest")({
 	server: {
 		handlers: {
 			GET: ({ request }) =>
 				runRouteEffect(
-					Effect.gen(function* () {
+					Effect.sync(() => {
 						const denied = sensitiveRequestErrorResponse(request);
 						if (denied) return denied;
 
@@ -78,70 +72,28 @@ export const Route = createFileRoute("/api/period-digest")({
 								{ status: 400 },
 							);
 						}
-						yield* maybeAutoUpdateBackupEffect();
-						let abortDigest: (() => void) | undefined;
-
-						return new Response(
-							new ReadableStream({
-								cancel() {
-									abortDigest?.();
+						return createEffectNdjsonResponse<PeriodDigestStreamEvent>({
+							request,
+							initialEvents: [
+								{
+									type: "status",
+									label: "Preparing local archive",
+									detail: "Checking for backup updates.",
 								},
-								start(controller) {
-									const abortController = new AbortController();
-									let closed = false;
-									const close = () => {
-										closed = true;
-										abortController.abort();
-									};
-									const closeController = () => {
-										request.signal.removeEventListener("abort", onAbort);
-										if (!closed) {
-											closed = true;
-											controller.close();
-										}
-									};
-									const onAbort = () => close();
-									request.signal.addEventListener("abort", onAbort, {
-										once: true,
-									});
-									abortDigest = close;
-									const enqueue = (event: PeriodDigestStreamEvent) => {
-										if (closed) return;
-										try {
-											controller.enqueue(encodeEvent(event));
-										} catch {
-											close();
-										}
-									};
-
-									runEffectBackground(
-										streamPeriodDigestEffect(
-											{ ...options, signal: abortController.signal },
-											{ onEvent: enqueue },
-										),
-										{
-											onSuccess: closeController,
-											onFailure: (error) => {
-												enqueue({
-													type: "error",
-													error:
-														error instanceof Error
-															? error.message
-															: "Digest failed",
-												});
-												closeController();
-											},
-										},
+							],
+							run: ({ signal, emit }) =>
+								Effect.gen(function* () {
+									yield* maybeAutoUpdateBackupEffect();
+									return yield* streamPeriodDigestEffect(
+										{ ...options, signal },
+										{ onEvent: emit },
 									);
-								},
+								}),
+							errorEvent: (error) => ({
+								type: "error",
+								error: error instanceof Error ? error.message : "Digest failed",
 							}),
-							{
-								headers: {
-									"cache-control": "no-store",
-									"content-type": "application/x-ndjson; charset=utf-8",
-								},
-							},
-						);
+						});
 					}),
 				),
 		},