birdclaw 0.7.0 → 0.8.1

package/src/lib/ndjson-stream.ts

@@ -0,0 +1,106 @@
+import { Effect } from "effect";
+import { runEffectBackground } from "./effect-runtime";
+
+const encoder = new TextEncoder();
+const HEARTBEAT_INTERVAL_MS = 15_000;
+const STREAM_START_DELAY_MS = 25;
+const FLUSH_PADDING = `${" ".repeat(16_384)}\n`;
+
+export function createEffectNdjsonResponse<Event>({
+	request,
+	initialEvents = [],
+	run,
+	errorEvent,
+}: {
+	request: Request;
+	initialEvents?: Event[];
+	run: (context: {
+		signal: AbortSignal;
+		emit: (event: Event) => void;
+	}) => Effect.Effect<unknown, unknown>;
+	errorEvent: (error: unknown) => Event;
+}) {
+	let abortStream: (() => void) | undefined;
+
+	return new Response(
+		new ReadableStream<Uint8Array>({
+			cancel() {
+				abortStream?.();
+			},
+			start(controller) {
+				const abortController = new AbortController();
+				let closed = false;
+				let heartbeat: ReturnType<typeof setInterval> | undefined;
+				let startTimer: ReturnType<typeof setTimeout> | undefined;
+
+				const cleanup = () => {
+					request.signal.removeEventListener("abort", abort);
+					if (heartbeat !== undefined) clearInterval(heartbeat);
+					if (startTimer !== undefined) clearTimeout(startTimer);
+				};
+				const abort = () => {
+					if (closed) return;
+					closed = true;
+					cleanup();
+					abortController.abort();
+				};
+				const close = () => {
+					if (closed) return;
+					closed = true;
+					cleanup();
+					abortController.abort();
+					controller.close();
+				};
+				const enqueue = (value: string) => {
+					if (closed) return;
+					try {
+						controller.enqueue(encoder.encode(value));
+					} catch {
+						abort();
+					}
+				};
+				const emit = (event: Event) => {
+					enqueue(`${JSON.stringify(event)}\n`);
+				};
+
+				request.signal.addEventListener("abort", abort, { once: true });
+				abortStream = abort;
+				if (request.signal.aborted) {
+					abort();
+					controller.close();
+					return;
+				}
+				for (const event of initialEvents) emit(event);
+				enqueue(FLUSH_PADDING);
+				heartbeat = setInterval(
+					() => enqueue(FLUSH_PADDING),
+					HEARTBEAT_INTERVAL_MS,
+				);
+
+				startTimer = setTimeout(() => {
+					if (closed) return;
+					try {
+						runEffectBackground(run({ signal: abortController.signal, emit }), {
+							onSuccess: close,
+							onFailure: (error) => {
+								emit(errorEvent(error));
+								close();
+							},
+						});
+					} catch (error) {
+						emit(errorEvent(error));
+						close();
+					}
+				}, STREAM_START_DELAY_MS);
+			},
+		}),
+		{
+			headers: {
+				"cache-control": "no-store, no-transform",
+				"content-encoding": "identity",
+				"content-type": "application/x-ndjson; charset=utf-8",
+				"x-accel-buffering": "no",
+			},
+		},
+	);
+}

package/src/lib/period-digest.ts

@@ -6,10 +6,14 @@ import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { getLinkInsights } from "./link-insights";
 import { syncMentionThreadsEffect } from "./mention-threads-live";
 import { syncMentionsEffect } from "./mentions-live";
-import { listDmConversations, listTimelineItems } from "./queries";
+import {
+	getTweetsByIds,
+	listDmConversations,
+	listTimelineItems,
+} from "./queries";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
-import type { ProfileRecord, TweetEntities } from "./types";
+import type { EmbeddedTweet, ProfileRecord, TweetEntities } from "./types";
 
 export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
 export type PeriodDigestSourceKind =
@@ -28,6 +32,7 @@ export interface PeriodDigestOptions {
 	includeDms?: boolean;
 	refresh?: boolean;
 	model?: string;
+	language?: string;
 	reasoningEffort?: "minimal" | "low" | "medium" | "high";
 	serviceTier?: "default" | "flex" | "priority";
 	signal?: AbortSignal;
@@ -108,6 +113,32 @@ const PeriodDigestSchema = z.object({
 	sourceTweetIds: z.array(z.string()).default([]),
 });
 
+const MAX_DIGEST_LANGUAGE_LENGTH = 64;
+
+export function normalizeDigestLanguage(
+	value: string | undefined,
+): string | undefined {
+	const trimmed = value?.trim();
+	if (!trimmed) return undefined;
+	if (
+		trimmed.length > MAX_DIGEST_LANGUAGE_LENGTH ||
+		!/^[a-z0-9]+(?:-[a-z0-9]+)*$/i.test(trimmed)
+	) {
+		throw new Error(
+			"Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
+		);
+	}
+	try {
+		const [canonical] = Intl.getCanonicalLocales(trimmed);
+		if (!canonical) throw new Error("missing canonical locale");
+		return canonical;
+	} catch {
+		throw new Error(
+			"Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
+		);
+	}
+}
+
 export type PeriodDigest = z.infer<typeof PeriodDigestSchema>;
 
 interface CompactTweet {
@@ -195,6 +226,7 @@ const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
 const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
 const DEFAULT_LIVE_THREAD_LIMIT = 12;
 const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
+const DEFAULT_DIGEST_FRESHNESS_MS = 5 * 60_000;
 const MAX_PROMPT_DATA_CHARS = 1_200_000;
 const DELIMITER_PATTERN = /\n---\s*\n/;
 const VISIBLE_DELIMITER_HOLD = 8;
@@ -336,6 +368,26 @@ function compactTweet(
 	};
 }
 
+function compactEmbeddedTweet(item: EmbeddedTweet): CompactTweet {
+	return {
+		id: item.id,
+		url: tweetUrl(item.author.handle, item.id),
+		source: "home",
+		author: item.author.handle,
+		name: item.author.displayName,
+		authorProfile: item.author,
+		createdAt: item.createdAt,
+		text: item.text,
+		entities: item.entities,
+		likeCount: item.likeCount ?? 0,
+		liked: Boolean(item.liked),
+		bookmarked: Boolean(item.bookmarked),
+		needsReply: !item.isReplied,
+		replyToId: item.replyToId ?? null,
+		replyToTweet: null,
+	};
+}
+
 function dedupeTweets(tweets: CompactTweet[]) {
 	const seen = new Set<string>();
 	const items: CompactTweet[] = [];
@@ -563,6 +615,12 @@ export function collectPeriodDigestContext(
 	};
 }
 
+function languageFromOptions(options: PeriodDigestOptions) {
+	return normalizeDigestLanguage(
+		options.language ?? process.env.BIRDCLAW_DIGEST_LANGUAGE,
+	);
+}
+
 function modelFromOptions(options: PeriodDigestOptions) {
 	return options.model ?? process.env.BIRDCLAW_AI_MODEL ?? DEFAULT_MODEL;
 }
@@ -834,16 +892,127 @@ function digestCacheKey(
 	context: PeriodDigestContext,
 	options: PeriodDigestOptions,
 ) {
-	return [
+	const parts = [
 		"period-digest:v2",
 		modelFromOptions(options),
 		reasoningEffortFromOptions(options),
 		serviceTierFromOptions(options),
 		context.hash,
-	].join(":");
+	];
+	const lang = languageFromOptions(options);
+	if (lang) parts.push(`lang:${lang}`);
+	return parts.join(":");
+}
+
+function latestDigestCacheKey(options: PeriodDigestOptions) {
+	const period = normalizePeriod(options.period);
+	const window = resolvePeriodDigestWindow(options);
+	const identity = {
+		period,
+		day:
+			period === "today" || period === "yesterday"
+				? window.since
+				: localDateStart(new Date()).toISOString(),
+		since: options.since?.trim() || null,
+		until: options.until?.trim() || null,
+		account: options.account?.trim() || null,
+		includeDms: Boolean(options.includeDms),
+		maxTweets: Math.max(
+			20,
+			Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
+		),
+		maxLinks: Math.max(3, Math.trunc(options.maxLinks ?? DEFAULT_MAX_LINKS)),
+		model: modelFromOptions(options),
+		language: languageFromOptions(options) ?? null,
+		reasoningEffort: reasoningEffortFromOptions(options),
+		serviceTier: serviceTierFromOptions(options),
+	};
+	return `period-digest-latest:v1:${createHash("sha1")
+		.update(JSON.stringify(identity))
+		.digest("hex")}`;
+}
+
+function collectDigestTweetIds(digest: PeriodDigest) {
+	const tweetIds = new Set(digest.sourceTweetIds);
+	for (const topic of digest.keyTopics) {
+		for (const tweetId of topic.tweetIds) tweetIds.add(tweetId);
+	}
+	for (const link of digest.notableLinks) {
+		for (const tweetId of link.sourceTweetIds) tweetIds.add(tweetId);
+	}
+	for (const action of digest.actionItems) {
+		if (action.tweetId) tweetIds.add(action.tweetId);
+	}
+	return [...tweetIds];
+}
+
+function enrichContextWithCitedTweets(
+	context: PeriodDigestContext,
+	digest: PeriodDigest,
+) {
+	const existingIds = new Set(context.tweets.map((tweet) => tweet.id));
+	const missingIds = collectDigestTweetIds(digest).filter(
+		(tweetId) => !existingIds.has(tweetId.replace(/^tweet_/, "")),
+	);
+	if (missingIds.length === 0) return context;
+	const citedTweets = getTweetsByIds(missingIds, context.account).map(
+		compactEmbeddedTweet,
+	);
+	return citedTweets.length > 0
+		? { ...context, tweets: [...context.tweets, ...citedTweets] }
+		: context;
+}
+
+interface CachedPeriodDigestValue {
+	context?: PeriodDigestContext;
+	digest: PeriodDigest;
+	markdown: string;
+	model: string;
+	reasoningEffort: string;
+	serviceTier: string;
+	updatedAt?: string;
+}
+
+function cachedDigestResult(
+	cached: { value: CachedPeriodDigestValue; updatedAt: string },
+	context: PeriodDigestContext,
+): PeriodDigestRunResult {
+	const digest = PeriodDigestSchema.parse(cached.value.digest);
+	return {
+		context: enrichContextWithCitedTweets(context, digest),
+		digest,
+		markdown: cached.value.markdown,
+		model: cached.value.model,
+		reasoningEffort: cached.value.reasoningEffort,
+		serviceTier: cached.value.serviceTier,
+		cached: true,
+		updatedAt: cached.value.updatedAt ?? cached.updatedAt,
+	};
+}
+
+function isFreshDigestCache(updatedAt: string) {
+	const timestamp = Date.parse(updatedAt);
+	return (
+		Number.isFinite(timestamp) &&
+		Date.now() - timestamp <= DEFAULT_DIGEST_FRESHNESS_MS
+	);
 }
 
-function buildPrompt(context: PeriodDigestContext) {
+function emitCachedDigest(
+	result: PeriodDigestRunResult,
+	handlers: PeriodDigestStreamHandlers,
+) {
+	handlers.onEvent?.({ type: "start", context: result.context, cached: true });
+	handlers.onDelta?.(result.markdown);
+	handlers.onEvent?.({ type: "delta", delta: result.markdown });
+	handlers.onEvent?.({ type: "done", result });
+}
+
+function buildPrompt(
+	context: PeriodDigestContext,
+	options?: { language?: string },
+) {
+	const language = normalizeDigestLanguage(options?.language);
 	const promptTweets = context.tweets.map((tweet) => ({
 		id: tweet.id,
 		url: tweet.url,
@@ -928,7 +1097,7 @@ Requirements:
 - Stream one readable Markdown report first. The UI will show this text directly; do not rely on separate cards or structured summaries.
 - Target 700-1100 words when there is enough data.
 - Start with a 2-3 sentence lead that immediately says what people are talking about.
-- Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies.
+- Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies. Translate these section titles when a report language is requested.
 - When a tweet has replyToTweet, use that parent context to understand what the author was replying to and whether Peter already joined the conversation.
 - Use bullets under each section. Each bullet should be specific and explain why it matters.
 - For tweets: cite every claim with inline tweet ids at the end of the relevant sentence or bullet, e.g. (tweet_123, tweet_456). These citations become hoverable source links.
@@ -942,6 +1111,7 @@ Requirements:
 - Keep actionItems empty unless you wrote a "Worth replying to" section.
 - Put every tweet id cited in the Markdown into sourceTweetIds.
 - JSON shape: { "title": string, "summary": string, "keyTopics": [{ "title": string, "summary": string, "tweetIds": string[], "handles": string[] }], "notableLinks": [{ "title": string, "url": string, "why": string, "sourceTweetIds": string[] }], "people": [{ "handle": string, "name"?: string, "why": string }], "actionItems": [{ "kind": "reply"|"follow_up"|"read"|"sync", "label": string, "tweetId"?: string, "dmConversationId"?: string }], "sourceTweetIds": string[] }
+${language ? `- Write all human-readable prose, including section titles and JSON prose fields, in ${language}.\n- Preserve handles, URLs, tweet ids, and JSON property names exactly.` : ""}
 
 Dataset:
 ${JSON.stringify(dataset)}`;
@@ -950,12 +1120,22 @@ ${JSON.stringify(dataset)}`;
 function fallbackDigest(
 	context: PeriodDigestContext,
 	markdown: string,
+	language?: string,
 ): PeriodDigest {
-	const title = `${context.window.label} digest`;
+	const normalized = markdown.replaceAll(/\s+/g, " ").trim();
+	const heading = markdown
+		.split("\n")
+		.map((line) => line.match(/^#{1,6}\s+(.+)$/)?.[1]?.trim())
+		.find(Boolean);
+	const neutralFallback = language ? `[${language}]` : undefined;
 	return {
-		title,
+		title:
+			heading?.slice(0, 160) ??
+			neutralFallback ??
+			`${context.window.label} digest`,
 		summary:
-			markdown.replaceAll(/\s+/g, " ").trim().slice(0, 280) ||
+			normalized.slice(0, 280) ||
+			neutralFallback ||
 			"No model summary was returned.",
 		keyTopics: [],
 		notableLinks: [],
@@ -968,6 +1148,7 @@ function fallbackDigest(
 function parseDigestFromHybridText(
 	context: PeriodDigestContext,
 	rawText: string,
+	language?: string,
 ): { digest: PeriodDigest; markdown: string } {
 	const [markdownPart, jsonPart] = rawText.split(DELIMITER_PATTERN);
 	const markdown = (markdownPart ?? rawText).trim();
@@ -982,10 +1163,13 @@ function parseDigestFromHybridText(
 				digest: PeriodDigestSchema.parse(JSON.parse(candidate)),
 			};
 		} catch {
-			return { markdown, digest: fallbackDigest(context, markdown) };
+			return {
+				markdown,
+				digest: fallbackDigest(context, markdown, language),
+			};
 		}
 	}
-	return { markdown, digest: fallbackDigest(context, markdown) };
+	return { markdown, digest: fallbackDigest(context, markdown, language) };
 }
 
 function emitVisibleDelta(
@@ -1129,7 +1313,9 @@ function createOpenAIRequestBody(
 			},
 			{
 				role: "user",
-				content: buildPrompt(context),
+				content: buildPrompt(context, {
+					language: languageFromOptions(options),
+				}),
 			},
 		],
 	};
@@ -1172,7 +1358,14 @@ function readOpenAIStreamEffect(
 			}
 
 			const parsed = yield* tryDigestSync(() =>
-				parseDigestFromHybridText(context, state.rawText),
+				parseDigestFromHybridText(
+					context,
+					state.rawText,
+					languageFromOptions(options),
+				),
+			);
+			const enrichedContext = yield* tryDigestSync(() =>
+				enrichContextWithCitedTweets(context, parsed.digest),
 			);
 			const cacheKey = digestCacheKey(context, options);
 			const updatedAt = yield* tryDigestSync(() =>
@@ -1187,7 +1380,7 @@ function readOpenAIStreamEffect(
 				}),
 			);
 			const result: PeriodDigestRunResult = {
-				context,
+				context: enrichedContext,
 				digest: parsed.digest,
 				markdown: parsed.markdown,
 				model: modelFromOptions(options),
@@ -1196,6 +1389,17 @@ function readOpenAIStreamEffect(
 				cached: false,
 				updatedAt,
 			};
+			yield* tryDigestSync(() =>
+				writeSyncCache(latestDigestCacheKey(options), {
+					context: result.context,
+					digest: result.digest,
+					markdown: result.markdown,
+					model: result.model,
+					reasoningEffort: result.reasoningEffort,
+					serviceTier: result.serviceTier,
+					updatedAt: result.updatedAt,
+				}),
+			);
 			handlers.onEvent?.({ type: "done", result });
 			return result;
 		}
@@ -1213,47 +1417,68 @@ export function streamPeriodDigestEffect(
 	handlers: PeriodDigestStreamHandlers = {},
 ): Effect.Effect<PeriodDigestRunResult, Error> {
 	return Effect.gen(function* () {
+		const resolvedOptions = {
+			...options,
+			language: yield* tryDigestSync(() => languageFromOptions(options)),
+		};
+		const latestCached = resolvedOptions.refresh
+			? null
+			: !resolvedOptions.liveSync
+				? yield* tryDigestSync(() =>
+						readSyncCache<CachedPeriodDigestValue>(
+							latestDigestCacheKey(resolvedOptions),
+						),
+					)
+				: null;
+		const latestContext = latestCached?.value.context;
+		if (
+			latestCached &&
+			latestContext &&
+			isFreshDigestCache(latestCached.value.updatedAt ?? latestCached.updatedAt)
+		) {
+			const result = yield* tryDigestSync(() =>
+				cachedDigestResult(latestCached, latestContext),
+			);
+			emitCachedDigest(result, handlers);
+			return result;
+		}
+
 		yield* refreshPeriodDigestInputsEffect(
-			options,
+			resolvedOptions,
 			{ threads: false },
 			handlers,
 		).pipe(Effect.catchAll(() => Effect.void));
 		let context = yield* tryDigestSync(() =>
-			collectPeriodDigestContext(options),
+			collectPeriodDigestContext(resolvedOptions),
 		);
-		let cacheKey = digestCacheKey(context, options);
-		const cached = options.refresh
+		let cacheKey = digestCacheKey(context, resolvedOptions);
+		const cached = resolvedOptions.refresh
 			? null
 			: yield* tryDigestSync(() =>
-					readSyncCache<{
-						digest: PeriodDigest;
-						markdown: string;
-						model: string;
-						reasoningEffort: string;
-						serviceTier: string;
-					}>(cacheKey),
+					readSyncCache<CachedPeriodDigestValue>(cacheKey),
 				);
 
 		if (cached) {
-			const result: PeriodDigestRunResult = yield* tryDigestSync(() => ({
-				context,
-				digest: PeriodDigestSchema.parse(cached.value.digest),
-				markdown: cached.value.markdown,
-				model: cached.value.model,
-				reasoningEffort: cached.value.reasoningEffort,
-				serviceTier: cached.value.serviceTier,
-				cached: true,
-				updatedAt: cached.updatedAt,
-			}));
-			handlers.onEvent?.({ type: "start", context, cached: true });
-			handlers.onDelta?.(result.markdown);
-			handlers.onEvent?.({ type: "delta", delta: result.markdown });
-			handlers.onEvent?.({ type: "done", result });
+			const result = yield* tryDigestSync(() =>
+				cachedDigestResult(cached, context),
+			);
+			yield* tryDigestSync(() =>
+				writeSyncCache(latestDigestCacheKey(resolvedOptions), {
+					context: result.context,
+					digest: result.digest,
+					markdown: result.markdown,
+					model: result.model,
+					reasoningEffort: result.reasoningEffort,
+					serviceTier: result.serviceTier,
+					updatedAt: result.updatedAt,
+				}),
+			);
+			emitCachedDigest(result, handlers);
 			return result;
 		}
 
 		yield* refreshPeriodDigestInputsEffect(
-			options,
+			resolvedOptions,
 			{
 				timeline: false,
 				mentions: false,
@@ -1264,8 +1489,10 @@ export function streamPeriodDigestEffect(
 			},
 			handlers,
 		).pipe(Effect.catchAll(() => Effect.void));
-		context = yield* tryDigestSync(() => collectPeriodDigestContext(options));
-		cacheKey = digestCacheKey(context, options);
+		context = yield* tryDigestSync(() =>
+			collectPeriodDigestContext(resolvedOptions),
+		);
+		cacheKey = digestCacheKey(context, resolvedOptions);
 
 		const apiKey = process.env.OPENAI_API_KEY;
 		if (!apiKey) {
@@ -1277,12 +1504,12 @@ export function streamPeriodDigestEffect(
 		const response = yield* tryDigestPromise(() =>
 			fetch("https://api.openai.com/v1/responses", {
 				method: "POST",
-				signal: options.signal,
+				signal: resolvedOptions.signal,
 				headers: {
 					authorization: `Bearer ${apiKey}`,
 					"content-type": "application/json",
 				},
-				body: JSON.stringify(createOpenAIRequestBody(context, options)),
+				body: JSON.stringify(createOpenAIRequestBody(context, resolvedOptions)),
 			}),
 		);
 		if (!response.ok) {
@@ -1296,7 +1523,12 @@ export function streamPeriodDigestEffect(
 				),
 			);
 		}
-		return yield* readOpenAIStreamEffect(response, context, options, handlers);
+		return yield* readOpenAIStreamEffect(
+			response,
+			context,
+			resolvedOptions,
+			handlers,
+		);
 	});
 }
 
@@ -1310,6 +1542,9 @@ export function streamPeriodDigest(
 export const __test__ = {
 	PeriodDigestSchema,
 	buildPrompt,
+	digestCacheKey,
+	languageFromOptions,
+	normalizeDigestLanguage,
 	readOpenAIStreamEffect,
 	parseDigestFromHybridText,
 	processSseChunk,