birdclaw 0.8.0 → 0.8.2

package/src/lib/client-cache.ts

@@ -0,0 +1,109 @@
+const DEFAULT_MAX_ENTRIES = 100;
+
+interface ClientCacheEntry<T> {
+	value: T;
+	updatedAt: number;
+}
+
+interface LoadClientCacheOptions {
+	force?: boolean;
+	maxAgeMs?: number;
+}
+
+const values = new Map<string, ClientCacheEntry<unknown>>();
+const pending = new Map<string, Promise<unknown>>();
+const revisions = new Map<string, number>();
+let generation = 0;
+
+function cacheToken(key: string) {
+	return `${String(generation)}:${String(revisions.get(key) ?? 0)}`;
+}
+
+function invalidateKey(key: string) {
+	revisions.set(key, (revisions.get(key) ?? 0) + 1);
+}
+
+function pruneCache() {
+	while (values.size > DEFAULT_MAX_ENTRIES) {
+		const oldestKey = values.keys().next().value as string | undefined;
+		if (!oldestKey) return;
+		values.delete(oldestKey);
+	}
+}
+
+export function readClientCache<T>(
+	key: string,
+	maxAgeMs = Number.POSITIVE_INFINITY,
+) {
+	const entry = values.get(key) as ClientCacheEntry<T> | undefined;
+	if (!entry) return undefined;
+	if (Date.now() - entry.updatedAt > maxAgeMs) {
+		values.delete(key);
+		return undefined;
+	}
+	return entry.value;
+}
+
+export function writeClientCache<T>(key: string, value: T) {
+	values.delete(key);
+	values.set(key, { value, updatedAt: Date.now() });
+	pruneCache();
+	return value;
+}
+
+export function loadClientCache<T>(
+	key: string,
+	load: () => Promise<T>,
+	{
+		force = false,
+		maxAgeMs = Number.POSITIVE_INFINITY,
+	}: LoadClientCacheOptions = {},
+) {
+	if (force) {
+		invalidateKey(key);
+		values.delete(key);
+		pending.delete(key);
+	} else {
+		const cached = readClientCache<T>(key, maxAgeMs);
+		if (cached !== undefined) return Promise.resolve(cached);
+	}
+
+	const active = pending.get(key) as Promise<T> | undefined;
+	if (active) return active;
+
+	const token = cacheToken(key);
+	let request: Promise<T>;
+	request = load()
+		.then((value) => {
+			if (cacheToken(key) === token) writeClientCache(key, value);
+			return value;
+		})
+		.finally(() => {
+			if (pending.get(key) === request) pending.delete(key);
+		});
+	pending.set(key, request);
+	return request;
+}
+
+export function deleteClientCache(key: string) {
+	invalidateKey(key);
+	values.delete(key);
+	pending.delete(key);
+}
+
+export function deleteClientCacheByPrefix(prefix: string) {
+	const keys = new Set([...values.keys(), ...pending.keys()]);
+	for (const key of keys) {
+		if (!key.startsWith(prefix)) continue;
+		invalidateKey(key);
+		values.delete(key);
+		pending.delete(key);
+	}
+}
+
+export function clearClientCache() {
+	generation += 1;
+	values.clear();
+	pending.clear();
+	revisions.clear();
+}

package/src/lib/db.ts

@@ -1,4 +1,7 @@
-import NativeSqliteDatabase, { type Database } from "./sqlite";
+import NativeSqliteDatabase, {
+	type Database,
+	SQLITE_BUSY_TIMEOUT_MS,
+} from "./sqlite";
 import { Kysely, SqliteDialect } from "kysely";
 import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
 import { seedDemoData } from "./seed";
@@ -326,7 +329,7 @@ export interface InitDatabaseOptions {
 
 const BASE_SCHEMA_SQL = `
   pragma journal_mode = wal;
-  pragma busy_timeout = 5000;
+  pragma busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS};
   pragma foreign_keys = on;
 
   create table if not exists accounts (
@@ -948,6 +951,34 @@ function ensureFollowGraphTables(db: Database) {
 }
 
 function backfillTweetCollections(db: Database) {
+	const missingKinds = (
+		[
+			["likes", "liked"],
+			["bookmarks", "bookmarked"],
+		] as const
+	).filter(([, column]) =>
+		db
+			.prepare(
+				`
+        select 1
+        from tweets as tweet
+        where tweet.${column} = 1
+          and not exists (
+            select 1
+            from tweet_collections as collection
+            where collection.account_id = tweet.account_id
+              and collection.tweet_id = tweet.id
+              and collection.kind = ?
+          )
+        limit 1
+      `,
+			)
+			.get(column === "liked" ? "likes" : "bookmarks"),
+	);
+	if (missingKinds.length === 0) {
+		return;
+	}
+
 	const now = new Date().toISOString();
 	const insert = db.prepare(`
     insert or ignore into tweet_collections (
@@ -963,12 +994,34 @@ function backfillTweetCollections(db: Database) {
   `);
 
 	db.transaction(() => {
-		insert.run("likes", now, "likes");
-		insert.run("bookmarks", now, "bookmarks");
+		for (const [kind] of missingKinds) {
+			insert.run(kind, now, kind);
+		}
 	})();
 }
 
 function backfillTweetAccountEdges(db: Database) {
+	const missing = db
+		.prepare(
+			`
+      select 1
+      from tweets as tweet
+      where tweet.kind in ('home', 'mention')
+        and not exists (
+          select 1
+          from tweet_account_edges as edge
+          where edge.account_id = tweet.account_id
+            and edge.tweet_id = tweet.id
+            and edge.kind = tweet.kind
+        )
+      limit 1
+    `,
+		)
+		.get();
+	if (!missing) {
+		return;
+	}
+
 	const now = new Date().toISOString();
 	db.prepare(`
     insert or ignore into tweet_account_edges (

package/src/lib/mention-threads-live.ts

@@ -4,6 +4,7 @@ import { listThreadViaBirdEffect } from "./bird";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
 import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	XurlMentionData,
 	XurlMentionsResponse,
@@ -415,7 +416,7 @@ function mergeMentionThreadIntoLocalStore({
 				replyToId ?? null,
 				Number(tweet.public_metrics?.like_count ?? 0),
 				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
 				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
 			);
 			if (writeThreadContextEdges) {

package/src/lib/mentions-live.ts

@@ -11,6 +11,7 @@ import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
 import { serializeMentionItemsAsXurlCompatible } from "./mentions-export";
 import { listTimelineItems } from "./queries";
 import { deleteSyncCache, readSyncCache, writeSyncCache } from "./sync-cache";
+import { tweetEntitiesFromXurl } from "./tweet-render";
 import type {
 	ReplyFilter,
 	TweetEntities,
@@ -521,52 +522,7 @@ function findNewestArchiveMentionId(db: Database, accountId: string) {
 }
 
 function toLocalEntities(tweet: XurlMentionData): TweetEntities {
-	const raw = tweet.entities;
-	if (!raw || typeof raw !== "object") {
-		return {};
-	}
-
-	const entities = raw as Record<string, unknown>;
-	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
-	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
-
-	return {
-		...(rawMentions.length
-			? {
-					mentions: rawMentions.map((mention) => {
-						const value =
-							mention && typeof mention === "object"
-								? (mention as Record<string, unknown>)
-								: {};
-						return {
-							username: String(value.username ?? ""),
-							id: typeof value.id === "string" ? String(value.id) : undefined,
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-		...(rawUrls.length
-			? {
-					urls: rawUrls.map((url) => {
-						const value =
-							url && typeof url === "object"
-								? (url as Record<string, unknown>)
-								: {};
-						return {
-							url: String(value.url ?? ""),
-							expandedUrl: String(value.expanded_url ?? value.url ?? ""),
-							displayUrl: String(
-								value.display_url ?? value.expanded_url ?? value.url ?? "",
-							),
-							start: Number(value.start ?? 0),
-							end: Number(value.end ?? 0),
-						};
-					}),
-				}
-			: {}),
-	};
+	return tweetEntitiesFromXurl(tweet.entities);
 }
 
 function replaceTweetFts(db: Database, tweetId: string, text: string) {

package/src/lib/ndjson-stream.ts

@@ -0,0 +1,106 @@
+import { Effect } from "effect";
+import { runEffectBackground } from "./effect-runtime";
+
+const encoder = new TextEncoder();
+const HEARTBEAT_INTERVAL_MS = 15_000;
+const STREAM_START_DELAY_MS = 25;
+const FLUSH_PADDING = `${" ".repeat(16_384)}\n`;
+
+export function createEffectNdjsonResponse<Event>({
+	request,
+	initialEvents = [],
+	run,
+	errorEvent,
+}: {
+	request: Request;
+	initialEvents?: Event[];
+	run: (context: {
+		signal: AbortSignal;
+		emit: (event: Event) => void;
+	}) => Effect.Effect<unknown, unknown>;
+	errorEvent: (error: unknown) => Event;
+}) {
+	let abortStream: (() => void) | undefined;
+
+	return new Response(
+		new ReadableStream<Uint8Array>({
+			cancel() {
+				abortStream?.();
+			},
+			start(controller) {
+				const abortController = new AbortController();
+				let closed = false;
+				let heartbeat: ReturnType<typeof setInterval> | undefined;
+				let startTimer: ReturnType<typeof setTimeout> | undefined;
+
+				const cleanup = () => {
+					request.signal.removeEventListener("abort", abort);
+					if (heartbeat !== undefined) clearInterval(heartbeat);
+					if (startTimer !== undefined) clearTimeout(startTimer);
+				};
+				const abort = () => {
+					if (closed) return;
+					closed = true;
+					cleanup();
+					abortController.abort();
+				};
+				const close = () => {
+					if (closed) return;
+					closed = true;
+					cleanup();
+					abortController.abort();
+					controller.close();
+				};
+				const enqueue = (value: string) => {
+					if (closed) return;
+					try {
+						controller.enqueue(encoder.encode(value));
+					} catch {
+						abort();
+					}
+				};
+				const emit = (event: Event) => {
+					enqueue(`${JSON.stringify(event)}\n`);
+				};
+
+				request.signal.addEventListener("abort", abort, { once: true });
+				abortStream = abort;
+				if (request.signal.aborted) {
+					abort();
+					controller.close();
+					return;
+				}
+				for (const event of initialEvents) emit(event);
+				enqueue(FLUSH_PADDING);
+				heartbeat = setInterval(
+					() => enqueue(FLUSH_PADDING),
+					HEARTBEAT_INTERVAL_MS,
+				);
+
+				startTimer = setTimeout(() => {
+					if (closed) return;
+					try {
+						runEffectBackground(run({ signal: abortController.signal, emit }), {
+							onSuccess: close,
+							onFailure: (error) => {
+								emit(errorEvent(error));
+								close();
+							},
+						});
+					} catch (error) {
+						emit(errorEvent(error));
+						close();
+					}
+				}, STREAM_START_DELAY_MS);
+			},
+		}),
+		{
+			headers: {
+				"cache-control": "no-store, no-transform",
+				"content-encoding": "identity",
+				"content-type": "application/x-ndjson; charset=utf-8",
+				"x-accel-buffering": "no",
+			},
+		},
+	);
+}

package/src/lib/period-digest.ts

@@ -6,10 +6,14 @@ import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { getLinkInsights } from "./link-insights";
 import { syncMentionThreadsEffect } from "./mention-threads-live";
 import { syncMentionsEffect } from "./mentions-live";
-import { listDmConversations, listTimelineItems } from "./queries";
+import {
+	getTweetsByIds,
+	listDmConversations,
+	listTimelineItems,
+} from "./queries";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
-import type { ProfileRecord, TweetEntities } from "./types";
+import type { EmbeddedTweet, ProfileRecord, TweetEntities } from "./types";
 
 export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
 export type PeriodDigestSourceKind =
@@ -222,6 +226,7 @@ const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
 const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
 const DEFAULT_LIVE_THREAD_LIMIT = 12;
 const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
+const DEFAULT_DIGEST_FRESHNESS_MS = 5 * 60_000;
 const MAX_PROMPT_DATA_CHARS = 1_200_000;
 const DELIMITER_PATTERN = /\n---\s*\n/;
 const VISIBLE_DELIMITER_HOLD = 8;
@@ -363,6 +368,26 @@ function compactTweet(
 	};
 }
 
+function compactEmbeddedTweet(item: EmbeddedTweet): CompactTweet {
+	return {
+		id: item.id,
+		url: tweetUrl(item.author.handle, item.id),
+		source: "home",
+		author: item.author.handle,
+		name: item.author.displayName,
+		authorProfile: item.author,
+		createdAt: item.createdAt,
+		text: item.text,
+		entities: item.entities,
+		likeCount: item.likeCount ?? 0,
+		liked: Boolean(item.liked),
+		bookmarked: Boolean(item.bookmarked),
+		needsReply: !item.isReplied,
+		replyToId: item.replyToId ?? null,
+		replyToTweet: null,
+	};
+}
+
 function dedupeTweets(tweets: CompactTweet[]) {
 	const seen = new Set<string>();
 	const items: CompactTweet[] = [];
@@ -879,6 +904,110 @@ function digestCacheKey(
 	return parts.join(":");
 }
 
+function latestDigestCacheKey(options: PeriodDigestOptions) {
+	const period = normalizePeriod(options.period);
+	const window = resolvePeriodDigestWindow(options);
+	const identity = {
+		period,
+		day:
+			period === "today" || period === "yesterday"
+				? window.since
+				: localDateStart(new Date()).toISOString(),
+		since: options.since?.trim() || null,
+		until: options.until?.trim() || null,
+		account: options.account?.trim() || null,
+		includeDms: Boolean(options.includeDms),
+		maxTweets: Math.max(
+			20,
+			Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
+		),
+		maxLinks: Math.max(3, Math.trunc(options.maxLinks ?? DEFAULT_MAX_LINKS)),
+		model: modelFromOptions(options),
+		language: languageFromOptions(options) ?? null,
+		reasoningEffort: reasoningEffortFromOptions(options),
+		serviceTier: serviceTierFromOptions(options),
+	};
+	return `period-digest-latest:v1:${createHash("sha1")
+		.update(JSON.stringify(identity))
+		.digest("hex")}`;
+}
+
+function collectDigestTweetIds(digest: PeriodDigest) {
+	const tweetIds = new Set(digest.sourceTweetIds);
+	for (const topic of digest.keyTopics) {
+		for (const tweetId of topic.tweetIds) tweetIds.add(tweetId);
+	}
+	for (const link of digest.notableLinks) {
+		for (const tweetId of link.sourceTweetIds) tweetIds.add(tweetId);
+	}
+	for (const action of digest.actionItems) {
+		if (action.tweetId) tweetIds.add(action.tweetId);
+	}
+	return [...tweetIds];
+}
+
+function enrichContextWithCitedTweets(
+	context: PeriodDigestContext,
+	digest: PeriodDigest,
+) {
+	const existingIds = new Set(context.tweets.map((tweet) => tweet.id));
+	const missingIds = collectDigestTweetIds(digest).filter(
+		(tweetId) => !existingIds.has(tweetId.replace(/^tweet_/, "")),
+	);
+	if (missingIds.length === 0) return context;
+	const citedTweets = getTweetsByIds(missingIds, context.account).map(
+		compactEmbeddedTweet,
+	);
+	return citedTweets.length > 0
+		? { ...context, tweets: [...context.tweets, ...citedTweets] }
+		: context;
+}
+
+interface CachedPeriodDigestValue {
+	context?: PeriodDigestContext;
+	digest: PeriodDigest;
+	markdown: string;
+	model: string;
+	reasoningEffort: string;
+	serviceTier: string;
+	updatedAt?: string;
+}
+
+function cachedDigestResult(
+	cached: { value: CachedPeriodDigestValue; updatedAt: string },
+	context: PeriodDigestContext,
+): PeriodDigestRunResult {
+	const digest = PeriodDigestSchema.parse(cached.value.digest);
+	return {
+		context: enrichContextWithCitedTweets(context, digest),
+		digest,
+		markdown: cached.value.markdown,
+		model: cached.value.model,
+		reasoningEffort: cached.value.reasoningEffort,
+		serviceTier: cached.value.serviceTier,
+		cached: true,
+		updatedAt: cached.value.updatedAt ?? cached.updatedAt,
+	};
+}
+
+function isFreshDigestCache(updatedAt: string) {
+	const timestamp = Date.parse(updatedAt);
+	return (
+		Number.isFinite(timestamp) &&
+		Date.now() - timestamp <= DEFAULT_DIGEST_FRESHNESS_MS
+	);
+}
+
+function emitCachedDigest(
+	result: PeriodDigestRunResult,
+	handlers: PeriodDigestStreamHandlers,
+) {
+	handlers.onEvent?.({ type: "start", context: result.context, cached: true });
+	handlers.onDelta?.(result.markdown);
+	handlers.onEvent?.({ type: "delta", delta: result.markdown });
+	handlers.onEvent?.({ type: "done", result });
+}
+
 function buildPrompt(
 	context: PeriodDigestContext,
 	options?: { language?: string },
@@ -1235,6 +1364,9 @@ function readOpenAIStreamEffect(
 					languageFromOptions(options),
 				),
 			);
+			const enrichedContext = yield* tryDigestSync(() =>
+				enrichContextWithCitedTweets(context, parsed.digest),
+			);
 			const cacheKey = digestCacheKey(context, options);
 			const updatedAt = yield* tryDigestSync(() =>
 				writeSyncCache(cacheKey, {
@@ -1248,7 +1380,7 @@ function readOpenAIStreamEffect(
 				}),
 			);
 			const result: PeriodDigestRunResult = {
-				context,
+				context: enrichedContext,
 				digest: parsed.digest,
 				markdown: parsed.markdown,
 				model: modelFromOptions(options),
@@ -1257,6 +1389,17 @@ function readOpenAIStreamEffect(
 				cached: false,
 				updatedAt,
 			};
+			yield* tryDigestSync(() =>
+				writeSyncCache(latestDigestCacheKey(options), {
+					context: result.context,
+					digest: result.digest,
+					markdown: result.markdown,
+					model: result.model,
+					reasoningEffort: result.reasoningEffort,
+					serviceTier: result.serviceTier,
+					updatedAt: result.updatedAt,
+				}),
+			);
 			handlers.onEvent?.({ type: "done", result });
 			return result;
 		}
@@ -1278,6 +1421,28 @@ export function streamPeriodDigestEffect(
 			...options,
 			language: yield* tryDigestSync(() => languageFromOptions(options)),
 		};
+		const latestCached = resolvedOptions.refresh
+			? null
+			: !resolvedOptions.liveSync
+				? yield* tryDigestSync(() =>
+						readSyncCache<CachedPeriodDigestValue>(
+							latestDigestCacheKey(resolvedOptions),
+						),
+					)
+				: null;
+		const latestContext = latestCached?.value.context;
+		if (
+			latestCached &&
+			latestContext &&
+			isFreshDigestCache(latestCached.value.updatedAt ?? latestCached.updatedAt)
+		) {
+			const result = yield* tryDigestSync(() =>
+				cachedDigestResult(latestCached, latestContext),
+			);
+			emitCachedDigest(result, handlers);
+			return result;
+		}
+
 		yield* refreshPeriodDigestInputsEffect(
 			resolvedOptions,
 			{ threads: false },
@@ -1290,30 +1455,25 @@ export function streamPeriodDigestEffect(
 		const cached = resolvedOptions.refresh
 			? null
 			: yield* tryDigestSync(() =>
-					readSyncCache<{
-						digest: PeriodDigest;
-						markdown: string;
-						model: string;
-						reasoningEffort: string;
-						serviceTier: string;
-					}>(cacheKey),
+					readSyncCache<CachedPeriodDigestValue>(cacheKey),
 				);
 
 		if (cached) {
-			const result: PeriodDigestRunResult = yield* tryDigestSync(() => ({
-				context,
-				digest: PeriodDigestSchema.parse(cached.value.digest),
-				markdown: cached.value.markdown,
-				model: cached.value.model,
-				reasoningEffort: cached.value.reasoningEffort,
-				serviceTier: cached.value.serviceTier,
-				cached: true,
-				updatedAt: cached.updatedAt,
-			}));
-			handlers.onEvent?.({ type: "start", context, cached: true });
-			handlers.onDelta?.(result.markdown);
-			handlers.onEvent?.({ type: "delta", delta: result.markdown });
-			handlers.onEvent?.({ type: "done", result });
+			const result = yield* tryDigestSync(() =>
+				cachedDigestResult(cached, context),
+			);
+			yield* tryDigestSync(() =>
+				writeSyncCache(latestDigestCacheKey(resolvedOptions), {
+					context: result.context,
+					digest: result.digest,
+					markdown: result.markdown,
+					model: result.model,
+					reasoningEffort: result.reasoningEffort,
+					serviceTier: result.serviceTier,
+					updatedAt: result.updatedAt,
+				}),
+			);
+			emitCachedDigest(result, handlers);
 			return result;
 		}