birdclaw 0.6.0 → 0.7.0

package/src/lib/period-digest.ts

@@ -1,11 +1,15 @@
 import { createHash } from "node:crypto";
 import { Effect } from "effect";
 import { z } from "zod";
+import { maybeAutoSyncBackupEffect } from "./backup";
 import { runEffectPromise, tryPromise } from "./effect-runtime";
 import { getLinkInsights } from "./link-insights";
+import { syncMentionThreadsEffect } from "./mention-threads-live";
+import { syncMentionsEffect } from "./mentions-live";
 import { listDmConversations, listTimelineItems } from "./queries";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
-import type { ProfileRecord } from "./types";
+import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
+import type { ProfileRecord, TweetEntities } from "./types";
 
 export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
 export type PeriodDigestSourceKind =
@@ -29,6 +33,13 @@ export interface PeriodDigestOptions {
 	signal?: AbortSignal;
 	maxTweets?: number;
 	maxLinks?: number;
+	liveSync?: boolean;
+	liveSyncMode?: HomeTimelineMode;
+	liveTimelineLimit?: number;
+	liveTimelineMaxPages?: number;
+	liveMentionsLimit?: number;
+	liveMentionsMaxPages?: number;
+	liveThreadLimit?: number;
 }
 
 export interface PeriodDigestWindow {
@@ -54,6 +65,7 @@ export interface PeriodDigestStreamHandlers {
 }
 
 export type PeriodDigestStreamEvent =
+	| { type: "status"; label: string; detail?: string }
 	| { type: "start"; context: PeriodDigestContext; cached: boolean }
 	| { type: "delta"; delta: string }
 	| { type: "done"; result: PeriodDigestRunResult }
@@ -107,10 +119,20 @@ interface CompactTweet {
 	authorProfile: ProfileRecord;
 	createdAt: string;
 	text: string;
+	entities?: TweetEntities;
 	likeCount: number;
 	liked: boolean;
 	bookmarked: boolean;
 	needsReply: boolean;
+	replyToId?: string | null;
+	replyToTweet?: {
+		id: string;
+		url: string;
+		author: string;
+		name: string;
+		createdAt: string;
+		text: string;
+	} | null;
 }
 
 interface CompactDm {
@@ -166,8 +188,14 @@ interface OpenAIStreamState {
 const DEFAULT_MODEL = "gpt-5.5";
 const DEFAULT_REASONING_EFFORT = "medium";
 const DEFAULT_SERVICE_TIER = "priority";
-const DEFAULT_MAX_TWEETS = 300;
+const DEFAULT_MAX_TWEETS = 2_500;
 const DEFAULT_MAX_LINKS = 12;
+const DEFAULT_LIVE_TIMELINE_MAX_PAGES = undefined;
+const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
+const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
+const DEFAULT_LIVE_THREAD_LIMIT = 12;
+const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
+const MAX_PROMPT_DATA_CHARS = 1_200_000;
 const DELIMITER_PATTERN = /\n---\s*\n/;
 const VISIBLE_DELIMITER_HOLD = 8;
 
@@ -204,6 +232,12 @@ function parseDate(value: string | undefined) {
 	return Number.isNaN(parsed.getTime()) ? null : parsed;
 }
 
+function floorIsoToHour(value: string) {
+	const date = new Date(value);
+	date.setUTCMinutes(0, 0, 0);
+	return date.toISOString();
+}
+
 function normalizePeriod(value: string | undefined): PeriodDigestPreset {
 	const normalized = value?.trim().toLowerCase();
 	if (normalized === "yesterday") return "yesterday";
@@ -273,6 +307,16 @@ function compactTweet(
 	source: PeriodDigestSourceKind,
 	item: ReturnType<typeof listTimelineItems>[number],
 ): CompactTweet {
+	const replyToTweet = item.replyToTweet
+		? {
+				id: item.replyToTweet.id,
+				url: tweetUrl(item.replyToTweet.author.handle, item.replyToTweet.id),
+				author: item.replyToTweet.author.handle,
+				name: item.replyToTweet.author.displayName,
+				createdAt: item.replyToTweet.createdAt,
+				text: item.replyToTweet.text,
+			}
+		: null;
 	return {
 		id: item.id,
 		url: tweetUrl(item.author.handle, item.id),
@@ -282,10 +326,13 @@ function compactTweet(
 		authorProfile: item.author,
 		createdAt: item.createdAt,
 		text: item.text,
+		entities: item.entities,
 		likeCount: item.likeCount,
 		liked: item.liked,
 		bookmarked: item.bookmarked,
 		needsReply: !item.isReplied,
+		replyToId: item.replyToId ?? null,
+		replyToTweet,
 	};
 }
 
@@ -417,6 +464,9 @@ function contextHash(context: Omit<PeriodDigestContext, "hash">) {
 					tweet.liked,
 					tweet.bookmarked,
 					tweet.needsReply,
+					tweet.replyToId,
+					tweet.replyToTweet?.id,
+					tweet.replyToTweet?.text,
 				]),
 				dms: context.dms.map((dm) => [
 					dm.id,
@@ -537,6 +587,249 @@ function serviceTierFromOptions(options: PeriodDigestOptions) {
 	);
 }
 
+function boundedPositiveInteger(
+	value: number | undefined,
+	fallback: number,
+	max: number,
+) {
+	if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
+		return fallback;
+	}
+	return Math.min(max, Math.floor(value));
+}
+
+function emitDigestStatus(
+	handlers: PeriodDigestStreamHandlers,
+	label: string,
+	detail?: string,
+) {
+	handlers.onEvent?.({
+		type: "status",
+		label,
+		...(detail ? { detail } : {}),
+	});
+}
+
+function formatFetchedStatus({
+	fetched,
+	total,
+	noun,
+}: {
+	fetched: number;
+	total: number;
+	noun: string;
+}) {
+	const count = `${String(Math.min(fetched, total))}/${String(total)}`;
+	return `Fetched ${count} ${noun}`;
+}
+
+function formatPageDetail({
+	source,
+	page,
+	maxPages,
+	done,
+}: {
+	source: string;
+	page?: number;
+	maxPages?: number;
+	done: boolean;
+}) {
+	const pageText =
+		page === undefined
+			? undefined
+			: `page ${String(page)}${maxPages === undefined ? "" : `/${String(maxPages)}`}`;
+	return [source, pageText, done ? "done" : undefined]
+		.filter(Boolean)
+		.join(" · ");
+}
+
+function refreshPeriodDigestInputsEffect(
+	options: PeriodDigestOptions,
+	phase: {
+		timeline?: boolean;
+		mentions?: boolean;
+		threads?: boolean;
+		threadTweetIds?: string[];
+	} = {},
+	handlers: PeriodDigestStreamHandlers = {},
+): Effect.Effect<void, unknown> {
+	if (!options.liveSync) {
+		return Effect.void;
+	}
+	const includeTimeline = phase.timeline ?? true;
+	const includeMentions = phase.mentions ?? true;
+	const includeThreads = phase.threads ?? true;
+	const window = resolvePeriodDigestWindow(options);
+	const liveStartTime = floorIsoToHour(window.since);
+	const mode = options.liveSyncMode ?? "xurl";
+	const contextTweetBudget = Math.max(
+		20,
+		Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
+	);
+	const timelineLimit =
+		options.liveTimelineLimit === undefined
+			? undefined
+			: boundedPositiveInteger(options.liveTimelineLimit, 300, 100_000);
+	const mentionsLimit = boundedPositiveInteger(
+		options.liveMentionsLimit,
+		DEFAULT_LIVE_MENTIONS_LIMIT,
+		100,
+	);
+	const threadLimit = boundedPositiveInteger(
+		options.liveThreadLimit,
+		DEFAULT_LIVE_THREAD_LIMIT,
+		100,
+	);
+	const timelineMaxPages =
+		options.liveTimelineMaxPages === undefined
+			? DEFAULT_LIVE_TIMELINE_MAX_PAGES
+			: boundedPositiveInteger(options.liveTimelineMaxPages, 3, 1_000);
+	const mentionsMaxPages =
+		options.liveMentionsMaxPages === undefined
+			? DEFAULT_LIVE_MENTIONS_MAX_PAGES
+			: boundedPositiveInteger(options.liveMentionsMaxPages, 3, 1_000);
+
+	return Effect.gen(function* () {
+		if (includeTimeline) {
+			yield* Effect.sync(() =>
+				emitDigestStatus(
+					handlers,
+					"Fetching home timeline from X",
+					"Walking the selected time window with xurl.",
+				),
+			);
+			const result = yield* syncHomeTimelineEffect({
+				account: options.account,
+				mode,
+				limit: timelineLimit,
+				maxPages: timelineMaxPages,
+				startTime: liveStartTime,
+				following: true,
+				refresh: Boolean(options.refresh),
+				cacheTtlMs: 2 * 60_000,
+				timeoutMs: 30_000,
+				onProgress: (progress) =>
+					emitDigestStatus(
+						handlers,
+						formatFetchedStatus({
+							fetched: progress.fetched,
+							total: progress.total ?? contextTweetBudget,
+							noun: "home tweets",
+						}),
+						formatPageDetail({
+							source: progress.source,
+							page: progress.page,
+							maxPages: progress.maxPages,
+							done: progress.done,
+						}),
+					),
+			}).pipe(
+				Effect.match({
+					onFailure: () => null,
+					onSuccess: (value) => value,
+				}),
+			);
+			yield* Effect.sync(() =>
+				emitDigestStatus(
+					handlers,
+					result
+						? `Fetched ${String(result.count)} home tweets from ${result.source}`
+						: "Home timeline fetch failed; using local data",
+				),
+			);
+		}
+		if (includeMentions) {
+			yield* Effect.sync(() =>
+				emitDigestStatus(
+					handlers,
+					"Fetching mentions from X",
+					"Reading replies and mentions for the selected window.",
+				),
+			);
+			const result = yield* syncMentionsEffect({
+				account: options.account,
+				mode: "xurl",
+				limit: mentionsLimit,
+				maxPages: mentionsMaxPages,
+				startTime: liveStartTime,
+				refresh: Boolean(options.refresh),
+				cacheTtlMs: 2 * 60_000,
+				onProgress: (progress) =>
+					emitDigestStatus(
+						handlers,
+						formatFetchedStatus({
+							fetched: progress.fetched,
+							total: progress.total ?? contextTweetBudget,
+							noun: "mentions",
+						}),
+						formatPageDetail({
+							source: progress.source,
+							page: progress.page,
+							maxPages: progress.maxPages,
+							done: progress.done,
+						}),
+					),
+			}).pipe(
+				Effect.match({
+					onFailure: () => null,
+					onSuccess: (value) => value,
+				}),
+			);
+			yield* Effect.sync(() =>
+				emitDigestStatus(
+					handlers,
+					result
+						? `Fetched ${String(result.count)} mentions from ${result.source}`
+						: "Mention fetch failed; using local data",
+				),
+			);
+		}
+		if (includeThreads) {
+			yield* Effect.sync(() =>
+				emitDigestStatus(
+					handlers,
+					"Fetching mention conversations",
+					"Pulling parent tweets so the AI sees what replies refer to.",
+				),
+			);
+			const result = yield* syncMentionThreadsEffect({
+				account: options.account,
+				mode: "xurl",
+				limit: threadLimit,
+				tweetIds: phase.threadTweetIds,
+				delayMs: 100,
+				timeoutMs: DEFAULT_LIVE_THREAD_TIMEOUT_MS,
+				maxPages: 2,
+				onProgress: (progress) =>
+					emitDigestStatus(
+						handlers,
+						`Fetched conversations for ${String(progress.processed)}/${String(progress.total)} mentions`,
+						`${String(progress.fetched)} tweets · ${progress.source}${
+							progress.done ? " · done" : ""
+						}`,
+					),
+			}).pipe(
+				Effect.match({
+					onFailure: () => null,
+					onSuccess: (value) => value,
+				}),
+			);
+			yield* Effect.sync(() =>
+				emitDigestStatus(
+					handlers,
+					result
+						? `Fetched ${String(result.uniqueTweets)} conversation tweets`
+						: "Conversation fetch failed; using available context",
+				),
+			);
+		}
+		yield* Effect.sync(() =>
+			emitDigestStatus(handlers, "Preparing local AI context"),
+		);
+		yield* maybeAutoSyncBackupEffect().pipe(Effect.catchAll(() => Effect.void));
+	}).pipe(Effect.asVoid);
+}
+
 function digestCacheKey(
 	context: PeriodDigestContext,
 	options: PeriodDigestOptions,
@@ -565,12 +858,69 @@ function buildPrompt(context: PeriodDigestContext) {
 		liked: tweet.liked,
 		bookmarked: tweet.bookmarked,
 		needsReply: tweet.needsReply,
+		replyToId: tweet.replyToId,
+		replyToTweet: tweet.replyToTweet,
 	}));
+	const fitDataset = () => {
+		let tweetCount = promptTweets.length;
+		let dmCount = context.dms.length;
+		let linkCount = context.links.length;
+		const datasetFor = (tweets: number, dms: number, links: number) => ({
+			tweets: promptTweets.slice(0, tweets),
+			dms: context.dms.slice(0, dms),
+			links: context.links.slice(0, links),
+		});
+		const lengthFor = (tweets: number, dms: number, links: number) =>
+			JSON.stringify(datasetFor(tweets, dms, links)).length;
+		const fitCount = (max: number, fits: (count: number) => boolean) => {
+			let low = 0;
+			let high = max;
+			let best = 0;
+			while (low <= high) {
+				const mid = Math.floor((low + high) / 2);
+				if (fits(mid)) {
+					best = mid;
+					low = mid + 1;
+				} else {
+					high = mid - 1;
+				}
+			}
+			return best;
+		};
+		if (lengthFor(tweetCount, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS) {
+			return {
+				dataset: datasetFor(tweetCount, dmCount, linkCount),
+				tweetCount,
+			};
+		}
+		dmCount = fitCount(
+			dmCount,
+			(count) =>
+				lengthFor(tweetCount, count, linkCount) <= MAX_PROMPT_DATA_CHARS,
+		);
+		if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
+			linkCount = fitCount(
+				linkCount,
+				(count) =>
+					lengthFor(tweetCount, dmCount, count) <= MAX_PROMPT_DATA_CHARS,
+			);
+		}
+		if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
+			tweetCount = fitCount(
+				tweetCount,
+				(count) =>
+					lengthFor(count, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS,
+			);
+		}
+		return { dataset: datasetFor(tweetCount, dmCount, linkCount), tweetCount };
+	};
+	const { dataset, tweetCount } = fitDataset();
 
 	return `Window: ${context.window.label}
 Since: ${context.window.since}
 Until: ${context.window.until}
 Sources: ${JSON.stringify(context.counts)}
+Prompt tweets: ${String(tweetCount)} of ${String(context.tweets.length)} selected context tweets
 
 Write a high-signal "what happened" report from this local Twitter/X dataset.
 
@@ -579,6 +929,7 @@ Requirements:
 - Target 700-1100 words when there is enough data.
 - Start with a 2-3 sentence lead that immediately says what people are talking about.
 - Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies.
+- When a tweet has replyToTweet, use that parent context to understand what the author was replying to and whether Peter already joined the conversation.
 - Use bullets under each section. Each bullet should be specific and explain why it matters.
 - For tweets: cite every claim with inline tweet ids at the end of the relevant sentence or bullet, e.g. (tweet_123, tweet_456). These citations become hoverable source links.
 - For links: emit normal Markdown links with no space between the label and URL, e.g. [title](https://example.com), then cite the sharing tweet ids in the same bullet.
@@ -593,15 +944,7 @@ Requirements:
 - JSON shape: { "title": string, "summary": string, "keyTopics": [{ "title": string, "summary": string, "tweetIds": string[], "handles": string[] }], "notableLinks": [{ "title": string, "url": string, "why": string, "sourceTweetIds": string[] }], "people": [{ "handle": string, "name"?: string, "why": string }], "actionItems": [{ "kind": "reply"|"follow_up"|"read"|"sync", "label": string, "tweetId"?: string, "dmConversationId"?: string }], "sourceTweetIds": string[] }
 
 Dataset:
-${JSON.stringify(
-	{
-		tweets: promptTweets,
-		dms: context.dms,
-		links: context.links,
-	},
-	null,
-	2,
-)}`;
+${JSON.stringify(dataset)}`;
 }
 
 function fallbackDigest(
@@ -870,10 +1213,15 @@ export function streamPeriodDigestEffect(
 	handlers: PeriodDigestStreamHandlers = {},
 ): Effect.Effect<PeriodDigestRunResult, Error> {
 	return Effect.gen(function* () {
-		const context = yield* tryDigestSync(() =>
+		yield* refreshPeriodDigestInputsEffect(
+			options,
+			{ threads: false },
+			handlers,
+		).pipe(Effect.catchAll(() => Effect.void));
+		let context = yield* tryDigestSync(() =>
 			collectPeriodDigestContext(options),
 		);
-		const cacheKey = digestCacheKey(context, options);
+		let cacheKey = digestCacheKey(context, options);
 		const cached = options.refresh
 			? null
 			: yield* tryDigestSync(() =>
@@ -904,12 +1252,28 @@ export function streamPeriodDigestEffect(
 			return result;
 		}
 
+		yield* refreshPeriodDigestInputsEffect(
+			options,
+			{
+				timeline: false,
+				mentions: false,
+				threads: true,
+				threadTweetIds: context.tweets
+					.filter((tweet) => tweet.source === "mentions")
+					.map((tweet) => tweet.id),
+			},
+			handlers,
+		).pipe(Effect.catchAll(() => Effect.void));
+		context = yield* tryDigestSync(() => collectPeriodDigestContext(options));
+		cacheKey = digestCacheKey(context, options);
+
 		const apiKey = process.env.OPENAI_API_KEY;
 		if (!apiKey) {
 			return yield* Effect.fail(new Error("OPENAI_API_KEY is not set"));
 		}
 
 		handlers.onEvent?.({ type: "start", context, cached: false });
+		emitDigestStatus(handlers, "Streaming AI summary");
 		const response = yield* tryDigestPromise(() =>
 			fetch("https://api.openai.com/v1/responses", {
 				method: "POST",