birdclaw 0.8.2 → 0.8.3

package/src/lib/query-status.ts

@@ -0,0 +1,164 @@
+import { Effect } from "effect";
+import type { Database } from "./sqlite";
+import { findArchivesCachedEffect } from "./archive-finder";
+import { getReadDb } from "./db";
+import { runEffectPromise } from "./effect-runtime";
+import type { AccountRecord, QueryEnvelope } from "./types";
+import { getTransportStatusEffect } from "./xurl";
+
+export type { QueryEnvelope } from "./types";
+
+function toError(error: unknown) {
+	return error instanceof Error ? error : new Error(String(error));
+}
+
+function trySync<T>(try_: () => T) {
+	return Effect.try({ try: try_, catch: toError });
+}
+
+function countTimelineEdges(db: Database, kind: "home" | "mention") {
+	const row = db
+		.prepare(
+			`
+      select count(distinct tweet_id) as count
+      from (
+        select edge.tweet_id
+        from tweet_account_edges edge
+        where edge.kind = ?
+          and exists (
+            select 1
+            from tweets t
+            where t.id = edge.tweet_id
+          )
+        union all
+        select legacy.id as tweet_id
+        from tweets legacy
+        where legacy.kind = ?
+          and not exists (
+            select 1
+            from tweet_account_edges edge
+            where edge.account_id = legacy.account_id
+              and edge.tweet_id = legacy.id
+              and edge.kind = legacy.kind
+          )
+      )
+      `,
+		)
+		.get(kind, kind) as { count: number | bigint } | undefined;
+	return Number(row?.count ?? 0);
+}
+
+function getAccountProfileMeta(
+	db: Database,
+	account: { handle: string; external_user_id: string | null },
+) {
+	const handle = account.handle.replace(/^@/, "");
+	const externalProfileId = account.external_user_id
+		? `profile_user_${account.external_user_id}`
+		: "";
+	return db
+		.prepare(
+			`
+      select id, avatar_hue, avatar_url
+      from profiles
+      where id = ?
+         or lower(handle) = lower(?)
+      order by case
+        when id = 'profile_me' then 0
+        when id = ? then 1
+        else 2
+      end
+      limit 1
+    `,
+		)
+		.get(externalProfileId, handle, externalProfileId) as
+		| { id: string; avatar_hue: number; avatar_url: string | null }
+		| undefined;
+}
+
+export function getQueryEnvelopeEffect({
+	includeArchives = true,
+}: { includeArchives?: boolean } = {}): Effect.Effect<QueryEnvelope, unknown> {
+	return Effect.gen(function* () {
+		const nativeDb = yield* trySync(() => getReadDb());
+		const homeCount = yield* trySync(() =>
+			countTimelineEdges(nativeDb, "home"),
+		);
+		const mentionCount = yield* trySync(() =>
+			countTimelineEdges(nativeDb, "mention"),
+		);
+		const counts = yield* Effect.all({
+			dms: trySync(
+				() =>
+					nativeDb
+						.prepare("select count(*) as count from dm_conversations")
+						.get() as { count: number },
+			),
+			needsReply: trySync(
+				() =>
+					nativeDb
+						.prepare(
+							"select count(*) as count from dm_conversations where needs_reply = 1",
+						)
+						.get() as { count: number },
+			),
+			accounts: trySync(
+				() =>
+					nativeDb
+						.prepare(
+							"select * from accounts order by is_default desc, name asc",
+						)
+						.all() as Array<{
+						id: string;
+						name: string;
+						handle: string;
+						external_user_id: string | null;
+						transport: string;
+						is_default: number;
+						created_at: string;
+					}>,
+			),
+			archives: includeArchives
+				? findArchivesCachedEffect()
+				: Effect.succeed([]),
+			transport: getTransportStatusEffect(),
+		});
+
+		return {
+			stats: {
+				home: homeCount,
+				mentions: mentionCount,
+				dms: Number(counts.dms.count),
+				needsReply: Number(counts.needsReply.count),
+				inbox: mentionCount + Number(counts.needsReply.count),
+			},
+			accounts: counts.accounts.map((row) => {
+				const profile = getAccountProfileMeta(nativeDb, row);
+				return {
+					id: row.id,
+					name: row.name,
+					handle: row.handle,
+					externalUserId: row.external_user_id,
+					...(profile
+						? {
+								profileId: profile.id,
+								avatarHue: Number(profile.avatar_hue),
+								...(profile.avatar_url
+									? { avatarUrl: profile.avatar_url }
+									: {}),
+							}
+						: {}),
+					transport: row.transport,
+					isDefault: row.is_default,
+					createdAt: row.created_at,
+				};
+			}) satisfies AccountRecord[],
+			archives: counts.archives,
+			transport: counts.transport,
+		};
+	});
+}
+
+export function getQueryEnvelope(): Promise<QueryEnvelope> {
+	return runEffectPromise(getQueryEnvelopeEffect());
+}

package/src/lib/research.ts

@@ -3,7 +3,7 @@ import path from "node:path";
 import { Effect } from "effect";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
-import { listTimelineItems } from "./queries";
+import { listTimelineItems } from "./timeline-read-model";
 import { lookupTweetsByIdsEffect } from "./tweet-lookup";
 import { renderTweetMarkdown, renderTweetPlainText } from "./tweet-render";
 import type { TweetEntities, XurlMentionUser } from "./types";

package/src/lib/runtime-services.ts

@@ -0,0 +1,20 @@
+export interface RuntimeServices {
+	fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
+	now(): Date;
+	random(): number;
+	env(name: string): string | undefined;
+}
+
+export const defaultRuntimeServices: RuntimeServices = {
+	fetch: (input, init) => globalThis.fetch(input, init),
+	now: () => new Date(),
+	random: () => Math.random(),
+	env: (name) =>
+		typeof process === "undefined" ? undefined : process.env[name],
+};
+
+export function createRuntimeServices(
+	overrides: Partial<RuntimeServices> = {},
+): RuntimeServices {
+	return { ...defaultRuntimeServices, ...overrides };
+}

package/src/lib/search-discussion.ts

@@ -1,14 +1,22 @@
 import { createHash } from "node:crypto";
 import { Effect } from "effect";
 import { z } from "zod";
-import { prefetchCachedAvatarsForProfileIdsEffect } from "./avatar-cache";
 import {
-	runEffectBackground,
-	runEffectPromise,
-	tryPromise,
-} from "./effect-runtime";
+	createAnalysisRequestBody,
+	type HybridAnalysisResult,
+	parseHybridAnalysis,
+	resolveAnalysisModelSettings,
+	streamHybridAnalysisEffect,
+} from "./analysis-runtime";
+import { prefetchCachedAvatarsForProfileIdsEffect } from "./avatar-cache";
+import { runEffectBackground, runEffectPromise } from "./effect-runtime";
 import { getNativeDb } from "./db";
-import { listDmConversations, listTimelineItems } from "./queries";
+import { listDmConversations } from "./dm-read-model";
+import {
+	type OpenAIStreamState,
+	processOpenAIResponseSseChunk,
+} from "./openai-response-runtime";
+import { listTimelineItems } from "./timeline-read-model";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import {
 	syncTweetSearchEffect,
@@ -129,24 +137,10 @@ export type SearchDiscussionStreamEvent =
 	| { type: "done"; result: SearchDiscussionRunResult }
 	| { type: "error"; error: string };
 
-interface OpenAIStreamState {
-	eventBuffer: string;
-	rawText: string;
-	pendingVisible: string;
-	jsonMode: boolean;
-	responseId?: string;
-	usage?: unknown;
-	error?: string;
-}
-
-const DEFAULT_MODEL = "gpt-5.5";
-const DEFAULT_REASONING_EFFORT = "medium";
-const DEFAULT_SERVICE_TIER = "priority";
 const DEFAULT_LIMIT = 20_000;
 const DEFAULT_MAX_PAGES = 200;
 const MAX_PROMPT_DATA_CHARS = 1_200_000;
 const DELIMITER_PATTERN = /\n---\s*\n/;
-const VISIBLE_DELIMITER_HOLD = 8;
 
 function toError(error: unknown) {
 	return error instanceof Error ? error : new Error(String(error));
@@ -159,12 +153,6 @@ function trySearchSync<T>(try_: () => T): Effect.Effect<T, Error> {
 	});
 }
 
-function trySearchPromise<T>(
-	try_: () => PromiseLike<T>,
-): Effect.Effect<T, Error> {
-	return tryPromise(try_).pipe(Effect.mapError(toError));
-}
-
 function tweetUrl(handle: string, id: string) {
 	return `https://x.com/${handle}/status/${id}`;
 }
@@ -524,27 +512,15 @@ function prefetchDiscussionAvatars(context: SearchDiscussionContext) {
 }
 
 function modelFromOptions(options: SearchDiscussionOptions) {
-	return options.model ?? process.env.BIRDCLAW_AI_MODEL ?? DEFAULT_MODEL;
+	return resolveAnalysisModelSettings(options).model;
 }
 
 function reasoningEffortFromOptions(options: SearchDiscussionOptions) {
-	return (
-		options.reasoningEffort ??
-		(process.env.BIRDCLAW_OPENAI_REASONING_EFFORT as
-			| SearchDiscussionOptions["reasoningEffort"]
-			| undefined) ??
-		DEFAULT_REASONING_EFFORT
-	);
+	return resolveAnalysisModelSettings(options).reasoningEffort;
 }
 
 function serviceTierFromOptions(options: SearchDiscussionOptions) {
-	return (
-		options.serviceTier ??
-		(process.env.BIRDCLAW_OPENAI_SERVICE_TIER as
-			| SearchDiscussionOptions["serviceTier"]
-			| undefined) ??
-		DEFAULT_SERVICE_TIER
-	);
+	return resolveAnalysisModelSettings(options).serviceTier;
 }
 
 function cacheKey(
@@ -665,115 +641,13 @@ function parseDiscussionFromHybridText(
 	context: SearchDiscussionContext,
 	rawText: string,
 ): { discussion: SearchDiscussion; markdown: string } {
-	const [markdownPart, jsonPart] = rawText.split(DELIMITER_PATTERN);
-	const markdown = (markdownPart ?? rawText).trim();
-	const candidate = jsonPart?.slice(
-		jsonPart.indexOf("{"),
-		jsonPart.lastIndexOf("}") + 1,
-	);
-	if (candidate?.startsWith("{")) {
-		try {
-			return {
-				markdown,
-				discussion: SearchDiscussionSchema.parse(JSON.parse(candidate)),
-			};
-		} catch {
-			return { markdown, discussion: fallbackDiscussion(context, markdown) };
-		}
-	}
-	return { markdown, discussion: fallbackDiscussion(context, markdown) };
-}
-
-function emitVisibleDelta(
-	state: OpenAIStreamState,
-	delta: string,
-	handlers: SearchDiscussionStreamHandlers,
-) {
-	state.rawText += delta;
-	if (state.jsonMode) return;
-
-	const combined = state.pendingVisible + delta;
-	const delimiterIndex = combined.search(DELIMITER_PATTERN);
-	if (delimiterIndex >= 0) {
-		const visible = combined.slice(0, delimiterIndex);
-		if (visible) {
-			handlers.onDelta?.(visible);
-			handlers.onEvent?.({ type: "delta", delta: visible });
-		}
-		state.pendingVisible = "";
-		state.jsonMode = true;
-		return;
-	}
-
-	if (combined.length <= VISIBLE_DELIMITER_HOLD) {
-		state.pendingVisible = combined;
-		return;
-	}
-
-	const visible = combined.slice(0, -VISIBLE_DELIMITER_HOLD);
-	state.pendingVisible = combined.slice(-VISIBLE_DELIMITER_HOLD);
-	if (visible) {
-		handlers.onDelta?.(visible);
-		handlers.onEvent?.({ type: "delta", delta: visible });
-	}
-}
-
-function flushPendingVisible(
-	state: OpenAIStreamState,
-	handlers: SearchDiscussionStreamHandlers,
-) {
-	if (state.jsonMode || !state.pendingVisible) return;
-	const delta = state.pendingVisible;
-	state.pendingVisible = "";
-	handlers.onDelta?.(delta);
-	handlers.onEvent?.({ type: "delta", delta });
-}
-
-function handleOpenAIEvent(
-	state: OpenAIStreamState,
-	event: Record<string, unknown>,
-	handlers: SearchDiscussionStreamHandlers,
-) {
-	const type = typeof event.type === "string" ? event.type : "";
-	if (
-		type === "response.output_text.delta" &&
-		typeof event.delta === "string"
-	) {
-		emitVisibleDelta(state, event.delta, handlers);
-		return;
-	}
-	if (type === "response.completed") {
-		const response = event.response;
-		if (response && typeof response === "object") {
-			const record = response as Record<string, unknown>;
-			state.responseId = typeof record.id === "string" ? record.id : undefined;
-			state.usage = record.usage;
-		}
-		return;
-	}
-	if (type === "response.error" || type === "error") {
-		const error = event.error;
-		state.error =
-			error && typeof error === "object" && "message" in error
-				? String((error as { message?: unknown }).message)
-				: "OpenAI stream failed";
-		return;
-	}
-	if (type === "response.failed" || type === "response.incomplete") {
-		const response = event.response;
-		const record =
-			response && typeof response === "object"
-				? (response as Record<string, unknown>)
-				: {};
-		const error = record.error;
-		const incomplete = record.incomplete_details;
-		state.error =
-			error && typeof error === "object" && "message" in error
-				? String((error as { message?: unknown }).message)
-				: incomplete && typeof incomplete === "object" && "reason" in incomplete
-					? `OpenAI response incomplete: ${String((incomplete as { reason?: unknown }).reason)}`
-					: "OpenAI stream failed";
-	}
+	const parsed = parseHybridAnalysis({
+		rawText,
+		parse: (value) => SearchDiscussionSchema.parse(value),
+		fallback: (markdown) => fallbackDiscussion(context, markdown),
+		delimiterPattern: DELIMITER_PATTERN,
+	});
+	return { markdown: parsed.markdown, discussion: parsed.value };
 }
 
 function processSseChunk(
@@ -781,126 +655,59 @@ function processSseChunk(
 	chunk: string,
 	handlers: SearchDiscussionStreamHandlers,
 ) {
-	state.eventBuffer += chunk;
-	let boundary = state.eventBuffer.indexOf("\n\n");
-	while (boundary >= 0) {
-		const block = state.eventBuffer.slice(0, boundary);
-		state.eventBuffer = state.eventBuffer.slice(boundary + 2);
-		const data = block
-			.split("\n")
-			.filter((line) => line.startsWith("data:"))
-			.map((line) => line.slice(5).trimStart())
-			.join("\n");
-		if (data && data !== "[DONE]") {
-			try {
-				handleOpenAIEvent(
-					state,
-					JSON.parse(data) as Record<string, unknown>,
-					handlers,
-				);
-			} catch {
-				// Final hybrid parse decides whether malformed output can be used.
-			}
-		}
-		boundary = state.eventBuffer.indexOf("\n\n");
-	}
+	processOpenAIResponseSseChunk(state, chunk, {
+		delimiterPattern: DELIMITER_PATTERN,
+		onDelta: (delta) => {
+			handlers.onDelta?.(delta);
+			handlers.onEvent?.({ type: "delta", delta });
+		},
+	});
 }
 
 function createOpenAIRequestBody(
 	context: SearchDiscussionContext,
 	options: SearchDiscussionOptions,
 ) {
-	return {
-		model: modelFromOptions(options),
-		reasoning: { effort: reasoningEffortFromOptions(options) },
-		service_tier: serviceTierFromOptions(options),
-		store: false,
+	return createAnalysisRequestBody({
+		settings: resolveAnalysisModelSettings(options),
+		system:
+			"You are a precise local Twitter archive analyst. Stream Markdown first, then emit the requested JSON object after the delimiter. Do not invent events not present in the dataset.",
+		prompt: buildPrompt(context),
 		stream: true,
-		max_output_tokens: 7000,
-		input: [
-			{
-				role: "system",
-				content:
-					"You are a precise local Twitter archive analyst. Stream Markdown first, then emit the requested JSON object after the delimiter. Do not invent events not present in the dataset.",
-			},
-			{
-				role: "user",
-				content: buildPrompt(context),
-			},
-		],
-	};
+	});
 }
 
-function readOpenAIStreamEffect(
-	response: Response,
+function completeOpenAIStreamEffect(
+	stream: HybridAnalysisResult<SearchDiscussion>,
 	context: SearchDiscussionContext,
 	options: SearchDiscussionOptions,
 	handlers: SearchDiscussionStreamHandlers,
 ): Effect.Effect<SearchDiscussionRunResult, Error> {
-	const reader = response.body?.getReader();
-	if (!reader) {
-		return Effect.fail(new Error("OpenAI response did not include a stream"));
-	}
-
-	const decoder = new TextDecoder();
-	const state: OpenAIStreamState = {
-		eventBuffer: "",
-		rawText: "",
-		pendingVisible: "",
-		jsonMode: false,
-	};
-
 	return Effect.gen(function* () {
-		for (;;) {
-			const { done, value } = yield* trySearchPromise(() => reader.read());
-			if (!done) {
-				processSseChunk(
-					state,
-					decoder.decode(value, { stream: true }),
-					handlers,
-				);
-				continue;
-			}
-
-			flushPendingVisible(state, handlers);
-			if (state.error) {
-				return yield* Effect.fail(new Error(state.error));
-			}
-
-			const parsed = yield* trySearchSync(() =>
-				parseDiscussionFromHybridText(context, state.rawText),
-			);
-			const updatedAt = yield* trySearchSync(() =>
-				writeSyncCache(cacheKey(context, options), {
-					discussion: parsed.discussion,
-					markdown: parsed.markdown,
-					model: modelFromOptions(options),
-					reasoningEffort: reasoningEffortFromOptions(options),
-					serviceTier: serviceTierFromOptions(options),
-					usage: state.usage,
-					responseId: state.responseId,
-				}),
-			);
-			const result = {
-				context,
-				discussion: parsed.discussion,
-				markdown: parsed.markdown,
+		const updatedAt = yield* trySearchSync(() =>
+			writeSyncCache(cacheKey(context, options), {
+				discussion: stream.value,
+				markdown: stream.markdown,
 				model: modelFromOptions(options),
 				reasoningEffort: reasoningEffortFromOptions(options),
 				serviceTier: serviceTierFromOptions(options),
-				cached: false,
-				updatedAt,
-			};
-			handlers.onEvent?.({ type: "done", result });
-			return result;
-		}
-	}).pipe(
-		Effect.ensuring(
-			Effect.sync(() => {
-				reader.releaseLock();
+				usage: stream.usage,
+				responseId: stream.responseId,
 			}),
-		),
-	);
+		);
+		const result = {
+			context,
+			discussion: stream.value,
+			markdown: stream.markdown,
+			model: modelFromOptions(options),
+			reasoningEffort: reasoningEffortFromOptions(options),
+			serviceTier: serviceTierFromOptions(options),
+			cached: false,
+			updatedAt,
+		};
+		handlers.onEvent?.({ type: "done", result });
+		return result;
+	});
 }
 
 export function streamSearchDiscussionEffect(
@@ -969,35 +776,24 @@ export function streamSearchDiscussionEffect(
 			return result;
 		}
 
-		const apiKey = process.env.OPENAI_API_KEY;
-		if (!apiKey) {
-			return yield* Effect.fail(new Error("OPENAI_API_KEY is not set"));
-		}
-
 		handlers.onEvent?.({ type: "start", context, cached: false });
-		const response = yield* trySearchPromise(() =>
-			fetch("https://api.openai.com/v1/responses", {
-				method: "POST",
-				signal: options.signal,
-				headers: {
-					authorization: `Bearer ${apiKey}`,
-					"content-type": "application/json",
-				},
-				body: JSON.stringify(createOpenAIRequestBody(context, options)),
-			}),
+		const stream = yield* streamHybridAnalysisEffect({
+			body: createOpenAIRequestBody(context, options),
+			signal: options.signal,
+			parse: (value) => SearchDiscussionSchema.parse(value),
+			fallback: (markdown) => fallbackDiscussion(context, markdown),
+			delimiterPattern: DELIMITER_PATTERN,
+			onDelta: (delta) => {
+				handlers.onDelta?.(delta);
+				handlers.onEvent?.({ type: "delta", delta });
+			},
+		});
+		return yield* completeOpenAIStreamEffect(
+			stream,
+			context,
+			options,
+			handlers,
 		);
-		if (!response.ok) {
-			const text = yield* trySearchPromise(() => response.text());
-			return yield* Effect.fail(
-				new Error(
-					`OpenAI request failed: ${String(response.status)} ${text.slice(
-						0,
-						400,
-					)}`,
-				),
-			);
-		}
-		return yield* readOpenAIStreamEffect(response, context, options, handlers);
 	});
 }
 

package/src/lib/server-runtime-services.ts

@@ -0,0 +1,30 @@
+import { getBirdclawPaths, type BirdclawPaths } from "./config";
+import { getNativeDb, type InitDatabaseOptions } from "./db";
+import {
+	createRuntimeServices,
+	defaultRuntimeServices,
+	type RuntimeServices,
+} from "./runtime-services";
+import type { Database } from "./sqlite";
+
+export interface ServerRuntimeServices extends RuntimeServices {
+	getDatabase(options?: InitDatabaseOptions): Database;
+	getPaths(): BirdclawPaths;
+}
+
+export const defaultServerRuntimeServices: ServerRuntimeServices = {
+	...defaultRuntimeServices,
+	getDatabase: (options) => getNativeDb(options),
+	getPaths: () => getBirdclawPaths(),
+};
+
+export function createServerRuntimeServices(
+	overrides: Partial<ServerRuntimeServices> = {},
+): ServerRuntimeServices {
+	return {
+		...createRuntimeServices(overrides),
+		getDatabase:
+			overrides.getDatabase ?? defaultServerRuntimeServices.getDatabase,
+		getPaths: overrides.getPaths ?? defaultServerRuntimeServices.getPaths,
+	};
+}