birdclaw 0.8.1 → 0.8.3

package/src/lib/sqlite.ts

@@ -5,6 +5,8 @@ export type Database = NativeSqliteDatabase;
 type DatabaseOptions = {
 	readonly?: boolean;
 	fileMustExist?: boolean;
+	timeout?: number;
+	onStatement?: (sql: string, durationMs: number) => void;
 };
 
 type PragmaOptions = {
@@ -16,6 +18,8 @@ type RunResult = {
 	lastInsertRowid: number;
 };
 
+export const SQLITE_BUSY_TIMEOUT_MS = 30_000;
+
 function bindArgs(parameters: unknown[]) {
 	if (parameters.length === 1 && Array.isArray(parameters[0])) {
 		return parameters[0];
@@ -50,44 +54,75 @@ function normalizeRow(row: unknown): unknown {
 class NativeSqliteStatement {
 	readonly reader: boolean;
 
-	constructor(private readonly statement: StatementSync) {
+	constructor(
+		private readonly statement: StatementSync,
+		private readonly sql: string,
+		private readonly onStatement?: (sql: string, durationMs: number) => void,
+	) {
 		this.reader = statement.columns().length > 0;
 	}
 
+	private track<T>(operation: () => T) {
+		const startedAt = performance.now();
+		try {
+			return operation();
+		} finally {
+			this.onStatement?.(this.sql, performance.now() - startedAt);
+		}
+	}
+
 	all(...parameters: unknown[]): unknown[] {
-		return this.statement.all(...bindArgs(parameters)).map(normalizeRow);
+		return this.track(() =>
+			this.statement.all(...bindArgs(parameters)).map(normalizeRow),
+		);
 	}
 
 	get(...parameters: unknown[]): unknown {
-		return normalizeRow(this.statement.get(...bindArgs(parameters)));
+		return this.track(() =>
+			normalizeRow(this.statement.get(...bindArgs(parameters))),
+		);
 	}
 
 	run(...parameters: unknown[]): RunResult {
-		const result = this.statement.run(...bindArgs(parameters));
-		return {
-			changes: Number(result.changes),
-			lastInsertRowid: Number(result.lastInsertRowid),
-		};
+		return this.track(() => {
+			const result = this.statement.run(...bindArgs(parameters));
+			return {
+				changes: Number(result.changes),
+				lastInsertRowid: Number(result.lastInsertRowid),
+			};
+		});
 	}
 
 	iterate(...parameters: unknown[]): IterableIterator<unknown> {
 		const rows = this.statement.iterate(...bindArgs(parameters));
+		const startedAt = performance.now();
+		const onStatement = this.onStatement;
+		const sql = this.sql;
 		return (function* () {
-			for (const row of rows) {
-				yield normalizeRow(row);
+			try {
+				for (const row of rows) {
+					yield normalizeRow(row);
+				}
+			} finally {
+				onStatement?.(sql, performance.now() - startedAt);
 			}
 		})();
 	}
 }
 
 export class NativeSqliteDatabase {
+	readonly writeIdentity: string | object;
 	private transactionDepth = 0;
 	private readonly db: DatabaseSync;
 
-	constructor(path: string, options: DatabaseOptions = {}) {
+	constructor(
+		path: string,
+		private readonly options: DatabaseOptions = {},
+	) {
+		this.writeIdentity = path === ":memory:" ? this : path;
 		this.db = new DatabaseSync(path, {
 			readOnly: options.readonly,
-			timeout: 5000,
+			timeout: options.timeout ?? SQLITE_BUSY_TIMEOUT_MS,
 		});
 	}
 
@@ -114,7 +149,11 @@ export class NativeSqliteDatabase {
 	}
 
 	prepare(sql: string): NativeSqliteStatement {
-		return new NativeSqliteStatement(this.db.prepare(sql));
+		return new NativeSqliteStatement(
+			this.db.prepare(sql),
+			sql,
+			this.options.onStatement,
+		);
 	}
 
 	transaction<TArgs extends unknown[], TResult>(
@@ -123,7 +162,7 @@ export class NativeSqliteDatabase {
 		return (...args: TArgs) => {
 			const nested = this.db.isTransaction;
 			const savepoint = `__birdclaw_tx_${++this.transactionDepth}`;
-			this.exec(nested ? `savepoint ${savepoint}` : "begin");
+			this.exec(nested ? `savepoint ${savepoint}` : "begin immediate");
 			try {
 				const result = fn(...args);
 				this.exec(nested ? `release ${savepoint}` : "commit");

package/src/lib/streaming-ingestion.ts

@@ -0,0 +1,240 @@
+import { createReadStream } from "node:fs";
+import { createInterface } from "node:readline";
+import { Effect } from "effect";
+
+export interface IngestionCheckpoint {
+	processed: number;
+}
+
+export interface IngestionSource<T> {
+	id: string;
+	stream: () => AsyncIterable<T>;
+}
+
+export interface IngestionSourceCheckpoint extends IngestionCheckpoint {
+	sourceId: string;
+	sourceIndex: number;
+	sourceProcessed: number;
+	sources: number;
+}
+
+export async function* streamJsonLines(
+	filePath: string,
+): AsyncGenerator<{ lineNumber: number; value: Record<string, unknown> }> {
+	const input = createReadStream(filePath, { encoding: "utf8" });
+	const lines = createInterface({ input, crlfDelay: Infinity });
+	let lineNumber = 0;
+	for await (const line of lines) {
+		lineNumber += 1;
+		if (!line.trim()) continue;
+		yield {
+			lineNumber,
+			value: JSON.parse(line) as Record<string, unknown>,
+		};
+	}
+}
+
+export async function* streamAssignedJsonArray(
+	source: AsyncIterable<Buffer | string>,
+): AsyncGenerator<Record<string, unknown>> {
+	let started = false;
+	let item = "";
+	let depth = 0;
+	let inString = false;
+	let escaped = false;
+
+	const flush = () => {
+		const value = item.trim();
+		item = "";
+		return value ? (JSON.parse(value) as Record<string, unknown>) : undefined;
+	};
+
+	for await (const chunk of source) {
+		for (const character of String(chunk)) {
+			if (!started) {
+				if (character === "[") started = true;
+				continue;
+			}
+
+			if (inString) {
+				item += character;
+				if (escaped) {
+					escaped = false;
+				} else if (character === "\\") {
+					escaped = true;
+				} else if (character === '"') {
+					inString = false;
+				}
+				continue;
+			}
+
+			if (character === '"') {
+				inString = true;
+				item += character;
+				continue;
+			}
+			if (character === "{" || character === "[") {
+				depth += 1;
+				item += character;
+				continue;
+			}
+			if (character === "}" || (character === "]" && depth > 0)) {
+				depth -= 1;
+				item += character;
+				continue;
+			}
+			if (character === "]" && depth === 0) {
+				const value = flush();
+				if (value) yield value;
+				return;
+			}
+			if (character === "," && depth === 0) {
+				const value = flush();
+				if (value) yield value;
+				continue;
+			}
+			if (item.length > 0 || !/\s/.test(character)) {
+				item += character;
+			}
+		}
+	}
+
+	if (!started) return;
+	const value = flush();
+	if (value) yield value;
+}
+
+export function ingestStreamInBatchesEffect<T>({
+	batchSize = 250,
+	onCheckpoint,
+	processBatch,
+	resumeAfter = 0,
+	source,
+}: {
+	batchSize?: number;
+	onCheckpoint?: (checkpoint: IngestionCheckpoint) => void | Promise<void>;
+	processBatch: (
+		batch: T[],
+		checkpoint: IngestionCheckpoint,
+	) => void | Promise<void>;
+	resumeAfter?: number;
+	source: () => AsyncIterable<T>;
+}): Effect.Effect<IngestionCheckpoint, unknown> {
+	return Effect.tryPromise({
+		try: async () => {
+			const normalizedBatchSize = Math.max(1, Math.trunc(batchSize));
+			const normalizedResumeAfter = Math.max(0, Math.trunc(resumeAfter));
+			let visited = 0;
+			let processed = normalizedResumeAfter;
+			let batch: T[] = [];
+
+			const flush = async () => {
+				if (batch.length === 0) return;
+				processed += batch.length;
+				const checkpoint = { processed };
+				await processBatch(batch, checkpoint);
+				await onCheckpoint?.(checkpoint);
+				batch = [];
+			};
+
+			for await (const item of source()) {
+				visited += 1;
+				if (visited <= normalizedResumeAfter) continue;
+				batch.push(item);
+				if (batch.length >= normalizedBatchSize) {
+					await flush();
+				}
+			}
+			await flush();
+			return { processed };
+		},
+		catch: (error) =>
+			error instanceof Error ? error : new Error(String(error)),
+	});
+}
+
+export function ingestSourcesInBatchesEffect<T>({
+	batchSize,
+	onCheckpoint,
+	onSourceComplete,
+	processBatch,
+	sources,
+}: {
+	batchSize?: number;
+	onCheckpoint?: (
+		checkpoint: IngestionSourceCheckpoint,
+	) => void | Promise<void>;
+	onSourceComplete?: (
+		checkpoint: IngestionSourceCheckpoint,
+	) => void | Promise<void>;
+	processBatch: (
+		batch: T[],
+		checkpoint: IngestionSourceCheckpoint,
+	) => void | Promise<void>;
+	sources: IngestionSource<T>[];
+}): Effect.Effect<IngestionCheckpoint, unknown> {
+	return Effect.gen(function* () {
+		let processed = 0;
+
+		for (const [sourceIndex, source] of sources.entries()) {
+			const sourceResult = yield* ingestStreamInBatchesEffect({
+				batchSize,
+				source: source.stream,
+				processBatch: (batch, checkpoint) =>
+					processBatch(batch, {
+						...checkpoint,
+						processed: processed + checkpoint.processed,
+						sourceId: source.id,
+						sourceIndex,
+						sourceProcessed: checkpoint.processed,
+						sources: sources.length,
+					}),
+				onCheckpoint: onCheckpoint
+					? (checkpoint) =>
+							onCheckpoint({
+								...checkpoint,
+								processed: processed + checkpoint.processed,
+								sourceId: source.id,
+								sourceIndex,
+								sourceProcessed: checkpoint.processed,
+								sources: sources.length,
+							})
+					: undefined,
+			});
+			processed += sourceResult.processed;
+			yield* Effect.tryPromise({
+				try: () =>
+					Promise.resolve(
+						onSourceComplete?.({
+							processed,
+							sourceId: source.id,
+							sourceIndex,
+							sourceProcessed: sourceResult.processed,
+							sources: sources.length,
+						}),
+					),
+				catch: (error) =>
+					error instanceof Error ? error : new Error(String(error)),
+			});
+		}
+
+		return { processed };
+	});
+}
+
+export function collectIngestionSourcesEffect<T>(
+	sources: IngestionSource<T>[],
+	options: { batchSize?: number } = {},
+): Effect.Effect<T[], unknown> {
+	return Effect.gen(function* () {
+		const rows: T[] = [];
+		yield* ingestSourcesInBatchesEffect({
+			...options,
+			sources,
+			processBatch: (batch) => {
+				rows.push(...batch);
+			},
+		});
+		return rows;
+	});
+}

package/src/lib/sync-cache.ts

@@ -1,4 +1,8 @@
 import { getNativeDb } from "./db";
+import {
+	defaultServerRuntimeServices,
+	type ServerRuntimeServices,
+} from "./server-runtime-services";
 
 export interface SyncCacheEntry<T> {
 	value: T;
@@ -45,8 +49,9 @@ export function writeSyncCache(
 	cacheKey: string,
 	value: unknown,
 	db = getNativeDb(),
+	runtime: ServerRuntimeServices = defaultServerRuntimeServices,
 ) {
-	const updatedAt = new Date().toISOString();
+	const updatedAt = runtime.now().toISOString();
 	db.prepare(
 		`
     insert into sync_cache (cache_key, value_json, updated_at)

package/src/lib/sync-plan.ts

@@ -0,0 +1,175 @@
+import { Effect } from "effect";
+
+export type PaginationStopReason =
+	| "boundary"
+	| "exhausted"
+	| "item-limit"
+	| "page-limit"
+	| "repeated-cursor";
+
+export interface PaginationPageContext<Page> {
+	cursor?: string;
+	fetched: number;
+	page: Page;
+	pageIndex: number;
+	pageNumber: number;
+	stopReason?: PaginationStopReason;
+	done: boolean;
+}
+
+export type SyncPlanStopReason = PaginationStopReason | "error";
+
+export interface SyncPlanPageContext<Page> extends Omit<
+	PaginationPageContext<Page>,
+	"stopReason"
+> {
+	nextCursor?: string;
+	stopReason?: SyncPlanStopReason;
+}
+
+export interface SyncPlanResult<Page, ErrorType> {
+	complete: boolean;
+	fetched: number;
+	nextCursor?: string;
+	pages: Page[];
+	stopReason: SyncPlanStopReason;
+	error?: ErrorType;
+}
+
+function normalizeCursor(value: string | null | undefined) {
+	const normalized = value?.trim();
+	return normalized || undefined;
+}
+
+export function runSyncPlanEffect<Page, FetchError, PageError = never>({
+	fetchPage,
+	getItemCount,
+	getNextCursor,
+	initialCursor,
+	maxItems,
+	maxPages,
+	onPage,
+	pageDelayMs,
+	persistPage,
+	shouldStop,
+	allowPartialFailure = false,
+}: {
+	fetchPage: (context: {
+		cursor?: string;
+		fetched: number;
+		pageIndex: number;
+	}) => Effect.Effect<Page, FetchError>;
+	getItemCount?: (page: Page) => number;
+	getNextCursor: (page: Page) => string | null | undefined;
+	initialCursor?: string;
+	maxItems?: number;
+	maxPages?: number;
+	onPage?: (context: SyncPlanPageContext<Page>) => void;
+	pageDelayMs?: number;
+	persistPage?: (
+		context: SyncPlanPageContext<Page>,
+	) => Effect.Effect<void, PageError>;
+	shouldStop?: (
+		context: Omit<SyncPlanPageContext<Page>, "done" | "nextCursor">,
+	) => boolean;
+	allowPartialFailure?: boolean;
+}): Effect.Effect<SyncPlanResult<Page, FetchError>, FetchError | PageError> {
+	return Effect.gen(function* () {
+		const pages: Page[] = [];
+		const seenCursors = new Set<string>();
+		let cursor = normalizeCursor(initialCursor);
+		let fetched = 0;
+		const pageLimit =
+			maxPages === undefined ? Number.POSITIVE_INFINITY : Math.max(1, maxPages);
+		const itemLimit =
+			maxItems === undefined ? Number.POSITIVE_INFINITY : Math.max(1, maxItems);
+
+		if (cursor) seenCursors.add(cursor);
+
+		while (pages.length < pageLimit) {
+			const pageIndex = pages.length;
+			const outcome = yield* fetchPage({ cursor, fetched, pageIndex }).pipe(
+				Effect.map((page) => ({ ok: true as const, page })),
+				Effect.catchAll((error) =>
+					Effect.succeed({ ok: false as const, error }),
+				),
+			);
+			if (!outcome.ok) {
+				if (!allowPartialFailure || pages.length === 0) {
+					return yield* Effect.fail(outcome.error);
+				}
+				return {
+					complete: false,
+					fetched,
+					...(cursor ? { nextCursor: cursor } : {}),
+					pages,
+					stopReason: "error",
+					error: outcome.error,
+				};
+			}
+
+			const page = outcome.page;
+			pages.push(page);
+			fetched += Math.max(0, getItemCount?.(page) ?? 0);
+			const nextCursor = normalizeCursor(getNextCursor(page));
+			const baseContext = {
+				cursor,
+				fetched,
+				page,
+				pageIndex,
+				pageNumber: pageIndex + 1,
+			};
+			let stopReason: PaginationStopReason | undefined;
+
+			if (!nextCursor) {
+				stopReason = "exhausted";
+			} else if (shouldStop?.(baseContext)) {
+				stopReason = "boundary";
+			} else if (fetched >= itemLimit) {
+				stopReason = "item-limit";
+			} else if (pages.length >= pageLimit) {
+				stopReason = "page-limit";
+			} else if (seenCursors.has(nextCursor)) {
+				stopReason = "repeated-cursor";
+			}
+
+			const context: SyncPlanPageContext<Page> = {
+				...baseContext,
+				...(nextCursor ? { nextCursor } : {}),
+				done: Boolean(stopReason),
+				...(stopReason ? { stopReason } : {}),
+			};
+			if (persistPage) {
+				yield* persistPage(context);
+			}
+			onPage?.(context);
+
+			if (stopReason) {
+				return {
+					complete:
+						stopReason === "exhausted" ||
+						stopReason === "boundary" ||
+						stopReason === "item-limit",
+					fetched,
+					...(nextCursor ? { nextCursor } : {}),
+					pages,
+					stopReason,
+				};
+			}
+
+			cursor = nextCursor;
+			seenCursors.add(nextCursor!);
+			if (typeof pageDelayMs === "number" && pageDelayMs > 0) {
+				yield* Effect.sleep(pageDelayMs);
+			}
+		}
+
+		return {
+			complete: false,
+			fetched,
+			...(cursor ? { nextCursor: cursor } : {}),
+			pages,
+			stopReason: "page-limit",
+		};
+	});
+}