birdclaw 0.8.3 → 0.8.4

package/src/lib/sqlite.ts

@@ -1,183 +0,0 @@
-import { DatabaseSync, type StatementSync } from "node:sqlite";
-
-export type Database = NativeSqliteDatabase;
-
-type DatabaseOptions = {
-	readonly?: boolean;
-	fileMustExist?: boolean;
-	timeout?: number;
-	onStatement?: (sql: string, durationMs: number) => void;
-};
-
-type PragmaOptions = {
-	simple?: boolean;
-};
-
-type RunResult = {
-	changes: number;
-	lastInsertRowid: number;
-};
-
-export const SQLITE_BUSY_TIMEOUT_MS = 30_000;
-
-function bindArgs(parameters: unknown[]) {
-	if (parameters.length === 1 && Array.isArray(parameters[0])) {
-		return parameters[0];
-	}
-	return parameters;
-}
-
-function normalizeValue(value: unknown): unknown {
-	if (value instanceof Uint8Array && !Buffer.isBuffer(value)) {
-		return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
-	}
-	return value;
-}
-
-function normalizeRow(row: unknown): unknown {
-	if (
-		!row ||
-		typeof row !== "object" ||
-		Array.isArray(row) ||
-		Buffer.isBuffer(row)
-	) {
-		return normalizeValue(row);
-	}
-	return Object.fromEntries(
-		Object.entries(row as Record<string, unknown>).map(([key, value]) => [
-			key,
-			normalizeValue(value),
-		]),
-	);
-}
-
-class NativeSqliteStatement {
-	readonly reader: boolean;
-
-	constructor(
-		private readonly statement: StatementSync,
-		private readonly sql: string,
-		private readonly onStatement?: (sql: string, durationMs: number) => void,
-	) {
-		this.reader = statement.columns().length > 0;
-	}
-
-	private track<T>(operation: () => T) {
-		const startedAt = performance.now();
-		try {
-			return operation();
-		} finally {
-			this.onStatement?.(this.sql, performance.now() - startedAt);
-		}
-	}
-
-	all(...parameters: unknown[]): unknown[] {
-		return this.track(() =>
-			this.statement.all(...bindArgs(parameters)).map(normalizeRow),
-		);
-	}
-
-	get(...parameters: unknown[]): unknown {
-		return this.track(() =>
-			normalizeRow(this.statement.get(...bindArgs(parameters))),
-		);
-	}
-
-	run(...parameters: unknown[]): RunResult {
-		return this.track(() => {
-			const result = this.statement.run(...bindArgs(parameters));
-			return {
-				changes: Number(result.changes),
-				lastInsertRowid: Number(result.lastInsertRowid),
-			};
-		});
-	}
-
-	iterate(...parameters: unknown[]): IterableIterator<unknown> {
-		const rows = this.statement.iterate(...bindArgs(parameters));
-		const startedAt = performance.now();
-		const onStatement = this.onStatement;
-		const sql = this.sql;
-		return (function* () {
-			try {
-				for (const row of rows) {
-					yield normalizeRow(row);
-				}
-			} finally {
-				onStatement?.(sql, performance.now() - startedAt);
-			}
-		})();
-	}
-}
-
-export class NativeSqliteDatabase {
-	readonly writeIdentity: string | object;
-	private transactionDepth = 0;
-	private readonly db: DatabaseSync;
-
-	constructor(
-		path: string,
-		private readonly options: DatabaseOptions = {},
-	) {
-		this.writeIdentity = path === ":memory:" ? this : path;
-		this.db = new DatabaseSync(path, {
-			readOnly: options.readonly,
-			timeout: options.timeout ?? SQLITE_BUSY_TIMEOUT_MS,
-		});
-	}
-
-	close(): void {
-		if (!this.db.isOpen) {
-			return;
-		}
-		this.db.close();
-	}
-
-	exec(sql: string): void {
-		this.db.exec(sql);
-	}
-
-	pragma(sql: string, options: PragmaOptions = {}): unknown {
-		const rows = this.prepare(`pragma ${sql}`).all() as Array<
-			Record<string, unknown>
-		>;
-		if (!options.simple) {
-			return rows;
-		}
-		const first = rows[0];
-		return first ? Object.values(first)[0] : undefined;
-	}
-
-	prepare(sql: string): NativeSqliteStatement {
-		return new NativeSqliteStatement(
-			this.db.prepare(sql),
-			sql,
-			this.options.onStatement,
-		);
-	}
-
-	transaction<TArgs extends unknown[], TResult>(
-		fn: (...args: TArgs) => TResult,
-	): (...args: TArgs) => TResult {
-		return (...args: TArgs) => {
-			const nested = this.db.isTransaction;
-			const savepoint = `__birdclaw_tx_${++this.transactionDepth}`;
-			this.exec(nested ? `savepoint ${savepoint}` : "begin immediate");
-			try {
-				const result = fn(...args);
-				this.exec(nested ? `release ${savepoint}` : "commit");
-				return result;
-			} catch (error) {
-				if (nested) {
-					this.exec(`rollback to ${savepoint}`);
-					this.exec(`release ${savepoint}`);
-				} else {
-					this.exec("rollback");
-				}
-				throw error;
-			}
-		};
-	}
-}
-
-export default NativeSqliteDatabase;

package/src/lib/streaming-ingestion.ts

@@ -1,240 +0,0 @@
-import { createReadStream } from "node:fs";
-import { createInterface } from "node:readline";
-import { Effect } from "effect";
-
-export interface IngestionCheckpoint {
-	processed: number;
-}
-
-export interface IngestionSource<T> {
-	id: string;
-	stream: () => AsyncIterable<T>;
-}
-
-export interface IngestionSourceCheckpoint extends IngestionCheckpoint {
-	sourceId: string;
-	sourceIndex: number;
-	sourceProcessed: number;
-	sources: number;
-}
-
-export async function* streamJsonLines(
-	filePath: string,
-): AsyncGenerator<{ lineNumber: number; value: Record<string, unknown> }> {
-	const input = createReadStream(filePath, { encoding: "utf8" });
-	const lines = createInterface({ input, crlfDelay: Infinity });
-	let lineNumber = 0;
-	for await (const line of lines) {
-		lineNumber += 1;
-		if (!line.trim()) continue;
-		yield {
-			lineNumber,
-			value: JSON.parse(line) as Record<string, unknown>,
-		};
-	}
-}
-
-export async function* streamAssignedJsonArray(
-	source: AsyncIterable<Buffer | string>,
-): AsyncGenerator<Record<string, unknown>> {
-	let started = false;
-	let item = "";
-	let depth = 0;
-	let inString = false;
-	let escaped = false;
-
-	const flush = () => {
-		const value = item.trim();
-		item = "";
-		return value ? (JSON.parse(value) as Record<string, unknown>) : undefined;
-	};
-
-	for await (const chunk of source) {
-		for (const character of String(chunk)) {
-			if (!started) {
-				if (character === "[") started = true;
-				continue;
-			}
-
-			if (inString) {
-				item += character;
-				if (escaped) {
-					escaped = false;
-				} else if (character === "\\") {
-					escaped = true;
-				} else if (character === '"') {
-					inString = false;
-				}
-				continue;
-			}
-
-			if (character === '"') {
-				inString = true;
-				item += character;
-				continue;
-			}
-			if (character === "{" || character === "[") {
-				depth += 1;
-				item += character;
-				continue;
-			}
-			if (character === "}" || (character === "]" && depth > 0)) {
-				depth -= 1;
-				item += character;
-				continue;
-			}
-			if (character === "]" && depth === 0) {
-				const value = flush();
-				if (value) yield value;
-				return;
-			}
-			if (character === "," && depth === 0) {
-				const value = flush();
-				if (value) yield value;
-				continue;
-			}
-			if (item.length > 0 || !/\s/.test(character)) {
-				item += character;
-			}
-		}
-	}
-
-	if (!started) return;
-	const value = flush();
-	if (value) yield value;
-}
-
-export function ingestStreamInBatchesEffect<T>({
-	batchSize = 250,
-	onCheckpoint,
-	processBatch,
-	resumeAfter = 0,
-	source,
-}: {
-	batchSize?: number;
-	onCheckpoint?: (checkpoint: IngestionCheckpoint) => void | Promise<void>;
-	processBatch: (
-		batch: T[],
-		checkpoint: IngestionCheckpoint,
-	) => void | Promise<void>;
-	resumeAfter?: number;
-	source: () => AsyncIterable<T>;
-}): Effect.Effect<IngestionCheckpoint, unknown> {
-	return Effect.tryPromise({
-		try: async () => {
-			const normalizedBatchSize = Math.max(1, Math.trunc(batchSize));
-			const normalizedResumeAfter = Math.max(0, Math.trunc(resumeAfter));
-			let visited = 0;
-			let processed = normalizedResumeAfter;
-			let batch: T[] = [];
-
-			const flush = async () => {
-				if (batch.length === 0) return;
-				processed += batch.length;
-				const checkpoint = { processed };
-				await processBatch(batch, checkpoint);
-				await onCheckpoint?.(checkpoint);
-				batch = [];
-			};
-
-			for await (const item of source()) {
-				visited += 1;
-				if (visited <= normalizedResumeAfter) continue;
-				batch.push(item);
-				if (batch.length >= normalizedBatchSize) {
-					await flush();
-				}
-			}
-			await flush();
-			return { processed };
-		},
-		catch: (error) =>
-			error instanceof Error ? error : new Error(String(error)),
-	});
-}
-
-export function ingestSourcesInBatchesEffect<T>({
-	batchSize,
-	onCheckpoint,
-	onSourceComplete,
-	processBatch,
-	sources,
-}: {
-	batchSize?: number;
-	onCheckpoint?: (
-		checkpoint: IngestionSourceCheckpoint,
-	) => void | Promise<void>;
-	onSourceComplete?: (
-		checkpoint: IngestionSourceCheckpoint,
-	) => void | Promise<void>;
-	processBatch: (
-		batch: T[],
-		checkpoint: IngestionSourceCheckpoint,
-	) => void | Promise<void>;
-	sources: IngestionSource<T>[];
-}): Effect.Effect<IngestionCheckpoint, unknown> {
-	return Effect.gen(function* () {
-		let processed = 0;
-
-		for (const [sourceIndex, source] of sources.entries()) {
-			const sourceResult = yield* ingestStreamInBatchesEffect({
-				batchSize,
-				source: source.stream,
-				processBatch: (batch, checkpoint) =>
-					processBatch(batch, {
-						...checkpoint,
-						processed: processed + checkpoint.processed,
-						sourceId: source.id,
-						sourceIndex,
-						sourceProcessed: checkpoint.processed,
-						sources: sources.length,
-					}),
-				onCheckpoint: onCheckpoint
-					? (checkpoint) =>
-							onCheckpoint({
-								...checkpoint,
-								processed: processed + checkpoint.processed,
-								sourceId: source.id,
-								sourceIndex,
-								sourceProcessed: checkpoint.processed,
-								sources: sources.length,
-							})
-					: undefined,
-			});
-			processed += sourceResult.processed;
-			yield* Effect.tryPromise({
-				try: () =>
-					Promise.resolve(
-						onSourceComplete?.({
-							processed,
-							sourceId: source.id,
-							sourceIndex,
-							sourceProcessed: sourceResult.processed,
-							sources: sources.length,
-						}),
-					),
-				catch: (error) =>
-					error instanceof Error ? error : new Error(String(error)),
-			});
-		}
-
-		return { processed };
-	});
-}
-
-export function collectIngestionSourcesEffect<T>(
-	sources: IngestionSource<T>[],
-	options: { batchSize?: number } = {},
-): Effect.Effect<T[], unknown> {
-	return Effect.gen(function* () {
-		const rows: T[] = [];
-		yield* ingestSourcesInBatchesEffect({
-			...options,
-			sources,
-			processBatch: (batch) => {
-				rows.push(...batch);
-			},
-		});
-		return rows;
-	});
-}

package/src/lib/sync-cache.ts

@@ -1,70 +0,0 @@
-import { getNativeDb } from "./db";
-import {
-	defaultServerRuntimeServices,
-	type ServerRuntimeServices,
-} from "./server-runtime-services";
-
-export interface SyncCacheEntry<T> {
-	value: T;
-	updatedAt: string;
-}
-
-function readSyncCacheRow(cacheKey: string, db = getNativeDb()) {
-	return db
-		.prepare(
-			`
-      select value_json, updated_at
-      from sync_cache
-      where cache_key = ?
-      `,
-		)
-		.get(cacheKey) as
-		| {
-				value_json: string;
-				updated_at: string;
-		  }
-		| undefined;
-}
-
-export function readSyncCache<T>(
-	cacheKey: string,
-	db = getNativeDb(),
-): SyncCacheEntry<T> | null {
-	const row = readSyncCacheRow(cacheKey, db);
-	if (!row) {
-		return null;
-	}
-
-	try {
-		return {
-			value: JSON.parse(row.value_json) as T,
-			updatedAt: row.updated_at,
-		};
-	} catch {
-		return null;
-	}
-}
-
-export function writeSyncCache(
-	cacheKey: string,
-	value: unknown,
-	db = getNativeDb(),
-	runtime: ServerRuntimeServices = defaultServerRuntimeServices,
-) {
-	const updatedAt = runtime.now().toISOString();
-	db.prepare(
-		`
-    insert into sync_cache (cache_key, value_json, updated_at)
-    values (?, ?, ?)
-    on conflict(cache_key) do update set
-      value_json = excluded.value_json,
-      updated_at = excluded.updated_at
-    `,
-	).run(cacheKey, JSON.stringify(value), updatedAt);
-
-	return updatedAt;
-}
-
-export function deleteSyncCache(cacheKey: string, db = getNativeDb()) {
-	db.prepare("delete from sync_cache where cache_key = ?").run(cacheKey);
-}

package/src/lib/sync-plan.ts

@@ -1,175 +0,0 @@
-import { Effect } from "effect";
-
-export type PaginationStopReason =
-	| "boundary"
-	| "exhausted"
-	| "item-limit"
-	| "page-limit"
-	| "repeated-cursor";
-
-export interface PaginationPageContext<Page> {
-	cursor?: string;
-	fetched: number;
-	page: Page;
-	pageIndex: number;
-	pageNumber: number;
-	stopReason?: PaginationStopReason;
-	done: boolean;
-}
-
-export type SyncPlanStopReason = PaginationStopReason | "error";
-
-export interface SyncPlanPageContext<Page> extends Omit<
-	PaginationPageContext<Page>,
-	"stopReason"
-> {
-	nextCursor?: string;
-	stopReason?: SyncPlanStopReason;
-}
-
-export interface SyncPlanResult<Page, ErrorType> {
-	complete: boolean;
-	fetched: number;
-	nextCursor?: string;
-	pages: Page[];
-	stopReason: SyncPlanStopReason;
-	error?: ErrorType;
-}
-
-function normalizeCursor(value: string | null | undefined) {
-	const normalized = value?.trim();
-	return normalized || undefined;
-}
-
-export function runSyncPlanEffect<Page, FetchError, PageError = never>({
-	fetchPage,
-	getItemCount,
-	getNextCursor,
-	initialCursor,
-	maxItems,
-	maxPages,
-	onPage,
-	pageDelayMs,
-	persistPage,
-	shouldStop,
-	allowPartialFailure = false,
-}: {
-	fetchPage: (context: {
-		cursor?: string;
-		fetched: number;
-		pageIndex: number;
-	}) => Effect.Effect<Page, FetchError>;
-	getItemCount?: (page: Page) => number;
-	getNextCursor: (page: Page) => string | null | undefined;
-	initialCursor?: string;
-	maxItems?: number;
-	maxPages?: number;
-	onPage?: (context: SyncPlanPageContext<Page>) => void;
-	pageDelayMs?: number;
-	persistPage?: (
-		context: SyncPlanPageContext<Page>,
-	) => Effect.Effect<void, PageError>;
-	shouldStop?: (
-		context: Omit<SyncPlanPageContext<Page>, "done" | "nextCursor">,
-	) => boolean;
-	allowPartialFailure?: boolean;
-}): Effect.Effect<SyncPlanResult<Page, FetchError>, FetchError | PageError> {
-	return Effect.gen(function* () {
-		const pages: Page[] = [];
-		const seenCursors = new Set<string>();
-		let cursor = normalizeCursor(initialCursor);
-		let fetched = 0;
-		const pageLimit =
-			maxPages === undefined ? Number.POSITIVE_INFINITY : Math.max(1, maxPages);
-		const itemLimit =
-			maxItems === undefined ? Number.POSITIVE_INFINITY : Math.max(1, maxItems);
-
-		if (cursor) seenCursors.add(cursor);
-
-		while (pages.length < pageLimit) {
-			const pageIndex = pages.length;
-			const outcome = yield* fetchPage({ cursor, fetched, pageIndex }).pipe(
-				Effect.map((page) => ({ ok: true as const, page })),
-				Effect.catchAll((error) =>
-					Effect.succeed({ ok: false as const, error }),
-				),
-			);
-			if (!outcome.ok) {
-				if (!allowPartialFailure || pages.length === 0) {
-					return yield* Effect.fail(outcome.error);
-				}
-				return {
-					complete: false,
-					fetched,
-					...(cursor ? { nextCursor: cursor } : {}),
-					pages,
-					stopReason: "error",
-					error: outcome.error,
-				};
-			}
-
-			const page = outcome.page;
-			pages.push(page);
-			fetched += Math.max(0, getItemCount?.(page) ?? 0);
-			const nextCursor = normalizeCursor(getNextCursor(page));
-			const baseContext = {
-				cursor,
-				fetched,
-				page,
-				pageIndex,
-				pageNumber: pageIndex + 1,
-			};
-			let stopReason: PaginationStopReason | undefined;
-
-			if (!nextCursor) {
-				stopReason = "exhausted";
-			} else if (shouldStop?.(baseContext)) {
-				stopReason = "boundary";
-			} else if (fetched >= itemLimit) {
-				stopReason = "item-limit";
-			} else if (pages.length >= pageLimit) {
-				stopReason = "page-limit";
-			} else if (seenCursors.has(nextCursor)) {
-				stopReason = "repeated-cursor";
-			}
-
-			const context: SyncPlanPageContext<Page> = {
-				...baseContext,
-				...(nextCursor ? { nextCursor } : {}),
-				done: Boolean(stopReason),
-				...(stopReason ? { stopReason } : {}),
-			};
-			if (persistPage) {
-				yield* persistPage(context);
-			}
-			onPage?.(context);
-
-			if (stopReason) {
-				return {
-					complete:
-						stopReason === "exhausted" ||
-						stopReason === "boundary" ||
-						stopReason === "item-limit",
-					fetched,
-					...(nextCursor ? { nextCursor } : {}),
-					pages,
-					stopReason,
-				};
-			}
-
-			cursor = nextCursor;
-			seenCursors.add(nextCursor!);
-			if (typeof pageDelayMs === "number" && pageDelayMs > 0) {
-				yield* Effect.sleep(pageDelayMs);
-			}
-		}
-
-		return {
-			complete: false,
-			fetched,
-			...(cursor ? { nextCursor: cursor } : {}),
-			pages,
-			stopReason: "page-limit",
-		};
-	});
-}