@oh-my-pi/pi-utils 18.2.1 → 18.2.3

package/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## [Unreleased]
 
+## [18.2.3] - 2026-09-17
+
+### Fixed
+
+- Optimized model configuration command execution by deduplicating requests and adding failure backoff
+- Prevented unnecessary credential command execution when runtime API keys are configured
+- Retained `readLines()` results no longer change when later chunks reuse the internal buffer.
+- Long sleeps honor elapsed time and re-arm after premature timer wakes without overflowing native timer delays.
+
+## [18.2.2] - 2026-09-16
+
+### Added
+
+- Added asynchronous and synchronous SQLite database opening APIs with path-attributed errors, optional corruption recovery that preserves private database and sidecar backups, and automatic retries for transient busy errors during asynchronous opens.
+
 ## [18.2.1] - 2026-09-15
 
 ### Added

package/dist/types/async.d.ts

@@ -12,15 +12,8 @@ export declare const MAX_TIMER_DELAY_MS = 2147483647;
  * so no single timer overflows; an abort during any chunk rejects like
  * `scheduler.wait`.
  *
- * The remainder is deliberately consumed by chunk, not recomputed from a
- * monotonic deadline: deadline tracking never terminates under the repo's
- * instant `scheduler.wait` mocks (retry-cap suites spy it to resolve
- * immediately, so `deadline - performance.now()` never reaches zero and the
- * loop spins forever). A premature native wake (Bun `uv_async_send`, see
- * `sleepAtLeast` in `packages/agent/src/utils/yield.ts`) can therefore
- * under-wait by the unelapsed chunk time — but that self-corrects downstream:
- * credential blocks carry the true deadline independently of this sleep, so
- * an early retry re-hits 429 and re-sleeps on a fresh server hint.
+ * Uses a monotonic deadline so a timer that wakes prematurely is re-armed for
+ * the unelapsed duration instead of shortening the requested sleep.
  */
 export declare function sleepLong(delayMs: number, signal?: AbortSignal): Promise<void>;
 /**

package/dist/types/sqlite.d.ts

@@ -1,13 +1,33 @@
+/** Shared SQLite opening, error attribution, and result-code classification for persistent stores. */
+import { Database } from "bun:sqlite";
+/** Controls opt-in replacement of an unrecoverably corrupt SQLite store. */
+export interface SqliteOpenOptions {
+    /**
+     * Preserve a corrupt store and its sidecars, recreate it, and run the
+     * initializer once more. Disabled by default.
+     */
+    recoverCorruption?: boolean;
+    /** Runs after preservation and before the replacement is initialized. */
+    onCorruptionPreserved?: (backupPath: string, error: unknown) => void;
+}
 /**
- * Shared classifiers for `bun:sqlite` error result codes.
+ * Opens and initializes a store, retrying BUSY failures up to four total attempts.
+ * Installs the busy handler before initialization and closes failed connections.
+ * The initializer may run again on a fresh connection; on success it owns the handle.
  *
- * Every omp SQLite store (`agent.db` credential/usage store, `models.db` model
- * cache, `history.db`) needs the same two distinctions: a transient BUSY that
- * clears by retrying, and an unrecoverable corruption that never does. Keeping
- * one implementation here prevents the classifiers from drifting between the
- * credential store and the model cache.
+ * With corruption recovery enabled, recovery is serialized across processes.
+ * The identity observed by the failed handle is checked under that lock, so a
+ * waiter adopts a replacement made by a peer instead of quarantining it.
+ * Final failures retain their SQLite codes and include the database path.
  */
-import type { Database } from "bun:sqlite";
+export declare function openSqliteDatabase<T>(dbPath: string, initialize: (db: Database) => T | Promise<T>, options?: SqliteOpenOptions): Promise<T>;
+/**
+ * Synchronous counterpart to {@link openSqliteDatabase}. It performs no BUSY
+ * retry loop; corruption recovery, when enabled, is bounded to one replacement.
+ */
+export declare function openSqliteDatabaseSync<T>(dbPath: string, initialize: (db: Database) => T, options?: SqliteOpenOptions): T;
+/** Adds the failing store's path to an error without losing SQLite result codes or its original stack. */
+export declare function annotateSqliteError(error: unknown, dbPath: string): Error;
 /** Checkpoints committed WAL frames without waiting for concurrent readers. */
 export declare function checkpointWal(db: Database): void;
 /**

package/dist/types/stream.d.ts

@@ -1,3 +1,9 @@
+/**
+ * Split a byte stream on LF boundaries.
+ *
+ * Every yielded line owns its bytes and remains unchanged after the generator
+ * advances or drains. Line terminators are excluded.
+ */
 export declare function readLines(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<Uint8Array>;
 export declare function readJsonl<T>(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<T>;
 /**
@@ -22,7 +28,13 @@ export declare class ConcatSink {
     /** Drop the first `count` buffered bytes, keeping the remainder. */
     consume(count: number): void;
     clear(): void;
-    appendAndFlushLines(chunk: Uint8Array): Generator<Uint8Array<ArrayBufferLike>, void, unknown>;
+    /**
+     * Append a chunk and yield each complete LF-delimited line.
+     *
+     * Yielded lines are owned snapshots. Unlike {@link flush}, they remain
+     * valid after this sink or the input chunk is mutated.
+     */
+    appendAndFlushLines(chunk: Uint8Array): Generator<Uint8Array>;
     appendAndFlushText(chunk: Uint8Array, decoder: TextDecoder): string | undefined;
     pullJSONL<T>(chunk: Uint8Array, beg: number, end: number): Generator<T, void, unknown>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@oh-my-pi/pi-utils",
-	"version": "18.2.1",
+	"version": "18.2.3",
 	"description": "Shared utilities for pi packages",
 	"keywords": [
 		"cli",
@@ -54,7 +54,7 @@
 		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-natives": "18.2.1"
+		"@oh-my-pi/pi-natives": "18.2.3"
 	},
 	"devDependencies": {
 		"@types/bun": "^1.3.14"

package/src/async.ts

@@ -15,22 +15,16 @@ export const MAX_TIMER_DELAY_MS = 2_147_483_647;
  * so no single timer overflows; an abort during any chunk rejects like
  * `scheduler.wait`.
  *
- * The remainder is deliberately consumed by chunk, not recomputed from a
- * monotonic deadline: deadline tracking never terminates under the repo's
- * instant `scheduler.wait` mocks (retry-cap suites spy it to resolve
- * immediately, so `deadline - performance.now()` never reaches zero and the
- * loop spins forever). A premature native wake (Bun `uv_async_send`, see
- * `sleepAtLeast` in `packages/agent/src/utils/yield.ts`) can therefore
- * under-wait by the unelapsed chunk time — but that self-corrects downstream:
- * credential blocks carry the true deadline independently of this sleep, so
- * an early retry re-hits 429 and re-sleeps on a fresh server hint.
+ * Uses a monotonic deadline so a timer that wakes prematurely is re-armed for
+ * the unelapsed duration instead of shortening the requested sleep.
  */
 export async function sleepLong(delayMs: number, signal?: AbortSignal): Promise<void> {
 	signal?.throwIfAborted();
-	let remaining = delayMs;
-	while (remaining > 0) {
+	const deadline = performance.now() + delayMs;
+	while (true) {
+		const remaining = deadline - performance.now();
+		if (!(remaining > 0)) return;
 		await scheduler.wait(Math.min(remaining, MAX_TIMER_DELAY_MS), { signal });
-		remaining -= MAX_TIMER_DELAY_MS;
 		signal?.throwIfAborted();
 	}
 }

package/src/sqlite.ts

@@ -1,13 +1,248 @@
+/** Shared SQLite opening, error attribution, and result-code classification for persistent stores. */
+import { Database } from "bun:sqlite";
+import * as fs from "node:fs";
+import { getDbBusyTimeoutMs } from "./env";
+import { withFileLockSync } from "./file-lock";
+import { isEnoent } from "./fs-error";
+import * as logger from "./logger";
+
+const BUSY_MAX_ATTEMPTS = 4;
+const BUSY_BASE_DELAY_MS = 100;
+const SQLITE_STORE_SUFFIXES = ["-wal", "-shm", "-journal", ""];
+
+type SqliteFileIdentity = string | null | undefined;
+
+class SqliteAttemptFailure extends Error {
+	readonly original: unknown;
+	readonly identity: SqliteFileIdentity;
+	readonly canRecover: boolean;
+	readonly db?: Database;
+
+	constructor(original: unknown, identity: SqliteFileIdentity, options: { canRecover?: boolean; db?: Database } = {}) {
+		super(original instanceof Error ? original.message : String(original));
+		this.original = original;
+		this.identity = identity;
+		this.canRecover = options.canRecover ?? true;
+		this.db = options.db;
+	}
+}
+
+function sqliteFileIdentity(dbPath: string): SqliteFileIdentity {
+	try {
+		const stat = fs.statSync(dbPath);
+		return `${stat.dev}:${stat.ino}:${stat.birthtimeMs}`;
+	} catch (error) {
+		return isEnoent(error) ? null : undefined;
+	}
+}
+
+function closeFailedDatabase(db: Database | undefined, error: unknown, identity: SqliteFileIdentity): void {
+	try {
+		db?.close();
+	} catch (closeError) {
+		const original = error instanceof Error ? error : new Error(String(error));
+		const detail = closeError instanceof Error ? closeError.message : String(closeError);
+		original.message += `; failed to close the SQLite handle: ${detail}`;
+		throw new SqliteAttemptFailure(original, identity, { canRecover: false });
+	}
+}
+
+/** Controls opt-in replacement of an unrecoverably corrupt SQLite store. */
+export interface SqliteOpenOptions {
+	/**
+	 * Preserve a corrupt store and its sidecars, recreate it, and run the
+	 * initializer once more. Disabled by default.
+	 */
+	recoverCorruption?: boolean;
+	/** Runs after preservation and before the replacement is initialized. */
+	onCorruptionPreserved?: (backupPath: string, error: unknown) => void;
+}
+
+async function openWithBusyRetries<T>(
+	dbPath: string,
+	initialize: (db: Database) => T | Promise<T>,
+	options: SqliteOpenOptions,
+): Promise<T> {
+	for (let attempt = 0; ; attempt++) {
+		let db: Database | undefined;
+		const identity = sqliteFileIdentity(dbPath);
+		try {
+			db = new Database(dbPath);
+			// WAL recovery can bypass the busy handler; both it and retries are needed (#2421).
+			db.run(`PRAGMA busy_timeout = ${getDbBusyTimeoutMs()}`);
+			return await initialize(db);
+		} catch (error) {
+			if (options.recoverCorruption && isSqliteCorruptionError(error)) {
+				throw new SqliteAttemptFailure(error, identity, { db });
+			}
+			closeFailedDatabase(db, error, identity);
+			if (!isSqliteBusyError(error) || attempt + 1 >= BUSY_MAX_ATTEMPTS) {
+				throw new SqliteAttemptFailure(error, identity);
+			}
+			await Bun.sleep(BUSY_BASE_DELAY_MS * 2 ** attempt);
+		}
+	}
+}
+
+function openOnce<T>(dbPath: string, initialize: (db: Database) => T, options: SqliteOpenOptions): T {
+	let db: Database | undefined;
+	const identity = sqliteFileIdentity(dbPath);
+	try {
+		db = new Database(dbPath);
+		db.run(`PRAGMA busy_timeout = ${getDbBusyTimeoutMs()}`);
+		return initialize(db);
+	} catch (error) {
+		if (options.recoverCorruption && isSqliteCorruptionError(error)) {
+			throw new SqliteAttemptFailure(error, identity, { db });
+		}
+		closeFailedDatabase(db, error, identity);
+		throw new SqliteAttemptFailure(error, identity);
+	}
+}
+
+function quarantineCorruptSqliteStore(dbPath: string, db: Database | undefined): string {
+	const backupPath = `${dbPath}.corrupt-${Date.now()}-${crypto.randomUUID()}`;
+	const preserved: string[] = [];
+	// Closing a failed WAL connection can truncate its WAL. Copy evidence
+	// before closing, and remove originals only after every copy succeeds.
+	for (const suffix of SQLITE_STORE_SUFFIXES) {
+		try {
+			fs.chmodSync(`${dbPath}${suffix}`, 0o600);
+			fs.copyFileSync(`${dbPath}${suffix}`, `${backupPath}${suffix}`, fs.constants.COPYFILE_EXCL);
+			preserved.push(suffix);
+		} catch (error) {
+			if (isEnoent(error) && suffix !== "") continue;
+			throw error;
+		}
+	}
+	db?.close();
+
+	const removed: string[] = [];
+	try {
+		// Remove the main file last so a failed sidecar removal cannot leave
+		// a path at which another startup creates an empty database.
+		for (const suffix of preserved) {
+			try {
+				fs.unlinkSync(`${dbPath}${suffix}`);
+				removed.push(suffix);
+			} catch (error) {
+				if (!isEnoent(error)) throw error;
+			}
+		}
+	} catch (error) {
+		for (const suffix of removed) {
+			try {
+				fs.copyFileSync(`${backupPath}${suffix}`, `${dbPath}${suffix}`, fs.constants.COPYFILE_EXCL);
+			} catch (rollbackError) {
+				logger.error("SQLite quarantine rollback failed; original preserved at backup path", {
+					path: `${dbPath}${suffix}`,
+					backupPath: `${backupPath}${suffix}`,
+					error: String(rollbackError),
+				});
+			}
+		}
+		throw error;
+	}
+	return backupPath;
+}
+
+function corruptionPreservationError(corruption: unknown, dbPath: string, preservationError: unknown): Error {
+	const annotated = annotateSqliteError(corruption, dbPath);
+	const detail = preservationError instanceof Error ? preservationError.message : String(preservationError);
+	annotated.message += `; failed to preserve the corrupt database: ${detail}`;
+	return annotated;
+}
+
+function recoverCorruptDatabase(dbPath: string, error: unknown, options: SqliteOpenOptions): void {
+	if (!(error instanceof SqliteAttemptFailure)) throw annotateSqliteError(error, dbPath);
+	const failure = error;
+	if (!options.recoverCorruption || !failure.canRecover || !isSqliteCorruptionError(failure.original)) {
+		throw annotateSqliteError(failure.original, dbPath);
+	}
+
+	let backupPath: string | null;
+	try {
+		try {
+			backupPath = withFileLockSync(`${dbPath}.recovery`, () => {
+				const currentIdentity = sqliteFileIdentity(dbPath);
+				if (failure.identity === undefined || currentIdentity === undefined) {
+					throw new Error("could not verify the corrupt database file identity");
+				}
+				if (currentIdentity !== failure.identity) return null;
+				return quarantineCorruptSqliteStore(dbPath, failure.db);
+			});
+		} finally {
+			closeFailedDatabase(failure.db, failure.original, failure.identity);
+		}
+	} catch (preservationError) {
+		throw corruptionPreservationError(failure.original, dbPath, preservationError);
+	}
+
+	if (backupPath === null) return;
+	logger.warn("SQLite database corrupt; preserved damaged store before recreating it", {
+		path: dbPath,
+		backupPath,
+		warning: "Stored credentials from this database may require re-login.",
+	});
+	options.onCorruptionPreserved?.(backupPath, failure.original);
+}
+
 /**
- * Shared classifiers for `bun:sqlite` error result codes.
+ * Opens and initializes a store, retrying BUSY failures up to four total attempts.
+ * Installs the busy handler before initialization and closes failed connections.
+ * The initializer may run again on a fresh connection; on success it owns the handle.
  *
- * Every omp SQLite store (`agent.db` credential/usage store, `models.db` model
- * cache, `history.db`) needs the same two distinctions: a transient BUSY that
- * clears by retrying, and an unrecoverable corruption that never does. Keeping
- * one implementation here prevents the classifiers from drifting between the
- * credential store and the model cache.
+ * With corruption recovery enabled, recovery is serialized across processes.
+ * The identity observed by the failed handle is checked under that lock, so a
+ * waiter adopts a replacement made by a peer instead of quarantining it.
+ * Final failures retain their SQLite codes and include the database path.
  */
-import type { Database } from "bun:sqlite";
+export async function openSqliteDatabase<T>(
+	dbPath: string,
+	initialize: (db: Database) => T | Promise<T>,
+	options: SqliteOpenOptions = {},
+): Promise<T> {
+	try {
+		return await openWithBusyRetries(dbPath, initialize, options);
+	} catch (error) {
+		recoverCorruptDatabase(dbPath, error, options);
+	}
+
+	try {
+		return await openWithBusyRetries(dbPath, initialize, {});
+	} catch (error) {
+		throw annotateSqliteError(error instanceof SqliteAttemptFailure ? error.original : error, dbPath);
+	}
+}
+
+/**
+ * Synchronous counterpart to {@link openSqliteDatabase}. It performs no BUSY
+ * retry loop; corruption recovery, when enabled, is bounded to one replacement.
+ */
+export function openSqliteDatabaseSync<T>(
+	dbPath: string,
+	initialize: (db: Database) => T,
+	options: SqliteOpenOptions = {},
+): T {
+	try {
+		return openOnce(dbPath, initialize, options);
+	} catch (error) {
+		recoverCorruptDatabase(dbPath, error, options);
+	}
+
+	try {
+		return openOnce(dbPath, initialize, {});
+	} catch (error) {
+		throw annotateSqliteError(error instanceof SqliteAttemptFailure ? error.original : error, dbPath);
+	}
+}
+
+/** Adds the failing store's path to an error without losing SQLite result codes or its original stack. */
+export function annotateSqliteError(error: unknown, dbPath: string): Error {
+	const annotated = error instanceof Error ? error : new Error(String(error));
+	annotated.message = `Database ${JSON.stringify(dbPath)}: ${annotated.message}`;
+	return annotated;
+}
 
 /** Checkpoints committed WAL frames without waiting for concurrent readers. */
 export function checkpointWal(db: Database): void {

package/src/stream.ts

@@ -6,6 +6,12 @@ import { parseStreamingJson } from "./json-parse";
 const LF = 0x0a;
 const CR = 0x0d;
 
+/**
+ * Split a byte stream on LF boundaries.
+ *
+ * Every yielded line owns its bytes and remains unchanged after the generator
+ * advances or drains. Line terminators are excluded.
+ */
 export async function* readLines(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<Uint8Array> {
 	const buffer = new ConcatSink();
 	const source = abortableSource(stream, signal);
@@ -131,7 +137,13 @@ export class ConcatSink {
 		this.#length = 0;
 	}
 
-	*appendAndFlushLines(chunk: Uint8Array) {
+	/**
+	 * Append a chunk and yield each complete LF-delimited line.
+	 *
+	 * Yielded lines are owned snapshots. Unlike {@link flush}, they remain
+	 * valid after this sink or the input chunk is mutated.
+	 */
+	*appendAndFlushLines(chunk: Uint8Array): Generator<Uint8Array> {
 		let pos = 0;
 		while (pos < chunk.length) {
 			const nl = chunk.indexOf(LF, pos);
@@ -142,12 +154,12 @@ export class ConcatSink {
 			const suffix = chunk.subarray(pos, nl);
 			pos = nl + 1;
 			if (this.isEmpty) {
-				yield suffix;
+				yield new Uint8Array(suffix);
 			} else {
 				this.append(suffix);
 				const payload = this.flush();
 				if (payload) {
-					yield payload;
+					yield new Uint8Array(payload);
 					this.clear();
 				}
 			}