@harperfast/harper 5.2.0-beta.4 → 5.2.0

package/dataLayer/blobBackup.ts

@@ -0,0 +1,286 @@
+'use strict';
+
+import { existsSync } from 'node:fs';
+import { copyFile, link, mkdir, readdir, rename, rm, unlink, writeFile } from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+import { ClientError } from '../utility/errors/hdbError.ts';
+import logger from '../utility/logging/harper_logger.ts';
+
+/**
+ * Managed-backup snapshotting of a database's file-backed blobs.
+ *
+ * A database's blobs live in one or more root directories *outside* the RocksDB directory (see
+ * `resources/blob.ts` `getBlobPathsForDatabaseName`), so the engine's own backup does not capture
+ * them. These helpers snapshot the blob roots alongside a RocksDB managed backup and restore them
+ * with it, so a restored database's records still resolve their file-backed blobs.
+ *
+ * Layout mirrors the binding's transaction-log snapshots (`<backupDir>/transaction_logs/<id>/`): a
+ * full, non-incremental copy per backup at `<backupDir>/blobs/<backupId>/<rootIndex>/<relpath>`,
+ * where `rootIndex` is the position of the source root in the database's blob-root list (preserved
+ * so a multi-root database restores each root back to its original slot). Files are hard-linked when
+ * possible (cheap, no extra space on the same filesystem) and copied otherwise (never symlinked, so
+ * a snapshot is a standalone set of files that survives independent of the live blob).
+ *
+ * Consistency is best-effort and point-in-time-ish, matching how the engine treats the transaction
+ * log: the walk captures whatever files exist at snapshot time. A blob deleted mid-walk is skipped;
+ * a blob being written mid-walk is captured as-is (a hard link shares the inode, so it reflects the
+ * writer's final bytes; a cross-filesystem copy captures the bytes present at copy time). Harper
+ * does not freeze blob writes for the duration of a backup.
+ *
+ * Hard-linking is safe against later mutation because Harper blobs are content-addressed and
+ * write-once: each write allocates a fresh monotonic file id (a new path), and an update or delete
+ * unlinks the old path rather than rewriting it in place — so a snapshot's hard link keeps the exact
+ * bytes alive even after the live blob is deleted, and no in-place overwrite can retroactively alter
+ * a snapshot.
+ */
+
+/** Directory holding all blob snapshots for a backup repository. */
+export function blobsRootDir(backupDir: string): string {
+	return join(backupDir, 'blobs');
+}
+
+/** Directory holding a single backup id's blob snapshot. */
+export function blobSnapshotDir(backupDir: string, backupId: number): string {
+	return join(blobsRootDir(backupDir), String(backupId));
+}
+
+/**
+ * Hard-link `src` to `dest`, falling back to a copy when the two are on different filesystems (or
+ * the filesystem does not support additional hard links). Never creates a symlink. A source that
+ * vanished mid-walk (a concurrent blob delete) is skipped rather than failing the whole snapshot.
+ */
+async function linkOrCopy(src: string, dest: string): Promise<void> {
+	await mkdir(dirname(dest), { recursive: true });
+	try {
+		await link(src, dest);
+	} catch (error: any) {
+		if (error.code === 'ENOENT') {
+			// src disappeared (concurrent delete) — nothing to snapshot
+			if (!existsSync(src)) return;
+			throw error;
+		}
+		if (
+			error.code === 'EXDEV' || // cross-device link
+			error.code === 'EMLINK' || // link count exhausted
+			error.code === 'EPERM' || // filesystem forbids hard links
+			error.code === 'ENOTSUP' ||
+			error.code === 'EOPNOTSUPP'
+		) {
+			await copyFile(src, dest);
+			return;
+		}
+		if (error.code === 'EEXIST') {
+			await unlink(dest);
+			await linkOrCopy(src, dest);
+			return;
+		}
+		throw error;
+	}
+}
+
+/**
+ * Recursively copy every file under `srcRoot` into `destRoot` (hard-link-else-copy), preserving the
+ * relative directory structure. Missing `srcRoot` is a no-op (a database with no blobs yet).
+ */
+async function copyTree(srcRoot: string, destRoot: string): Promise<void> {
+	if (!existsSync(srcRoot)) return;
+	const stack: string[] = [srcRoot];
+	while (stack.length > 0) {
+		const dir = stack.pop() as string;
+		let entries;
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch (error: any) {
+			if (error.code === 'ENOENT') continue; // directory removed mid-walk
+			throw error;
+		}
+		for (const entry of entries) {
+			const srcPath = join(dir, entry.name);
+			if (entry.isDirectory()) {
+				stack.push(srcPath);
+			} else if (entry.isFile()) {
+				await linkOrCopy(srcPath, join(destRoot, relative(srcRoot, srcPath)));
+			}
+			// symlinks/other node types in a blob root are not expected and are intentionally skipped
+		}
+	}
+}
+
+/**
+ * Snapshot a database's blob roots into a backup's blob directory. Writes to a temporary sibling
+ * and atomically renames into place so a create_backup that fails mid-copy never leaves a partial
+ * `blobs/<backupId>/` that a later restore would trust. Overwrites any pre-existing snapshot for the
+ * same id (create_backup always produces a fresh id, so this only matters on a retried offline run).
+ */
+export async function snapshotBlobs(backupDir: string, backupId: number, blobRoots: string[]): Promise<void> {
+	const finalDir = blobSnapshotDir(backupDir, backupId);
+	const tempDir = join(blobsRootDir(backupDir), `.tmp-${backupId}`);
+	await rm(tempDir, { recursive: true, force: true });
+	await mkdir(tempDir, { recursive: true });
+	try {
+		for (let index = 0; index < blobRoots.length; index++) {
+			await copyTree(blobRoots[index], join(tempDir, String(index)));
+		}
+		await rm(finalDir, { recursive: true, force: true });
+		await rename(tempDir, finalDir);
+	} catch (error) {
+		await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+		throw error;
+	}
+	await writeBlobsReadme(backupDir, blobRoots);
+}
+
+/**
+ * Build the `blobs/README.md` documenting the blob snapshot layout, so an operator inspecting or
+ * hand-recovering a backup can decode the numeric directories. Two variants:
+ * - managed (default): a create_backup repository, where snapshots are keyed by backup id
+ *   (`<backupId>/<rootIndex>/…`) and restore is automatic via `restore_backup`.
+ * - archive (`archive: true`): a downloaded `get_backup` tar, which holds a single snapshot with no
+ *   backup-id level (`<rootIndex>/…`) and is restored by extracting the files back into the roots.
+ */
+export function blobsReadmeContent(blobRoots: string[], { archive = false }: { archive?: boolean } = {}): string {
+	const rootMapping =
+		blobRoots.length > 0 ? blobRoots.map((root, index) => `      ${index} -> ${root}`).join('\n') : '      (none)';
+	const layout = archive
+		? '<rootIndex>/<shard1>/<shard2>/<fileId>'
+		: '<backupId>/<rootIndex>/<shard1>/<shard2>/<fileId>';
+	const intro = archive
+		? `This directory holds this database's file-backed blobs within a downloaded \`get_backup\` archive.
+To restore them, extract each \`<rootIndex>/\` tree back into the matching blob root (see the mapping
+below and ../README.md).`
+		: `This directory holds point-in-time snapshots of this database's file-backed blobs, captured
+alongside each RocksDB managed backup. You do not restore these by hand — \`restore_backup\` puts
+them back automatically (see ../README.md); this file just documents the layout.`;
+	const backupIdBullet = archive
+		? ''
+		: `- **<backupId>** matches the RocksDB backup id (\`harper list_backups\`). Each id is a full,
+  independent snapshot (not incremental).
+`;
+	return `# Harper blob snapshots
+
+${intro}
+
+## Directory layout
+
+    ${layout}
+
+${backupIdBullet}- **<rootIndex>** is which of the database's blob roots the file came from — the index into
+  \`storage.blobPaths[n]\`. When \`storage.blobPaths\` is not configured there is a single default root
+  (\`<rootPath>/blobs/<db>\`) at index 0. Current mapping for this backup:
+
+${rootMapping}
+
+- **<shard1>/<shard2>/<fileId>** is Harper's on-disk blob layout, copied verbatim from the live
+  root: the hex blob file id split into two directory levels plus the file itself (keeping roughly
+  4096 entries per directory). E.g. a blob with id \`0x12345678\` lives at \`12/345/678\`; a short id
+  like \`0xc1a\` lives at \`0/0/c1a\`.
+
+Files are hard links to the live blobs when the backup is on the same filesystem, and copies
+otherwise.
+`;
+}
+
+/**
+ * Write the `blobs/README.md` into a managed backup's `blobs/` directory. Best-effort: a failure to
+ * write the doc must not fail an otherwise-successful backup.
+ */
+export async function writeBlobsReadme(backupDir: string, blobRoots: string[]): Promise<void> {
+	try {
+		await writeFile(join(blobsRootDir(backupDir), 'README.md'), blobsReadmeContent(blobRoots));
+	} catch (error) {
+		logger.warn(`Failed to write blob snapshot README in ${backupDir}: ${(error as Error).message}`);
+	}
+}
+
+/** The blob-root indices present in a snapshot (sorted); [] when the backup has no blob snapshot. */
+async function snapshotRootIndices(snapshotDir: string): Promise<number[]> {
+	if (!existsSync(snapshotDir)) return [];
+	return (await readdir(snapshotDir, { withFileTypes: true }))
+		.filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name))
+		.map((entry) => Number(entry.name))
+		.sort((a, b) => a - b);
+}
+
+/**
+ * Reject (before any destructive step) a restore whose snapshot has more blob roots than the
+ * database is currently configured with. File-backed blob references persist their `storageIndex`,
+ * so a record written under root index 1 keeps resolving through `blobRoots[1]`; collapsing the
+ * out-of-range index onto another root would preserve the bytes at the wrong address and the restore
+ * would "succeed" while reads of those blobs fail. The operator must reconcile `storage.blobPaths`
+ * to at least as many roots as the backup before restoring. Callers invoke this ahead of the engine
+ * restore so a mismatch never purges the database.
+ */
+export async function assertBlobSnapshotRestorable(
+	backupDir: string,
+	backupId: number,
+	blobRoots: string[]
+): Promise<void> {
+	const indices = await snapshotRootIndices(blobSnapshotDir(backupDir, backupId));
+	const maxIndex = indices.length > 0 ? indices[indices.length - 1] : -1;
+	if (maxIndex >= blobRoots.length) {
+		throw new ClientError(
+			`Cannot restore backup ${backupId}: it captured ${maxIndex + 1} blob root(s) but the database is now configured with ${blobRoots.length}. ` +
+				`Blob references persist their root index, so restoring would mis-address blobs — set 'storage.blobPaths' to at least ${maxIndex + 1} root(s) before restoring.`
+		);
+	}
+}
+
+/**
+ * Restore a backup's blob snapshot back into the database's blob roots. Each root is purged and
+ * rewritten from `blobs/<backupId>/<rootIndex>/` so the restored blob set matches the backup exactly
+ * (a newer blob written after the backup is removed, mirroring the engine's `purgeAllFiles` restore).
+ *
+ * A backup created with blobs excluded (or an older backup that predates blob snapshots) has no
+ * snapshot directory: in that case the live blob roots are left untouched and a warning is logged,
+ * since purging them would strip blobs the restored records may still reference. Roots are restored
+ * by index into the *same* configured root; an incompatible root count is rejected up front (see
+ * `assertBlobSnapshotRestorable`) rather than collapsed, so blobs are never mis-addressed.
+ */
+export async function restoreBlobSnapshot(
+	backupDir: string,
+	backupId: number,
+	databaseName: string,
+	blobRoots: string[]
+): Promise<void> {
+	const snapshotDir = blobSnapshotDir(backupDir, backupId);
+	if (!existsSync(snapshotDir)) {
+		logger.warn(
+			`Backup ${backupId} of database '${databaseName}' has no blob snapshot; leaving existing blob files in place (this backup did not include blobs)`
+		);
+		return;
+	}
+	// defense in depth: the restore flow pre-checks before the engine restore, but re-validate here
+	// so this function never mis-addresses blobs regardless of caller
+	await assertBlobSnapshotRestorable(backupDir, backupId, blobRoots);
+	const indexDirs = await snapshotRootIndices(snapshotDir);
+	// purge every current blob root first so nothing newer than the backup survives the restore
+	for (const root of blobRoots) {
+		await rm(root, { recursive: true, force: true });
+	}
+	for (const index of indexDirs) {
+		await copyTree(join(snapshotDir, String(index)), blobRoots[index]);
+	}
+}
+
+/**
+ * Remove the blob snapshot for a single backup id (paired with `backups.delete`). Best-effort:
+ * a missing snapshot directory is fine.
+ */
+export async function deleteBlobSnapshot(backupDir: string, backupId: number): Promise<void> {
+	await rm(blobSnapshotDir(backupDir, backupId), { recursive: true, force: true });
+}
+
+/**
+ * Remove blob snapshots for every backup id not in `keepIds` (paired with `backups.purge`, which
+ * reference-counts and removes the engine files; blob snapshots are full per-id copies, so they are
+ * simply deleted). Given the ids that survive the purge, this deletes the rest.
+ */
+export async function purgeBlobSnapshots(backupDir: string, keepIds: Set<number>): Promise<void> {
+	const root = blobsRootDir(backupDir);
+	if (!existsSync(root)) return;
+	for (const entry of await readdir(root, { withFileTypes: true })) {
+		if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue;
+		if (keepIds.has(Number(entry.name))) continue;
+		await rm(join(root, entry.name), { recursive: true, force: true });
+	}
+}

package/dataLayer/harperBridge/ResourceBridge.ts

@@ -27,6 +27,7 @@ import { errorToString } from '../../utility/logging/harper_logger.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 import { BridgeMethods } from './BridgeMethods.ts';
 import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.js';
+import { createBackupStream, resolveSingleRootStore } from '../rocksdbBackup.ts';
 import { DeleteTransactionLogsBeforeResults } from './DeleteTransactionLogsBeforeResults.ts';
 import type { Readable } from 'node:stream';
 
@@ -563,7 +564,49 @@ export class ResourceBridge extends BridgeMethods {
 		schema?: string;
 		table?: string;
 		tables?: string[];
+		include_audit?: boolean;
+		gzip?: boolean;
+		exclude_blobs?: boolean;
 	}): Promise<Readable> {
+		const databaseName = getBackupObj.database || getBackupObj.schema || 'data';
+		const database = getDatabases()[databaseName];
+		if (!database) {
+			throw new ClientError(`Database '${databaseName}' does not exist`, 404);
+		}
+		const firstTable = database[Object.keys(database)[0]];
+		if (!firstTable) {
+			throw new ClientError(`Database '${databaseName}' has no tables to back up`);
+		}
+		if (firstTable.primaryStore.rootStore instanceof RocksDatabase) {
+			// RocksDB: stream a fresh full-snapshot tar of the database's current state — no
+			// scratch disk, nothing to clean up. Per-engine params are validated descriptively.
+			if (getBackupObj.tables || getBackupObj.table) {
+				throw new ClientError(`'tables'/'table' are LMDB-only options; RocksDB backups are always whole-database`);
+			}
+			if (getBackupObj.include_audit !== undefined) {
+				throw new ClientError(
+					`'include_audit' is an LMDB-only option; RocksDB backups always include the transaction log`
+				);
+			}
+			if (getBackupObj.gzip !== undefined && typeof getBackupObj.gzip !== 'boolean') {
+				throw new ClientError(`'gzip' must be a boolean`);
+			}
+			if (getBackupObj.exclude_blobs !== undefined && typeof getBackupObj.exclude_blobs !== 'boolean') {
+				throw new ClientError(`'exclude_blobs' must be a boolean`);
+			}
+			const rootStore = resolveSingleRootStore(databaseName);
+			// gzip defaults on (it compresses the snapshot substantially); gzip=false opts out.
+			// blobs are included by default; exclude_blobs=true streams an engine-only tar.
+			return createBackupStream(
+				rootStore,
+				databaseName,
+				getBackupObj.gzip !== false,
+				getBackupObj.exclude_blobs === true
+			);
+		}
+		if (getBackupObj.gzip !== undefined) {
+			throw new ClientError(`'gzip' is a RocksDB-only option; LMDB backups are gzipped per the accept-encoding header`);
+		}
 		return lmdbGetBackup(getBackupObj);
 	}
 }

package/dataLayer/restoreMarker.ts

@@ -0,0 +1,276 @@
+'use strict';
+
+import {
+	closeSync,
+	existsSync,
+	fsyncSync,
+	mkdirSync,
+	openSync,
+	readdirSync,
+	readFileSync,
+	unlinkSync,
+	writeSync,
+} from 'node:fs';
+import { basename, dirname, join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { tryFileLock, fileLockRelease } from '@harperfast/rocksdb-js';
+
+/**
+ * Restore lock + marker protocol for RocksDB database restores (online operation and offline CLI),
+ * and the shared per-database exclusion used by `dropDatabase` so a drop and a restore can never
+ * mutate the same directory concurrently.
+ *
+ * Restore metadata lives in an isolated `` `restore` `` directory *beside* the database directory
+ * (never inside it, since a restore purges the destination). Each database's two files are keyed by
+ * a hash of the database directory name rather than being suffixed onto the name itself. That keeps
+ * them out of the database-name namespace — a legal database literally named `orders.restoring`
+ * would otherwise be mistaken for the restore marker of `orders`, and a 250-character name plus a
+ * `.restore.lock` suffix would exceed the 255-byte `NAME_MAX` on most filesystems. The directory
+ * name deliberately contains a backtick: `schemaRegex` (the database-name validator) forbids only
+ * `/` and `` ` `` among filesystem-legal characters, so no legal database can ever occupy this path
+ * — including a database literally named `.restore` (which *is* a legal name, so a plain `.restore`
+ * directory would collide with it and land the markers inside the live database). The directory is
+ * not itself a RocksDB/LMDB database (no `CURRENT`/`MANIFEST-`/`.mdb`), so the startup scan ignores
+ * it, and no user can create a database that resolves to it.
+ *
+ * - `<meta-dir>/<key>.lock` — an OS-level exclusive file lock (via rocksdb-js `tryFileLock`),
+ *   effective across processes, containers, and worker threads, auto-released on process exit.
+ *   Only *held-ness* is meaningful; the file itself persists after release (harmless). Held for the
+ *   duration of a restore, and briefly by `dropDatabase` so the two serialize on the same primitive.
+ *   Known limitation: the lock is owned by the process, so if the restore job's worker *thread*
+ *   dies without the process exiting, the lock stays held (restores 409) until Harper restarts.
+ * - `<meta-dir>/<key>.restoring` — the completion marker. Written (and fsynced) after the lock is
+ *   acquired and before the destructive restore begins; deleted only after the restore completes
+ *   successfully, while still holding the lock. Its *existence* means "a restore started and has
+ *   not finished successfully". Its first line records the database directory name so the startup
+ *   scan can map a marker back to the database it blocks without decoding the hashed key.
+ */
+
+// The backtick makes this an illegal database name (schemaRegex rejects `/` and backtick only), so
+// it can never collide with a real database directory — see the module header.
+export const RESTORE_META_DIR = '`restore`';
+export const RESTORE_LOCK_SUFFIX = '.lock';
+export const RESTORING_MARKER_SUFFIX = '.restoring';
+
+/**
+ * Directory holding the restore metadata for a database — the reserved `` `restore` `` sibling of
+ * the database directory (see the module header for why the name contains a backtick). Shared by
+ * every database under the same parent, so a single readdir surfaces all pending restores during
+ * the startup scan.
+ */
+export function restoreMetaDir(dbPath: string): string {
+	return join(dirname(dbPath), RESTORE_META_DIR);
+}
+
+/**
+ * Filesystem-safe, length-bounded key for a database's restore metadata files. Hashing the
+ * database directory name (not the full path, so it is stable regardless of where the databases
+ * root lives) keeps the metadata filenames short and collision-free while staying independent of
+ * the database-name namespace. Database directory names are unique within a databases root, so
+ * their hashes are too.
+ */
+function restoreMetaKey(dbPath: string): string {
+	return createHash('sha256').update(basename(dbPath)).digest('hex').slice(0, 32);
+}
+
+export function restoreLockPath(dbPath: string): string {
+	return join(restoreMetaDir(dbPath), restoreMetaKey(dbPath) + RESTORE_LOCK_SUFFIX);
+}
+
+export function restoringMarkerPath(dbPath: string): string {
+	return join(restoreMetaDir(dbPath), restoreMetaKey(dbPath) + RESTORING_MARKER_SUFFIX);
+}
+
+export type RestoreState = 'in-progress' | 'incomplete' | 'clear';
+
+/**
+ * Whether a `.restoring` marker exists for a database. Cheaper than `checkRestoreState` and, unlike
+ * it, safe to call while *this* thread holds the restore lock: `checkRestoreState` would re-probe
+ * the lock (which reads as held from the same thread) and report 'in-progress' rather than telling
+ * a caller that a *leftover* marker is present. `dropDatabase` uses this after acquiring the lock to
+ * distinguish debris from a crashed restore.
+ */
+export function restoreMarkerPresent(dbPath: string): boolean {
+	return existsSync(restoringMarkerPath(dbPath));
+}
+
+/** The lock and (optional) marker held by a begin/acquire call, threaded back to complete/abandon. */
+export type RestoreLock = {
+	/** rocksdb-js file-lock token; non-zero. */
+	token: number;
+	/** The database directory this lock guards. */
+	dbPath: string;
+	/**
+	 * True when a `.restoring` marker already existed at `beginRestore` time — i.e. this restore is a
+	 * recovery attempt over a possibly half-purged directory. A pre-existing marker must never be
+	 * cleared by a *failed* recovery attempt, or the directory could be reloaded as healthy while
+	 * still partial. Only set on `beginRestore`; always false for a bare `acquireRestoreLock`.
+	 */
+	preexisting: boolean;
+};
+
+/**
+ * Determine the restore state of a database directory. Used by startup database detection and
+ * the open-database guards:
+ * - 'in-progress': marker present and the restore lock is held (a restore is running in some
+ *   process) — do not load.
+ * - 'incomplete': marker present but the lock is free (crashed mid-restore; the directory may
+ *   be partial garbage) — do not load; rerun the restore.
+ * - 'clear': no marker — load normally (a stale, unheld lock file alone is fine).
+ *
+ * The marker is checked FIRST and the lock is only probed when the marker exists. Probing takes
+ * and releases the flock, and probes are mutually exclusive across threads — if every rescan on
+ * every thread probed the (persistent) lock file of a long-ago-restored database, concurrent
+ * rescans would collide and misclassify healthy databases as 'in-progress'. Marker-first is
+ * safe: `beginRestore` writes (and fsyncs) the marker immediately after taking the lock and
+ * before any destructive step, so a database without a marker has nothing to protect yet.
+ */
+export function checkRestoreState(dbPath: string): RestoreState {
+	if (!existsSync(restoringMarkerPath(dbPath))) return 'clear';
+	const lockPath = restoreLockPath(dbPath);
+	if (existsSync(lockPath)) {
+		const token = tryFileLock(lockPath);
+		if (token === 0) return 'in-progress';
+		fileLockRelease(token);
+	}
+	return 'incomplete';
+}
+
+/**
+ * fsync a directory so a create/unlink of an entry within it is durable. Best-effort: Windows (and
+ * some filesystems) reject opening a directory for fsync with EPERM/EISDIR/ENOTSUP — the durability
+ * flush is a POSIX nicety, so treat those as a no-op rather than failing the restore.
+ */
+function fsyncDir(dir: string): void {
+	let dirFd: number;
+	try {
+		dirFd = openSync(dir, 'r');
+	} catch (error: any) {
+		if (error.code === 'EPERM' || error.code === 'EISDIR' || error.code === 'ENOTSUP') return;
+		throw error;
+	}
+	try {
+		fsyncSync(dirFd);
+	} finally {
+		closeSync(dirFd);
+	}
+}
+
+/**
+ * Take the per-database restore lock without writing a marker. Used by `dropDatabase` so a drop and
+ * a restore serialize on the same primitive: whichever takes the lock first runs to completion; the
+ * other gets a 409. Throws (statusCode 409) if the lock is already held.
+ */
+export function acquireRestoreLock(dbPath: string): RestoreLock {
+	mkdirSync(restoreMetaDir(dbPath), { recursive: true });
+	const token = tryFileLock(restoreLockPath(dbPath));
+	if (token === 0) {
+		const error: any = new Error(`Restore already in progress for database at ${dbPath}`);
+		error.statusCode = 409;
+		throw error;
+	}
+	return { token, dbPath, preexisting: false };
+}
+
+/**
+ * Release a lock taken by `acquireRestoreLock` (no marker to remove).
+ */
+export function releaseRestoreLock(lock: RestoreLock): void {
+	fileLockRelease(lock.token);
+}
+
+/**
+ * Acquire the per-database restore lock and write the restoring marker. Call before any
+ * destructive step. Returns the lock (with `preexisting` set when a marker was already present, so
+ * a failed recovery attempt knows not to clear it). Throws (statusCode 409) if another restore
+ * already holds the lock.
+ */
+export function beginRestore(dbPath: string): RestoreLock {
+	const markerPath = restoringMarkerPath(dbPath);
+	const preexisting = existsSync(markerPath);
+	const lock = acquireRestoreLock(dbPath);
+	try {
+		const fd = openSync(markerPath, 'w');
+		try {
+			// first line is the database directory name so the startup scan can map this marker back to
+			// the database it blocks without reversing the hashed key
+			writeSync(fd, `${basename(dbPath)}\nrestore started ${new Date().toISOString()}\n`);
+			fsyncSync(fd);
+		} finally {
+			closeSync(fd);
+		}
+		// fsync the metadata directory so the marker's directory entry is durable — without this a
+		// power loss can lose the entry, and a half-purged database would load as healthy
+		fsyncDir(restoreMetaDir(dbPath));
+	} catch (error) {
+		fileLockRelease(lock.token);
+		throw error;
+	}
+	return { ...lock, preexisting };
+}
+
+/**
+ * Mark the restore successful: delete the marker (while still holding the lock), then release
+ * the lock.
+ */
+export function completeRestore(lock: RestoreLock): void {
+	try {
+		unlinkSync(restoringMarkerPath(lock.dbPath));
+		// fsync the metadata directory so the marker's *removal* is durable — symmetric with the
+		// creation fsync in beginRestore. Without it, a power loss could resurrect the marker's
+		// directory entry and misclassify a fully-restored database as incomplete.
+		fsyncDir(restoreMetaDir(lock.dbPath));
+	} finally {
+		fileLockRelease(lock.token);
+	}
+}
+
+/**
+ * Release the lock after a failed restore, leaving the marker in place so the database is
+ * detected as an incomplete restore (and not loaded) until a rerun succeeds.
+ */
+export function abandonRestore(lock: RestoreLock): void {
+	fileLockRelease(lock.token);
+}
+
+/**
+ * Remove a database's restore marker if one is present, then release the lock. Used by
+ * `dropDatabase`: a dropped database that carried an incomplete-restore marker should not leave the
+ * marker behind to block a future database of the same name. No-op on the marker if none exists.
+ */
+export function clearRestoreMarker(lock: RestoreLock): void {
+	try {
+		const markerPath = restoringMarkerPath(lock.dbPath);
+		if (existsSync(markerPath)) {
+			unlinkSync(markerPath);
+			fsyncDir(restoreMetaDir(lock.dbPath));
+		}
+	} finally {
+		fileLockRelease(lock.token);
+	}
+}
+
+/**
+ * Scan a databases root's reserved `` `restore` `` metadata directory and report every database currently blocked from
+ * loading, mapping each surviving marker back to its database name via the marker's first line.
+ * Returns `[dbName, state]` pairs for markers whose state is `in-progress` or `incomplete`
+ * (a `clear` result means the marker was removed concurrently and the database is loadable).
+ */
+export function scanBlockedRestores(databasesRoot: string): Array<[string, RestoreState]> {
+	const metaDir = join(databasesRoot, RESTORE_META_DIR);
+	if (!existsSync(metaDir)) return [];
+	const blocked: Array<[string, RestoreState]> = [];
+	for (const entry of readdirSync(metaDir, { withFileTypes: true })) {
+		if (!entry.isFile() || !entry.name.endsWith(RESTORING_MARKER_SUFFIX)) continue;
+		let dbName: string;
+		try {
+			dbName = readFileSync(join(metaDir, entry.name), 'utf8').split('\n', 1)[0];
+		} catch {
+			continue; // marker removed concurrently
+		}
+		if (!dbName) continue;
+		const state = checkRestoreState(join(databasesRoot, dbName));
+		if (state !== 'clear') blocked.push([dbName, state]);
+	}
+	return blocked;
+}