@harperfast/harper 5.3.0 → 5.3.1

package/components/operations.js

@@ -40,6 +40,7 @@ const {
 } = require('./Application.ts');
 const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require('./componentPreparationLock.ts');
 const { DEPLOYMENT_PROVENANCE_FILE } = require('./deploymentProvenance.ts');
+const { compareInstallFingerprints, describeInstallDrift } = require('./installFingerprint.ts');
 const {
 	applyRootConfigEffect,
 	assertRootConfigEffectPublishable,
@@ -888,9 +889,11 @@ async function deployComponent(req) {
 				emit('phase', { phase: 'prepare', status: 'done' });
 				await validateComponentLoads(candidateDirPath, emit);
 			},
+			fingerprintInstall: !isActivation,
 		});
 		// Nothing ran `validateCandidate`, which ends this phase.
 		if (application.alreadyActive) emit('phase', { phase: 'prepare', status: 'done' });
+		if (recorder && application.installFingerprint) recorder.row.install_fingerprint = application.installFingerprint;
 		// The build is certified on disk from here on, so every later failure — a peer result, or a rejection
 		// thrown by the replication layer itself — still leaves an artifact this id can activate.
 		if (mode === 'stage') stagedOnOrigin = true;
@@ -933,6 +936,13 @@ async function deployComponent(req) {
 			timeoutMs: peerDeployAnswerTimeoutMs(req),
 		});
 		emit('phase', { phase: 'replicate', status: 'done' });
+		if (application.installFingerprint) response.install = application.installFingerprint;
+		// Marked on the aggregate, which the recorder re-records below, so a peer's per-peer entry is replaced
+		// rather than contradicted.
+		const installDrift = recorder
+			? markInstallComparisons(application.installFingerprint, response?.replicated)
+			: undefined;
+		if (installDrift) emit('warning', { message: installDrift });
 		if (recorder && response?.replicated) {
 			// Fallback path for replicators that don't honor onPeerResult: re-record the
 			// aggregate. recordPeer's upsert-by-node-name semantics make this idempotent
@@ -1081,6 +1091,7 @@ async function deployComponent(req) {
 					`Staged: ${application.name}. Deploy it with deploy_component ` + `deployment_id=${recorder.deploymentId}`;
 			}
 		}
+		if (installDrift) response.message = `${response.message} ${installDrift}`;
 		return response;
 	} catch (err) {
 		// Pack phase, install output tail, and deployment_id into http_resp_msg so the
@@ -1145,6 +1156,23 @@ function unconfirmedStagingPeers(replicated) {
 	return replicated.filter((peer) => peer && !confirmed(peer));
 }
 
+/** The fingerprint is read flat or from a wrapped body, as `unconfirmedStagingPeers` reads its marker. */
+function markInstallComparisons(ownFingerprint, replicated) {
+	if (!ownFingerprint || !Array.isArray(replicated)) return undefined;
+	const peers = [];
+	for (const peer of replicated) {
+		if (!peer || typeof peer !== 'object') continue;
+		const comparison = compareInstallFingerprints(
+			ownFingerprint,
+			peer.install ?? peer.value?.install ?? peer.body?.install
+		);
+		peer.install_matches = comparison.matches;
+		peer.install_differs = comparison.differs;
+		peers.push({ node: peer.node ?? peer.name ?? peer.hostname ?? null, comparison });
+	}
+	return describeInstallDrift(peers);
+}
+
 // Ring buffer of install stdout/stderr lines, capped by both line count and bytes so
 // a chatty install can't unbounded-grow the error response. snapshot() reports whether
 // the head was dropped so callers can flag truncation.
@@ -1634,6 +1662,7 @@ exports.dropCustomFunctionProject = dropCustomFunctionProject;
 exports.packageComponent = packageComponent;
 exports.deployComponent = deployComponent;
 exports.unconfirmedStagingPeers = unconfirmedStagingPeers;
+exports.markInstallComparisons = markInstallComparisons;
 exports.peerDeployAnswerTimeoutMs = peerDeployAnswerTimeoutMs;
 exports.getComponents = getComponents;
 exports.getComponentFile = getComponentFile;

package/config/DESIGN.md

@@ -11,7 +11,7 @@ Index of every design note: [DESIGN.md](../DESIGN.md).
 ## `set_configuration` replication is opt-in; `replicateOperation` is default-on (`config/configUtils.ts`)
 
 `server.replication.replicateOperation` (installed by harper-pro's replicator) fans out whenever
-`req.replicated \!== false` — absence of the flag means "replicate". That default-on contract is what
+`req.replicated !== false` — absence of the flag means "replicate". That default-on contract is what
 DDL ops rely on (`dropSchema`/`dropTable` call it unconditionally), so a handler that mirrors the
 drop_schema pattern without a guard silently becomes replicate-by-default. `setConfiguration` must
 stay **opt-in** (`if (replicated)` truthy guard) because config bodies routinely carry node-local
@@ -211,13 +211,13 @@ What a scope does about a config that arrives late is the other half of settling
 config of its own to having one, which is both the recreated-config-file path and a scope that
 booted while the file was unreadable. Nothing downstream re-runs on it — `componentLoader` is long
 past its `await scope.ready` — so `Scope` answers a repeat `ready` the same way it answers `remove`,
-by requesting a restart. Without that, one worker keeps serving the defaults while every worker
+by requesting a restart, unless the plugin listens for `ready` itself. Without that, one worker keeps serving the defaults while every worker
 that read the file cleanly serves the operator's config.
 
 Arming is a terminal outcome of its own: chokidar reports a scan that found no file by emitting
 `ready` and nothing else, so `RootConfigWatcher` always re-reads when the gate opens rather than
-publishing what an earlier read staged — a missing config file takes the ladder and settles on the
-defaults instead of holding the barrier open. That fallback must also discard the staged value:
+publishing what an earlier read staged — a missing config file settles on the defaults at once (it
+never takes the ladder) instead of holding the barrier open. That fallback must also discard the staged value:
 the arming re-read is authoritative precisely because a write in the unarmed window may have
 superseded it, including by replacing the file with an unusable or missing one. A watcher scan error
 also settles the barrier, but preserves a successfully staged value because no read superseded it.
@@ -258,7 +258,7 @@ cleanup that frees space needs a started process (#847). Two rules follow.
 **Derived boot writes are best-effort; user-requested ones are not.** `persistConfigDuringBoot()`
 swallows exactly ENOSPC/EDQUOT (matching on `errno` as well as `code`, because Linux has no libuv
 mapping for EDQUOT and reports `Unknown system error -122`) and lets the boot proceed on the
-in-memory config. `updateConfig`/`set_configuration`, `addConfig`, `deleteConfigFromFile` and the
+in-memory config. `set_configuration` (`updateConfigValue`), deploy/drop's `applyRootConfigEffect` and the
 install path keep persist-or-throw: a caller who asked to persist must not get a silent success, and
 an install has no last-known-good config to fall back on.
 

package/dataLayer/DESIGN.md

@@ -10,7 +10,7 @@ Index of every design note: [DESIGN.md](../DESIGN.md).
 
 ## Version gate at startup: downgrades prompt, and only the minor direction is confirmable
 
-`getVersionUpdateInfo()` (`dataLayer/hdbInfoController.ts`) compares the store's `data_version_num` (latest `system.hdb_info` record) against the binary's `packageJson.version` on every start. Data newer than binary by a **major** version → hard refusal. Newer by a **minor** version → `forceDowngradePrompt()` asks for confirmation; answering yes records the data version back down to the binary's version and boots (upgrade directives are deliberately additive/downgrade-compatible — see the struct-mode section above and `patchHdbSecretIsHashAttribute` in `upgrade/directives/5-2-0.ts`).
+`getVersionUpdateInfo()` (`dataLayer/hdbInfoController.ts`) compares the store's `data_version_num` (latest `system.hdb_info` record) against the binary's `packageJson.version` on every start. Data newer than binary by a **major** version → hard refusal. Newer by a **minor** version → `forceDowngradePrompt()` asks for confirmation; answering yes records the data version back down to the binary's version and boots (upgrade directives are deliberately additive/downgrade-compatible — see [Struct mode is gated to primary DBIs](../resources/DESIGN.md#struct-mode-is-gated-to-primary-dbis-downgrade-compatibility) and `patchHdbSecretIsHashAttribute` in `upgrade/directives/5-2-0.ts`).
 
 - The prompt's answer can be supplied non-interactively via `CONFIRM_DOWNGRADE` — env var or `--CONFIRM_DOWNGRADE` CLI arg; argv wins (`assignCMDENVVariables`). With no override and no TTY on stdin, the prompt throws instead of blocking on stdin forever (#2046 — services/CI hung with nothing in the log; the mismatch is also logged to hdb.log now).
 - Upgrades never prompt (see the rationale comment in `bin/upgrade.js`); only the downgrade direction confirms. `upgradeCertsPrompt()` (4.x upgrade path; currently has no in-core caller) has the same no-TTY guard as `forceDowngradePrompt()` — a `GENERATE_CERTS` override is honored first, and with no TTY and no override it throws instead of blocking.
@@ -22,9 +22,9 @@ When `migrateOnStart` opens a source LMDB primary store to read records out for
 
 Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match.
 
-The persisted `compression` value itself is LMDB-era and loosely shaped: `getDefaultCompression()` historically stored whatever falsy value the config resolved to (`''`, `false`, `null`) when `storage.compression` was disabled, and `{ startingOffset, threshold, dictionary? }` when enabled. lmdb-js interprets falsy as "no compression", but rocksdb-js >= 2.6 validates the option strictly (`''`/booleans throw `Unsupported compression algorithm`) and treats UNSET as "use the build default (lz4)" — the inverse default of lmdb. Every RocksDB open must therefore route through `toRocksCompression()` in `resources/databases.ts` (applied inside `openRocksDatabase`, the single chokepoint), which maps defined-falsy → `'none'` and enabled-without-an-algorithm → an explicit lz4 request when available. Don't pass persisted attribute compression to a RocksDB open directly.
+The persisted `compression` value itself is LMDB-era and loosely shaped: `getDefaultCompression()` historically stored whatever falsy value the config resolved to (`''`, `false`, `null`) when `storage.compression` was disabled, and `{ startingOffset, threshold, dictionary? }` when enabled. lmdb-js interprets falsy as "no compression", but rocksdb-js >= 2.6 validates the option strictly (`''`/booleans throw `Unsupported compression algorithm`) and treats UNSET as "keep the family's persisted codec, or the build default (lz4) for a new family" — the inverse default of lmdb. Every RocksDB open therefore takes one per-process deployment codec, `getRocksCompression()` (applied inside `openRocksDatabase`; see [The RocksDB codec is a deployment setting, resolved once per process](../resources/DESIGN.md#the-rocksdb-codec-is-a-deployment-setting-resolved-once-per-process)), derived through `toRocksCompression()` in `resources/databases.ts`, which maps defined-falsy → `'none'` and enabled-without-an-algorithm → an explicit lz4 request when available. Don't pass persisted attribute compression to a RocksDB open directly.
 
-`bin/copyDb.ts`'s `openRocksDb` is part of that chokepoint, not an exception to it. This is about the bytes migration writes, not about a later failure: `copyDbToRocks()` closes every target handle before the staging directory is renamed, and rocksdb-js permits an explicit codec change across a close/reopen, so the runtime would open the migrated database fine either way. But a migration that ignores the configured codec writes the entire dataset uncompressed, and those SST/blob files then keep their original codec until write traffic rewrites them — a full LMDB→RocksDB migration is the one moment the whole dataset is written at once, so it is exactly when the deployment's codec should apply.
+`bin/copyDb.ts`'s `openRocksDb` applies the same codec (`getRocksCompression() ?? toRocksCompression(...)`), not an exception to it. This is about the bytes migration writes, not about a later failure: `copyDbToRocks()` closes every target handle before the staging directory is renamed, and rocksdb-js permits an explicit codec change across a close/reopen, so the runtime would open the migrated database fine either way. But a migration that ignores the configured codec writes the entire dataset uncompressed, and those SST/blob files then keep their original codec until write traffic rewrites them — a full LMDB→RocksDB migration is the one moment the whole dataset is written at once, so it is exactly when the deployment's codec should apply.
 
 ## System table bootstrap: `systemSchema.json` + upgrade directive
 
@@ -32,13 +32,13 @@ Adding a new system table (e.g. `hdb_deployment` in #641 Slice A) requires three
 
 1. **`json/systemSchema.json`** — the table entry. Fresh installs auto-create it via `utility/mount_hdb.ts:createTables()`, which iterates `Object.keys(systemSchema)` on first boot.
 2. **`utility/hdbTerms.ts`** — add the table name to `SYSTEM_TABLE_NAMES`.
-3. **`upgrade/directives/<version>.ts`** — provisions the table on existing installs that already have a system schema. Registered in `upgrade/directives/directivesController.ts` (which is otherwise empty — its `versions` Map gets populated by these imports). The directive shape is `{ version, sync_functions, async_functions }`; copy `5-1-0.ts` for the canonical pattern (uses `bridge.createTable` to match what `mount_hdb` does on a fresh install).
+3. **`upgrade/directives/<version>.ts`** — provisions the table on existing installs that already have a system schema. Registered in `upgrade/directives/directivesController.ts` (which is otherwise empty — its `versions` Map gets populated by these imports). Each directive module default-exports an array of `{ version, sync_functions, async_functions }`; copy `5-1-0.ts` for the canonical pattern (uses `bridge.createTable` to match what `mount_hdb` does on a fresh install).
 
    **Version the directive to the first release that ships the dependent code, not a later one.** Directives only run when `current_version < directive_version <= upgrade_version` (`directivesController.getVersionsForUpgrade`). The `hdb_deployment` directive was originally mis-tagged `5.2.0` while the deployment-recorder code shipped in `5.1.0`, so on every `5.0.x -> 5.1.x` upgrade the directive was filtered out (`5.2.0 > 5.1.x`) and the table never got created — breaking replicated `deploy_component` on peer nodes for the entire existing customer base. Caveat: `utility/common_utils.ts:compareVersions` strips trailing `.0` and therefore sorts a pre-release (`5.1.0-beta.1`) _above_ its GA (`5.1.0`), so an install already on a `5.1.0-beta.x` data version will not pick up a `5.1.0` directive when upgrading to GA; those pre-release installs need the table created by other means.
 
 System tables replicate by default. To opt out, add the name to `NON_REPLICATING_SYSTEM_TABLES` in `resources/databases.ts`. The check happens after table init and sets `table.replicate = false` per-node.
 
-If the table needs `audit: true`, set it both in the schema (for fresh installs) **and** on the `CreateTableObject` instance in the directive (for upgrades) — otherwise the two paths diverge.
+Set `audit = true` on the directive's `CreateTableObject` (for upgrades): a fresh install audits every system table — `utility/mount_hdb.ts:createTables()` sets `createTable.audit = true` regardless of the schema entry — so a directive that leaves it off makes upgraded nodes diverge from fresh ones.
 
 **A replicated system table whose owner declares more than a name list** — a table-level `expiration`, an index — needs that declaration on every node, every boot. `hdb_oidc_token_use` is the case: `security/authn/oidc/tokenUseTable.ts` holds its one definition, and `bin/run.ts initialize()` applies it after the upgrade step and before worker threads start (skipped in read-only mode; a failure is logged and retried on the next start), as do the 5.3.0 directive and the exchange path. Each replay row carries its own expiry as record metadata, which replication applies on the receiving node; the table's stored `expiration` is what arms the cleanup scan that removes those rows once a node loads the table. Metadata rather than an `@expiresAt` attribute, because the attribute needs an index and a sweep of its own on every node, while reads and the cleanup scan already honor the metadata. `systemSchema.json` keeps only its primary key; the declaration completes the table on the first boot after install. Every other route to a copy leaves it partial, which is why the declaration cannot live only on the exchange path or in the directive:
 
@@ -76,7 +76,7 @@ Three non-obvious mechanics keep that safe:
   be exactly that database's directory). Because the startup scan opens any `CURRENT`+`MANIFEST-`
   directory without re-applying `schemaRegex`, it also explicitly skips the reserved `` `restore` ``
   entry so an out-of-band directory at that name is never loaded as a database. Startup/rescan
-  detection (`databasesBlockedByRestore` → `scanBlockedRestores` in `dataLayer/restoreMarker.ts`)
+  detection (`databasesBlockedByRestore` in `resources/databases.ts` → `scanBlockedRestores` in `dataLayer/restoreMarker.ts`)
   reads the metadata directory and checks the **marker first**, only probing the lock when the marker exists —
   probes take the flock and are mutually exclusive across threads, so probing the (persistent) lock
   file of every long-ago-restored database on every rescan would make concurrent rescans misclassify
@@ -127,15 +127,16 @@ Three non-obvious mechanics keep that safe:
 - **Online restore is impossible for a database a component holds open — and that failure is
   correct.** rocksdb-js's registry is process-global but records only a per-path refCount, with no
   attribution to a thread or component; Harper keeps no component→database ownership map. So when a
-  loaded component (or the `system` database, which Harper itself never stops while running) holds
-  its own handle on the target database, `registryStatus()` stays non-zero, Harper can neither
-  identify nor force-close that handle, and an in-place purge would corrupt a live instance.
+  loaded component holds its own handle on the target database, `registryStatus()` stays non-zero,
+  Harper can neither identify nor force-close that handle, and an in-place purge would corrupt a live
+  instance.
   `verifyDatabaseClosed` therefore waits only a short grace period (`DATABASE_CLOSE_WAIT_MS`, for a
   just-finished job worker's own close to drain) and then fails fast with a 409 that points at
   running the operation offline (`harper restore_backup` with the server stopped, where no
   components are loaded and nothing holds the database open). Offline restore is the supported path
   for component-held and `system` databases; online restore serves databases not actively held by a
-  component. The CLI exposes each backup operation under its operation name only (`create_backup`,
+  component. `system`, which Harper itself never stops while running, never reaches this check:
+  `validateRestoreBackup` refuses it up front with a 400 naming the offline command. The CLI exposes each backup operation under its operation name only (`create_backup`,
   `restore_backup`, …) — no hyphenated alias — and `bin/backup.ts` routes it to a reachable server
   or, when the local server is stopped, to the equivalent offline function.
 - **Job workers must release their RocksDB handles on exit, or the closure check can never pass.**
@@ -149,6 +150,11 @@ Three non-obvious mechanics keep that safe:
   calls `closeLoadedDatabases()` (`resources/databases.ts`) in its `finally`, closing every loaded
   user database on that thread (the non-enumerable `system` DB is intentionally skipped), so an
   exited job worker leaves no residual handle to be mistaken for a live holder.
+- **A restore stamps a new database generation before `completeRestore`.** The restored files carry
+  the backup's generation, so both paths open the restored directory privately, stamp it and flush
+  (`stampDatabaseDirectory`, [database generation](../resources/DESIGN.md#database-generation-and-resumable-positions))
+  inside the destructive section: a failed stamp leaves the marker, and the rerun re-purges and
+  re-stamps. The stamp is as durable as the marker protocol it runs inside.
 - **`dropDatabase` and `restore_backup` serialize on the same lock, not a check-then-act probe.**
   A drop's `destroy()` interleaving with a restore's purge-and-copy on the same directory would gut
   a "successful" restore (or vice versa). `dropDatabase` takes the restore lock for every RocksDB or
@@ -170,22 +176,31 @@ Three non-obvious mechanics keep that safe:
   `create_table`/`create_schema` must not resurrect a half-purged directory as a fresh empty DB), but
   the destructive drop path now uses the exclusive lock so the race is closed, not merely narrowed.
 - **The offline restore probes RocksDB's own `LOCK` file, and fails closed.** The offline path runs
-  only when the CLI sees no server (a PID heuristic; the PID file is briefly absent mid-`harper
-restart`), and `backups.restore`'s `purgeAllFiles` never takes RocksDB's lock — so before purging,
-  `restoreBackupOffline` opens the database to probe. It now takes the restore lock+marker _before_
-  probing (so a server that starts afterward sees the marker and refuses to load), and recognizes the
-  pinned rocksdb-js 2.5.0 lock error — a plain `Error` with no `code` and message
-  `IO error: While lock file: <db>/LOCK: Resource temporarily unavailable` (`isRocksDbLockError`) —
-  aborting with a 409 rather than purging a database another process holds open. Any _other_ open
-  failure (corrupt/half-restored) is exactly what restore recovers, so only a lock conflict aborts.
+  only when the CLI sees no server (a PID heuristic; the PID file is briefly absent
+  mid-`harper restart`), and `backups.restore`'s `purgeAllFiles` never takes RocksDB's lock — so
+  before purging, `restoreBackupOffline` opens the database to probe. It now takes the restore
+  lock+marker _before_ probing (so a server that starts afterward sees the marker and refuses to
+  load), and recognizes the rocksdb-js lock error by message (`isRocksDbLockError`; at 2.5.0, when
+  this was written, a plain `Error` with no `code`) —
+  `IO error: While lock file: <db>/LOCK: Resource temporarily unavailable` — aborting with a 409
+  rather than purging a database another process holds open. Any _other_ open failure
+  (corrupt/half-restored) is exactly what restore recovers, so only a lock conflict aborts.
 
 Known limitation: the flock is process-owned; if the restore job's worker _thread_ dies without
-the process exiting, the lock stays held (restores 409) until Harper restarts. There is no typed
-native lock signal in rocksdb-js 2.5.0, so the offline probe relies on message matching; a native
+the process exiting, the lock stays held (restores 409) until Harper restarts. rocksdb-js had no typed
+native lock signal at 2.5.0 (the pin is now 2.10.0), so the offline probe relies on message matching; a native
 lock primitive is a rocksdb-js follow-on.
 
 ## RocksDB managed backups: blob snapshots (`dataLayer/blobBackup.ts`)
 
+Repository mutations hold the management lock before entering the engine. Restore admission
+rechecks the source under that lock, then `beginRestore` takes the destination lock and runs a
+synchronous callback that durably writes the source claim before publishing the restoring marker.
+A crash after publication therefore leaves both marker and claim; a crash before publication leaves
+only a claim that lapses when no marker exists. Failed claims preserve any preexisting marker.
+Delete and purge reconcile Harper-managed files even after engine failure: a cleanup failure must
+propagate when the engine succeeded, but remains secondary when the engine already failed.
+
 A database's file-backed blobs live in one or more roots _outside_ the RocksDB directory
 (`getBlobPathsForDatabaseName` in `resources/blob.ts` — one per configured `storage.blobPaths`, else
 `<hdb_root>/blobs/<database>`), so the engine's backup does not capture them. `create_backup`,
@@ -201,7 +216,7 @@ engine-only backup):
   before capture, a terminal ERROR (`0xff`) marker preserves its file id. A file reclaimed before its
   parent directory is read is outside the snapshot. This keeps a snapshot inode from changing as a
   live write finishes, while complete blobs remain safe to hard-link because published blob paths are
-  write-once. The snapshot is built in a `.tmp-<id>` sibling and atomically renamed so a failed create
+  write-once. The snapshot is built in a `<backupId>.tmp` sibling and atomically renamed so a failed create
   leaves no partial snapshot. `restore_backup` purges each blob root and rewrites it from the snapshot;
   `delete_backup` / `purge_backups` remove the corresponding snapshot directories.
 - **`get_backup`** appends the blob files to the same tar under `blobs/<rootIndex>/<relpath>`. The
@@ -228,4 +243,4 @@ flag — not the mere presence of a snapshot dir — to decide whether to restor
 backup leaves live blobs untouched, and a manifest that claims blobs but has no snapshot is flagged
 corrupt by verify). This closes the "healthy-looking but incomplete" and concurrent-restore races;
 the remaining engine/blob point-in-time skew (a blob unlinked between the engine cut and the blob
-walk) is the documented best-effort limitation above.
+walk) is best-effort: Harper does not freeze blob writes for a backup.

package/dataLayer/backupRepository.ts

@@ -0,0 +1,248 @@
+'use strict';
+
+import { accessSync, constants, existsSync, mkdirSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { fileLockRelease, tryFileLock } from '@harperfast/rocksdb-js';
+import { ClientError } from '../utility/errors/hdbError.ts';
+import logger from '../utility/logging/harper_logger.ts';
+import { restoreMarkerPresent } from './restoreMarker.ts';
+import { fsyncDirectory, removeFileDurably, writeFileDurably } from '../utility/durableFile.ts';
+
+/**
+ * Harper-level coordination for a backup repository: one exclusive management lock, and pins that
+ * keep a backup alive while something is depending on it.
+ *
+ * rocksdb-js has its own `.backup.lock`, but it only covers the engine files. Harper's blob
+ * snapshots and completion manifests live beside them and are written and removed by separate
+ * Harper code, so a `purge_backups` can still remove a blob snapshot out from under a
+ * `create_backup` that is mid-finalization, or out from under a restore that is reading it
+ * (harper#2031). Everything that mutates the *Harper-managed* parts of a repository therefore takes
+ * this lock, in this order: management lock first, then any engine call that takes `.backup.lock`.
+ *
+ * A pin is the other half. A restore selects a backup and then acts on it, and a delete or purge in
+ * between would leave it with no source. Pins are installed and checked under the same lock, so "is
+ * this id pinned?" cannot be answered stale: a delete could otherwise read "unpinned", pause, and
+ * resume after a pin landed.
+ *
+ * A pin holds no owner or expiry. It names the database directory it is protecting, and it counts
+ * only while that database still carries a restoring marker — the marker being present is exactly
+ * the condition under which a restore is either running or waiting to be rerun, and therefore the
+ * condition under which its source must survive. A process killed mid-restore leaves both, and the
+ * rerun clears both; a process killed between clearing the marker and releasing the pin leaves a pin
+ * that is ignored and swept the next time anything looks. Nothing has to be reaped on a timer, and
+ * no pin can outlive what it protects.
+ */
+
+const MANAGEMENT_LOCK_FILE = '.management.lock';
+const PINS_DIR = 'pins';
+const PIN_FILE_SUFFIX = '.json';
+/** Filesystem-safe, and short enough to stay well inside NAME_MAX once suffixed. */
+const PIN_ID_PATTERN = /^[A-Za-z0-9._-]{1,128}$/;
+
+const LOCK_POLL_INTERVAL_MS = 50;
+const LOCK_WAIT_TIMEOUT_MS = 30_000;
+
+export function managementLockPath(backupDir: string): string {
+	return join(backupDir, MANAGEMENT_LOCK_FILE);
+}
+
+export function backupPinsDir(backupDir: string): string {
+	return join(backupDir, PINS_DIR);
+}
+
+export interface BackupPin {
+	pin_id: string;
+	backup_id: number;
+	reason: string;
+	created_at: number;
+	/** Directory of the database whose restore this pin protects; the pin counts while it is marked. */
+	database_path?: string;
+}
+
+/** Codes that mean the repository cannot be written at all, not that this attempt lost a race. */
+const REPOSITORY_NOT_WRITABLE = new Set(['EROFS', 'EACCES', 'EPERM']);
+const { W_OK } = constants;
+
+/**
+ * A repository Harper cannot write is refused by name rather than by whatever errno surfaced first.
+ * Every managed operation — restore included, which only reads the backup itself — records a lock
+ * and a claim inside the repository, so read-only media (a `ro` snapshot mount, a read-only share)
+ * cannot be used in place. Inferring "immutable, so no claim needed" from a failed write would be
+ * wrong: a read-only *view* of a repository another writer can still purge looks identical.
+ */
+function assertRepositoryWritable(databaseName: string, error: any): never {
+	if (!REPOSITORY_NOT_WRITABLE.has(error?.code)) throw error;
+	throw new ClientError(
+		`Backup repository for database '${databaseName}' is not writable (${error.code}). Harper records a ` +
+			`management lock and a claim inside the repository for every backup operation, including restore, so ` +
+			`it cannot be used read-only. Remount it writable, or copy it to writable storage, and retry.`,
+		409
+	);
+}
+
+/**
+ * Run `operation` holding the repository's management lock.
+ *
+ * Waits rather than failing immediately: the operations it serializes are short (a manifest write,
+ * a snapshot removal) except for a blob snapshot of a large database, and a `delete_backup` that
+ * 409s because an unrelated `create_backup` was finalizing is a worse answer than a brief wait. A
+ * holder that dies releases the lock with its process, so the wait cannot outlive a crash.
+ */
+export async function withBackupRepositoryLock<T>(
+	backupDir: string,
+	databaseName: string,
+	operation: () => Promise<T>,
+	createRepository = false
+): Promise<T> {
+	const lockPath = managementLockPath(backupDir);
+	const deadline = Date.now() + LOCK_WAIT_TIMEOUT_MS;
+	if (!existsSync(backupDir)) {
+		// Only a create may bring a repository into existence. Creating it here for a delete or purge
+		// would leave an empty directory behind after the 404, and a later list_backups for that name
+		// would answer [] instead of the 404 a typo is owed. With nothing on disk there is also
+		// nothing to exclude, so the operation runs unlocked and refuses on its own.
+		if (!createRepository) return operation();
+		try {
+			mkdirSync(backupDir, { recursive: true });
+		} catch (error: any) {
+			assertRepositoryWritable(databaseName, error);
+		}
+	}
+	// The binding's lock error carries a message but no errno, so writability is checked here.
+	try {
+		accessSync(backupDir, W_OK);
+	} catch (error: any) {
+		assertRepositoryWritable(databaseName, error);
+	}
+	let token = tryFileLock(lockPath);
+	while (token === 0) {
+		if (Date.now() >= deadline) {
+			throw new ClientError(
+				`Another backup management operation is in progress for database '${databaseName}'; retry once it finishes`,
+				409
+			);
+		}
+		await delay(LOCK_POLL_INTERVAL_MS);
+		token = tryFileLock(lockPath);
+	}
+	try {
+		return await operation();
+	} finally {
+		fileLockRelease(token);
+	}
+}
+
+function pinPath(backupDir: string, pinId: string): string {
+	if (!PIN_ID_PATTERN.test(pinId)) throw new ClientError(`Invalid backup pin id '${pinId}'`);
+	return join(backupPinsDir(backupDir), pinId + PIN_FILE_SUFFIX);
+}
+
+/** Every pin currently held on a repository. A malformed pin file is reported, never silently ignored. */
+export function readBackupPins(backupDir: string): BackupPin[] {
+	const pinsDir = backupPinsDir(backupDir);
+	if (!existsSync(pinsDir)) return [];
+	const pins: BackupPin[] = [];
+	for (const entry of readdirSync(pinsDir, { withFileTypes: true })) {
+		if (!entry.isFile() || !entry.name.endsWith(PIN_FILE_SUFFIX)) continue;
+		const pinId = entry.name.slice(0, -PIN_FILE_SUFFIX.length);
+		let parsed: any;
+		try {
+			parsed = JSON.parse(readFileSync(join(pinsDir, entry.name), 'utf8'));
+			if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+				throw new Error('pin file is not an object');
+			}
+			// A parseable record is not a usable one: `{"backup_id":"7"}` matches no requested id, so
+			// without this the claim permits exactly the delete it exists to refuse.
+			if (!Number.isInteger(parsed.backup_id) || parsed.backup_id < 1) {
+				throw new Error('pin file has no usable backup id');
+			}
+		} catch (error: any) {
+			// A released claim, unlinked between the readdir and the read. Anything else still means
+			// something claimed a backup: fail closed against every id, because a torn pin file must not
+			// become permission to delete whatever it was protecting.
+			if (error?.code === 'ENOENT') continue;
+			pins.push({ pin_id: pinId, backup_id: Number.NaN, reason: 'unreadable pin', created_at: 0 });
+			continue;
+		}
+		pins.push({
+			pin_id: pinId,
+			backup_id: parsed.backup_id,
+			reason: typeof parsed.reason === 'string' ? parsed.reason : 'unspecified',
+			created_at: parsed.created_at ?? 0,
+			database_path: typeof parsed.database_path === 'string' ? parsed.database_path : undefined,
+		});
+	}
+	return pins;
+}
+
+/**
+ * Claim a backup so it cannot be deleted or purged. The caller must hold the management lock, which
+ * is what makes the claim atomic with the delete/purge admission check.
+ */
+export function pinBackup(
+	backupDir: string,
+	pinId: string,
+	backupId: number,
+	reason: string,
+	databasePath?: string
+): void {
+	const path = pinPath(backupDir, pinId);
+	const pinsDir = backupPinsDir(backupDir);
+	const created = mkdirSync(pinsDir, { recursive: true });
+	// writeFileDurably flushes the pins directory, but on the first pin that directory's own entry is
+	// new too — without this the whole directory, and so the pin, can be lost to a power cut.
+	if (created !== undefined) fsyncDirectory(backupDir);
+	const pin: BackupPin = {
+		pin_id: pinId,
+		backup_id: backupId,
+		reason,
+		created_at: Date.now(),
+		...(databasePath ? { database_path: databasePath } : {}),
+	};
+	writeFileDurably(path, JSON.stringify(pin), `${pinId}.tmp`);
+}
+
+/** Whether a pin still protects anything: a restore pin counts only while its database is marked. */
+function pinIsLive(pin: BackupPin): boolean {
+	if (!pin.database_path) return true;
+	return restoreMarkerPresent(pin.database_path);
+}
+
+/** Release a claim. Missing is success — the goal is that nothing holds this backup any more. */
+export function unpinBackup(backupDir: string, pinId: string): void {
+	removeFileDurably(pinPath(backupDir, pinId));
+}
+
+/**
+ * Refuse to remove any of `backupIds` while something holds a pin on it. An unreadable pin blocks
+ * every id: what it was protecting is exactly what cannot be determined.
+ */
+export function assertBackupsUnpinned(backupDir: string, backupIds: number[], databaseName: string): void {
+	const pins = readBackupPins(backupDir);
+	if (pins.length === 0) return;
+	const requested = new Set(backupIds);
+	const live: BackupPin[] = [];
+	for (const pin of pins) {
+		if (pinIsLive(pin)) live.push(pin);
+		else {
+			// Opportunistic: a lapsed claim we cannot unlink is debris, not a reason to refuse the
+			// removal it no longer protects.
+			try {
+				unpinBackup(backupDir, pin.pin_id);
+			} catch (error) {
+				logger.warn(`Could not sweep a lapsed backup claim in ${backupDir}`, error);
+			}
+		}
+	}
+	const blocking = live.filter((pin) => Number.isNaN(pin.backup_id) || requested.has(pin.backup_id));
+	if (blocking.length === 0) return;
+	const described = blocking
+		.map((pin) => `${Number.isNaN(pin.backup_id) ? 'unknown backup' : `backup ${pin.backup_id}`} (${pin.reason})`)
+		.join(', ');
+	throw new ClientError(
+		`Cannot remove backups of database '${databaseName}' that are in use: ${described}. ` +
+			`Finish or cancel the operation holding them first.`,
+		409
+	);
+}