@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/config/harperConfigEnvVars.ts

@@ -25,13 +25,26 @@
 import type { Logger } from '../utility/logging/logger.ts';
 import * as fs from 'fs-extra';
 import * as path from 'node:path';
+import { isMainThread } from 'node:worker_threads';
 import * as crypto from 'node:crypto';
 import { cloneDeep } from 'lodash';
 import { getBackupDirPath } from './configHelpers.ts';
-import { atomicWriteFile } from './configUtils.ts';
+import { atomicWriteFile, renameWithRetry } from './configUtils.ts';
 import * as hdbTerms from '../utility/hdbTerms.ts';
 
 const STATE_FILE_NAME = '.harper-config-state.json';
+// Staged beside the confirmed state while a config-file write is in flight, then renamed over it.
+// Per-process, because every CLI invocation and worker runs this and one shared name would let one
+// clear another's in-flight commit. See DESIGN.md, boot-path config persistence.
+const PENDING_STATE_PREFIX = '.harper-config-state.pending.';
+const PENDING_STATE_SUFFIX = '.json';
+const pendingStateFileName = () => `${PENDING_STATE_PREFIX}${process.pid}${PENDING_STATE_SUFFIX}`;
+// Recovery from a recycled pid only has to be eventual, and deleting a slow-but-live writer's
+// sidecar is the worse error - it strands that writer's config file against an unpromoted state. So
+// the age-out is far longer than a commit (three synchronous steps) could ever legitimately take:
+// long enough that a stalled writer or clock skew between containers sharing a volume cannot reach
+// it, short enough that leaked wreckage does not suspend drift detection indefinitely.
+const PENDING_STATE_STALE_MS = 60 * 60 * 1000;
 
 /**
  * Get logger instance with tag - lazy loaded to avoid circular dependencies
@@ -546,15 +559,7 @@ function parseConfigEnvVar(envVarValue: string | undefined, envVarName: string):
 function loadConfigState(rootPath: string): ConfigState {
 	const statePath = path.join(getBackupDirPath(rootPath), STATE_FILE_NAME);
 
-	if (!fs.existsSync(statePath)) {
-		return {
-			version: '1.0',
-			sources: {},
-			originalValues: {},
-			emptyScopeOriginals: {},
-			snapshots: {},
-		};
-	}
+	if (!fs.existsSync(statePath)) return freshConfigState();
 
 	try {
 		const state = fs.readJsonSync(statePath) as ConfigState;
@@ -573,20 +578,85 @@ function loadConfigState(rootPath: string): ConfigState {
 		// If state file is corrupted, start fresh
 		const logger = getLogger();
 		logger.warn(`Failed to load config state file, starting fresh: ${(error as Error).message}`);
-		return {
-			version: '1.0',
-			sources: {},
-			originalValues: {},
-			emptyScopeOriginals: {},
-			snapshots: {},
-		};
+		return freshConfigState();
 	}
 }
 
+/**
+ * Clear a sidecar left by an interrupted commit, reporting whether one was there. The caller skips
+ * drift detection for that boot: it cannot tell a manual user edit from the write that was in
+ * flight, and calling it an edit hands those paths to 'user' for good.
+ */
+function takeInterruptedCommit(rootPath: string): boolean {
+	const backupDir = getBackupDirPath(rootPath);
+	let entries: string[];
+	try {
+		entries = fs.readdirSync(backupDir);
+	} catch {
+		return false;
+	}
+	let interrupted = false;
+	for (const entry of entries) {
+		if (!entry.startsWith(PENDING_STATE_PREFIX) || !entry.endsWith(PENDING_STATE_SUFFIX)) continue;
+		const pid = Number(entry.slice(PENDING_STATE_PREFIX.length, -PENDING_STATE_SUFFIX.length));
+		const entryPath = path.join(backupDir, entry);
+		if (pid !== process.pid && isProcessAlive(pid) && !isStale(entryPath)) {
+			// A live owner is mid-commit, so its sidecar is not wreckage to clear - but the pair it is
+			// halfway through is no more comparable than an interrupted one, so drift detection is off
+			// for this boot either way.
+			interrupted = true;
+			continue;
+		}
+		try {
+			fs.removeSync(entryPath);
+		} catch (error) {
+			// Keep drift detection on rather than disabling it every boot over a sidecar we cannot clear
+			getLogger().warn(`Could not remove an interrupted env config commit (${entry}): ${(error as Error).message}`);
+			continue;
+		}
+		interrupted = true;
+	}
+	if (interrupted) getLogger().warn('An env config commit was interrupted; skipping drift detection for this boot');
+	return interrupted;
+}
+
+function isStale(entryPath: string): boolean {
+	try {
+		return Date.now() - fs.statSync(entryPath).mtimeMs > PENDING_STATE_STALE_MS;
+	} catch {
+		return false;
+	}
+}
+
+function isProcessAlive(pid: number): boolean {
+	if (!Number.isInteger(pid) || pid <= 0) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (error) {
+		return (error as NodeJS.ErrnoException).code === 'EPERM';
+	}
+}
+
+function freshConfigState(): ConfigState {
+	return {
+		version: '1.0',
+		sources: {},
+		originalValues: {},
+		emptyScopeOriginals: {},
+		snapshots: {},
+	};
+}
+
 /**
  * Save configuration state to file
  */
-function saveConfigState(rootPath: string, state: ConfigState): void {
+function serializeConfigState(state: ConfigState): string {
+	return JSON.stringify(state, null, 2) + '\n';
+}
+
+// Returns true when the file was rewritten, false when it already held this state.
+function saveConfigState(rootPath: string, state: ConfigState): boolean {
 	const backupDir = getBackupDirPath(rootPath);
 	const statePath = path.join(backupDir, STATE_FILE_NAME);
 
@@ -595,7 +665,49 @@ function saveConfigState(rootPath: string, state: ConfigState): void {
 
 	// Atomic write: a torn state file resets to fresh on the next load, losing every
 	// restoration record — the blast radius is user config-file content
-	atomicWriteFile(statePath, JSON.stringify(state, null, 2) + '\n');
+	return atomicWriteFile(statePath, serializeConfigState(state), { skipIfUnchanged: true });
+}
+
+function configStateMatchesDisk(rootPath: string, state: ConfigState): boolean {
+	try {
+		const statePath = path.join(getBackupDirPath(rootPath), STATE_FILE_NAME);
+		return fs.readFileSync(statePath, 'utf8') === serializeConfigState(state);
+	} catch {
+		return false;
+	}
+}
+
+function stageConfigState(rootPath: string, state: ConfigState): boolean {
+	const backupDir = getBackupDirPath(rootPath);
+	fs.ensureDirSync(backupDir);
+	return atomicWriteFile(path.join(backupDir, pendingStateFileName()), serializeConfigState(state));
+}
+
+// A rename, so an exhausted volume cannot refuse it and leave the config file described by nothing.
+
+export function commitStagedConfigState(rootPath: string): boolean {
+	const backupDir = getBackupDirPath(rootPath);
+	const pendingPath = path.join(backupDir, pendingStateFileName());
+	try {
+		renameWithRetry(pendingPath, path.join(backupDir, STATE_FILE_NAME));
+		return true;
+	} catch (error) {
+		// Loud: the config file is already on disk, so the confirmed state now describes values it no
+		// longer has, and the next boot reads that difference as a manual user edit.
+		getLogger().error(`Could not promote the staged env config state at ${pendingPath}: ${(error as Error).message}`);
+		return false;
+	}
+}
+
+// Drops the staged state for a config-file write that did not happen; the confirmed record stays.
+
+export function discardConfigState(rootPath: string): void {
+	const pendingPath = path.join(getBackupDirPath(rootPath), pendingStateFileName());
+	try {
+		fs.removeSync(pendingPath);
+	} catch (error) {
+		getLogger().warn(`Could not remove the staged env config state at ${pendingPath}: ${(error as Error).message}`);
+	}
 }
 
 /**
@@ -957,20 +1069,49 @@ export function applyRuntimeEnvConfig(
 	rootPath: string,
 	options: { isInstall?: boolean } = {}
 ): ConfigObject {
+	const { config, commitState } = prepareRuntimeEnvConfig(fileConfig, rootPath, options);
+	commitState();
+	return config;
+}
+
+/**
+ * Apply the env layers and hand the state writes back to the caller, so a caller that also persists
+ * the merged config file can commit the pair as a unit (#847): saveState() stages the new state
+ * beside the confirmed one, confirmConfigWritten() renames it over the confirmed one after the
+ * config file lands, and discardConfigState() drops the staged copy if that write never happens.
+ */
+export function prepareRuntimeEnvConfig(
+	fileConfig: ConfigObject,
+	rootPath: string,
+	options: { isInstall?: boolean } = {}
+): {
+	config: ConfigObject;
+	saveState: () => boolean;
+	confirmConfigWritten: () => boolean;
+	commitState: () => boolean;
+} {
 	const defaultEnvValue = process.env.HARPER_DEFAULT_CONFIG;
 	const configEnvValue = process.env.HARPER_CONFIG;
 	const setEnvValue = process.env.HARPER_SET_CONFIG;
 
-	// Load existing state
+	// Load existing state. Only the main thread persists, so only the main thread has wreckage to
+	// clear - and a worker shares its pid, so letting one scan would delete the main thread's
+	// in-flight sidecar as if it were last boot's.
+	const interruptedCommit = isMainThread && takeInterruptedCommit(rootPath);
 	const state = loadConfigState(rootPath);
 
 	// No env vars set and no previous state, nothing to do
 	if (!defaultEnvValue && !configEnvValue && !setEnvValue && Object.keys(state.snapshots).length === 0) {
-		return fileConfig;
+		return { config: fileConfig, saveState: () => false, confirmConfigWritten: () => false, commitState: () => false };
 	}
 
-	// Detect drift (user manual edits) - only at runtime, not install
-	if (!options.isInstall) {
+	// Detect drift (user manual edits) - only at runtime, not install, not on a boot that found an
+	// interrupted commit (where a difference could equally be the write that was in flight), and only
+	// on the main thread: a worker never owns the state, and one re-deriving inside the main thread's
+	// commit window would call its half-written config file a user edit and drop the env-supplied
+	// value for itself alone - serving different config than its siblings, with nothing on disk to
+	// show why.
+	if (!options.isInstall && !interruptedCommit && isMainThread) {
 		const driftedPaths = detectConfigDrift(fileConfig, state);
 		for (const path of driftedPaths) {
 			state.sources[path] = 'user';
@@ -998,8 +1139,10 @@ export function applyRuntimeEnvConfig(
 	processEnvVar(fileConfig, state, 'HARPER_CONFIG', 'HARPER_CONFIG', options);
 	processEnvVar(fileConfig, state, 'HARPER_SET_CONFIG', 'HARPER_SET_CONFIG', options);
 
-	// Save updated state
-	saveConfigState(rootPath, state);
-
-	return fileConfig;
+	return {
+		config: fileConfig,
+		saveState: () => (configStateMatchesDisk(rootPath, state) ? false : stageConfigState(rootPath, state)),
+		confirmConfigWritten: () => commitStagedConfigState(rootPath),
+		commitState: () => saveConfigState(rootPath, state),
+	};
 }

package/config-root.schema.json

@@ -519,6 +519,10 @@
 					"type": "string",
 					"description": "Max write queue time before rejecting (e.g. '45s')."
 				},
+				"longTransactionReportThreshold": {
+					"type": ["string", "number"],
+					"description": "How old a RocksDB transaction handle must be before it is reported in the log (e.g. '5m'). A handle this old can hold write intents that park other writers' commits and pin a read snapshot. Set to 0 to disable the reporting. Default: '5m'"
+				},
 				"noReadAhead": { "type": "boolean", "description": "Advise OS to not read ahead. Default: false" },
 				"randomAccessFields": {
 					"type": "boolean",
@@ -534,6 +538,31 @@
 					"oneOf": [{ "type": "string" }, { "type": "array", "items": { "type": "string" } }],
 					"description": "Path or array of paths for blob storage. Default: <rootPath>/blobs"
 				},
+				"blobs": {
+					"type": "object",
+					"description": "File-backed blob storage settings.",
+					"properties": {
+						"compression": {
+							"type": "object",
+							"description": "Opt-in compression for newly written file-backed blobs, keyed by content type: an exact type ('application/json'), a wildcard ('text/*'), or 'default'. Absent means no compression. Each entry is false (never compress the type) or an object { codec, threshold }: codec currently only 'deflate' (the default); threshold is the minimum content size in bytes to compress (default 8192). Matching precedence is exact type, then 'type/*', then 'default'. Types that are already compressed (application/gzip, application/zip, application/zstd, image/*, video/*, audio/*) ship as false and stay uncompressed unless explicitly overridden. Blobs streamed without a known size are not compressed, and a threshold below 8192 only affects streamed writes — buffers under 8KB are stored inline in the record (covered by record-level compression) and never reach blob files. Applies to new writes only; existing blob files are never rewritten.",
+							"propertyNames": { "pattern": "^([\\w.+-]+/(\\*|[\\w.+-]+)|default)$" },
+							"additionalProperties": {
+								"oneOf": [
+									{ "const": false },
+									{
+										"type": "object",
+										"properties": {
+											"codec": { "const": "deflate" },
+											"threshold": { "type": "number", "minimum": 0 }
+										},
+										"additionalProperties": false
+									}
+								]
+							}
+						}
+					},
+					"additionalProperties": false
+				},
 				"blobReadTimeout": {
 					"type": "number",
 					"description": "Max time (ms) a blob read will wait for an in-progress write to finish before failing with a 503, rather than holding the connection. Default: 20000"

package/dataLayer/blobBackup.ts

@@ -5,6 +5,12 @@ import { copyFile, link, mkdir, readdir, rename, rm, unlink, writeFile } from 'n
 import { dirname, join, relative } from 'node:path';
 import { ClientError } from '../utility/errors/hdbError.ts';
 import logger from '../utility/logging/harper_logger.ts';
+import {
+	type BlobCaptureDisposition,
+	classifyBlobFileForCapture,
+	createCaptureMarker,
+	isSystemicIoError,
+} from '../resources/blob.ts';
 
 /**
  * Managed-backup snapshotting of a database's file-backed blobs.
@@ -21,17 +27,17 @@ import logger from '../utility/logging/harper_logger.ts';
  * possible (cheap, no extra space on the same filesystem) and copied otherwise (never symlinked, so
  * a snapshot is a standalone set of files that survives independent of the live blob).
  *
- * Consistency is best-effort and point-in-time-ish, matching how the engine treats the transaction
- * log: the walk captures whatever files exist at snapshot time. A blob deleted mid-walk is skipped;
- * a blob being written mid-walk is captured as-is (a hard link shares the inode, so it reflects the
- * writer's final bytes; a cross-filesystem copy captures the bytes present at copy time). Harper
- * does not freeze blob writes for the duration of a backup.
+ * Harper does not freeze blob writes for a backup, and a blob is written in place at its final path,
+ * so the walk can meet one that is still growing. Hard-linking that inode would put bytes in the
+ * snapshot that keep changing after it was taken. Every entry returned by the walk is therefore
+ * classified (`captureBlobFile`): a complete blob is linked, and one that becomes unavailable or is
+ * not yet complete is replaced by a marker rather than dropped. This closes the classify-to-capture
+ * race and keeps that file id reserved; a file reclaimed before its parent directory is read remains
+ * outside the snapshot.
  *
- * Hard-linking is safe against later mutation because Harper blobs are content-addressed and
- * write-once: each write allocates a fresh monotonic file id (a new path), and an update or delete
- * unlinks the old path rather than rewriting it in place — so a snapshot's hard link keeps the exact
- * bytes alive even after the live blob is deleted, and no in-place overwrite can retroactively alter
- * a snapshot.
+ * Past that gate, hard-linking is safe against later mutation: each write takes a fresh monotonic
+ * file id, and an update, delete, or in-place repair replaces or unlinks the path rather than
+ * rewriting the inode.
  */
 
 /** Directory holding all blob snapshots for a backup repository. */
@@ -46,17 +52,17 @@ export function blobSnapshotDir(backupDir: string, backupId: number): string {
 
 /**
  * Hard-link `src` to `dest`, falling back to a copy when the two are on different filesystems (or
- * the filesystem does not support additional hard links). Never creates a symlink. A source that
- * vanished mid-walk (a concurrent blob delete) is skipped rather than failing the whole snapshot.
+ * the filesystem does not support additional hard links). Never creates a symlink. Returns false
+ * when the source vanishes so the caller can substitute a marker for it. `counts.copied` is what a
+ * caller on a latency budget needs: the fallback turns a constant-time clone into an O(bytes) one.
  */
-async function linkOrCopy(src: string, dest: string): Promise<void> {
+async function linkOrCopy(src: string, dest: string, counts?: { copied: number }): Promise<boolean> {
 	await mkdir(dirname(dest), { recursive: true });
 	try {
 		await link(src, dest);
 	} catch (error: any) {
 		if (error.code === 'ENOENT') {
-			// src disappeared (concurrent delete) — nothing to snapshot
-			if (!existsSync(src)) return;
+			if (!existsSync(src)) return false; // vanished since it was classified
 			throw error;
 		}
 		if (
@@ -66,24 +72,86 @@ async function linkOrCopy(src: string, dest: string): Promise<void> {
 			error.code === 'ENOTSUP' ||
 			error.code === 'EOPNOTSUPP'
 		) {
-			await copyFile(src, dest);
-			return;
+			try {
+				await copyFile(src, dest);
+			} catch (copyError: any) {
+				if (copyError.code === 'ENOENT') return false;
+				throw copyError;
+			}
+			if (counts) counts.copied++;
+			return true;
 		}
 		if (error.code === 'EEXIST') {
 			await unlink(dest);
-			await linkOrCopy(src, dest);
-			return;
+			return linkOrCopy(src, dest, counts);
 		}
 		throw error;
 	}
+	return true;
+}
+
+/** What a substituted marker says happened, so a branch clone doesn't report itself as a backup. */
+export interface CaptureMarkerReasons {
+	gone: string;
+	pending: string;
+}
+
+const BACKUP_MARKER_REASONS: CaptureMarkerReasons = {
+	gone: 'blob was deleted while this backup was being taken',
+	pending: 'blob was not yet complete when this backup was taken',
+};
+
+/** Put one blob file into the destination, returning how the entry was captured. */
+async function captureBlobFile(
+	srcPath: string,
+	destPath: string,
+	reasons: CaptureMarkerReasons = BACKUP_MARKER_REASONS,
+	counts?: { copied: number }
+): Promise<BlobCaptureDisposition> {
+	let disposition: BlobCaptureDisposition;
+	try {
+		disposition = await classifyBlobFileForCapture(srcPath);
+	} catch (error) {
+		// This rethrow is unverified: only the membership of SYSTEMIC_IO_ERRORS is tested, because
+		// reaching it needs a real host-level fault no test here can produce.
+		if (isSystemicIoError(error)) throw error;
+		// Fall back to what this walk did before it classified anything. `link()` needs no read permission
+		// on the source, so a blob this failed to *read* may still be perfectly capturable; substituting
+		// here would turn an unreadable-but-valid root into a backup of nothing but stubs.
+		logger.warn(`Could not verify blob ${srcPath} for snapshot; capturing it unverified`, error);
+		disposition = 'capture';
+	}
+	if (disposition === 'skip') return disposition;
+	if (disposition === 'capture') {
+		if (await linkOrCopy(srcPath, destPath, counts)) return disposition;
+		disposition = 'gone';
+	}
+	await mkdir(dirname(destPath), { recursive: true });
+	await writeFile(destPath, createCaptureMarker(disposition, disposition === 'gone' ? reasons.gone : reasons.pending));
+	return disposition;
 }
 
 /**
  * Recursively copy every file under `srcRoot` into `destRoot` (hard-link-else-copy), preserving the
  * relative directory structure. Missing `srcRoot` is a no-op (a database with no blobs yet).
+ *
+ * `classify` substitutes a marker for a file that is mid-write or already gone, so the destination
+ * fails loudly on that blob rather than carrying a truncated one. It belongs to the capture direction
+ * only; restore must replace exactly what the snapshot holds.
+ *
+ * Also used by branch materialization (harper#644), which clones a base's blob roots into the
+ * branch's own so the OS inode refcount does the reference counting. Its walk runs inside a window
+ * other threads wait on, which is what `onProgress` is for.
  */
-async function copyTree(srcRoot: string, destRoot: string): Promise<void> {
-	if (!existsSync(srcRoot)) return;
+export async function copyTree(
+	srcRoot: string,
+	destRoot: string,
+	classify = false,
+	reasons?: CaptureMarkerReasons,
+	onProgress?: () => void
+): Promise<{ substituted: number; captured: number; copied: number }> {
+	const counts = { substituted: 0, captured: 0, copied: 0 };
+	if (!existsSync(srcRoot)) return counts;
 	const stack: string[] = [srcRoot];
 	while (stack.length > 0) {
 		const dir = stack.pop() as string;
@@ -99,63 +167,103 @@ async function copyTree(srcRoot: string, destRoot: string): Promise<void> {
 			if (entry.isDirectory()) {
 				stack.push(srcPath);
 			} else if (entry.isFile()) {
-				await linkOrCopy(srcPath, join(destRoot, relative(srcRoot, srcPath)));
+				const destPath = join(destRoot, relative(srcRoot, srcPath));
+				if (classify) {
+					const disposition = await captureBlobFile(srcPath, destPath, reasons, counts);
+					if (disposition === 'pending' || disposition === 'gone') counts.substituted++;
+					else if (disposition === 'capture') counts.captured++;
+				} else {
+					await linkOrCopy(srcPath, destPath, counts);
+				}
+				onProgress?.();
 			}
 			// symlinks/other node types in a blob root are not expected and are intentionally skipped
 		}
 	}
+	return counts;
 }
 
 /**
- * Snapshot a database's blob roots into a backup's blob directory. Writes to a temporary sibling
- * and atomically renames into place so a create_backup that fails mid-copy never leaves a partial
- * `blobs/<backupId>/` that a later restore would trust. Overwrites any pre-existing snapshot for the
- * same id (create_backup always produces a fresh id, so this only matters on a retried offline run).
+ * Copy every blob root into `destDir` as `<rootIndex>/<relpath>`, hard-linking where possible.
+ * Writes to a temporary sibling and atomically renames into place so a run that fails mid-copy never
+ * leaves a partial directory a later restore would trust, and replaces any pre-existing `destDir`.
+ * Shared by managed-backup snapshots and `copy-db`'s standalone blob copy (harper#2048).
  */
-export async function snapshotBlobs(backupDir: string, backupId: number, blobRoots: string[]): Promise<void> {
-	const finalDir = blobSnapshotDir(backupDir, backupId);
-	const tempDir = join(blobsRootDir(backupDir), `.tmp-${backupId}`);
+export async function copyBlobRootsByIndex(destDir: string, blobRoots: string[]): Promise<void> {
+	const tempDir = destDir + '.tmp';
 	await rm(tempDir, { recursive: true, force: true });
 	await mkdir(tempDir, { recursive: true });
 	try {
+		let substituted = 0;
+		let captured = 0;
 		for (let index = 0; index < blobRoots.length; index++) {
-			await copyTree(blobRoots[index], join(tempDir, String(index)));
+			const counts = await copyTree(blobRoots[index], join(tempDir, String(index)), true);
+			substituted += counts.substituted;
+			captured += counts.captured;
 		}
-		await rm(finalDir, { recursive: true, force: true });
-		await rename(tempDir, finalDir);
+		if (substituted > 0) {
+			logger.warn(
+				`Blob copy into ${destDir} substituted ${substituted} of ${substituted + captured} blob ` +
+					`file(s) with PENDING or ERROR markers because they were not capturable whole.`
+			);
+		}
+		await rm(destDir, { recursive: true, force: true });
+		await rename(tempDir, destDir);
 	} catch (error) {
 		await rm(tempDir, { recursive: true, force: true }).catch(() => {});
 		throw error;
 	}
+}
+
+/**
+ * Snapshot a database's blob roots into a backup's blob directory. Overwrites any pre-existing
+ * snapshot for the same id (create_backup always produces a fresh id, so this only matters on a
+ * retried offline run).
+ */
+export async function snapshotBlobs(backupDir: string, backupId: number, blobRoots: string[]): Promise<void> {
+	await copyBlobRootsByIndex(blobSnapshotDir(backupDir, backupId), blobRoots);
 	await writeBlobsReadme(backupDir, blobRoots);
 }
 
 /**
  * Build the `blobs/README.md` documenting the blob snapshot layout, so an operator inspecting or
- * hand-recovering a backup can decode the numeric directories. Two variants:
- * - managed (default): a create_backup repository, where snapshots are keyed by backup id
+ * hand-recovering a backup can decode the numeric directories. Three variants:
+ * - `managed` (default): a create_backup repository, where snapshots are keyed by backup id
  *   (`<backupId>/<rootIndex>/…`) and restore is automatic via `restore_backup`.
- * - archive (`archive: true`): a downloaded `get_backup` tar, which holds a single snapshot with no
- *   backup-id level (`<rootIndex>/…`) and is restored by extracting the files back into the roots.
+ * - `archive`: a downloaded `get_backup` tar, which holds a single snapshot with no backup-id level
+ *   (`<rootIndex>/…`) and is restored by extracting the files back into the roots.
+ * - `copy`: the companion directory `copy-db` writes beside a database copy, restored by hand.
  */
-export function blobsReadmeContent(blobRoots: string[], { archive = false }: { archive?: boolean } = {}): string {
+export function blobsReadmeContent(
+	blobRoots: string[],
+	{ variant = 'managed' }: { variant?: 'managed' | 'archive' | 'copy' } = {}
+): string {
 	const rootMapping =
 		blobRoots.length > 0 ? blobRoots.map((root, index) => `      ${index} -> ${root}`).join('\n') : '      (none)';
-	const layout = archive
-		? '<rootIndex>/<shard1>/<shard2>/<fileId>'
-		: '<backupId>/<rootIndex>/<shard1>/<shard2>/<fileId>';
-	const intro = archive
-		? `This directory holds this database's file-backed blobs within a downloaded \`get_backup\` archive.
+	const layout =
+		variant === 'managed'
+			? '<backupId>/<rootIndex>/<shard1>/<shard2>/<fileId>'
+			: '<rootIndex>/<shard1>/<shard2>/<fileId>';
+	const intro =
+		variant === 'copy'
+			? `This directory holds the file-backed blobs of a \`copy-db\` database copy; the database file itself
+is the sibling \`.mdb\` this directory is named after. Blobs are addressed by database NAME and the
+configured blob roots — never by the database file's path — so a copy is only restorable with these
+files: put each \`<rootIndex>/\` tree into the matching blob root of whatever database name you
+restore the copy as (mapping below).`
+			: variant === 'archive'
+				? `This directory holds this database's file-backed blobs within a downloaded \`get_backup\` archive.
 To restore them, extract each \`<rootIndex>/\` tree back into the matching blob root (see the mapping
 below and ../README.md).`
-		: `This directory holds point-in-time snapshots of this database's file-backed blobs, captured
+				: `This directory holds point-in-time snapshots of this database's file-backed blobs, captured
 alongside each RocksDB managed backup. You do not restore these by hand — \`restore_backup\` puts
 them back automatically (see ../README.md); this file just documents the layout.`;
-	const backupIdBullet = archive
-		? ''
-		: `- **<backupId>** matches the RocksDB backup id (\`harper list_backups\`). Each id is a full,
+	const backupIdBullet =
+		variant === 'managed'
+			? `- **<backupId>** matches the RocksDB backup id (\`harper list_backups\`). Each id is a full,
   independent snapshot (not incremental).
-`;
+`
+			: '';
 	return `# Harper blob snapshots
 
 ${intro}
@@ -166,7 +274,7 @@ ${intro}
 
 ${backupIdBullet}- **<rootIndex>** is which of the database's blob roots the file came from — the index into
   \`storage.blobPaths[n]\`. When \`storage.blobPaths\` is not configured there is a single default root
-  (\`<rootPath>/blobs/<db>\`) at index 0. Current mapping for this backup:
+  (\`<rootPath>/blobs/<db>\`) at index 0. Current mapping:
 
 ${rootMapping}
 
@@ -175,8 +283,10 @@ ${rootMapping}
   4096 entries per directory). E.g. a blob with id \`0x12345678\` lives at \`12/345/678\`; a short id
   like \`0xc1a\` lives at \`0/0/c1a\`.
 
-Files are hard links to the live blobs when the backup is on the same filesystem, and copies
-otherwise.
+Complete blobs are hard links to the live blobs when the destination is on the same filesystem, and
+copies otherwise. A blob that was not capturable whole is stored as a marker instead: header type
+\`0xfe\` is retryable (PENDING), while \`0xff\` is terminal (ERROR). The marker preserves the file id
+but does not contain the original blob bytes.
 `;
 }
 

package/dataLayer/delete.ts

@@ -110,7 +110,12 @@ export async function deleteAuditLogsBefore(deleteObj: any) {
 	harperLogger.info(`Finished deleting audit logs before ${deleteObj.timestamp}`);
 
 	// `deleteTransactionLogsBefore` returns `entries_deleted`; the legacy result exposes it as `transactions_deleted`.
-	return new DeleteAuditLogsBeforeResults(results.start_timestamp, results.end_timestamp, results.entries_deleted);
+	return new DeleteAuditLogsBeforeResults(
+		results.start_timestamp,
+		results.end_timestamp,
+		results.entries_deleted,
+		results.log_files_deleted
+	);
 }
 
 /**