@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/dist/resources/blob.js

@@ -12,23 +12,31 @@
  *   - Note that for compressed data, the size is the uncompressed size, and the compressed size in the file
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.databasePaths = exports.BLOB_UNAVAILABLE_STATUS = exports.blobsWereEncoded = exports.Blob = void 0;
+exports.databasePaths = exports.BLOB_UNAVAILABLE_STATUS = exports.blobsWereEncoded = exports.Blob = exports.DEFLATE_TYPE = void 0;
 exports.registerBlobReceiveInFlight = registerBlobReceiveInFlight;
 exports.unregisterBlobReceiveInFlight = unregisterBlobReceiveInFlight;
 exports.isBlobReceiveInFlight = isBlobReceiveInFlight;
 exports.createPendingMarkerBarrier = createPendingMarkerBarrier;
+exports.watchInProgressFile = watchInProgressFile;
 exports.getBlobHoldStateForTesting = getBlobHoldStateForTesting;
 exports.holdBlobFile = holdBlobFile;
 exports.deleteBlob = deleteBlob;
 exports.setDeletionDelay = setDeletionDelay;
+exports.resolveBlobCompression = resolveBlobCompression;
 exports.createBlob = createBlob;
+exports.createBlobFromStoredBody = createBlobFromStoredBody;
 exports.saveBlob = saveBlob;
 exports.shouldDestroyIdleBlobSource = shouldDestroyIdleBlobSource;
+exports.createStoredDeflateVerifier = createStoredDeflateVerifier;
 exports.getFileId = getFileId;
 exports.isSaving = isSaving;
 exports.getFilePathForBlob = getFilePathForBlob;
 exports.repairBlobFile = repairBlobFile;
+exports.isSystemicIoError = isSystemicIoError;
+exports.classifyBlobFileForCapture = classifyBlobFileForCapture;
+exports.createCaptureMarker = createCaptureMarker;
 exports.blobHeaderIndicatesIncomplete = blobHeaderIndicatesIncomplete;
+exports.blobFileMissingOrIncompleteAsync = blobFileMissingOrIncompleteAsync;
 exports.blobFileMissingOrIncomplete = blobFileMissingOrIncomplete;
 exports.getRootBlobPathsForDB = getRootBlobPathsForDB;
 exports.getBlobPathsForDatabaseName = getBlobPathsForDatabaseName;
@@ -47,7 +55,9 @@ exports.startPreCommitBlobsForRecord = startPreCommitBlobsForRecord;
 exports.cleanupUnusedBlobs = cleanupUnusedBlobs;
 exports.collectRetainedFileIds = collectRetainedFileIds;
 exports.cleanupOrphans = cleanupOrphans;
+exports.inflatesToExactly = inflatesToExactly;
 exports.isBlobComplete = isBlobComplete;
+exports.openStoredBlobBody = openStoredBlobBody;
 exports.findIncompleteBlobRefs = findIncompleteBlobRefs;
 const msgpackr_1 = require("msgpackr");
 const promises_1 = require("node:fs/promises");
@@ -59,25 +69,31 @@ const environmentManager_ts_1 = require("../utility/environment/environmentManag
 const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
 const path_1 = require("path");
 const logger_ts_1 = require("../utility/logging/logger.js");
+const watchPath_ts_1 = require("../utility/watchPath.js");
 const contentTypes_ts_1 = require("../server/serverHelpers/contentTypes.js");
 const node_v8_1 = require("node:v8");
 const promises_2 = require("node:timers/promises");
 const globals_js_1 = require("../globals.js");
+function discardStorage(storageInfo) {
+    (storageInfo.fileState ??= {}).discarded = true;
+}
 const FILE_STORAGE_THRESHOLD = 8192; // if the file is below this size, we will store it in memory, or within the record itself, otherwise we will store it in a file
 // We want to keep the file path private (but accessible to the extension)
 const HEADER_SIZE = 8;
 const UNCOMPRESSED_TYPE = 0;
-const DEFLATE_TYPE = 1;
+exports.DEFLATE_TYPE = 1;
 const ERROR_TYPE = 0xff;
 // A write that aborted on a re-streamable external source (replication receive / origin fetch) stamps the
 // file with this type so a downstream read returns 503 (retry) rather than 500 (confidently incomplete).
-// The bytes are still expected — the receive side holds a blob gap and re-streams on reconnect, which
-// overwrites this stub; a terminal give-up unlinks the file (→ 404). Distinct from ERROR_TYPE (a permanent
-// corrupt/error stub, replicated as-is). See harper-pro#481.
+// The re-stream builds a fresh blob and takes a NEW file id (harper-pro's `createBlob` → `saveBlob`), so
+// this stub is orphaned rather than overwritten; a terminal give-up unlinks it (→ 404). No path rewrites
+// a published blob file in place: `repairBlobFile` is the only same-id writer and it publishes via a
+// `.repair` temp + rename. Distinct from ERROR_TYPE (a permanent corrupt/error stub, replicated as-is).
+// See harper-pro#481.
 const PENDING_TYPE = 0xfe;
 const BLOB_REPAIR_SUFFIX = '.repair';
 const DEFAULT_HEADER = new Uint8Array([0, UNCOMPRESSED_TYPE, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
-const COMPRESS_HEADER = new Uint8Array([0, DEFLATE_TYPE, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
+const COMPRESS_HEADER = new Uint8Array([0, exports.DEFLATE_TYPE, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
 const UNKNOWN_SIZE = 0xffffffffffff;
 const storageInfoForBlob = new WeakMap();
 // Cross-thread receive-in-flight signal. The lock lives on the shared store (same primitive
@@ -215,6 +231,50 @@ const BLOB_GONE_STATUS = 404;
 // retryable class from the permanent ones without duplicating the status code.
 exports.BLOB_UNAVAILABLE_STATUS = 503;
 const BLOB_CORRUPT_STATUS = 500;
+/**
+ * Arm the one-shot watch that wakes an in-progress blob read of `filePath`, or return `undefined`
+ * for a read that has to poll instead. `isLive` decides whether the read still owns a callback's
+ * watcher: a callback from one it has already replaced must not act, or it would close the live
+ * watcher and start a second read at the same position. `onFailure` runs only for a live watcher
+ * that fails after registration — a registration that throws latches `mustPoll` and leaves the
+ * caller in its own no-watcher branch.
+ */
+function watchInProgressFile(filePath, watchTarget, handlers, watchFile = node_fs_1.watch) {
+    if (watchTarget.mustPoll)
+        return undefined;
+    let watcher;
+    try {
+        watcher = watchFile(watchTarget.path, { persistent: false }, () => {
+            if (handlers.isLive(watcher))
+                handlers.onChange();
+        });
+    }
+    catch (error) {
+        // fs.watch throws synchronously when the OS watcher pool is exhausted (EMFILE/ENOSPC). Latch
+        // on the stream-scoped target, or the caller's re-poll re-enters here and re-attempts the same
+        // failing registration for the rest of the read.
+        logger_ts_1.logger.debug?.(`Could not watch ${filePath} for in-progress writes, polling instead:`, error);
+        watchTarget.mustPoll = true;
+        return undefined;
+    }
+    // An FSWatcher that fails after registration emits 'error'; with no listener Node rethrows it out
+    // of the watcher callback.
+    watcher.on('error', (error) => {
+        if (!handlers.isLive(watcher))
+            return;
+        logger_ts_1.logger.debug?.(`Watch of ${filePath} failed, polling instead:`, error);
+        watchTarget.mustPoll = true;
+        try {
+            watcher.close();
+        }
+        catch {
+            // A close() that throws here must not skip onFailure — the poll fallback is what recovers
+            // the read, and the exception would otherwise escape this listener uncaught.
+        }
+        handlers.onFailure();
+    });
+    return watcher;
+}
 class BlobReadError extends Error {
     statusCode;
     code;
@@ -231,6 +291,41 @@ class BlobReadError extends Error {
             this.code = 'ENOENT';
     }
 }
+/**
+ * Wait until no writer holds the blob's write lock, bounded by the blob read timeout. This is the
+ * completeness authority for a compressed body: its header stores the uncompressed size, so unlike
+ * an uncompressed blob its length on disk proves nothing, and inflating a partial deflate stream
+ * errors. Resolving means the writer (if any) has finished — not that this caller holds the lock.
+ */
+function waitForBlobWriteCompletion(storageInfo) {
+    const store = storageInfo.store;
+    const lockKey = storageInfo.fileId + ':blob';
+    return new Promise((resolve, reject) => {
+        // Probe first: the writer lock is free on the overwhelming majority of reads, so arm the
+        // timeout only when a writer actually holds it — an uncontended compressed read pays a
+        // tryLock, not a setTimeout.
+        let settled = false;
+        let timer;
+        function onReleased() {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve();
+        }
+        if (store.tryLock(lockKey, onReleased)) {
+            store.unlock(lockKey);
+            return resolve();
+        }
+        timer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            reject(new BlobReadError(`Blob ${storageInfo.fileId} is unavailable; the in-progress write did not complete in time`, exports.BLOB_UNAVAILABLE_STATUS));
+        }, getBlobReadTimeout());
+        timer.unref();
+    });
+}
 // We want FileBackedBlob instances to be an instanceof Blob, but we don't want to actually extend the class and call Blob's constructor, which is quite expensive because it has to set it up as a transferrable.
 function InstanceOfBlobWithNoConstructor() { }
 InstanceOfBlobWithNoConstructor.prototype = exports.Blob.prototype;
@@ -329,8 +424,6 @@ class FileBackedBlob extends exports.Blob {
                         throw new BlobReadError('Blob pending replication for ' + filePath, exports.BLOB_UNAVAILABLE_STATUS);
                     }
                     size = Number(headerValue & 0xffffffffffffn);
-                    if (size < end)
-                        size = end;
                     if (size < UNKNOWN_SIZE) {
                         if (isFullRead && descriptorSize != null && descriptorSize < UNKNOWN_SIZE && size !== descriptorSize) {
                             // the header claims a different (uncompressed) size than the record descriptor: the file
@@ -449,7 +542,7 @@ class FileBackedBlob extends exports.Blob {
             // Only sniff the storage-type byte and take the decompress branch when a full header is present.
             // A file shorter than the header (truncated/corrupted, #1424) would otherwise be mis-read as a
             // deflate body and yield garbage; instead it falls through to checkCompletion as incomplete.
-            if (rawBytes.length >= HEADER_SIZE && rawBytes[1] === DEFLATE_TYPE) {
+            if (rawBytes.length >= HEADER_SIZE && rawBytes[1] === exports.DEFLATE_TYPE) {
                 // An in-flight compressed file still carrying the UNKNOWN_SIZE placeholder is definitely
                 // incomplete; otherwise the size is finalized but the compressed body may still be streaming,
                 // so verify via the write lock before inflating (inflating a partial deflate stream errors).
@@ -457,10 +550,22 @@ class FileBackedBlob extends exports.Blob {
                 if (pending)
                     return pending;
                 return new Promise((resolve, reject) => {
-                    // start/end index into the UNCOMPRESSED content, so inflate first, then slice
-                    (0, node_zlib_1.inflate)(rawBytes.subarray(HEADER_SIZE), (error, result) => {
-                        if (error)
-                            reject(error);
+                    // start/end index into the UNCOMPRESSED content, so inflate first, then slice — with
+                    // output capped at the header's declared size, so a body that inflates past its own
+                    // header (a lying or corrupted header) is refused instead of allocating without bound,
+                    // matching the streaming read path's ceiling. zlib rejects a cap of 0, so an empty
+                    // blob's cap is 1 and the exact-length check below covers it.
+                    (0, node_zlib_1.inflate)(rawBytes.subarray(HEADER_SIZE), { maxOutputLength: Math.max(size, 1) }, (error, result) => {
+                        if (error) {
+                            const code = error.code ?? '';
+                            reject(code === 'ERR_BUFFER_TOO_LARGE'
+                                ? new BlobReadError(`Blob ${filePath} inflates past its declared size of ${size}`, BLOB_CORRUPT_STATUS)
+                                : code.startsWith('Z_')
+                                    ? new BlobReadError(`Blob ${filePath} is not a valid deflate stream: ${error.message}`, BLOB_CORRUPT_STATUS)
+                                    : error);
+                        }
+                        else if (result.length !== size)
+                            reject(new BlobReadError(`Blob ${filePath} inflates to ${result.length} bytes, not its declared size of ${size}`, BLOB_CORRUPT_STATUS));
                         else
                             resolve(sliceContent(result));
                     });
@@ -507,6 +612,20 @@ class FileBackedBlob extends exports.Blob {
         let position = 0;
         let totalContentRead = 0;
         let watcher;
+        // Drop the live watcher before closing it, so a callback racing the close cannot pass its
+        // `isLive` check — and so a close() on an already-failed handle cannot abandon the teardown
+        // it was part of. Every site that retires this watcher goes through here.
+        const closeWatcher = () => {
+            const opened = watcher;
+            watcher = null;
+            try {
+                opened?.close();
+            }
+            catch (error) {
+                logger_ts_1.logger.debug?.(`Could not close the in-progress watch of ${filePath}:`, error);
+            }
+        };
+        let watchTarget;
         let timer;
         // The start() open-retry timer lives in a different scope/phase than pull()'s `timer`; track it
         // separately so a cancel() during the file-creation wait clears it instead of leaking an fd (#1457).
@@ -535,6 +654,174 @@ class FileBackedBlob extends exports.Blob {
         // full reads can be compared against the header.
         const descriptorSize = this.size;
         const isFullRead = start === undefined && end === undefined;
+        let compressedRead;
+        let teardownCompressedRead;
+        // `bodyPrefix` is what the first raw read already pulled past the header; `bodyPosition` is where
+        // the file read continues. Both stay valid across the writer wait because a body is only ever
+        // appended to, and the stamps that do overwrite it (PENDING/ERROR) change the header re-read below.
+        async function* readCompressedBlob(bodyPrefix, bodyPosition) {
+            // The writer's lock is the only completeness authority for a compressed body (the header
+            // stores the uncompressed size, so body length proves nothing; a partial deflate stream
+            // errors on inflate). Wait for it — bounded, like every other blob read wait (#1423).
+            await waitForBlobWriteCompletion(storageInfo);
+            if (cancelled || fd == null)
+                return;
+            // An in-place repair renames a fresh (uncompressed) file over the path while holding the
+            // writer lock, so the descriptor we opened before the wait can be left on the orphaned inode.
+            // The repair may finish either during our wait or in the gap between the first read and the
+            // lock probe, so detect it by file identity rather than by whether we waited: if the path now
+            // resolves to a different inode than our descriptor, the file was swapped. Reopen so the header
+            // re-read and the stream below see the current file — a repaired (uncompressed) header is then
+            // reported retryable 503 (mirroring openStoredBlobBody), and the prefix read from the old inode
+            // is discarded. If the inode is unchanged the descriptor is still valid and the same-fd re-read
+            // still catches an in-place PENDING/ERROR stamp (harper-pro#481).
+            let fileSwapped = false;
+            try {
+                const currentFd = fd;
+                const [openStat, pathStat] = await Promise.all([
+                    new Promise((res, rej) => (0, node_fs_1.fstat)(currentFd, (error, s) => (error ? rej(error) : res(s)))),
+                    new Promise((res, rej) => (0, node_fs_1.stat)(filePath, (error, s) => (error ? rej(error) : res(s)))),
+                ]);
+                fileSwapped = openStat.ino !== pathStat.ino;
+            }
+            catch {
+                // the path is momentarily unstatable, or the descriptor was closed by a concurrent cancel;
+                // fall through — the cancellation recheck below returns, and otherwise the header re-read
+                // reports the outcome. The blob hold makes a vanished path unlikely here.
+            }
+            if (cancelled || fd == null)
+                return; // the identity stat awaited; a cancel may have landed
+            if (fileSwapped) {
+                // Null `fd` before closing it: a closed descriptor number left in `fd` across the await
+                // below would be closed a second time by a concurrent cancel()/closeFd() — after the OS may
+                // have reassigned it to an unrelated file or socket (#1457).
+                const staleFd = fd;
+                fd = null;
+                (0, node_fs_1.close)(staleFd);
+                fd = await new Promise((resolveOpen, rejectOpen) => (0, node_fs_1.open)(filePath, 'r', (error, openedFd) => (error ? rejectOpen(error) : resolveOpen(openedFd))));
+                if (cancelled) {
+                    closeFd();
+                    return;
+                }
+                bodyPrefix = Buffer.alloc(0);
+                bodyPosition = HEADER_SIZE;
+            }
+            // Re-read the header from the (current) fd: the writer we just waited for finalizes the size
+            // in place, and an aborted receive stamps PENDING/ERROR over the same inode (harper-pro#481).
+            const header = Buffer.allocUnsafe(HEADER_SIZE);
+            const headerBytes = await new Promise((resolveRead, rejectRead) => (0, node_fs_1.read)(fd, header, 0, HEADER_SIZE, 0, (error, bytesRead) => (error ? rejectRead(error) : resolveRead(bytesRead))));
+            if (cancelled || fd == null)
+                return;
+            if (headerBytes < HEADER_SIZE)
+                throw new BlobReadError(`Incomplete blob for ${filePath}`, BLOB_CORRUPT_STATUS);
+            const headerValue = new DataView(header.buffer, header.byteOffset, HEADER_SIZE).getBigUint64(0);
+            const type = Number(headerValue >> 48n);
+            if (type === ERROR_TYPE || type === PENDING_TYPE) {
+                const message = Buffer.allocUnsafe(256);
+                const messageBytes = await new Promise((resolveRead) => (0, node_fs_1.read)(fd, message, 0, message.length, HEADER_SIZE, (error, bytesRead) => resolveRead(error ? 0 : bytesRead)));
+                throw type === ERROR_TYPE
+                    ? new BlobReadError('Error in blob: ' + message.subarray(0, messageBytes), BLOB_CORRUPT_STATUS)
+                    : new BlobReadError('Blob pending replication for ' + filePath, exports.BLOB_UNAVAILABLE_STATUS);
+            }
+            const declaredSize = Number(headerValue & 0xffffffffffffn);
+            if (type === UNCOMPRESSED_TYPE) {
+                // We sniffed DEFLATE, waited, and the file is now uncompressed: an in-place repair
+                // replaced it with a healthy uncompressed body. Report retryable so the caller re-reads
+                // the current file through the ordinary path rather than misclassifying it as corrupt.
+                throw new BlobReadError(`Blob ${filePath} was replaced while reading; retry`, exports.BLOB_UNAVAILABLE_STATUS);
+            }
+            if (type !== exports.DEFLATE_TYPE || declaredSize === UNKNOWN_SIZE) {
+                // the writer is confirmed done, so a placeholder size (or a header no longer readable as
+                // deflate) is a write that never finished — confidently incomplete, not merely in progress
+                throw new BlobReadError(`Incomplete blob for ${filePath}`, BLOB_CORRUPT_STATUS);
+            }
+            if (isFullRead && descriptorSize != null && descriptorSize < UNKNOWN_SIZE && declaredSize !== descriptorSize) {
+                throw new BlobReadError(`Blob size mismatch for ${filePath}: record descriptor expects ${descriptorSize} bytes, on-disk header reports ${declaredSize}`, BLOB_CORRUPT_STATUS);
+            }
+            if (blob.size !== declaredSize) {
+                blob.size = declaredSize;
+                if (blob.#onSize) {
+                    for (const callback of blob.#onSize)
+                        callback(declaredSize);
+                }
+            }
+            // The read stream takes sole ownership of the descriptor (nulling `fd` leaves closeFd() only
+            // the hold to release): an fs stream's worker-thread close(2) races any other user of the fd.
+            const source = (0, node_fs_1.createReadStream)(filePath, { fd, start: bodyPosition, autoClose: true });
+            fd = null;
+            const inflater = (0, node_zlib_1.createInflate)();
+            teardownCompressedRead = () => {
+                source.destroy();
+                inflater.destroy();
+            };
+            // pipe() does not forward source errors to its destination; without this a disk fault
+            // mid-read would be an unhandled 'error' on the source stream.
+            source.on('error', (error) => inflater.destroy(error));
+            if (bodyPrefix.length > 0)
+                inflater.write(bodyPrefix);
+            source.pipe(inflater);
+            const sliceStart = start ?? 0;
+            let inflatedTotal = 0; // content offset consumed so far (emitted or discarded)
+            try {
+                for await (const chunk of inflater) {
+                    const chunkStart = inflatedTotal;
+                    inflatedTotal += chunk.length;
+                    if (inflatedTotal > declaredSize) {
+                        // refused before emitting, so a body that lies about its length cannot stream unbounded output
+                        throw new BlobReadError(`Blob ${filePath} inflates past its declared size of ${declaredSize}`, BLOB_CORRUPT_STATUS);
+                    }
+                    const emitFrom = Math.max(sliceStart - chunkStart, 0);
+                    const emitTo = end === undefined ? chunk.length : Math.min(end - chunkStart, chunk.length);
+                    if (emitTo > emitFrom)
+                        yield emitFrom === 0 && emitTo === chunk.length ? chunk : chunk.subarray(emitFrom, emitTo);
+                    if (end !== undefined && inflatedTotal >= end)
+                        return; // slice satisfied; stop inflating
+                }
+                if (inflatedTotal !== declaredSize) {
+                    // clean deflate end short of the declared size: torn despite the writer being done
+                    throw new BlobReadError(`Incomplete blob for ${filePath}: inflated to ${inflatedTotal} bytes of a declared ${declaredSize}`, BLOB_CORRUPT_STATUS);
+                }
+            }
+            catch (error) {
+                if (cancelled)
+                    return;
+                if (!(error instanceof BlobReadError) && error.code?.startsWith?.('Z_')) {
+                    // the writer is confirmed done, so a zlib failure is real corruption, not an in-progress body
+                    throw new BlobReadError(`Error reading compressed blob for ${filePath}: ${error.message}`, BLOB_CORRUPT_STATUS);
+                }
+                throw error;
+            }
+            finally {
+                teardownCompressedRead();
+                teardownCompressedRead = undefined;
+            }
+        }
+        const pumpCompressedRead = (controller) => compressedRead.next().then((step) => {
+            if (cancelled)
+                return;
+            if (step.done) {
+                closeFd();
+                try {
+                    controller.close();
+                }
+                catch {
+                    // controller may already be closed
+                }
+                return;
+            }
+            try {
+                controller.enqueue(step.value);
+            }
+            catch (error) {
+                logger_ts_1.logger.debug?.('Error enqueuing chunk', error);
+            }
+        }, (error) => {
+            closeFd();
+            if (cancelled)
+                return;
+            blob.#onError?.forEach((callback) => callback(error));
+            throw error;
+        });
         return new ReadableStream({
             start() {
                 // Bound the wait for the file to appear so a write that was abandoned (e.g. the file was
@@ -586,6 +873,8 @@ class FileBackedBlob extends exports.Blob {
                 return new Promise(openFile);
             },
             pull: (controller) => {
+                if (compressedRead)
+                    return pumpCompressedRead(controller);
                 let size = 0;
                 let retries = 100;
                 // No-progress deadline for the incomplete-content wait below, mirroring the open-retry loop's
@@ -608,8 +897,7 @@ class FileBackedBlob extends exports.Blob {
                         settled = true;
                         closeFd();
                         clearTimeout(timer);
-                        if (watcher)
-                            watcher.close();
+                        closeWatcher();
                         reject(error);
                         blob.#onError?.forEach((callback) => callback(error));
                     }
@@ -652,35 +940,12 @@ class FileBackedBlob extends exports.Blob {
                                 // half-replicated blob whose source stream aborted; the bytes are still expected — retry (harper-pro#481)
                                 return onError(new BlobReadError('Blob pending replication for ' + filePath, exports.BLOB_UNAVAILABLE_STATUS));
                             }
-                            if (buffer[1] === DEFLATE_TYPE) {
-                                // We can't seek/slice a deflate stream by uncompressed offset, so hand off to the
-                                // buffered inflate path (bytes() inflates then slices) and emit it as one chunk.
-                                // Safe by construction: the read loop never streams the raw compressed body.
-                                return blob.bytes().then((bytes) => {
-                                    // bytes() resolves asynchronously; the consumer may have cancelled meanwhile.
-                                    // Settle exactly once and route the close through closeFd() so we never touch a
-                                    // reassigned/nulled descriptor (#1457).
-                                    if (settled || cancelled)
-                                        return resolve();
-                                    settled = true;
-                                    closeFd();
-                                    if (bytes.length > 0) {
-                                        try {
-                                            controller.enqueue(bytes);
-                                        }
-                                        catch (error) {
-                                            logger_ts_1.logger.debug?.('Error enqueuing chunk', error);
-                                            return resolve();
-                                        }
-                                    }
-                                    try {
-                                        controller.close();
-                                    }
-                                    catch {
-                                        // controller may already be closed
-                                    }
-                                    resolve();
-                                }, (error) => onError(error));
+                            if (buffer[1] === exports.DEFLATE_TYPE) {
+                                // a deflate body cannot be length-checked or offset-seeked: this and every later
+                                // pull go through the streaming inflate iterator instead of the raw read loop
+                                settled = true;
+                                compressedRead = readCompressedBlob(buffer.subarray(HEADER_SIZE, bytesRead), bytesRead);
+                                return pumpCompressedRead(controller).then(resolve, reject);
                             }
                             size = Number(headerValue & 0xffffffffffffn);
                             if (size < UNKNOWN_SIZE) {
@@ -728,10 +993,7 @@ class FileBackedBlob extends exports.Blob {
                                             if (updatedSize === UNKNOWN_SIZE)
                                                 return false;
                                             size = updatedSize;
-                                            if (watcher) {
-                                                watcher.close();
-                                                watcher = null;
-                                            }
+                                            closeWatcher();
                                             // The header reports a known final size but the bytes at `position` have not arrived.
                                             // Re-entering readMore() synchronously here busy-spins the worker at ~100% CPU on a
                                             // present-but-truncated blob (header rewritten to a self-consistent smaller size, lock
@@ -749,26 +1011,39 @@ class FileBackedBlob extends exports.Blob {
                                             return true;
                                         };
                                         // the file is not finished being written, watch the file for changes to resume reading
-                                        // set up a watcher to be notified of file changes
-                                        watcher = (0, node_fs_1.watch)(filePath, { persistent: false }, () => {
-                                            if (watcher) {
-                                                watcher.close();
-                                                watcher = null;
+                                        watchTarget ??= (0, watchPath_ts_1.resolveWatchTarget)(filePath);
+                                        watcher = watchInProgressFile(filePath, watchTarget, {
+                                            isLive: (candidate) => watcher === candidate,
+                                            onChange: () => {
+                                                closeWatcher();
                                                 clearTimeout(timer); // clear it
                                                 readMore(resolve, reject);
-                                            }
+                                            },
+                                            onFailure: () => {
+                                                watcher = null;
+                                                clearTimeout(timer);
+                                                timer = setTimeout(() => readMore(resolve, reject), 20).unref();
+                                            },
                                         });
                                         // immediately try to read again in case there was a change before we started watching,
                                         // readSync should be fine here, the data should be in memory
                                         if ((0, node_fs_1.readSync)(fd, buffer, 0, buffer.length, position) > 0) {
                                             // never mind with the watcher, let's read more data
-                                            if (watcher) {
-                                                watcher.close();
-                                                watcher = null;
-                                            }
+                                            closeWatcher();
                                             readMore(resolve, reject);
                                         }
                                         else if (!resumeIfWriterFinished()) {
+                                            if (!watcher) {
+                                                // Nothing will wake this read, so poll on the deadline resumeIfWriterFinished uses
+                                                // rather than sitting out the full read timeout and 503-ing a healthy write.
+                                                if (Date.now() >= incompleteDeadline) {
+                                                    onError(new BlobReadError(`File read timed out reading from ${filePath}, read ${totalContentRead} bytes, but size is supposed to be ${size} bytes`, exports.BLOB_UNAVAILABLE_STATUS));
+                                                }
+                                                else {
+                                                    timer = setTimeout(() => readMore(resolve, reject), 20).unref();
+                                                }
+                                                return;
+                                            }
                                             // set a timer for the watcher too. A write that stalls past the bound returns a
                                             // prompt 503 (retryable) instead of holding the connection for the full 60s (#1423).
                                             timer = setTimeout(() => {
@@ -851,11 +1126,11 @@ class FileBackedBlob extends exports.Blob {
             },
             cancel() {
                 cancelled = true;
+                teardownCompressedRead?.();
                 closeFd(); // releases the hold, including when cancelled before any open succeeded
                 clearTimeout(timer);
                 clearTimeout(openTimer);
-                if (watcher)
-                    watcher.close();
+                closeWatcher();
             },
         });
         function checkIfIsBeingWritten() {
@@ -877,6 +1152,7 @@ class FileBackedBlob extends exports.Blob {
         if (sourceStorageInfo?.fileId) {
             const slicedStorageInfo = {
                 ...sourceStorageInfo,
+                fileState: (sourceStorageInfo.fileState ??= {}),
                 start,
                 end,
             };
@@ -1153,8 +1429,24 @@ function deleteBlob(blob) {
             Atomics.store(state.table, state.slot + REREFERENCED, 0);
     }
     // Reusing the queued entry when two writes supersede the same file keeps the age cap measuring
-    // from the first supersession.
-    const pending = pendingReclamation.get(filePath) ?? { blob, deadline: 0, enqueuedAt: now, supersededAt: now };
+    // from the first supersession and tracks every live blob instance that carries the condemned fileId.
+    const pending = pendingReclamation.get(filePath) ?? {
+        blobs: [],
+        seenBlobs: new WeakSet(),
+        fileInfo: { store: storageInfo?.store, fileId: storageInfo?.fileId },
+        deadline: 0,
+        enqueuedAt: now,
+        supersededAt: now,
+    };
+    if (!pending.seenBlobs.has(blob)) {
+        pending.seenBlobs.add(blob);
+        pending.blobs.push(new WeakRef(blob));
+    }
+    if (pending.unlinking) {
+        if (storageInfo)
+            discardStorage(storageInfo);
+        return;
+    }
     scheduleReclamation(enqueue(filePath, pending, Math.max(pending.deadline, now + getReclamationDelay())));
 }
 /**
@@ -1201,7 +1493,7 @@ function runReclamation() {
             earliest = pending.deadline;
             break; // insertion order is deadline order; nothing behind this entry is due
         }
-        const storageInfo = storageInfoForBlob.get(pending.blob);
+        const storageInfo = pending.fileInfo;
         const expired = now - pending.enqueuedAt >= ageCap;
         let held;
         try {
@@ -1255,6 +1547,16 @@ function runReclamation() {
         // Keep the entry until the unlink lands so a concurrent re-reference can tell that the file is
         // already going away instead of silently adopting a doomed path.
         pending.unlinking = true;
+        // Once reclamation has claimed this file, a later write must not preserve its soon-to-be-deleted
+        // fileId. The retention window above still permits legitimate re-references before this point.
+        for (const blobRef of pending.blobs) {
+            const blob = blobRef.deref();
+            if (!blob)
+                continue;
+            const instanceStorageInfo = storageInfoForBlob.get(blob);
+            if (instanceStorageInfo)
+                discardStorage(instanceStorageInfo);
+        }
         (0, node_fs_1.unlink)(filePath, (error) => {
             pendingReclamation.delete(filePath);
             if (pendingReclamation.size === 0)
@@ -1276,6 +1578,87 @@ function runReclamation() {
 function setDeletionDelay(delay) {
     reclamationDelayOverride = delay;
 }
+/**
+ * `StorageInfo.contentBuffer` must always hold a `Buffer`: `Buffer.prototype.toString()` decodes
+ * UTF-8, while the inherited `Uint8Array.prototype.toString()` joins the byte values with commas.
+ */
+function asBuffer(bytes) {
+    return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+}
+const DEFAULT_COMPRESSION_THRESHOLD = FILE_STORAGE_THRESHOLD;
+/**
+ * Already-compressed containers (the deployment recorder writes `application/gzip`). An operator entry
+ * for the same key overrides these; a broader operator wildcard does not — specificity beats origin.
+ */
+const BUILT_IN_COMPRESSION_ENTRIES = {
+    'application/gzip': false,
+    'application/zip': false,
+    'application/zstd': false,
+    'image/*': false,
+    'video/*': false,
+    'audio/*': false,
+};
+/** The matcher works on the canonical lowercase base media type, with any `;` parameters stripped. */
+function normalizeContentType(type) {
+    if (!type)
+        return '';
+    const parametersStart = type.indexOf(';');
+    return (parametersStart === -1 ? type : type.slice(0, parametersStart)).trim().toLowerCase();
+}
+// The normalized policy map is rebuilt only when the configured value object changes identity, so
+// the per-save cost is one config lookup and a reference compare.
+let compressionPolicyForConfig;
+let compressionPolicy = null;
+let invalidCompressionConfigLogged = false;
+function getBlobCompressionPolicy() {
+    const configured = (0, environmentManager_ts_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_BLOBS_COMPRESSION);
+    if (configured === compressionPolicyForConfig)
+        return compressionPolicy;
+    compressionPolicyForConfig = configured;
+    compressionPolicy = null;
+    if (configured == null)
+        return null;
+    if (typeof configured !== 'object' || Array.isArray(configured)) {
+        if (!invalidCompressionConfigLogged) {
+            invalidCompressionConfigLogged = true;
+            logger_ts_1.logger.warn?.('Ignoring invalid storage.blobs.compression value; expected a map of content types');
+        }
+        return null;
+    }
+    const policy = new Map(Object.entries(BUILT_IN_COMPRESSION_ENTRIES));
+    for (const [key, entry] of Object.entries(configured)) {
+        policy.set(normalizeContentType(key) || 'default', entry);
+    }
+    compressionPolicy = policy;
+    return policy;
+}
+/**
+ * Whether a new blob of `type` and (uncompressed) `size` should be deflate-compressed on disk,
+ * per the opt-in `storage.blobs.compression` map. Absent config means never. Matching precedence
+ * is exact type, then `type/*`, then `default`. An unknown size never compresses: the threshold
+ * cannot be evaluated, and compressing a small streamed write under a large configured threshold
+ * would violate the operator's limit.
+ */
+function resolveBlobCompression(type, size) {
+    const policy = getBlobCompressionPolicy();
+    if (!policy)
+        return false;
+    const normalized = normalizeContentType(type);
+    let entry = policy.get(normalized);
+    if (entry === undefined && normalized) {
+        const slash = normalized.indexOf('/');
+        if (slash > 0)
+            entry = policy.get(normalized.slice(0, slash + 1) + '*');
+    }
+    entry ??= policy.get('default');
+    if (!entry)
+        return false;
+    if (entry.codec !== undefined && entry.codec !== 'deflate')
+        return false; // enforced by config validation; fail safe
+    if (size === undefined)
+        return false;
+    return size >= (entry.threshold ?? DEFAULT_COMPRESSION_THRESHOLD);
+}
 /**
  * Create a blob from a readable stream or a buffer by creating a file in the blob storage path with a new unique internal id, that
  * can be saved/stored.
@@ -1292,7 +1675,7 @@ function createBlob(source, options) {
     storageInfoForBlob.set(blob, storageInfo);
     if (source instanceof Uint8Array) {
         blob.size = source.length;
-        storageInfo.contentBuffer = source;
+        storageInfo.contentBuffer = asBuffer(source);
     }
     else if (source instanceof node_stream_1.Readable) {
         storageInfo.source = source;
@@ -1306,6 +1689,22 @@ function createBlob(source, options) {
     return blob;
 }
 (0, globals_js_1._assignPackageExport)('createBlob', createBlob);
+/**
+ * Create a blob whose source stream carries the STORED representation — an already-deflated body —
+ * rather than content bytes, so a replication receiver can land a peer's compressed blob without
+ * recompressing it. `size` is the uncompressed content length (it goes in the header; a raw
+ * compressed source cannot derive it) and the body is verified on write: it must be a well-formed
+ * deflate stream inflating to exactly `size` bytes, or the save rejects (see
+ * createStoredDeflateVerifier). Internal to replication on purpose — not part of BlobCreationOptions
+ * and not exported to components.
+ */
+function createBlobFromStoredBody(source, options) {
+    if (options.codec !== 'deflate')
+        throw new Error(`Unsupported stored blob codec: ${options.codec}`);
+    const blob = createBlob(source, { type: options.type, size: options.size });
+    storageInfoForBlob.get(blob).storedCodec = options.codec;
+    return blob;
+}
 // When set (during a migration via encodeBlobsWithFilePath), saveBlob pushes the in-flight
 // writeBlob save promise here so the migration can await every blob's durable write before
 // declaring the database done. Without this, the migration's `await targetDbi.put(...)`
@@ -1319,6 +1718,13 @@ function saveBlob(blob, deleteOnFailure = false) {
         storageInfoForBlob.set(blob, storageInfo);
     }
     else {
+        if (storageInfo.fileState?.discarded) {
+            // The file this blob was saved to has been deleted (an aborted/skipped write's cleanup, or an
+            // explicit delete). Re-storing it would commit a reference to a file that no longer exists —
+            // a permanently unreadable record, and for a replicated one a blob the peer can never fetch.
+            // Fail here, where the cause is still known, instead of at the eventual read (issue #2062).
+            throw new Error('Blob was discarded (its file was deleted by an aborted or superseded write) and can no longer be stored; the data must be re-supplied');
+        }
         if (storageInfo.fileId)
             return storageInfo; // if there is any file id, we are already saving and can return the info
         storageInfo.store = currentStore;
@@ -1331,6 +1737,14 @@ function saveBlob(blob, deleteOnFailure = false) {
         }
         return storageInfo; // nothing more to do if it supposed to be saved in the record
     }
+    // The policy resolves here, not in createBlob, because the HTTP-upload path never sees creation
+    // options — every local write funnels through saveBlob. An explicit `compress` option (either
+    // value) and a pre-compressed source both take precedence.
+    if (storageInfo.compress === undefined && !storageInfo.storedCodec) {
+        const size = blob.size ?? storageInfo.contentBuffer?.length;
+        if (resolveBlobCompression(blob.type, size))
+            storageInfo.compress = true;
+    }
     generateFilePath(storageInfo);
     if (storageInfo.source)
         writeBlobWithStream(blob, storageInfo.source, storageInfo);
@@ -1406,11 +1820,92 @@ function getBlobStreamIdleTimeoutMs(stream) {
 function shouldDestroyIdleBlobSource(paused, bytesWritten, lastProgressBytes) {
     return !(paused && bytesWritten > lastProgressBytes);
 }
+/**
+ * Pass-through for a pre-compressed (stored deflate) blob body that concurrently inflates what it
+ * forwards and fails the pipeline unless the body is a well-formed deflate stream inflating to
+ * exactly `expectedSize` bytes. This is what lets a raw peer-preserved body land under a finalized
+ * DEFLATE header without trusting the peer: a truncated, corrupt, oversized (bomb), or trailing-
+ * garbage body rejects the save instead of being published.
+ */
+function createStoredDeflateVerifier(expectedSize, fileId) {
+    const inflater = (0, node_zlib_1.createInflate)();
+    let inflatedLength = 0;
+    let failure;
+    // The in-flight transform() callback, kept so a failure can settle it directly. A mid-write
+    // zlib data error (a corrupt, not merely truncated, body) fires 'error'/'close' but NEVER invokes
+    // the pending `inflater.write` callback, so a transform that waited only on that callback would
+    // stall forever — hanging the save pipeline and leaking the blob write lock. settlePending() is
+    // the escape: whichever of the write callback or the failure runs first clears it; the other is a
+    // no-op, so callback() is still called exactly once per chunk.
+    let pending;
+    const settlePending = () => {
+        if (!pending)
+            return;
+        const { callback, chunk } = pending;
+        pending = undefined;
+        callback(failure, failure ? undefined : chunk);
+    };
+    const fail = (error) => {
+        failure ??= error;
+        inflater.destroy();
+        settlePending();
+    };
+    inflater.on('data', (inflated) => {
+        inflatedLength += inflated.length;
+        if (inflatedLength > expectedSize)
+            fail(new Error(`Pre-compressed blob body for ${fileId} inflates past its declared size of ${expectedSize}`));
+    });
+    inflater.on('error', (error) => fail(new Error(`Pre-compressed blob body for ${fileId} is not a valid deflate stream`, { cause: error })));
+    // Trailing data past the final deflate block is detected best-effort (zlib may acknowledge a write
+    // before emitting the early 'end' it triggers); readers ignore trailing bytes, so the residue is
+    // only wasted disk, while truncation, wrong length, invalid stream, and oversize are deterministic.
+    const trailingData = () => new Error(`Pre-compressed blob body for ${fileId} continues past the end of its deflate stream`);
+    let sawDeflateEnd = false;
+    inflater.on('end', () => {
+        if (!inflater.writableEnded)
+            fail(trailingData());
+        sawDeflateEnd = true;
+    });
+    return new node_stream_1.Transform({
+        transform(chunk, _encoding, callback) {
+            if (!failure && sawDeflateEnd)
+                fail(trailingData());
+            if (failure)
+                return callback(failure);
+            // Forward the chunk only once zlib has consumed it: this is the backpressure coupling and
+            // what keeps the end/overflow/error detections ordered ahead of the next chunk. If zlib
+            // data-errors while consuming this chunk it drops the write callback, so fail()/settlePending()
+            // settles this transform instead (see the `pending` note above).
+            pending = { callback, chunk };
+            inflater.write(chunk, (writeError) => {
+                if (!pending)
+                    return; // already settled by a failure; do not call callback twice
+                pending = undefined;
+                const error = failure ?? writeError;
+                callback(error, error ? undefined : chunk);
+            });
+        },
+        flush(callback) {
+            if (failure)
+                return callback(failure);
+            inflater.on('close', () => {
+                if (!failure && inflatedLength !== expectedSize)
+                    failure = new Error(`Pre-compressed blob body for ${fileId} inflated to ${inflatedLength} bytes; expected ${expectedSize}`);
+                callback(failure);
+            });
+            inflater.end();
+        },
+        destroy(error, callback) {
+            inflater.destroy();
+            callback(error);
+        },
+    });
+}
 function writeBlobWithStream(blob, stream, storageInfo, options) {
     const repairTargetPath = options?.repairTargetPath;
     const repairing = repairTargetPath !== undefined;
     const repairTempLockKey = options?.repairTempLockKey;
-    const { filePath, fileId, store, compress, flush } = storageInfo;
+    const { filePath, fileId, store, compress, storedCodec, flush } = storageInfo;
     storageInfo.saving = new Promise((resolve, reject) => {
         const lockKey = fileId + ':blob';
         let lockAcquired = repairing;
@@ -1458,7 +1953,7 @@ function writeBlobWithStream(blob, stream, storageInfo, options) {
             let headerValue = BigInt(size);
             const header = new Uint8Array(HEADER_SIZE);
             const headerView = new DataView(header.buffer);
-            headerValue |= BigInt(compress ? DEFLATE_TYPE : UNCOMPRESSED_TYPE) << 48n;
+            headerValue |= BigInt(compress || storedCodec ? exports.DEFLATE_TYPE : UNCOMPRESSED_TYPE) << 48n;
             headerView.setBigInt64(0, headerValue);
             return header;
         }
@@ -1499,8 +1994,8 @@ function writeBlobWithStream(blob, stream, storageInfo, options) {
                     // half-replicated blob returns 503 (retry) instead of 500 (confidently incomplete → the peer
                     // advances its resume cursor past it = silent loss, harper-pro#481). Hold the write lock until the
                     // marker is durable so no concurrent read/send observes the bare partial file (lock-free + short =
-                    // classified 500 = the very loss this prevents). The re-stream overwrites this stub
-                    // (createWriteStream flags 'w'); a terminal give-up on the receive side unlinks it (→ 404). Build
+                    // classified 500 = the very loss this prevents). The re-stream builds a fresh blob and so takes a
+                    // new file id, leaving this stub for orphan GC; a terminal give-up unlinks it (→ 404). Build
                     // the header directly rather than via createHeader so its compress-type OR can't collide with the
                     // PENDING type bits.
                     // Bounded so a `writeFile` that never calls back cannot leave `saving` un-settled for the
@@ -1606,6 +2101,20 @@ function writeBlobWithStream(blob, stream, storageInfo, options) {
                         void replaceTarget();
                     return;
                 }
+                if (compressedStream && blob.size !== compressedStream.bytesWritten) {
+                    // A known-size compressed write stamped the declared size into the header up front, but
+                    // the source ended with a different content length. Unlike the uncompressed path — whose
+                    // short body reads as retryable-incomplete (503) — this file would inflate to the wrong
+                    // size and every reader would reject it as permanently corrupt (500). Fail the save so the
+                    // record never commits a reference to an unreadable file (matches the storedCodec verifier
+                    // and the repair size check); the unreferenced file is left for orphan GC. The unknown-size
+                    // path syncs blob.size to bytesWritten before reaching here, so it never trips this.
+                    store.unlock(lockKey);
+                    reject(new Error(`Blob ${fileId} deflated ${compressedStream.bytesWritten} bytes but its header declares ${blob.size}`));
+                    (0, node_fs_1.close)(fd);
+                    writeStream.fd = null;
+                    return;
+                }
                 store.unlock(lockKey);
                 if (flush) {
                     // we just use fdatasync because we really aren't that concerned with flushing file metadata
@@ -1625,6 +2134,12 @@ function writeBlobWithStream(blob, stream, storageInfo, options) {
             }
         }
         try {
+            if (storedCodec && blob.size === undefined) {
+                // The header must carry the uncompressed length, and it cannot be derived from a raw
+                // compressed source (bytesWritten would be the compressed count). Refuse before any file
+                // exists at the path so the failure reads as a cleanly absent blob, not a torn one.
+                throw new Error(`Cannot store a pre-compressed blob without a known uncompressed size (fileId=${fileId})`);
+            }
             if (!lockAcquired) {
                 if (!store.tryLock(lockKey))
                     throw new Error(`Unable to get lock for blob file ${fileId}`);
@@ -1654,7 +2169,14 @@ function writeBlobWithStream(blob, stream, storageInfo, options) {
                 stream.on('resume', armIdleTimer);
                 armIdleTimer();
             }
-            if (compress) {
+            if (storedCodec) {
+                // The source bytes are already the stored deflate representation (a peer preserving its
+                // codec). Verify — not just copy — so a truncated or forged body can never land under a
+                // finalized DEFLATE header: the verifier inflates the pass-through bytes concurrently and
+                // fails the pipeline unless they inflate to exactly the declared uncompressed size.
+                (0, node_stream_1.pipeline)(stream, createStoredDeflateVerifier(blob.size, fileId), writeStream, finished);
+            }
+            else if (compress) {
                 if (!wroteSize)
                     writeStream.write(COMPRESS_HEADER);
                 compressedStream = (0, node_zlib_1.createDeflate)();
@@ -1701,9 +2223,11 @@ function getFilePathForBlob(blob) {
  * Repairs a damaged file-backed blob under its existing fileId. The damage check and atomic file
  * replacement share the blob lock, so a failed repair never modifies the referenced file. Callers
  * must establish an exact record identity tie (same version and source node) and positional blob
- * pairing before calling. The received byte count must match the reported source size and, when
- * present, the stored descriptor. On a synchronous `undefined`, no repair started and `source`
- * remains owned by the caller.
+ * pairing before calling, and classify the file with `blobFileMissingOrIncompleteAsync` first — a
+ * compressed body's damage is only provable by inflating it, which the locked recheck here will
+ * not do. The received byte count must match the reported source size and, when present, the
+ * stored descriptor. On a synchronous `undefined`, no repair started and `source` remains owned
+ * by the caller.
  */
 function repairBlobFile(blob, source, sourceSize = blob.size) {
     let blobLockKey;
@@ -1765,7 +2289,9 @@ function repairBlobFile(blob, source, sourceSize = blob.size) {
             filePath: repairFilePath,
             saving: undefined,
             flush: true,
+            // the repair source is inflated content; its size verification counts uncompressed bytes
             compress: false,
+            storedCodec: undefined,
         };
         writeBlobWithStream(blob, source, repairStorageInfo, {
             expectedSize,
@@ -1792,11 +2318,84 @@ function repairBlobFile(blob, source, sourceSize = blob.size) {
         return undefined;
     }
 }
+/** Fail-closed storage and descriptor errors that must abort the capture instead of producing an unverified backup. */
+const SYSTEMIC_IO_ERRORS = new Set(['EMFILE', 'ENFILE', 'ENOSPC', 'EIO', 'EROFS']);
+function isSystemicIoError(error) {
+    return SYSTEMIC_IO_ERRORS.has(error?.code ?? '');
+}
+/**
+ * How a consumer that captures a blob root (backup snapshot, backup archive) should treat one file.
+ * `skip` is not a blob to capture at all; `pending` and `gone` require markers; `capture` is taken as-is.
+ * `capture` means settled as far as the path can show, which is short of a guarantee: a known-size write
+ * that has landed every byte is indistinguishable here from a finished one, and if it then aborts, the
+ * PENDING stamp rewrites that inode in place, truncating any same-filesystem hard link taken from it.
+ * Telling the two apart needs the blob write lock, which is keyed by file id and unreachable from a walk.
+ */
+async function classifyBlobFileForCapture(filePath) {
+    if (filePath.endsWith(BLOB_REPAIR_SUFFIX))
+        return 'skip';
+    let header;
+    let fileSize;
+    let handle;
+    try {
+        handle = await (0, promises_1.open)(filePath, 'r');
+    }
+    catch (error) {
+        // Reclamation can unlink a superseded blob after the engine checkpoint but before this walk. The
+        // checkpointed record may still reference it, so reserve the id. Treat local absence as terminal
+        // rather than making every read wait for repair; this gives up peer repair of that checkpointed version.
+        if (error.code === 'ENOENT')
+            return 'gone';
+        throw error;
+    }
+    try {
+        fileSize = (await handle.stat()).size;
+        header = Buffer.alloc(HEADER_SIZE);
+        const { bytesRead } = await handle.read(header, 0, HEADER_SIZE, 0);
+        header = header.subarray(0, bytesRead);
+    }
+    finally {
+        await handle.close();
+    }
+    if (blobHeaderIsAbortMarker(header))
+        return 'capture';
+    if (blobHeaderIndicatesIncomplete(header, fileSize))
+        return 'pending';
+    // A deflate header records the *uncompressed* length, so the check above compares lengths only for
+    // UNCOMPRESSED_TYPE and a short compressed body reaches here looking whole. Both producers of one
+    // need this: saveBlob stamps a known size before the first compressed byte, so a live write is
+    // invisible above and caught only here, and an unclean shutdown can leave a torn body the
+    // asynchronous repair sweep has not reached yet. Compression is opt-in via storage.blobs.compression
+    // (off by default): on a corpus with it disabled this branch is never taken, but where an operator
+    // enables it each completeness check fully inflates the compressed body it reaches.
+    if (header[1] !== exports.DEFLATE_TYPE)
+        return 'capture';
+    const uncompressedSize = Number(new DataView(header.buffer, header.byteOffset, HEADER_SIZE).getBigUint64(0) & 0xffffffffffffn);
+    return (await inflatesToExactly(filePath, uncompressedSize)) ? 'capture' : 'pending';
+}
+/**
+ * Whether the header is a deliberate abort marker rather than content. Written once and never
+ * rewritten, so a consumer sharing blob inodes can keep them: dropping a PENDING marker downgrades a
+ * retryable 503 to a 404 the replication layer reads as "cleanly gone" (harper-pro#481).
+ */
+function blobHeaderIsAbortMarker(header) {
+    if (header.length < HEADER_SIZE)
+        return false;
+    const type = header.readUInt16BE(0);
+    return type === PENDING_TYPE || type === ERROR_TYPE;
+}
+/** The stand-in bytes for a blob that could not be captured whole. */
+function createCaptureMarker(disposition, message) {
+    const messageBuffer = Buffer.from(message);
+    const header = Buffer.alloc(HEADER_SIZE);
+    new DataView(header.buffer, header.byteOffset, HEADER_SIZE).setBigInt64(0, BigInt(messageBuffer.length) | (BigInt(disposition === 'gone' ? ERROR_TYPE : PENDING_TYPE) << 48n));
+    return Buffer.concat([header, messageBuffer]);
+}
 function blobHeaderIndicatesIncomplete(header, fileSize) {
     if (fileSize < HEADER_SIZE || header.length < HEADER_SIZE)
         return true;
     const type = header.readUInt16BE(0);
-    if (type !== UNCOMPRESSED_TYPE && type !== DEFLATE_TYPE)
+    if (type !== UNCOMPRESSED_TYPE && type !== exports.DEFLATE_TYPE)
         return true;
     const storedSize = header.readUIntBE(2, 6);
     if (storedSize === UNKNOWN_SIZE)
@@ -1804,10 +2403,83 @@ function blobHeaderIndicatesIncomplete(header, fileSize) {
     return type === UNCOMPRESSED_TYPE && fileSize !== HEADER_SIZE + storedSize;
 }
 /**
- * Whether a file-backed blob's backing file is missing or incomplete on disk — the gate for the
- * copy-delivery repair. This blocking probe is the locked final recheck after the caller's async
- * exact-duplicate prefilter. Compressed bodies require the asynchronous repair sweep for
- * verification. Returns undefined for blobs the question does not apply to.
+ * What a blob file's header and length alone say about its completeness: `true` when the file is
+ * missing, short, unfinished, an error/pending stub, or disagrees with the record descriptor; `false`
+ * when an uncompressed body is whole; `null` for a deflate body, whose header records the uncompressed
+ * length and so proves nothing about the body — only inflating it can. `undefined` when the question
+ * does not apply or the file could not be inspected.
+ */
+function classifyBlobFileByHeader(blob, fileSize, header, headerBytes) {
+    if (fileSize < HEADER_SIZE || headerBytes < HEADER_SIZE)
+        return true;
+    if (blobHeaderIndicatesIncomplete(header, fileSize))
+        return true;
+    const descriptorSize = blob.size;
+    if (descriptorSize !== undefined && header.readUIntBE(2, 6) !== descriptorSize)
+        return true;
+    return header.readUInt16BE(0) === exports.DEFLATE_TYPE ? null : false;
+}
+/**
+ * Whether a file-backed blob's backing file is missing or incomplete on disk — the unlocked,
+ * non-blocking classification a copy-delivery repair candidate gets first (harper-pro#699). A deflate
+ * body is inflated (streamed, off the event loop) to decide; the file it judged damaged is remembered
+ * so the locked recheck in `repairBlobFile` can recognize it without inflating again. Returns
+ * undefined for blobs the question does not apply to, or that could not be inspected.
+ */
+async function blobFileMissingOrIncompleteAsync(blob) {
+    if (!(blob instanceof FileBackedBlob))
+        return undefined;
+    const storageInfo = storageInfoForBlob.get(blob);
+    if (!storageInfo?.fileId)
+        return undefined;
+    storageInfo.probedDamage = undefined; // every probe starts over; no early exit may leave a stale verdict behind
+    const filePath = getFilePath(storageInfo);
+    let file;
+    try {
+        file = await (0, promises_1.open)(filePath, 'r');
+    }
+    catch (error) {
+        if (error?.code === 'ENOENT')
+            return true;
+        logger_ts_1.logger.debug?.('Unable to open blob file for in-place repair inspection', error);
+        return undefined;
+    }
+    let fileSize;
+    const header = Buffer.allocUnsafe(HEADER_SIZE);
+    let verdict;
+    try {
+        fileSize = (await file.stat()).size;
+        const { bytesRead } = await file.read(header, 0, HEADER_SIZE, 0);
+        verdict = classifyBlobFileByHeader(blob, fileSize, header, bytesRead);
+    }
+    catch (error) {
+        logger_ts_1.logger.debug?.('Unable to inspect blob file for in-place repair', error);
+        return undefined;
+    }
+    finally {
+        await file.close().catch(() => { });
+    }
+    if (verdict !== null)
+        return verdict;
+    try {
+        verdict = !(await inflatesToExactly(filePath, header.readUIntBE(2, 6)));
+    }
+    catch (error) {
+        logger_ts_1.logger.debug?.('Unable to inspect blob file for in-place repair', error);
+        return undefined;
+    }
+    if (verdict)
+        storageInfo.probedDamage = { fileSize, header };
+    return verdict;
+}
+/**
+ * The locked, synchronous recheck of `blobFileMissingOrIncompleteAsync`'s verdict, run by
+ * `repairBlobFile` under the blob's write lock so a repair can only replace a file no writer is
+ * still producing. It repeats the header classification, and for a deflate body — which it cannot
+ * afford to inflate on the event loop — answers `true` only while the file is still the one the
+ * probe found damaged: a body can only grow (writers append; a repair replaces the file) so an
+ * unchanged length and header mean unchanged bytes, and anything else is `undefined` (not provably
+ * damaged — decline, and let the next probe or the repair sweep classify it).
  */
 function blobFileMissingOrIncomplete(blob) {
     try {
@@ -1828,16 +2500,13 @@ function blobFileMissingOrIncomplete(blob) {
             return undefined;
         }
         try {
-            const size = (0, node_fs_1.fstatSync)(fd).size;
-            if (size < HEADER_SIZE)
-                return true;
+            const fileSize = (0, node_fs_1.fstatSync)(fd).size;
             const header = Buffer.allocUnsafe(HEADER_SIZE);
-            if ((0, node_fs_1.readSync)(fd, header, 0, HEADER_SIZE, 0) < HEADER_SIZE)
-                return true;
-            if (blobHeaderIndicatesIncomplete(header, size))
-                return true;
-            const descriptorSize = blob.size;
-            return descriptorSize !== undefined && header.readUIntBE(2, 6) !== descriptorSize;
+            const verdict = classifyBlobFileByHeader(blob, fileSize, header, (0, node_fs_1.readSync)(fd, header, 0, HEADER_SIZE, 0));
+            if (verdict !== null)
+                return verdict;
+            const probed = storageInfo.probedDamage;
+            return probed?.fileSize === fileSize && probed.header.equals(header) ? true : undefined;
         }
         finally {
             (0, node_fs_1.closeSync)(fd);
@@ -2349,6 +3018,10 @@ function cleanupUnusedBlobs(blobs, retainedFileIds) {
             continue; // no file written, nothing to clean up
         if (retainedFileIds?.has(storageInfo.fileId))
             continue; // the committed record still references this blob
+        // Tombstone the instance as soon as the deletion is DECIDED, not when the unlink is issued: the
+        // unlink waits for an in-flight save to settle, and a re-store in that window would otherwise
+        // mint a reference to a file that is already condemned (issue #2062).
+        discardStorage(storageInfo);
         const settle = storageInfo.saving ?? Promise.resolve();
         settle.then(() => deleteBlob(blob), () => deleteBlob(blob) // even on save failure, attempt cleanup in case a partial file remains
         );
@@ -2402,7 +3075,7 @@ const copyingUnpacker = new msgpackr_1.Packr({ copyBuffers: true, mapsAsObjects:
             storageInfoForBlob.set(blob, {
                 storageIndex: 0,
                 fileId: null,
-                contentBuffer: blobInfo[1],
+                contentBuffer: asBuffer(blobInfo[1]),
             });
             blob.size = blobInfo[1]?.length;
         }
@@ -2436,6 +3109,12 @@ const copyingUnpacker = new msgpackr_1.Packr({ copyBuffers: true, mapsAsObjects:
                 throw new Error('Unable to save blob without file id');
             }
             storageInfo.recordId = encodeForStorageForRecordId;
+            // Per-node hint only — a replication sender uses it to skip the header sniff for the
+            // uncompressed majority, but always confirms against the local file header before sending
+            // raw: a relayed record (or an in-place repair) can outlive the storage form recorded here.
+            // Namespaced away from `codec`, which a media blob plausibly carries as its own property.
+            if (storageInfo.compress || storageInfo.storedCodec)
+                options.storedCodec = 'deflate';
             // A record version being written now references this file, so any reclamation queued by an
             // earlier supersession is void — the retain-on-update check in RecordEncoder only covers the
             // write that supersedes, not a file already awaiting reclamation from a previous one.
@@ -2548,7 +3227,7 @@ function polyfillBlob() {
  * @param database
  */
 async function cleanupOrphans(database, databaseName) {
-    const { HAS_BLOBS } = await import("./auditStore.js");
+    const { HAS_BLOBS, isAuditEntryWrite } = await import("./auditStore.js");
     let store;
     let auditStore;
     let orphansDeleted = 0;
@@ -2632,7 +3311,10 @@ async function cleanupOrphans(database, databaseName) {
                 if (!primaryStore)
                     continue;
                 const entry = primaryStore?.getEntry(auditRecord.recordId);
-                if (!entry || entry.version !== auditRecord.version || !entry.value) {
+                // Only the write this audit record describes had its blobs scanned by the table loop above.
+                // Identity is the log key, not the version: a version match can name a different write and
+                // would skip an audit value whose blobs are still referenced.
+                if (!entry?.value || !isAuditEntryWrite(entry, auditRecord)) {
                     checkObjectForReferences(auditRecord.getValue(primaryStore));
                 }
                 // slow this down a bit to reduce excessive load, this runs approximately at 10k per second
@@ -2697,10 +3379,11 @@ async function cleanupOrphans(database, databaseName) {
     }
 }
 async function isBlobFileComplete(storageInfo) {
-    let filePath;
+    return isBlobFileCompleteAtPath(getFilePath(storageInfo));
+}
+async function isBlobFileCompleteAtPath(filePath) {
     let fileSize;
     try {
-        filePath = getFilePath(storageInfo);
         fileSize = (0, node_fs_1.statSync)(filePath).size;
     }
     catch (e) {
@@ -2726,32 +3409,56 @@ async function isBlobFileComplete(storageInfo) {
     // uncompressed byte count). For an uncompressed blob that equals the on-disk body length;
     // for a compressed blob it does not, so the body length can't be compared to it directly.
     const size = Number(headerValue & 0xffffffffffffn);
-    if (header[1] === DEFLATE_TYPE) {
-        // A compressed blob's header size is the uncompressed length, so it can't be compared to the
-        // compressed on-disk body. Verify by streaming the body through inflate and counting the
-        // decompressed bytes: a fully-written deflate stream inflates to exactly `size` bytes; a
-        // truncated one errors (Z_BUF_ERROR) or yields fewer. Streaming (rather than inflateSync on
-        // the whole buffer) keeps memory bounded during the repair sweep, which may touch many large
-        // blobs.
-        return new Promise((resolve) => {
-            let inflatedLength = 0;
-            const source = (0, node_fs_1.createReadStream)(filePath, { start: HEADER_SIZE });
-            const inflate = (0, node_zlib_1.createInflate)();
-            const fail = () => {
-                source.destroy();
-                resolve(false);
-            };
-            source.on('error', fail);
-            inflate.on('error', fail);
-            inflate.on('data', (chunk) => {
-                inflatedLength += chunk.length;
-            });
-            inflate.on('end', () => resolve(inflatedLength === size));
-            source.pipe(inflate);
-        });
+    if (header[1] === exports.DEFLATE_TYPE) {
+        // This function's contract is to resolve true/false; harper-pro's repair sweep awaits it without a
+        // catch (replication/blobRepair.ts), so an I/O fault must not become a rejection here. The capture
+        // classifier calls inflatesToExactly directly, where the distinction does matter.
+        return inflatesToExactly(filePath, size).catch(() => false);
     }
     return true;
 }
+/**
+ * Whether the deflate body after the header inflates to exactly `size` bytes. A compressed blob's
+ * header records the uncompressed length, so it cannot be compared to the on-disk body; a truncated
+ * stream errors (Z_BUF_ERROR) or yields fewer bytes. Streamed rather than inflateSync so memory stays
+ * bounded over a sweep that may touch many large blobs, and abandoned as soon as the output passes
+ * `size` so a body that lies about its length costs no more than the length it claimed.
+ */
+function inflatesToExactly(filePath, size) {
+    return new Promise((resolve, reject) => {
+        let inflatedLength = 0;
+        const source = (0, node_fs_1.createReadStream)(filePath, { start: HEADER_SIZE });
+        const inflate = (0, node_zlib_1.createInflate)();
+        // Both: pipe() does not tear down the destination when the source errors, so the inflate's
+        // native zlib handle would sit allocated until GC — once per blob, on every backup walk.
+        const stop = () => {
+            source.destroy();
+            inflate.destroy();
+        };
+        // A zlib error is the answer (body truncated or corrupt); an I/O error is a failure to answer and
+        // must propagate, or a systemic fault would classify a corpus of complete blobs as incomplete.
+        // The reject arm is unverified: it needs a read fault raised mid-inflate on a file that opened
+        // cleanly, which no test here can produce.
+        const fail = (error) => {
+            stop();
+            if (error?.code?.startsWith('Z_'))
+                resolve(false);
+            else
+                reject(error);
+        };
+        source.on('error', fail);
+        inflate.on('error', fail);
+        inflate.on('data', (chunk) => {
+            inflatedLength += chunk.length;
+            if (inflatedLength > size) {
+                stop();
+                resolve(false);
+            }
+        });
+        inflate.on('end', () => resolve(inflatedLength === size));
+        source.pipe(inflate);
+    });
+}
 /**
  * Resolves true if the given blob's backing file is present and complete: no error stub, not an
  * in-flight placeholder, and the content matches the header (uncompressed blobs by body length,
@@ -2766,6 +3473,177 @@ async function isBlobComplete(blob) {
         return false;
     return isBlobFileComplete(storageInfo);
 }
+/**
+ * Open a blob's STORED body for transfer, without inflating it — the sender half of replication's
+ * codec preservation. Returns undefined whenever the stored form is not a settled deflate file
+ * (uncompressed, sliced, still being written, missing, or already condemned); the caller then falls
+ * back to the ordinary content stream, which knows how to wait and how to report errors.
+ *
+ * Deliberately synchronous through the decision: the caller decides raw-vs-inflated inside the same
+ * synchronous block that puts the owning record frame on the wire. The stream opens its own
+ * descriptor later and re-reads the header first: a settled file's header never changes except by
+ * a repair publishing a fresh (uncompressed) file over the path, so a header that differs from the
+ * sniffed one means the announced form is gone — reported as transient (503), since a retry
+ * re-decides against the new file. The stream verifies as it sends — a concurrent inflate must
+ * land on exactly `size` bytes — so a torn body that survived the header checks becomes a loud,
+ * permanently-classified error instead of replicated garbage.
+ */
+function openStoredBlobBody(blob) {
+    if (!(blob instanceof FileBackedBlob))
+        return undefined;
+    const storageInfo = storageInfoForBlob.get(blob);
+    if (!storageInfo?.fileId || !storageInfo.store)
+        return undefined;
+    if (storageInfo.start !== undefined || storageInfo.end !== undefined)
+        return undefined;
+    if (storageInfo.fileState?.discarded)
+        return undefined;
+    const releaseHold = holdBlobFile(blob);
+    if (!releaseHold)
+        return undefined; // already being reclaimed
+    let fd;
+    let filePath;
+    const decline = () => {
+        if (fd !== undefined) {
+            try {
+                (0, node_fs_1.closeSync)(fd);
+            }
+            catch (error) {
+                logger_ts_1.logger.debug?.('Error closing stored blob body descriptor', error);
+            }
+        }
+        releaseHold();
+        return undefined;
+    };
+    let size;
+    const header = Buffer.allocUnsafe(HEADER_SIZE);
+    try {
+        const lockKey = storageInfo.fileId + ':blob';
+        // A held write lock means the body may still be streaming to disk; the normal read path knows
+        // how to wait for it, so decline rather than duplicate that machinery here.
+        if (!storageInfo.store.tryLock(lockKey))
+            return decline();
+        storageInfo.store.unlock(lockKey);
+        filePath = getFilePath(storageInfo);
+        fd = (0, node_fs_1.openSync)(filePath, 'r');
+        if ((0, node_fs_1.readSync)(fd, header, 0, HEADER_SIZE, 0) < HEADER_SIZE)
+            return decline();
+        const headerValue = new DataView(header.buffer, header.byteOffset, HEADER_SIZE).getBigUint64(0);
+        if (Number(headerValue >> 48n) !== exports.DEFLATE_TYPE)
+            return decline();
+        size = Number(headerValue & 0xffffffffffffn);
+        if (size === UNKNOWN_SIZE)
+            return decline();
+        const descriptorSize = blob.size;
+        if (descriptorSize !== undefined && descriptorSize < UNKNOWN_SIZE && descriptorSize !== size)
+            return decline();
+        (0, node_fs_1.closeSync)(fd);
+        fd = undefined;
+    }
+    catch (error) {
+        logger_ts_1.logger.debug?.('Unable to open stored blob body', storageInfo.fileId, error);
+        return decline();
+    }
+    let holdReleased = false;
+    const releaseHoldOnce = () => {
+        if (holdReleased)
+            return;
+        holdReleased = true;
+        releaseHold();
+    };
+    let streamStarted = false;
+    const failVerification = (detail) => new BlobReadError(`Stored blob body for ${filePath} ${detail}`, BLOB_CORRUPT_STATUS);
+    return {
+        codec: 'deflate',
+        size,
+        close: () => {
+            streamStarted = true;
+            releaseHoldOnce();
+        },
+        stream: async function* () {
+            if (streamStarted)
+                throw new Error(`Stored blob body for ${filePath} was already consumed or closed`);
+            streamStarted = true;
+            let streamFd;
+            let source;
+            try {
+                streamFd = (0, node_fs_1.openSync)(filePath, 'r');
+                const currentHeader = Buffer.allocUnsafe(HEADER_SIZE);
+                const headerBytes = (0, node_fs_1.readSync)(streamFd, currentHeader, 0, HEADER_SIZE, 0);
+                if (headerBytes < HEADER_SIZE || !currentHeader.equals(header)) {
+                    throw new BlobReadError(`Stored blob body for ${filePath} was replaced before it was streamed`, exports.BLOB_UNAVAILABLE_STATUS);
+                }
+                // the stream owns the descriptor from here (an fs stream's worker-thread close(2) races any other user)
+                source = (0, node_fs_1.createReadStream)(filePath, { fd: streamFd, start: HEADER_SIZE, autoClose: true });
+            }
+            catch (error) {
+                if (streamFd !== undefined)
+                    (0, node_fs_1.closeSync)(streamFd);
+                releaseHoldOnce();
+                throw error;
+            }
+            const verifier = (0, node_zlib_1.createInflate)();
+            let inflated = 0;
+            let verifyFailure;
+            // A mid-write zlib data error (a corrupt, not merely truncated, body) fires 'error'/'close'
+            // but drops the pending `verifier.write` callback, so the per-chunk await below would hang
+            // forever — suspending this generator inside the try and leaking the descriptor and the file
+            // hold its finally releases. settleWrite() unblocks it from error/close too; whichever runs
+            // first resolves the promise and clears the ref, so the write callback is a harmless no-op.
+            let resolvePendingWrite;
+            const settleWrite = () => {
+                const resolve = resolvePendingWrite;
+                resolvePendingWrite = undefined;
+                resolve?.();
+            };
+            verifier.on('data', (chunk) => {
+                inflated += chunk.length;
+                if (inflated > size && !verifyFailure) {
+                    verifyFailure = failVerification(`inflates past its declared size of ${size}`);
+                    verifier.destroy();
+                }
+            });
+            verifier.on('error', (error) => {
+                verifyFailure ??= failVerification(`is not a valid deflate stream: ${error.message}`);
+                settleWrite();
+            });
+            verifier.on('close', settleWrite); // a destroy() with no 'error' (oversize) also drops the write callback
+            try {
+                for await (const chunk of source) {
+                    if (verifyFailure)
+                        throw verifyFailure;
+                    if (!verifier.destroyed) {
+                        // Yield only once the verifier consumed the chunk, so a detection surfaces before the
+                        // bytes it condemns are handed to the caller's wire frame (and never after the caller
+                        // has already seen a clean end).
+                        await new Promise((resolveWrite) => {
+                            resolvePendingWrite = resolveWrite;
+                            verifier.write(chunk, settleWrite);
+                        });
+                        if (verifyFailure)
+                            throw verifyFailure;
+                    }
+                    yield chunk;
+                }
+                if (!verifier.destroyed) {
+                    await new Promise((resolveEnd) => {
+                        verifier.on('close', resolveEnd);
+                        verifier.end();
+                    });
+                }
+                if (verifyFailure)
+                    throw verifyFailure;
+                if (inflated !== size)
+                    throw failVerification(`inflated to ${inflated} bytes of a declared ${size}`);
+            }
+            finally {
+                source.destroy();
+                verifier.destroy();
+                releaseHoldOnce();
+            }
+        },
+    };
+}
 /**
  * Async generator that yields records whose referenced blob files are missing, truncated, or in an
  * error state. Used by the blob repair sweep to find candidates for peer-fetched recovery.