harper 5.1.20 → 5.2.0-alpha.5

package/resources/blob.ts

@@ -34,7 +34,7 @@ import {
 	type FSWatcher,
 } from 'node:fs';
 import type { StatsFs } from 'node:fs';
-import { createDeflate, createInflate, deflate } from 'node:zlib';
+import { createDeflate, createInflate, inflate } from 'node:zlib';
 import { Readable, pipeline } from 'node:stream';
 import { ensureDirSync } from 'fs-extra';
 import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.ts';
@@ -294,11 +294,17 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 				rawBytes = Buffer.alloc(0);
 				fileMissing = true;
 			}
-			function checkCompletion(rawBytes: Buffer): Buffer | Promise<Buffer> {
-				if (size > rawBytes.length) {
-					// the file is not finished being written, wait for the write lock to complete
-					const store = storageInfo.store;
-					const lockKey = storageInfo.fileId + ':blob';
+			// The file may still be mid-write. Wait for the writer's `:blob` lock to release, then re-read.
+			// `incomplete` is the caller's "definitely not done yet" signal: for an uncompressed blob the
+			// on-disk body length must reach the (uncompressed) header size. For a compressed blob the body
+			// is shorter than the uncompressed size AND the header size is finalized up front when the size
+			// is known, so neither is a reliable completeness signal — the writer's lock is. The caller
+			// passes `mustVerifyViaLock` for compressed blobs so we probe the lock even when the bytes look
+			// plausible, and only treat the file as complete once the writer is confirmed done.
+			function waitForCompletion(incomplete: boolean, mustVerifyViaLock = false): Promise<Buffer> | undefined {
+				const store = storageInfo.store;
+				const lockKey = storageInfo.fileId + ':blob';
+				if (incomplete) {
 					if (writeFinished) {
 						// the writer released its lock but the content is still short: the file is cleanly gone
 						// (404) or confidently incomplete/corrupt (500) — not merely mid-write (#1423/#1424).
@@ -337,24 +343,77 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 							resolve(readContents());
 						}
 					});
+				} else if (mustVerifyViaLock && !writeFinished) {
+					// Bytes look complete but we can't prove it for a compressed body; probe whether the writer
+					// still holds the lock. If we acquire it, the write has finished, but the bytes we read may
+					// predate the final flush, so re-read once to get the complete body. Otherwise fall through
+					// to wait for the lock to release.
+					const acquiredImmediately = store.tryLock(lockKey);
+					if (acquiredImmediately) {
+						writeFinished = true;
+						store.unlock(lockKey);
+						return readContents(); // writer done — re-read to ensure we have the final bytes
+					}
+				} else {
+					return undefined; // confirmed complete (uncompressed exact-size match)
 				}
+				return new Promise((resolve, reject) => {
+					let settled = false;
+					const timer = setTimeout(() => {
+						if (settled) return;
+						settled = true;
+						reject(
+							new BlobReadError(
+								`Blob ${storageInfo.fileId} is unavailable; the in-progress write did not complete in time`,
+								BLOB_UNAVAILABLE_STATUS
+							)
+						);
+					}, getBlobReadTimeout());
+					timer.unref();
+					const callback = () => {
+						if (settled) return;
+						settled = true;
+						clearTimeout(timer);
+						writeFinished = true;
+						resolve(readContents());
+					};
+					const lockAcquired = store.tryLock(lockKey, callback);
+					if (lockAcquired) {
+						settled = true;
+						clearTimeout(timer);
+						writeFinished = true;
+						store.unlock(lockKey);
+						resolve(readContents());
+					}
+				});
+			}
+			function sliceContent(bytes: Buffer): Buffer {
 				if (end != undefined || start != undefined) {
-					rawBytes = rawBytes.subarray(start ?? 0, end ?? rawBytes.length);
+					bytes = bytes.subarray(start ?? 0, end ?? bytes.length);
 				}
-				return rawBytes;
+				return bytes;
 			}
 			// Only sniff the storage-type byte and take the decompress branch when a full header is present.
 			// A file shorter than the header (truncated/corrupted, #1424) would otherwise be mis-read as a
 			// deflate body and yield garbage; instead it falls through to checkCompletion as incomplete.
 			if (rawBytes.length >= HEADER_SIZE && rawBytes[1] === DEFLATE_TYPE) {
+				// An in-flight compressed file still carrying the UNKNOWN_SIZE placeholder is definitely
+				// incomplete; otherwise the size is finalized but the compressed body may still be streaming,
+				// so verify via the write lock before inflating (inflating a partial deflate stream errors).
+				const pending = waitForCompletion(size === UNKNOWN_SIZE, true);
+				if (pending) return pending;
 				return new Promise<Buffer>((resolve, reject) => {
-					deflate(rawBytes.subarray(HEADER_SIZE), (error, result) => {
+					// start/end index into the UNCOMPRESSED content, so inflate first, then slice
+					inflate(rawBytes.subarray(HEADER_SIZE), (error, result) => {
 						if (error) reject(error);
-						else resolve(checkCompletion(result));
+						else resolve(sliceContent(result));
 					});
 				});
 			}
-			return checkCompletion(rawBytes.subarray(HEADER_SIZE));
+			const body = rawBytes.subarray(HEADER_SIZE);
+			const pending = waitForCompletion(size > body.length);
+			if (pending) return pending;
+			return sliceContent(body);
 		};
 		return readContents();
 	}
@@ -388,7 +447,24 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 		let totalContentRead = 0;
 		let watcher: FSWatcher;
 		let timer: NodeJS.Timeout;
+		// The start() open-retry timer lives in a different scope/phase than pull()'s `timer`; track it
+		// separately so a cancel() during the file-creation wait clears it instead of leaking an fd (#1457).
+		let openTimer: NodeJS.Timeout;
+		// Stream-wide cancellation gate. cancel() can null `fd` but cannot reach pull()'s per-pull
+		// `settled` flag, so this is the cross-phase signal that stops start()/pull() from opening or
+		// reading once the consumer has cancelled (e.g. an aborted ranged response) (#1457).
+		let cancelled = false;
 		let isBeingWritten: boolean;
+		// Close the descriptor at most once. Without nulling `fd`, a later close()/cancel() (e.g. a
+		// GC-driven cancel after a successful read) would close a descriptor the OS may have reassigned to
+		// an unrelated file/socket; and a cancel() before start()'s open() resolves would call
+		// close(undefined) and throw. Guarding on `fd != null` fixes both (#1457).
+		const closeFd = () => {
+			if (fd != null) {
+				close(fd);
+				fd = null;
+			}
+		};
 		let previouslyFinishedWriting = false;
 		const blob = this;
 		// Authoritative (uncompressed) content length from the record descriptor, captured before the
@@ -406,6 +482,7 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 				// lookup) is computed lazily on the first miss, so a healthy read never pays for it.
 				let deadline = 0;
 				const openFile = (resolve: (value: any) => void, reject: (error: Error) => void) => {
+					if (cancelled) return resolve(undefined); // consumer cancelled before/while opening; stop retrying
 					open(filePath, 'r', (error, openedFd) => {
 						if (error) {
 							if (error.code === 'ENOENT' && isBeingWritten !== false) {
@@ -413,10 +490,11 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 								if (Date.now() < deadline) {
 									logger.debug?.('File does not exist yet, waiting for it to be created', filePath);
 									// the file doesn't exist, so we need to wait for it to be created
-									return setTimeout(() => {
+									openTimer = setTimeout(() => {
 										checkIfIsBeingWritten();
 										openFile(resolve, reject);
 									}, 20).unref();
+									return openTimer;
 								}
 							}
 							// ENOENT: 503 if a local writer holds the lock or a replication receive is in
@@ -436,6 +514,9 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 							blob.#onError?.forEach((callback) => callback(readError));
 						} else {
 							fd = openedFd;
+							// the consumer cancelled while open() was in flight: pull()/cancel() will not run
+							// again, so close the descriptor we just acquired rather than leaking it (#1457)
+							if (cancelled) closeFd();
 							resolve(openedFd);
 						}
 					});
@@ -450,9 +531,18 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 				// it; bounds the case where the header reports a known size but the bytes never arrive (a
 				// present-but-truncated blob whose writer lock stays held) (#1454).
 				let incompleteDeadline = 0;
+				// Settle this pull exactly once. A slow async read that completes with an error after the
+				// watcher-timeout path already called onError() (and closed the fd) would otherwise run
+				// onError twice — double close + duplicated #onError callbacks (#1457).
+				let settled = false;
 				return new Promise(function readMore(resolve: () => void, reject: (error: Error) => void) {
+					// A queued timer/watcher callback can re-enter readMore after the stream was cancelled (fd
+					// is gone); stop here so we never issue read() on a closed/nulled descriptor (#1457).
+					if (cancelled || fd == null) return resolve();
 					function onError(error) {
-						close(fd);
+						if (settled) return;
+						settled = true;
+						closeFd();
 						clearTimeout(timer);
 						if (watcher) watcher.close();
 						reject(error);
@@ -461,7 +551,11 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 					// allocate a buffer for reading. Note that we could do a stat to get the size, but that is a little more complicated, and might be a little extra overhead
 					const buffer = Buffer.allocUnsafe(0x40000);
 					read(fd, buffer, 0, buffer.length, position, (error, bytesRead, buffer) => {
-						// TODO: Implement support for decompression
+						// A late read completion after the pull already settled (via onError or terminal
+						// close) or after the stream was cancelled (fd nulled, EBADF/partial read) must not
+						// re-enter the state machine — otherwise a recursive read() would hit a null fd and
+						// throw synchronously, or onError() would reject an already-cancelled stream (#1457).
+						if (settled || cancelled) return resolve();
 						totalContentRead += bytesRead;
 						if (error) return onError(error);
 						if (position === 0) {
@@ -497,6 +591,36 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 								// half-replicated blob whose source stream aborted; the bytes are still expected — retry (harper-pro#481)
 								return onError(new BlobReadError('Blob pending replication for ' + filePath, BLOB_UNAVAILABLE_STATUS));
 							}
+							if (buffer[1] === DEFLATE_TYPE) {
+								// We can't seek/slice a deflate stream by uncompressed offset, so hand off to the
+								// buffered inflate path (bytes() inflates then slices) and emit it as one chunk.
+								// Safe by construction: the read loop never streams the raw compressed body.
+								return blob.bytes().then(
+									(bytes: Buffer) => {
+										// bytes() resolves asynchronously; the consumer may have cancelled meanwhile.
+										// Settle exactly once and route the close through closeFd() so we never touch a
+										// reassigned/nulled descriptor (#1457).
+										if (settled || cancelled) return resolve();
+										settled = true;
+										closeFd();
+										if (bytes.length > 0) {
+											try {
+												controller.enqueue(bytes);
+											} catch (error) {
+												logger.debug?.('Error enqueuing chunk', error);
+												return resolve();
+											}
+										}
+										try {
+											controller.close();
+										} catch {
+											// controller may already be closed
+										}
+										resolve();
+									},
+									(error: Error) => onError(error)
+								);
+							}
 							size = Number(headerValue & 0xffffffffffffn);
 							if (size < UNKNOWN_SIZE) {
 								if (isFullRead && descriptorSize != null && descriptorSize < UNKNOWN_SIZE && size !== descriptorSize) {
@@ -522,6 +646,7 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 						} else if (bytesRead === 0) {
 							const buffer = Buffer.allocUnsafe(8);
 							return read(fd, buffer, 0, HEADER_SIZE, 0, (error) => {
+								if (settled || cancelled) return resolve();
 								if (error) return onError(error);
 								size = Number(new DataView(buffer.buffer, buffer.byteOffset, 8).getBigUint64(0) & 0xffffffffffffn);
 								if (size > totalContentRead) {
@@ -621,7 +746,8 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 									}
 									return;
 								}
-								close(fd);
+								settled = true;
+								closeFd();
 								controller.close();
 								resolve();
 							});
@@ -629,19 +755,26 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 							buffer = buffer.subarray(0, bytesRead);
 						}
 						if (start !== undefined || end !== undefined) {
+							// content offset of the first byte currently in `buffer`. The 8-byte header is only
+							// present in — and stripped from — the first read at file position 0; every later read
+							// starts HEADER_SIZE into the file relative to its content offset.
+							const contentStart = position === 0 ? 0 : position - HEADER_SIZE;
 							if (start && totalContentRead < start) {
-								// we are before the start of the slice, so we need to read more
-								position += bytesRead;
+								// we are before the start of the slice. Seek straight to it (HEADER_SIZE + start)
+								// instead of reading and discarding a 256KB buffer per chunk from byte 0. The header
+								// has already been validated and the size detected on the first read (#1457).
+								position = HEADER_SIZE + start;
+								totalContentRead = start;
 								return readMore(resolve, reject);
 							}
 							if (end && totalContentRead >= end) {
 								// we are past or reached the end of the slice, so we have reached the end, indicate
-								if (totalContentRead > end) buffer = buffer.subarray(0, end - position);
+								if (totalContentRead > end) buffer = buffer.subarray(0, end - contentStart);
 								totalContentRead = size = end;
 							}
-							if (start && start > position) {
-								// we need to skip ahead to the start of the slice
-								buffer = buffer.subarray(start - position);
+							if (start && start > contentStart) {
+								// we need to skip ahead to the start of the slice within this chunk
+								buffer = buffer.subarray(start - contentStart);
 							}
 						}
 						position += bytesRead;
@@ -654,7 +787,8 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 							return resolve();
 						}
 						if (totalContentRead === size) {
-							close(fd);
+							settled = true;
+							closeFd();
 							controller.close();
 						}
 						resolve();
@@ -662,8 +796,10 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 				});
 			},
 			cancel() {
-				close(fd);
+				cancelled = true;
+				closeFd();
 				clearTimeout(timer);
+				clearTimeout(openTimer);
 				if (watcher) watcher.close();
 			},
 		});

package/resources/crdt.ts

@@ -1,11 +1,18 @@
+// Single definition of the numeric `add` fold, shared by the storage-time merge (crdt.add, via
+// updateAndFreeze) and the read/serialize path (tracked.ts Addition.update). One implementation
+// keeps the two from diverging: a numeric string was concatenated on the storage path but coerced
+// on the read path, and a bigint folded on the storage path but threw (`+bigint`) on the read path.
+export function addValues(previousValue: any, value: any) {
+	if (typeof previousValue === 'bigint') return previousValue + BigInt(value);
+	if (previousValue == null) return value; // no prior value: the add establishes it
+	const previous = +previousValue; // coerce so a numeric string adds instead of concatenating
+	if (isNaN(previous)) return value; // non-numeric prior: the add establishes it
+	// A number-typed field stays a number even if a stray bigint delta arrives, rather than throwing
+	// a number+bigint TypeError on the apply path.
+	return previous + (typeof value === 'bigint' ? Number(value) : value);
+}
 export function add(record, property, action) {
-	const previousValue = record[property];
-	if (typeof previousValue === 'bigint') {
-		record[property] = previousValue + BigInt(action.value);
-	} else if (isNaN(record[property])) record[property] = action.value;
-	else {
-		record[property] = previousValue + action.value;
-	}
+	record[property] = addValues(record[property], action.value);
 }
 add.reverse = function (record, property, action) {
 	const previousValue = record[property];
@@ -15,9 +22,16 @@ add.reverse = function (record, property, action) {
 		record[property] = previousValue - action.value;
 	}
 };
-const operations = {
+// The CRDT operation registry. Exported so the storage/apply path (tracked.ts updateAndFreeze)
+// resolves ops against this explicit set rather than the module's export namespace — otherwise a
+// crafted/corrupt `__op__` naming any exported function (addValues, getRecordAtTime, …) would
+// resolve truthy and be invoked with the wrong arguments (throwing and wedging the apply path, or
+// silently corrupting the field). applyReverse/applyForward below already resolve through this.
+// Null-prototype so a crafted `__op__` naming an Object.prototype member (toString, valueOf,
+// constructor, …) resolves to undefined rather than an inherited function.
+export const operations = Object.assign(Object.create(null), {
 	add,
-};
+});
 
 /**
  * Rebuild a record update that has a timestamp before the provided newer update
@@ -179,30 +193,22 @@ export function getRecordAtTime(currentEntry, timestamp, store, tableId: number,
 		const boundaryEntry = auditStore.get(auditTime, tableId, recordId);
 		if (boundaryEntry?.type === 'delete') return null;
 	}
-	// some patches may leave properties in an unknown state, so we need to fill in the blanks
-	// first we determine if there any unknown properties
-	// then continue to iterate back through the audit history, filling in the blanks
-	while (unknowns.size > 0 && auditTime > 0) {
-		const auditEntry = auditStore.get(auditTime, tableId, recordId);
-		// The history needed to resolve the remaining unknowns may have been pruned away; stop
-		// rather than dereferencing a missing entry (mirrors the reverse-walk guard above).
-		if (!auditEntry) break;
-		let priorRecord: any;
-		switch (auditEntry.type) {
-			case 'put':
-				priorRecord = auditEntry.getValue(store);
-				break;
-			case 'patch':
-				priorRecord = auditEntry.getValue(store);
-				break;
-		}
-		for (const key in priorRecord) {
-			if (unknowns.has(key)) {
-				record[key] = priorRecord[key];
-				unknowns.delete(key);
+	// A reversed patch that set a field to a plain value can't be undone (a plain set has no inverse),
+	// so those fields are left "unknown": their value at `timestamp` must come from older history.
+	// Reconstruct that state forward from the nearest base so CRDT ops accumulate correctly — e.g. a
+	// counter built up by `add` patches and later overwritten by a plain set. The prior code copied
+	// the nearest audit entry's raw field, which for a patch is an unresolved `{ __op__ }` object or a
+	// single delta rather than the folded value at `timestamp`. If the history needed to reconstruct a
+	// key is unavailable (pruned), the key keeps its live value (best effort, as before).
+	if (unknowns.size > 0 && auditTime > 0) {
+		const priorRecord = reconstructForward(auditStore, store, tableId, recordId, auditTime, timestamp);
+		if (priorRecord) {
+			for (const key of unknowns) {
+				// Object.hasOwn, not `in`: an unknown field named like a prototype member (toString,
+				// constructor, …) must not be filled from priorRecord's prototype chain.
+				if (Object.hasOwn(priorRecord, key)) record[key] = priorRecord[key];
 			}
 		}
-		auditTime = auditEntry.previousVersion;
 	}
 	// finally return the record in the state it was at the requested timestamp
 	return record;