@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.1

package/resources/Resource.ts

@@ -147,20 +147,55 @@ export class Resource<Record extends object = any> implements ResourceInterface<
 	static put = transactional(
 		function (resource: any, query: RequestTarget, request: Context, data: any) {
 			if (Array.isArray(data) && resource.#isCollection && resource.constructor.loadAsInstance !== false) {
+				const resourceClass = resource.constructor;
+				const primaryKey = resourceClass.primaryKey;
+				// Before dispatching anything: a malformed body must not race a sibling's write, and a bare
+				// dereference below would reach the client as a 500 carrying V8's wording. A primitive is
+				// malformed for the same reason — only a Table's own key validation rejects one later, so a
+				// plain Resource's override would otherwise be handed it.
+				for (let index = 0; index < data.length; index++) {
+					const element = data[index];
+					if (element == null || typeof element !== 'object')
+						throw new ClientError(
+							`Array element at index ${index} is ${element === null ? 'null' : typeof element}, expected a record`
+						);
+				}
+				const elementTarget = elementTargetFactory(query);
 				const results = [];
-				for (const element of data) {
-					const resourceClass = resource.constructor;
-					const id = element[resourceClass.primaryKey];
-					let target = new RequestTarget();
-					target.id = id;
-					const elementResource = resourceClass.getResource(target, request, {
-						async: true,
+				try {
+					for (let index = 0; index < data.length; index++) {
+						const element = data[index];
+						const target = elementTarget(element[primaryKey]);
+						const elementResource = resourceClass.getResource(target, request, {
+							async: true,
+						});
+						// `target`, not `request`: this is the target slot, and Table's back-compat shift only
+						// recognizes a RequestTarget there — a context is taken for the record.
+						// `missingMethod` rather than an unguarded call, so a resource without `put` answers 405
+						// here exactly as it does on the single-record path below.
+						if (typeof elementResource.then === 'function')
+							results.push(
+								elementResource.then((resource) =>
+									resource.put ? resource.put(element, target) : missingMethod(resource, 'put')
+								)
+							);
+						else if (elementResource.put) results.push(elementResource.put(element, target));
+						else missingMethod(elementResource, 'put');
+					}
+				} catch (error) {
+					// Settle the already-dispatched elements or a rejection among them goes unhandled. Then
+					// report the earliest-index failure, the same rule `settleElements` applies: every sibling
+					// already dispatched precedes the element that threw, so the batch reports the same failure
+					// whether that element's dispatch resolved synchronously or asynchronously.
+					return Promise.allSettled(results).then((settled) => {
+						const failed = settled.find((sibling) => sibling.status === 'rejected');
+						if (!failed) throw error;
+						throw (failed as PromiseRejectedResult).reason;
 					});
-					if (typeof elementResource.then === 'function')
-						results.push(elementResource.then((resource) => resource.put(element, request)));
-					else results.push(elementResource.put(element, query));
 				}
-				return Promise.all(results);
+				// Not `Promise.all`: settling the batch while a slower element is still resolving lets that
+				// element commit outside the failed batch — a partially applied array PUT.
+				return settleElements(results);
 			}
 			return resource.put
 				? resource.constructor.loadAsInstance === false
@@ -571,6 +606,53 @@ export function snakeCase(camelCase: string) {
 	);
 }
 
+/**
+ * Mint per-element targets for a fanned-out batch write. Each element needs its own object — its id
+ * must not be visible to a sibling whose dispatch resolves later — but it also needs the request's
+ * query and route metadata, which a `put()` override can read. The request's contribution is
+ * therefore resolved once here rather than deep-cloned per element, and nested metadata is shared
+ * rather than copied. `checkPermission` never carries: the collection receiver's single verdict is
+ * authoritative and per-element dispatch must not re-arm it.
+ */
+function elementTargetFactory(query: any): (id: any) => RequestTarget {
+	const base = cloneRequestTarget(query);
+	const search = base.toString();
+	// Only what the element target would not otherwise have. `RequestTarget` declares `pathname`,
+	// `search` and `sort` without `declare`, so they are own properties on every instance and would
+	// otherwise always be "carried" — overwriting what this element's own construction just derived,
+	// and making the skip below dead. A nullish own value carries no metadata either.
+	const carried: Record<string, any> = {};
+	for (const key of Object.keys(base)) {
+		if (key === 'id' || key === 'isCollection' || key === 'pathname' || key === 'search') continue;
+		const value = (base as any)[key];
+		if (value != null) carried[key] = value;
+	}
+	const hasCarried = Object.keys(carried).length > 0;
+	return (id) => {
+		const target = search ? new RequestTarget(search) : new RequestTarget();
+		if (hasCarried) Object.assign(target, carried);
+		target.id = id;
+		target.isCollection = false;
+		return target;
+	};
+}
+
+/**
+ * Report a fanned-out batch write's outcome only once every element has staged or failed its own
+ * write, so the enclosing transaction cannot unwind under one. Rejects with the first failure.
+ */
+function settleElements(results: any[]): Promise<any[]> {
+	return Promise.allSettled(results).then((settled) => {
+		const values = new Array(settled.length);
+		for (let index = 0; index < settled.length; index++) {
+			const element = settled[index];
+			if (element.status === 'rejected') throw element.reason;
+			values[index] = element.value;
+		}
+		return values;
+	});
+}
+
 /**
  * This is responsible for arranging arguments in the main static methods and creating the appropriate context and default transaction wrapping
  * @param action
@@ -733,8 +815,10 @@ function transactional(
 		if (!query) {
 			query = new RequestTarget();
 			query.id = id;
-			if (isCollection && options.method === 'put' && Array.isArray(data) && this.loadAsInstance === false)
-				query.isCollection = true;
+			// The collection inferred from the batch's null id has to survive onto the synthesized target,
+			// or the array is dispatched as one record update against a null primary key. Array-only:
+			// a null id on non-array data means auto-generate, not collection.
+			if (isCollection && options.method === 'put' && Array.isArray(data)) query.isCollection = true;
 		}
 		if (options.method === 'query' && data && typeof data === 'object') {
 			// QUERY executes the independently parsed body target. Make its requested projection part of

package/resources/ResourceInterface.ts

@@ -201,6 +201,10 @@ interface TypedDirectCondition<Record extends object, Property extends keyof Rec
 	search_type?: Comparator;
 	value?: Record[Property] | Record[Property][];
 	search_value?: Record[Property] | Record[Property][];
+	/** Native HNSW coverage tolerance in milliseconds; defaults to 3000, with 0 requiring current coverage. */
+	maxIndexLagMilliseconds?: number;
+	/** Wait for coverage of prior committed writes, up to 30,000 ms; 0 (default) does not wait. */
+	waitForIndexMilliseconds?: number;
 	/**
 	 * If true, the condition is negated. Phase 1: filter-only — forces a
 	 * full scan unless paired with another indexed condition.
@@ -223,6 +227,10 @@ export type Conditions<Record extends object = any> = Condition<Record>[];
 
 export interface Sort<Record extends object = any> {
 	attribute: keyof Record;
+	/** Native HNSW coverage tolerance in milliseconds; defaults to 3000, with 0 requiring current coverage. */
+	maxIndexLagMilliseconds?: number;
+	/** Wait for coverage of prior committed writes, up to 30,000 ms; 0 (default) does not wait. */
+	waitForIndexMilliseconds?: number;
 	descending?: boolean;
 	next?: Sort<Record>;
 }

package/resources/RocksIndexStore.ts

@@ -1,3 +1,4 @@
+import { trackReadRange } from './DatabaseTransaction.ts';
 import {
 	type CountEstimate,
 	type CountEstimateOptions,
@@ -48,7 +49,7 @@ export class RocksIndexStore extends RocksDatabase {
 	 * @param options
 	 */
 	getRange(options: StoreIteratorOptions): Iterable<any> {
-		return super.getRange(translateIndexBounds(options)).map(({ key }) => {
+		return trackReadRange(options.transaction, () => super.getRange(translateIndexBounds(options))).map(({ key }) => {
 			return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] };
 		});
 	}

package/resources/RocksTransactionLogStore.ts

@@ -29,10 +29,16 @@ type TransactionLogIterator = Iterator<TransactionEntry | number> & {
 };
 
 type TrackedIterator = IterableIterator<TransactionEntry> & { lastVersion?: number; lastEndTxn?: boolean };
+type NamedTransactionEntry = TransactionEntry & { logName?: string };
+export type ExactStartFailure = 'missing' | 'incomplete' | 'duplicate';
 
 export type TransactionLogIterable = Iterable<AuditRecord> & {
 	/** Corrupt frames that ended a log's iteration during this range. */
 	corruptFrameStop: CorruptFrameStop;
+	/** Physical logs whose iterators ended with an unexpected, non-corruption error. */
+	failedLogs: Set<string>;
+	/** Physical logs whose requested exact transaction could not form one unambiguous resume boundary. */
+	exactStartFailures: Map<string, ExactStartFailure>;
 };
 
 const reportCorruptFrame = createCorruptFrameReporter(harperLogger);
@@ -242,6 +248,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 	getRange(options: {
 		start?: number;
 		exactStart?: boolean;
+		exclusiveStart?: boolean;
 		end?: number;
 		log?: string | number;
 		excludeLogs?: string[];
@@ -249,6 +256,10 @@ export class RocksTransactionLogStore extends EventEmitter {
 		startByLog?: Map<string, number>;
 		startFromLastFlushed?: boolean;
 		readUncommitted?: boolean;
+		/** Include the source transaction-log name on each returned audit record. */
+		includeLogName?: boolean;
+		/** Validate and consume one complete exact-start transaction per entry in `startByLog`. */
+		resumeAfterExactStart?: boolean;
 		/**
 		 * Track which version a break truncated, in `corruptFrameStop.truncatedVersions`. Costs two
 		 * property stores per yielded entry (see below), so it defaults off: only boot replay reads
@@ -262,6 +273,46 @@ export class RocksTransactionLogStore extends EventEmitter {
 		let singleLogIterator: TrackedIterator;
 		const attributeCorruption = options.trackCorruptTransactions === true;
 		const corruptFrameStop: CorruptFrameStop = { breaks: 0, truncatedVersions: new Set(), midLogBreak: false };
+		const failedLogs = new Set<string>();
+		const exactStartFailures = new Map<string, ExactStartFailure>();
+		const failedIterators = new WeakSet<IterableIterator<TransactionEntry>>();
+		const safeNext = (iterator: IterableIterator<TransactionEntry>, log: TransactionLog) => {
+			if (failedIterators.has(iterator)) return { value: undefined, done: true } as IteratorResult<TransactionEntry>;
+			try {
+				return iterator.next();
+			} catch (error) {
+				failedIterators.add(iterator);
+				failedLogs.add(log.name);
+				harperLogger.error('Transaction log iterator failed; terminating this log', error, {
+					log: log.name,
+				});
+				return { value: undefined, done: true } as IteratorResult<TransactionEntry>;
+			}
+		};
+		const resumePastExactStart = (
+			result: IteratorResult<TransactionEntry>,
+			iterator: TrackedIterator,
+			log: TransactionLog,
+			expected: number
+		): IteratorResult<TransactionEntry> => {
+			if (result.done || result.value.timestamp !== expected) {
+				exactStartFailures.set(log.name, 'missing');
+				return { value: undefined, done: true };
+			}
+			while (!result.value.endTxn) {
+				result = safeNext(iterator, log);
+				if (result.done || result.value.timestamp !== expected) {
+					exactStartFailures.set(log.name, 'incomplete');
+					return { value: undefined, done: true };
+				}
+			}
+			result = safeNext(iterator, log);
+			if (!result.done && result.value.timestamp === expected) {
+				exactStartFailures.set(log.name, 'duplicate');
+				return { value: undefined, done: true };
+			}
+			return result;
+		};
 		// Each log's iterator carries the version and endTxn of the last entry it yielded, so a break
 		// can be attributed to the source transaction whose remaining entries it swallowed — unless
 		// that entry's own endTxn already closed the transaction, in which case the break fell after
@@ -299,9 +350,34 @@ export class RocksTransactionLogStore extends EventEmitter {
 					log = this.rootStore.useLog(options.log);
 				}
 			}
-			const queryIterator = trackCorruptFrames(log, options);
+			const resumeAfterExactStart =
+				options.resumeAfterExactStart === true && options.exactStart === true && options.start !== undefined;
+			const queryOptions = resumeAfterExactStart ? { ...options, exclusiveStart: false } : options;
+			const queryIterator = trackCorruptFrames(log, queryOptions);
 			singleLogIterator = queryIterator;
-			iterable.iterate = () => queryIterator;
+			let exactStartObserved = !resumeAfterExactStart;
+			iterable.iterate =
+				options.includeLogName || resumeAfterExactStart
+					? () => ({
+							next: () => {
+								let result = safeNext(queryIterator, log);
+								if (!exactStartObserved) {
+									exactStartObserved = true;
+									if (resumeAfterExactStart) result = resumePastExactStart(result, queryIterator, log, options.start);
+								}
+								if (!result.done && options.includeLogName) (result.value as NamedTransactionEntry).logName = log.name;
+								return result;
+							},
+							[Symbol.iterator]() {
+								return this;
+							},
+							return: (value?: any) => queryIterator.return?.(value) ?? { value, done: true },
+							throw: (error?: any) => {
+								if (queryIterator.throw) return queryIterator.throw(error);
+								throw error;
+							},
+						})
+					: () => queryIterator;
 		} else {
 			const onlyKeys = options.onlyKeys;
 			let logs: TransactionLog[] = [];
@@ -309,30 +385,8 @@ export class RocksTransactionLogStore extends EventEmitter {
 			let nextEntries: any[];
 			let latestUpdates: number;
 			const iterators: TrackedIterator[] = [];
-			// Iterators that have permanently failed (corrupt entry stuck at the same
-			// position). Tracked by identity so the retry-poll path in next() and
-			// updateIterators() never calls .next() on them again — otherwise every
-			// subsequent drain cycle would re-throw the same RangeError, spamming logs
-			// and burning CPU.
-			const failedIterators = new WeakSet<IterableIterator<TransactionEntry>>();
-			// Per-log advance that converts a thrown corrupt-entry error from rocksdb-js
-			// into a clean `done: true` for that iterator. The reader's RangeError leaves
-			// `position` at the bad entry; re-calling next() would re-throw indefinitely.
-			// Terminating just this log lets the aggregate keep draining the other peers'
-			// logs and prevents the throw from escaping into setImmediate-scheduled
-			// consumers (notifyFromTransactionData) where it becomes an uncaughtException.
-			const safeNext = (iterator: IterableIterator<TransactionEntry>, log?: TransactionLog) => {
-				if (failedIterators.has(iterator)) return { value: undefined, done: true };
-				try {
-					return iterator.next();
-				} catch (error) {
-					failedIterators.add(iterator);
-					harperLogger.error('Transaction log iterator failed; terminating this log', error, {
-						log: log?.name,
-					});
-					return { value: undefined, done: true };
-				}
-			};
+			const expectedExactStarts: Array<number | undefined> = [];
+			const observedExactStarts = new Set<string>();
 			const updateIterators = () => {
 				if (latestUpdates !== this.updates) {
 					const latestLogs = (this.nodeLogs || this.loadLogs()).filter(
@@ -342,9 +396,18 @@ export class RocksTransactionLogStore extends EventEmitter {
 						if (!logs.includes(log)) {
 							logs.push(log);
 							let queryOptions = options;
+							let expectedExactStart: number | undefined;
 							if (options.startByLog) {
-								// if the startByLog is provided, we use that
-								queryOptions = { ...options, start: options.startByLog.get(log.name) ?? 0 };
+								if (options.startByLog.has(log.name)) {
+									expectedExactStart = options.startByLog.get(log.name)!;
+									queryOptions = {
+										...options,
+										start: expectedExactStart,
+										exclusiveStart: options.resumeAfterExactStart ? false : options.exclusiveStart,
+									};
+								} else {
+									queryOptions = { ...options, start: 0, exactStart: false, exclusiveStart: false };
+								}
 							} else if (latestUpdates >= 0) {
 								// if this is not the first update, that means that this is a brand new log and if start wasn't specified
 								// that means we are taking all future requests, so we need to start at zero so we don't introduce a race
@@ -352,6 +415,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 								queryOptions = { ...options, start: options.start ?? 0 };
 							}
 							iterators.push(trackCorruptFrames(log, queryOptions));
+							expectedExactStarts.push(expectedExactStart);
 						}
 					}
 					latestUpdates = this.updates;
@@ -360,12 +424,22 @@ export class RocksTransactionLogStore extends EventEmitter {
 							let log = logs[i];
 							if (!latestLogs.includes(log)) {
 								logs.splice(i, 1);
-								iterators.splice(i--, 1);
+								iterators.splice(i, 1);
+								expectedExactStarts.splice(i--, 1);
 							}
 						}
 					}
 				}
-				nextEntries = iterators.map((iterator, i) => safeNext(iterator, logs[i]));
+				nextEntries = iterators.map((iterator, i) => {
+					const result = safeNext(iterator, logs[i]);
+					const expected = expectedExactStarts[i];
+					if (expected !== undefined && !observedExactStarts.has(logs[i].name)) {
+						observedExactStarts.add(logs[i].name);
+						if (options.resumeAfterExactStart) return resumePastExactStart(result, iterator, logs[i], expected);
+						if (result.done || result.value.timestamp !== expected) exactStartFailures.set(logs[i].name, 'missing');
+					}
+					return result;
+				});
 			};
 			updateIterators();
 
@@ -408,6 +482,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 							}
 						}
 						if (earliestIndex >= 0) {
+							if (options.includeLogName) (earliest as NamedTransactionEntry).logName = logs[earliestIndex].name;
 							if (attributeCorruption) {
 								// before the refill, which is where a break surfaces and needs this entry's version
 								iterators[earliestIndex].lastVersion = earliest.timestamp;
@@ -438,6 +513,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 					if (index >= 0) {
 						logs.splice(index, 1);
 						iterators.splice(index, 1);
+						expectedExactStarts.splice(index, 1);
 						nextEntries.splice(index, 1);
 						options.excludeLogs.push(logName);
 					}
@@ -445,7 +521,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 			};
 			iterable.iterate = () => aggregateIterator;
 		}
-		const mappedAggregateIterable = iterable.map(({ timestamp, data, endTxn }: TransactionEntry) => {
+		const mappedAggregateIterable = iterable.map(({ timestamp, data, endTxn, logName }: NamedTransactionEntry) => {
 			// A break surfaces on the pull after this entry, so recording it here is in time to attribute
 			// that break to this entry's transaction. The aggregate branch records its own, per source
 			// log, because there this callback cannot tell which log an entry came from.
@@ -478,6 +554,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 					position += 8;
 				}
 				const auditRecord = readAuditEntry(data, position, undefined);
+				if (options.includeLogName) auditRecord.logName = logName;
 				auditRecord.txnLogKey = timestamp;
 				auditRecord.endTxn = endTxn;
 				auditRecord.previousResidencyId = previousResidencyId;
@@ -493,6 +570,7 @@ export class RocksTransactionLogStore extends EventEmitter {
 					// the log key is all this entry still yields; its record version is undecodable
 					version: timestamp,
 					txnLogKey: timestamp,
+					logName: options.includeLogName ? logName : undefined,
 					endTxn,
 					type: undefined,
 					tableId: undefined,
@@ -509,6 +587,8 @@ export class RocksTransactionLogStore extends EventEmitter {
 			mappedAggregateIterable.removeLog = aggregateIterator.removeLog;
 		}
 		mappedAggregateIterable.corruptFrameStop = corruptFrameStop;
+		mappedAggregateIterable.failedLogs = failedLogs;
+		mappedAggregateIterable.exactStartFailures = exactStartFailures;
 		return mappedAggregateIterable as TransactionLogIterable;
 	}
 	getKeys(_options?: any) {