@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/resources/search.ts

@@ -1,13 +1,15 @@
 import { ClientError, IndexRebuildingError, Violation } from '../utility/errors/hdbError.ts';
 import { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH, SEARCH_TYPES } from '../utility/lmdb/terms.ts';
 import { compareKeys, MAXIMUM_KEY, writeKey } from 'ordered-binary';
-import { SKIP } from '@harperfast/extended-iterable';
+import { SKIP, ExtendedIterable } from '@harperfast/extended-iterable';
 import { INVALIDATED, EVICTED, freezeRecord } from './Table.ts';
 import type { DirectCondition, Id } from './ResourceInterface.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import { lastMetadata } from './RecordEncoder.ts';
+import { writeKeyId, getReadTransactionGuard } from './DatabaseTransaction.ts';
 import { recordAction } from './analytics/write';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
+import { appendHeader } from '../server/serverHelpers/Headers.ts';
 
 // these are ratios/percentages of overall table size
 const OPEN_RANGE_ESTIMATE = 0.3;
@@ -140,16 +142,13 @@ export function executeConditions(
 				filtered
 				// recordAccess intentionally omitted: guards run once, at the top level (see above).
 			);
-		return searchByIndex(
-			condition,
-			txn,
-			condition.descending || request.reverse === true,
-			table,
-			request.allowFullScan,
+		return searchByIndex(condition, txn, condition.descending || request.reverse === true, table, {
+			allowFullScan: request.allowFullScan,
 			filtered,
 			context,
-			request.limit !== undefined ? (request.offset || 0) + request.limit : undefined
-		);
+			minResults:
+				request.limit === 0 ? 0 : request.limit !== undefined ? (request.offset || 0) + request.limit : undefined,
+		});
 	}
 	function mapConditionsToFilters(conditions, intersection, estimatedIncomingCount) {
 		return conditions
@@ -233,27 +232,70 @@ function composeRecordFilter(recordFilters, table, context): (primaryKey: Id) =>
 }
 
 /**
- * Search for records or keys, based on the search condition, using an index if available
+ * A record's index entries are not adjacent — the composite `[indexedValue, primaryKey]` key sorts
+ * on the indexed value first — so comparing neighbours cannot collapse them and the scan has to
+ * remember what it already yielded (#2434).
+ *
+ * Heap cost is one key per distinct record in the scanned range, held for the life of the scan, and
+ * a page window does not cap it: sibling-condition and row filters run downstream, so a selective
+ * filter keeps the scan running and the set growing while the page fills. On a multi-million-row
+ * range that is hundreds of MB where main streamed at O(1). Accepted for #2434 over the bounded
+ * alternative, which costs a primary read per index entry.
+ */
+function distinctRecords(entries: any): AsyncIterable<Id> {
+	const distinct = new ExtendedIterable();
+	(distinct as any).iterate = (options) => {
+		// A non-scalar key decodes to a fresh instance per entry, so identity never matches it;
+		// `writeKeyId` is the store's own key equality. Its bytes get their own set because a scalar
+		// string id can legitimately carry exactly those bytes, and one set would fold the two —
+		// the collision `flattenKey` has, and the reason Table.ts prefixes its key ids.
+		const yieldedKeys = new Set();
+		const yieldedEncodedKeys = new Set();
+		// map rather than filter: filter clears `continueOnRecoverableError` for everything below it
+		return entries
+			.map((primaryKey) => {
+				const isEncoded = typeof primaryKey === 'object' && primaryKey !== null;
+				const yielded = isEncoded ? yieldedEncodedKeys : yieldedKeys;
+				const identity = isEncoded ? writeKeyId(primaryKey) : primaryKey;
+				if (yielded.has(identity)) return SKIP;
+				yielded.add(identity);
+				return primaryKey;
+			})
+			.iterate(options);
+	};
+	return distinct as any;
+}
+
+/**
+ * Search for records or keys, based on the search condition, using an index if available. The four
+ * leading parameters are required at every call site; everything optional is named, so a new
+ * capability can be added without shifting positions at callers that don't use it.
  * @param searchCondition
  * @param transaction
  * @param reverse
  * @param Table
- * @param allowFullScan
- * @param filtered
+ * @param options
  */
 export function searchByIndex(
 	searchCondition: DirectCondition,
 	transaction: any,
 	reverse: boolean,
 	Table: any,
-	allowFullScan?: boolean,
-	filtered?: any,
-	context?: any,
-	// How many rows the query will ultimately consume (offset + limit), when it is bounded. An
-	// approximate index returns a fixed-size candidate list, so without this a query asking for more
-	// rows than that list holds silently gets a short result set. Only custom indexes read it.
-	minResults?: number
+	options: {
+		allowFullScan?: boolean;
+		filtered?: any;
+		context?: any;
+		// How many rows the query will ultimately consume (offset + limit), when it is bounded. An
+		// approximate index returns a fixed-size candidate list, so without this a query asking for more
+		// rows than that list holds silently gets a short result set. Only custom indexes read it.
+		minResults?: number;
+	} = {}
 ): AsyncIterable<Id | { key: Id; value: any }> {
+	// A stale positional caller passes `allowFullScan` here. Type checking only covers .ts callers, so
+	// fail loud rather than silently reading every option as undefined.
+	if (typeof options !== 'object' || options === null)
+		throw new TypeError('searchByIndex: the 5th argument is an options object (#2165), not a positional value');
+	const { allowFullScan, filtered, context, minResults } = options;
 	let attribute_name = searchCondition[0] ?? searchCondition.attribute;
 	let value = searchCondition[1] ?? searchCondition.value;
 	const comparator = searchCondition.comparator;
@@ -261,6 +303,9 @@ export function searchByIndex(
 		throw new ClientError(`Search condition for ${attribute_name} must have a value`);
 	}
 	let needFullScan;
+	// `[indexedValue, primaryKey]` is unique, so a scan that stays inside one indexed value cannot
+	// reach a record twice and skips the collapse below
+	let scansOneIndexedValue;
 	if (Array.isArray(attribute_name)) {
 		const firstAttributeName = attribute_name[0];
 		// get the potential relationship attribute
@@ -284,8 +329,7 @@ export function searchByIndex(
 				transaction,
 				reverse,
 				relatedTable,
-				allowFullScan,
-				joined
+				{ allowFullScan, filtered: joined }
 			);
 			if (attribute.relationship.to) {
 				// this is one-to-many or many-to-many, so we need to track the filtering of related entries that match
@@ -302,8 +346,7 @@ export function searchByIndex(
 						transaction,
 						reverse,
 						Table,
-						allowFullScan,
-						joined
+						{ allowFullScan, filtered: joined }
 					);
 				};
 				if (attribute.elements) {
@@ -388,6 +431,7 @@ export function searchByIndex(
 				start = value;
 				end = value;
 				inclusiveEnd = true;
+				scansOneIndexedValue = true;
 				break;
 			case 'in':
 				// Phase 1: route through filter — index-merge optimization is a Phase 2 follow-up.
@@ -510,28 +554,119 @@ export function searchByIndex(
 			// exploring until it has enough MATCHING results, rather than post-filtering an under-filled
 			// candidate set. Only indexes that opt in (filteredSearch) receive it; others post-filter as before.
 			const recordFilter = index.customIndex.filteredSearch ? searchCondition.recordFilter : undefined;
-			const loaded = index.customIndex.search(searchCondition, context, recordFilter, minResults).map((entry) => {
-				// if the custom index returns an entry with metadata, merge it with the loaded entry
-				if (typeof entry === 'object' && entry) {
-					const { key, ...otherProps } = entry;
-					if (key == null) return SKIP; // primaryKey missing from HNSW node — skip rather than crash
-					const loadedEntry = Table.primaryStore.getEntry(key, {
-						transaction: context && Table._readTxnForContext(context),
-					});
-					if (!loadedEntry) return SKIP; // record was deleted/expired or not yet visible
-					freezeRecord(loadedEntry?.value);
-					recordRead(loadedEntry);
-					return { ...otherProps, ...loadedEntry };
-				}
-				return entry;
+			const waiting = index.customIndex.filePrimary && searchCondition.waitForIndexMilliseconds > 0;
+			const start = waiting ? Promise.withResolvers<void>() : undefined;
+			const controller = waiting ? new AbortController() : undefined;
+			const checkActive = waiting ? getReadTransactionGuard(transaction) : undefined;
+			const signal = waiting
+				? context.signal
+					? AbortSignal.any([context.signal, controller.signal])
+					: controller.signal
+				: undefined;
+			const searchContext = waiting
+				? Object.create(context, {
+						signal: { value: signal },
+						indexSearchStart: { value: start.promise.then(() => checkActive?.()) },
+					})
+				: context;
+			const searched = index.customIndex.search(searchCondition, searchContext, {
+				filter:
+					waiting && recordFilter
+						? (id) => {
+								if (signal.aborted) return false;
+								checkActive?.();
+								return recordFilter(id);
+							}
+						: recordFilter,
+				minResults,
 			});
-			if (index.customIndex.rescoreResults) {
-				const rescored = index.customIndex.rescoreResults(loaded, searchCondition, comparator, attribute_name);
-				if (rescored != null) return rescored as any;
+			const coverage = (searched as any).indexCoverage;
+			if (!waiting && coverage && context?.responseHeaders) {
+				appendHeader(
+					context.responseHeaders,
+					'Harper-Index-Coverage',
+					`${coverage.state}; lag=${coverage.lagUpperBoundMilliseconds}; tolerance=${coverage.maxLagMilliseconds}`,
+					true
+				);
+				appendHeader(context.responseHeaders, 'Access-Control-Expose-Headers', 'Harper-Index-Coverage', true);
 			}
-			return loaded;
+			const processEntries = (entries: any[]) => {
+				const loaded = entries
+					.map((entry) => {
+						// if the custom index returns an entry with metadata, merge it with the loaded entry
+						if (typeof entry === 'object' && entry) {
+							const { key, ...otherProps } = entry;
+							if (key == null) return SKIP; // primaryKey missing from HNSW node — skip rather than crash
+							const loadedEntry = Table.primaryStore.getEntry(key, {
+								transaction: waiting ? transaction : context && Table._readTxnForContext(context),
+							});
+							if (!loadedEntry) return SKIP; // record was deleted/expired or not yet visible
+							freezeRecord(loadedEntry?.value);
+							recordRead(loadedEntry);
+							return { ...otherProps, ...loadedEntry };
+						}
+						return entry;
+					})
+					.filter((entry) => entry !== SKIP);
+				if (index.customIndex.rescoreResults) {
+					const rescored = index.customIndex.rescoreResults(loaded, searchCondition, comparator, attribute_name);
+					if (rescored != null) return rescored as any;
+				}
+				return loaded;
+			};
+			if (typeof (searched as any)?.then === 'function') {
+				let settled = false;
+				const pending = waiting
+					? (searched as Promise<any[]>)
+							.then((entries) => {
+								if (controller.signal.aborted) return [];
+								if (signal.aborted) throw signal.reason ?? new Error('Index search aborted', { cause: signal.reason });
+								checkActive?.();
+								return processEntries(entries);
+							})
+							.finally(() => {
+								settled = true;
+							})
+					: (searched as Promise<any[]>).then(processEntries);
+				// A consumer may abandon this lazy iterable without calling next().
+				pending.catch(() => {});
+				const results: any = new ExtendedIterable();
+				results.iterate = (options?: { async?: boolean }) => {
+					if (!options?.async) {
+						throw new Error(
+							'This index resolves search results asynchronously; the results must be consumed with async iteration'
+						);
+					}
+					// Overlapping next() calls share one cursor and cannot duplicate its first entry.
+					const iteratorPromise = pending.then((entries) => entries[Symbol.iterator]());
+					iteratorPromise.catch(() => {});
+					let closed = false;
+					return {
+						next() {
+							if (closed) return Promise.resolve({ done: true, value: undefined });
+							start?.resolve();
+							return iteratorPromise.then((inner) => {
+								if (closed) return { done: true, value: undefined };
+								checkActive?.();
+								return inner.next();
+							});
+						},
+						return(value?: any) {
+							closed = true;
+							if (!settled) controller?.abort();
+							iteratorPromise.then(
+								(inner) => (inner as any).return?.(value),
+								() => {}
+							);
+							return Promise.resolve({ done: true, value });
+						},
+					};
+				};
+				return results;
+			}
+			return processEntries(searched);
 		}
-		return index.getRange(rangeOptions).map(
+		const scanned = index.getRange(rangeOptions).map(
 			filter
 				? function ({ key, value }) {
 						let recordMatcher: any;
@@ -554,6 +689,14 @@ export function searchByIndex(
 					}
 				: ({ value }) => value
 		);
+		// Collapsing AFTER the map is required, not incidental: the condition's own filter tests one
+		// element at a time, so a record whose first entry fails it and whose second passes must
+		// still be reached. Collapsing first would settle that record on the failing entry and drop
+		// it. Scoped to attributes the schema declares multi-valued; a runtime array under an
+		// undeclared attribute repeats the same way but stays outside this boundary.
+		return scansOneIndexedValue || !findAttribute(Table.attributes, attribute_name)?.elements
+			? scanned
+			: distinctRecords(scanned);
 	} else {
 		return Table.primaryStore
 			.getRange(reverse ? { end: true, transaction, reverse: true } : { start: true, transaction })
@@ -1112,6 +1255,91 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar
 	}
 }
 
+/**
+ * Estimates a range comparator's entry count from the store's statistical estimate, blended with
+ * the table-fraction heuristic by the estimate's confidence so low-confidence estimates degrade
+ * to the old behavior. Returns undefined (caller falls back to the heuristic) when the store
+ * cannot estimate, the shape is unexpected, or the executed range wouldn't match this one.
+ */
+function estimateRangeCondition(table, condition, searchType, fraction) {
+	const attributeName = condition[0] ?? condition.attribute;
+	const isPrimaryKey = attributeName === table.primaryKey;
+	const store = isPrimaryKey ? table.primaryStore : usableIndex(table, attributeName);
+	// LMDB-backed and custom index stores do not implement estimateCount
+	if (typeof store?.estimateCount !== 'function') return undefined;
+	let value = condition[1] ?? condition.value;
+	if (value instanceof Date) value = value.getTime();
+	let range;
+	switch (searchType) {
+		case 'lt':
+			// `start: true` mirrors searchByIndex: `true` sorts above `null`, so an indexNulls
+			// index's `[null, primaryKey]` entries are outside the executed range
+			range = { start: true, end: value };
+			break;
+		case 'le':
+			range = { start: true, end: value, inclusiveEnd: true };
+			break;
+		case 'gt':
+			range = { start: value, exclusiveStart: true };
+			break;
+		case 'ge':
+			range = { start: value };
+			break;
+		case 'between':
+		case 'gele':
+		case 'gelt':
+		case 'gtlt':
+		case 'gtle': {
+			if (!Array.isArray(value)) return undefined;
+			let [start, end] = value;
+			if (start instanceof Date) start = start.getTime();
+			if (end instanceof Date) end = end.getTime();
+			range = {
+				start,
+				end,
+				inclusiveEnd: searchType === 'between' || searchType === 'gele' || searchType === 'gtle',
+				exclusiveStart: searchType === 'gtlt' || searchType === 'gtle',
+			};
+			break;
+		}
+		case 'starts_with': {
+			const prefix = value?.toString();
+			if (!prefix) return undefined;
+			range = { start: prefix, end: getStringPrefixUpperBound(prefix) };
+			break;
+		}
+		case 'prefix': {
+			let start = Array.isArray(value) ? value : [value, null];
+			if (start[start.length - 1] != null) start = start.concat(null);
+			const end = start.slice(0);
+			end[end.length - 1] = MAXIMUM_KEY;
+			range = { start, end };
+			break;
+		}
+		default:
+			return undefined;
+	}
+	// Long string bounds get truncated + filtered at execution (searchByIndex), so the
+	// executed range is wider than this one; don't estimate what won't be iterated.
+	if (
+		(typeof range.start === 'string' && range.start.length > MAX_SEARCH_KEY_LENGTH) ||
+		(typeof range.end === 'string' && range.end.length > MAX_SEARCH_KEY_LENGTH)
+	) {
+		return undefined;
+	}
+	let count, confidence;
+	try {
+		({ count, confidence } = store.estimateCount(range) ?? {});
+	} catch {
+		// a concurrently closing/dropped store must degrade the plan, not fail the query
+		return undefined;
+	}
+	if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+		return undefined;
+	const heuristic = fraction * estimatedEntryCount(table.primaryStore) + 1;
+	return Math.max(1, Math.round(confidence * count + (1 - confidence) * heuristic));
+}
+
 /** The index a condition can be driven by; searchByIndex refuses a rebuilding one, so it reads as absent. */
 function usableIndex(table, attributeName): any {
 	const index = attributeName == null ? undefined : table.indices[attributeName];
@@ -1166,6 +1394,13 @@ export function estimateCondition(table) {
 			// before comparator dispatch: several branches fall back to a finite table-fraction heuristic
 			if (drivesRebuildingIndex(table, condition[0] ?? condition.attribute)) {
 				condition.estimated_count = Infinity;
+			} else if (condition.negated) {
+				// a negated condition always executes as a full scan (searchByIndex forces
+				// needFullScan), so follow the filter-only convention used by contains/ends_with:
+				// estimate Infinity so its positive-range estimate can never win the driving-condition
+				// ordering. Short-circuit here rather than computing (and discarding) a positive-range
+				// estimate that would cross the native FFI boundary for nothing.
+				condition.estimated_count = Infinity;
 			} else if (searchType === SEARCH_TYPES.EQUALS || !searchType) {
 				const attribute_name = condition[0] ?? condition.attribute;
 				if (attribute_name == null || attribute_name === table.primaryKey) condition.estimated_count = 1;
@@ -1217,11 +1452,14 @@ export function estimateCondition(table) {
 				} else if (Array.isArray(condition.value)) {
 					condition.estimated_count = Infinity;
 				} else condition.estimated_count = Infinity;
-				// for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess
 			} else if (searchType === 'starts_with' || searchType === 'prefix')
-				condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+				condition.estimated_count =
+					estimateRangeCondition(table, condition, searchType, STARTS_WITH_ESTIMATE) ??
+					STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
 			else if (searchType === 'between')
-				condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+				condition.estimated_count =
+					estimateRangeCondition(table, condition, searchType, BETWEEN_ESTIMATE) ??
+					BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
 			else if (searchType === 'sort') {
 				const attribute_name = condition[0] ?? condition.attribute;
 				const index = usableIndex(table, attribute_name);
@@ -1236,7 +1474,10 @@ export function estimateCondition(table) {
 				if (index?.customIndex?.estimateCount)
 					// allow custom index to define its own estimation of counts
 					condition.estimated_count = index.customIndex.estimateCount(condition.value);
-				else condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+				else
+					condition.estimated_count =
+						estimateRangeCondition(table, condition, searchType, OPEN_RANGE_ESTIMATE) ??
+						OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
 			}
 			// we give a condition significantly more weight/preference if we will be ordering by it
 			if (typeof condition.descending === 'boolean') condition.estimated_count /= 2;
@@ -1422,6 +1663,7 @@ function parseBlock(query, expectedEnd) {
 						break;
 					case 'group-by':
 						recordError('group by is not implemented yet');
+						break;
 					case 'sort':
 						query.sort = toSortObject(args);
 						break;

package/resources/tracked.ts

@@ -3,7 +3,15 @@ import * as crdtOperations from './crdt.ts';
 import { Blob } from './blob.ts';
 import * as harperLogger from '../utility/logging/harper_logger.ts';
 
+export const ASSERT_TRACKED_WRITABLE = Symbol('assert-tracked-writable');
+export const GET_TRACKED_WRITE_GENERATION = Symbol('get-tracked-write-generation');
+
+function assertWritable(target, generation?) {
+	target[ASSERT_TRACKED_WRITABLE]?.(generation);
+}
+
 function getChanges(target) {
+	assertWritable(target);
 	let changes = target.getChanges();
 	if (!changes) {
 		changes = Object.create(null);
@@ -49,6 +57,7 @@ export function assignTrackedAccessors(Target, typeDef, useFullPropertyProxy = f
 					// still load), this is a user assignment to a resolved attribute, and a read-only resolver
 					// (a scalar @computed has no attribute.set) has nowhere to put it (harper#2359).
 					if (!attribute.set) throw new ClientError(`${name} is a computed attribute and can not be assigned`);
+					assertWritable(this);
 					return attribute.set(this, related);
 				},
 				configurable: true,
@@ -188,7 +197,7 @@ export function assignTrackedAccessors(Target, typeDef, useFullPropertyProxy = f
 					}
 					const sourceValue = this.getRecord()?.[name];
 					if (sourceValue && typeof sourceValue === 'object') {
-						const updatedValue = trackObject(sourceValue, attribute);
+						const updatedValue = trackObject(sourceValue, attribute, this);
 						if (updatedValue) {
 							if (!changes) {
 								this._setChanges((changes = Object.create(null)));
@@ -288,7 +297,7 @@ function getProxiedProperty(target, name, receiver) {
 		}
 		const sourceValue = receiver.getRecord?.()?.[name];
 		if (sourceValue && typeof sourceValue === 'object') {
-			const updatedValue = trackObject(sourceValue);
+			const updatedValue = trackObject(sourceValue, undefined, receiver);
 			if (updatedValue) {
 				if (!changes) {
 					changes = Object.create(null);
@@ -303,6 +312,7 @@ function getProxiedProperty(target, name, receiver) {
 }
 function setProxiedProperty(target: any, name: string | symbol, value, receiver) {
 	if (typeof name === 'string') {
+		assertWritable(receiver);
 		let changes = receiver.getChanges?.();
 		if (!changes) {
 			changes = {};
@@ -315,7 +325,7 @@ function setProxiedProperty(target: any, name: string | symbol, value, receiver)
 	return true;
 }
 
-function trackObject(sourceObject: any, typeDef?: any) {
+function trackObject(sourceObject: any, typeDef?: any, writableOwner?: any) {
 	// lazily instantiate in case of recursive structures
 	let TrackedObject;
 	switch (sourceObject.constructor) {
@@ -326,18 +336,41 @@ function trackObject(sourceObject: any, typeDef?: any) {
 					typeDef.TrackedObject = TrackedObject = class extends GenericTrackedObject {};
 					assignTrackedAccessors(TrackedObject, typeDef);
 				}
-				return new TrackedObject(sourceObject);
+				return new TrackedObject(sourceObject, writableOwner);
 			} else {
-				return new GenericTrackedObject(sourceObject);
+				return new GenericTrackedObject(sourceObject, writableOwner);
 			}
 		case Array:
-			const trackedArray = new TrackedArray(sourceObject.length, sourceObject);
+			const trackedArray = new TrackedArray(sourceObject.length, sourceObject, writableOwner);
 			for (let i = 0, l = sourceObject.length; i < l; i++) {
 				let element = sourceObject[i];
-				if (element && typeof element === 'object') element = trackObject(element, typeDef?.elements);
+				if (element && typeof element === 'object') element = trackObject(element, typeDef?.elements, trackedArray);
 				trackedArray[i] = element;
 			}
-			return trackedArray;
+			if (!writableOwner) return trackedArray;
+			return new Proxy(trackedArray, {
+				set(target, name, value) {
+					if (typeof name === 'string') {
+						assertWritable(target);
+						target[HAS_ARRAY_CHANGES] = true;
+					}
+					return Reflect.set(target, name, value, target);
+				},
+				defineProperty(target, name, descriptor) {
+					if (typeof name === 'string') {
+						assertWritable(target);
+						target[HAS_ARRAY_CHANGES] = true;
+					}
+					return Reflect.defineProperty(target, name, descriptor);
+				},
+				deleteProperty(target, name) {
+					if (typeof name === 'string') {
+						assertWritable(target);
+						target[HAS_ARRAY_CHANGES] = true;
+					}
+					return Reflect.deleteProperty(target, name);
+				},
+			});
 		// any other objects (like Date) are left unchanged
 		default:
 			return sourceObject;
@@ -346,10 +379,20 @@ function trackObject(sourceObject: any, typeDef?: any) {
 export class GenericTrackedObject<T extends object = any> {
 	#record: T;
 	#changes: Partial<T>;
-	constructor(sourceObject?: GenericTrackedObject<T> | T) {
+	#writableOwner: any;
+	#writableGeneration: any;
+	constructor(sourceObject?: GenericTrackedObject<T> | T, writableOwner?: any) {
 		if ((sourceObject as GenericTrackedObject<T>)?.getRecord)
 			throw new Error('Can not track an already tracked object, check for circular references');
 		this.#record = sourceObject as any;
+		this.#writableOwner = writableOwner;
+		this.#writableGeneration = writableOwner?.[GET_TRACKED_WRITE_GENERATION]?.();
+	}
+	[ASSERT_TRACKED_WRITABLE]() {
+		this.#writableOwner?.[ASSERT_TRACKED_WRITABLE]?.(this.#writableGeneration);
+	}
+	[GET_TRACKED_WRITE_GENERATION]() {
+		return this.#writableGeneration;
 	}
 	getRecord(): T {
 		return this.#record;
@@ -404,18 +447,13 @@ export function updateAndFreeze(target, changes = target.getChanges?.()) {
 	let mergedUpdatedObject: any;
 	if (!target) return changes;
 	if (target.getRecord && target.constructor === Array && !Object.isFrozen(target)) {
-		// a tracked array, by default we can freeze the tracked array itself
-		mergedUpdatedObject = target;
+		if (!hasChanges(target)) return (target as any).getRecord();
+		// Materialize a plain array so the stored value retains neither the mutation proxy nor its owner.
+		mergedUpdatedObject = new Array(target.length);
 		for (let i = 0, l = target.length; i < l; i++) {
 			let value = target[i];
 			if (value && typeof value === 'object') {
-				const newValue = updateAndFreeze(value);
-				if (newValue !== value && mergedUpdatedObject === target) {
-					// if we need to make any changes to the user's array, we make a copy so we don't modify
-					// an array that the user may be using with transient properties
-					mergedUpdatedObject = target.slice(0);
-				}
-				value = newValue;
+				value = updateAndFreeze(value);
 			}
 			mergedUpdatedObject[i] = value;
 		}
@@ -493,15 +531,28 @@ export function hasChanges(target) {
 }
 
 const HAS_ARRAY_CHANGES = Symbol.for('has-array-changes');
+const TRACKED_ARRAY_RECORD = Symbol('tracked-array-record');
+const TRACKED_WRITABLE_OWNER = Symbol('tracked-writable-owner');
+const TRACKED_WRITABLE_GENERATION = Symbol('tracked-writable-generation');
 class TrackedArray extends Array {
-	#record: any;
+	[TRACKED_ARRAY_RECORD]: any;
+	[TRACKED_WRITABLE_OWNER]: any;
+	[TRACKED_WRITABLE_GENERATION]: any;
 	[HAS_ARRAY_CHANGES]: boolean;
-	constructor(length, record) {
+	constructor(length, record, writableOwner) {
 		super(length);
-		this.#record = record;
+		this[TRACKED_ARRAY_RECORD] = record;
+		this[TRACKED_WRITABLE_OWNER] = writableOwner;
+		this[TRACKED_WRITABLE_GENERATION] = writableOwner?.[GET_TRACKED_WRITE_GENERATION]?.();
+	}
+	[ASSERT_TRACKED_WRITABLE]() {
+		this[TRACKED_WRITABLE_OWNER]?.[ASSERT_TRACKED_WRITABLE]?.(this[TRACKED_WRITABLE_GENERATION]);
+	}
+	[GET_TRACKED_WRITE_GENERATION]() {
+		return this[TRACKED_WRITABLE_GENERATION];
 	}
 	getRecord() {
-		return this.#record;
+		return this[TRACKED_ARRAY_RECORD];
 	}
 	splice(...args) {
 		this[HAS_ARRAY_CHANGES] = true;

package/resources/transactionBroadcast.ts

@@ -95,7 +95,7 @@ export function addSubscription(table, key, listener?: (key) => any, startTime?:
  * subscription and get the initial state.
  */
 class Subscription extends IterableEventQueue {
-	listener: (recordId: Id, auditEntry: any, localTime: number, beginTxn: boolean) => void;
+	listener: (recordId: Id, auditEntry: any, txnLogKey: number, beginTxn: boolean) => void;
 	subscriptions: any;
 	startTime?: number;
 	includeDescendants?: boolean;
@@ -178,10 +178,10 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 			}
 			if (result.done) break;
 			const auditRecord = result.value;
-			const timestamp: number = auditRecord.localTime ?? auditRecord.version;
+			const timestamp: number = auditRecord.txnLogKey;
 			subscriptions.lastTxnTime = timestamp;
 			// the transaction extent: RocksDB entries committed together share the log key (record
-			// versions may differ); LMDB's localTime is a per-entry audit key, so version delimits there
+			// versions may differ); LMDB's transaction-log key is per-entry, so version delimits there
 			const txnKey = auditStore.reusableIterable ? timestamp : auditRecord.version;
 			if (ACTIONS_OF_INTEREST.includes(auditRecord.type)) {
 				const tableSubscriptions = subscriptions[auditRecord.tableId];