@harperfast/harper 5.2.0 → 5.2.2

package/resources/Table.ts

@@ -75,7 +75,7 @@ import { rebuildUpdateBefore } from './crdt.ts';
 import { appendHeader } from '../server/serverHelpers/Headers.ts';
 import fs from 'node:fs';
 import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts';
-import { onStorageReclamation } from '../server/storageReclamation.ts';
+import { onStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
 import { throttle } from '../server/throttle.ts';
@@ -389,10 +389,20 @@ export function makeTable(options) {
 	if (!attributes) attributes = [];
 	if (!properties) properties = projectAttributesToProperties(attributes);
 	const updateRecord = recordUpdater(primaryStore, tableId, auditStore);
+	let warnedNullSourcePut = false; // latched: one warn per table per worker (see _writeUpdate)
 	let sourceLoad: any; // if a source has a load function (replicator), record it here
 	let hasSourceGet: any;
 	let primaryKeyAttribute: Attribute | undefined;
 	let lastEvictionCompletion: Promise<void> = Promise.resolve();
+	// getFromSource() intentionally resolves its caller before the resolved record's cache
+	// write has committed (see there) so GET latency doesn't pay for the write. dropTable()
+	// must not drop this table's column families while one of those writes is still landing —
+	// racing a drop against an in-flight write is what corrupts the column family handle and
+	// produces "Invalid column family specified in write batch" (harper#1381). Track the
+	// in-flight commit promises here so dropTable() can drain them first, and stop admitting
+	// new ones (droppingTable) once a drop has actually started.
+	const pendingSourceCommits = new Set<Promise<any>>();
+	let droppingTable = false;
 	let createdTimeProperty: Attribute | undefined,
 		updatedTimeProperty: Attribute | undefined,
 		expiresAtProperty: Attribute | undefined;
@@ -1375,12 +1385,54 @@ export function makeTable(options) {
 					if (tombstoneWrite?.then) await tombstoneWrite;
 				}
 			}
+			// A get() against a sourcedFrom table resolves to its caller before the resolved
+			// record's cache write has committed (see getFromSource) - the write lands "in the
+			// background" for latency reasons. Flip this BEFORE removing the table from the
+			// schema below: getFromSource() checks it and skips caching (treats the load as
+			// noCacheStore) for any call it admits from here on, including one that slipped in
+			// through a stale reference to this Table between the two steps.
+			droppingTable = true;
 			// Remove the table from the in-memory schema immediately so concurrent
 			// requests get "table does not exist" instead of racing the column
 			// family drops below. If a drop fails past this point the table stays
 			// invisible, and the tombstone guarantees the drop completes on the
 			// next startup (or on a same-name create).
 			delete databases[databaseName][tableName];
+			// The above stops new source-fill writes from starting, but a write from a get()
+			// that already returned to its caller may still be in flight. Dropping the column
+			// families out from under that write is a genuine invariant violation, not just a
+			// benign race: RocksDB rejects the still-open write batch with "Invalid column
+			// family specified in write batch" (or "Could not access column family N"), which
+			// can also abort this drop before it removes the tombstoned catalog rows - leaving
+			// the table stuck "dropping" for completeInterruptedDrop to retry (and fail
+			// identically) on every subsequent load (harper#1381). Drain any in-flight commits
+			// before the blob sweep below (so it observes every row a drain-caught write just
+			// committed) and before touching a single column family.
+			//
+			// Bounded, and fails CLOSED: the tracked promise covers the whole source round-trip
+			// plus the local commit (see getFromSource), so a hung/slow source or a slow commit
+			// (e.g. a large blob write) could otherwise wedge this drop forever. Rather than
+			// give up and drop anyway - which would reopen exactly the race this drain exists to
+			// close, just less often - a timeout FAILS the drop. The tombstone written above is
+			// already durable, so completeInterruptedDrop picks the drop back up on the next
+			// load, once the stuck write has had time to finish.
+			if (pendingSourceCommits.size) {
+				const pending = [...pendingSourceCommits];
+				let timer: NodeJS.Timeout;
+				const timedOut = Symbol('timedOut');
+				const result = await Promise.race([
+					Promise.allSettled(pending),
+					new Promise<typeof timedOut>((resolve) => {
+						timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT);
+					}),
+				]);
+				clearTimeout(timer);
+				if (result === timedOut) {
+					throw new Error(
+						`dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.`
+					);
+				}
+			}
 			for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) {
 				if (entry.metadataFlags & HAS_BLOBS && entry.value) {
 					deleteBlobsInObject(entry.value);
@@ -2185,6 +2237,22 @@ export function makeTable(options) {
 			const context = this.getContext();
 			const transaction = txnForContext(context);
 			checkValidId(id);
+			if (fullUpdate && recordUpdate == null && options?.isNotification) {
+				// A source/replication-applied put must carry the record; these applies skip record
+				// validation, so this is the one path a nullish full update reaches (isNotification scopes
+				// this to the apply dispatcher, not instance flows that fill from #changes). Applying it
+				// stores nothing and mints an audit-only entry misrepresenting the write (#2153) — skip it;
+				// a redelivery re-skips and a later real write supersedes.
+				logger.trace?.('Skipped valueless source put', tableName, id, options?.nodeId);
+				if (!warnedNullSourcePut) {
+					warnedNullSourcePut = true;
+					logger.warn?.(
+						`Skipping a source-applied put with no record content for ${tableName} id ${id} from node ${options?.nodeId}`,
+						new Error('valueless source put')
+					);
+				}
+				return;
+			}
 			const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
 			const writeToSource = () => {
 				if (!(this.constructor as any).source || (context as any)?.source) return;
@@ -3295,6 +3363,11 @@ export function makeTable(options) {
 			const operator = target.operator;
 			if (conditions.length > 0 || operator) conditions = prepareConditions(conditions, operator);
 			const sort = typeof target.sort === 'object' && target.sort;
+			for (let order = sort; order; order = order.next) {
+				if (typeof order.attribute !== 'string') continue;
+				const customIndex = indices[order.attribute]?.customIndex;
+				if (customIndex?.exactDistance) customIndex.exactDistance(order, null);
+			}
 			let postOrdering;
 			if (sort) {
 				// TODO: Support index-assisted sorts of unions, which will require potentially recursively adding/modifying an order aligned condition and be able to recursively undo it if necessary
@@ -3342,9 +3415,12 @@ export function makeTable(options) {
 					if (sort.next) {
 						postOrdering = {
 							dbOrderedAttribute: sort.attribute,
+							dbOrderedSort: sort,
 							attribute: sort.next.attribute,
 							descending: sort.next.descending,
 							next: sort.next.next,
+							target: (sort.next as any).target,
+							distance: (sort.next as any).distance,
 						};
 					}
 				} else {
@@ -3426,7 +3502,8 @@ export function makeTable(options) {
 				ensure_loaded,
 				true,
 				boundRowFilter,
-				includeExpired
+				includeExpired,
+				postOrdering
 			);
 			let results = TableResource.transformToOrderedSelect(
 				entries,
@@ -3487,10 +3564,9 @@ export function makeTable(options) {
 					function createComparator(order: Sort) {
 						const nextComparator = order.next && createComparator(order.next);
 						const descending = order.descending;
-						(context as any).sort = order; // make sure this is set to the current sort order
 						return (entryA, entryB) => {
-							const a = getAttributeValue(entryA, order.attribute, context);
-							const b = getAttributeValue(entryB, order.attribute, context);
+							const a = getAttributeValue(entryA, order.attribute, context, order);
+							const b = getAttributeValue(entryB, order.attribute, context, order);
 							const diff = descending
 								? compareKeys(convertToComparableKeys(b), convertToComparableKeys(a))
 								: compareKeys(convertToComparableKeys(a), convertToComparableKeys(b));
@@ -3531,7 +3607,12 @@ export function makeTable(options) {
 									// if the index has already provided the first order of sorting, we only need to sort
 									// within each grouping
 									if (dbOrderedAttribute) {
-										const groupingValue = getAttributeValue(entry, dbOrderedAttribute, context);
+										const groupingValue = getAttributeValue(
+											entry,
+											dbOrderedAttribute,
+											context,
+											(sort as any).dbOrderedSort
+										);
 										if (firstEntry) {
 											firstEntry = false;
 											lastGroupingValue = groupingValue;
@@ -3630,6 +3711,7 @@ export function makeTable(options) {
 		 * authorization verdict can't be made on bytes that differ from what the caller receives.
 		 * @param includeExpired when true, a row past its TTL but not yet swept is treated as a live
 		 * match rather than gone (used by the SQL engine's UPDATE/DELETE row-finder).
+		 * @param sort post-ordering owned by this selection
 		 * @returns
 		 */
 		static transformEntryForSelect(
@@ -3640,7 +3722,8 @@ export function makeTable(options) {
 			ensure_loaded?,
 			canSkip?,
 			rowFilter?,
-			includeExpired?
+			includeExpired?,
+			sort?
 		) {
 			let checkLoaded;
 			if (
@@ -3716,7 +3799,7 @@ export function makeTable(options) {
 						swrResource.setRecord(record);
 						const loadingFromSource = ensureLoadedFromSource(source, entry.key ?? entry, entry, context, swrResource);
 						if (loadingFromSource?.then) {
-							return loadingFromSource.then(transform);
+							return loadingFromSource.then(transform.bind(this));
 						}
 					}
 				}
@@ -3746,7 +3829,7 @@ export function makeTable(options) {
 									value = filterMap.fromRecord?.(record);
 								}
 							} else {
-								value = resolver(record, context, entry, true);
+								value = resolver(record, context, entry, true, sort);
 							}
 							const handleResolvedValue = (value: any) => {
 								if (resolver.directReturn) return callback(value, attribute_name);
@@ -3768,7 +3851,11 @@ export function makeTable(options) {
 											context,
 											targetReadTxn,
 											filterMap,
-											ensure_loaded
+											ensure_loaded,
+											undefined,
+											undefined,
+											undefined,
+											typeof attribute.sort === 'object' && attribute.sort
 										));
 									if (Array.isArray(value)) {
 										const results = [];
@@ -4707,13 +4794,13 @@ export function makeTable(options) {
 					(stats.treeBranchPageCount + stats.treeLeafPageCount + stats.overflowPages) * stats.pageSize)
 			);
 		}
-		static getStorageStats() {
-			const stats = fs.statfsSync(primaryStore.path);
-			return {
-				available: stats.bavail * stats.bsize,
-				free: stats.bfree * stats.bsize,
-				size: stats.blocks * stats.bsize,
-			};
+		/**
+		 * Get available/free/size storage stats for the table's underlying volume. Async because
+		 * this may need to read quota-status.json (#1976); getSize/getAuditSize stay sync because
+		 * they only read in-memory store stats.
+		 */
+		static async getStorageStats() {
+			return getStorageSpaceStats(primaryStore.path);
 		}
 		static async getRecordCount(options?: any) {
 			// iterate through the metadata entries to exclude their count and exclude the deletion counts
@@ -4820,8 +4907,24 @@ export function makeTable(options) {
 				$updatedTime: (object, context, entry) => entry.version,
 				$expiresAt: (object, context, entry) => entry.expiresAt,
 				$record: (object, context, entry) => (entry ? { value: object } : object),
-				$distance: (object, context, entry) => {
-					return entry && (entry.distance ?? context?.vectorDistances?.get(entry));
+				$distance: (object, context, entry, returnEntry, sort) => {
+					if (!entry) return;
+					if (entry.distance !== undefined) return entry.distance;
+					let distanceSort = sort;
+					while (
+						distanceSort &&
+						(typeof distanceSort.attribute !== 'string' ||
+							!Array.isArray(distanceSort.target) ||
+							!indices[distanceSort.attribute]?.customIndex?.propertyResolver)
+					)
+						distanceSort = distanceSort.next;
+					if (!distanceSort) return;
+					const customIndex = indices[distanceSort.attribute].customIndex;
+					const vector = object[distanceSort.attribute];
+					const distanceCache = context?.vectorDistanceCaches?.get(distanceSort);
+					const cachedDistance = distanceCache?.get(entry) ?? distanceCache?.get(vector);
+					if (cachedDistance !== undefined) return cachedDistance;
+					return customIndex.propertyResolver(vector, context, entry, distanceSort);
 				},
 			};
 			for (const attribute of this.attributes) {
@@ -4969,9 +5072,14 @@ export function makeTable(options) {
 					attribute.resolve.directReturn = true;
 				} else if (indices[attribute.name]?.customIndex?.propertyResolver) {
 					const customIndex = indices[attribute.name].customIndex;
-					propertyResolvers[attribute.name] = (object, context, entry) => {
+					propertyResolvers[attribute.name] = (object, context, entry, returnEntry, sort, comparing) => {
 						const value = object[attribute.name];
-						return customIndex.propertyResolver(value, context, entry);
+						const sortAttribute = sort?.attribute;
+						const resolvesSort =
+							comparing === true &&
+							(sortAttribute === attribute.name ||
+								(Array.isArray(sortAttribute) && sortAttribute[sortAttribute.length - 1] === attribute.name));
+						return customIndex.propertyResolver(value, context, entry, resolvesSort ? sort : undefined);
 					};
 					propertyResolvers[attribute.name].directReturn = true;
 				}
@@ -5506,14 +5614,16 @@ export function makeTable(options) {
 					// Inherit never-drop-on-conflict so a source-applied multi-store transaction doesn't
 					// drop the canonical write when a secondary store hits a transient conflict.
 					transaction.next.sourceApply = transaction.sourceApply;
+					transaction.next.timeoutBudget = transaction.timeoutBudget;
 					// Inherit the replay marker so a multi-table replay transaction skips validation on
 					// every store, not just the first (harper#1316).
 					transaction.next.isReplay = transaction.isReplay;
 					// Inherit which resource/method started this logical operation, so a chained (second
 					// table) transaction's long-transaction-abort log (DatabaseTransaction.ts) can still
-					// identify the request that caused it instead of leaving that field blank. (The
-					// checkOverloaded() stuck-commit log reads this too, but a chained link's own commit
-					// rarely arms outstandingCommit — see the note at DatabaseTransaction.ts's arming site.)
+					// identify the request that caused it instead of leaving that field blank. The
+					// checkOverloaded() stuck-commit log reads this too — every chained link's own native
+					// commit is tracked with its own identity (DatabaseTransaction.ts's trackOutstandingCommit),
+					// so a wedged second-store commit is named just as precisely as a wedged first one.
 					transaction.next.startedFrom = transaction.startedFrom;
 					if (transaction.open === TRANSACTION_STATE.CLOSED) {
 						// if the current transaction is already closed, we need to retain that state on new databases we work with
@@ -5536,7 +5646,7 @@ export function makeTable(options) {
 			return transaction;
 		}
 	}
-	function getAttributeValue(entry, attribute_name, context) {
+	function getAttributeValue(entry, attribute_name, context, sort?) {
 		if (!entry) {
 			return;
 		}
@@ -5548,14 +5658,17 @@ export function makeTable(options) {
 			for (let i = 0, l = attribute_name.length; i < l; i++) {
 				const attribute = attribute_name[i];
 				const resolver = resolvers?.[attribute];
-				value = resolver && value ? resolver(value, context, entry) : value?.[attribute];
+				value =
+					resolver && value
+						? resolver(value, context, entry, false, i === l - 1 ? sort : undefined, true)
+						: value?.[attribute];
 				entry = null; // can't use this in the nested object
 				resolvers = resolver?.definition?.tableClass?.propertyResolvers;
 			}
 			return value;
 		}
 		const resolver = propertyResolvers[attribute_name];
-		return resolver ? resolver(record, context, entry) : record[attribute_name];
+		return resolver ? resolver(record, context, entry, false, sort, true) : record[attribute_name];
 	}
 	function transformToEntries(ids, select, context, readTxn, filters?) {
 		// TODO: Test and ensure that we break out of these loops when a connection is lost
@@ -5700,7 +5813,10 @@ export function makeTable(options) {
 			replacingRecord: existingRecord,
 			replacingEntry: existingEntry,
 			replacingVersion: existingVersion,
-			noCacheStore: false,
+			// Once dropTable() has started, no new source-fill write may begin (dropTable()
+			// only drains writes already in flight - see there); still resolve the caller's
+			// read with fresh source data, just don't cache it into a table that's going away.
+			noCacheStore: droppingTable,
 			source: null,
 			transaction: undefined,
 			expiresAt: undefined,
@@ -5711,264 +5827,287 @@ export function makeTable(options) {
 			// we don't want to wait for the transaction because we want to return as fast as possible
 			// and let the transaction commit in the background
 			let resolved;
-			when(
-				transaction(sourceContext, async (_txn) => {
-					const start = performance.now();
-					let updatedRecord;
-					let hasChanges, invalidated;
-					try {
-						updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
-						invalidated = metadataFlags & INVALIDATED;
-						let version = sourceContext.lastModified || (invalidated && existingVersion);
-						hasChanges = invalidated || version > existingVersion || !existingRecord;
-						const resolveDuration = performance.now() - start;
-						recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success');
-						if (responseHeaders)
-							appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
-						if (expirationMs && sourceContext.expiresAt == undefined)
-							sourceContext.expiresAt = Date.now() + expirationMs;
-						if (updatedRecord) {
-							if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables');
-							if (updatedRecord.status > 0 && updatedRecord.headers) {
-								// if the source has a status code and headers, treat it as a response
-								const status = updatedRecord.status;
-								if (status === 304) {
-									// revalidation of our current cached record
-									updatedRecord = existingRecord;
-									version = existingVersion;
-								} else if (!CACHEABLE_STATUS_CODES.has(status)) {
-									// non-cacheable status - propagate to client without caching
-									throw new ServerError(updatedRecord.body || 'Error from source', status);
-								} else {
-									let headers: any;
-									const sourceHeaders = updatedRecord.headers;
-									if (sourceHeaders[Symbol.iterator]) {
-										headers = {};
-										for (let [name, value] of sourceHeaders) {
-											headers[name.toLowerCase()] = value;
-										}
-									} else {
-										headers = sourceHeaders; // just a plain object
-									}
-									const contentType = sourceHeaders.get?.('Content-Type');
-									let data: any;
-									if (contentType === 'application/json' && updatedRecord.json) {
-										// use native .json() if possible
-										data = await updatedRecord.json();
-									} else {
-										const contentTypeHandler = contentType && contentTypes.get(contentType);
-										if (contentTypeHandler?.deserialize) {
-											data = contentTypeHandler.deserialize(
-												await (contentType.startsWith('text/') ? updatedRecord.text() : updatedRecord.bytes())
-											);
-										}
+			// Tracked in pendingSourceCommits (below) for the full lifetime of this transaction -
+			// including the source round-trip, not just from the point it's actually staged -
+			// so dropTable() can wait for it before dropping the table's column families (see the
+			// comment there). Registering only once a write reaches staging would track less (no
+			// Set churn for a plain cache miss, and a slow/hung source couldn't delay a drop) and
+			// is safe on its own given the live droppingTable re-checks in this function - but it
+			// would make dropTable()'s correctness depend on every future early-return path in
+			// this function remembering to check droppingTable, rather than on dropTable() simply
+			// waiting for whatever this function is doing. Tracking the whole lifetime is the
+			// belt to that suspenders, at the cost of a bounded wait on a merely slow source
+			// before the drain's fail-closed timeout below.
+			const commitPromise = transaction(sourceContext, async (_txn) => {
+				const start = performance.now();
+				let updatedRecord;
+				let hasChanges, invalidated;
+				try {
+					updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
+					invalidated = metadataFlags & INVALIDATED;
+					let version = sourceContext.lastModified || (invalidated && existingVersion);
+					hasChanges = invalidated || version > existingVersion || !existingRecord;
+					const resolveDuration = performance.now() - start;
+					recordAction(resolveDuration, 'cache-resolution', tableName, null, 'success');
+					if (responseHeaders)
+						appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
+					if (expirationMs && sourceContext.expiresAt == undefined) sourceContext.expiresAt = Date.now() + expirationMs;
+					if (updatedRecord) {
+						if (typeof updatedRecord !== 'object') throw new Error('Only objects can be cached and stored in tables');
+						if (updatedRecord.status > 0 && updatedRecord.headers) {
+							// if the source has a status code and headers, treat it as a response
+							const status = updatedRecord.status;
+							if (status === 304) {
+								// revalidation of our current cached record
+								updatedRecord = existingRecord;
+								version = existingVersion;
+							} else if (!CACHEABLE_STATUS_CODES.has(status)) {
+								// non-cacheable status - propagate to client without caching
+								throw new ServerError(updatedRecord.body || 'Error from source', status);
+							} else {
+								let headers: any;
+								const sourceHeaders = updatedRecord.headers;
+								if (sourceHeaders[Symbol.iterator]) {
+									headers = {};
+									for (let [name, value] of sourceHeaders) {
+										headers[name.toLowerCase()] = value;
 									}
-									if (data !== undefined) {
-										// we have structured data that we have parsed
-										delete headers['content-type']; // don't store the content type if we have already parsed it
-										updatedRecord = { headers, data };
-									} else {
-										updatedRecord = { headers, body: createBlob(updatedRecord.body) };
+								} else {
+									headers = sourceHeaders; // just a plain object
+								}
+								const contentType = sourceHeaders.get?.('Content-Type');
+								let data: any;
+								if (contentType === 'application/json' && updatedRecord.json) {
+									// use native .json() if possible
+									data = await updatedRecord.json();
+								} else {
+									const contentTypeHandler = contentType && contentTypes.get(contentType);
+									if (contentTypeHandler?.deserialize) {
+										data = contentTypeHandler.deserialize(
+											await (contentType.startsWith('text/') ? updatedRecord.text() : updatedRecord.bytes())
+										);
 									}
-									if (status !== 200) updatedRecord.status = status;
 								}
+								if (data !== undefined) {
+									// we have structured data that we have parsed
+									delete headers['content-type']; // don't store the content type if we have already parsed it
+									updatedRecord = { headers, data };
+								} else {
+									updatedRecord = { headers, body: createBlob(updatedRecord.body) };
+								}
+								if (status !== 200) updatedRecord.status = status;
 							}
-							if (typeof updatedRecord.toJSON === 'function') updatedRecord = updatedRecord.toJSON();
-							// updatedRecord may still be a frozen record (e.g. a reused existingRecord); copy-on-mutate
-							// before stamping the primary key and created/updated times below (records are immutable —
-							// 5.2 record caching relies on it — so we must not write through the frozen object).
-							if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord };
-							if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
 						}
-						resolved = true;
-						const resolvedEntry: Entry = {
-							key: id,
-							version,
-							value: updatedRecord,
-							expiresAt: sourceContext.expiresAt,
-							metadataFlags: 0,
-							size: 0,
-							localTime: 0,
-							nodeId: 0,
-							residencyId: 0,
-						} as any;
-						// Give the plain object the RecordObject prototype so getExpiresAt/getUpdatedTime
-						// are available on the immediately-resolved entry. We mutate the prototype
-						// in-place rather than copying so that the commit callback (which adds
-						// createdAt/updatedAt to updatedRecord) is still reflected in the entry value.
-						if (updatedRecord && updatedRecord.constructor === Object) {
-							Object.setPrototypeOf(updatedRecord, primaryStore.encoder.structPrototype);
-							entryMap.set(updatedRecord, resolvedEntry);
-						}
-						resolve(resolvedEntry);
-					} catch (error) {
-						error.message += ` while resolving record ${id} for ${tableName}`;
-						if (
-							existingRecord &&
-							(((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
-								!context?.mustRevalidate) ||
-								(context?.staleIfError &&
-									(error.statusCode === 500 ||
-										error.statusCode === 502 ||
-										error.statusCode === 503 ||
-										error.statusCode === 504)))
-						) {
-							// these are conditions under which we can use stale data after an error
-							resolve({
-								key: id,
-								version: existingVersion,
-								value: existingRecord,
-							} as any);
-							logger.trace?.(error.message, '(returned stale record)');
-						} else reject(error);
-						const resolveDuration = performance.now() - start;
-						recordAction(resolveDuration, 'cache-resolution', tableName, null, 'fail');
-						if (responseHeaders)
-							appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
-						sourceContext.transaction.abort();
-						return;
-					}
-					if (context?.noCacheStore || sourceContext.noCacheStore) {
-						// abort before we write any change
-						sourceContext.transaction.abort();
-						return;
+						if (typeof updatedRecord.toJSON === 'function') updatedRecord = updatedRecord.toJSON();
+						// updatedRecord may still be a frozen record (e.g. a reused existingRecord); copy-on-mutate
+						// before stamping the primary key and created/updated times below (records are immutable —
+						// 5.2 record caching relies on it — so we must not write through the frozen object).
+						if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord };
+						if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
 					}
-					const dbTxn = txnForContext(sourceContext);
-					const sourceWrite: any = {
+					resolved = true;
+					const resolvedEntry: Entry = {
 						key: id,
-						store: primaryStore,
-						entry: existingEntry,
-						nodeName: 'source',
-						commit: (txnTime, existingEntry, _retry, transaction: any) => {
-							sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
-							if (existingEntry?.version !== existingVersion) {
-								// don't do anything if the version has changed
-								sourceWrite.skipped = true;
-								return;
+						version,
+						value: updatedRecord,
+						expiresAt: sourceContext.expiresAt,
+						metadataFlags: 0,
+						size: 0,
+						localTime: 0,
+						nodeId: 0,
+						residencyId: 0,
+					} as any;
+					// Give the plain object the RecordObject prototype so getExpiresAt/getUpdatedTime
+					// are available on the immediately-resolved entry. We mutate the prototype
+					// in-place rather than copying so that the commit callback (which adds
+					// createdAt/updatedAt to updatedRecord) is still reflected in the entry value.
+					if (updatedRecord && updatedRecord.constructor === Object) {
+						Object.setPrototypeOf(updatedRecord, primaryStore.encoder.structPrototype);
+						entryMap.set(updatedRecord, resolvedEntry);
+					}
+					resolve(resolvedEntry);
+				} catch (error) {
+					error.message += ` while resolving record ${id} for ${tableName}`;
+					if (
+						existingRecord &&
+						(((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
+							!context?.mustRevalidate) ||
+							(context?.staleIfError &&
+								(error.statusCode === 500 ||
+									error.statusCode === 502 ||
+									error.statusCode === 503 ||
+									error.statusCode === 504)))
+					) {
+						// these are conditions under which we can use stale data after an error
+						resolve({
+							key: id,
+							version: existingVersion,
+							value: existingRecord,
+						} as any);
+						logger.trace?.(error.message, '(returned stale record)');
+					} else reject(error);
+					const resolveDuration = performance.now() - start;
+					recordAction(resolveDuration, 'cache-resolution', tableName, null, 'fail');
+					if (responseHeaders)
+						appendHeader(responseHeaders, 'Server-Timing', `cache-resolve;dur=${resolveDuration.toFixed(2)}`, true);
+					sourceContext.transaction.abort();
+					return;
+				}
+				if (context?.noCacheStore || sourceContext.noCacheStore || droppingTable) {
+					// abort before we write any change. droppingTable is re-checked live (not just
+					// the noCacheStore snapshot taken at call start) because a call admitted before
+					// dropTable() began can still be sitting here after it started - the await above
+					// waited on the source, which may take arbitrarily long.
+					sourceContext.transaction.abort();
+					return;
+				}
+				const dbTxn = txnForContext(sourceContext);
+				const sourceWrite: any = {
+					key: id,
+					store: primaryStore,
+					entry: existingEntry,
+					nodeName: 'source',
+					commit: (txnTime, existingEntry, _retry, transaction: any) => {
+						sourceWrite.skipped = false; // reset on each retry; cleanup happens after commit if still true
+						if (existingEntry?.version !== existingVersion) {
+							// don't do anything if the version has changed
+							sourceWrite.skipped = true;
+							return;
+						}
+						updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
+						if (updatedRecord) {
+							if (existingEntry) {
+								context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
 							}
-							updateIndices(id, existingRecord, updatedRecord, transaction && { transaction });
-							if (updatedRecord) {
-								if (existingEntry) {
-									context.previousResidency = TableResource.getResidencyRecord(existingEntry.residencyId);
-								}
-								let auditRecord: any;
-								let omitLocalRecord = false;
-								let residencyId: number;
-								if (updatedTimeProperty) {
-									updatedRecord[updatedTimeProperty.name] =
-										updatedTimeProperty.type === 'Date'
+							let auditRecord: any;
+							let omitLocalRecord = false;
+							let residencyId: number;
+							if (updatedTimeProperty) {
+								updatedRecord[updatedTimeProperty.name] =
+									updatedTimeProperty.type === 'Date'
+										? new Date(txnTime)
+										: updatedTimeProperty.type === 'String'
+											? new Date(txnTime).toISOString()
+											: txnTime;
+							}
+							if (createdTimeProperty && updatedRecord[createdTimeProperty.name] == null) {
+								const existingCreatedTime = existingEntry?.value?.[createdTimeProperty.name];
+								if (existingCreatedTime != null) {
+									updatedRecord[createdTimeProperty.name] = existingCreatedTime;
+								} else {
+									updatedRecord[createdTimeProperty.name] =
+										createdTimeProperty.type === 'Date'
 											? new Date(txnTime)
-											: updatedTimeProperty.type === 'String'
+											: createdTimeProperty.type === 'String'
 												? new Date(txnTime).toISOString()
 												: txnTime;
 								}
-								if (createdTimeProperty && updatedRecord[createdTimeProperty.name] == null) {
-									const existingCreatedTime = existingEntry?.value?.[createdTimeProperty.name];
-									if (existingCreatedTime != null) {
-										updatedRecord[createdTimeProperty.name] = existingCreatedTime;
+							}
+							const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
+							if (residency) {
+								if (!residency.includes(server.hostname)) {
+									// if we aren't in the residency list, specify that our local record should be omitted or be partial
+									auditRecord = updatedRecord;
+									omitLocalRecord = true;
+									if (TableResource.getResidencyById) {
+										// complete omission of the record that doesn't belong here
+										updatedRecord = undefined;
 									} else {
-										updatedRecord[createdTimeProperty.name] =
-											createdTimeProperty.type === 'Date'
-												? new Date(txnTime)
-												: createdTimeProperty.type === 'String'
-													? new Date(txnTime).toISOString()
-													: txnTime;
-									}
-								}
-								const residency = residencyFromFunction(TableResource.getResidency(updatedRecord, context));
-								if (residency) {
-									if (!residency.includes(server.hostname)) {
-										// if we aren't in the residency list, specify that our local record should be omitted or be partial
-										auditRecord = updatedRecord;
-										omitLocalRecord = true;
-										if (TableResource.getResidencyById) {
-											// complete omission of the record that doesn't belong here
-											updatedRecord = undefined;
-										} else {
-											// store the partial record
-											updatedRecord = null;
-											for (const name in indices) {
-												if (!updatedRecord) {
-													updatedRecord = {};
-												}
-												// if there are any indices, we need to preserve a partial invalidated record to ensure we can still do searches
-												updatedRecord[name] = auditRecord[name];
-											}
-											if (createdTimeProperty && auditRecord[createdTimeProperty.name] != null) {
-												// preserve the created timestamp in the partial record so it isn't lost when we don't have residency
-												if (!updatedRecord) updatedRecord = {};
-												updatedRecord[createdTimeProperty.name] = auditRecord[createdTimeProperty.name];
+										// store the partial record
+										updatedRecord = null;
+										for (const name in indices) {
+											if (!updatedRecord) {
+												updatedRecord = {};
 											}
+											// if there are any indices, we need to preserve a partial invalidated record to ensure we can still do searches
+											updatedRecord[name] = auditRecord[name];
+										}
+										if (createdTimeProperty && auditRecord[createdTimeProperty.name] != null) {
+											// preserve the created timestamp in the partial record so it isn't lost when we don't have residency
+											if (!updatedRecord) updatedRecord = {};
+											updatedRecord[createdTimeProperty.name] = auditRecord[createdTimeProperty.name];
 										}
 									}
-									residencyId = getResidencyId(residency);
 								}
-								logger.trace?.(
-									`Writing resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
-								);
-								// TODO: We are doing a double check for ifVersion that should probably be cleaned out
+								residencyId = getResidencyId(residency);
+							}
+							logger.trace?.(
+								`Writing resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
+							);
+							// TODO: We are doing a double check for ifVersion that should probably be cleaned out
+							updateRecord(
+								id,
+								updatedRecord,
+								existingEntry,
+								txnTime,
+								omitLocalRecord ? INVALIDATED : 0,
+								(audit && (hasChanges || omitLocalRecord)) || null,
+								{
+									user: (sourceContext as any)?.user,
+									expiresAt: sourceContext.expiresAt,
+									residencyId,
+									transaction,
+									tableToTrack: tableName,
+								},
+								'put',
+								Boolean(invalidated),
+								auditRecord
+							);
+							// arm the eviction scanner, mirroring the .put() path
+							if (sourceContext.expiresAt) scheduleCleanup();
+						} else if (existingEntry) {
+							logger.trace?.(
+								`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
+							);
+							if (audit || trackDeletes) {
 								updateRecord(
 									id,
-									updatedRecord,
+									null,
 									existingEntry,
 									txnTime,
-									omitLocalRecord ? INVALIDATED : 0,
-									(audit && (hasChanges || omitLocalRecord)) || null,
-									{
-										user: (sourceContext as any)?.user,
-										expiresAt: sourceContext.expiresAt,
-										residencyId,
-										transaction,
-										tableToTrack: tableName,
-									},
-									'put',
-									Boolean(invalidated),
-									auditRecord
-								);
-								// arm the eviction scanner, mirroring the .put() path
-								if (sourceContext.expiresAt) scheduleCleanup();
-							} else if (existingEntry) {
-								logger.trace?.(
-									`Deleting resolved record from source with id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`
+									0,
+									(audit && hasChanges) || null,
+									{ user: (sourceContext as any)?.user, transaction, tableToTrack: tableName },
+									'delete',
+									Boolean(invalidated)
 								);
-								if (audit || trackDeletes) {
-									updateRecord(
-										id,
-										null,
-										existingEntry,
-										txnTime,
-										0,
-										(audit && hasChanges) || null,
-										{ user: (sourceContext as any)?.user, transaction, tableToTrack: tableName },
-										'delete',
-										Boolean(invalidated)
-									);
-								} else {
-									removeEntry(primaryStore, existingEntry, existingVersion);
-								}
+							} else {
+								removeEntry(primaryStore, existingEntry, existingVersion);
 							}
-						},
-					};
-					// The cache-from-source write bypasses `_writeUpdate`, so wire the embed hook here
-					// too (always the originating node). It runs after the client GET has resolved with
-					// fresh source data, so it's a background commit: an embedder failure aborts the cache
-					// write via the outer error handler (row re-embeds next read) and never reaches the
-					// caller. Source-resolution errors are handled earlier, with the stale-data fallback.
-					const embedBefore = buildEmbedBefore(
-						updatedRecord,
-						sourceContext,
-						undefined,
-						TableResource.embedAttributes,
-						TableResource.userEmbedders
-					);
-					if (embedBefore) await embedBefore();
-					sourceWrite.before = preCommitBlobsForRecordBefore(sourceWrite, updatedRecord);
-					dbTxn.addWrite(sourceWrite);
-				}),
+						}
+					},
+				};
+				// The cache-from-source write bypasses `_writeUpdate`, so wire the embed hook here
+				// too (always the originating node). It runs after the client GET has resolved with
+				// fresh source data, so it's a background commit: an embedder failure aborts the cache
+				// write via the outer error handler (row re-embeds next read) and never reaches the
+				// caller. Source-resolution errors are handled earlier, with the stale-data fallback.
+				const embedBefore = buildEmbedBefore(
+					updatedRecord,
+					sourceContext,
+					undefined,
+					TableResource.embedAttributes,
+					TableResource.userEmbedders
+				);
+				if (embedBefore) await embedBefore();
+				if (droppingTable) {
+					// Re-check right before staging the write: dropTable() may have started
+					// while we were awaiting the embed step above (harper#1381).
+					sourceContext.transaction.abort();
+					return;
+				}
+				sourceWrite.before = preCommitBlobsForRecordBefore(sourceWrite, updatedRecord);
+				dbTxn.addWrite(sourceWrite);
+			});
+			pendingSourceCommits.add(commitPromise);
+			when(
+				commitPromise,
 				() => {
+					pendingSourceCommits.delete(commitPromise);
 					primaryStore.unlock(id);
 				},
 				(error) => {
+					pendingSourceCommits.delete(commitPromise);
 					primaryStore.unlock(id);
 					if (resolved) logger.error?.('Error committing cache update', error);
 					// else the error was already propagated as part of the promise that we returned