@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/resources/LMDBTransaction.ts

@@ -1,11 +1,15 @@
 import {
 	DatabaseTransaction,
+	closeWriteInstance,
+	validateWrite,
+	shouldSpareCommitPhase,
 	transactionOpenTooLongError,
 	type CommitOptions,
 	type TransactionWrite,
 	type CommitResolution,
 } from './DatabaseTransaction';
 import { cleanupUnusedBlobs, collectRetainedFileIds } from './blob.ts';
+import { ServerError } from '../utility/errors/hdbError.ts';
 import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
 import * as harperLogger from '../utility/logging/harper_logger.ts';
 import type { Context } from './ResourceInterface.ts';
@@ -35,10 +39,14 @@ export class LMDBTransaction extends DatabaseTransaction {
 	declare overloadChecked: boolean;
 	open = TRANSACTION_STATE.OPEN;
 
+	renewIdleTimeout(): void {
+		this.timeout = Math.max(txnExpiration, this.timeoutBudget ?? 0);
+	}
+
 	getReadTxn(): any {
 		// used optimistically
 		this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
-		this.timeout = txnExpiration; // reset the timeout
+		this.renewIdleTimeout();
 		if (this.stale) this.stale = false;
 		if (this.readTxn) {
 			if ((this.readTxn as any).openTimer) (this.readTxn as any).openTimer = 0;
@@ -111,12 +119,14 @@ export class LMDBTransaction extends DatabaseTransaction {
 
 		this.linkWrite(operation);
 		this.writes.push(operation); // standard path, add to current transaction
+		(this.ownedWrites ??= new WeakSet()).add(operation);
 		operation.stagedIn = this;
 	}
 
 	removeWrite(operation: TransactionWrite) {
 		const index = this.writes.indexOf(operation);
 		if (index > -1) this.writes[index] = null;
+		this.ownedWrites?.delete(operation);
 	}
 
 	/**
@@ -138,7 +148,11 @@ export class LMDBTransaction extends DatabaseTransaction {
 				this.validated = this.writes.length;
 				for (let i = start; i < this.validated; i++) {
 					const write = this.writes[i];
-					write?.validate?.(this.timestamp, this);
+					try {
+						if (write) validateWrite(write, this.timestamp, this);
+					} finally {
+						closeWriteInstance(write);
+					}
 				}
 				let hasBefore;
 				for (let i = start; i < this.validated; i++) {
@@ -153,6 +167,9 @@ export class LMDBTransaction extends DatabaseTransaction {
 				// source writes will finish (with right to refuse/abort) before proceeeding to less
 				// canonical sources.
 				if (hasBefore) {
+					// see `committing` in DatabaseTransaction (issue #2062)
+					this.setCommitPhase(true);
+					const stagedWrites = this.writes.length;
 					return (async () => {
 						try {
 							for (let phase = 0; phase < 2; phase++) {
@@ -172,9 +189,15 @@ export class LMDBTransaction extends DatabaseTransaction {
 								if (completion) await (completion.push ? Promise.all(completion) : completion);
 							}
 						} catch (error) {
+							this.setCommitPhase(false);
 							this.abort();
 							throw error;
 						}
+						this.setCommitPhase(false);
+						// aborted underneath us while parked above — see DatabaseTransaction's twin guard
+						if (this.timedOut) throw transactionOpenTooLongError();
+						if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
+							throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
 						return this.commit(options);
 					})();
 				}
@@ -183,6 +206,7 @@ export class LMDBTransaction extends DatabaseTransaction {
 				throw error;
 			}
 		}
+		if (!retries && this.writes.some((write) => write)) this.renewChainForNativeCommit();
 		// release the read snapshot so we don't keep it open longer than necessary
 		if (!retries) this.doneReadTxn();
 		this.open = options?.doneWriting ? TRANSACTION_STATE.LINGERING : TRANSACTION_STATE.OPEN;
@@ -192,7 +216,12 @@ export class LMDBTransaction extends DatabaseTransaction {
 		let writeIndex = 0;
 		this.writes = this.writes.filter((write) => write); // filter out removed entries
 		const doWrite = (write) => {
-			const completion = write.commit(txnTime, write.entry, retries);
+			let completion;
+			try {
+				completion = write.commit(txnTime, write.entry, retries);
+			} finally {
+				closeWriteInstance(write);
+			}
 			if (typeof completion?.then === 'function') {
 				// the aggregating Promise.all is attached a turn or more later (after the conditional batch
 				// or the exclusive transaction resolves), so handle rejection here to keep the gap from
@@ -372,10 +401,21 @@ let timer;
 
 function startMonitoringTxns() {
 	timer = setInterval(function () {
+		const checkedCommitPhaseChains = new Set<DatabaseTransaction>();
 		for (const txn of trackedTxns) {
+			const commitChainHead = txn.commitChainHead ?? txn;
 			if (txn.timeout <= 0) {
 				const url = (txn.getContext() as any)?.url;
-				if (txn.hasPendingWrites() && !txn.sourceApply && !txn.isReplay) {
+				if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
+					// see DatabaseTransaction's monitor (issue #2062)
+					harperLogger.warn?.(
+						`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work (e.g. a large blob write); letting it complete, from table: ${
+							(txn.db as any)?.name + (url ? ' path: ' + url : '')
+						}`,
+						...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : [])
+					);
+					txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
+				} else if (txn.hasPendingWrites() && !txn.sourceApply && !txn.isReplay) {
 					// Abort and surface an error rather than force-committing a partial write set: silently
 					// committing on the application's behalf breaks atomicity and can leave orphaned
 					// secondary-index entries that only a full index rebuild repairs (issue #1407). The app
@@ -390,7 +430,7 @@ function startMonitoringTxns() {
 						}`
 					);
 					try {
-						txn.abortDueToTimeout();
+						commitChainHead.abortDueToTimeout();
 					} catch (error) {
 						harperLogger.debug?.(`Error aborting timed out transaction: ${error.message}`);
 					}
@@ -408,7 +448,7 @@ function startMonitoringTxns() {
 					} catch (error) {
 						harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
 					}
-					txn.timeout = txnExpiration;
+					txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
 				}
 			} else {
 				txn.timeout -= txnExpiration;

package/resources/PrimaryRocksDatabase.ts

@@ -1,10 +1,10 @@
 import { trackReadRange } from './DatabaseTransaction.ts';
-import { RocksDatabase, type RocksDatabaseOptions, constants, type Store } from '@harperfast/rocksdb-js';
+import { RocksDatabase, type RocksDatabaseOptions, constants, type Store, Transaction } from '@harperfast/rocksdb-js';
 
 const FRESH_VERSION_FLAG = constants.FRESH_VERSION_FLAG;
 import { WeakLRUCache } from 'weak-lru-cache';
 import { when } from '../utility/when.ts';
-import { assignStoredFields, entryMap, METADATA, type Entry } from './RecordEncoder.ts';
+import { assignStoredFields, entryMap, METADATA, VERSION_REUSED, type Entry } from './RecordEncoder.ts';
 
 /**
  * RocksDatabase subclass that owns all primary-store behaviour for Harper tables:
@@ -42,6 +42,33 @@ export class PrimaryRocksDatabase extends RocksDatabase {
 		}
 	}
 
+	// Match LMDB's remove(key, version) contract without splitting the version check from the delete.
+	async removeIfVersion(id: any, version: number): Promise<boolean> {
+		let retried = false;
+		while (true) {
+			let transaction: Transaction | undefined;
+			try {
+				transaction = new Transaction(this.store);
+				const entry = this.#processEntry(super.getSync(id, { transaction }), id);
+				if (!entry || entry.version !== version) {
+					transaction.abort();
+					return false;
+				}
+				this.#cache?.delete(id);
+				super.removeSync(id, { transaction });
+				await transaction.commit();
+				return true;
+			} catch (error: any) {
+				try {
+					transaction?.abort();
+				} catch {}
+				// rocksdb-js reports a writer that committed between our read and our commit as ERR_BUSY
+				if (retried || error?.code !== 'ERR_BUSY') throw error;
+				retried = true;
+			}
+		}
+	}
+
 	/**
 	 * Initialises encoder/decoder state. Must be called once after open() with the root
 	 * RocksDatabase. Equivalent to the RocksDB branch of handleLocalTimeForGets, but as
@@ -95,6 +122,13 @@ export class PrimaryRocksDatabase extends RocksDatabase {
 	 */
 	getEntry(id: any, options?: any): any {
 		this.readCount++;
+		// Commit-path base reads must reflect the caller's transaction snapshot; the cache vouch
+		// answers "latest committed" — the wrong question there, and wrong outright for a version a
+		// resequenced write reused — so read the store directly, touching neither cache nor VT.
+		if (options?.uncachedRead) {
+			if (options.async) return when(super.get(id, options), (result) => this.#processEntry(result, id));
+			return this.#processEntry(super.getSync(id, options), id);
+		}
 		const cache = this.#cache;
 		// The cache stores the record *value* (weakly, via setValue) rather than
 		// the Entry: a WeakRef-wrapped value lets the LRFU expirer release it once
@@ -132,11 +166,16 @@ export class PrimaryRocksDatabase extends RocksDatabase {
 				if (cache && cachedValue !== undefined) cache.delete(id);
 				return undefined;
 			}
-			// Only object values can be weakly cached and mapped back to their Entry;
-			// primitive/empty values fall through uncached (no fast path, still correct).
-			if (entry.version != null && cache && entry.value != null && typeof entry.value === 'object') {
-				entryMap.set(entry.value, entry);
-				cache.setValue(id, entry.value, (entry.size ?? 0) >> 10);
+			if (entry.version != null && cache) {
+				// its version no longer identifies its value, so drop any copy already held
+				if (entry.metadataFlags & VERSION_REUSED) {
+					if (cachedValue !== undefined) cache.delete(id);
+				} else if (entry.value != null && typeof entry.value === 'object') {
+					// Only object values can be weakly cached and mapped back to their Entry;
+					// primitive/empty values fall through uncached (no fast path, still correct).
+					entryMap.set(entry.value, entry);
+					cache.setValue(id, entry.value, (entry.size ?? 0) >> 10);
+				}
 			}
 			return entry;
 		});

package/resources/RecordEncoder.ts

@@ -30,7 +30,7 @@ import {
 } from './blob.ts';
 import { getThisNodeId } from './nodeIdMapping.ts';
 import { recordAction } from './analytics/write.ts';
-import { RocksDatabase } from '@harperfast/rocksdb-js';
+import { constants, RocksDatabase } from '@harperfast/rocksdb-js';
 import { when } from '../utility/when.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 import * as envMngr from '../utility/environment/environmentManager.js';
@@ -42,6 +42,33 @@ const StructonEncoder = createStructon(Encoder) as typeof Encoder;
 // monitoring; the store name is passed as the metric path.
 const MISSING_STRUCTURE_METRIC = 'decode-missing-structure';
 
+// Reaching this bound is not an error: novel shapes fall back to plain msgpackr encoding, which
+// round-trips correctly but gives up random-access field reads.
+export const DEFAULT_MAX_TYPED_STRUCTURES = 256;
+
+export interface StructureCounts {
+	typed: number;
+	classic: number;
+	typedLimit: number;
+	typedEnabled: boolean;
+}
+
+// Mirrors structon's onLoadedStructures over the four durable payload shapes it accepts. The
+// bare-array and cbor-x forms carry named structures only; structon keeps the encoder's own typed
+// dictionary for them, and they are written by the capped fallback path (msgpackr's base save
+// overwriting the combined payload, before structon's best-effort re-save). That window is by
+// definition a saturated store, so reporting zero there would show an empty dictionary at the one
+// moment the count matters most.
+function countDurableStructures(sharedData: any, localTyped: number): { typed: number; classic: number } {
+	if (!sharedData) return { typed: 0, classic: 0 };
+	if (sharedData instanceof Map)
+		return { typed: sharedData.get('typed')?.length ?? 0, classic: sharedData.get('named')?.length ?? 0 };
+	if (Array.isArray(sharedData)) return { typed: localTyped, classic: sharedData.length };
+	if (!sharedData.named && !sharedData.typed && Array.isArray(sharedData.structures))
+		return { typed: localTyped, classic: sharedData.structures.length };
+	return { typed: sharedData.typed?.length ?? 0, classic: sharedData.named?.length ?? 0 };
+}
+
 // Terminal error messages msgpackr/structon throw when a record references a shared structure that
 // is not in this node's structures buffer. Both the typed (random-access) path (structon's
 // readStruct) and the classic path (msgpackr's createSecondByteReader) reload the structures from
@@ -77,6 +104,8 @@ export type Entry = {
 export const TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 4, 0x40, 0, 0]);
 // the first byte here indicates that we use the last timestamp
 export const LAST_TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 1, 0, 0, 0]);
+// Never write this into an audit entry value: the substitution resolves to 2.0 when no previous time
+// was recorded, and the audit format reads a leading byte other than 0x42 as "no field here". See harper#2247.
 export const PREVIOUS_TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 3, 0x40, 0, 0]);
 export const NEW_TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 0, 0x40, 0, 0]);
 export const LOCAL_TIMESTAMP = Symbol('local-timestamp');
@@ -95,6 +124,26 @@ export const HAS_NODE_ID = 64;
 export const PENDING_LOCAL_TIME = 1;
 export const HAS_STRUCTURE_UPDATE = 0x100;
 export const HAS_ADDITIONAL_AUDIT_REFS = 0x80;
+// A resequenced write keeps the (newer) version it merged onto, so one version identifies two
+// different stored values and version equality proves nothing. The metadata word this is set in is
+// the same header word the VerificationTable reads at value offset 8 — ACTION_32_BIT is the tag
+// byte its predicate requires — so the bit is what stops the native layer vouching for the version.
+export const VERSION_REUSED = constants.VERSION_NOT_UNIQUE_FLAG;
+// The bit is persisted in every resequenced record, so a rocksdb-js that moved it down onto one of
+// the flags above (or up into the tag byte) would silently change what records already on disk
+// mean: it must stay a single bit strictly between them.
+if (
+	typeof VERSION_REUSED !== 'number' ||
+	(VERSION_REUSED & (VERSION_REUSED - 1)) !== 0 ||
+	VERSION_REUSED < 0x10000 ||
+	VERSION_REUSED > 0x800000
+)
+	throw new Error(
+		`rocksdb-js VERSION_NOT_UNIQUE_FLAG (${VERSION_REUSED}) is not a single bit in the range Harper record metadata reserves for it — requires @harperfast/rocksdb-js >= 2.8.0`
+	);
+function versionIsReused(newVersion: number, existingEntry: { version?: number } | undefined): boolean {
+	return existingEntry?.version != null && newVersion <= existingEntry.version;
+}
 
 const TRACKED_WRITE_TYPES = new Set(['put', 'patch', 'delete', 'message', 'publish']);
 // For now we use this as the private property mechanism for mapping records to entries.
@@ -197,6 +246,10 @@ export class RecordEncoder extends StructonEncoder {
 	declare saveStructures: any;
 	declare getStructures: any;
 	declare _writeStruct: any;
+	declare typedStructs: any[];
+	declare structures: any[];
+	declare maxOwnStructures: number;
+	randomAccessStructure = false;
 	structureUpdate?: any;
 	// Set by TableResource.updatedAttributes. Undefined for a store with no resolved attributes, which
 	// is what keeps the projection off that store's writes entirely.
@@ -217,7 +270,7 @@ export class RecordEncoder extends StructonEncoder {
 		// Bound the per-encoder typed-structure dictionary. It is append-only and pinned on the
 		// long-lived primary store, so a wide/sparse schema (whose records vary by per-field value
 		// width) can grow it unbounded and exhaust memory. Caller-overridable; default caps it.
-		options.maxOwnStructures ??= 256;
+		options.maxOwnStructures ??= DEFAULT_MAX_TYPED_STRUCTURES;
 		/**
 		 * The base class for records that provides the read-only methods for accessing
 		 * metadata and will be assigned computed property getters. On its own, these instances
@@ -233,6 +286,7 @@ export class RecordEncoder extends StructonEncoder {
 		// (e.g. __dbis__, useVersions=false) must not — see the encode hook + harper#1307. Default to
 		// true when unspecified so an option that doesn't propagate can't silently strip prefixes.
 		this.useVersions = options.useVersions !== false;
+		this.randomAccessStructure = options.randomAccessStructure === true;
 		// structon (the StructonEncoder base) always installs the struct write hook. For DBIs
 		// that don't opt into struct mode (non-primary, e.g. __dbis__), force it to bail (return
 		// 0) so objects are written in plain msgpackr records mode — decodable by readers without
@@ -417,12 +471,14 @@ export class RecordEncoder extends StructonEncoder {
 				// HAS_STRUCTURE_UPDATE in the audit log for a structure that was never persisted.
 				if (committed === true) {
 					this.structureUpdate = structures;
+					this.checkStructureCapacity();
 					return true;
 				}
 				return false;
 			} else {
 				const result = superSaveStructures.call(this, structures, isCompatible);
 				this.structureUpdate = structures;
+				if (result !== false) this.checkStructureCapacity();
 				return result;
 			}
 		};
@@ -432,10 +488,51 @@ export class RecordEncoder extends StructonEncoder {
 				const buffer = this.rootStore.getBinarySync(sharedStructuresKey);
 				return buffer ? this.decode(buffer) : undefined;
 			} else {
-				return superGetStructures.call(this);
+				// msgpackr only defines getStructures when the option was supplied; a store with no
+				// shared-structures mechanism has no durable dictionary to report.
+				return superGetStructures?.call(this);
 			}
 		};
 	}
+	/**
+	 * Sizes of this store's record-structure dictionaries. Read from the durable payload, not this
+	 * encoder's arrays: each worker owns an encoder and loads lazily, so a worker that never encoded
+	 * for this store holds empty arrays for a store whose dictionary is full.
+	 */
+	getStructureCounts(): StructureCounts {
+		const { typed, classic } = countDurableStructures(this.getStructures(), this.typedStructs?.length ?? 0);
+		return {
+			typed,
+			classic,
+			typedLimit: this.maxOwnStructures ?? DEFAULT_MAX_TYPED_STRUCTURES,
+			typedEnabled: this.randomAccessStructure,
+		};
+	}
+	/**
+	 * Warn once per encoder when its typed dictionary is at the bound. Fires from the post-save path,
+	 * so it needs some structure save to happen -- any save, including a classic one -- and it needs
+	 * this encoder to have loaded the dictionary. Several workers can therefore each warn for one
+	 * table, and a worker that never saves after loading a full dictionary stays silent. The counts
+	 * above are the reliable signal; this is the heads-up. It must not escape into the save's return
+	 * value -- the structures are committed.
+	 */
+	#reportedStructureSaturation = false;
+	checkStructureCapacity() {
+		if (this.#reportedStructureSaturation) return;
+		const typedLimit = this.maxOwnStructures ?? DEFAULT_MAX_TYPED_STRUCTURES;
+		if ((this.typedStructs?.length ?? 0) < typedLimit) return;
+		this.#reportedStructureSaturation = true;
+		try {
+			harperLogger.warn(
+				`Typed-structure dictionary for store ${this.name ?? this.rootStore?.name ?? '(unnamed)'} reached its limit of ` +
+					`${typedLimit} structures; new record shapes will be stored without random-access field ` +
+					'encoding. Dictionary size follows the variety of shapes written -- field set, key order, ' +
+					'and per-field value width -- not column count (see HarperFast/harper#2220).'
+			);
+		} catch {
+			// the structures are already committed; a failing sink must not fail the write
+		}
+	}
 	decode(buffer, options) {
 		lastMetadata = null;
 		const start = options?.start || 0;
@@ -809,6 +906,10 @@ export function recordUpdater(store, tableId, auditStore) {
 		if (expiresAt >= 0) assignMetadata |= HAS_EXPIRATION;
 		metadataInNextEncoding = assignMetadata;
 		expiresAtNextEncoding = expiresAt;
+		// Math.max normalizes the -1 "no metadata word" sentinel to 0 first: OR-ing the flag into
+		// -1 directly would stay -1 and silently drop the metadata word (and the flag with it).
+		if (isRocksDB && record !== undefined && versionIsReused(newVersion, existingEntry))
+			metadataInNextEncoding = Math.max(metadataInNextEncoding, 0) | VERSION_REUSED;
 		const putOptions: {
 			version: number;
 			instructedWrite?: boolean;
@@ -829,9 +930,9 @@ export function recordUpdater(store, tableId, auditStore) {
 				metadataInNextEncoding |= HAS_RESIDENCY_ID;
 				extendedType |= HAS_CURRENT_RESIDENCY_ID;
 			} else residencyIdAtNextEncoding = 0;
-			const nodeId = options?.nodeId ?? (audit ? getThisNodeId(auditStore) : undefined);
-			if (nodeId >= 0) {
-				nodeIdAtNextEncoding = nodeId;
+			const recordNodeId = options?.recordNodeId ?? options?.nodeId ?? (audit ? getThisNodeId(auditStore) : undefined);
+			if (recordNodeId >= 0) {
+				nodeIdAtNextEncoding = recordNodeId;
 				metadataInNextEncoding |= HAS_NODE_ID;
 			} else nodeIdAtNextEncoding = -1;
 			const additionalAuditRefs = options?.additionalAuditRefs;
@@ -904,8 +1005,22 @@ export function recordUpdater(store, tableId, auditStore) {
 				const nodeId = options?.nodeId ?? getThisNodeId(auditStore) ?? 0;
 				const viaNodeId = options?.viaNodeId ?? nodeId;
 				if (resolveRecord && existingEntry?.localTime) {
-					const replacingId = existingEntry?.localTime;
-					const replacingEntry = auditStore.get(replacingId, tableId, id);
+					let replacingId = existingEntry.localTime;
+					let replacingEntry;
+					if (isRocksDB && existingEntry.additionalAuditRefs) {
+						for (const ref of existingEntry.additionalAuditRefs) {
+							const candidate = auditStore.get(ref.version, tableId, id, ref.nodeId);
+							if (
+								candidate?.version === existingEntry.version &&
+								(candidate.nodeId ?? 0) === (existingEntry.nodeId ?? 0)
+							) {
+								replacingId = ref.version;
+								replacingEntry = candidate;
+								break;
+							}
+						}
+					}
+					replacingEntry ??= auditStore.get(replacingId, tableId, id, existingEntry.nodeId);
 					if (replacingEntry) {
 						const previousVersion = replacingEntry.previousVersion;
 						result = auditStore[isRocksDB ? 'putSync' : 'put'](
@@ -934,7 +1049,7 @@ export function recordUpdater(store, tableId, auditStore) {
 				result = auditStore[isRocksDB ? 'putSync' : 'put'](
 					record === undefined ? NEW_TIMESTAMP_PLACEHOLDER : LAST_TIMESTAMP_PLACEHOLDER,
 					{
-						version: newVersion,
+						version: options?.recordVersion ?? newVersion,
 						tableId,
 						recordId: id,
 						previousVersion: isRocksDB ? existingEntry?.version : existingEntry?.localTime,

package/resources/RequestTarget.ts

@@ -36,6 +36,8 @@ export class RequestTarget extends URLSearchParams {
 	declare select?: Select;
 	/**	 Return an explanation of the query order */
 	declare explain?: boolean;
+	/** Request a total count of matching records for pagination (REST `Prefer: count=exact|estimated`). */
+	declare count?: 'exact' | 'estimated';
 	/**	 Force the query to be executed in the order of conditions */
 	declare enforceExecutionOrder?: boolean;
 	declare lazy?: boolean;

package/resources/Resource.ts

@@ -147,20 +147,55 @@ export class Resource<Record extends object = any> implements ResourceInterface<
 	static put = transactional(
 		function (resource: any, query: RequestTarget, request: Context, data: any) {
 			if (Array.isArray(data) && resource.#isCollection && resource.constructor.loadAsInstance !== false) {
+				const resourceClass = resource.constructor;
+				const primaryKey = resourceClass.primaryKey;
+				// Before dispatching anything: a malformed body must not race a sibling's write, and a bare
+				// dereference below would reach the client as a 500 carrying V8's wording. A primitive is
+				// malformed for the same reason — only a Table's own key validation rejects one later, so a
+				// plain Resource's override would otherwise be handed it.
+				for (let index = 0; index < data.length; index++) {
+					const element = data[index];
+					if (element == null || typeof element !== 'object')
+						throw new ClientError(
+							`Array element at index ${index} is ${element === null ? 'null' : typeof element}, expected a record`
+						);
+				}
+				const elementTarget = elementTargetFactory(query);
 				const results = [];
-				for (const element of data) {
-					const resourceClass = resource.constructor;
-					const id = element[resourceClass.primaryKey];
-					let target = new RequestTarget();
-					target.id = id;
-					const elementResource = resourceClass.getResource(target, request, {
-						async: true,
+				try {
+					for (let index = 0; index < data.length; index++) {
+						const element = data[index];
+						const target = elementTarget(element[primaryKey]);
+						const elementResource = resourceClass.getResource(target, request, {
+							async: true,
+						});
+						// `target`, not `request`: this is the target slot, and Table's back-compat shift only
+						// recognizes a RequestTarget there — a context is taken for the record.
+						// `missingMethod` rather than an unguarded call, so a resource without `put` answers 405
+						// here exactly as it does on the single-record path below.
+						if (typeof elementResource.then === 'function')
+							results.push(
+								elementResource.then((resource) =>
+									resource.put ? resource.put(element, target) : missingMethod(resource, 'put')
+								)
+							);
+						else if (elementResource.put) results.push(elementResource.put(element, target));
+						else missingMethod(elementResource, 'put');
+					}
+				} catch (error) {
+					// Settle the already-dispatched elements or a rejection among them goes unhandled. Then
+					// report the earliest-index failure, the same rule `settleElements` applies: every sibling
+					// already dispatched precedes the element that threw, so the batch reports the same failure
+					// whether that element's dispatch resolved synchronously or asynchronously.
+					return Promise.allSettled(results).then((settled) => {
+						const failed = settled.find((sibling) => sibling.status === 'rejected');
+						if (!failed) throw error;
+						throw (failed as PromiseRejectedResult).reason;
 					});
-					if (typeof elementResource.then === 'function')
-						results.push(elementResource.then((resource) => resource.put(element, request)));
-					else results.push(elementResource.put(element, query));
 				}
-				return Promise.all(results);
+				// Not `Promise.all`: settling the batch while a slower element is still resolving lets that
+				// element commit outside the failed batch — a partially applied array PUT.
+				return settleElements(results);
 			}
 			return resource.put
 				? resource.constructor.loadAsInstance === false
@@ -571,6 +606,53 @@ export function snakeCase(camelCase: string) {
 	);
 }
 
+/**
+ * Mint per-element targets for a fanned-out batch write. Each element needs its own object — its id
+ * must not be visible to a sibling whose dispatch resolves later — but it also needs the request's
+ * query and route metadata, which a `put()` override can read. The request's contribution is
+ * therefore resolved once here rather than deep-cloned per element, and nested metadata is shared
+ * rather than copied. `checkPermission` never carries: the collection receiver's single verdict is
+ * authoritative and per-element dispatch must not re-arm it.
+ */
+function elementTargetFactory(query: any): (id: any) => RequestTarget {
+	const base = cloneRequestTarget(query);
+	const search = base.toString();
+	// Only what the element target would not otherwise have. `RequestTarget` declares `pathname`,
+	// `search` and `sort` without `declare`, so they are own properties on every instance and would
+	// otherwise always be "carried" — overwriting what this element's own construction just derived,
+	// and making the skip below dead. A nullish own value carries no metadata either.
+	const carried: Record<string, any> = {};
+	for (const key of Object.keys(base)) {
+		if (key === 'id' || key === 'isCollection' || key === 'pathname' || key === 'search') continue;
+		const value = (base as any)[key];
+		if (value != null) carried[key] = value;
+	}
+	const hasCarried = Object.keys(carried).length > 0;
+	return (id) => {
+		const target = search ? new RequestTarget(search) : new RequestTarget();
+		if (hasCarried) Object.assign(target, carried);
+		target.id = id;
+		target.isCollection = false;
+		return target;
+	};
+}
+
+/**
+ * Report a fanned-out batch write's outcome only once every element has staged or failed its own
+ * write, so the enclosing transaction cannot unwind under one. Rejects with the first failure.
+ */
+function settleElements(results: any[]): Promise<any[]> {
+	return Promise.allSettled(results).then((settled) => {
+		const values = new Array(settled.length);
+		for (let index = 0; index < settled.length; index++) {
+			const element = settled[index];
+			if (element.status === 'rejected') throw element.reason;
+			values[index] = element.value;
+		}
+		return values;
+	});
+}
+
 /**
  * This is responsible for arranging arguments in the main static methods and creating the appropriate context and default transaction wrapping
  * @param action
@@ -733,8 +815,10 @@ function transactional(
 		if (!query) {
 			query = new RequestTarget();
 			query.id = id;
-			if (isCollection && options.method === 'put' && Array.isArray(data) && this.loadAsInstance === false)
-				query.isCollection = true;
+			// The collection inferred from the batch's null id has to survive onto the synthesized target,
+			// or the array is dispatched as one record update against a null primary key. Array-only:
+			// a null id on non-array data means auto-generate, not collection.
+			if (isCollection && options.method === 'put' && Array.isArray(data)) query.isCollection = true;
 		}
 		if (options.method === 'query' && data && typeof data === 'object') {
 			// QUERY executes the independently parsed body target. Make its requested projection part of
@@ -963,15 +1047,23 @@ function registerLiveSubscriptionForContext(subscription: any, resource: any, ad
 		// JWT exp of the bearer credential (set by the auth layer); undefined for password/mTLS/session.
 		authExpiresAt: user.authExpiresAt,
 		recheck: async () => {
-			// Re-fetch current user state — the user/role cache is rebuilt on mutations — so a dropped or
-			// role-stripped user no longer authorizes.
-			const { findAndValidateUser } = require('../security/user');
-			const fresh: any = await findAndValidateUser(username, undefined, false);
-			if (!fresh?.role) return false;
-			// Advance the subscription's context to the fresh user so downstream checks — context.user
-			// and getCurrentUser() (which reads the resource's context) — evaluate against current state,
-			// not the stale user captured at subscribe time.
-			if (context) (context as any).user = fresh;
+			let fresh: any;
+			if (user._scopedToken) {
+				// A scoped token's identity IS its embedded role — never re-resolve its attribution
+				// username against hdb_user (it may not exist, or may name an unrelated principal
+				// created later). Expiry (authExpiresAt above) is its only revocation.
+				fresh = user;
+			} else {
+				// Re-fetch current user state — the user/role cache is rebuilt on mutations — so a dropped or
+				// role-stripped user no longer authorizes.
+				const { findAndValidateUser } = require('../security/user');
+				fresh = await findAndValidateUser(username, undefined, false);
+				if (!fresh?.role) return false;
+				// Advance the subscription's context to the fresh user so downstream checks — context.user
+				// and getCurrentUser() (which reads the resource's context) — evaluate against current state,
+				// not the stale user captured at subscribe time.
+				if (context) (context as any).user = fresh;
+			}
 			// Re-run the same operation-level allowRead that granted the subscription.
 			const reTarget: any = cloneRequestTarget(admittedTarget);
 			reTarget.checkPermission = fresh.role?.permission;