harper 5.1.20 → 5.2.0-alpha.5

package/resources/PrimaryRocksDatabase.ts

@@ -0,0 +1,201 @@
+import { RocksDatabase, type RocksDatabaseOptions, constants, type Store } from '@harperfast/rocksdb-js';
+
+const FRESH_VERSION_FLAG = constants.FRESH_VERSION_FLAG;
+import { WeakLRUCache } from 'weak-lru-cache';
+import { when } from '../utility/when.ts';
+import { entryMap, METADATA, type Entry } from './RecordEncoder.ts';
+
+/**
+ * RocksDatabase subclass that owns all primary-store behaviour for Harper tables:
+ *   - RecordEncoder metadata extraction (version, metadataFlags, etc.)
+ *   - Optional WeakLRUCache keyed on entry version, verified via the process-wide VerificationTable
+ *
+ * Replaces both the old CachingRocksDatabase and the RocksDB-specific patches applied by
+ * handleLocalTimeForGets. Call initStore(rootStore) after open() — or let
+ * handleLocalTimeForGets delegate to it automatically via the isPrimaryRocksDatabase marker.
+ *
+ * Caching is enabled by default; pass { cache: false } to disable (useful for benchmarking
+ * or stores where version tracking is not desired).
+ *
+ * Cache freshness pattern (using the rocksdb-js VT API):
+ *   1. verifyVersion(key, cached.version)  → true  → return cached entry (no disk I/O)
+ *   2. verifyVersion(key, cached.version)  → false → read from DB, populateVersion, update cache
+ */
+export class PrimaryRocksDatabase extends RocksDatabase {
+	readonly isPrimaryRocksDatabase = true;
+	#cache?: WeakLRUCache;
+	readCount = 0;
+	cachePuts = false;
+	declare rootStore: any;
+	declare decoder: any;
+
+	get #enc(): any {
+		return (this as any).encoder;
+	}
+
+	constructor(pathOrStore: string | Store, options?: RocksDatabaseOptions & { cache?: boolean }) {
+		const enableCache = (options as any)?.cache === true;
+		super(pathOrStore, enableCache ? { ...options, verificationTable: true } : options);
+		if (enableCache) {
+			this.#cache = new WeakLRUCache();
+		}
+	}
+
+	/**
+	 * Initialises encoder/decoder state. Must be called once after open() with the root
+	 * RocksDatabase. Equivalent to the RocksDB branch of handleLocalTimeForGets, but as
+	 * a real method rather than instance-level monkey-patching.
+	 */
+	initStore(rootStore: RocksDatabase) {
+		this.readCount = 0;
+		this.cachePuts = false;
+		this.rootStore = rootStore;
+		this.#enc.rootStore = rootStore;
+		this.#enc.isRocksDB = true;
+		this.decoder = this.#enc;
+	}
+
+	#withEntry(entry: Entry, id: any): Entry {
+		if (entry.value) {
+			if (entry.value.constructor === Object && this.#enc.structPrototype) {
+				const originalValue = entry.value;
+				entry.value = new this.#enc.structPrototype.constructor();
+				Object.assign(entry.value, originalValue);
+			}
+			if (typeof entry.value === 'object' && entry.value !== null) {
+				entryMap.set(entry.value, entry);
+			}
+		}
+		entry.key = id;
+		return entry;
+	}
+
+	#processEntry(raw: any, id: any): Entry | undefined {
+		if (raw == null) return undefined;
+		if (raw[METADATA]) {
+			raw.metadataFlags = raw[METADATA];
+			return this.#withEntry(raw, id);
+		}
+		const entry = { value: raw, key: id } as Entry;
+		// map metadata-less values too, so getEntry guarantees the value→Entry
+		// mapping for every object value it returns (getSync/get rely on this)
+		if (typeof raw === 'object') entryMap.set(raw, entry);
+		return entry;
+	}
+
+	/**
+	 * Core read method. Returns a full Entry (with version, metadataFlags, value, …) or
+	 * undefined. When caching is enabled, passes `expectedVersion` to the native layer so
+	 * a single call handles both verification (returns FRESH_VERSION_FLAG on hit) and VT
+	 * population (auto-seeded on DB read). Only cold reads (no cached version) need a
+	 * separate populateVersion call.
+	 */
+	getEntry(id: any, options?: any): any {
+		this.readCount++;
+		const cache = this.#cache;
+		// The cache stores the record *value* (weakly, via setValue) rather than
+		// the Entry: a WeakRef-wrapped value lets the LRFU expirer release it once
+		// it cycles out of the LRU stages (a raw Entry stored via set() has neither
+		// a .deref nor a .cache hook, so it would never be reclaimed — an unbounded
+		// leak). The value→Entry WeakMap (entryMap) recovers the Entry metadata
+		// (version, flags) needed for the fast path while the value is live.
+		const cachedValue = cache?.getValue(id);
+		const cached =
+			cachedValue != null && typeof cachedValue === 'object'
+				? (entryMap.get(cachedValue) as Entry | undefined)
+				: undefined;
+		const expectedVersion = cached?.version;
+
+		// Build get options, always merging with caller options to preserve
+		// transaction snapshot. Pass expectedVersion when cached:
+		//   VT hit  → native returns FRESH_VERSION_FLAG, no DB read
+		//   VT miss → native reads DB and auto-populates VT slot
+		// For cold reads (no cached version), use populateVersion flag so the
+		// native layer seeds the VT slot in the same call.
+		let getOptions: any;
+		if (expectedVersion != null) {
+			getOptions = options ? { ...options, expectedVersion } : { expectedVersion };
+		} else if (cache) {
+			getOptions = options ? { ...options, populateVersion: true } : { populateVersion: true };
+		} else {
+			getOptions = options;
+		}
+		const raw = options?.async ? super.get(id, getOptions) : super.getSync(id, getOptions);
+
+		return when(raw, (result) => {
+			if (result === FRESH_VERSION_FLAG) return cached;
+			const entry = this.#processEntry(result, id);
+			if (entry == null) {
+				if (cache && cachedValue !== undefined) cache.delete(id);
+				return undefined;
+			}
+			// Only object values can be weakly cached and mapped back to their Entry;
+			// primitive/empty values fall through uncached (no fast path, still correct).
+			if (entry.version != null && cache && entry.value != null && typeof entry.value === 'object') {
+				entryMap.set(entry.value, entry);
+				cache.setValue(id, entry.value, (entry.size ?? 0) >> 10);
+			}
+			return entry;
+		});
+	}
+
+	getSync(id: any, options?: any): any {
+		const entry = this.getEntry(id, options) as Entry;
+		return entry?.value;
+	}
+
+	get(id: any, options?: any): any {
+		return when(this.getEntry(id, { ...options, async: true }), (entry: Entry) => entry?.value);
+	}
+
+	getRange(options?: any): any {
+		const iterable = super.getRange(options);
+		if (options?.valuesForKey) return iterable.map((v: any) => v?.value);
+		if (options?.values === false || options?.onlyCount) return iterable;
+		if (!this.#enc.isRocksDB) return iterable;
+		const enc = this.#enc;
+		return iterable.map((entry: any) => {
+			if (entry.value?.[METADATA]) {
+				entry.metadataFlags = entry.value[METADATA];
+				Object.assign(entry, entry.value);
+			}
+			if (entry.value?.constructor === Object && enc.structPrototype) {
+				const originalValue = entry.value;
+				entry.value = new enc.structPrototype.constructor();
+				for (const key in originalValue) entry.value[key] = originalValue[key];
+			}
+			return entry;
+		});
+	}
+
+	putSync(id: any, value: any, options?: any): any {
+		this.#cache?.delete(id);
+		return super.putSync(id, value, options);
+	}
+
+	removeSync(id: any, options?: any): any {
+		this.#cache?.delete(id);
+		return super.removeSync(id, options);
+	}
+
+	clearSync(): void {
+		// clearSync wipes every record; the per-instance cache must be dropped
+		// too, otherwise a cached entry would survive the clear and be served as
+		// a stale "still exists" hit on the next read.
+		this.#cache?.clear();
+		return super.clearSync();
+	}
+
+	clear(): Promise<void> {
+		this.#cache?.clear();
+		return super.clear();
+	}
+
+	open(): PrimaryRocksDatabase {
+		return super.open() as PrimaryRocksDatabase;
+	}
+
+	static open(pathOrStore: string | Store, options?: RocksDatabaseOptions): PrimaryRocksDatabase {
+		return new PrimaryRocksDatabase(pathOrStore, options).open();
+	}
+}

package/resources/RecordEncoder.ts

@@ -18,6 +18,7 @@ import {
 	LOCAL_ONLY,
 } from './auditStore.ts';
 import * as harperLogger from '../utility/logging/harper_logger.ts';
+import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
 import './blob.ts';
 import {
 	blobsWereEncoded,
@@ -118,6 +119,12 @@ export class RecordEncoder extends StructonEncoder {
 	isRocksDB: boolean;
 	name: string;
 	useVersions: boolean;
+	// Self-versioning mode for stores whose writes never stage a timestamp (HNSW/custom-index
+	// object stores). When set, each encode stamps a fresh monotonic version into the 8-byte
+	// metadata prefix so the value carries a version the RocksDB Verification Table can extract —
+	// enabling race-safe cache verification of graph nodes during traversal. Unlike primary-record
+	// writes, this never reads or clears the shared timestampNextEncoding global (harper#1307).
+	autoVersion = false;
 	constructor(options) {
 		options.useBigIntExtension = true;
 		// Bound the per-encoder typed-structure dictionary. It is append-only and pinned on the
@@ -169,6 +176,33 @@ export class RecordEncoder extends StructonEncoder {
 				lastValueEncoding = superEncode.call(this, record, options);
 				return lastValueEncoding;
 			}
+			if (this.autoVersion && this.isRocksDB) {
+				// Self-versioned index store (RocksDB only — the Verification Table is RocksDB-specific):
+				// prefix the record-encoder metadata header so each node carries a version the VT can
+				// extract and the decode path recognises. The header is [8-byte BE float64 version]
+				// [4-byte BE metadata word], exactly the format decode() expects: the version's first
+				// byte (0x42 for current-era ms timestamps) triggers metadata detection, and the
+				// ACTION_32_BIT-tagged flags word (flags = 0 — no expiration/residency/etc.) is required
+				// so decode consumes a metadata word rather than mis-reading the struct's first byte as
+				// flags — the flags word being mandatory was the actual bug behind earlier failures,
+				// NOT any struct incompatibility. Use the same reserve-start mechanism the primary
+				// metadata path below uses (the 2048|valueStart option reserves valueStart bytes at the
+				// front of the encode buffer; we write the header in place), which works fine with the
+				// forced randomAccessStructure/typed-struct mode of custom-object indexes — verified
+				// against the cold/frozen-read (#1161) and concurrent multi-worker (#386) suites. This
+				// avoids a per-write extra allocation + full-payload copy. Never touch the shared
+				// timestampNextEncoding/metadataInNextEncoding globals — those belong to the enclosing
+				// primary-record write whose commit nests this index encode (harper#1307).
+				const valueStart = 12; // 8-byte version + 4-byte metadata word
+				const encoded = superEncode.call(this, record, options | 2048 | valueStart);
+				const position = encoded.start || 0;
+				const dataView =
+					encoded.dataView || (encoded.dataView = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength));
+				dataView.setFloat64(position, getNextMonotonicTime()); // version (big-endian, rocksdb stores it directly)
+				dataView.setUint32(position + 8, ACTION_32_BIT << 24); // metadata word: ACTION_32_BIT tag, flags = 0
+				lastValueEncoding = encoded.subarray(position + valueStart, encoded.end);
+				return encoded;
+			}
 			// this handles our custom metadata encoding, prefixing the record with metadata, including the local
 			// timestamp into the audit record, invalidation status and residency information
 			if (timestampNextEncoding || metadataInNextEncoding >= 0) {
@@ -457,52 +491,37 @@ function getTimestamp() {
 }
 
 export function handleLocalTimeForGets(store, rootStore) {
-	const isRocksDB = store instanceof RocksDatabase;
+	if ((store as any).isPrimaryRocksDatabase) {
+		(store as any).initStore(rootStore);
+		return store;
+	}
 	store.readCount = 0;
 	store.cachePuts = false;
 	store.rootStore = rootStore;
 	store.encoder.rootStore = rootStore;
-	store.encoder.isRocksDB = isRocksDB;
 	store.decoder = store.encoder;
 	const storeGetEntry = store.getEntry;
-	const storeGetSync = store.getSync;
-	const storeGet = store.get;
 
 	store.getEntry = function (id, options) {
 		store.readCount++;
 		lastMetadata = null;
-		if (isRocksDB) {
-			return when(
-				options?.async ? storeGet.call(store, id, options) : storeGetSync.call(store, id, options),
-				(entry) => {
-					if (entry) {
-						if (entry[METADATA]) {
-							entry.metadataFlags = entry[METADATA];
-							return withEntry(entry);
-						} else return { value: entry };
-					} else return entry;
-				}
-			);
-		} else {
-			let entry: Entry = storeGetEntry.call(this, id, options);
-			if (lastMetadata) {
-				entry.metadataFlags = lastMetadata[METADATA];
-				entry.localTime = lastMetadata.localTime;
-				entry.residencyId = lastMetadata.residencyId;
-				entry.nodeId = lastMetadata.nodeId;
-				entry.additionalAuditRefs = lastMetadata.additionalAuditRefs;
-				entry.size = lastMetadata.size;
-				if (lastMetadata.expiresAt >= 0) {
-					entry.expiresAt = lastMetadata.expiresAt;
-				}
-				if (isRocksDB) entry.version = lastMetadata.localTime;
-				if (entry.value) {
-					entryMap.set(entry.value, entry); // allow the record to access the entry
-				}
-				entry.key = id;
+		let entry: Entry = storeGetEntry.call(this, id, options);
+		if (lastMetadata) {
+			entry.metadataFlags = lastMetadata[METADATA];
+			entry.localTime = lastMetadata.localTime;
+			entry.residencyId = lastMetadata.residencyId;
+			entry.nodeId = lastMetadata.nodeId;
+			entry.additionalAuditRefs = lastMetadata.additionalAuditRefs;
+			entry.size = lastMetadata.size;
+			if (lastMetadata.expiresAt >= 0) {
+				entry.expiresAt = lastMetadata.expiresAt;
+			}
+			if (entry.value) {
+				entryMap.set(entry.value, entry); // allow the record to access the entry
 			}
-			return entry && withEntry(entry);
+			entry.key = id;
 		}
+		return entry && withEntry(entry);
 		// if we have decoded with metadata, we want to pull it out and assign to this entry
 		function withEntry(entry) {
 			if (entry.value) {
@@ -547,15 +566,9 @@ export function handleLocalTimeForGets(store, rootStore) {
 		if (options.values === false || options.onlyCount) return iterable;
 		return iterable.map((entry) => {
 			// if we have metadata, move the metadata to the entry
-			if (isRocksDB) {
-				if (entry.value?.[METADATA]) {
-					entry.metadataFlags = entry.value[METADATA];
-					Object.assign(entry, entry.value);
-				}
-			} else if (lastMetadata) {
+			if (lastMetadata) {
 				entry.metadataFlags = lastMetadata[METADATA];
 				entry.localTime = lastMetadata.localTime;
-				if (isRocksDB) entry.version = lastMetadata.localTime;
 				entry.residencyId = lastMetadata.residencyId;
 				entry.nodeId = lastMetadata.nodeId;
 				entry.additionalAuditRefs = lastMetadata.additionalAuditRefs;
@@ -575,36 +588,35 @@ export function handleLocalTimeForGets(store, rootStore) {
 		});
 	};
 
-	if (!isRocksDB) {
-		// add read transaction tracking
-		const txn = store.useReadTransaction();
-		txn.done();
-		if (!txn.done.isTracked) {
-			const Txn = txn.constructor;
-			const use = txn.use;
-			const done = txn.done;
-			Txn.prototype.use = function () {
-				if (!this.timerTracked) {
-					this.timerTracked = true;
-					trackedTxns.push(new WeakRef(this));
-				}
-				use.call(this);
-			};
-			Txn.prototype.done = function () {
-				if (this.isDone) return;
-				done.call(this);
-				this.openTimer = 0; // reset so idle pool time doesn't accumulate toward the stale-open threshold
-				if (this.isDone) {
-					for (let i = 0; i < trackedTxns.length; i++) {
-						const txn = trackedTxns[i].deref();
-						if (!txn || txn.isDone || txn.isCommitted) {
-							trackedTxns.splice(i--, 1);
-						}
+	// add read transaction tracking (LMDB only — RocksDB primary stores are PrimaryRocksDatabase and
+	// return early above; only LMDB stores reach here)
+	const txn = store.useReadTransaction();
+	txn.done();
+	if (!txn.done.isTracked) {
+		const Txn = txn.constructor;
+		const use = txn.use;
+		const done = txn.done;
+		Txn.prototype.use = function () {
+			if (!this.timerTracked) {
+				this.timerTracked = true;
+				trackedTxns.push(new WeakRef(this));
+			}
+			use.call(this);
+		};
+		Txn.prototype.done = function () {
+			if (this.isDone) return;
+			done.call(this);
+			this.openTimer = 0; // reset so idle pool time doesn't accumulate toward the stale-open threshold
+			if (this.isDone) {
+				for (let i = 0; i < trackedTxns.length; i++) {
+					const txn = trackedTxns[i].deref();
+					if (!txn || txn.isDone || txn.isCommitted) {
+						trackedTxns.splice(i--, 1);
 					}
 				}
-			};
-			Txn.prototype.done.isTracked = true;
-		}
+			}
+		};
+		Txn.prototype.done.isTracked = true;
 	}
 
 	return store;

package/resources/RequestTarget.ts

@@ -56,6 +56,8 @@ export class RequestTarget extends URLSearchParams {
 	declare previousResidency?: string[];
 
 	// Action tracking
+	/** Cache disposition of this get on a caching table: true if loaded from the source, false if
+	 * served from cache. Set per-get; read it on the RequestTarget you passed to the get. */
 	declare loadedFromSource?: boolean;
 	declare createdNewId?: string;
 
@@ -65,6 +67,25 @@ export class RequestTarget extends URLSearchParams {
 	declare allowFullScan?: boolean;
 	declare allowConditionsOnDynamicAttributes?: boolean;
 
+	/**
+	 * Predicate-aware vector search (#1241). A `(record) => boolean` filter evaluated during HNSW
+	 * traversal (and as a post-filter for other paths), so a vector sort keeps exploring until it has
+	 * enough MATCHING nearest neighbors instead of post-filtering an under-filled candidate set. The
+	 * function must be synchronous and side-effect free; the record it receives is frozen. JS-API only —
+	 * never parsed from a REST query string (no eval of user-supplied code over HTTP).
+	 */
+	declare vectorFilter?: (record: any) => boolean;
+
+	/**
+	 * When `false`, the query reads against the latest committed data without holding a consistent
+	 * read snapshot open for the duration of the iteration. This trades read consistency (rows
+	 * written after the scan starts may be observed) for not pinning a snapshot that blocks
+	 * compaction and ties up resources — useful for long-running scans such as analytics queries.
+	 * Defaults to `true` (a stable snapshot is held). Currently only honored by RocksDB-backed
+	 * tables; see `DatabaseTransaction.getReadTxn`.
+	 */
+	declare snapshot?: boolean;
+
 	constructor(target?: string) {
 		let searchIndex: number | undefined;
 		let path: string | undefined;

package/resources/Resource.ts

@@ -10,7 +10,7 @@ import {
 	RequestTargetOrId,
 } from './ResourceInterface.ts';
 import { randomUUID } from 'crypto';
-import { DatabaseTransaction, type Transaction } from './DatabaseTransaction.ts';
+import { DatabaseTransaction, TRANSACTION_STATE, type Transaction } from './DatabaseTransaction.ts';
 import { IterableEventQueue } from './IterableEventQueue.ts';
 import { _assignPackageExport } from '../globals.js';
 import { ClientError, AccessViolation } from '../utility/errors/hdbError.ts';
@@ -18,6 +18,7 @@ import { transaction, contextStorage } from './transaction.ts';
 import { parseQuery } from './search.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import { when, promiseNormalize } from '../utility/when.ts';
+import { registerLiveSubscription } from '../server/liveSubscriptionAuth.ts';
 import type { JsonSchemaFragment } from './jsonSchemaTypes.ts';
 
 const EXTENSION_TYPES = {
@@ -68,10 +69,6 @@ export class Resource<Record extends object = any> implements ResourceInterface<
 		return true; // Subclasses should override if needed
 	}
 
-	wasLoadedFromSource(): boolean | void {
-		// Subclasses should override if needed
-	}
-
 	addTo(_property: keyof Record, _value: Record[keyof Record]): void {
 		throw new Error('Not implemented');
 	}
@@ -661,8 +658,34 @@ function transactional(
 			if (isCollection) resourceOptions.isCollection = true;
 		} else resourceOptions = options;
 		const loadAsInstance = this.loadAsInstance;
-		if (context?.transaction) {
-			// we are already in a transaction, proceed
+		// Only join an existing transaction if it is still genuinely OPEN (mirrors the reuse check
+		// resources/transaction.ts's transaction() helper already applies to itself). A `context`
+		// object can carry a *stale* `.transaction` left over from an earlier, unrelated call that
+		// already ran to completion: ambient contexts obtained via contextStorage.getStore() are
+		// no longer guaranteed to be fresh, one-shot objects now that processLocalTransaction (#1591/
+		// #1592) installs one shared, long-lived context for the lifetime of an entire operation
+		// handler. Without this check, a second, logically-independent static Resource API call made
+		// later in the same handler (no explicit context of its own) would see that leftover
+		// `.transaction` reference and wrongly treat it as "still open," silently folding its write
+		// into a transaction whose commit lifecycle was already driven to completion by the first
+		// call — coalescing two unrelated writes and risking the first one being dropped. See
+		// harper-pro's replication/subscriptionManager.ts ensureNode(), which is called twice in
+		// sequence (once for this node, once for a peer) from a single add_node/add_node_back
+		// operation handler and reproduced this exact loss in a live cluster-formation test.
+		//
+		// A transaction that is no longer OPEN reached that state one of two ways, and they need
+		// opposite treatment:
+		//  - committed: the prior call ran to completion normally. Safe to silently start a fresh
+		//    transaction for this independent call (the case above).
+		//  - aborted due to exceeding storage.maxTransactionOpenTime (#1411, DatabaseTransaction.ts
+		//    abortDueToTimeout(), marked via the `timedOut` poison flag): the whole logical operation
+		//    must fail atomically, not have this write quietly land on a brand-new transaction while
+		//    the earlier write(s) were rolled back. Joining the poisoned transaction here (instead of
+		//    starting fresh) makes the write throw transactionOpenTooLongError via addWrite()/commit()'s
+		//    poison check, correctly propagating the abort to the caller. See
+		//    integrationTests/resources/txn-overtime-atomicity.test.ts.
+		if (context?.transaction?.open === TRANSACTION_STATE.OPEN || context?.transaction?.timedOut) {
+			// we are already in a transaction (or it was poisoned by a timeout abort and must fail), proceed
 			const resource = this.getResource(query, context, resourceOptions);
 			return resource.then
 				? resource.then(authorizeActionOnResource)
@@ -683,6 +706,20 @@ function transactional(
 			);
 		}
 		function authorizeActionOnResource(resource: ResourceInterface) {
+			// For subscribe, register the resulting subscription for continuous re-authorization so it is
+			// terminated if the principal later loses access or its token expires (#1414). Non-subscribe
+			// actions run unchanged.
+			// 'subscribe' is the direct MQTT path; 'connect' is the SSE/WebSocket path (REST CONNECT) —
+			// both resolve to the same subscription iterable.
+			const isSubscribeAction = options.method === 'subscribe' || options.method === 'connect';
+			const runAction = (data: any) => {
+				const result = action(resource, query, context, data);
+				if (!isSubscribeAction) return result;
+				return when(result, (subscription: any) => {
+					registerLiveSubscriptionForContext(subscription, resource, query, context);
+					return subscription;
+				});
+			};
 			let checkPermission = false;
 			if (query.checkPermission) {
 				checkPermission = true;
@@ -697,26 +734,38 @@ function transactional(
 			if (checkPermission) {
 				if (loadAsInstance !== false) {
 					// do permission checks, with allow methods
-					const allowed =
-						options.type === 'read'
-							? resource.allowRead(context.user, query, context)
-							: options.type === 'update'
-								? resource.doesExist?.() === false
-									? resource.allowCreate(context.user, data, context)
-									: resource.allowUpdate(context.user, data, context)
-								: options.type === 'create'
-									? resource.allowCreate(context.user, data, context)
-									: resource.allowDelete(context.user, query, context);
+					let allowed;
+					try {
+						allowed =
+							options.type === 'read'
+								? resource.allowRead(context.user, query, context)
+								: options.type === 'update'
+									? resource.doesExist?.() === false
+										? resource.allowCreate(context.user, data, context)
+										: resource.allowUpdate(context.user, data, context)
+									: options.type === 'create'
+										? resource.allowCreate(context.user, data, context)
+										: resource.allowDelete(context.user, query, context);
+					} catch {
+						// allow* threw — fail closed rather than letting the request proceed
+						throw new AccessViolation(context.user);
+					}
 					if ((allowed as any)?.then) {
-						return (allowed as any).then((allowed) => {
-							query.checkPermission = false;
-							if (!allowed) {
+						return (allowed as any).then(
+							(allowed) => {
+								query.checkPermission = false;
+								if (!allowed) {
+									throw new AccessViolation(context.user);
+								}
+								return when(data, (data) => {
+									return runAction(data);
+								});
+							},
+							() => {
+								// async allow* rejected — fail closed
 								throw new AccessViolation(context.user);
 							}
-							return when(data, (data) => {
-								return action(resource, query, context, data);
-							});
-						});
+						);
 					}
 					query.checkPermission = false;
 					if (!allowed) {
@@ -725,11 +774,46 @@ function transactional(
 				}
 			}
 			return when(data, (data) => {
-				return action(resource, query, context, data);
+				return runAction(data);
 			});
 		}
 	}
 }
+function registerLiveSubscriptionForContext(subscription: any, resource: any, query: any, context: Context) {
+	const user: any = context?.user;
+	const username = user?.username;
+	// Internal watchers, replication and local-bypass have no user principal — nothing to re-authorize.
+	if (!username) return;
+	const capturedId = query?.id;
+	const capturedIsCollection = query?.isCollection;
+	const capturedSelect = query?.select;
+	registerLiveSubscription({
+		subscription,
+		username,
+		// JWT exp of the bearer credential (set by the auth layer); undefined for password/mTLS/session.
+		authExpiresAt: user.authExpiresAt,
+		recheck: async () => {
+			// Re-fetch current user state — the user/role cache is rebuilt on mutations — so a dropped or
+			// role-stripped user no longer authorizes.
+			const { findAndValidateUser } = require('../security/user');
+			const fresh: any = await findAndValidateUser(username, undefined, false);
+			if (!fresh?.role) return false;
+			// Advance the subscription's context to the fresh user so downstream checks — context.user
+			// and getCurrentUser() (which reads the resource's context) — evaluate against current state,
+			// not the stale user captured at subscribe time.
+			if (context) (context as any).user = fresh;
+			// Re-run the same table/RBAC-level allowRead the subscription was granted with, against the
+			// fresh user. No per-record evaluation — this matches how access was originally granted.
+			const reTarget: any = new RequestTarget();
+			reTarget.id = capturedId;
+			reTarget.isCollection = capturedIsCollection;
+			reTarget.select = capturedSelect;
+			reTarget.checkPermission = fresh.role?.permission;
+			return !!(await resource.allowRead(fresh, reTarget, context));
+		},
+	});
+}
+
 const KNOWN_METHODS = ['get', 'head', 'put', 'post', 'delete', 'patch', 'query', 'move', 'copy'];
 type ClientErrorWithMethods = ClientError & { allow: string[]; method: string };
 export function missingMethod(resource: any, method: string) {

package/resources/ResourceInterface.ts

@@ -49,7 +49,6 @@ export interface ResourceInterface<Record extends object = any>
 	subscribe?(request: SubscriptionRequest): AsyncIterable<Record> | Promise<AsyncIterable<Record>>;
 
 	doesExist(): boolean;
-	wasLoadedFromSource(): boolean | void;
 
 	getCurrentUser(): User | undefined;
 }
@@ -95,7 +94,6 @@ export interface Context {
 	sourceApply?: boolean;
 	originatingOperation?: OperationFunctionName;
 	previousResidency?: string[];
-	loadedFromSource?: boolean;
 	nodeName?: string;
 	resourceCache?: Map<Id, any>;
 	_freezeRecords?: boolean; // until v5, we conditionally freeze records for back-compat
@@ -177,6 +175,12 @@ interface TypedDirectCondition<Record extends object, Property extends keyof Rec
 	 * full scan unless paired with another indexed condition.
 	 */
 	negated?: boolean;
+	/**
+	 * Internal (#1241): a `(primaryKey) => boolean` predicate composed by the query executor from
+	 * companion conditions, a caller `vectorFilter`, and record-level RBAC, pushed into a filterable
+	 * custom index (HNSW) so filtering happens during traversal. Not part of the public query surface.
+	 */
+	recordFilter?: (primaryKey: Id) => boolean;
 }
 
 interface ConditionGroup<Record extends object = any> {

package/resources/Resources.ts

@@ -17,9 +17,7 @@ export interface ResourceEntry {
 }
 
 export type RouteSegment =
-	| { type: 'static'; value: string }
-	| { type: 'param'; value: string }
-	| { type: 'wildcard'; value: string };
+	{ type: 'static'; value: string } | { type: 'param'; value: string } | { type: 'wildcard'; value: string };
 
 export interface CompiledRoute {
 	/** Normalized pattern (leading/trailing slashes stripped) — used as the identity key for the route. */