@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.1

package/resources/nodeIdMapping.ts

@@ -50,6 +50,7 @@ function getIdMappingRecord(auditStore) {
 		// now we can take over the local node id
 		nameToId[node_name] = 0;
 		auditStore.putSync(REMOTE_NODE_IDS, pack(idMappingRecord));
+		invalidateNodeNames(auditStore);
 	}
 	return idMappingRecord;
 }
@@ -84,6 +85,7 @@ export function remoteToLocalNodeId(remoteMapping: any, auditStore: any) {
 	}
 	if (hasChanges) {
 		auditStore.putSync(REMOTE_NODE_IDS, pack(idMappingRecord));
+		invalidateNodeNames(auditStore);
 	}
 	return remoteToLocalId;
 }
@@ -103,6 +105,7 @@ export function getIdOfRemoteNode(remoteNodeName, auditStore) {
 		id = lastId + 1;
 		nameToId[remoteNodeName] = id;
 		auditStore.putSync(REMOTE_NODE_IDS, pack(idMappingRecord));
+		invalidateNodeNames(auditStore);
 	}
 	logger.trace?.('The remote node name map', remoteNodeName, nameToId, id);
 	return id;
@@ -123,3 +126,50 @@ export function lastTimeInAuditStore(auditStore: Database) {
 export function getThisNodeId(auditStore: any) {
 	return exportIdMapping(auditStore)?.[server.hostname];
 }
+
+// Inverted id -> name map. exportIdMapping() re-reads and unpacks the mapping record on every call,
+// which is far too much per replicated entry on the apply thread; ids are stable once minted, so the
+// inversion is cached and dropped wherever nameToId is written. Keyed per audit store because short
+// ids are minted per database in first-seen order — id 1 names a different node in each one.
+// Refresh window for a MISS. A hit is always trusted, but a miss cannot be: ids are minted by
+// whichever worker first talks to a peer, and invalidation only reaches that worker's own copy — so
+// a newly admitted node is absent from every other worker's map until it is rebuilt. Rebuilding on
+// every miss would put a store read and unpack back on the apply thread for each unmapped entry, so
+// misses re-read at most this often — short enough that a joining node loses at most a stray entry,
+// long enough that a burst of unmapped ids cannot drive the store. `onMiss` records whether the
+// cached map was built by a `rebuildOnMiss` caller, which is what bounds that caller to one read per
+// window too rather than exempting it from the interval entirely.
+const NODE_NAME_REFRESH_MS = 50;
+const idToNodeName = new WeakMap<object, { names: Map<number, string>; refreshedAt: number; onMiss: boolean }>();
+function invalidateNodeNames(auditStore: any) {
+	idToNodeName.delete(auditStore);
+}
+
+/**
+ * The node name a local short id refers to. Callers that must attribute a replicated entry to its
+ * origin node need the globally stable name, not the id, which is assigned per node.
+ */
+export function getNodeNameForId(
+	auditStore: any,
+	nodeId: number | undefined,
+	rebuildOnMiss = false
+): string | undefined {
+	if (typeof nodeId !== 'number' || !auditStore) return undefined;
+	const cached = idToNodeName.get(auditStore);
+	const hit = cached?.names.get(nodeId);
+	if (hit !== undefined) return hit;
+	const now = Date.now();
+	// `rebuildOnMiss` is for entry classes whose loss costs more than the read: a dropped record lock
+	// release leaves the home holding a grant until its own deadline, so a just-minted id is worth one
+	// store read rather than up to `NODE_NAME_REFRESH_MS` of dropped entries. ONE read, though, not one
+	// per entry — an id this map will never resolve (a node whose mapping was purged, an origin
+	// relayed from a database that never minted one) is exactly the burst the interval exists to keep
+	// off the apply thread, and a rebuild inside the window is not stale: a write to the mapping drops
+	// this entry outright (`invalidateNodeNames`).
+	if (cached && now - cached.refreshedAt < NODE_NAME_REFRESH_MS && (!rebuildOnMiss || cached.onMiss)) return undefined;
+	const nameToId = exportIdMapping(auditStore);
+	const names = new Map<number, string>();
+	for (const name in nameToId) names.set(nameToId[name], name);
+	idToNodeName.set(auditStore, { names, refreshedAt: now, onMiss: rebuildOnMiss });
+	return names.get(nodeId);
+}

package/resources/recordLock.ts

@@ -1,12 +1,19 @@
+import { performance } from 'node:perf_hooks';
 import harperLogger from '../utility/logging/harper_logger.ts';
 import { ClientError } from '../utility/errors/hdbError.ts';
 
 /**
- * Exclusive per-record locks (harper#483, Phase 0: one node, every worker thread).
+ * Exclusive per-record locks (harper#483).
  *
- * The sole authority is the rocksdb-js process-wide key lock: one `locks` map per DBDescriptor
- * shared by every worker thread's handle. Nothing is written to the store or audit log. Lock and
- * unlock are pure in-memory operations; the record's version and bytes are unchanged.
+ * The intra-node authority is the rocksdb-js process-wide key lock: one `locks` map per DBDescriptor
+ * shared by every worker thread's handle. Acquiring and releasing it write nothing to the store, and
+ * the record's version and bytes are unchanged.
+ *
+ * Phase 1 layers cluster-wide exclusion on top without changing that: once the native key is held,
+ * `Table.lock()` obtains a delegation for the key from its home node (see `recordLockCoordinator.ts`)
+ * and calls `joinClusterRound()` on the handle. A live delegation serves repeat locks with no cluster
+ * message at all. A node-scoped lock (`{ scope: 'node' }`), or any lock on a database with no
+ * transport registered, skips the cluster step entirely and behaves exactly as it did in Phase 0.
  */
 
 export const DEFAULT_LOCK_LEASE_MS = 30_000;
@@ -24,12 +31,20 @@ export interface RecordLockOptions {
 	timeout?: number;
 	/** A held lock outlives the acquiring transaction; it is released by `unlock()` or by its lease */
 	hold?: boolean;
+	/**
+	 * `'cluster'` (default) is exclusive across every participating node; `'node'` keeps Phase 0
+	 * semantics and is exclusive only across this node's worker threads.
+	 */
+	scope?: 'cluster' | 'node';
 }
 
 export interface ResolvedRecordLockOptions {
 	lease: number;
 	timeout: number;
 	hold: boolean;
+	scope: 'cluster' | 'node';
+	/** True when the caller named `scope` explicitly, which makes `'cluster'` fail-closed. */
+	scopeRequested: boolean;
 }
 
 /**
@@ -55,6 +70,35 @@ export interface RecordLockHandle {
 	noteCandidateFloor(version: number): void;
 	/** Compute a holder-write version without advancing the floor until that write commits. */
 	holderVersionCandidate(floor?: number): number;
+	/** `ts_R` of the granted cluster round; undefined while the hold is node-scoped (Phase 0). */
+	clusterTsR?: number;
+	/**
+	 * Whether this handle may still authorize a write, evaluated NOW rather than trusting the lease
+	 * timer to have run. A holder whose event loop stalled past its deadline would otherwise commit
+	 * between the deadline and its own timer callback, while peers had already expired the hold.
+	 * Expiry is detected on a monotonic clock, so a wall-clock jump cannot extend a lease.
+	 */
+	isExpired(): boolean;
+	/**
+	 * Whether the LEASE elapsed. Distinct from `isExpired()` in that a deliberate release does NOT
+	 * make it true on its own: a write staged while the lock was held stays valid at commit if the
+	 * caller merely unlocked. Once the deadline passes it is true either way, because by then peers
+	 * have passed their own bound and the key may already belong to someone else.
+	 */
+	isLeaseExpired(): boolean;
+	/** Revoke authority ahead of the lease, so a staged-but-uncommitted write is fenced at commit. */
+	revokeLease(): void;
+	/**
+	 * Re-anchor a granted cluster round: the hold now runs from `tsR` (no margin, so this node
+	 * always expires before any participant does) and `onRelease` emits the durable LOCK_RELEASE.
+	 * Returns false when the round completed after the lease had already elapsed.
+	 *
+	 * `mintedMono` is `performance.now()` at the instant `tsR` was minted. The remaining lease is
+	 * measured from it rather than from `tsR - Date.now()`: `tsR` comes from a never-decreasing
+	 * monotonic source, so after a backward wall-clock step that subtraction would hand back MORE
+	 * lease than the round was granted, past what every peer bounded from its own observation.
+	 */
+	joinClusterRound(tsR: number, leaseMs: number, mintedMono: number, onRelease: () => void): boolean;
 	/**
 	 * Return the next version to stamp a holder write with. Starts at acquiredAt when no transaction
 	 * version has been noted; otherwise advances by the minimum monotonic step from the recorded floor.
@@ -83,10 +127,15 @@ export function resolveLockOptions(options?: RecordLockOptions | null): Resolved
 		throw new ClientError(`Lock options must be an object, but received ${typeof options}`);
 	const hold = options?.hold ?? false;
 	if (typeof hold !== 'boolean') throw new ClientError(`Lock option hold must be a boolean, but received ${hold}`);
+	const scope = options?.scope ?? 'cluster';
+	if (scope !== 'cluster' && scope !== 'node')
+		throw new ClientError(`Lock option scope must be 'cluster' or 'node', but received ${scope}`);
 	return {
 		lease: requireDuration('lease', options?.lease, DEFAULT_LOCK_LEASE_MS, MIN_LOCK_LEASE_MS, MAX_LOCK_LEASE_MS),
 		timeout: requireDuration('timeout', options?.timeout, DEFAULT_LOCK_TIMEOUT_MS, 1, MAX_LOCK_TIMEOUT_MS),
 		hold,
+		scope,
+		scopeRequested: options?.scope !== undefined,
 	};
 }
 
@@ -105,6 +154,19 @@ export function lockAttemptKey(tableId: number, id: any): any[] {
 }
 
 const warnedLeaseTimerStores = new WeakSet();
+/**
+ * Rate-limited rather than latched, for the reason `warnOnce` in the coordinator gives: a release
+ * that cannot be written leaves peers waiting out the lease, and a latch would show an operator the
+ * first occurrence and then hide the condition for as long as it lasts.
+ */
+const CLUSTER_RELEASE_WARN_INTERVAL_MS = 60_000;
+let lastClusterReleaseWarn = -Infinity;
+function warnClusterReleaseFailure(err: unknown) {
+	const now = performance.now();
+	if (now - lastClusterReleaseWarn < CLUSTER_RELEASE_WARN_INTERVAL_MS) return;
+	lastClusterReleaseWarn = now;
+	harperLogger.warn?.('cluster record lock release could not be written; peers will expire the hold', err);
+}
 
 class KeyLockHandle implements RecordLockHandle {
 	store: any;
@@ -115,9 +177,15 @@ class KeyLockHandle implements RecordLockHandle {
 	released = false;
 	expired = false;
 	acquiredAt: number;
+	clusterTsR: number | undefined;
 	#lastHolderVersion: number | undefined;
 	#candidateFloor: number | undefined;
 	#timer: ReturnType<typeof setTimeout> | undefined;
+	// Monotonic mirror of expiresAt. The wall-clock field stays public (callers and the cluster
+	// protocol both speak in timestamps) but the enforcement compares monotonic readings, so neither
+	// an NTP step nor a manual clock change can extend a lease past what peers assumed.
+	#deadlineMono: number | undefined;
+	#onRelease: (() => void) | undefined;
 
 	constructor(store: any, key: any[], keyId: unknown, lease?: number, hold = false, acquiredAt?: number) {
 		this.store = store;
@@ -127,6 +195,7 @@ class KeyLockHandle implements RecordLockHandle {
 		this.hold = hold;
 		this.acquiredAt = acquiredAt ?? store.getMonotonicTimestamp();
 		if (lease != null) {
+			this.#deadlineMono = performance.now() + lease;
 			this.#timer = setTimeout(() => this.#onLeaseExpire(), lease).unref();
 		}
 	}
@@ -150,7 +219,70 @@ class KeyLockHandle implements RecordLockHandle {
 		);
 	}
 
+	isExpired(): boolean {
+		return this.released || this.expired || this.#leaseLapsed();
+	}
+
+	isLeaseExpired(): boolean {
+		return this.expired || this.#leaseLapsed();
+	}
+
+	// The single place the deadline is evaluated, so the two predicates cannot disagree about it. A
+	// deliberate release does not make a lapsed lease valid again: by then peers have passed their own
+	// bound and the key may belong to someone else.
+	#leaseLapsed(): boolean {
+		if (this.#deadlineMono === undefined || performance.now() < this.#deadlineMono) return false;
+		// Finish the expiry rather than only reporting it: the native key must actually go back so a
+		// stalled holder does not keep a key every participant has already written off.
+		if (!this.released) this.#onLeaseExpire();
+		return true;
+	}
+
+	/**
+	 * Revoke this handle's authority immediately, ahead of its lease. A recall must stop a delegate
+	 * from COMMITTING, not merely from admitting: a caller that already staged a write and then called
+	 * `unlock()` has nothing outstanding for a drain to wait on, but its write is still in the
+	 * transaction and would land after the successor was admitted. Expiring the handle is what makes
+	 * the commit-time fence reject it (§6).
+	 */
+	revokeLease(): void {
+		if (this.expired) return;
+		// `expired` is what the commit-time fence reads, and it must be set even when the caller has
+		// ALREADY unlocked: a write staged before `unlock()` is still in its transaction, and that is
+		// precisely the handoff §6 exists to fence. Gating this on `released` made the revocation a
+		// no-op in the one state it was written for.
+		this.expired = true;
+		if (this.released) {
+			// The native key is already back and `onRelease` already fired; only the fence flag and the
+			// lease timer are left to settle.
+			clearTimeout(this.#timer);
+			return;
+		}
+		this.#onLeaseExpire();
+	}
+
+	joinClusterRound(tsR: number, leaseMs: number, mintedMono: number, onRelease: () => void): boolean {
+		// A round that completed after the native lease already fired yields no usable hold: the key
+		// has gone back and another worker may hold it, so every write through this handle would 409.
+		if (this.isLeaseExpired()) return false;
+		const remaining = leaseMs - (performance.now() - mintedMono);
+		if (remaining <= 0) return false;
+		this.clusterTsR = tsR;
+		this.acquiredAt = tsR;
+		this.#lastHolderVersion = undefined;
+		this.#onRelease = onRelease;
+		this.expiresAt = tsR + leaseMs;
+		// From the mint origin, not a second clock read: the design says this bound has no margin.
+		this.#deadlineMono = mintedMono + leaseMs;
+		clearTimeout(this.#timer);
+		this.#timer = setTimeout(() => this.#onLeaseExpire(), remaining).unref();
+		return true;
+	}
+
 	nextHolderVersion(): number {
+		// TODO(harper#2412): under the settled dual-clock model the lock-ordered stamp belongs in the
+		// distinct-version second word (HAS_DISTINCT_VERSION_FLAG), not the transaction timestamp.
+		// Until that lands, Phase 1 keeps this Phase 0 mechanism, anchored at ts_R in cluster mode.
 		const next = this.holderVersionCandidate();
 		this.#lastHolderVersion = next;
 		return next;
@@ -160,6 +292,7 @@ class KeyLockHandle implements RecordLockHandle {
 		if (this.released) return;
 		this.released = true;
 		this.expired = true;
+		clearTimeout(this.#timer);
 		try {
 			this.store.unlock(this.key);
 		} catch (err) {
@@ -168,6 +301,24 @@ class KeyLockHandle implements RecordLockHandle {
 				harperLogger.warn?.('record lock lease timer failed (store may have been dropped)', err);
 			}
 		}
+		this.#emitRelease();
+	}
+
+	// The cluster release is a durability optimization, not the safety mechanism — peers expire the
+	// hold on their own lease bound — so neither a throw nor a rejected promise may escape here. This
+	// runs from a lease timer and from post-commit cleanup, where an escaping rejection would take
+	// down the worker or turn a committed transaction into a 500.
+	#emitRelease() {
+		const onRelease = this.#onRelease;
+		if (!onRelease) return;
+		this.#onRelease = undefined;
+		try {
+			const result = onRelease() as unknown;
+			if (result && typeof (result as Promise<void>).then === 'function')
+				(result as Promise<void>).then(undefined, warnClusterReleaseFailure);
+		} catch (err) {
+			warnClusterReleaseFailure(err);
+		}
 	}
 
 	release(): boolean {
@@ -182,14 +333,29 @@ class KeyLockHandle implements RecordLockHandle {
 				harperLogger.warn?.('record lock release failed (store may have been dropped)', err);
 			}
 		}
+		this.#emitRelease();
 		return true;
 	}
 
 	upgradeToHold(lease: number): void {
 		this.hold = true;
 		clearTimeout(this.#timer);
-		this.expiresAt = Date.now() + lease;
-		this.#timer = setTimeout(() => this.#onLeaseExpire(), lease).unref();
+		// A granted cluster round may not be extended locally: peers bound the hold from the request
+		// they saw, and a longer local lease is exactly the two-holder window this protocol removes.
+		// Clamped on the monotonic deadline, not on expiresAt, so a backward wall-clock step cannot
+		// turn the upgrade into the extension this is here to refuse.
+		const now = performance.now();
+		const extended = now + lease;
+		const deadline = this.clusterTsR === undefined ? extended : Math.min(extended, this.#deadlineMono ?? extended);
+		const remaining = deadline - now;
+		this.#deadlineMono = deadline;
+		this.expiresAt = Date.now() + remaining;
+		this.#timer = setTimeout(() => this.#onLeaseExpire(), remaining).unref();
+		// Prime the nextHolderVersion counter so a later CLOSED-path save (e.g. after the
+		// enclosing transaction commits) gets acquiredAt+MIN_STEP rather than acquiredAt again
+		// — the same priming the fresh-hold path performs.  Without this, a post-commit save
+		// and any in-transaction save both get acquiredAt, causing a LWW tie that drops the write.
+		this.nextHolderVersion();
 	}
 }
 
@@ -219,7 +385,7 @@ export async function acquireRecordKey(
 	hold = false
 ): Promise<RecordLockHandle> {
 	const existing = txn.recordLockFor(store, keyId);
-	if (existing && !existing.released && !existing.expired) return existing;
+	if (existing && !existing.isExpired()) return existing;
 
 	const deadline = Date.now() + waitMs;
 	// One persistent wake slot: at most one onUnlocked callback is registered per waiter at any