@harperfast/harper 5.2.0-alpha.6 → 5.2.0-beta.2

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@harperfast/harper",
 	"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
-	"version": "5.2.0-alpha.6",
+	"version": "5.2.0-beta.2",
 	"license": "Apache-2.0",
 	"homepage": "https://harper.fast",
 	"bugs": {
@@ -167,14 +167,14 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.1012.0",
-		"@aws-sdk/lib-storage": "3.1080.0",
+		"@aws-sdk/lib-storage": "3.1086.0",
 		"@endo/static-module-record": "^1.1.2",
 		"@fastify/autoload": "^6.3.1",
 		"@fastify/compress": "^8.3.1",
 		"@fastify/cors": "^11.2.0",
 		"@fastify/static": "^9.1.3",
 		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^2.4.0",
+		"@harperfast/rocksdb-js": "^2.5.0",
 		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",

package/resources/DESIGN.md

@@ -78,22 +78,23 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource`
 
 ## "Where is X" cheat sheet
 
-| Question                                            | Where                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| How is a CRUD request authorized?                   | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| Where does versioning / conflict resolution happen? | `Table.ts → _writeUpdate` (`#section: write-path-internals`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| How does `search()` choose an index?                | `Table.ts → search` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| How are subscriptions replayed?                     | `Table.ts → subscribe` (`#section: pub-sub`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| How is the response body shaped (select clause)?    | `Table.ts → transformEntryForSelect` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| Where is record-level TTL evaluated?                | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| How are residencies enforced (replication)?         | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| How is the RecordObject prototype applied?          | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| Where is the per-request transaction stored?        | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| How does a query opt out of a read snapshot?        | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`).                                                                                                                                                                                                                                                                                                                                                                      |
-| How does a URL path map to a Resource?              | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| How does HNSW keep the graph connected on delete?   | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| How is a filter applied _during_ a vector search?   | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a record-scoped `allowRead` override into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering.                                                                                          |
-| How is row-level read access control enforced?      | Unified `allowRead` (#1422 gap 2 / #1241): an application-OVERRIDDEN `allowRead` (detected via the `isDefaultAllowRead` marker on the framework defaults) is record-scoped — evaluated once per record with `this` = the (frozen) record during query execution, fail-closed on throw, dispatched via the resolved method (never `record.allowRead` lookup — data shadowing). The authorize wrapper (`Resource.ts → authorizeActionOnResource`) defers collection reads on tables (`supportsRowLevelAllowRead`) to this per-record path; single-record `get` keeps the entry check (record loaded, proxied reads work). Records also expose a non-enumerable `allowRead` delegate on the per-table `structPrototype`. |
+| Question                                                              | Where                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| --------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| How is a CRUD request authorized?                                     | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| Where does versioning / conflict resolution happen?                   | `Table.ts → _writeUpdate` (`#section: write-path-internals`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| How does `search()` choose an index?                                  | `Table.ts → search` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| How are subscriptions replayed?                                       | `Table.ts → subscribe` (`#section: pub-sub`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| How is the response body shaped (select clause)?                      | `Table.ts → transformEntryForSelect` (`#section: search-query`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| Where is record-level TTL evaluated?                                  | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`). Stored expiry metadata is resolved in the `_writeUpdate` commit closure: `options.expiresAt ?? context.expiresAt ?? (record @expiresAt field, if finite & ≥ 0) ?? table default`. This metadata drives read-hiding + the cleanup sweep. The `@expiresAt` attribute is authoritative for **direct** put/patch only; cache/source fills persist via `recordUpdater` and derive expiry from `sourceContext.expiresAt` (source freshness / table default), not the field.                                                                                                                      |
+| Why does `search()` hide a row that's past its TTL but not yet swept? | `Table.ts → transformEntryForSelect` unconditionally treats `entry.expiresAt < Date.now()` as gone (lazy eviction on read) — correct for a SELECT, but a mutation locating rows to overwrite needs the opposite: pass `target.includeExpired = true` (read by the SQL engine's `runUpdate`/`runDelete` via `SqlEngineContext.includeExpiredRows`) to treat such a row as a live match, matching the leniency a direct by-id `put`/`patch` already has (they skip this check entirely, since `Resource.patch`'s static options don't request `ensureLoaded`).                                                                                                                                                          |
+| How are residencies enforced (replication)?                           | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| How is the RecordObject prototype applied?                            | `RecordEncoder.ts` (see `../DESIGN.md`)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| Where is the per-request transaction stored?                          | `transaction.ts` + `contextStorage` (AsyncLocalStorage)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| How does a query opt out of a read snapshot?                          | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`).                                                                                                                                                                                                                                                                                                                                                                      |
+| How does a URL path map to a Resource?                                | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| How does HNSW keep the graph connected on delete?                     | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| How is a filter applied _during_ a vector search?                     | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a record-scoped `allowRead` override into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering.                                                                                          |
+| How is row-level read access control enforced?                        | Unified `allowRead` (#1422 gap 2 / #1241): an application-OVERRIDDEN `allowRead` (detected via the `isDefaultAllowRead` marker on the framework defaults) is record-scoped — evaluated once per record with `this` = the (frozen) record during query execution, fail-closed on throw, dispatched via the resolved method (never `record.allowRead` lookup — data shadowing). The authorize wrapper (`Resource.ts → authorizeActionOnResource`) defers collection reads on tables (`supportsRowLevelAllowRead`) to this per-record path; single-record `get` keeps the entry check (record loaded, proxied reads work). Records also expose a non-enumerable `allowRead` delegate on the per-table `structPrototype`. |
 
 ---
 

package/resources/DatabaseTransaction.ts

@@ -27,6 +27,80 @@ const MAX_RETRIES = 40;
 // cap (see the commit rejection handler), don't grow the delay unbounded.
 const MAX_RETRY_DELAY_MS = 1000;
 let outstandingCommit, outstandingCommitStart;
+
+// The analytics module registers a recorder here at load (dependency inversion, mirroring
+// `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
+// modules. Unset until analytics loads, and when analytics is disabled the recorder call is cheap.
+let recordCommitLatencyMs: ((durationMs: number) => void) | undefined;
+export function setCommitLatencyRecorder(recorder: ((durationMs: number) => void) | undefined) {
+	recordCommitLatencyMs = recorder;
+}
+
+// Emit the submit→settle duration of a write commit as the `transaction-commit-time` distribution
+// metric. Recorded on both fulfilment and rejection since a slow-then-failed commit still consumed
+// queue time. The recorder is wrapped so it can never throw — a metrics failure must neither break the
+// commit nor surface as an unhandled rejection on this floating `.then`. The thenable guard protects
+// against a future caller passing a non-Promise `commitResolution` (today it is always the rocksdb-js
+// async `Transaction.commit()` result, which is guaranteed to be a Promise).
+function recordCommitLatency(commitResolution: Promise<void>, submittedAt: number) {
+	if (!recordCommitLatencyMs) return;
+	const record = () => {
+		try {
+			recordCommitLatencyMs(performance.now() - submittedAt);
+		} catch {
+			// analytics recording is best-effort and must never disturb the commit path
+		}
+	};
+	if (commitResolution && typeof (commitResolution as any).then === 'function') {
+		commitResolution.then(record, record);
+	}
+}
+
+// Queue-depth gauges surfaced through the analytics pipeline (write-transaction-queue-depth /
+// read-transaction-queue-depth). Per-thread state; the analytics aggregator sums across threads.
+// `writeTxnQueueDepth` counts write commits handed to the storage engine but not yet resolved —
+// this is the backlog that, when it drains too slowly, produces the "Outstanding write transactions
+// have too long of queue" overload error. Read depth is derived from the live `trackedTxns` set
+// (every tracked transaction holds an open read snapshot). We also retain a high-water mark per
+// sampling window because the queue can fill and drain within a single (~1s) analytics period, so an
+// instantaneous sample taken at emit time would routinely miss the spike operators need to see.
+// RocksDB-write-path only: LMDB routes through the separate LMDBTransaction.commit()/getReadTxn()
+// overrides (resources/LMDBTransaction.ts), which maintain their own unrelated `trackedTxns` set and
+// do not call into this accounting.
+let writeTxnQueueDepth = 0;
+let writeTxnQueueDepthHighWater = 0;
+let readTxnQueueDepthHighWater = 0;
+
+function enterWriteQueue() {
+	if (++writeTxnQueueDepth > writeTxnQueueDepthHighWater) writeTxnQueueDepthHighWater = writeTxnQueueDepth;
+}
+function leaveWriteQueue() {
+	// Floor at zero: accounting is balanced by construction (every enterWriteQueue has exactly one
+	// matching settlement), but the guard is cheap insurance against a future call-site imbalance
+	// producing a negative depth that would corrupt every subsequent sample.
+	if (writeTxnQueueDepth > 0) writeTxnQueueDepth--;
+}
+
+/**
+ * Returns the current write/read transaction queue depths for this thread along with the high-water
+ * mark observed since the previous call, then resets the high-water marks to the current depth so the
+ * next sampling window starts fresh. Consumed by the analytics writer (see analytics/write.ts).
+ */
+export function getTransactionQueueDepths() {
+	// `readTxnQueueDepthHighWater` is maintained at the single trackedTxns growth site, so it already
+	// dominates the current size here — no need to reconcile against `readDepth` before reporting.
+	const readDepth = trackedTxns.size;
+	const depths = {
+		writeDepth: writeTxnQueueDepth,
+		writeMaxDepth: writeTxnQueueDepthHighWater,
+		readDepth,
+		readMaxDepth: readTxnQueueDepthHighWater,
+	};
+	writeTxnQueueDepthHighWater = writeTxnQueueDepth;
+	readTxnQueueDepthHighWater = readDepth;
+	return depths;
+}
+
 let confirmReplication;
 export function replicationConfirmation(callback) {
 	confirmReplication = callback;
@@ -161,6 +235,7 @@ export class DatabaseTransaction implements Transaction {
 		}
 		if ((this.transaction as any).openTimer) (this.transaction as any).openTimer = 0;
 		trackedTxns.add(this);
+		if (trackedTxns.size > readTxnQueueDepthHighWater) readTxnQueueDepthHighWater = trackedTxns.size;
 		return this.transaction;
 	}
 
@@ -329,6 +404,26 @@ export class DatabaseTransaction implements Transaction {
 							// cast to Promise<void> is safe — the sentinel never propagates
 							// past that branch.
 							commitResolution = transaction.commit() as Promise<void>;
+							// Record how long this commit stays outstanding (submit → settle) as a distribution
+							// metric. This is the same clock the overload check uses (outstandingCommitStart is
+							// stamped at submit), so a rising p99/p999 is the leading indicator for the
+							// "Outstanding write transactions have too long of queue" (503) rejection. A transient-
+							// conflict retry rejects this promise and issues a fresh commit(), and outstandingCommit
+							// re-arms per attempt, so recording per attempt matches the overload semantics.
+							recordCommitLatency(commitResolution, performance.now());
+							// Count this commit against the write queue depth until the storage engine
+							// resolves it. A transient-conflict retry rejects this promise and issues a
+							// fresh commit() (re-entering here), so the enter/leave stays balanced. leaveWriteQueue
+							// never throws, so the settled promise resolves and needs no rejection handling of its own.
+							// The thenable guard protects against a future caller passing a non-Promise
+							// `commitResolution` (today it is always rocksdb-js's async Transaction.commit()
+							// result, guaranteed to be a Promise).
+							enterWriteQueue();
+							if (commitResolution && typeof (commitResolution as any).then === 'function') {
+								commitResolution.then(leaveWriteQueue, leaveWriteQueue);
+							} else {
+								leaveWriteQueue();
+							}
 						} else {
 							try {
 								commitResolution = transaction.abort();

package/resources/RequestTarget.ts

@@ -111,17 +111,26 @@ export class RequestTarget extends URLSearchParams {
 		if (path) {
 			// parse for properties and set the id
 			if (path.startsWith('/')) path = path.substring(1);
-		} else {
-			return; // leave this.id undefined
+		} else if (target === undefined) {
+			return; // constructed with no target at all (internal use) — leave id/isCollection unset
 		}
 		if (path) {
 			if (path.endsWith('/')) {
 				this.isCollection = true;
 			}
 			this.id = decodeURIComponent(path);
-		} else {
+		} else if (this.pathname === '/') {
+			// a bare trailing slash is the documented way to address a resource's collection
 			this.isCollection = true;
 			this.id = null;
+		} else {
+			// an exact resource-path match with nothing left to parse and no trailing slash
+			// (e.g. `/redirects` instead of the required `/redirects/`, harper#678): this is
+			// neither a valid collection request nor a specific record, so id/isCollection must
+			// still be well-defined (never left `undefined`) for dispatch to reject it cleanly
+			// instead of letting downstream code assume one of them is always set.
+			this.isCollection = false;
+			this.id = null;
 		}
 	}
 	toString() {

package/resources/Resource.ts

@@ -689,8 +689,11 @@ function transactional(
 				query = new RequestTarget();
 				query.id = id;
 				if (id == null) {
-					if (options.method === 'get') {
-						throw new Error(`Using an argument with a value of ${id} for ${options.method}, is not allowed`);
+					if (options.method === 'get' || options.method === 'delete') {
+						// For delete, a bare null/undefined id would be coerced into a whole-collection
+						// target and silently delete every record (studio#1199); a deliberate delete-all
+						// must use an explicit query object instead.
+						throw new ClientError(`Using an argument with a value of ${id} for ${options.method}, is not allowed`);
 					}
 					query.isCollection = true;
 				}
@@ -701,6 +704,24 @@ function transactional(
 			query.id = id;
 		}
 		isCollection = query.isCollection;
+		if (
+			options.method === 'post' &&
+			query.id === null &&
+			!isCollection &&
+			this.prototype.post === Resource.prototype.post
+		) {
+			// the matched path had nothing left to resolve into a collection or a specific record —
+			// i.e. it exactly matched a resource's base path without the required trailing slash
+			// (harper#678). This only matters for the base/default post() dispatch: it reads
+			// this.#isCollection (set from this same query) and falls back to missingMethod() for
+			// this state anyway, so rejecting early here just gives a clearer, purpose-built message.
+			// A resource with its own post() override (e.g. a component doing a bulk import via
+			// POST to its collection root, like the redirector template's Redirect.post()) is
+			// trusted to handle a null-id/non-collection target itself — it may not use id/isCollection
+			// at all, and forcing the trailing slash on it would break a currently-supported no-slash
+			// bulk-POST convention. See harper#678's regression on PR #1807.
+			throw new ClientError(`A trailing slash is required to POST to the ${this.name} collection`, 404);
+		}
 		let resourceOptions;
 		if (!context) {
 			// try to get the context from the async context if possible

package/resources/Table.ts

@@ -1244,7 +1244,7 @@ export function makeTable(options) {
 				// go back to the static search method so it gets a chance to override
 				return constructor.search(target, this.getContext());
 			}
-			if (target && target.id === undefined && !target.toString()) {
+			if (target && target.id == null && !target.toString()) {
 				const description = {
 					// basically a describe call
 					records: './', // an href to the records themselves
@@ -2081,8 +2081,10 @@ export function makeTable(options) {
 					const type = fullUpdate ? 'put' : 'patch';
 					let residencyId: number | undefined;
 					if (options?.residencyId != undefined) residencyId = options.residencyId;
-					const expiresAt: number =
-						options?.expiresAt ?? context?.expiresAt ?? (expirationMs ? expirationMs + Date.now() : -1);
+					// options/context expiresAt are the most specific overrides; a record @expiresAt field
+					// (resolved below, once recordToStore is merged) overrides the table default in both
+					// directions; the table default is the final fallback. -1 means no expiration.
+					let expiresAt: number | undefined = options?.expiresAt ?? context?.expiresAt;
 					const additionalAuditRefs: Array<{ version: number; nodeId: number }> = []; // track additional audit refs to store
 					// Bulk base-copy snapshot apply: store current-state directly with no audit/transaction-log entry
 					// and no out-of-order resequencing/dedup (the source of the O(n) keyed-lookup spin in
@@ -2499,6 +2501,35 @@ export function makeTable(options) {
 						}
 						residencyId = getResidencyId(residency);
 					}
+					if (expiresAt == undefined) {
+						// A schema @expiresAt attribute makes the record field authoritative over the table
+						// default, in both directions: stamp it into the stored expiry metadata that governs
+						// read-hiding and the cleanup sweep, not just the separate index-pruning sweep (which
+						// only removes already-past records and so can never extend past the table default).
+						// Read from recordToStore so the metadata matches exactly what the pruning sweep later
+						// reads back. Falls back to the table default when the field is unset or not a timestamp.
+						const fieldExpiresAt = expiresAtProperty ? recordToStore?.[expiresAtProperty.name] : undefined;
+						// Coerce only genuine timestamp shapes: a number/bigint epoch, a Date, or a numeric/ISO
+						// string. Booleans, empty/whitespace strings, and null/undefined fall through to NaN so a
+						// nonsensical field value uses the table default rather than expiring the record at epoch 0.
+						let fieldExpiresAtMs = NaN;
+						if (typeof fieldExpiresAt === 'number' || typeof fieldExpiresAt === 'bigint')
+							fieldExpiresAtMs = Number(fieldExpiresAt);
+						else if (fieldExpiresAt instanceof Date) fieldExpiresAtMs = fieldExpiresAt.getTime();
+						else if (typeof fieldExpiresAt === 'string' && fieldExpiresAt.trim() !== '') {
+							const numeric = Number(fieldExpiresAt);
+							fieldExpiresAtMs = Number.isFinite(numeric) ? numeric : Date.parse(fieldExpiresAt);
+						}
+						// Only a finite, non-negative epoch counts: negatives collide with the -1 "no expiration"
+						// sentinel (the encoder omits HAS_EXPIRATION for <0, but the field sweep would still evict a
+						// negative field value), so treat a negative/NaN field as unset and use the table default.
+						expiresAt =
+							Number.isFinite(fieldExpiresAtMs) && fieldExpiresAtMs >= 0
+								? fieldExpiresAtMs
+								: expirationMs
+									? expirationMs + Date.now()
+									: -1;
+					}
 					if (!fullUpdate) {
 						// we use our own data as the basis for the audit record, which will include information about the incremental updates, even if it was overwritten by CRDT resolution
 						auditRecordToStore = recordUpdate;
@@ -2989,6 +3020,14 @@ export function makeTable(options) {
 			// there, after materialization (the earlier evaluation stays as a prune that also bounds HNSW
 			// traversal). vectorFilter and condition filters intentionally keep the local-record
 			// semantics all query filters have on caching tables.
+			//
+			// A row that is past its TTL but not yet swept by the background eviction
+			// scan is still physically present. A write that is about to overwrite it
+			// anyway (e.g. the SQL engine locating UPDATE/DELETE targets) needs to see
+			// it as a match — the same leniency a direct by-id put/patch already gets,
+			// since those never run the ensureLoaded-gated freshness check this transform
+			// otherwise applies unconditionally to every read.
+			const includeExpired = (target as any).includeExpired === true;
 			const transformToRecord = TableResource.transformEntryForSelect(
 				select,
 				context,
@@ -2996,7 +3035,8 @@ export function makeTable(options) {
 				filtered,
 				ensure_loaded,
 				true,
-				recordGuard
+				recordGuard,
+				includeExpired
 			);
 			let results = TableResource.transformToOrderedSelect(
 				entries,
@@ -3210,9 +3250,20 @@ export function makeTable(options) {
 		 * @param recordGuard record-level read check (#1241) applied to the record actually being
 		 * returned — i.e. AFTER any caching-source revalidation replaces a stale local copy — so an
 		 * authorization verdict can't be made on bytes that differ from what the caller receives.
+		 * @param includeExpired when true, a row past its TTL but not yet swept is treated as a live
+		 * match rather than gone (used by the SQL engine's UPDATE/DELETE row-finder).
 		 * @returns
 		 */
-		static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded?, canSkip?, recordGuard?) {
+		static transformEntryForSelect(
+			select,
+			context,
+			readTxn,
+			filtered,
+			ensure_loaded?,
+			canSkip?,
+			recordGuard?,
+			includeExpired?
+		) {
 			let checkLoaded;
 			if (
 				ensure_loaded &&
@@ -3260,7 +3311,7 @@ export function makeTable(options) {
 					}
 					if (
 						(checkLoaded && entry?.metadataFlags & (INVALIDATED | EVICTED)) || // invalidated or evicted should go to load from source
-						(entry?.expiresAt != undefined && entry?.expiresAt < Date.now())
+						(!includeExpired && entry?.expiresAt != undefined && entry?.expiresAt < Date.now())
 					) {
 						// should expiration really apply?
 						if (context.onlyIfCached) {

package/resources/analytics/metadata.ts

@@ -8,6 +8,9 @@ export const METRIC = {
 	NODE_STORAGE: 'node-storage',
 	ROCKSDB_STATS: 'rocksdb-stats',
 	ROCKSDB_TXNLOG_STATS: 'rocksdb-txnlog-stats',
+	TRANSACTION_COMMIT_TIME: 'transaction-commit-time',
+	WRITE_TRANSACTION_QUEUE_DEPTH: 'write-transaction-queue-depth',
+	READ_TRANSACTION_QUEUE_DEPTH: 'read-transaction-queue-depth',
 } as const;
 
 export type BuiltInMetricName = (typeof METRIC)[keyof typeof METRIC];

package/resources/analytics/write.ts

@@ -14,6 +14,7 @@ import { server } from '../../server/Server.ts';
 import * as fs from 'node:fs';
 import { getAnalyticsHostnameTable, nodeIds, stableNodeId } from './hostnames.ts';
 import { METRIC } from './metadata.ts';
+import { getTransactionQueueDepths, setCommitLatencyRecorder } from '../DatabaseTransaction.ts';
 import { RocksDatabase, type TransactionLogStats } from '@harperfast/rocksdb-js';
 
 const log = forComponent('analytics').conditional;
@@ -130,6 +131,9 @@ export function recordAction(value: Value, metric: string, path?: string, method
 
 server.recordAnalytics = recordAction;
 
+// Let the storage layer emit write-commit latency without statically depending on this module.
+setCommitLatencyRecorder((durationMs) => recordAction(durationMs, METRIC.TRANSACTION_COMMIT_TIME));
+
 export function recordActionBinary(value, metric, path?, method?, type?) {
 	recordAction(Boolean(value), metric, path, method, type);
 }
@@ -213,6 +217,25 @@ function sendAnalytics() {
 			byThread: true,
 			...memoryUsage,
 		});
+		// Transaction queue depth gauges. `depth` is the instantaneous depth at emit time; `maxDepth` is
+		// the high-water mark over this sampling window (the queue can fill and drain within a single
+		// period, so the instantaneous sample alone would miss short spikes). Reported per-thread and
+		// summed across threads by the aggregator, mirroring the `memory` gauge above.
+		const queueDepths = getTransactionQueueDepths();
+		metrics.push({
+			metric: METRIC.WRITE_TRANSACTION_QUEUE_DEPTH,
+			threadId,
+			byThread: true,
+			depth: queueDepths.writeDepth,
+			maxDepth: queueDepths.writeMaxDepth,
+		});
+		metrics.push({
+			metric: METRIC.READ_TRANSACTION_QUEUE_DEPTH,
+			threadId,
+			byThread: true,
+			depth: queueDepths.readDepth,
+			maxDepth: queueDepths.readMaxDepth,
+		});
 		for (const listener of analyticsListeners) {
 			listener(metrics);
 		}

package/resources/databases.ts

@@ -13,6 +13,7 @@ import { makeTable } from './Table.ts';
 import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts';
 import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
 import { getConfigPath } from '../config/configUtils.ts';
+import { ClientError } from '../utility/errors/hdbError.ts';
 import { _assignPackageExport } from '../globals.js';
 import { getIndexedValues } from '../utility/lmdb/commonUtility.ts';
 import * as signalling from '../utility/signalling.ts';
@@ -1120,6 +1121,29 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 		if (Table.primaryStore.rootStore.status === 'closed') {
 			throw new Error(`Can not use a closed data store from ${tableName} class`);
 		}
+		// Reject moving the primary key to a different attribute on a table that already has records.
+		// The storage key (Table.primaryKey) is never re-pointed here, so honoring the change would
+		// leave describe reporting the new attribute while every record — old and newly inserted — stays
+		// keyed by the original one; search_by_id/update/delete by the declared key then all miss. Only
+		// schema-authored callers (@table / defineTable / create_table) reassert the declaration, so
+		// gate on schemaDefinedExplicit to leave cluster schema-replication / data-loader callers alone.
+		// See HarperFast/studio#1199.
+		const declaredPrimaryKey = attributes.find((attribute) => attribute.isPrimaryKey)?.name;
+		if (schemaDefinedExplicit && declaredPrimaryKey && declaredPrimaryKey !== Table.primaryKey) {
+			let hasRecords = false;
+			for (const _entry of Table.primaryStore.getRange({ start: true })) {
+				hasRecords = true;
+				break;
+			}
+			if (hasRecords) {
+				throw new ClientError(
+					`Cannot change the primary key of table '${databaseName}.${tableName}' from '${Table.primaryKey}' to ` +
+						`'${declaredPrimaryKey}' because it already contains records. Recreate the table with the new primary ` +
+						`key, or migrate the existing records.`,
+					400
+				);
+			}
+		}
 		// it table already exists, get the split segments setting
 		if (splitSegments == undefined) splitSegments = Table.splitSegments;
 		Table.attributes.splice(0, Table.attributes.length, ...attributes);

package/resources/defineResource.ts

@@ -550,12 +550,25 @@ function applyContractMetadata(carrier: any, contract: Contract): void {
  */
 function wrapStaticVerb(original: Function, compiled: CompiledVerb): Function {
 	const { queryFragment, bodyFragment, hasBody } = compiled;
+	function validateAndDispatch(this: any, target: any, rest: any[], body: any) {
+		const issues: ValidationIssue[] = [];
+		if (queryFragment) coerceAndValidateQuery(queryFragment, target, issues);
+		if (hasBody && bodyFragment) rest[0] = validateBody(bodyFragment, body, issues);
+		if (issues.length) throw new ValidationError(issues);
+		return original.call(this, target, ...rest);
+	}
 	return function (this: any, target: any, ...rest: any[]) {
 		if (target instanceof URLSearchParams) {
-			const issues: ValidationIssue[] = [];
-			if (queryFragment) coerceAndValidateQuery(queryFragment, target, issues);
-			if (hasBody && bodyFragment) rest[0] = validateBody(bodyFragment, rest[0], issues);
-			if (issues.length) throw new ValidationError(issues);
+			const body = rest[0];
+			// REST.ts's streaming deserializer hands the body over as a still-pending promise
+			// (`request.data`); await it before validating, or it reads as an empty object and every
+			// declared-required field is (incorrectly) reported missing. A plain value (as passed by a
+			// programmatic call or a unit test) validates synchronously, preserving the synchronous-throw
+			// contract those callers rely on.
+			if (hasBody && bodyFragment && body && typeof body.then === 'function') {
+				return body.then((resolvedBody: any) => validateAndDispatch.call(this, target, rest, resolvedBody));
+			}
+			return validateAndDispatch.call(this, target, rest, body);
 		}
 		return original.call(this, target, ...rest);
 	};

package/resources/jsResource.ts

@@ -71,9 +71,39 @@ export class ResourceLoadError extends Error {
  *
  * Thus, this plugin only handle files as they are added (`add` event). All other events result in a restart request.
  *
+ * A redeploy tears down and reinstalls the component's files while this scope's watcher is paused
+ * (see `Scope`/`EntryHandler` deploy lifecycle); on resume the fresh chokidar scan re-emits every
+ * existing file as `'add'` — including ones whose contents just changed. Treating those as plain
+ * adds would silently re-run against the stale module cache and never flag a restart (harper#1817).
+ * So we track which files this scope has already loaded: a re-`add` of a known file is a redeploy of
+ * loaded code we cannot hot-swap, and is handled like a `change` — request a restart. A first-time
+ * `add` (initial load, or a genuinely new file added at runtime) still loads without a restart.
+ *
+ * A redeploy that *deletes* a loaded file is a different shape of the same problem: the fresh
+ * chokidar scan only reports what's currently on disk, so a file that's gone produces no event at
+ * all — no re-`add`, no `unlink` — and the modified-file handling above never sees it. Left
+ * unhandled, the deleted resource stays registered and active in memory (harper#1817 follow-up). So
+ * we also track which files the post-redeploy scan pass reports, and once that scan's `ready` fires,
+ * diff it against everything this scope has ever loaded: anything missing was deleted, and is
+ * handled the same way as a modified file — request a restart.
+ *
+ * That diff must only run for an actual redeploy rescan, not every time `EntryHandler` emits
+ * `ready` — it also refires after each ordinary runtime add/change once that file's read settles
+ * (its initial-scan-complete latch never resets outside a full rescan), and diffing against that
+ * would falsely treat every other already-loaded file as deleted. So the diff window is gated by
+ * the scope's own `deploy:start`/`deploy:end` bracket (see `Scope`): `deploy:start` pauses the
+ * watcher and opens the window (and is where we reset the scan-file tracking, since no file events
+ * can land while paused), and the first `ready` afterward — the resumed watcher's fresh scan
+ * completing — closes it and runs the diff.
  */
 export async function handleApplication(scope: Scope) {
-	scope.handleEntry(async function handleResourceEntry(entryEvent) {
+	const loadedResourceFiles = new Set<string>();
+	// Files reported as `add` since the most recent `deploy:start`, populated only while
+	// `awaitingPostRedeployScan` is true — see the gating note above.
+	let currentScanFiles = new Set<string>();
+	let awaitingPostRedeployScan = false;
+
+	const entryHandler = scope.handleEntry(async function handleResourceEntry(entryEvent) {
 		if (entryEvent.entryType !== 'file') {
 			scope.logger.warn(
 				`jsResource plugin cannot handle entry type ${entryEvent.entryType}. Modify the 'files' option in ${scope.configFilePath} to only include files.`
@@ -81,7 +111,13 @@ export async function handleApplication(scope: Scope) {
 			return;
 		}
 
-		if (entryEvent.eventType !== 'add') {
+		if (awaitingPostRedeployScan && entryEvent.eventType === 'add') {
+			// Recorded unconditionally — before the loaded/re-add branch below — so the post-scan
+			// deletion diff sees every file this scan reported, whether newly loaded or already known.
+			currentScanFiles.add(entryEvent.absolutePath);
+		}
+
+		if (entryEvent.eventType !== 'add' || loadedResourceFiles.has(entryEvent.absolutePath)) {
 			scope.requestRestart();
 			return;
 		}
@@ -97,6 +133,9 @@ export async function handleApplication(scope: Scope) {
 				scope.logger.debug?.(`Registered root resource: ${path}`);
 			}
 			recurseForResources(scope, resourceModule, root);
+			// Record the load so a later re-`add` of this same file (a redeploy re-scan) is treated
+			// as a change and requests a restart rather than silently re-serving stale cached code.
+			loadedResourceFiles.add(entryEvent.absolutePath);
 			// A JS resource that extends an exported @table is the one carrying author opt-ins
 			// (`static mcpTools`/`mcpPrompts`), and it registers here — after the schema-derived
 			// table class and after the MCP component's boot scan. Signal so listing surfaces
@@ -107,6 +146,26 @@ export async function handleApplication(scope: Scope) {
 			throw new ResourceLoadError(entryEvent.absolutePath, error);
 		}
 	});
+
+	// Optional chaining: a mock/test scope may not implement EventEmitter, and Scope#handleEntry
+	// itself can return undefined (e.g. MissingDefaultFilesOptionError). In real use `scope` is
+	// always an EventEmitter and `entryHandler` is always the EntryHandler backing this watcher.
+	scope.on?.('deploy:start', () => {
+		awaitingPostRedeployScan = true;
+		currentScanFiles = new Set();
+	});
+
+	entryHandler?.on?.('ready', () => {
+		if (!awaitingPostRedeployScan) return;
+		awaitingPostRedeployScan = false;
+		for (const loadedFile of loadedResourceFiles) {
+			if (!currentScanFiles.has(loadedFile)) {
+				// Known file that the just-completed scan never reported — deleted during the redeploy.
+				loadedResourceFiles.delete(loadedFile);
+				scope.requestRestart();
+			}
+		}
+	});
 }
 
 function recurseForResources(scope: Scope, resourceModule: any, prefix: string) {