@harperfast/harper 5.3.0-beta.1 → 5.3.0-beta.2

package/resources/RecordEncoder.ts

@@ -163,6 +163,10 @@ export class RecordObject {
 		return entryMap.get(this)?.expiresAt;
 	}
 }
+// msgpackr's RESERVE_START_SPACE option carries the reserved byte count in the LOW BYTE of the same
+// word as its flags (`encodeOptions & 0xff`, msgpackr `pack`), so the metadata prefix this encoder
+// reserves cannot exceed it.
+const MAX_RESERVED_START_SPACE = 255;
 export let lastValueEncoding: Buffer | undefined;
 let timestampNextEncoding = 0,
 	metadataInNextEncoding = -1,
@@ -355,6 +359,7 @@ export class RecordEncoder extends StructonEncoder {
 				const residencyId = residencyIdAtNextEncoding;
 				const nodeId = nodeIdAtNextEncoding;
 				const additionalAuditRefs = additionalAuditRefsNextEncoding;
+				let auditRefsToEncode = 0;
 				if (metadata >= 0) {
 					valueStart += 4; // make room for metadata bytes
 					metadataInNextEncoding = -1; // reset indicator to mean no metadata
@@ -380,7 +385,18 @@ export class RecordEncoder extends StructonEncoder {
 						}
 					}
 					if (additionalAuditRefs && additionalAuditRefs.length > 0) {
-						valueStart += 1 + additionalAuditRefs.length * 12; // 1 byte for count + 8 bytes version + 4 bytes nodeId per ref
+						// `valueStart` is OR'd into msgpackr's encode options below, where the reserved-space field
+						// is the LOW BYTE (RESERVE_START_SPACE): a valueStart over 255 wraps, so the record is
+						// written with a prefix neither side agrees on and decodes as garbage. The persisted ref
+						// count is one byte too. Refs are best-effort — a later in-order write drops them outright —
+						// so over budget the MIDDLE is dropped: the first entry is the addressable audit head when
+						// the record and log clocks diverge, and the last is the identity this write is matched on
+						// by the re-delivery guard, so neither end may go. (harper#2642)
+						auditRefsToEncode = Math.min(
+							additionalAuditRefs.length,
+							Math.floor((MAX_RESERVED_START_SPACE - valueStart - 1) / 12)
+						);
+						valueStart += 1 + auditRefsToEncode * 12; // 1 byte for count + 8 bytes version + 4 bytes nodeId per ref
 						additionalAuditRefsNextEncoding = undefined;
 					}
 				}
@@ -402,7 +418,7 @@ export class RecordEncoder extends StructonEncoder {
 					position += 8;
 				}
 				if (blobsWereEncoded) metadata |= HAS_BLOBS;
-				if (additionalAuditRefs && additionalAuditRefs.length > 0) metadata |= HAS_ADDITIONAL_AUDIT_REFS;
+				if (auditRefsToEncode > 0) metadata |= HAS_ADDITIONAL_AUDIT_REFS;
 				if (metadata >= 0) {
 					dataView.setUint32(position, metadata | (ACTION_32_BIT << 24)); // use the extended action byte
 					position += 4;
@@ -418,9 +434,15 @@ export class RecordEncoder extends StructonEncoder {
 						dataView.setUint32(position, nodeId);
 						position += 4;
 					}
-					if (additionalAuditRefs && additionalAuditRefs.length > 0) {
-						encoded[position++] = additionalAuditRefs.length;
-						for (const ref of additionalAuditRefs) {
+					if (auditRefsToEncode > 0) {
+						encoded[position++] = auditRefsToEncode;
+						// Written as [0, …tail]: the dropped window is the middle, and it is empty when nothing
+						// had to be dropped.
+						const dropFrom = 1;
+						const dropTo = additionalAuditRefs.length - auditRefsToEncode + 1;
+						for (let index = 0; index < additionalAuditRefs.length; index++) {
+							if (index >= dropFrom && index < dropTo) continue;
+							const ref = additionalAuditRefs[index];
 							dataView.setFloat64(position, ref.version);
 							position += 8;
 							dataView.setUint32(position, ref.nodeId);

package/resources/Table.ts

@@ -108,6 +108,7 @@ import {
 	LOCAL_ONLY,
 	auditRetention,
 	removeAuditEntry,
+	getAuditFloor,
 	raiseAuditFloor,
 	boundedAuditPruneEnd,
 	isLockControlType,
@@ -135,7 +136,7 @@ import {
 	storedFieldsOnly,
 } from './RecordEncoder.ts';
 import { recordAction, recordActionBinary } from './analytics/write.ts';
-import { rebuildUpdateBefore } from './crdt.ts';
+import { commutativeOpsOf, rebuildUpdateBefore } from './crdt.ts';
 import { appendHeader } from '../server/serverHelpers/Headers.ts';
 import fs from 'node:fs';
 import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts';
@@ -3642,6 +3643,8 @@ export function makeTable(options) {
 						// existing timestamp, which means that we received updates out of order, and must resequence the application
 						// of the updates to the record to ensure consistency across the cluster
 						// TODO: can the previous version be older, but even more previous version be newer?
+						let belowAuditFloor = false;
+						let dedupVersionCouldBeRetained: (version: number) => boolean;
 						if (audit) {
 							// A re-delivered out-of-order write (full-copy audit-replay re-delivers writes) must not have
 							// its commutative ops re-folded. additionalAuditRefs is the record's own list of folded
@@ -3687,7 +3690,7 @@ export function makeTable(options) {
 							// Resolve the oldest retained entry once, for the same log the dedup reads.
 							let oldestRetainedAuditTime: number | undefined;
 							let oldestRetainedAuditTimeResolved = false;
-							const dedupVersionCouldBeRetained = (version: number): boolean => {
+							dedupVersionCouldBeRetained = (version: number): boolean => {
 								if (!isRocksDB) return true; // LMDB keeps its exact, unbounded lookup (keyed by local audit time)
 								if (!oldestRetainedAuditTimeResolved) {
 									oldestRetainedAuditTimeResolved = true;
@@ -3734,6 +3737,19 @@ export function makeTable(options) {
 									return; // duplicate already applied; avoid the resequencing walk
 								}
 							}
+							// The walk terminates at this write only by reaching an audit entry whose log key is at or below
+							// txnTime (the loop condition below), so below the floor it cannot: it runs the whole retained
+							// chain to an outcome the floor already determines (harper#2642). txnTime is the coordinate
+							// because it is the one that loop compares; txnLogKey addresses this write's own entry, a
+							// different question that dedupVersionCouldBeRetained already answers. An unknown floor is
+							// Infinity, which fails closed for a cursor check but has to fail OPEN here — walk rather than
+							// discard. RocksDB only: LMDB keeps its exact, unbounded reconciliation.
+							if (isRocksDB && precedesExisting < 0) {
+								const auditFloor = getAuditFloor(auditStore);
+								belowAuditFloor = Number.isFinite(auditFloor) && txnTime < auditFloor;
+							}
+						}
+						if (audit && !belowAuditFloor) {
 							// incremental CRDT updates are only available with audit logging on
 							const initialAuditHead = isRocksDB
 								? resolveAuditHead(id, existingEntry.version, existingEntry.nodeId, existingEntry.additionalAuditRefs)
@@ -3751,7 +3767,12 @@ export function makeTable(options) {
 								new Date(localTime)
 							);
 
-							let nodeId = initialAuditHead.nodeId;
+							// Normalized here and not at the lookup, because the two sources of an undefined nodeId do not
+							// mean the same thing: a record's own nodeId is resolved by the same expression as the id its
+							// audit entry is logged under, which applies `?? 0` (RecordEncoder), so absent means log 0;
+							// `previousNodeId` is never encoded, so absent there means the log is unknown and only the
+							// aggregate lookup can resolve a cross-origin predecessor.
+							let nodeId = initialAuditHead.nodeId ?? 0;
 							const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
 							const auditRefsToVisit: Array<{ localTime: number; nodeId: number }> = existingEntry.additionalAuditRefs
 								? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
@@ -4015,6 +4036,31 @@ export function makeTable(options) {
 								write.skipped = true;
 								return;
 							}
+						} else if (belowAuditFloor) {
+							// The walk cannot reach this write, so it may contribute only what is order-independent: its
+							// commutative ops. Whether a plain field survives depends on what newer writes did to that key,
+							// which is what the purged history no longer answers, so applying one would resurrect state a
+							// newer put may have erased. Everything else loses to the strictly newer head: a full update, a
+							// head carrying no record (a delete or a residency-omitted record, which an op must not
+							// resurrect), and an op-less patch. Bare return, no writeCommit, so no audit record references
+							// the losing update's pre-saved blobs.
+							incrementalUpdateToApply = fullUpdate || existingRecord == null ? null : commutativeOpsOf(recordUpdate);
+							if (!incrementalUpdateToApply) {
+								write.skipped = true;
+								return;
+							}
+							// The surviving head's addressable log-key pointer lives in these, wherever the record and log
+							// clocks differ.
+							if (existingEntry.additionalAuditRefs) {
+								for (const ref of existingEntry.additionalAuditRefs) {
+									additionalAuditRefs.push(ref);
+								}
+							}
+							// Once the walk no longer runs, this ref is what the read-your-writes check above matches a
+							// re-delivery of these ops on. Best-effort, like every other guard here: the encoder bounds
+							// the persisted list, so an identity can age out of it (harper#1148's full-copy convergence
+							// is the backstop).
+							additionalAuditRefs.push({ version: txnLogKey, nodeId: options?.nodeId });
 						} else if (fullUpdate) {
 							// if no audit, we can't accurately do incremental updates, so we just assume the last update
 							// was the same type. Assuming a full update this record update loses and there are no changes —

package/resources/analytics/DESIGN.md

@@ -0,0 +1,38 @@
+# resources/analytics/ — Design notes
+
+Analytics recording and aggregation.
+
+**Read this when:** touching `write.ts` aggregation or the raw-analytics cursor.
+
+Index of every design note: [DESIGN.md](../../DESIGN.md).
+
+---
+
+## Analytics aggregation resumes from a raw cursor, which is not a clock (`resources/analytics/write.ts`)
+
+`aggregation()` rolls up the raw reports in `hdb_raw_analytics` one `toPeriod` window at a time —
+it stops at `firstForPeriod + toPeriod` so one cycle's main-thread msgpackr decode stays bounded
+(#1538) — and resumes, exclusive, from `rawCursor`. Two rules hold that cursor:
+
+- **It only ever moves to a record a cycle actually read.** It used to be one variable with the
+  cadence marker, stamped with the cycle's end-of-run `Date.now()`, which discarded the remainder of
+  a backlog longer than one window and every report written while the cycle ran, because nothing
+  reads below the cursor again.
+- **A cycle that reads nothing leaves it where it is.** An empty scan is not proof that nothing is
+  there: `recordAnalytics` does not await its `primaryStore.put`, so a report can be uncommitted and
+  invisible to the scan while carrying a key older than anything the cycle could compare against. No
+  wall-clock reading is safe to put in the cursor for the same reason.
+
+`lastAggregationTime` stays a pure cadence marker, stamped `now` at the end of every completed
+cycle, so an idle node still aggregates once per period (#1538). A cycle that stopped at its window
+edge sets `aggregationBehind`, which lets the next tick skip that guard: a backlog then drains one
+window per half-period tick instead of waiting out the cadence it is already behind on.
+
+Cycles must not overlap. `setInterval` does not await its async callback, so `runAggregationCycle`
+holds a single-flight flag — two cycles reading the same cursor roll the same window up twice and
+double every count in it.
+
+Still open: `findLastAggregationTime` seeds both markers after a restart from the newest stored
+analytics record, whose `time` is an end-of-cycle stamp, so a restart mid-drain can skip the rest of
+the backlog; and `storeMetric` discards `table.put()`'s result, so the cursor advances past raw
+records whose aggregate rows failed to commit. Both need a cursor persisted with the rows.

package/resources/analytics/write.ts

@@ -877,6 +877,12 @@ async function storeNodeStorageMetric(analyticsTable: Table) {
 const MAX_LAST_AGGREGATION_SCAN = 1000;
 // Time of this node's last completed aggregation, tracked in O(1) across cycles within a process.
 let lastAggregationTime: number | undefined;
+// Resume point into the raw analytics table: it moves only over records a cycle read, so it is not
+// a clock and never passes an unread report.
+let rawCursor: number | undefined;
+// A cycle stopped at its window edge with raw records still behind it, so the next one skips the
+// cadence guard and drains the next window.
+let aggregationBehind = false;
 
 /**
  * Find the time of the local node's most recent aggregation record by scanning back from the
@@ -901,6 +907,20 @@ export function findLastAggregationTime(
 	return undefined;
 }
 
+let aggregationRunning = false;
+
+/** Two cycles reading the same cursor roll the same window up twice, and the scheduler does not
+ * await its callback. */
+export async function runAggregationCycle(fromPeriod, toPeriod = 60000) {
+	if (aggregationRunning) return;
+	aggregationRunning = true;
+	try {
+		await aggregation(fromPeriod, toPeriod);
+	} finally {
+		aggregationRunning = false;
+	}
+}
+
 async function aggregation(fromPeriod, toPeriod = 60000) {
 	const rawAnalyticsTable = getRawAnalyticsTable();
 	const analyticsTable = getAnalyticsTable();
@@ -924,28 +944,35 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	// first never touch the table. On the first cycle after boot we seed it once from the table
 	// via a bounded reverse scan (see findLastAggregationTime). Caching the seeded value keeps
 	// the next cycle O(1) even when this run early-returns below as "too recent"; a bound-hit
-	// (no match) leaves it undefined so we don't early-return, proceed to aggregate, and set the
-	// marker to `now` at the end of the cycle instead (#1538).
+	// (no match) leaves it undefined so we don't early-return, proceed to aggregate, and stamp the
+	// cadence marker at the end of the cycle instead (#1538).
 	let lastForPeriod = lastAggregationTime;
 	if (lastForPeriod === undefined) {
 		lastForPeriod = findLastAggregationTime(analyticsTable.primaryStore, localNodeId);
-		if (lastForPeriod !== undefined) lastAggregationTime = lastForPeriod;
+		if (lastForPeriod !== undefined) {
+			lastAggregationTime = lastForPeriod;
+			rawCursor ??= lastForPeriod;
+		}
 	}
 	// was the last aggregation too recent to calculate a whole period?
-	if (lastForPeriod !== undefined && Date.now() - toPeriod < lastForPeriod) return;
+	if (!aggregationBehind && lastForPeriod !== undefined && Date.now() - toPeriod < lastForPeriod) return;
 	let firstForPeriod;
 	const aggregateActions = new Map();
 	const distributions = new Map();
 	const threadsToAverage = [];
-	let lastTime: number;
+	let lastTime: number | undefined;
+	let stoppedAtWindowEdge = false;
 	for (const { key, value } of rawAnalyticsTable.primaryStore.getRange({
-		start: lastForPeriod || false,
+		start: rawCursor || false,
 		exclusiveStart: true,
 		end: Infinity,
 	})) {
 		if (!value) continue;
 		if (firstForPeriod) {
-			if (key > firstForPeriod + toPeriod) break; // outside the period of interest
+			if (key > firstForPeriod + toPeriod) {
+				stoppedAtWindowEdge = true;
+				break; // outside the period of interest
+			}
 		} else firstForPeriod = key;
 		lastTime = key;
 		const { metrics, threadId } = value;
@@ -1091,12 +1118,11 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	};
 	storeMetric(analyticsTable, cruMetric);
 	lastResourceUsage = resourceUsage;
-	// Advance the O(1) last-aggregation marker to this cycle's time. The resource-usage metric
-	// just written is a composite-id record stamped `time: now`, so it is exactly what the
-	// reverse-scan seed would find as the newest matching record. Updating it every completed
-	// cycle — not only when there were raw updates — keeps the `Date.now() - toPeriod` period
-	// guard enforcing the configured cadence during idle stretches, matching the prior
-	// scan-based behavior rather than letting a stale marker run aggregation on every tick (#1538).
+	// The cursor may only pass records this cycle read, and nothing reads below it again: the scan
+	// covers one `toPeriod` window, and a raw report whose `put` has not committed is invisible to
+	// it however recent its key is. An empty scan therefore leaves the cursor where it was.
+	if (lastTime !== undefined) rawCursor = lastTime;
+	aggregationBehind = stoppedAtWindowEdge;
 	lastAggregationTime = now;
 
 	// `system` is set as non-enumerable on the object returned by getDatabases() so most
@@ -1208,7 +1234,7 @@ function startScheduledTasks() {
 		const aggregateRetentionMs = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATERETENTIONMS) ?? AGGREGATE_EXPIRATION;
 		setInterval(
 			async () => {
-				await aggregation(analyticsDelay, AGGREGATE_PERIOD);
+				await runAggregationCycle(analyticsDelay, AGGREGATE_PERIOD);
 				await cleanup(getRawAnalyticsTable(), rawRetentionMs);
 				// 0 means "keep forever" — skip aggregate cleanup, matching storageInterval: 0 convention
 				if (aggregateRetentionMs) await cleanup(getAnalyticsTable(), aggregateRetentionMs);

package/resources/crdt.ts

@@ -74,6 +74,28 @@ export function rebuildUpdateBefore(update: any, newerUpdate: any, fullUpdate?:
 	}
 	return newUpdate;
 }
+
+/**
+ * The commutative operations in an out-of-order `update` — the only part of it whose outcome does not
+ * depend on where it lands in the write order. Used when the audit history the update would have been
+ * resequenced against is no longer retained.
+ *
+ * Not `rebuildUpdateBefore`, which folds an update against another UPDATE: against a materialized
+ * record its plain-newer-value rule drops the incoming `add` on any key the record already has a
+ * value for.
+ */
+export function commutativeOpsOf(update: any) {
+	let ops = null;
+	for (const key in update) {
+		const value = update[key];
+		if (value?.__op__) {
+			if (!ops) ops = {};
+			ops[key] = value;
+		}
+	}
+	return ops;
+}
+
 export function applyReverse(record, update, unknowns: Set<string>) {
 	for (const key in update) {
 		const value = update[key];

package/resources/databases.ts

@@ -1014,7 +1014,7 @@ function initStores(
 	{ defaultTable, auditPath, isLegacy, destination, storeName, openedStores }: InitStoresOptions = {}
 ) {
 	// a store with no tables never reaches the per-table loop below, and blob roots resolve from this
-	rootStore.databaseName = storeName ?? databaseName;
+	rootStore.databaseName ??= storeName ?? databaseName;
 	const envInit = new OpenEnvironmentObject(path, isReadOnlyMode());
 	const internalDbiInit = createOpenDBIObject(false);
 	let attributesDbi = rootStore.dbisDb;