@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

package/resources/analytics/DESIGN.md

@@ -0,0 +1,38 @@
+# resources/analytics/ — Design notes
+
+Analytics recording and aggregation.
+
+**Read this when:** touching `write.ts` aggregation or the raw-analytics cursor.
+
+Index of every design note: [DESIGN.md](../../DESIGN.md).
+
+---
+
+## Analytics aggregation resumes from a raw cursor, which is not a clock (`resources/analytics/write.ts`)
+
+`aggregation()` rolls up the raw reports in `hdb_raw_analytics` one `toPeriod` window at a time —
+it stops at `firstForPeriod + toPeriod` so one cycle's main-thread msgpackr decode stays bounded
+(#1538) — and resumes, exclusive, from `rawCursor`. Two rules hold that cursor:
+
+- **It only ever moves to a record a cycle actually read.** It used to be one variable with the
+  cadence marker, stamped with the cycle's end-of-run `Date.now()`, which discarded the remainder of
+  a backlog longer than one window and every report written while the cycle ran, because nothing
+  reads below the cursor again.
+- **A cycle that reads nothing leaves it where it is.** An empty scan is not proof that nothing is
+  there: `recordAnalytics` does not await its `primaryStore.put`, so a report can be uncommitted and
+  invisible to the scan while carrying a key older than anything the cycle could compare against. No
+  wall-clock reading is safe to put in the cursor for the same reason.
+
+`lastAggregationTime` stays a pure cadence marker, stamped `now` at the end of every completed
+cycle, so an idle node still aggregates once per period (#1538). A cycle that stopped at its window
+edge sets `aggregationBehind`, which lets the next tick skip that guard: a backlog then drains one
+window per half-period tick instead of waiting out the cadence it is already behind on.
+
+Cycles must not overlap. `setInterval` does not await its async callback, so `runAggregationCycle`
+holds a single-flight flag — two cycles reading the same cursor roll the same window up twice and
+double every count in it.
+
+Still open: `findLastAggregationTime` seeds both markers after a restart from the newest stored
+analytics record, whose `time` is an end-of-cycle stamp, so a restart mid-drain can skip the rest of
+the backlog; and `storeMetric` discards `table.put()`'s result, so the cursor advances past raw
+records whose aggregate rows failed to commit. Both need a cursor persisted with the rows.

package/resources/analytics/write.ts

@@ -157,6 +157,10 @@ function sendAnalytics() {
 	analyticsStart ||= performance.now();
 	sendAnalyticsTimeout = setTimeout(async () => {
 		sendAnalyticsTimeout = null;
+		// Rotate before the first yield below: a sample recorded mid-flush would otherwise be added to
+		// an entry already reported, and dropped with it.
+		const reportingActions = activeActions;
+		activeActions = new Map();
 		const period = performance.now() - analyticsStart;
 		analyticsStart = 0;
 		const metrics = [];
@@ -166,7 +170,7 @@ function sendAnalytics() {
 			threadId,
 			metrics,
 		};
-		for (const [_name, action] of activeActions) {
+		for (const [_name, action] of reportingActions) {
 			if (action.values) {
 				const values = action.values.subarray(0, (action.values as any).index);
 				values.sort();
@@ -239,7 +243,6 @@ function sendAnalytics() {
 		for (const listener of analyticsListeners) {
 			listener(metrics);
 		}
-		activeActions = new Map();
 		if (parentPort)
 			parentPort.postMessage({
 				type: ANALYTICS_REPORT_TYPE,
@@ -874,6 +877,12 @@ async function storeNodeStorageMetric(analyticsTable: Table) {
 const MAX_LAST_AGGREGATION_SCAN = 1000;
 // Time of this node's last completed aggregation, tracked in O(1) across cycles within a process.
 let lastAggregationTime: number | undefined;
+// Resume point into the raw analytics table: it moves only over records a cycle read, so it is not
+// a clock and never passes an unread report.
+let rawCursor: number | undefined;
+// A cycle stopped at its window edge with raw records still behind it, so the next one skips the
+// cadence guard and drains the next window.
+let aggregationBehind = false;
 
 /**
  * Find the time of the local node's most recent aggregation record by scanning back from the
@@ -898,13 +907,31 @@ export function findLastAggregationTime(
 	return undefined;
 }
 
+let aggregationRunning = false;
+
+/** Two cycles reading the same cursor roll the same window up twice, and the scheduler does not
+ * await its callback. */
+export async function runAggregationCycle(fromPeriod, toPeriod = 60000) {
+	if (aggregationRunning) return;
+	aggregationRunning = true;
+	try {
+		await aggregation(fromPeriod, toPeriod);
+	} finally {
+		aggregationRunning = false;
+	}
+}
+
 async function aggregation(fromPeriod, toPeriod = 60000) {
 	const rawAnalyticsTable = getRawAnalyticsTable();
 	const analyticsTable = getAnalyticsTable();
 	const taskQueueLatency = (async () => {
 		const start = performance.now();
 		// measure how long it takes to enqueue and get a callback from a simple/fast task:
-		await stat(getLogFilePath());
+		// The result is discarded — only the round trip is being measured — so a missing file is not a
+		// failure. It rejects whenever the probe races a rotation, which the write-path size guard
+		// makes a routine event rather than a once-a-minute one, and an unhandled rejection here takes
+		// the process down over a latency measurement.
+		await stat(getLogFilePath()).catch(() => {});
 		const delay = performance.now() - start;
 		if (delay > 5000) {
 			log.warn?.('Unusually high task queue latency on the main thread of ' + Math.round(delay) + 'ms');
@@ -917,28 +944,35 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	// first never touch the table. On the first cycle after boot we seed it once from the table
 	// via a bounded reverse scan (see findLastAggregationTime). Caching the seeded value keeps
 	// the next cycle O(1) even when this run early-returns below as "too recent"; a bound-hit
-	// (no match) leaves it undefined so we don't early-return, proceed to aggregate, and set the
-	// marker to `now` at the end of the cycle instead (#1538).
+	// (no match) leaves it undefined so we don't early-return, proceed to aggregate, and stamp the
+	// cadence marker at the end of the cycle instead (#1538).
 	let lastForPeriod = lastAggregationTime;
 	if (lastForPeriod === undefined) {
 		lastForPeriod = findLastAggregationTime(analyticsTable.primaryStore, localNodeId);
-		if (lastForPeriod !== undefined) lastAggregationTime = lastForPeriod;
+		if (lastForPeriod !== undefined) {
+			lastAggregationTime = lastForPeriod;
+			rawCursor ??= lastForPeriod;
+		}
 	}
 	// was the last aggregation too recent to calculate a whole period?
-	if (lastForPeriod !== undefined && Date.now() - toPeriod < lastForPeriod) return;
+	if (!aggregationBehind && lastForPeriod !== undefined && Date.now() - toPeriod < lastForPeriod) return;
 	let firstForPeriod;
 	const aggregateActions = new Map();
 	const distributions = new Map();
 	const threadsToAverage = [];
-	let lastTime: number;
+	let lastTime: number | undefined;
+	let stoppedAtWindowEdge = false;
 	for (const { key, value } of rawAnalyticsTable.primaryStore.getRange({
-		start: lastForPeriod || false,
+		start: rawCursor || false,
 		exclusiveStart: true,
 		end: Infinity,
 	})) {
 		if (!value) continue;
 		if (firstForPeriod) {
-			if (key > firstForPeriod + toPeriod) break; // outside the period of interest
+			if (key > firstForPeriod + toPeriod) {
+				stoppedAtWindowEdge = true;
+				break; // outside the period of interest
+			}
 		} else firstForPeriod = key;
 		lastTime = key;
 		const { metrics, threadId } = value;
@@ -1084,12 +1118,11 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	};
 	storeMetric(analyticsTable, cruMetric);
 	lastResourceUsage = resourceUsage;
-	// Advance the O(1) last-aggregation marker to this cycle's time. The resource-usage metric
-	// just written is a composite-id record stamped `time: now`, so it is exactly what the
-	// reverse-scan seed would find as the newest matching record. Updating it every completed
-	// cycle — not only when there were raw updates — keeps the `Date.now() - toPeriod` period
-	// guard enforcing the configured cadence during idle stretches, matching the prior
-	// scan-based behavior rather than letting a stale marker run aggregation on every tick (#1538).
+	// The cursor may only pass records this cycle read, and nothing reads below it again: the scan
+	// covers one `toPeriod` window, and a raw report whose `put` has not committed is invisible to
+	// it however recent its key is. An empty scan therefore leaves the cursor where it was.
+	if (lastTime !== undefined) rawCursor = lastTime;
+	aggregationBehind = stoppedAtWindowEdge;
 	lastAggregationTime = now;
 
 	// `system` is set as non-enumerable on the object returned by getDatabases() so most
@@ -1201,7 +1234,7 @@ function startScheduledTasks() {
 		const aggregateRetentionMs = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATERETENTIONMS) ?? AGGREGATE_EXPIRATION;
 		setInterval(
 			async () => {
-				await aggregation(analyticsDelay, AGGREGATE_PERIOD);
+				await runAggregationCycle(analyticsDelay, AGGREGATE_PERIOD);
 				await cleanup(getRawAnalyticsTable(), rawRetentionMs);
 				// 0 means "keep forever" — skip aggregate cleanup, matching storageInterval: 0 convention
 				if (aggregateRetentionMs) await cleanup(getAnalyticsTable(), aggregateRetentionMs);