@edgehero/pi-dispatch 4.3.1 → 5.0.0

package/src/run-mirror.mjs

@@ -1,4 +1,5 @@
 import { UNREADABLE_RECORD } from "./run-history.mjs";
+import { RUN_MIRROR_MAX_RUNS_DEFAULT } from "./retention.mjs";
 
 /**
  * A fleet-visible copy of the run history (issue #57, Gap 3).
@@ -201,13 +202,17 @@ export const runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
 export const MIRROR_MAX_DAYS = 92;
 
 /**
- * A hard ceiling on index members, independent of the time window.
+ * A hard ceiling on index members, independent of the time window: the DEFAULT of `PI_RUN_MIRROR_MAX_RUNS`
+ * (issue #761), which the worker hands `makeRunMirror` as `indexMax`. Kept under this name for its importers (a
+ * reader's default, the admin's local index).
  *
  * The window alone does not bound memory: a deployment running thousands of jobs a day would hold a
  * quarter of a million members for ninety-two days. This caps what the fleet view can cost at roughly the
- * depth a panel can display, and the file on disk remains the complete history either way.
+ * depth a panel can display, and the file on disk remains the complete history either way. Every writer trims
+ * the SHARED index to its own cap, so in a fleet the lowest setting wins; each count trim raises the horizon, so
+ * a reader sees the cut, never idle time.
  */
-export const RUNS_INDEX_MAX = 5_000;
+export const RUNS_INDEX_MAX = RUN_MIRROR_MAX_RUNS_DEFAULT;
 
 const DAY_MS = 24 * 60 * 60 * 1000;
 const OP_TIMEOUT_MS = 2_000;
@@ -290,8 +295,13 @@ export function makeRunMirror({ redis, retentionDays, now = () => Date.now(), lo
  *
  * Two round trips regardless of how many runs come back: one `ZREVRANGEBYSCORE` for the ids, one `MGET`
  * for the bodies. Never one read per run.
+ *
+ * `prune: false` makes the read READ-ONLY (issue #641): the console's pages read the mirror on every request, and a
+ * read that writes is a read that needs a writer's care (a page refresh that moves the horizon is a page refresh that
+ * changes what capacity reports). An expired body is then skipped and left in the index for a pruning reader or the
+ * writer's own trim; the row it stood for is not shown either way.
  */
-export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = () => Date.now(), timeoutMs = OP_TIMEOUT_MS } = {}) {
+export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, prune = true, now = () => Date.now(), timeoutMs = OP_TIMEOUT_MS } = {}) {
 	if (!redis) return { runs: [], degraded: "off" };
 	let ids;
 	try {
@@ -326,7 +336,7 @@ export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = (
 			stale.push(ids[i]); // unparseable is indistinguishable from gone, and equally not showable
 		}
 	}
-	if (stale.length > 0) {
+	if (stale.length > 0 && prune !== false) {
 		try {
 			// Removed with the cut recorded (`PRUNE_SCRIPT`): the time up to a pruned run is no longer whole.
 			await bounded(redis.eval(PRUNE_SCRIPT, 3, RUNS_INDEX, RUNS_HORIZON, RUNS_SINCE, mirrorWindowMs(0), RUNS_HORIZON_MEMBER, ...stale), timeoutMs);
@@ -337,6 +347,60 @@ export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = (
 	return { runs, degraded: runs.length >= limit ? "truncated" : "ok" };
 }
 
+/**
+ * One PAGE of the index below a score cursor, READ-ONLY (issue #641): the console's runs screen. Returns
+ * `{ runs, fetched, frontier, degraded }` and never throws.
+ *
+ * `before` is an EXCLUSIVE score (millis) or null for the newest; `offset` skips that many members under the same
+ * cursor, so a caller fetching in batches under ONE cursor reads a stable sequence while new runs land above it (a new
+ * run scores at its own end, which is after any cursor a page handed out). `fetched` is how many members the range
+ * returned, expired ones included, and `frontier` the lowest score among them (null when none): together they tell a
+ * caller how deep it has read, which the runs alone cannot, since an expired body leaves no run behind. Nothing is
+ * pruned and nothing is written: two reads, the range WITHSCORES and one MGET.
+ */
+export async function readMirroredPage(redis, { before = null, offset = 0, limit = 50, timeoutMs = OP_TIMEOUT_MS } = {}) {
+	if (!redis) return { runs: [], fetched: 0, frontier: null, degraded: "off" };
+	const max = Number.isFinite(before) ? `(${before}` : "+inf";
+	let flat;
+	try {
+		flat = await bounded(redis.zrevrangebyscore(RUNS_INDEX, max, "-inf", "WITHSCORES", "LIMIT", Math.max(0, Math.floor(offset) || 0), Math.max(1, Math.floor(limit) || 1)), timeoutMs);
+	} catch (err) {
+		return { runs: [], fetched: 0, frontier: null, degraded: `unreachable (${err?.message ?? "?"})` };
+	}
+	const ids = [];
+	let frontier = null;
+	for (let i = 0; Array.isArray(flat) && i + 1 < flat.length; i += 2) {
+		ids.push(flat[i]);
+		const score = Number(flat[i + 1]);
+		if (Number.isFinite(score)) frontier = frontier === null ? score : Math.min(frontier, score);
+	}
+	if (ids.length === 0) return { runs: [], fetched: 0, frontier: null, degraded: "ok" };
+	let bodies;
+	try {
+		bodies = await bounded(redis.mget(...ids.map(runRecordKey)), timeoutMs);
+	} catch (err) {
+		return { runs: [], fetched: 0, frontier: null, degraded: `unreachable (${err?.message ?? "?"})` };
+	}
+	const runs = [];
+	for (let i = 0; i < ids.length; i++) {
+		const raw = bodies?.[i];
+		if (typeof raw !== "string" || raw === "") continue;
+		try {
+			const rec = JSON.parse(raw);
+			if (rec && typeof rec === "object" && !Array.isArray(rec)) runs.push(rec);
+		} catch {
+			// unparseable is not showable, and this read prunes nothing
+		}
+	}
+	return { runs, fetched: ids.length, frontier, degraded: "ok" };
+}
+
+/**
+ * What `readMirroredRecord` answers, when asked to tell it (`tellUnread`), for a GET that did not complete: the server
+ * unreachable, or no answer within the bound. Nothing was read, which is not the same as no record.
+ */
+export const UNREAD_RECORD = Object.freeze({ unread: true });
+
 /**
  * One run's mirrored record, by its sanitized id: the parsed object, `null`, or `UNREADABLE_RECORD`. Never throws,
  * never rejects.
@@ -346,8 +410,12 @@ export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = (
  * is there but is not a record (unparseable, empty, not an object) is `UNREADABLE_RECORD`, exactly as the local
  * reader says it, so the check can log `unreadable` from `mirror`. Neither is ever accepted: the caller keeps
  * today's path.
+ *
+ * `tellUnread: true` answers `UNREAD_RECORD` for a GET that did not complete instead of `null`, for a caller that must
+ * not take an unread record for an absent one: the console's run page, whose "no run with this id" is a claim that
+ * every place a run is kept was read. The worker's own reads keep `null`, which fails toward today's path.
  */
-export async function readMirroredRecord(redis, sanitizedJobId, { timeoutMs = OP_TIMEOUT_MS } = {}) {
+export async function readMirroredRecord(redis, sanitizedJobId, { timeoutMs = OP_TIMEOUT_MS, tellUnread = false } = {}) {
 	if (!redis || !sanitizedJobId) return null;
 	try {
 		const raw = await bounded(redis.get(runRecordKey(sanitizedJobId)), timeoutMs);
@@ -361,7 +429,7 @@ export async function readMirroredRecord(redis, sanitizedJobId, { timeoutMs = OP
 		}
 		return rec !== null && typeof rec === "object" && !Array.isArray(rec) ? rec : UNREADABLE_RECORD;
 	} catch {
-		return null; // unreachable or timed out: nothing was read
+		return tellUnread === true ? UNREAD_RECORD : null; // unreachable or timed out: nothing was read
 	}
 }