@edgehero/pi-dispatch 4.3.1 → 5.0.0

package/src/packages.mjs

@@ -27,6 +27,7 @@
  *     manifest must degrade to "no staged packages", not crash the worker mid-queue.
  */
 
+import { EXACT_VERSION_RE, NPM_NAME_RE } from "./package-names.mjs";
 import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { configError } from "./config.mjs";
@@ -50,9 +51,7 @@ export const STAGE_MANIFEST = "packages.json";
 export const CONTAINER_PACKAGES_ROOT = `${CONTAINER_GLOBAL_PI_DIR}/${PACKAGES_SUBDIR}`;
 
 /** An exact semver -- prerelease and build metadata allowed, ranges/tags/wildcards are not. */
-export const EXACT_VERSION_RE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
-/** An npm package name: lowercase, optionally `@scope/`-prefixed. */
-export const NPM_NAME_RE = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/;
+export { EXACT_VERSION_RE, NPM_NAME_RE };
 
 /**
  * The pi resource kinds a package may contribute by convention dir, when it carries no `pi` manifest.
@@ -61,8 +60,8 @@ export const NPM_NAME_RE = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/;
  * would add an edge to the import-pi <-> packages cycle that only survives because both sides use their
  * bindings at call time.
  *
- * This list is pi's, not ours: at the 0.99.1 pin (as at 0.80.7) `collectPackageResources` falls through to exactly these
- * four directory names when `readPiManifest` returns null, so a package with no `pi` key and a `skills/`
+ * This list is pi's, not ours: at the pin `collectPackageResources` falls through to exactly these four directory
+ * names when `readPiManifest` returns null or lists no resource (1.5.0), so a package with no `pi` key and a `skills/`
  * dir IS a pi package. See host-pi.mjs's PINNED_PI_NEEDLES for the assertion that keeps that true.
  */
 export const RESOURCE_DIRS = ["extensions", "skills", "prompts", "themes"];
@@ -76,6 +75,12 @@ const MAX_NAME_LENGTH = 214;
 /** The staged dir name doubles as a path segment in the container, so it stays short and flat. */
 const MAX_DIR_LENGTH = 64;
 
+/**
+ * A `dir` on the stage manifest: a plain path segment. Since issue #796 it names one stage (`package-stage.mjs`, name, version and a unique
+ * suffix) and may be longer than the package's own directory name, which `MAX_DIR_LENGTH` still bounds. A receipt from before is still valid.
+ */
+const STAGED_TREE_RE = /^[a-z0-9](?:[a-z0-9_.-]{0,142}[a-z0-9])?$/i;
+
 function isNonEmptyString(value) {
 	return typeof value === "string" && value.trim() !== "";
 }
@@ -266,7 +271,7 @@ export function readStageManifest({ globalPiDir, readFile = readFileSync, fileEx
 			if (entry === null || typeof entry !== "object") return null;
 			const { name, version, dir } = entry;
 			if (!isNonEmptyString(name) || !isNonEmptyString(version) || !isNonEmptyString(dir)) return null;
-			if (!ENTRY_NAME_RE.test(dir) || dir.length > MAX_DIR_LENGTH) return null;
+			if (!STAGED_TREE_RE.test(dir)) return null;
 			packages.push({ name, version, dir, from: entry.from === FROM_HOST ? FROM_HOST : FROM_DECLARED });
 		}
 		return { stagedAt: typeof parsed.stagedAt === "string" ? parsed.stagedAt : null, packages };
@@ -281,12 +286,14 @@ export function readStageManifest({ globalPiDir, readFile = readFileSync, fileEx
  * readStageManifest's policy, because the consumers are advisory (doctor's per-trigger flow lines,
  * and issue #188's topology) and a half-staged tree must degrade to "nothing visible", not a crash.
  *
- * The semantics mirror pi's collectPackageResources at the 0.99.1 pin EXACTLY (unchanged since 0.80.7), because an enumerator
- * that agrees with pi by hand is how doctor comes to report a tier pi then ignores:
- *   - a `pi` manifest object means its `skills` entries are the ONLY sources -- a manifest WITHOUT a
- *     `skills` key contributes NO skills and gets NO convention fallback (readPiManifest short-circuits
- *     before the dir walk);
- *   - no `pi` key at all falls through to the convention `skills/` dir (RESOURCE_DIRS);
+ * The semantics mirror pi's collectPackageResources at the pin EXACTLY, and worker/test/packages.test.mjs holds them to
+ * pi's own package manager, because an enumerator that agrees with pi by hand is how doctor comes to report a tier pi then
+ * ignores:
+ *   - a `pi` object that lists a resource (a field that is an array of strings, which is all readPiManifest keeps) means
+ *     its `skills` entries are the ONLY sources -- one listing other resources but no `skills` contributes NO skills and
+ *     gets NO convention fallback;
+ *   - no `pi` key, or since 1.5.0 a `pi` object holding only metadata such as `changelogPath`, falls through to the
+ *     convention `skills/` dir (RESOURCE_DIRS);
  *   - a manifest entry may be a SKILL.md file, a skill dir, or a dir of skill dirs -- pi walks
  *     recursively, so this walks the same shape (bounded, it is host-advisory);
  *   - a glob (`*`/`?`) or override (`!`/`+`/`-` prefix) entry makes the whole package UNENUMERABLE
@@ -307,10 +314,11 @@ export function readStagedSkills({ globalPiDir, readFile = readFileSync, fileExi
 		const root = join(globalPiDir, PACKAGES_SUBDIR, pkg.dir);
 		let sources;
 		try {
-			const parsed = JSON.parse(readFile(join(root, "package.json"), "utf8"));
-			const pi = parsed !== null && typeof parsed === "object" && parsed.pi !== null && typeof parsed.pi === "object" ? parsed.pi : null;
-			if (pi) {
-				const entries = Array.isArray(pi.skills) ? pi.skills.filter((e) => typeof e === "string") : [];
+			const parsed = JSON.parse(readFile(join(root, "package.json"), "utf8").replace(/^\uFEFF/, ""));
+			const pi = parsed !== null && typeof parsed === "object" && parsed.pi !== null && typeof parsed.pi === "object" && !Array.isArray(parsed.pi) ? parsed.pi : null;
+			const listed = (field) => Array.isArray(pi?.[field]) && pi[field].every((e) => typeof e === "string");
+			if (RESOURCE_DIRS.some(listed)) {
+				const entries = listed("skills") ? pi.skills : [];
 				if (entries.some((e) => e.startsWith("!") || e.startsWith("+") || e.startsWith("-") || e.includes("*") || e.includes("?"))) {
 					out.unenumerable.push(pkg.name);
 					continue;

package/src/parked.mjs

@@ -11,8 +11,9 @@
  * widens WHICH comments start a job, never WHO may: the comment trigger's author gate still runs first.
  */
 export const PARKED_LABEL = "pi-awaiting-input";
+export const APPROVAL_LABEL = "pi-awaiting-approval";
 
 /** Whether a subset's `[{ name }]` label list (or Azure's tag list, the same shape) carries the marker. */
 export function hasParkedLabel(labels) {
-	return Array.isArray(labels) && labels.some((label) => label?.name === PARKED_LABEL);
+	return Array.isArray(labels) && labels.some((label) => (label?.name === PARKED_LABEL || label?.name === APPROVAL_LABEL));
 }

package/src/pause-windows.mjs

@@ -17,9 +17,106 @@ import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node
 import { isAbsolute } from "node:path";
 import { configError } from "./config.mjs";
 import { FORGE_KINDS, isForgeKind } from "./forges.mjs";
+import { heldDocument } from "./held-document.mjs";
+import { findDuplicateKey } from "./json-duplicates.mjs";
+import { neededVersion, underDeclared, valueNeededVersion, valueUnderDeclared, versionProblem } from "./store-version.mjs";
 
-// Sunday-first to match JS getUTCDay() and the Intl weekday index used in zonedParts().
-const DAYS = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"];
+/**
+ * The highest pause windows file version this build reads (issue #761, `DES-STORE-VERSION-WHEN-NEEDED`): a top-level
+ * `version`, absent meaning 1, written only when a window uses a field that needs more, so every file of today's
+ * fields stays byte-identical.
+ */
+export const PAUSE_WINDOWS_VERSION = 1;
+
+/**
+ * The window fields `normalizeWindow` reads, in its order. The console's whole-list action checks a submitted window
+ * against this list, and the version table's keys must come from it; a test holds it to the normalizer's own reads.
+ */
+export const PAUSE_WINDOW_FIELDS = Object.freeze(["scope", "from", "to", "tz", "days", "dateFrom", "dateTo"]);
+
+/**
+ * Each window field whose ABSENCE an older build would misread (a field that narrows when a window pauses, so a build
+ * that drops it pauses at the wrong time), with the version that introduced it. Empty until a field needs one.
+ */
+export const PAUSE_WINDOW_FIELD_VERSIONS = Object.freeze({});
+
+/**
+ * Each EXISTING window field whose range a later release widens, as `field -> (value) => version` (the shared rule,
+ * `valueNeededVersion`): a new `scope` form, say, or a new `days` form. An older build would refuse the whole file as
+ * broken, keep its last good windows and run the jobs the new file pauses; a stamp makes it defer them instead.
+ * Empty until a value needs one.
+ */
+export const PAUSE_WINDOW_VALUE_VERSIONS = Object.freeze({});
+
+/**
+ * The lowest version a list of RAW windows needs, from each window's own keys and their values: 1 while nothing needs
+ * more. The highest window wins, whatever its place in the list.
+ */
+export function pauseWindowsVersionFor(windows, { table = PAUSE_WINDOW_FIELD_VERSIONS, valueTable = PAUSE_WINDOW_VALUE_VERSIONS } = {}) {
+	let needed = 1;
+	for (const w of Array.isArray(windows) ? windows : []) {
+		if (w === null || typeof w !== "object") continue;
+		const needs = Math.max(neededVersion(Object.keys(w), table), valueNeededVersion(w, valueTable));
+		if (needs > needed) needed = needs;
+	}
+	return needed;
+}
+
+/**
+ * Whether a writer must refuse the parsed document `doc`: `{ newer, message }` when its raw `version` is newer than
+ * this build reads (`newer: true`) or is not an integer of 1 or more (`newer: false`), else null. The RAW version
+ * only, never the full parser, so a list with a bad window in it can still be replaced whole; a writer refuses the newer case (it would lose what a newer release wrote) and replaces a broken `version`.
+ */
+export function pauseWindowsVersionRefusal(doc, { supported = PAUSE_WINDOWS_VERSION } = {}) {
+	if (doc === null || typeof doc !== "object" || Array.isArray(doc) || !Object.hasOwn(doc, "version")) return null;
+	return versionProblem(doc.version, { supported, what: "pause-windows file" });
+}
+
+/**
+ * What a write of this file starts from, `{ doc, windows }`, or `{ problem }` with the sentence saying why it could not keep
+ * what the file holds, and `doc` when it parsed, for the caller's newer-release answer, which comes first. One function for
+ * `writePauseWindows` and the console's whole-list action, so a preview and its write cannot read a file two ways. Only a file
+ * that holds nothing starts from no windows: missing (`null`), whitespace, or a byte-order mark alone. A file with one bad
+ * window is not such a file, since its raw list is still there to replace. A key written twice is refused too: `JSON.parse`
+ * keeps the last, so a write would drop the windows a person reading the file can see.
+ */
+export function heldPauseWindows(text, path) {
+	if ((text ?? "").trim() === "") return { doc: null, windows: [] };
+	const { value, problem } = heldDocument(text, "windows");
+	const refused = problem ?? (findDuplicateKey(text) ? "has a key written twice" : null);
+	if (refused) return { doc: value ?? null, problem: `the pause-windows file ${path} ${refused}, so its windows cannot be kept` };
+	return { doc: value, windows: Array.isArray(value.windows) ? value.windows : [] };
+}
+
+/**
+ * The pause windows file text for `windows`, written over the parsed document `doc` the write started from (issue
+ * #761): 2-space JSON and a trailing newline. Every top-level key of `doc` other than `windows` and `version` is
+ * carried in its original place (the file is shared, and a writer from an older release must not delete a block a
+ * newer one added), `windows` keeps its position (first when `doc` had none), and
+ * `version` leads only when the windows need more than 1. A `doc` whose only key is `windows`, or no `doc`, gives
+ * exactly the bytes this file was always written with, `JSON.stringify({ windows }, null, 2)` plus the newline.
+ * Built with `Object.defineProperty`, so a carried key named `__proto__` stays data.
+ */
+export function pauseWindowsText(doc, windows, { table = PAUSE_WINDOW_FIELD_VERSIONS, valueTable = PAUSE_WINDOW_VALUE_VERSIONS } = {}) {
+	const rest = doc !== null && typeof doc === "object" && !Array.isArray(doc) ? doc : {};
+	const out = {};
+	const put = (key, value) => Object.defineProperty(out, key, { value, enumerable: true, writable: true, configurable: true });
+	const version = pauseWindowsVersionFor(windows, { table, valueTable });
+	if (version > 1) put("version", version);
+	if (!Object.hasOwn(rest, "windows")) put("windows", windows);
+	for (const key of Object.keys(rest)) {
+		if (key === "version") continue;
+		put(key, key === "windows" ? windows : rest[key]);
+	}
+	return `${JSON.stringify(out, null, 2)}\n`;
+}
+
+/**
+ * The weekdays a window's `days` may name, Sunday-first to match JS getUTCDay() and the Intl weekday index used in
+ * zonedParts(). A `days` entry is read by its first three letters, lowercased. EXPORTED for the version bolt (issue
+ * #761): a form added here is one an older build refuses, so it needs a `PAUSE_WINDOW_VALUE_VERSIONS.days` rule.
+ */
+export const DAYS = Object.freeze(["sun", "mon", "tue", "wed", "thu", "fri", "sat"]);
 const DAY_SET = new Set(DAYS);
 
 function isNonEmptyString(value) {
@@ -60,19 +157,38 @@ function validateTz(tz, at, path) {
  * Parse, validate, and normalize the pause-windows file text. Returns an array of normalized windows
  * (unknown fields dropped; `fromMin`/`toMin` precomputed). Throws `configError` (fail-loud) on any malformed
  * entry. `path` is for messages only.
+ *
+ * The file's `version` is judged FIRST, before the windows (issue #761): a newer file's windows may use a shape this
+ * build would call malformed, and the true message is "upgrade". A newer file throws a configError with
+ * `.newer === true` (the worker defers jobs rather than misread a pause); a broken `version`, or a window using a field
+ * or a value that needs a higher version than the file declares, throws the ordinary configError. `supported`, `table`
+ * and `valueTable` are injectable for tests.
  */
-export function parsePauseWindows(text, path) {
+export function parsePauseWindows(text, path, { supported = PAUSE_WINDOWS_VERSION, table = PAUSE_WINDOW_FIELD_VERSIONS, valueTable = PAUSE_WINDOW_VALUE_VERSIONS } = {}) {
 	let parsed;
 	try {
 		parsed = JSON.parse(text);
 	} catch (error) {
 		throw configError(`pause-windows file is not valid JSON: ${path} (${error.message})`);
 	}
+	const problem = pauseWindowsVersionRefusal(parsed, { supported });
+	if (problem) {
+		const error = configError(`${problem.message}: ${path}`);
+		if (problem.newer) error.newer = true;
+		throw error;
+	}
 	const windows = parsed?.windows;
 	if (!Array.isArray(windows)) {
 		throw configError(`pause-windows file must have a "windows" array: ${path}`);
 	}
-	return windows.map((w, index) => normalizeWindow(w, index, path));
+	const normalized = windows.map((w, index) => normalizeWindow(w, index, path));
+	windows.forEach((w, index) => {
+		const short = underDeclared(Object.keys(w), table, parsed.version);
+		if (short) throw configError(`pause window at index ${index}: ${short.field} needs pause-windows file version ${short.needs}, and the file declares ${parsed.version ?? 1}: ${path}`);
+		const shortValue = valueUnderDeclared(w, valueTable, parsed.version);
+		if (shortValue) throw configError(`pause window at index ${index}: ${shortValue.field} holds a value that needs pause-windows file version ${shortValue.needs}, and the file declares ${parsed.version ?? 1}: ${path}`);
+	});
+	return normalized;
 }
 
 function normalizeWindow(w, index, path) {
@@ -142,11 +258,12 @@ export function loadPauseWindows(config, { readFileSync = fsReadFileSync, exists
 const WEEKDAY_INDEX = { Sun: 0, Mon: 1, Tue: 2, Wed: 3, Thu: 4, Fri: 5, Sat: 6 };
 
 /** The wall-clock parts of instant `ms` in `tz`: `{ y, mo, d, hh, mi, dow }` (dow 0=Sun..6=Sat). */
-function zonedParts(tz, ms) {
-	const fmt = new Intl.DateTimeFormat("en-US", {
+function zonedParts(tz, ms, formatters = null) {
+	const fmt = formatters?.get(tz) ?? new Intl.DateTimeFormat("en-US", {
 		timeZone: tz, hour12: false,
 		year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", weekday: "short",
 	});
+	if (formatters && !formatters.has(tz)) formatters.set(tz, fmt);
 	const p = {};
 	for (const part of fmt.formatToParts(new Date(ms))) p[part.type] = part.value;
 	let hh = Number(p.hour);
@@ -159,17 +276,17 @@ function zonedParts(tz, ms) {
  * guess back in `tz`, diff to recover the zone offset, subtract. Correct except the ~1h DST-transition seam,
  * which is immaterial for a pause boundary.
  */
-function wallToMs(tz, y, mo, d, hh, mi) {
+function wallToMs(tz, y, mo, d, hh, mi, formatters = null) {
 	const guess = Date.UTC(y, mo - 1, d, hh, mi);
-	const shown = zonedParts(tz, guess);
+	const shown = zonedParts(tz, guess, formatters);
 	const shownAsUtc = Date.UTC(shown.y, shown.mo - 1, shown.d, shown.hh, shown.mi);
 	return guess - (shownAsUtc - guess);
 }
 
 /** The date `delta` days from `parts`' date, computed at local noon to dodge DST edges. Returns zoned parts. */
-function shiftDay(tz, parts, delta) {
-	const noon = wallToMs(tz, parts.y, parts.mo, parts.d, 12, 0);
-	return zonedParts(tz, noon + delta * 86400000);
+function shiftDay(tz, parts, delta, formatters = null) {
+	const noon = wallToMs(tz, parts.y, parts.mo, parts.d, 12, 0, formatters);
+	return zonedParts(tz, noon + delta * 86400000, formatters);
 }
 
 function ymd(parts) {
@@ -186,11 +303,13 @@ function qualifies(w, parts) {
 }
 
 /** The end epoch ms of the window occurrence active at `nowMs`, or null if `now` is not inside `w`. Exported
- * so the admin panel can mark a single window "currently paused · resumes in …" without a synthetic job. */
-export function windowEndAt(w, nowMs) {
-	const p = zonedParts(w.tz, nowMs);
+ * so the admin panel can mark a single window "currently paused · resumes in …" without a synthetic job.
+ * An optional caller-owned formatter cache lets a bounded calendar projection reuse clocks; the worker default
+ * still constructs its clock per call, and both paths evaluate exactly the same wall-time rules. */
+export function windowEndAt(w, nowMs, formatters = null) {
+	const p = zonedParts(w.tz, nowMs, formatters);
 	const nowMin = p.hh * 60 + p.mi;
-	const endAt = (dateParts) => wallToMs(w.tz, dateParts.y, dateParts.mo, dateParts.d, Math.floor(w.toMin / 60), w.toMin % 60);
+	const endAt = (dateParts) => wallToMs(w.tz, dateParts.y, dateParts.mo, dateParts.d, Math.floor(w.toMin / 60), w.toMin % 60, formatters);
 
 	if (w.fromMin < w.toMin) {
 		// Same-day window: [from, to) today, starting today.
@@ -199,8 +318,8 @@ export function windowEndAt(w, nowMs) {
 	}
 	// Overnight window (from > to): started tonight (>= from, today qualifies) -> ends tomorrow at `to`,
 	// or started last night (< to, yesterday qualified) -> ends today at `to`.
-	if (nowMin >= w.fromMin && qualifies(w, p)) return endAt(shiftDay(w.tz, p, 1));
-	if (nowMin < w.toMin && qualifies(w, shiftDay(w.tz, p, -1))) return endAt(p);
+	if (nowMin >= w.fromMin && qualifies(w, p)) return endAt(shiftDay(w.tz, p, 1, formatters));
+	if (nowMin < w.toMin && qualifies(w, shiftDay(w.tz, p, -1, formatters))) return endAt(p);
 	return null;
 }
 

package/src/podman-stack.mjs

@@ -25,6 +25,7 @@
 import { execFileSync } from "node:child_process";
 import { connect as netConnect } from "node:net";
 import { dirname, join } from "node:path";
+import { openshellSetupOf, venuesOf } from "./backends.mjs";
 import { DEFAULT_EGRESS_PROXY, egressProxyName } from "./egress.mjs";
 import { MODEL_ENDPOINTS_INCLUDE_NAME } from "./model-endpoints.mjs";
 import { VALKEY_PASSWORD_KEY, valkeyEnvFileText } from "./valkey-auth.mjs";
@@ -593,8 +594,9 @@ export async function decideValkey({ venues, url, installed, probeTcp, lookup, e
 export async function resolveWorkerValkey({ url, venues, platform, env, envText, envPath, probeTcp, lookup, fs, euid, user, ownerName, interfaces, subuids, waitMs = 20_000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = () => Date.now(), configError: refuse }) {
 	// Gate round 3: judged on EVERY venue on Linux, not only podman's. Another account's listener is refused everywhere
 	// (item 7: a local-venue worker silently adopted another account's rootless Valkey); root's (docker-proxy) is refused
-	// only where the podman venue is this deployment's and `local` is not.
-	const rootRefused = venues.podmanUsed && !venues.localUsed;
+	// only where the podman venue is this deployment's and `local` is not, or (issue #707, review round 1 of #715) where an
+	// openshell venue on Podman is the queue's engine (`openshellSetupOf`), whose Quadlet Valkey is this account's too.
+	const rootRefused = (venues.podmanUsed && !venues.localUsed) || openshellSetupOf(venues, env).valkeyEngine === "podman";
 	const same = { url, servername: null, pinned: null, notes: [], rootRefused };
 	if (platform !== "linux") return same;
 	const notes = [];
@@ -956,7 +958,12 @@ export function foreignContainerRefusal(found, { forceHint }) {
  * docker-shaped worker for a podman deployment, and the opposite guess a stack for a docker one. So does a line systemd
  * splits differently from this reader (`envFileSystemdHazard`, issue #447), for the whole file.
  */
-export const STACK_KEYS = Object.freeze(["PI_BACKENDS", "PI_EGRESS", "PI_EGRESS_PROXY"]);
+// Issue #707: and the openshell venues' two keys, read from the same place for the same reason. `up` and `service install`
+// decide from PI_OPENSHELL_ENGINE which engine's store gets the job image and where Valkey runs (`openshellSetupOf`), so
+// a shell value the service never sees would stand the stack up on the wrong engine.
+const PODMAN_VENUE_KEYS = Object.freeze(["PI_BACKENDS", "PI_EGRESS", "PI_EGRESS_PROXY"]);
+const OPENSHELL_VENUE_KEYS = Object.freeze(["PI_OPENSHELL_ENGINE", "PI_OPENSHELL_GATEWAY"]);
+export const STACK_KEYS = Object.freeze([...PODMAN_VENUE_KEYS, ...OPENSHELL_VENUE_KEYS]);
 
 /** The longest venue-key value `readStackKeys` accepts, in bytes (issue #447 gate round 2; systemd's own limit is ~128 KiB). */
 export const STACK_VALUE_MAX = 4096;
@@ -967,7 +974,26 @@ function shapedRefusal(path, hazard, why) {
 	return `${path} line ${hazard.line} has ${shape.what}, and ${why}, so what the service reads for it is unknown: ${shape.fix}`;
 }
 
-export function readStackKeys(content, { loader = "systemd", path = ".env", assumeSpelled = false } = {}) {
+/**
+ * The venue keys of a deployment's `.env` (`STACK_KEYS`), read by `readVenueKeys` below in two passes since review round 2
+ * of #715. The three the podman venue decides from are read and refused as ever. The openshell venues' two are read the
+ * same way but REFUSED only where an openshell venue is blessed (the file's PI_BACKENDS, or `openshellBlessed` from the
+ * caller, which knows this shell's): a stray `export PI_OPENSHELL_ENGINE=...` or a continued line in a `local`-only
+ * deployment names a key nothing there reads, and stopping `up` or `service install` on it is a refusal with no reason.
+ */
+export function readStackKeys(content, { loader = "systemd", path = ".env", assumeSpelled = false, openshellBlessed = false } = {}) {
+	const base = readVenueKeys(content, { loader, path, assumeSpelled, keys: PODMAN_VENUE_KEYS, about: "whether this deployment runs the podman venue" });
+	if (base.error) return base;
+	let blessed = openshellBlessed === true;
+	try {
+		blessed ||= venuesOf({ PI_BACKENDS: base.keys.PI_BACKENDS }).openshellUsed;
+	} catch {}
+	const openshell = readVenueKeys(content, { loader, path, assumeSpelled, keys: OPENSHELL_VENUE_KEYS, about: "which engine and gateway the openshell venue uses (PI_OPENSHELL_ENGINE, PI_OPENSHELL_GATEWAY)" });
+	if (openshell.error) return blessed ? openshell : base;
+	return { keys: { ...base.keys, ...openshell.keys } };
+}
+
+function readVenueKeys(content, { loader, path, assumeSpelled, keys: KEYS, about }) {
 	// The file's BYTES where the caller has them (issue #447, gate round 1): systemd refuses to LOAD a file with a NUL
 	// or with invalid UTF-8 in a key or value, and the unit then fails with every key unset, which no reading of the
 	// decoded text can see. Refused whatever the file assigns, because the service does not start on it at all.
@@ -979,9 +1005,9 @@ export function readStackKeys(content, { loader = "systemd", path = ".env", assu
 	const lines = String(text ?? "").split("\n").map((l) => (l.endsWith("\r") ? l.slice(0, -1) : l));
 	// A line TOUCHES a key when the key is its leading word, `export ` allowed: that is where a loader could read an
 	// assignment of it. A key name inside another key's value (`PI_ENV_SETUP=/opt/PI_BACKENDS.sh`) touches nothing.
-	const touches = new RegExp(`^[ \\t]*(?:export[ \\t]+)?(${STACK_KEYS.join("|")})(?![A-Za-z0-9_])`);
+	const touches = new RegExp(`^[ \\t]*(?:export[ \\t]+)?(${KEYS.join("|")})(?![A-Za-z0-9_])`);
 	// `export K=v` is an assignment only to a loader that SOURCES the file (the macOS wrapper); systemd ignores it.
-	const exact = new RegExp(`^[ \\t]*${loader === "shell" ? "(?:export[ \\t]+)?" : ""}(${STACK_KEYS.join("|")})=`);
+	const exact = new RegExp(`^[ \\t]*${loader === "shell" ? "(?:export[ \\t]+)?" : ""}(${KEYS.join("|")})=`);
 	// A `#` line is a comment to every loader. A `;` line (a comment to systemd's EnvironmentFile=) needs no rule of its
 	// own: its leading word is `;`, never a key, so it touches nothing and is not refused (the round 2 nit).
 	const comment = /^[ \t]*#/;
@@ -1004,13 +1030,13 @@ export function readStackKeys(content, { loader = "systemd", path = ".env", assu
 	// value, and to a shell part of the joined line (`X=a\` + `#;PI_EGRESS=0` sets PI_EGRESS, gate round 2).
 	const afterLoneCr = (l) => (l.includes("\r") ? l.slice(l.indexOf("\r") + 1) : "");
 	const inValue = loader === "cmd" ? [] : envFileValueLines(text, { loader });
-	const spelledOutside = (commentLine) => (assumeSpelled ? STACK_KEYS[0] : STACK_KEYS.find((k) => lines.some((l, i) => (commentLine.test(l) && !inValue[i] ? afterLoneCr(l) : l).includes(k))));
+	const spelledOutside = (commentLine) => (assumeSpelled ? KEYS[0] : KEYS.find((k) => lines.some((l, i) => (commentLine.test(l) && !inValue[i] ? afterLoneCr(l) : l).includes(k))));
 	if (loader === "systemd") {
 		const h = envFileSystemdHazard(text);
 		const spelled = h === null ? undefined : spelledOutside(/^[ \t]*[#;]/);
 		if (h !== null && spelled !== undefined) {
 			const shape = SYSTEMD_HAZARD_SHAPES[h.shape];
-			return { error: `${path} line ${h.line} has ${shape.what}. The service's systemd would read the lines of this file differently from this command, so which venue keys (${STACK_KEYS.join(", ")}) it sets is unknown: ${shape.fix}` };
+			return { error: `${path} line ${h.line} has ${shape.what}. The service's systemd would read the lines of this file differently from this command, so which venue keys (${KEYS.join(", ")}) it sets is unknown: ${shape.fix}` };
 		}
 	}
 	let touched = null;
@@ -1021,7 +1047,7 @@ export function readStackKeys(content, { loader = "systemd", path = ".env", assu
 		if (!m) continue;
 		touched ??= { key: m[1], line: i + 1 };
 		if (!exact.test(line)) {
-			return { error: `${path} line ${i + 1} assigns ${m[1]} in a form other than a plain ${m[1]}=value line, which the loaders do not agree on (systemd sets \`${m[1]} =x\` and ignores \`export ${m[1]}=x\`; the shells do the opposite). Whether this deployment runs the podman venue is therefore unknown: write it as a plain ${m[1]}=value line` };
+			return { error: `${path} line ${i + 1} assigns ${m[1]} in a form other than a plain ${m[1]}=value line, which the loaders do not agree on (systemd sets \`${m[1]} =x\` and ignores \`export ${m[1]}=x\`; the shells do the opposite). ${about[0].toUpperCase()}${about.slice(1)} is therefore unknown: write it as a plain ${m[1]}=value line` };
 		}
 	}
 	if (touched) {
@@ -1060,12 +1086,12 @@ export function readStackKeys(content, { loader = "systemd", path = ".env", assu
 			return { error: `${path} line ${hazard.line} is one this command cannot read (an open quote, a continuation, a line that runs, or a second assignment on one line), and ${why}, so what the service reads for it is unknown. Fix that line first` };
 		}
 	}
-	const found = readEnvAssignments(text, STACK_KEYS, { loader });
+	const found = readEnvAssignments(text, KEYS, { loader });
 	const keys = {};
-	for (const key of STACK_KEYS) {
+	for (const key of KEYS) {
 		const read = found[key];
 		if (!read) continue;
-		if (!read.plain) return { error: `${path} line ${read.line} assigns ${key} in a form this command cannot read the way the service's loader will ($, quotes, spaces or a backslash in the value), so whether this deployment runs the podman venue is unknown. Write it as a plain ${key}=value line` };
+		if (!read.plain) return { error: `${path} line ${read.line} assigns ${key} in a form this command cannot read the way the service's loader will ($, quotes, spaces or a backslash in the value), so ${about} is unknown. Write it as a plain ${key}=value line` };
 		// THE PROJECT'S OWN CAP, not systemd's (gate round 3 corrected the wording): systemd 259 passes a value up to about
 		// 128 KiB, and past that the whole environment is `envFileLoadHazard`'s `exec-too-large`. A venue key names a list,
 		// a switch or a container, so 4096 bytes is far more than one needs, and a longer one is a mistake worth naming.
@@ -1291,9 +1317,9 @@ export function describeRollBack(rolled, { partial = false } = {}) {
  * also take the worker down with it, since the worker's own queue connection retries and says why, and a proxy that
  * is down refuses jobs pre-spend by itself. `After=` so a boot starts the queue before the worker reaches for it.
  */
-export function workerUnitDeps(units) {
+export function workerUnitDeps(units, { forWhat = "the podman venue (issue #430)" } = {}) {
 	if (units.length === 0) return "";
-	return `# Added by \`pi-dispatch service install\` for the podman venue (issue #430): the Quadlet units it installed.\nWants=${units.join(" ")}\nAfter=${units.join(" ")}\n`;
+	return `# Added by \`pi-dispatch service install\` for ${forWhat}: the Quadlet units it installed.\nWants=${units.join(" ")}\nAfter=${units.join(" ")}\n`;
 }
 
 /**

package/src/portfolio-snapshot.mjs

@@ -16,7 +16,9 @@
  * settles in), so the numbers are complete on every host. A counter holds what was spent AND what running jobs still
  * hold: that is the number the next job is admitted against, so it is the one a manager should plan with. Run counts
  * come from the local run history merged with the Valkey run mirror; without a mirror (no `PI_WORKER_NAME`) they are
- * this host's alone, and `fleet.runsComplete` says so.
+ * this host's alone, and `fleet.runsComplete` says so. A run record from a NEWER pi-dispatch (issue #761, run-history.mjs
+ * `recordReadable`) is not counted, from either source, and makes `fleet.runsComplete` false the same way: a count that
+ * reads a record past its stable core could count a field that release gave another meaning.
  */
 
 import * as nodeFs from "node:fs";
@@ -26,6 +28,7 @@ import { dayKey, monthKey, weekKey } from "./budget.mjs";
 import { OTHER, PLAN_FIELDS, PLAN_RULES, envelopeEntries, scopeRef } from "./priorities.mjs";
 import { InfraRetry } from "./processor.mjs";
 import { isProjectId } from "./project-id.mjs";
+import { recordReadable } from "./run-history.mjs";
 import { RUNS_INDEX_MAX, mergeRuns, readMirroredRuns } from "./run-mirror.mjs";
 import { scopeDollarKeyPrefix } from "./scoped-limits.mjs";
 
@@ -220,7 +223,9 @@ export async function buildPortfolioSnapshot({ envelope, digest, projects = [],
 /**
  * The run records of this host's history (`PI_LOGS_DIR/*.json`) that ended at or after `sinceMs`: `{ records, complete }`.
  * A file older than the window by its mtime is not opened (a record is written when its run ends). Never throws: a
- * missing directory is no runs, and any other fault reads as `complete: false`.
+ * missing directory is no runs, and any other fault reads as `complete: false`. A record from a newer pi-dispatch is
+ * left out and counted in `newerSkipped` (present only when there were any), judged against the window by its core
+ * times, the rule `countRuns` reads.
  */
 export function readLocalRuns({ logsDir, sinceMs, fs = nodeFs }) {
 	let names;
@@ -231,19 +236,22 @@ export function readLocalRuns({ logsDir, sinceMs, fs = nodeFs }) {
 	}
 	const records = [];
 	let complete = true;
+	let newerSkipped = 0;
 	for (const name of names) {
 		if (!name.endsWith(".json")) continue;
 		try {
 			const st = fs.statSync(join(logsDir, name));
 			if (!st.isFile() || st.mtimeMs < sinceMs) continue;
 			const rec = JSON.parse(fs.readFileSync(join(logsDir, name), "utf8"));
-			if (rec && typeof rec === "object" && typeof rec.jobId === "string") records.push(rec);
+			if (!rec || typeof rec !== "object" || typeof rec.jobId !== "string") continue;
+			if (recordReadable(rec)) records.push(rec);
+			else if (Date.parse(rec.endedAt ?? rec.startedAt ?? "") >= sinceMs) newerSkipped++;
 		} catch (error) {
 			// A record reaped between the listing and the read is simply gone; anything else leaves the count short.
 			if (error?.code !== "ENOENT") complete = false;
 		}
 	}
-	return { records, complete };
+	return { records, complete, ...(newerSkipped > 0 ? { newerSkipped } : {}) };
 }
 
 /**
@@ -257,8 +265,13 @@ export function readLocalRuns({ logsDir, sinceMs, fs = nodeFs }) {
  * anything yet, so it is retried rather than run on a snapshot with holes. The run mirror is a view and never throws; a
  * mirror that could not be read makes `runsComplete` false. A snapshot over 64 KiB is refused as
  * `portfolio-snapshot-oversize`, before any spend, and the log line names the project count.
+ *
+ * `indexMax` is this host's run index cap (`PI_RUN_MIRROR_MAX_RUNS`, issue #761), the most the mirror read asks for.
+ * Every writer trims the shared index to its own cap, this host included, so a read of this many is the whole index
+ * unless a peer with a larger cap has written since this host last did; a read cut short says `truncated`, which makes
+ * `runsComplete` false rather than leaving a short count unflagged.
  */
-export function makePortfolioSnapshot({ checkPortfolioFlag, governing, projects = () => [], limits = () => null, allocation, redis, logsDir, mirror = false, readMirror = readMirroredRuns, fs = nodeFs, now = () => new Date(), log = () => {} }) {
+export function makePortfolioSnapshot({ checkPortfolioFlag, governing, projects = () => [], limits = () => null, allocation, redis, logsDir, mirror = false, readMirror = readMirroredRuns, indexMax = RUNS_INDEX_MAX, fs = nodeFs, now = () => new Date(), log = () => {} }) {
 	return async function portfolioSnapshot(job) {
 		const flagged = await Promise.resolve()
 			.then(() => checkPortfolioFlag(job))
@@ -282,11 +295,17 @@ export function makePortfolioSnapshot({ checkPortfolioFlag, governing, projects
 			const local = readLocalRuns({ logsDir, sinceMs, fs });
 			let mirrored = [];
 			let complete = false;
+			let newerSkipped = local.newerSkipped ?? 0;
 			if (mirror) {
-				const read = await readMirror(redis, { limit: RUNS_INDEX_MAX, sinceMs, now: () => at.getTime() });
-				mirrored = read.runs;
+				const read = await readMirror(redis, { limit: indexMax, sinceMs, now: () => at.getTime() });
+				for (const r of Array.isArray(read?.runs) ? read.runs : []) {
+					if (recordReadable(r)) mirrored.push(r);
+					else if (r !== null && typeof r === "object" && !Array.isArray(r)) newerSkipped++;
+				}
 				complete = local.complete && (read.degraded === "ok" || read.degraded === "off");
 			}
+			// A newer release's record left out of the counts leaves them short, which is what `runsComplete` says.
+			if (newerSkipped > 0) complete = false;
 			const records = mergeRuns(local.records, mirrored, { limit: Number.POSITIVE_INFINITY });
 			snapshot = await buildPortfolioSnapshot({ envelope: g.envelope, digest: g.digest, projects: projects(), limits: limits(), allocation: { state, lastAttempt: last }, redis, runs: { records, complete }, now: at });
 		} catch (error) {