@edgehero/pi-dispatch-admin 1.7.0 → 1.9.0

package/dist/index.mjs

@@ -9215,6 +9215,84 @@ var init_queue = __esm({
   }
 });
 
+// ../worker/src/run-mirror.mjs
+function bounded2(promise, ms) {
+  let timer;
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error("mirror timeout")), ms);
+    })
+  ]).finally(() => clearTimeout(timer));
+}
+async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = () => Date.now(), timeoutMs = OP_TIMEOUT_MS } = {}) {
+  if (!redis) return { runs: [], degraded: "off" };
+  let ids;
+  try {
+    ids = await bounded2(redis.zrevrangebyscore(RUNS_INDEX, "+inf", `(${sinceMs}`, "LIMIT", 0, Math.max(1, limit)), timeoutMs);
+  } catch (err) {
+    return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
+  }
+  if (!Array.isArray(ids) || ids.length === 0) return { runs: [], degraded: "off" };
+  let bodies;
+  try {
+    bodies = await bounded2(redis.mget(...ids.map(runRecordKey)), timeoutMs);
+  } catch (err) {
+    return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
+  }
+  const runs = [];
+  const stale = [];
+  for (let i = 0; i < ids.length; i++) {
+    const raw = bodies?.[i];
+    if (typeof raw !== "string" || raw === "") {
+      stale.push(ids[i]);
+      continue;
+    }
+    try {
+      const rec = JSON.parse(raw);
+      if (rec && typeof rec === "object") runs.push(rec);
+    } catch {
+      stale.push(ids[i]);
+    }
+  }
+  if (stale.length > 0) {
+    try {
+      await bounded2(redis.zrem(RUNS_INDEX, ...stale), timeoutMs);
+    } catch {
+    }
+  }
+  return { runs, degraded: runs.length >= limit ? "truncated" : "ok" };
+}
+function mergeRuns(local, mirrored, { limit = 50 } = {}) {
+  const by = /* @__PURE__ */ new Map();
+  const at = (r) => {
+    const t = Date.parse(r?.endedAt ?? r?.startedAt ?? "");
+    return Number.isFinite(t) ? t : -Infinity;
+  };
+  for (const r of Array.isArray(mirrored) ? mirrored : []) if (r?.jobId) by.set(r.jobId, r);
+  for (const r of Array.isArray(local) ? local : []) {
+    if (!r?.jobId) continue;
+    const seen = by.get(r.jobId);
+    if (!seen || at(r) >= at(seen)) by.set(r.jobId, r);
+  }
+  const out = [...by.values()].sort((a, b) => at(b) - at(a));
+  return out.slice(0, Math.max(0, limit));
+}
+function hostsIn(runs) {
+  const names = /* @__PURE__ */ new Set();
+  for (const r of runs ?? []) if (typeof r?.host === "string" && r.host !== "") names.add(r.host);
+  return [...names].sort();
+}
+var RUNS_INDEX, runRecordKey, DAY_MS, OP_TIMEOUT_MS;
+var init_run_mirror = __esm({
+  "../worker/src/run-mirror.mjs"() {
+    RUNS_INDEX = "runs:index";
+    runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
+    DAY_MS = 24 * 60 * 60 * 1e3;
+    OP_TIMEOUT_MS = 2e3;
+  }
+});
+
 // ../worker/src/git-dirty.mjs
 import { execFileSync } from "node:child_process";
 function gitDirty(folder, { exec: exec4 = execFileSync } = {}) {
@@ -9269,7 +9347,7 @@ var init_import_pi = __esm({
 });
 
 // ../worker/src/docker-run.mjs
-function buildDockerRunArgs({
+function containerSpec({
   image,
   env,
   jobDir,
@@ -9286,21 +9364,44 @@ function buildDockerRunArgs({
   if (!image) throw new Error("docker run: image is required");
   if (!name) throw new Error("docker run: container name is required");
   if (!workspace) throw new Error("docker run: workspace mount is required");
-  const args = ["run", `--name=${name}`, ...ISOLATION_FLAGS, `--memory=${memory}`, `--cpus=${cpus}`];
-  if (network) args.push(`--network=${network}`);
-  args.push(...extraFlags);
-  for (const [k, v] of Object.entries(env ?? {})) {
+  const mounts = [];
+  if (jobDir) mounts.push({ host: jobDir, container: "/job", readOnly: true });
+  mounts.push({ host: workspace, container: "/workspace", readOnly: false });
+  if (outboxDir) mounts.push({ host: outboxDir, container: "/outbox", readOnly: false });
+  if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false });
+  if (globalPiDir) mounts.push({ host: globalPiDir, container: CONTAINER_GLOBAL_PI_DIR, readOnly: true });
+  return {
+    image,
+    name,
+    memory,
+    cpus,
+    network,
+    // UNCONDITIONALLY true, and there is deliberately no parameter that can unset it. The boundary is
+    // not a thing a caller opts into -- CONST-ISOLATION-CONTAINER-PER-JOB is why every other flag here
+    // exists -- so the spec is simply unable to describe an unisolated container, and the builder below
+    // refuses one it is handed. A field that could be false would be a way to ask for less.
+    isolated: true,
+    mounts,
+    env,
+    dockerExtra: extraFlags
+  };
+}
+function dockerArgsFromSpec(spec) {
+  if (spec?.isolated !== true) throw new Error("docker run: refusing to build an argv for a spec that is not isolated");
+  const args = ["run", `--name=${spec.name}`, ...ISOLATION_FLAGS, `--memory=${spec.memory}`, `--cpus=${spec.cpus}`];
+  if (spec.network) args.push(`--network=${spec.network}`);
+  args.push(...spec.dockerExtra ?? []);
+  for (const [k, v] of Object.entries(spec.env ?? {})) {
     if (v === void 0 || v === null) continue;
     args.push("-e", `${k}=${v}`);
   }
-  if (jobDir) args.push("-v", `${jobDir}:/job:ro`);
-  args.push("-v", `${workspace}:/workspace`);
-  if (outboxDir) args.push("-v", `${outboxDir}:/outbox`);
-  if (sessionDir) args.push("-v", `${sessionDir}:${CONTAINER_SESSION_DIR}`);
-  if (globalPiDir) args.push("-v", `${globalPiDir}:${CONTAINER_GLOBAL_PI_DIR}:ro`);
-  args.push(image);
+  for (const m of spec.mounts ?? []) args.push("-v", `${m.host}:${m.container}${m.readOnly ? ":ro" : ""}`);
+  args.push(spec.image);
   return args;
 }
+function buildDockerRunArgs(opts) {
+  return dockerArgsFromSpec(containerSpec(opts));
+}
 var CONTAINER_GLOBAL_PI_DIR, CONTAINER_SESSION_DIR, CONTAINER_SESSION_FILE, ISOLATION_FLAGS;
 var init_docker_run = __esm({
   "../worker/src/docker-run.mjs"() {
@@ -10028,8 +10129,8 @@ var init_graph_model = __esm({
 
 // src/costs.mjs
 function costsSinceMs(windowKey, nowMs) {
-  if (windowKey === "7d") return nowMs - 7 * DAY_MS;
-  if (windowKey === "30d") return nowMs - 30 * DAY_MS;
+  if (windowKey === "7d") return nowMs - 7 * DAY_MS2;
+  if (windowKey === "30d") return nowMs - 30 * DAY_MS2;
   const now = new Date(nowMs);
   return Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1);
 }
@@ -10152,7 +10253,7 @@ function foldCosts({ records, subscriptions, pricing, nowMs, piAiPin = null, sin
   const requested = Number.isFinite(sinceMs);
   const fromMs = requested ? Math.min(sinceMs, nowMs) : firstRunMs ?? nowMs;
   const toMs2 = nowMs;
-  const days = requested || runs.length > 0 ? Math.max(1, Math.ceil((toMs2 - fromMs) / DAY_MS)) : 0;
+  const days = requested || runs.length > 0 ? Math.max(1, Math.ceil((toMs2 - fromMs) / DAY_MS2)) : 0;
   return {
     // `firstRunMs` rides along (null when nothing ran) so a renderer can pad or trim the daily series
     // against the requested edge without re-deriving what the fold already knows.
@@ -10183,7 +10284,7 @@ function buildDaily(runs, fromMs, toMs2) {
     buckets.get(day).push(r.contribution);
   }
   const daily = [];
-  for (let cursor = Math.floor(fromMs / DAY_MS) * DAY_MS; cursor <= toMs2; cursor += DAY_MS) {
+  for (let cursor = Math.floor(fromMs / DAY_MS2) * DAY_MS2; cursor <= toMs2; cursor += DAY_MS2) {
     const day = utcDay(cursor);
     const contribs = buckets.get(day) ?? [];
     daily.push({ day, cost: combineContributions(contribs), runs: contribs.length });
@@ -10207,7 +10308,7 @@ function buildDailyByFlow(runs, fromMs, toMs2) {
   const rows = [];
   for (const [flow, buckets] of byFlow) {
     const days = [];
-    for (let cursor = Math.floor(fromMs / DAY_MS) * DAY_MS; cursor <= toMs2; cursor += DAY_MS) {
+    for (let cursor = Math.floor(fromMs / DAY_MS2) * DAY_MS2; cursor <= toMs2; cursor += DAY_MS2) {
       const day = utcDay(cursor);
       const contribs = buckets.get(day) ?? [];
       days.push({ day, cost: combineContributions(contribs), runs: contribs.length });
@@ -10451,7 +10552,7 @@ function peakWindow(w, attributed) {
   let peakRuns = 0;
   let peakTokens = 0;
   if (w.rolling) {
-    const span = w.per === "5h" ? 5 * HOUR_MS : w.per === "7d" ? 7 * DAY_MS : 30 * DAY_MS;
+    const span = w.per === "5h" ? 5 * HOUR_MS : w.per === "7d" ? 7 * DAY_MS2 : 30 * DAY_MS2;
     let left = 0;
     let runsInWindow = 0;
     let tokensInWindow = 0;
@@ -10542,11 +10643,11 @@ function medianQuad(quads) {
   }
   return quad;
 }
-var DAY_MS, HOUR_MS, COSTS_WINDOWS, QUAD_KEYS, TRIGGER_TAIL, TRIGGER_TAIL_LABELS;
+var DAY_MS2, HOUR_MS, COSTS_WINDOWS, QUAD_KEYS, TRIGGER_TAIL, TRIGGER_TAIL_LABELS;
 var init_costs = __esm({
   "src/costs.mjs"() {
     init_budget();
-    DAY_MS = 24 * 60 * 60 * 1e3;
+    DAY_MS2 = 24 * 60 * 60 * 1e3;
     HOUR_MS = 60 * 60 * 1e3;
     COSTS_WINDOWS = ["7d", "30d", "mtd"];
     QUAD_KEYS = ["input", "output", "cacheRead", "cacheWrite", "cacheWrite1h"];
@@ -11209,6 +11310,35 @@ function withTimeout(promise, ms, fallback) {
   });
   return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
 }
+async function mergedRunsOn(redis, { logsDir, limit = 10, fs = nodeFs3, timeoutMs = 2500 } = {}) {
+  const local = listRuns({ logsDir, limit, fs });
+  if (!Array.isArray(local)) return { runs: [], hosts: [], mirror: "off", ...local };
+  try {
+    const { runs: mirrored, degraded } = await readMirroredRuns(redis, { limit: Math.max(limit * 2, 50), timeoutMs });
+    const runs = mergeRuns(local, mirrored, { limit });
+    return { runs, hosts: hostsIn(runs), mirror: degraded };
+  } catch (err) {
+    return { runs: local, hosts: hostsIn(local), mirror: `unreachable (${err?.message ?? "?"})` };
+  }
+}
+async function listRunsMerged({ logsDir, limit = 10, url, fs = nodeFs3, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
+  let redis;
+  try {
+    redis = redisFn(url);
+    redis.on?.("error", () => {
+    });
+    return await mergedRunsOn(redis, { logsDir, limit, fs, timeoutMs });
+  } catch (err) {
+    const local = listRuns({ logsDir, limit, fs });
+    const runs = Array.isArray(local) ? local : [];
+    return { runs, hosts: hostsIn(runs), mirror: `unreachable (${err?.message ?? "?"})` };
+  } finally {
+    try {
+      redis?.disconnect?.();
+    } catch {
+    }
+  }
+}
 function listRuns({ logsDir, limit = 10, fs = nodeFs3 }) {
   const cap = clampLimit(limit);
   let names;
@@ -11251,12 +11381,13 @@ function scanRunRecords({ logsDir, sinceMs, nowMs = Date.now(), fs = nodeFs3 })
 function readRun({ logsDir, jobId, fs = nodeFs3 }) {
   return readJsonFile(join2(logsDir, `${sanitizeJobId(jobId)}.json`), fs);
 }
-function readLogTail({ logsDir, jobId, lines = 200, fs = nodeFs3 }) {
+function readLogTail({ logsDir, jobId, lines = 200, fs = nodeFs3, host = null, self = null }) {
   let text;
   try {
     text = fs.readFileSync(join2(logsDir, `${sanitizeJobId(jobId)}.log`), "utf8");
   } catch {
-    return { missing: true };
+    const foreign = typeof host === "string" && host !== "" && typeof self === "string" && self !== "" && host !== self;
+    return foreign ? { missing: true, elsewhere: host } : { missing: true };
   }
   const all = text.split("\n").filter((line) => line !== "");
   const cap = clampLines(lines);
@@ -11772,6 +11903,7 @@ var init_read_model = __esm({
     init_connection();
     init_host_registry();
     init_queue();
+    init_run_mirror();
     init_flow_gate();
     init_git_dirty();
     init_packages();
@@ -40066,10 +40198,11 @@ var init_render = __esm({
 });
 
 // ../worker/src/scheduler-stall-guard.mjs
-var STALL_KEY, STALL_WINDOW_SECONDS;
+var STALL_KEY, stallKey, STALL_WINDOW_SECONDS;
 var init_scheduler_stall_guard = __esm({
   "../worker/src/scheduler-stall-guard.mjs"() {
     STALL_KEY = "pi-dispatch:sched-stalls";
+    stallKey = (schedulerId) => `${STALL_KEY}:${schedulerId}`;
     STALL_WINDOW_SECONDS = 24 * 60 * 60;
   }
 });
@@ -40334,7 +40467,10 @@ function createDashboardDeps(paths, {
   return {
     async fetchSnapshot() {
       const queues = await fleetQueues();
-      const [pausedStates, countsPer, workerList, dayRaw, weekRaw, monthRaw, tokenRaw, schedulerLists, activeLists, stallHash] = await Promise.all([
+      const merged = await mergedRunsOn(redis, { logsDir: paths.logsDir, limit: RUNS_ON_DASHBOARD }).catch(() => ({ runs: [], hosts: [], mirror: "off" }));
+      const triggersView = readTriggers({ triggersPath: paths.triggersPath });
+      const cronIds = Array.isArray(triggersView?.triggers) ? triggersView.triggers.filter((t) => t?.type === "cron" && typeof t?.id === "string" && t.id !== "").map((t) => t.id) : [];
+      const [pausedStates, countsPer, workerList, dayRaw, weekRaw, monthRaw, tokenRaw, schedulerLists, activeLists, stallCounts] = await Promise.all([
         Promise.all(queues.map((q) => q.isPaused())),
         Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed"))),
         queue.getWorkers().catch(() => []),
@@ -40348,8 +40484,10 @@ function createDashboardDeps(paths, {
         Promise.all(queues.map((q, i) => i === 0 ? q.getJobSchedulers(0, -1, true) : q.getJobSchedulers(0, -1, true).catch(() => []))),
         Promise.all(queues.map((q) => q.getActive(0, 0).catch(() => []))),
         // Per-scheduler stall counts (money backstop) for the cron drill-in; reuses the held client like the
-        // budget GETs. HGETALL of an absent key is `{}`, so a never-stalled deployment shows 0 stalls.
-        redis.hgetall(STALL_KEY).catch(() => ({}))
+        // budget GETs. ONE MGET for every declared cron id, so this stays one round trip however many
+        // schedulers exist. An absent key reads as an honest 0 -- a scheduler that has never stalled -- which
+        // is the same posture `readScopedBudget` takes for a scope that has never run.
+        cronIds.length > 0 ? redis.mget(...cronIds.map((id) => stallKey(id))).catch(() => []) : Promise.resolve([])
       ]);
       const workers = Array.isArray(workerList) && workerList.length > 0 ? workerList.length : "unknown";
       const counts = {};
@@ -40382,11 +40520,18 @@ function createDashboardDeps(paths, {
         queue: { pausedState, pausedPartial, counts, workers, queues: queues.length, fleetDegraded },
         budget: { day: Number(dayRaw ?? 0), week: Number(weekRaw ?? 0), month: Number(monthRaw ?? 0), tokensToday: Number(tokenRaw ?? 0) },
         schedulers: mapSchedulers(schedulerList, Date.now()),
-        schedulerStalls: stallHash ?? {},
+        // Rebuilt into the `{ id -> count }` shape the drill-in and the LIST badge already index by, so
+        // neither reader changes.
+        schedulerStalls: Object.fromEntries(cronIds.map((id, i) => [id, Number(stallCounts?.[i] ?? 0) || 0])),
         schedulerStallMax: paths.schedulerStallMax,
-        runs: listRuns({ logsDir: paths.logsDir, limit: RUNS_ON_DASHBOARD }),
+        // Merged across the fleet (issue #57, Gap 3). This host's files stay the truth and the single-host
+        // path; the mirror adds the other hosts' records. On a shared `PI_LOGS_DIR` the local read is
+        // already the merged read and the mirror adds nothing, which is why there is no second code path.
+        runs: merged.runs,
+        runHosts: merged.hosts,
+        runMirror: merged.mirror,
         settings: readSettingsView({ settingsFile: paths.settingsFile }),
-        triggers: readTriggers({ triggersPath: paths.triggersPath }),
+        triggers: triggersView,
         pauseWindows: readPauseWindows({ pauseWindowsPath: paths.pauseWindowsPath }),
         // The operator's staged third-party pi packages (REQ-GLOBAL-PI-OVERLAY), for the armed triggers'
         // trust model. Like the four reads above it is a plain file read whose fs access lives entirely in
@@ -40847,7 +40992,11 @@ function renderPanel(snapshot, width, state, styler) {
       ]
     },
     { title: "TRIGGERS", lines: toLines(renderTriggers({ schedulers: snapshot.schedulers, triggers: snapshot.triggers })) },
-    { title: "RUNS", lines: renderRunList(buildRows(snapshot, runSort), selected, 24) },
+    // The section TITLE carries the fleet fact, not a seventh cell in each row (issue #57, Gap 3). A cell
+    // would compete with jobId and target inside one `fitLine(..., inner)` and clip silently at width 80;
+    // the title answers the completeness question once rather than fifty times. Absent on a single host,
+    // so that output is byte-identical.
+    { title: runsTitle(snapshot), lines: renderRunList(buildRows(snapshot, runSort), selected, 24) },
     { title: "SETTINGS", lines: toLines(renderSettingsView(snapshot.settings)) }
   ];
   const scopedText = renderScopedLimits({ limits: snapshot.scopedLimits, scopedBudget: snapshot.scopedBudget });
@@ -41148,6 +41297,13 @@ function limitRow(l, used, inner, styler) {
   if (Number.isInteger(l.concurrent)) bits.push(styler.fg("muted", `\u2264${l.concurrent} at once`));
   return fitLine(bits.join(styler.fg("dim", "  ")), inner, styler);
 }
+function runsTitle(snapshot) {
+  const hosts = Array.isArray(snapshot?.runHosts) ? snapshot.runHosts : [];
+  if (hosts.length > 1) return `RUNS \xB7 ${hosts.length} hosts`;
+  const mirror = snapshot?.runMirror;
+  if (typeof mirror === "string" && mirror.startsWith("unreachable")) return "RUNS \xB7 this host only";
+  return "RUNS";
+}
 function runLines(rows, selected, inner, styler) {
   if (!Array.isArray(rows) || rows.length === 0) return [styler.cell("(no runs)", inner, { color: "dim" })];
   const { top, count } = runsWindow(rows.length, selected);
@@ -42121,8 +42277,8 @@ var init_setup_wizard = __esm({
     init_deployment_pointer();
     init_read_model();
     init_index();
-    RUNTIME_VERSION = "1.7.0";
-    RECEIVER_VERSION = "1.4.0";
+    RUNTIME_VERSION = "1.9.0";
+    RECEIVER_VERSION = "1.5.0";
     RUNTIME_PKG = "@edgehero/pi-dispatch";
     RECEIVER_PKG = "@edgehero/pi-dispatch-receiver";
     NUDGE_MARKER_BASENAME = "pi-dispatch-setup.nudged";
@@ -42309,8 +42465,9 @@ function registerTools(pi) {
     }),
     async execute(_toolCallId, params) {
       const paths = resolvePaths(process.env);
-      const data = params.jobId ? readRun({ logsDir: paths.logsDir, jobId: params.jobId }) : listRuns({ logsDir: paths.logsDir, limit: params.limit ?? 10 });
-      return toolText(JSON.stringify(data));
+      if (params.jobId) return toolText(JSON.stringify(readRun({ logsDir: paths.logsDir, jobId: params.jobId })));
+      const merged = await listRunsMerged({ logsDir: paths.logsDir, limit: params.limit ?? 10, url: paths.valkeyUrl });
+      return toolText(JSON.stringify(merged));
     }
   });
   pi.registerTool({
@@ -43597,7 +43754,8 @@ async function showLogs(logsDir, tokens, ctx) {
     return;
   }
   const lines = tokens[2] ? Number(tokens[2]) : void 0;
-  const tail = readLogTail({ logsDir, jobId, lines });
+  const record = readRun({ logsDir, jobId });
+  const tail = readLogTail({ logsDir, jobId, lines, host: record?.host ?? null, self: process.env.PI_WORKER_NAME ?? null });
   const custom = ctx?.ui?.custom;
   if (typeof custom !== "function") {
     const message = "logs viewer unavailable in this pi version -- raw logs are never sent to the model";
@@ -43616,7 +43774,7 @@ function makeLogViewer(jobId, tail) {
     const component = {
       render(_width) {
         if (missing) {
-          return [`logs ${jobId} -- no captured log (PI_CAPTURE_JOB_LOGS off or not found). Esc to close.`, ""];
+          return tail.elsewhere ? [`logs ${jobId} -- this run happened on ${tail.elsewhere}; its raw log stays on that host. Esc to close.`, ""] : [`logs ${jobId} -- no captured log (PI_CAPTURE_JOB_LOGS off or not found). Esc to close.`, ""];
         }
         const out = [`logs ${jobId} -- ${lines.length} line(s). Up/Down scroll, PgUp/PgDn page, Esc close.`, ""];
         for (const line of lines.slice(top, top + VIEWPORT_LINES)) out.push(line);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch-admin",
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "Operator console (a pi extension) + skill for pi-dispatch: run the pi coding agent as a self-hosted service. A /dispatch TUI for the queue, spend caps, run history, editable GitHub, GitLab, Forgejo and Azure DevOps triggers (cron/label/comment/PR/MR/work item), and scheduled pause windows — plus AI-operable, human-confirmed controls. Runs against a live pi-dispatch deployment.",
   "keywords": [
     "pi-package",