@edgehero/pi-dispatch-admin 1.6.1 → 1.8.0

package/README.md

@@ -8,6 +8,7 @@
 
 > This npm package, **`@edgehero/pi-dispatch-admin`**, is the **operator console** (a pi extension). The service itself is `@edgehero/pi-dispatch` (worker + CLI) and `@edgehero/pi-dispatch-receiver` (the webhook edge); the [main repo](https://github.com/edgehero/pi-dispatch) has the container image, docs, and SECURITY.md.
 
+
 ## How it works
 
 Every trigger produces the same job, through the same path: one queue, one container, one budget.
@@ -73,6 +74,13 @@ One command puts a live TUI over the whole deployment:
 - **AI-operable, with a human gate.** Model-callable tools let an agent change limits (global and scoped) and manage triggers and pause windows, and every **config** write pops an operator confirmation the model cannot answer, refusing outright when no operator is present. Two tools sit outside that gate on purpose: `dispatch_pause` and `dispatch_resume` write durable queue state but are reversible and spend nothing, so they carry no confirm. One more sits outside it and is **not** money-safe: `dispatch_run` enqueues a **paid** run that edits a local folder in place with no undo, bounded instead by six independent limits (the `PI_DISPATCH_RUN_ROOTS` folder allowlist, a committed per-flow `ai-trigger: allow` opt-in read at a pre-agent SHA, a dirty-tree refusal with no force option, no spend knobs on the tool, a per-hour rate limit, and the worker's daily cap). Commands sit outside its reach entirely: `dispatch_run` speaks flows only, and a chained job's request naming a `command` is refused outright, with no opt-in. Read [`SECURITY.md`](https://github.com/edgehero/pi-dispatch/blob/main/SECURITY.md) on that one before you enable it. The bundled `operate-pi-dispatch` skill teaches the agent those gates.
 - **Logs stay put.** Raw container output renders only in the overlay viewer, never into model context.
 
+## More than one machine
+
+The console reads the whole deployment, not one host. The status line names the workers when they have
+been named, `RUN_DETAIL` names the machine that ran a job, the pause switch stops every queue rather than
+the shared one, and the scheduler view spans hosts. On a single host none of that is visible, because
+there is nothing to distinguish. See [`docs/multi-host.md`](../docs/multi-host.md).
+
 ## Install
 
 ```bash

package/dist/index.mjs

@@ -8326,6 +8326,57 @@ var init_connection = __esm({
   }
 });
 
+// ../worker/src/host-registry.mjs
+function bounded(promise, ms = REGISTRY_OP_TIMEOUT_MS) {
+  return new Promise((resolve3, reject) => {
+    const t = setTimeout(() => reject(new Error("registry timeout")), ms);
+    Promise.resolve(promise).then(
+      (v) => (clearTimeout(t), resolve3(v)),
+      (e) => (clearTimeout(t), reject(e))
+    );
+  });
+}
+async function readLiveHosts(redis, { now = () => Date.now(), timeoutMs = REGISTRY_OP_TIMEOUT_MS } = {}) {
+  let names;
+  try {
+    names = await bounded(redis.smembers(HOST_SET), timeoutMs);
+    if (names !== null && names !== void 0 && !Array.isArray(names)) throw new Error("smembers did not return a list");
+  } catch (err) {
+    return { unreachable: err?.message ?? "registry unreadable" };
+  }
+  const hosts = [];
+  for (const member of names ?? []) {
+    try {
+      const row = await bounded(redis.hgetall(hostKey(member)), timeoutMs);
+      if (!row || Object.keys(row).length === 0) {
+        await bounded(redis.srem(HOST_SET, member), timeoutMs).catch(() => {
+        });
+        continue;
+      }
+      const beatAt = typeof row.beatAt === "string" && row.beatAt.trim() !== "" ? Number(row.beatAt) : NaN;
+      hosts.push({
+        ...row,
+        name: row.name || member,
+        // Derived rather than stored, so the panel can say "stale 2m" about a row that still lives.
+        // A row whose clock is AHEAD of ours reads as 0 rather than negative: the difference is the
+        // skew between two machines, which is its own signal and not this field's to report.
+        staleMs: Number.isFinite(beatAt) ? Math.max(0, now() - beatAt) : null
+      });
+    } catch {
+    }
+  }
+  hosts.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+  return { hosts };
+}
+var HOST_SET, hostKey, REGISTRY_OP_TIMEOUT_MS;
+var init_host_registry = __esm({
+  "../worker/src/host-registry.mjs"() {
+    HOST_SET = "host:live";
+    hostKey = (name) => `host:h:${name}`;
+    REGISTRY_OP_TIMEOUT_MS = 2e3;
+  }
+});
+
 // ../worker/src/job-id.mjs
 import { createHash as createHash2 } from "node:crypto";
 function localJobId({ folder, flow, task, minute }) {
@@ -9069,8 +9120,18 @@ var init_triggers = __esm({
 
 // ../worker/src/queue.mjs
 import { Queue } from "bullmq";
-function makeQueue(connection) {
-  return new Queue(QUEUE, { connection });
+function fleetQueueNames(hosts) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const h of hosts ?? []) {
+    if (h?.routes !== true && h?.routes !== "true") continue;
+    const name = h?.name;
+    if (typeof name !== "string" || !WORKER_NAME_RE.test(name)) continue;
+    seen.add(name);
+  }
+  return [QUEUE, ...[...seen].sort().map(hostQueueName)];
+}
+function makeQueue(connection, { name = QUEUE } = {}) {
+  return new Queue(name, { connection });
 }
 async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, now = /* @__PURE__ */ new Date() }) {
   const minute = now.toISOString().slice(0, 16);
@@ -9110,7 +9171,37 @@ async function enqueueLocalJob(queue, { folder, flow, task, command, provider, m
   });
   return id;
 }
-var PR_CLOSE_WORDS, QUEUE, SEMANTIC_WINDOW_MS;
+async function discoverHostQueues(redis, { timeoutMs = 2e3, count = 500 } = {}) {
+  const prefix = `bull:${QUEUE}@`;
+  const names = /* @__PURE__ */ new Set();
+  try {
+    const deadline = Date.now() + timeoutMs;
+    let cursor = "0";
+    do {
+      let timer;
+      const [next, keys] = await Promise.race([
+        redis.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", count),
+        new Promise((_, reject) => {
+          timer = setTimeout(() => reject(new Error("scan timed out")), Math.max(1, deadline - Date.now()));
+        })
+      ]).finally(() => clearTimeout(timer));
+      cursor = next;
+      for (const key of keys ?? []) {
+        const name = String(key).slice(prefix.length, -":meta".length);
+        if (WORKER_NAME_RE.test(name)) names.add(name);
+      }
+    } while (cursor !== "0" && Date.now() < deadline);
+  } catch {
+    return [];
+  }
+  return [...names].sort().map(hostQueueName);
+}
+function unionQueueNames(fromRegistry, fromKeyspace) {
+  const seen = /* @__PURE__ */ new Set([...fromRegistry ?? [], ...fromKeyspace ?? []]);
+  seen.delete(QUEUE);
+  return [QUEUE, ...[...seen].sort()];
+}
+var PR_CLOSE_WORDS, QUEUE, hostQueueName, WORKER_NAME_RE, SEMANTIC_WINDOW_MS;
 var init_queue = __esm({
   "../worker/src/queue.mjs"() {
     init_job_id();
@@ -9118,10 +9209,90 @@ var init_queue = __esm({
     init_triggers();
     PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
     QUEUE = "pi-jobs";
+    hostQueueName = (worker) => `${QUEUE}@${worker}`;
+    WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
     SEMANTIC_WINDOW_MS = 10 * 60 * 1e3;
   }
 });
 
+// ../worker/src/run-mirror.mjs
+function bounded2(promise, ms) {
+  let timer;
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error("mirror timeout")), ms);
+    })
+  ]).finally(() => clearTimeout(timer));
+}
+async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = () => Date.now(), timeoutMs = OP_TIMEOUT_MS } = {}) {
+  if (!redis) return { runs: [], degraded: "off" };
+  let ids;
+  try {
+    ids = await bounded2(redis.zrevrangebyscore(RUNS_INDEX, "+inf", `(${sinceMs}`, "LIMIT", 0, Math.max(1, limit)), timeoutMs);
+  } catch (err) {
+    return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
+  }
+  if (!Array.isArray(ids) || ids.length === 0) return { runs: [], degraded: "off" };
+  let bodies;
+  try {
+    bodies = await bounded2(redis.mget(...ids.map(runRecordKey)), timeoutMs);
+  } catch (err) {
+    return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
+  }
+  const runs = [];
+  const stale = [];
+  for (let i = 0; i < ids.length; i++) {
+    const raw = bodies?.[i];
+    if (typeof raw !== "string" || raw === "") {
+      stale.push(ids[i]);
+      continue;
+    }
+    try {
+      const rec = JSON.parse(raw);
+      if (rec && typeof rec === "object") runs.push(rec);
+    } catch {
+      stale.push(ids[i]);
+    }
+  }
+  if (stale.length > 0) {
+    try {
+      await bounded2(redis.zrem(RUNS_INDEX, ...stale), timeoutMs);
+    } catch {
+    }
+  }
+  return { runs, degraded: runs.length >= limit ? "truncated" : "ok" };
+}
+function mergeRuns(local, mirrored, { limit = 50 } = {}) {
+  const by = /* @__PURE__ */ new Map();
+  const at = (r) => {
+    const t = Date.parse(r?.endedAt ?? r?.startedAt ?? "");
+    return Number.isFinite(t) ? t : -Infinity;
+  };
+  for (const r of Array.isArray(mirrored) ? mirrored : []) if (r?.jobId) by.set(r.jobId, r);
+  for (const r of Array.isArray(local) ? local : []) {
+    if (!r?.jobId) continue;
+    const seen = by.get(r.jobId);
+    if (!seen || at(r) >= at(seen)) by.set(r.jobId, r);
+  }
+  const out = [...by.values()].sort((a, b) => at(b) - at(a));
+  return out.slice(0, Math.max(0, limit));
+}
+function hostsIn(runs) {
+  const names = /* @__PURE__ */ new Set();
+  for (const r of runs ?? []) if (typeof r?.host === "string" && r.host !== "") names.add(r.host);
+  return [...names].sort();
+}
+var RUNS_INDEX, runRecordKey, DAY_MS, OP_TIMEOUT_MS;
+var init_run_mirror = __esm({
+  "../worker/src/run-mirror.mjs"() {
+    RUNS_INDEX = "runs:index";
+    runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
+    DAY_MS = 24 * 60 * 60 * 1e3;
+    OP_TIMEOUT_MS = 2e3;
+  }
+});
+
 // ../worker/src/git-dirty.mjs
 import { execFileSync } from "node:child_process";
 function gitDirty(folder, { exec: exec4 = execFileSync } = {}) {
@@ -9935,8 +10106,8 @@ var init_graph_model = __esm({
 
 // src/costs.mjs
 function costsSinceMs(windowKey, nowMs) {
-  if (windowKey === "7d") return nowMs - 7 * DAY_MS;
-  if (windowKey === "30d") return nowMs - 30 * DAY_MS;
+  if (windowKey === "7d") return nowMs - 7 * DAY_MS2;
+  if (windowKey === "30d") return nowMs - 30 * DAY_MS2;
   const now = new Date(nowMs);
   return Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1);
 }
@@ -10059,7 +10230,7 @@ function foldCosts({ records, subscriptions, pricing, nowMs, piAiPin = null, sin
   const requested = Number.isFinite(sinceMs);
   const fromMs = requested ? Math.min(sinceMs, nowMs) : firstRunMs ?? nowMs;
   const toMs2 = nowMs;
-  const days = requested || runs.length > 0 ? Math.max(1, Math.ceil((toMs2 - fromMs) / DAY_MS)) : 0;
+  const days = requested || runs.length > 0 ? Math.max(1, Math.ceil((toMs2 - fromMs) / DAY_MS2)) : 0;
   return {
     // `firstRunMs` rides along (null when nothing ran) so a renderer can pad or trim the daily series
     // against the requested edge without re-deriving what the fold already knows.
@@ -10090,7 +10261,7 @@ function buildDaily(runs, fromMs, toMs2) {
     buckets.get(day).push(r.contribution);
   }
   const daily = [];
-  for (let cursor = Math.floor(fromMs / DAY_MS) * DAY_MS; cursor <= toMs2; cursor += DAY_MS) {
+  for (let cursor = Math.floor(fromMs / DAY_MS2) * DAY_MS2; cursor <= toMs2; cursor += DAY_MS2) {
     const day = utcDay(cursor);
     const contribs = buckets.get(day) ?? [];
     daily.push({ day, cost: combineContributions(contribs), runs: contribs.length });
@@ -10114,7 +10285,7 @@ function buildDailyByFlow(runs, fromMs, toMs2) {
   const rows = [];
   for (const [flow, buckets] of byFlow) {
     const days = [];
-    for (let cursor = Math.floor(fromMs / DAY_MS) * DAY_MS; cursor <= toMs2; cursor += DAY_MS) {
+    for (let cursor = Math.floor(fromMs / DAY_MS2) * DAY_MS2; cursor <= toMs2; cursor += DAY_MS2) {
       const day = utcDay(cursor);
       const contribs = buckets.get(day) ?? [];
       days.push({ day, cost: combineContributions(contribs), runs: contribs.length });
@@ -10358,7 +10529,7 @@ function peakWindow(w, attributed) {
   let peakRuns = 0;
   let peakTokens = 0;
   if (w.rolling) {
-    const span = w.per === "5h" ? 5 * HOUR_MS : w.per === "7d" ? 7 * DAY_MS : 30 * DAY_MS;
+    const span = w.per === "5h" ? 5 * HOUR_MS : w.per === "7d" ? 7 * DAY_MS2 : 30 * DAY_MS2;
     let left = 0;
     let runsInWindow = 0;
     let tokensInWindow = 0;
@@ -10449,11 +10620,11 @@ function medianQuad(quads) {
   }
   return quad;
 }
-var DAY_MS, HOUR_MS, COSTS_WINDOWS, QUAD_KEYS, TRIGGER_TAIL, TRIGGER_TAIL_LABELS;
+var DAY_MS2, HOUR_MS, COSTS_WINDOWS, QUAD_KEYS, TRIGGER_TAIL, TRIGGER_TAIL_LABELS;
 var init_costs = __esm({
   "src/costs.mjs"() {
     init_budget();
-    DAY_MS = 24 * 60 * 60 * 1e3;
+    DAY_MS2 = 24 * 60 * 60 * 1e3;
     HOUR_MS = 60 * 60 * 1e3;
     COSTS_WINDOWS = ["7d", "30d", "mtd"];
     QUAD_KEYS = ["input", "output", "cacheRead", "cacheWrite", "cacheWrite1h"];
@@ -10625,32 +10796,73 @@ function parseNonNegInt(raw, fallback) {
   const n = Number.parseInt(raw, 10);
   return Number.isInteger(n) && n >= 0 && String(n) === String(raw).trim() ? n : fallback;
 }
-async function readQueueState({ url, makeQueueFn = makeQueue, parseConnectionFn = parseConnection } = {}) {
+async function readQueueState({ url, makeQueueFn = makeQueue, parseConnectionFn = parseConnection, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
   let queue;
+  const queuesToClose = [];
   try {
-    queue = makeQueueFn(parseConnectionFn(url, { failFast: true }));
-    const pausedState = await queue.isPaused();
-    const counts = await queue.getJobCounts("waiting", "active", "paused", "delayed", "failed");
-    const workers = await readWorkerCount(queue);
-    return { pausedState, counts, workers };
+    const { names, hosts, blind } = await readFleetQueues({ url, redisFn, timeoutMs });
+    const fleet = blind ? { unreachable: blind } : { hosts };
+    const queues = names.map((name) => makeQueueFn(parseConnectionFn(url, { failFast: true }), { name }));
+    queuesToClose.push(...queues);
+    queue = queues[0];
+    const pausedStates = await Promise.all(queues.map((q) => q.isPaused()));
+    const pausedState = pausedStates.every(Boolean);
+    const pausedPartial = pausedStates.some(Boolean) && !pausedState;
+    const perQueue = await Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed")));
+    const counts = perQueue.reduce((acc, c) => {
+      for (const [k, v] of Object.entries(c ?? {})) acc[k] = (acc[k] ?? 0) + (Number(v) || 0);
+      return acc;
+    }, {});
+    const workerCount = await readWorkerCount(queue);
+    const resolved = resolveWorkerCount({ hosts: fleet.hosts ?? [], workerCount: typeof workerCount === "number" ? workerCount : 0 });
+    return { pausedState, ...pausedPartial && { pausedPartial }, counts, workers: resolved.count, workerNames: resolved.names };
   } catch (err) {
     return { unreachable: err?.message ?? String(err) };
   } finally {
-    if (queue) await queue.close().catch(() => {
+    for (const q of queuesToClose) await q.close().catch(() => {
     });
   }
 }
-async function setQueuePaused({ url, paused, makeQueueFn = makeQueue, parseConnectionFn = parseConnection } = {}) {
-  let queue;
+async function readFleetQueues({ url, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
+  const redis = redisFn(url);
+  redis.on?.("error", () => {
+  });
+  try {
+    const [fleet, existing] = await Promise.all([
+      withTimeout(readLiveHosts(redis, { timeoutMs: Math.max(250, Math.floor(timeoutMs / 2)) }), timeoutMs, { unreachable: "timed out reaching the registry" }),
+      discoverHostQueues(redis, { timeoutMs })
+    ]);
+    return { names: unionQueueNames(fleetQueueNames(fleet?.hosts), existing), blind: fleet?.unreachable ?? null, hosts: fleet?.hosts ?? [] };
+  } catch (err) {
+    return { names: [QUEUE], blind: err?.message ?? String(err), hosts: [] };
+  } finally {
+    try {
+      redis.disconnect?.();
+    } catch {
+    }
+  }
+}
+async function setQueuePaused({ url, paused, makeQueueFn = makeQueue, parseConnectionFn = parseConnection, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
+  const opened = [];
   try {
-    queue = makeQueueFn(parseConnectionFn(url, { failFast: true }));
-    if (paused) await queue.pause();
-    else await queue.resume();
-    return { ok: true, paused };
+    const { names, blind } = await readFleetQueues({ url, redisFn, timeoutMs });
+    const done = [];
+    for (const name of names) {
+      const q = makeQueueFn(parseConnectionFn(url, { failFast: true }), { name });
+      opened.push(q);
+      try {
+        if (paused) await q.pause();
+        else await q.resume();
+      } catch (err) {
+        return { unreachable: err?.message ?? String(err), partial: { done, failed: name, error: err?.message ?? String(err) }, blind };
+      }
+      done.push(name);
+    }
+    return { ok: true, paused, queues: done, ...blind ? { blind } : {} };
   } catch (err) {
     return { unreachable: err?.message ?? String(err) };
   } finally {
-    if (queue) await queue.close().catch(() => {
+    for (const q of opened) await q.close().catch(() => {
     });
   }
 }
@@ -10725,7 +10937,8 @@ async function enqueueDispatchRun({
   }
   let queue;
   try {
-    queue = makeQueueFn(parseConnectionFn(valkeyUrl, { failFast: true }));
+    const workerName = process.env.PI_WORKER_NAME;
+    queue = makeQueueFn(parseConnectionFn(valkeyUrl, { failFast: true }), { ...workerName ? { name: hostQueueName(workerName) } : {} });
     const jobId = await enqueueLocalJob(queue, { folder, flow, task });
     return { ok: true, jobId };
   } catch (err) {
@@ -10872,6 +11085,29 @@ async function readScopedBudget({ url, limits, redisFn = makeRedisClient, timeou
     }
   }
 }
+async function readHosts({ url, redisFn = makeRedisClient, timeoutMs = 2500, now = () => Date.now() } = {}) {
+  let redis;
+  try {
+    parseConnection(url, { failFast: true });
+    redis = redisFn(url);
+    const res = await withTimeout(readLiveHosts(redis, { now, timeoutMs: Math.max(250, Math.floor(timeoutMs / 2)) }), timeoutMs, { unreachable: "timed out reaching the registry" });
+    if (res.unreachable) return { unreachable: res.unreachable };
+    return { hosts: res.hosts };
+  } catch (err) {
+    return { unreachable: err?.message ?? String(err) };
+  } finally {
+    if (redis) redis.disconnect?.();
+  }
+}
+function resolveWorkerCount({ hosts, workerCount }) {
+  if (Array.isArray(hosts) && hosts.length > 0) {
+    const declared = hosts.filter((h) => h?.routes === true || h?.routes === "true");
+    const names = declared.map((h) => h.name).filter(Boolean).sort();
+    return { count: hosts.length, names };
+  }
+  if (typeof workerCount === "number" && workerCount > 0) return { count: workerCount, names: [] };
+  return { count: "unknown", names: [] };
+}
 async function readHeldJobs({ url, limit = 20, redisFn = makeRedisClient, timeoutMs = 2500, now = () => Date.now() } = {}) {
   let redis;
   try {
@@ -10990,17 +11226,22 @@ async function readSchedulers({
   url,
   makeQueueFn = makeQueue,
   parseConnectionFn = parseConnection,
+  redisFn = makeRedisClient,
+  timeoutMs = 2500,
   now = Date.now
 } = {}) {
-  let queue;
+  const opened = [];
   try {
-    queue = makeQueueFn(parseConnectionFn(url, { failFast: true }));
-    const list = await queue.getJobSchedulers(0, -1, true);
-    return mapSchedulers(list, now());
+    const fleet = await readHosts({ url, redisFn, timeoutMs });
+    const queues = fleetQueueNames(fleet.hosts).map((name) => makeQueueFn(parseConnectionFn(url, { failFast: true }), { name }));
+    opened.push(...queues);
+    const [primary, ...rest] = queues;
+    const lists = [await primary.getJobSchedulers(0, -1, true), ...await Promise.all(rest.map((q) => q.getJobSchedulers(0, -1, true).catch(() => [])))];
+    return mapSchedulers(lists.flat(), now());
   } catch (err) {
     return { unreachable: err?.message ?? String(err) };
   } finally {
-    if (queue) await queue.close().catch(() => {
+    for (const q of opened) await q.close().catch(() => {
     });
   }
 }
@@ -11046,6 +11287,35 @@ function withTimeout(promise, ms, fallback) {
   });
   return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
 }
+async function mergedRunsOn(redis, { logsDir, limit = 10, fs = nodeFs3, timeoutMs = 2500 } = {}) {
+  const local = listRuns({ logsDir, limit, fs });
+  if (!Array.isArray(local)) return { runs: [], hosts: [], mirror: "off", ...local };
+  try {
+    const { runs: mirrored, degraded } = await readMirroredRuns(redis, { limit: Math.max(limit * 2, 50), timeoutMs });
+    const runs = mergeRuns(local, mirrored, { limit });
+    return { runs, hosts: hostsIn(runs), mirror: degraded };
+  } catch (err) {
+    return { runs: local, hosts: hostsIn(local), mirror: `unreachable (${err?.message ?? "?"})` };
+  }
+}
+async function listRunsMerged({ logsDir, limit = 10, url, fs = nodeFs3, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
+  let redis;
+  try {
+    redis = redisFn(url);
+    redis.on?.("error", () => {
+    });
+    return await mergedRunsOn(redis, { logsDir, limit, fs, timeoutMs });
+  } catch (err) {
+    const local = listRuns({ logsDir, limit, fs });
+    const runs = Array.isArray(local) ? local : [];
+    return { runs, hosts: hostsIn(runs), mirror: `unreachable (${err?.message ?? "?"})` };
+  } finally {
+    try {
+      redis?.disconnect?.();
+    } catch {
+    }
+  }
+}
 function listRuns({ logsDir, limit = 10, fs = nodeFs3 }) {
   const cap = clampLimit(limit);
   let names;
@@ -11088,12 +11358,13 @@ function scanRunRecords({ logsDir, sinceMs, nowMs = Date.now(), fs = nodeFs3 })
 function readRun({ logsDir, jobId, fs = nodeFs3 }) {
   return readJsonFile(join2(logsDir, `${sanitizeJobId(jobId)}.json`), fs);
 }
-function readLogTail({ logsDir, jobId, lines = 200, fs = nodeFs3 }) {
+function readLogTail({ logsDir, jobId, lines = 200, fs = nodeFs3, host = null, self = null }) {
   let text;
   try {
     text = fs.readFileSync(join2(logsDir, `${sanitizeJobId(jobId)}.log`), "utf8");
   } catch {
-    return { missing: true };
+    const foreign = typeof host === "string" && host !== "" && typeof self === "string" && self !== "" && host !== self;
+    return foreign ? { missing: true, elsewhere: host } : { missing: true };
   }
   const all = text.split("\n").filter((line) => line !== "");
   const cap = clampLines(lines);
@@ -11607,7 +11878,9 @@ var init_read_model = __esm({
     init_wait_state();
     init_subscriptions();
     init_connection();
+    init_host_registry();
     init_queue();
+    init_run_mirror();
     init_flow_gate();
     init_git_dirty();
     init_packages();
@@ -39692,7 +39965,9 @@ function renderStatus(queue) {
   const counts = queue.counts ?? {};
   const line = ["waiting", "active", "paused", "delayed", "failed"].map((k) => `${k} ${counts[k] ?? 0}`).join("  ");
   const workers = queue.workers === void 0 ? "unknown" : queue.workers;
-  return [`Queue: ${queue.pausedState ? "paused" : "running"}`, `  ${line}`, `  workers: ${workers}`].join("\n");
+  const named = Array.isArray(queue.workerNames) && queue.workerNames.length > 0 ? ` (${queue.workerNames.join(", ")})` : "";
+  const state = queue.pausedPartial ? "PARTIALLY paused" : queue.pausedState ? "paused" : "running";
+  return [`Queue: ${state}`, `  ${line}`, `  workers: ${workers}${named}`].join("\n");
 }
 function renderRuns(runs) {
   if (runs && runs.unreachable) return `Runs: unreachable (${runs.unreachable})`;
@@ -40131,27 +40406,68 @@ async function heldJobs(redis) {
   rows.sort((a, b) => (b.waitedMs ?? -1) - (a.waitedMs ?? -1));
   return { rows: rows.slice(0, HELD_LIMIT), more: Math.max(0, rows.length - HELD_LIMIT), truncated: ids.length > HELD_HYDRATE_MAX2 };
 }
-function createDashboardDeps(paths) {
-  const queue = makeQueue(parseConnection(paths.valkeyUrl, { failFast: true }));
-  const redis = makeRedisClient(paths.valkeyUrl);
+function createDashboardDeps(paths, {
+  makeQueueFn = makeQueue,
+  parseConnectionFn = parseConnection,
+  redisFn = makeRedisClient,
+  readLiveHostsFn = readLiveHosts,
+  discoverHostQueuesFn = discoverHostQueues
+} = {}) {
+  const queue = makeQueueFn(parseConnectionFn(paths.valkeyUrl, { failFast: true }));
+  const redis = redisFn(paths.valkeyUrl);
+  const pool = /* @__PURE__ */ new Map([[queue.name, queue]]);
+  let lastNames = [queue.name];
+  let fleetDegraded = null;
+  const fleetQueues = async () => {
+    const fleet = await readLiveHostsFn(redis, { timeoutMs: FLEET_READ_TIMEOUT_MS }).catch((err) => ({
+      unreachable: err?.message ?? String(err)
+    }));
+    if (fleet?.unreachable || !Array.isArray(fleet?.hosts)) {
+      fleetDegraded = fleet?.unreachable ?? "registry unreadable";
+    } else {
+      fleetDegraded = null;
+      lastNames = fleetQueueNames(fleet.hosts);
+    }
+    for (const name of lastNames) {
+      if (!pool.has(name)) pool.set(name, makeQueueFn(parseConnectionFn(paths.valkeyUrl, { failFast: true }), { name }));
+    }
+    return lastNames.map((n) => pool.get(n));
+  };
+  const killSwitchQueues = async () => {
+    const existing = await discoverHostQueuesFn(redis, { timeoutMs: FLEET_READ_TIMEOUT_MS });
+    for (const name of unionQueueNames(lastNames, existing)) {
+      if (!pool.has(name)) pool.set(name, makeQueueFn(parseConnectionFn(paths.valkeyUrl, { failFast: true }), { name }));
+    }
+    return unionQueueNames(lastNames, existing).map((n) => pool.get(n));
+  };
   return {
     async fetchSnapshot() {
-      const [pausedState, counts, workerList, dayRaw, weekRaw, monthRaw, tokenRaw, schedulerList, activeList, stallHash] = await Promise.all([
-        queue.isPaused(),
-        queue.getJobCounts("waiting", "active", "paused", "delayed", "failed"),
+      const queues = await fleetQueues();
+      const merged = await mergedRunsOn(redis, { logsDir: paths.logsDir, limit: RUNS_ON_DASHBOARD }).catch(() => ({ runs: [], hosts: [], mirror: "off" }));
+      const [pausedStates, countsPer, workerList, dayRaw, weekRaw, monthRaw, tokenRaw, schedulerLists, activeLists, stallHash] = await Promise.all([
+        Promise.all(queues.map((q) => q.isPaused())),
+        Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed"))),
         queue.getWorkers().catch(() => []),
         redis.get(dayKey()),
         redis.get(weekKey()),
         redis.get(monthKey()),
         redis.get(tokenDayKey()),
         // issue #25 daily token spend (budget:t:YYYY-MM-DD)
-        queue.getJobSchedulers(0, -1, true),
-        queue.getActive(0, 0).catch(() => []),
+        // The shared queue's failure is UNCAUGHT (the panel must not draw a scheduler list it could not
+        // read), a host queue's is caught: one unreachable host degrades one host's triggers, never the view.
+        Promise.all(queues.map((q, i) => i === 0 ? q.getJobSchedulers(0, -1, true) : q.getJobSchedulers(0, -1, true).catch(() => []))),
+        Promise.all(queues.map((q) => q.getActive(0, 0).catch(() => []))),
         // Per-scheduler stall counts (money backstop) for the cron drill-in; reuses the held client like the
         // budget GETs. HGETALL of an absent key is `{}`, so a never-stalled deployment shows 0 stalls.
         redis.hgetall(STALL_KEY).catch(() => ({}))
       ]);
       const workers = Array.isArray(workerList) && workerList.length > 0 ? workerList.length : "unknown";
+      const counts = {};
+      for (const c of countsPer) for (const [k, v] of Object.entries(c ?? {})) counts[k] = (counts[k] ?? 0) + Number(v ?? 0);
+      const pausedState = pausedStates.every(Boolean);
+      const pausedPartial = !pausedState && pausedStates.some(Boolean);
+      const schedulerList = schedulerLists.flat();
+      const activeList = activeLists.flat();
       const scopedLimits = readScopedLimits({ scopedLimitsPath: paths.scopedLimitsPath });
       const limitRows = Array.isArray(scopedLimits?.limits) ? scopedLimits.limits : [];
       const scopedNow = /* @__PURE__ */ new Date();
@@ -40173,12 +40489,17 @@ function createDashboardDeps(paths) {
         held,
         scopedLimits,
         scopedBudget,
-        queue: { pausedState, counts, workers },
+        queue: { pausedState, pausedPartial, counts, workers, queues: queues.length, fleetDegraded },
         budget: { day: Number(dayRaw ?? 0), week: Number(weekRaw ?? 0), month: Number(monthRaw ?? 0), tokensToday: Number(tokenRaw ?? 0) },
         schedulers: mapSchedulers(schedulerList, Date.now()),
         schedulerStalls: stallHash ?? {},
         schedulerStallMax: paths.schedulerStallMax,
-        runs: listRuns({ logsDir: paths.logsDir, limit: RUNS_ON_DASHBOARD }),
+        // Merged across the fleet (issue #57, Gap 3). This host's files stay the truth and the single-host
+        // path; the mirror adds the other hosts' records. On a shared `PI_LOGS_DIR` the local read is
+        // already the merged read and the mirror adds nothing, which is why there is no second code path.
+        runs: merged.runs,
+        runHosts: merged.hosts,
+        runMirror: merged.mirror,
         settings: readSettingsView({ settingsFile: paths.settingsFile }),
         triggers: readTriggers({ triggersPath: paths.triggersPath }),
         pauseWindows: readPauseWindows({ pauseWindowsPath: paths.pauseWindowsPath }),
@@ -40193,15 +40514,19 @@ function createDashboardDeps(paths) {
       };
     },
     async pause() {
-      await queue.pause();
+      await fleetQueues();
+      for (const q of await killSwitchQueues()) await q.pause();
     },
     async resume() {
-      await queue.resume();
+      await fleetQueues();
+      for (const q of await killSwitchQueues()) await q.resume();
     },
     async dispose() {
-      try {
-        await queue.close();
-      } catch {
+      for (const q of pool.values()) {
+        try {
+          await q.close();
+        } catch {
+        }
       }
       try {
         redis.disconnect();
@@ -40637,7 +40962,11 @@ function renderPanel(snapshot, width, state, styler) {
       ]
     },
     { title: "TRIGGERS", lines: toLines(renderTriggers({ schedulers: snapshot.schedulers, triggers: snapshot.triggers })) },
-    { title: "RUNS", lines: renderRunList(buildRows(snapshot, runSort), selected, 24) },
+    // The section TITLE carries the fleet fact, not a seventh cell in each row (issue #57, Gap 3). A cell
+    // would compete with jobId and target inside one `fitLine(..., inner)` and clip silently at width 80;
+    // the title answers the completeness question once rather than fifty times. Absent on a single host,
+    // so that output is byte-identical.
+    { title: runsTitle(snapshot), lines: renderRunList(buildRows(snapshot, runSort), selected, 24) },
     { title: "SETTINGS", lines: toLines(renderSettingsView(snapshot.settings)) }
   ];
   const scopedText = renderScopedLimits({ limits: snapshot.scopedLimits, scopedBudget: snapshot.scopedBudget });
@@ -40743,11 +41072,11 @@ function statusHeader(queue, inner, styler) {
     return styler.cell(`queue unreachable (${queue?.unreachable ?? "?"})`, inner, { color: "error" });
   }
   const c = queue.counts ?? {};
-  const running = !queue.pausedState;
+  const running = !queue.pausedState && !queue.pausedPartial;
   const failed = Number(c.failed ?? 0);
   const stateColor = running ? "success" : "warning";
   const dot = styler.fg(stateColor, "\u25CF");
-  const word = styler.bold(styler.fg(stateColor, running ? "RUNNING" : "PAUSED"));
+  const word = styler.bold(styler.fg(stateColor, running ? "RUNNING" : queue.pausedPartial ? "PART PAUSED" : "PAUSED"));
   const sep2 = styler.fg("dim", " \xB7 ");
   const delayed = Number(c.delayed ?? 0);
   const vitals = `${c.waiting ?? 0} waiting` + sep2 + `${c.active ?? 0} active` + sep2 + (delayed > 0 ? `${delayed} delayed` + sep2 : "") + (failed > 0 ? styler.fg("error", `${failed} failed`) : `${failed} failed`) + sep2 + `${queue.workers ?? "?"} workers`;
@@ -40938,6 +41267,13 @@ function limitRow(l, used, inner, styler) {
   if (Number.isInteger(l.concurrent)) bits.push(styler.fg("muted", `\u2264${l.concurrent} at once`));
   return fitLine(bits.join(styler.fg("dim", "  ")), inner, styler);
 }
+function runsTitle(snapshot) {
+  const hosts = Array.isArray(snapshot?.runHosts) ? snapshot.runHosts : [];
+  if (hosts.length > 1) return `RUNS \xB7 ${hosts.length} hosts`;
+  const mirror = snapshot?.runMirror;
+  if (typeof mirror === "string" && mirror.startsWith("unreachable")) return "RUNS \xB7 this host only";
+  return "RUNS";
+}
 function runLines(rows, selected, inner, styler) {
   if (!Array.isArray(rows) || rows.length === 0) return [styler.cell("(no runs)", inner, { color: "dim" })];
   const { top, count } = runsWindow(rows.length, selected);
@@ -41267,6 +41603,7 @@ function renderRunDetail(record, inner, styler, allRuns = [], sandbox = null) {
   const endMs = toMs(r.endedAt);
   const dur = Number.isFinite(startMs) && Number.isFinite(endMs) && endMs >= startMs ? ` (${formatDuration(endMs - startMs)})` : "";
   out.push(kv("timing", `${fmtStamp(r.startedAt)} \u2192 ${fmtStamp(r.endedAt)}${dur}`));
+  if (r.host) out.push(kv("host", String(r.host)));
   const turnBits = [`${show(r.turns)} turns`, `exit ${show(r.exitCode)}`];
   if (r.budgetReserved !== null && r.budgetReserved !== void 0) turnBits.push(`${r.budgetReserved} budget slot`);
   if (r.attempt !== null && r.attempt !== void 0) turnBits.push(`attempt ${r.attempt}`);
@@ -41346,12 +41683,13 @@ function runDetailHints(inner, styler, canOpen = false, canCopy = false, copiedN
   if (copiedNote) return fitLine(styler.fg("success", copiedNote) + styler.fg("dim", "  \xB7  ") + line, inner, styler);
   return fitLine(line, inner, styler);
 }
-var KEY_HINTS, RUNS_ON_DASHBOARD, RUNS_VIEWPORT, RUN_SORTS, REFRESH_MS, TAIL_LINES, TAIL_VIEWPORT, MIN_WIDTH2, DRILL_WIDTH, HELD_LIMIT, HELD_HYDRATE_MAX2, KIND_COLOR, KIND_WIDTH, FRAME_CHROME_ROWS, HELD_ON_DASHBOARD;
+var KEY_HINTS, RUNS_ON_DASHBOARD, RUNS_VIEWPORT, RUN_SORTS, REFRESH_MS, TAIL_LINES, TAIL_VIEWPORT, MIN_WIDTH2, DRILL_WIDTH, HELD_LIMIT, HELD_HYDRATE_MAX2, FLEET_READ_TIMEOUT_MS, KIND_COLOR, KIND_WIDTH, FRAME_CHROME_ROWS, HELD_ON_DASHBOARD;
 var init_dashboard = __esm({
   "src/dashboard.ts"() {
     init_budget();
     init_connection();
     init_queue();
+    init_host_registry();
     init_scheduler_stall_guard();
     init_pause_windows();
     init_read_model();
@@ -41371,6 +41709,7 @@ var init_dashboard = __esm({
     DRILL_WIDTH = 70;
     HELD_LIMIT = 20;
     HELD_HYDRATE_MAX2 = 200;
+    FLEET_READ_TIMEOUT_MS = 2e3;
     KIND_COLOR = { cron: "accent", label: "syntaxType", comment: "syntaxKeyword", pull_request: "syntaxFunction", issue: "syntaxString" };
     KIND_WIDTH = 13;
     FRAME_CHROME_ROWS = 4;
@@ -41908,8 +42247,8 @@ var init_setup_wizard = __esm({
     init_deployment_pointer();
     init_read_model();
     init_index();
-    RUNTIME_VERSION = "1.6.1";
-    RECEIVER_VERSION = "1.4.0";
+    RUNTIME_VERSION = "1.8.0";
+    RECEIVER_VERSION = "1.5.0";
     RUNTIME_PKG = "@edgehero/pi-dispatch";
     RECEIVER_PKG = "@edgehero/pi-dispatch-receiver";
     NUDGE_MARKER_BASENAME = "pi-dispatch-setup.nudged";
@@ -42089,15 +42428,16 @@ function registerTools(pi) {
   pi.registerTool({
     name: "dispatch_runs",
     label: "pi-dispatch runs",
-    description: "Read-only. Returns structured, PII-free run records from the durable run history. Raw job logs are not available to tools \u2014 ask the user to run /dispatch logs.",
+    description: "Read-only. Returns structured run records (PII-free by construction, plus the worker name when the operator declared one) from the durable run history. Raw job logs are not available to tools \u2014 ask the user to run /dispatch logs.",
     parameters: typebox_exports.Object({
       limit: typebox_exports.Optional(typebox_exports.Integer({ minimum: 1, maximum: 50 })),
       jobId: typebox_exports.Optional(typebox_exports.String())
     }),
     async execute(_toolCallId, params) {
       const paths = resolvePaths(process.env);
-      const data = params.jobId ? readRun({ logsDir: paths.logsDir, jobId: params.jobId }) : listRuns({ logsDir: paths.logsDir, limit: params.limit ?? 10 });
-      return toolText(JSON.stringify(data));
+      if (params.jobId) return toolText(JSON.stringify(readRun({ logsDir: paths.logsDir, jobId: params.jobId })));
+      const merged = await listRunsMerged({ logsDir: paths.logsDir, limit: params.limit ?? 10, url: paths.valkeyUrl });
+      return toolText(JSON.stringify(merged));
     }
   });
   pi.registerTool({
@@ -42128,7 +42468,10 @@ function registerTools(pi) {
       if (res.unreachable) {
         throw new Error(`could not reach the queue at ${paths.valkeyUrl}: ${res.unreachable}`);
       }
-      return toolText("paused");
+      if (res.partial) {
+        throw new Error(`paused ${res.partial.done.join(", ")} but FAILED at ${res.partial.failed} \u2014 the deployment is half paused: ${res.partial.error}`);
+      }
+      return toolText(`paused${res.queues.length > 1 ? ` (${res.queues.length} queues)` : ""}`);
     }
   });
   pi.registerTool({
@@ -42143,7 +42486,10 @@ function registerTools(pi) {
       if (res.unreachable) {
         throw new Error(`could not reach the queue at ${paths.valkeyUrl}: ${res.unreachable}`);
       }
-      return toolText("resumed");
+      if (res.partial) {
+        throw new Error(`resumed ${res.partial.done.join(", ")} but FAILED at ${res.partial.failed} \u2014 the deployment is half resumed: ${res.partial.error}`);
+      }
+      return toolText(`resumed${res.queues.length > 1 ? ` (${res.queues.length} queues)` : ""}`);
     }
   });
   pi.registerTool({
@@ -42758,11 +43104,15 @@ ${scopedText}` : ""}`);
       const paused = sub === "pause";
       const res = await setQueuePaused({ url: paths.valkeyUrl, paused });
       if (res.unreachable) {
-        notify?.(`could not reach Valkey at ${paths.valkeyUrl} \u2014 is it running? (docker compose up)`, "error");
+        notify?.(
+          res.partial ? `${paused ? "paused" : "resumed"} ${res.partial.done.join(", ")} but FAILED at ${res.partial.failed} \u2014 the deployment is half ${paused ? "paused" : "resumed"}` : `could not reach Valkey at ${paths.valkeyUrl} \u2014 is it running? (docker compose up)`,
+          "error"
+        );
         return;
       }
+      const span = res.blind ? ` (fleet unknown: ${res.blind}, only ${res.queues[0]})` : res.queues.length > 1 ? ` (${res.queues.length} queues)` : "";
       notify?.(
-        paused ? "paused \u2014 worker will stop taking new jobs (jobs still enqueue; durable, survives restart)" : "resumed",
+        paused ? `paused \u2014 worker will stop taking new jobs (jobs still enqueue; durable, survives restart)${span}` : `resumed${span}`,
         "info"
       );
       return;
@@ -43374,7 +43724,8 @@ async function showLogs(logsDir, tokens, ctx) {
     return;
   }
   const lines = tokens[2] ? Number(tokens[2]) : void 0;
-  const tail = readLogTail({ logsDir, jobId, lines });
+  const record = readRun({ logsDir, jobId });
+  const tail = readLogTail({ logsDir, jobId, lines, host: record?.host ?? null, self: process.env.PI_WORKER_NAME ?? null });
   const custom = ctx?.ui?.custom;
   if (typeof custom !== "function") {
     const message = "logs viewer unavailable in this pi version -- raw logs are never sent to the model";
@@ -43393,7 +43744,7 @@ function makeLogViewer(jobId, tail) {
     const component = {
       render(_width) {
         if (missing) {
-          return [`logs ${jobId} -- no captured log (PI_CAPTURE_JOB_LOGS off or not found). Esc to close.`, ""];
+          return tail.elsewhere ? [`logs ${jobId} -- this run happened on ${tail.elsewhere}; its raw log stays on that host. Esc to close.`, ""] : [`logs ${jobId} -- no captured log (PI_CAPTURE_JOB_LOGS off or not found). Esc to close.`, ""];
         }
         const out = [`logs ${jobId} -- ${lines.length} line(s). Up/Down scroll, PgUp/PgDn page, Esc close.`, ""];
         for (const line of lines.slice(top, top + VIEWPORT_LINES)) out.push(line);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch-admin",
-  "version": "1.6.1",
+  "version": "1.8.0",
   "description": "Operator console (a pi extension) + skill for pi-dispatch: run the pi coding agent as a self-hosted service. A /dispatch TUI for the queue, spend caps, run history, editable GitHub, GitLab, Forgejo and Azure DevOps triggers (cron/label/comment/PR/MR/work item), and scheduled pause windows — plus AI-operable, human-confirmed controls. Runs against a live pi-dispatch deployment.",
   "keywords": [
     "pi-package",