@edgehero/pi-dispatch-admin 1.6.1 → 1.7.0

package/README.md

@@ -8,6 +8,7 @@
 
 > This npm package, **`@edgehero/pi-dispatch-admin`**, is the **operator console** (a pi extension). The service itself is `@edgehero/pi-dispatch` (worker + CLI) and `@edgehero/pi-dispatch-receiver` (the webhook edge); the [main repo](https://github.com/edgehero/pi-dispatch) has the container image, docs, and SECURITY.md.
 
+
 ## How it works
 
 Every trigger produces the same job, through the same path: one queue, one container, one budget.
@@ -73,6 +74,13 @@ One command puts a live TUI over the whole deployment:
 - **AI-operable, with a human gate.** Model-callable tools let an agent change limits (global and scoped) and manage triggers and pause windows, and every **config** write pops an operator confirmation the model cannot answer, refusing outright when no operator is present. Two tools sit outside that gate on purpose: `dispatch_pause` and `dispatch_resume` write durable queue state but are reversible and spend nothing, so they carry no confirm. One more sits outside it and is **not** money-safe: `dispatch_run` enqueues a **paid** run that edits a local folder in place with no undo, bounded instead by six independent limits (the `PI_DISPATCH_RUN_ROOTS` folder allowlist, a committed per-flow `ai-trigger: allow` opt-in read at a pre-agent SHA, a dirty-tree refusal with no force option, no spend knobs on the tool, a per-hour rate limit, and the worker's daily cap). Commands sit outside its reach entirely: `dispatch_run` speaks flows only, and a chained job's request naming a `command` is refused outright, with no opt-in. Read [`SECURITY.md`](https://github.com/edgehero/pi-dispatch/blob/main/SECURITY.md) on that one before you enable it. The bundled `operate-pi-dispatch` skill teaches the agent those gates.
 - **Logs stay put.** Raw container output renders only in the overlay viewer, never into model context.
 
+## More than one machine
+
+The console reads the whole deployment, not one host. The status line names the workers when they have
+been named, `RUN_DETAIL` names the machine that ran a job, the pause switch stops every queue rather than
+the shared one, and the scheduler view spans hosts. On a single host none of that is visible, because
+there is nothing to distinguish. See [`docs/multi-host.md`](../docs/multi-host.md).
+
 ## Install
 
 ```bash

package/dist/index.mjs

@@ -8326,6 +8326,57 @@ var init_connection = __esm({
   }
 });
 
+// ../worker/src/host-registry.mjs
+function bounded(promise, ms = REGISTRY_OP_TIMEOUT_MS) {
+  return new Promise((resolve3, reject) => {
+    const t = setTimeout(() => reject(new Error("registry timeout")), ms);
+    Promise.resolve(promise).then(
+      (v) => (clearTimeout(t), resolve3(v)),
+      (e) => (clearTimeout(t), reject(e))
+    );
+  });
+}
+async function readLiveHosts(redis, { now = () => Date.now(), timeoutMs = REGISTRY_OP_TIMEOUT_MS } = {}) {
+  let names;
+  try {
+    names = await bounded(redis.smembers(HOST_SET), timeoutMs);
+    if (names !== null && names !== void 0 && !Array.isArray(names)) throw new Error("smembers did not return a list");
+  } catch (err) {
+    return { unreachable: err?.message ?? "registry unreadable" };
+  }
+  const hosts = [];
+  for (const member of names ?? []) {
+    try {
+      const row = await bounded(redis.hgetall(hostKey(member)), timeoutMs);
+      if (!row || Object.keys(row).length === 0) {
+        await bounded(redis.srem(HOST_SET, member), timeoutMs).catch(() => {
+        });
+        continue;
+      }
+      const beatAt = typeof row.beatAt === "string" && row.beatAt.trim() !== "" ? Number(row.beatAt) : NaN;
+      hosts.push({
+        ...row,
+        name: row.name || member,
+        // Derived rather than stored, so the panel can say "stale 2m" about a row that still lives.
+        // A row whose clock is AHEAD of ours reads as 0 rather than negative: the difference is the
+        // skew between two machines, which is its own signal and not this field's to report.
+        staleMs: Number.isFinite(beatAt) ? Math.max(0, now() - beatAt) : null
+      });
+    } catch {
+    }
+  }
+  hosts.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+  return { hosts };
+}
+var HOST_SET, hostKey, REGISTRY_OP_TIMEOUT_MS;
+var init_host_registry = __esm({
+  "../worker/src/host-registry.mjs"() {
+    HOST_SET = "host:live";
+    hostKey = (name) => `host:h:${name}`;
+    REGISTRY_OP_TIMEOUT_MS = 2e3;
+  }
+});
+
 // ../worker/src/job-id.mjs
 import { createHash as createHash2 } from "node:crypto";
 function localJobId({ folder, flow, task, minute }) {
@@ -9069,8 +9120,18 @@ var init_triggers = __esm({
 
 // ../worker/src/queue.mjs
 import { Queue } from "bullmq";
-function makeQueue(connection) {
-  return new Queue(QUEUE, { connection });
+function fleetQueueNames(hosts) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const h of hosts ?? []) {
+    if (h?.routes !== true && h?.routes !== "true") continue;
+    const name = h?.name;
+    if (typeof name !== "string" || !WORKER_NAME_RE.test(name)) continue;
+    seen.add(name);
+  }
+  return [QUEUE, ...[...seen].sort().map(hostQueueName)];
+}
+function makeQueue(connection, { name = QUEUE } = {}) {
+  return new Queue(name, { connection });
 }
 async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, now = /* @__PURE__ */ new Date() }) {
   const minute = now.toISOString().slice(0, 16);
@@ -9110,7 +9171,37 @@ async function enqueueLocalJob(queue, { folder, flow, task, command, provider, m
   });
   return id;
 }
-var PR_CLOSE_WORDS, QUEUE, SEMANTIC_WINDOW_MS;
+async function discoverHostQueues(redis, { timeoutMs = 2e3, count = 500 } = {}) {
+  const prefix = `bull:${QUEUE}@`;
+  const names = /* @__PURE__ */ new Set();
+  try {
+    const deadline = Date.now() + timeoutMs;
+    let cursor = "0";
+    do {
+      let timer;
+      const [next, keys] = await Promise.race([
+        redis.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", count),
+        new Promise((_, reject) => {
+          timer = setTimeout(() => reject(new Error("scan timed out")), Math.max(1, deadline - Date.now()));
+        })
+      ]).finally(() => clearTimeout(timer));
+      cursor = next;
+      for (const key of keys ?? []) {
+        const name = String(key).slice(prefix.length, -":meta".length);
+        if (WORKER_NAME_RE.test(name)) names.add(name);
+      }
+    } while (cursor !== "0" && Date.now() < deadline);
+  } catch {
+    return [];
+  }
+  return [...names].sort().map(hostQueueName);
+}
+function unionQueueNames(fromRegistry, fromKeyspace) {
+  const seen = /* @__PURE__ */ new Set([...fromRegistry ?? [], ...fromKeyspace ?? []]);
+  seen.delete(QUEUE);
+  return [QUEUE, ...[...seen].sort()];
+}
+var PR_CLOSE_WORDS, QUEUE, hostQueueName, WORKER_NAME_RE, SEMANTIC_WINDOW_MS;
 var init_queue = __esm({
   "../worker/src/queue.mjs"() {
     init_job_id();
@@ -9118,6 +9209,8 @@ var init_queue = __esm({
     init_triggers();
     PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
     QUEUE = "pi-jobs";
+    hostQueueName = (worker) => `${QUEUE}@${worker}`;
+    WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
     SEMANTIC_WINDOW_MS = 10 * 60 * 1e3;
   }
 });
@@ -10625,32 +10718,73 @@ function parseNonNegInt(raw, fallback) {
   const n = Number.parseInt(raw, 10);
   return Number.isInteger(n) && n >= 0 && String(n) === String(raw).trim() ? n : fallback;
 }
-async function readQueueState({ url, makeQueueFn = makeQueue, parseConnectionFn = parseConnection } = {}) {
+async function readQueueState({ url, makeQueueFn = makeQueue, parseConnectionFn = parseConnection, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
   let queue;
+  const queuesToClose = [];
   try {
-    queue = makeQueueFn(parseConnectionFn(url, { failFast: true }));
-    const pausedState = await queue.isPaused();
-    const counts = await queue.getJobCounts("waiting", "active", "paused", "delayed", "failed");
-    const workers = await readWorkerCount(queue);
-    return { pausedState, counts, workers };
+    const { names, hosts, blind } = await readFleetQueues({ url, redisFn, timeoutMs });
+    const fleet = blind ? { unreachable: blind } : { hosts };
+    const queues = names.map((name) => makeQueueFn(parseConnectionFn(url, { failFast: true }), { name }));
+    queuesToClose.push(...queues);
+    queue = queues[0];
+    const pausedStates = await Promise.all(queues.map((q) => q.isPaused()));
+    const pausedState = pausedStates.every(Boolean);
+    const pausedPartial = pausedStates.some(Boolean) && !pausedState;
+    const perQueue = await Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed")));
+    const counts = perQueue.reduce((acc, c) => {
+      for (const [k, v] of Object.entries(c ?? {})) acc[k] = (acc[k] ?? 0) + (Number(v) || 0);
+      return acc;
+    }, {});
+    const workerCount = await readWorkerCount(queue);
+    const resolved = resolveWorkerCount({ hosts: fleet.hosts ?? [], workerCount: typeof workerCount === "number" ? workerCount : 0 });
+    return { pausedState, ...pausedPartial && { pausedPartial }, counts, workers: resolved.count, workerNames: resolved.names };
   } catch (err) {
     return { unreachable: err?.message ?? String(err) };
   } finally {
-    if (queue) await queue.close().catch(() => {
+    for (const q of queuesToClose) await q.close().catch(() => {
     });
   }
 }
-async function setQueuePaused({ url, paused, makeQueueFn = makeQueue, parseConnectionFn = parseConnection } = {}) {
-  let queue;
+async function readFleetQueues({ url, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
+  const redis = redisFn(url);
+  redis.on?.("error", () => {
+  });
+  try {
+    const [fleet, existing] = await Promise.all([
+      withTimeout(readLiveHosts(redis, { timeoutMs: Math.max(250, Math.floor(timeoutMs / 2)) }), timeoutMs, { unreachable: "timed out reaching the registry" }),
+      discoverHostQueues(redis, { timeoutMs })
+    ]);
+    return { names: unionQueueNames(fleetQueueNames(fleet?.hosts), existing), blind: fleet?.unreachable ?? null, hosts: fleet?.hosts ?? [] };
+  } catch (err) {
+    return { names: [QUEUE], blind: err?.message ?? String(err), hosts: [] };
+  } finally {
+    try {
+      redis.disconnect?.();
+    } catch {
+    }
+  }
+}
+async function setQueuePaused({ url, paused, makeQueueFn = makeQueue, parseConnectionFn = parseConnection, redisFn = makeRedisClient, timeoutMs = 2500 } = {}) {
+  const opened = [];
   try {
-    queue = makeQueueFn(parseConnectionFn(url, { failFast: true }));
-    if (paused) await queue.pause();
-    else await queue.resume();
-    return { ok: true, paused };
+    const { names, blind } = await readFleetQueues({ url, redisFn, timeoutMs });
+    const done = [];
+    for (const name of names) {
+      const q = makeQueueFn(parseConnectionFn(url, { failFast: true }), { name });
+      opened.push(q);
+      try {
+        if (paused) await q.pause();
+        else await q.resume();
+      } catch (err) {
+        return { unreachable: err?.message ?? String(err), partial: { done, failed: name, error: err?.message ?? String(err) }, blind };
+      }
+      done.push(name);
+    }
+    return { ok: true, paused, queues: done, ...blind ? { blind } : {} };
   } catch (err) {
     return { unreachable: err?.message ?? String(err) };
   } finally {
-    if (queue) await queue.close().catch(() => {
+    for (const q of opened) await q.close().catch(() => {
     });
   }
 }
@@ -10725,7 +10859,8 @@ async function enqueueDispatchRun({
   }
   let queue;
   try {
-    queue = makeQueueFn(parseConnectionFn(valkeyUrl, { failFast: true }));
+    const workerName = process.env.PI_WORKER_NAME;
+    queue = makeQueueFn(parseConnectionFn(valkeyUrl, { failFast: true }), { ...workerName ? { name: hostQueueName(workerName) } : {} });
     const jobId = await enqueueLocalJob(queue, { folder, flow, task });
     return { ok: true, jobId };
   } catch (err) {
@@ -10872,6 +11007,29 @@ async function readScopedBudget({ url, limits, redisFn = makeRedisClient, timeou
     }
   }
 }
+async function readHosts({ url, redisFn = makeRedisClient, timeoutMs = 2500, now = () => Date.now() } = {}) {
+  let redis;
+  try {
+    parseConnection(url, { failFast: true });
+    redis = redisFn(url);
+    const res = await withTimeout(readLiveHosts(redis, { now, timeoutMs: Math.max(250, Math.floor(timeoutMs / 2)) }), timeoutMs, { unreachable: "timed out reaching the registry" });
+    if (res.unreachable) return { unreachable: res.unreachable };
+    return { hosts: res.hosts };
+  } catch (err) {
+    return { unreachable: err?.message ?? String(err) };
+  } finally {
+    if (redis) redis.disconnect?.();
+  }
+}
+function resolveWorkerCount({ hosts, workerCount }) {
+  if (Array.isArray(hosts) && hosts.length > 0) {
+    const declared = hosts.filter((h) => h?.routes === true || h?.routes === "true");
+    const names = declared.map((h) => h.name).filter(Boolean).sort();
+    return { count: hosts.length, names };
+  }
+  if (typeof workerCount === "number" && workerCount > 0) return { count: workerCount, names: [] };
+  return { count: "unknown", names: [] };
+}
 async function readHeldJobs({ url, limit = 20, redisFn = makeRedisClient, timeoutMs = 2500, now = () => Date.now() } = {}) {
   let redis;
   try {
@@ -10990,17 +11148,22 @@ async function readSchedulers({
   url,
   makeQueueFn = makeQueue,
   parseConnectionFn = parseConnection,
+  redisFn = makeRedisClient,
+  timeoutMs = 2500,
   now = Date.now
 } = {}) {
-  let queue;
+  const opened = [];
   try {
-    queue = makeQueueFn(parseConnectionFn(url, { failFast: true }));
-    const list = await queue.getJobSchedulers(0, -1, true);
-    return mapSchedulers(list, now());
+    const fleet = await readHosts({ url, redisFn, timeoutMs });
+    const queues = fleetQueueNames(fleet.hosts).map((name) => makeQueueFn(parseConnectionFn(url, { failFast: true }), { name }));
+    opened.push(...queues);
+    const [primary, ...rest] = queues;
+    const lists = [await primary.getJobSchedulers(0, -1, true), ...await Promise.all(rest.map((q) => q.getJobSchedulers(0, -1, true).catch(() => [])))];
+    return mapSchedulers(lists.flat(), now());
   } catch (err) {
     return { unreachable: err?.message ?? String(err) };
   } finally {
-    if (queue) await queue.close().catch(() => {
+    for (const q of opened) await q.close().catch(() => {
     });
   }
 }
@@ -11607,6 +11770,7 @@ var init_read_model = __esm({
     init_wait_state();
     init_subscriptions();
     init_connection();
+    init_host_registry();
     init_queue();
     init_flow_gate();
     init_git_dirty();
@@ -39692,7 +39856,9 @@ function renderStatus(queue) {
   const counts = queue.counts ?? {};
   const line = ["waiting", "active", "paused", "delayed", "failed"].map((k) => `${k} ${counts[k] ?? 0}`).join("  ");
   const workers = queue.workers === void 0 ? "unknown" : queue.workers;
-  return [`Queue: ${queue.pausedState ? "paused" : "running"}`, `  ${line}`, `  workers: ${workers}`].join("\n");
+  const named = Array.isArray(queue.workerNames) && queue.workerNames.length > 0 ? ` (${queue.workerNames.join(", ")})` : "";
+  const state = queue.pausedPartial ? "PARTIALLY paused" : queue.pausedState ? "paused" : "running";
+  return [`Queue: ${state}`, `  ${line}`, `  workers: ${workers}${named}`].join("\n");
 }
 function renderRuns(runs) {
   if (runs && runs.unreachable) return `Runs: unreachable (${runs.unreachable})`;
@@ -40131,27 +40297,67 @@ async function heldJobs(redis) {
   rows.sort((a, b) => (b.waitedMs ?? -1) - (a.waitedMs ?? -1));
   return { rows: rows.slice(0, HELD_LIMIT), more: Math.max(0, rows.length - HELD_LIMIT), truncated: ids.length > HELD_HYDRATE_MAX2 };
 }
-function createDashboardDeps(paths) {
-  const queue = makeQueue(parseConnection(paths.valkeyUrl, { failFast: true }));
-  const redis = makeRedisClient(paths.valkeyUrl);
+function createDashboardDeps(paths, {
+  makeQueueFn = makeQueue,
+  parseConnectionFn = parseConnection,
+  redisFn = makeRedisClient,
+  readLiveHostsFn = readLiveHosts,
+  discoverHostQueuesFn = discoverHostQueues
+} = {}) {
+  const queue = makeQueueFn(parseConnectionFn(paths.valkeyUrl, { failFast: true }));
+  const redis = redisFn(paths.valkeyUrl);
+  const pool = /* @__PURE__ */ new Map([[queue.name, queue]]);
+  let lastNames = [queue.name];
+  let fleetDegraded = null;
+  const fleetQueues = async () => {
+    const fleet = await readLiveHostsFn(redis, { timeoutMs: FLEET_READ_TIMEOUT_MS }).catch((err) => ({
+      unreachable: err?.message ?? String(err)
+    }));
+    if (fleet?.unreachable || !Array.isArray(fleet?.hosts)) {
+      fleetDegraded = fleet?.unreachable ?? "registry unreadable";
+    } else {
+      fleetDegraded = null;
+      lastNames = fleetQueueNames(fleet.hosts);
+    }
+    for (const name of lastNames) {
+      if (!pool.has(name)) pool.set(name, makeQueueFn(parseConnectionFn(paths.valkeyUrl, { failFast: true }), { name }));
+    }
+    return lastNames.map((n) => pool.get(n));
+  };
+  const killSwitchQueues = async () => {
+    const existing = await discoverHostQueuesFn(redis, { timeoutMs: FLEET_READ_TIMEOUT_MS });
+    for (const name of unionQueueNames(lastNames, existing)) {
+      if (!pool.has(name)) pool.set(name, makeQueueFn(parseConnectionFn(paths.valkeyUrl, { failFast: true }), { name }));
+    }
+    return unionQueueNames(lastNames, existing).map((n) => pool.get(n));
+  };
   return {
     async fetchSnapshot() {
-      const [pausedState, counts, workerList, dayRaw, weekRaw, monthRaw, tokenRaw, schedulerList, activeList, stallHash] = await Promise.all([
-        queue.isPaused(),
-        queue.getJobCounts("waiting", "active", "paused", "delayed", "failed"),
+      const queues = await fleetQueues();
+      const [pausedStates, countsPer, workerList, dayRaw, weekRaw, monthRaw, tokenRaw, schedulerLists, activeLists, stallHash] = await Promise.all([
+        Promise.all(queues.map((q) => q.isPaused())),
+        Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed"))),
         queue.getWorkers().catch(() => []),
         redis.get(dayKey()),
         redis.get(weekKey()),
         redis.get(monthKey()),
         redis.get(tokenDayKey()),
         // issue #25 daily token spend (budget:t:YYYY-MM-DD)
-        queue.getJobSchedulers(0, -1, true),
-        queue.getActive(0, 0).catch(() => []),
+        // The shared queue's failure is UNCAUGHT (the panel must not draw a scheduler list it could not
+        // read), a host queue's is caught: one unreachable host degrades one host's triggers, never the view.
+        Promise.all(queues.map((q, i) => i === 0 ? q.getJobSchedulers(0, -1, true) : q.getJobSchedulers(0, -1, true).catch(() => []))),
+        Promise.all(queues.map((q) => q.getActive(0, 0).catch(() => []))),
         // Per-scheduler stall counts (money backstop) for the cron drill-in; reuses the held client like the
         // budget GETs. HGETALL of an absent key is `{}`, so a never-stalled deployment shows 0 stalls.
         redis.hgetall(STALL_KEY).catch(() => ({}))
       ]);
       const workers = Array.isArray(workerList) && workerList.length > 0 ? workerList.length : "unknown";
+      const counts = {};
+      for (const c of countsPer) for (const [k, v] of Object.entries(c ?? {})) counts[k] = (counts[k] ?? 0) + Number(v ?? 0);
+      const pausedState = pausedStates.every(Boolean);
+      const pausedPartial = !pausedState && pausedStates.some(Boolean);
+      const schedulerList = schedulerLists.flat();
+      const activeList = activeLists.flat();
       const scopedLimits = readScopedLimits({ scopedLimitsPath: paths.scopedLimitsPath });
       const limitRows = Array.isArray(scopedLimits?.limits) ? scopedLimits.limits : [];
       const scopedNow = /* @__PURE__ */ new Date();
@@ -40173,7 +40379,7 @@ function createDashboardDeps(paths) {
         held,
         scopedLimits,
         scopedBudget,
-        queue: { pausedState, counts, workers },
+        queue: { pausedState, pausedPartial, counts, workers, queues: queues.length, fleetDegraded },
         budget: { day: Number(dayRaw ?? 0), week: Number(weekRaw ?? 0), month: Number(monthRaw ?? 0), tokensToday: Number(tokenRaw ?? 0) },
         schedulers: mapSchedulers(schedulerList, Date.now()),
         schedulerStalls: stallHash ?? {},
@@ -40193,15 +40399,19 @@ function createDashboardDeps(paths) {
       };
     },
     async pause() {
-      await queue.pause();
+      await fleetQueues();
+      for (const q of await killSwitchQueues()) await q.pause();
     },
     async resume() {
-      await queue.resume();
+      await fleetQueues();
+      for (const q of await killSwitchQueues()) await q.resume();
     },
     async dispose() {
-      try {
-        await queue.close();
-      } catch {
+      for (const q of pool.values()) {
+        try {
+          await q.close();
+        } catch {
+        }
       }
       try {
         redis.disconnect();
@@ -40743,11 +40953,11 @@ function statusHeader(queue, inner, styler) {
     return styler.cell(`queue unreachable (${queue?.unreachable ?? "?"})`, inner, { color: "error" });
   }
   const c = queue.counts ?? {};
-  const running = !queue.pausedState;
+  const running = !queue.pausedState && !queue.pausedPartial;
   const failed = Number(c.failed ?? 0);
   const stateColor = running ? "success" : "warning";
   const dot = styler.fg(stateColor, "\u25CF");
-  const word = styler.bold(styler.fg(stateColor, running ? "RUNNING" : "PAUSED"));
+  const word = styler.bold(styler.fg(stateColor, running ? "RUNNING" : queue.pausedPartial ? "PART PAUSED" : "PAUSED"));
   const sep2 = styler.fg("dim", " \xB7 ");
   const delayed = Number(c.delayed ?? 0);
   const vitals = `${c.waiting ?? 0} waiting` + sep2 + `${c.active ?? 0} active` + sep2 + (delayed > 0 ? `${delayed} delayed` + sep2 : "") + (failed > 0 ? styler.fg("error", `${failed} failed`) : `${failed} failed`) + sep2 + `${queue.workers ?? "?"} workers`;
@@ -41267,6 +41477,7 @@ function renderRunDetail(record, inner, styler, allRuns = [], sandbox = null) {
   const endMs = toMs(r.endedAt);
   const dur = Number.isFinite(startMs) && Number.isFinite(endMs) && endMs >= startMs ? ` (${formatDuration(endMs - startMs)})` : "";
   out.push(kv("timing", `${fmtStamp(r.startedAt)} \u2192 ${fmtStamp(r.endedAt)}${dur}`));
+  if (r.host) out.push(kv("host", String(r.host)));
   const turnBits = [`${show(r.turns)} turns`, `exit ${show(r.exitCode)}`];
   if (r.budgetReserved !== null && r.budgetReserved !== void 0) turnBits.push(`${r.budgetReserved} budget slot`);
   if (r.attempt !== null && r.attempt !== void 0) turnBits.push(`attempt ${r.attempt}`);
@@ -41346,12 +41557,13 @@ function runDetailHints(inner, styler, canOpen = false, canCopy = false, copiedN
   if (copiedNote) return fitLine(styler.fg("success", copiedNote) + styler.fg("dim", "  \xB7  ") + line, inner, styler);
   return fitLine(line, inner, styler);
 }
-var KEY_HINTS, RUNS_ON_DASHBOARD, RUNS_VIEWPORT, RUN_SORTS, REFRESH_MS, TAIL_LINES, TAIL_VIEWPORT, MIN_WIDTH2, DRILL_WIDTH, HELD_LIMIT, HELD_HYDRATE_MAX2, KIND_COLOR, KIND_WIDTH, FRAME_CHROME_ROWS, HELD_ON_DASHBOARD;
+var KEY_HINTS, RUNS_ON_DASHBOARD, RUNS_VIEWPORT, RUN_SORTS, REFRESH_MS, TAIL_LINES, TAIL_VIEWPORT, MIN_WIDTH2, DRILL_WIDTH, HELD_LIMIT, HELD_HYDRATE_MAX2, FLEET_READ_TIMEOUT_MS, KIND_COLOR, KIND_WIDTH, FRAME_CHROME_ROWS, HELD_ON_DASHBOARD;
 var init_dashboard = __esm({
   "src/dashboard.ts"() {
     init_budget();
     init_connection();
     init_queue();
+    init_host_registry();
     init_scheduler_stall_guard();
     init_pause_windows();
     init_read_model();
@@ -41371,6 +41583,7 @@ var init_dashboard = __esm({
     DRILL_WIDTH = 70;
     HELD_LIMIT = 20;
     HELD_HYDRATE_MAX2 = 200;
+    FLEET_READ_TIMEOUT_MS = 2e3;
     KIND_COLOR = { cron: "accent", label: "syntaxType", comment: "syntaxKeyword", pull_request: "syntaxFunction", issue: "syntaxString" };
     KIND_WIDTH = 13;
     FRAME_CHROME_ROWS = 4;
@@ -41908,7 +42121,7 @@ var init_setup_wizard = __esm({
     init_deployment_pointer();
     init_read_model();
     init_index();
-    RUNTIME_VERSION = "1.6.1";
+    RUNTIME_VERSION = "1.7.0";
     RECEIVER_VERSION = "1.4.0";
     RUNTIME_PKG = "@edgehero/pi-dispatch";
     RECEIVER_PKG = "@edgehero/pi-dispatch-receiver";
@@ -42089,7 +42302,7 @@ function registerTools(pi) {
   pi.registerTool({
     name: "dispatch_runs",
     label: "pi-dispatch runs",
-    description: "Read-only. Returns structured, PII-free run records from the durable run history. Raw job logs are not available to tools \u2014 ask the user to run /dispatch logs.",
+    description: "Read-only. Returns structured run records (PII-free by construction, plus the worker name when the operator declared one) from the durable run history. Raw job logs are not available to tools \u2014 ask the user to run /dispatch logs.",
     parameters: typebox_exports.Object({
       limit: typebox_exports.Optional(typebox_exports.Integer({ minimum: 1, maximum: 50 })),
       jobId: typebox_exports.Optional(typebox_exports.String())
@@ -42128,7 +42341,10 @@ function registerTools(pi) {
       if (res.unreachable) {
         throw new Error(`could not reach the queue at ${paths.valkeyUrl}: ${res.unreachable}`);
       }
-      return toolText("paused");
+      if (res.partial) {
+        throw new Error(`paused ${res.partial.done.join(", ")} but FAILED at ${res.partial.failed} \u2014 the deployment is half paused: ${res.partial.error}`);
+      }
+      return toolText(`paused${res.queues.length > 1 ? ` (${res.queues.length} queues)` : ""}`);
     }
   });
   pi.registerTool({
@@ -42143,7 +42359,10 @@ function registerTools(pi) {
       if (res.unreachable) {
         throw new Error(`could not reach the queue at ${paths.valkeyUrl}: ${res.unreachable}`);
       }
-      return toolText("resumed");
+      if (res.partial) {
+        throw new Error(`resumed ${res.partial.done.join(", ")} but FAILED at ${res.partial.failed} \u2014 the deployment is half resumed: ${res.partial.error}`);
+      }
+      return toolText(`resumed${res.queues.length > 1 ? ` (${res.queues.length} queues)` : ""}`);
     }
   });
   pi.registerTool({
@@ -42758,11 +42977,15 @@ ${scopedText}` : ""}`);
       const paused = sub === "pause";
       const res = await setQueuePaused({ url: paths.valkeyUrl, paused });
       if (res.unreachable) {
-        notify?.(`could not reach Valkey at ${paths.valkeyUrl} \u2014 is it running? (docker compose up)`, "error");
+        notify?.(
+          res.partial ? `${paused ? "paused" : "resumed"} ${res.partial.done.join(", ")} but FAILED at ${res.partial.failed} \u2014 the deployment is half ${paused ? "paused" : "resumed"}` : `could not reach Valkey at ${paths.valkeyUrl} \u2014 is it running? (docker compose up)`,
+          "error"
+        );
         return;
       }
+      const span = res.blind ? ` (fleet unknown: ${res.blind}, only ${res.queues[0]})` : res.queues.length > 1 ? ` (${res.queues.length} queues)` : "";
       notify?.(
-        paused ? "paused \u2014 worker will stop taking new jobs (jobs still enqueue; durable, survives restart)" : "resumed",
+        paused ? `paused \u2014 worker will stop taking new jobs (jobs still enqueue; durable, survives restart)${span}` : `resumed${span}`,
         "info"
       );
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch-admin",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "description": "Operator console (a pi extension) + skill for pi-dispatch: run the pi coding agent as a self-hosted service. A /dispatch TUI for the queue, spend caps, run history, editable GitHub, GitLab, Forgejo and Azure DevOps triggers (cron/label/comment/PR/MR/work item), and scheduled pause windows — plus AI-operable, human-confirmed controls. Runs against a live pi-dispatch deployment.",
   "keywords": [
     "pi-package",