@rjfan/dsh-presence 0.18.0 → 0.20.1

package/src/pool.js

@@ -1,6 +1,7 @@
 // The pool the plugin owns: one live connection per machine, plus the
 // terminals that belong to it.
 
+import { randomUUID } from 'node:crypto';
 import { listHosts, readSshConfig } from './core/ssh-config.mjs';
 import { PresenceConnection } from './core/connection.mjs';
 
@@ -9,15 +10,50 @@ const MAX_TERMINAL_BUFFER = 256 * 1024;
 // started: give the wrapper a moment, then check the process group.
 const JOB_EARLY_CHECK_MS = 800;
 const JOB_EARLY_TAIL_LINES = 20;
-// Finished jobs pile up in the local registry for the life of the app. Prune
-// them at most once a minute from `listJobs`, keeping the newest entries and
-// anything that finished in the last six hours.
+// Finished jobs pile up in the local registry and in the JSON store. Prune at
+// most once a minute from `listJobs`; the 0.20 store keeps a month of finished
+// jobs (a run is worth coming back to after a restart), bounded to 500 rows.
 const JOB_PRUNE_INTERVAL_MS = 60_000;
-const JOB_PRUNE_MAX_AGE_MS = 6 * 60 * 60 * 1000;
-const JOB_PRUNE_MAX_ENTRIES = 100;
+const JOB_PRUNE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
+const JOB_PRUNE_MAX_ENTRIES = 500;
+// A transient connect failure is retried twice, with these gaps, and a dead
+// cached connection is dropped before each retry.
+const CONNECT_RETRY_BACKOFF_MS = [200, 600];
+// How long a failure keeps its edge on the host state: recent enough to matter,
+// old enough to clear itself once the connection has been healthy again.
+const HOST_DEGRADED_WINDOW_MS = 60_000;
 
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
+// Auth, permission and unknown-host failures are configuration problems a retry
+// cannot fix; everything network-shaped is worth another attempt.
+const FATAL_CONNECTION_ERROR =
+  /no host "|authentication failed|authentication methods|passphrase-protected|password requested|permission denied|access denied|not authorized/i;
+const TRANSIENT_CONNECTION_ERROR =
+  /ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EPIPE|EHOSTUNREACH|ENETUNREACH|connection refused|could not be resolved|timed out|handshake|channel|closed|socket hang up|no route to host/i;
+
+/**
+ * Whether a failed connection attempt is worth retrying.
+ *
+ * `transient` covers the network-shaped failures (DNS, refused, timeout,
+ * dropped channel, reset); `fatal` covers configuration errors a second attempt
+ * cannot fix (authentication, permissions, an alias that is not configured) and
+ * is also the default for an unrecognised error, so a retry never masks a real
+ * problem.
+ *
+ * @param {unknown} error
+ * @returns {'transient'|'fatal'}
+ */
+export function classifyConnectionError(error) {
+  const message = String(error?.message ?? error ?? '');
+  const code = String(error?.code ?? '');
+  // ssh2 tags authentication failures with this level even before describeSshError.
+  if (String(error?.level ?? '') === 'client-authentication') return 'fatal';
+  if (FATAL_CONNECTION_ERROR.test(message)) return 'fatal';
+  if (TRANSIENT_CONNECTION_ERROR.test(`${code} ${message}`)) return 'transient';
+  return 'fatal';
+}
+
 export class PresencePool {
   #settings;
   #connections = new Map();
@@ -25,18 +61,34 @@ export class PresencePool {
   #terminals = new Map();
   #tunnels = new Map();
   #jobs = new Map();
+  #jobStore = null;
   #lastUsed = new Map();
   #lastPruneAt = 0;
   #secrets = new Map();
   #connectionHandlers = new Set();
+  #hostStates = new Map();
   #hostsCache;
   #reaper;
   #counter = 0;
 
-  constructor(settings) {
+  /**
+   * @param settings - resolved plugin config.
+   * @param options.jobStore - optional persistent job registry (`JobStore`, or
+   *   any object with `list/upsert/remove/prune`). It is captured once and
+   *   never replaced, so persistence cannot be attached halfway through a run.
+   */
+  constructor(settings, { jobStore = null } = {}) {
     this.#settings = settings;
+    this.#jobStore = jobStore;
+    this.#loadPersistedJobs();
     this.#reaper = setInterval(() => this.#reapIdle(), 60_000);
     this.#reaper.unref?.();
+    // A job started before a restart is `unknown` until the machine is asked;
+    // the first successful connection is exactly that chance. Fire-and-forget:
+    // a slow probe must never delay or fail `connection()`.
+    this.onConnectionEstablished((host) => {
+      this.#refreshPersistedJobs(host).catch(() => {});
+    });
   }
 
   get settings() {
@@ -50,20 +102,51 @@ export class PresencePool {
       this.#hostsCache = { at: now, list: listHosts() };
     }
     return this.#hostsCache.list.map((entry) => {
-      const connection = this.#connections.get(entry.alias);
+      const state = this.hostState(entry.alias);
       return {
         alias: entry.alias,
         hostname: entry.hostname,
         port: entry.port,
         user: entry.user,
         source: entry.source ?? 'config',
-        connected: Boolean(connection && !connection.closed),
+        connected: state.connected,
+        state: state.state,
+        lastError: state.lastError,
         identityFile: entry.identityFile?.[0],
         terminals: [...this.#terminals.values()].filter((terminal) => terminal.host === entry.alias).length,
       };
     });
   }
 
+  /**
+   * Current trust state of one machine.
+   *
+   * `connected` is the cached connection; `degraded` means that connection is
+   * usable but a failure happened in the last minute; `error` means the last
+   * attempt failed and nothing is cached; `idle` means no connection and no
+   * recorded failure. `retries` is how many retries the last attempt needed.
+   */
+  hostState(alias) {
+    const name = String(alias ?? '');
+    const connection = this.#connections.get(name);
+    const connected = Boolean(connection && !connection.closed);
+    const record = this.#hostStates.get(name) ?? null;
+    const failedAt = record?.lastErrorAt ?? null;
+    const recent = failedAt !== null && Date.now() - failedAt <= HOST_DEGRADED_WINDOW_MS;
+    let state;
+    if (connected) state = recent ? 'degraded' : 'connected';
+    else if (failedAt !== null) state = 'error';
+    else state = 'idle';
+    return {
+      alias: name,
+      connected,
+      state,
+      lastError: record?.lastError ?? null,
+      lastErrorAt: record?.lastErrorAt ?? null,
+      retries: record?.retries ?? 0,
+    };
+  }
+
   /** Resolve a live connection, opening one if needed. */
   async connection(host, { password, passphrase } = {}) {
     const existing = this.#connections.get(host);
@@ -71,6 +154,9 @@ export class PresencePool {
       this.#lastUsed.set(host, Date.now());
       return existing;
     }
+    // A cached connection that died between calls is not reusable: drop it (and
+    // anything that hung off it) before dialling again.
+    if (existing) this.#dropConnection(host);
     if (this.#connecting.has(host)) return this.#connecting.get(host);
 
     const entry = readSshConfig().get(host);
@@ -92,20 +178,72 @@ export class PresencePool {
     if (password || passphrase) this.#secrets.set(host, { password, passphrase });
     const secret = this.#secrets.get(host) ?? (envPassword ? { password: envPassword } : {});
 
-    const pending = PresenceConnection.connect(
-      { ...entry, maxChannels: this.#settings.maxChannels },
-      { password: secret.password, passphrase: secret.passphrase, timeoutMs: this.#settings.connectTimeoutMs },
-    )
-      .then((connection) => {
+    // The whole retry sequence is one shared promise, so concurrent callers do
+    // not dial the same machine three times over.
+    const pending = this.#connectWithRetry(host, entry, secret).finally(() => this.#connecting.delete(host));
+    this.#connecting.set(host, pending);
+    return pending;
+  }
+
+  /**
+   * Open one connection, retrying transient failures. Auth/permission/unknown
+   * host errors are configuration problems and fail on the first attempt; a
+   * network-shaped failure is retried (200 ms, 600 ms apart) after dropping the
+   * dead cached connection, and the final error is the last one observed.
+   */
+  async #connectWithRetry(host, entry, secret) {
+    const previous = this.#hostStates.get(host) ?? {};
+    const state = {
+      lastError: previous.lastError ?? null,
+      lastErrorAt: previous.lastErrorAt ?? null,
+      retries: 0,
+    };
+    let lastError = null;
+    for (let attempt = 0; attempt <= CONNECT_RETRY_BACKOFF_MS.length; attempt += 1) {
+      if (attempt > 0) {
+        this.#dropConnection(host);
+        state.retries = attempt;
+        await sleep(CONNECT_RETRY_BACKOFF_MS[attempt - 1]);
+      }
+      try {
+        const connection = await PresenceConnection.connect(
+          { ...entry, maxChannels: this.#settings.maxChannels },
+          { password: secret.password, passphrase: secret.passphrase, timeoutMs: this.#settings.connectTimeoutMs },
+        );
         this.#connections.set(host, connection);
         this.#lastUsed.set(host, Date.now());
+        // Keep the failure that led here: a host that needed a retry is usable
+        // but degraded for the next minute.
+        this.#hostStates.set(host, state);
         this.#notifyConnectionEstablished(host, connection);
         return connection;
-      })
-      .finally(() => this.#connecting.delete(host));
+      } catch (error) {
+        lastError = error;
+        state.lastError = String(error?.message ?? error);
+        state.lastErrorAt = Date.now();
+        if (classifyConnectionError(error) !== 'transient' || attempt === CONNECT_RETRY_BACKOFF_MS.length) {
+          this.#hostStates.set(host, state);
+          throw error;
+        }
+      }
+    }
+    this.#hostStates.set(host, state);
+    throw lastError ?? new Error(`could not connect to ${host}`);
+  }
 
-    this.#connecting.set(host, pending);
-    return pending;
+  /** Forget a cached connection and close anything that hung off it. */
+  #dropConnection(host) {
+    const connection = this.#connections.get(host);
+    this.#connections.delete(host);
+    this.#lastUsed.delete(host);
+    if (!connection) return;
+    try {
+      connection.end();
+    } catch {
+      /* already gone */
+    }
+    // A forward whose SSH connection is gone still owns a local listener.
+    this.#closeTunnelsForHost(host).catch(() => {});
   }
 
   /**
@@ -141,6 +279,8 @@ export class PresencePool {
     const connection = this.#connections.get(host);
     this.#connections.delete(host);
     this.#lastUsed.delete(host);
+    // A deliberate disconnect is not a failure: the host goes back to `idle`.
+    this.#hostStates.delete(host);
     for (const terminal of [...this.#terminals.values()]) {
       if (terminal.host === host) this.#closeTerminal(terminal.id);
     }
@@ -249,12 +389,19 @@ export class PresencePool {
     if (!command) throw new Error('command is required');
     const connection = await this.connection(host);
     const started = await connection.startBackground(command, { cwd, logPath });
-    const id = `j${(this.#counter += 1)}`;
+    const id = this.#nextJobId();
+    const pid = Number(started.pid);
     const job = {
       id,
       host,
-      pid: started.pid,
+      pid,
+      pgid: Number.isFinite(Number(started.pgid)) ? Number(started.pgid) : pid,
       logPath: started.logPath,
+      // The 0.20 factory returns a distinct command-output log when the
+      // wrapper exposes one; the current kernel returns only `logPath`, which
+      // IS the command's stdout/stderr log. Keep the public field non-null and
+      // truthful either way (it equals logPath on today's kernels).
+      commandLogPath: started.commandLogPath ?? started.logPath ?? null,
       statusPath: started.statusPath,
       command,
       cwd,
@@ -268,13 +415,29 @@ export class PresencePool {
       logSize: null,
       logTail: '',
       stopped: false,
+      persisted: false,
+      lastPersistedJson: null,
     };
     this.#jobs.set(id, job);
+    // Persist before the early-death check: a job that dies instantly must
+    // still leave a record, otherwise a restart cannot explain it.
+    this.#syncJob(job);
 
     await sleep(JOB_EARLY_CHECK_MS);
     const alive = await this.#groupAlive(connection, job.pid);
     if (alive !== false) {
-      return { id, host, pid: job.pid, logPath: job.logPath, statusPath: job.statusPath, label: job.label, sessionId: job.sessionId };
+      this.#syncJob(job);
+      return {
+        id,
+        host,
+        pid: job.pid,
+        pgid: job.pgid,
+        logPath: job.logPath,
+        commandLogPath: job.commandLogPath,
+        statusPath: job.statusPath,
+        label: job.label,
+        sessionId: job.sessionId,
+      };
     }
     const status = await connection.backgroundStatus(job.statusPath);
     const output = await this.#readOutput(connection, job, JOB_EARLY_TAIL_LINES);
@@ -283,11 +446,14 @@ export class PresencePool {
     job.exitedAt = Date.now();
     job.logSize = output.logSize;
     job.logTail = output.logTail;
+    this.#syncJob(job);
     return {
       id,
       host,
       pid: job.pid,
+      pgid: job.pgid,
       logPath: job.logPath,
+      commandLogPath: job.commandLogPath,
       statusPath: job.statusPath,
       label: job.label,
       sessionId: job.sessionId,
@@ -334,6 +500,7 @@ export class PresencePool {
         job.exitedAt ??= Date.now();
         job.logSize = output.logSize ?? job.logSize;
         job.logTail = output.logTail;
+        this.#syncJob(job);
         return this.#jobResult(job, {
           state: 'exited',
           exitCode: status.exitCode,
@@ -354,6 +521,7 @@ export class PresencePool {
         job.exitedAt ??= Date.now();
         job.logSize = output.logSize ?? job.logSize;
         job.logTail = output.logTail;
+        this.#syncJob(job);
         return this.#jobResult(job, {
           state: 'exited',
           exitCode: null,
@@ -396,7 +564,9 @@ export class PresencePool {
         id: job?.id ?? null,
         host: target,
         pid: numeric,
+        pgid: job?.pgid ?? numeric,
         logPath: job?.logPath ?? null,
+        commandLogPath: job?.commandLogPath ?? null,
         stopped: result.stopped,
         wasAlive: result.wasAlive,
         reason: result.reason ?? null,
@@ -412,7 +582,9 @@ export class PresencePool {
       id,
       host: job.host,
       pid: job.pid,
+      pgid: job.pgid ?? job.pid ?? null,
       logPath: job.logPath,
+      commandLogPath: job.commandLogPath ?? null,
       stopped: result.stopped,
       wasAlive: result.wasAlive,
       reason: result.reason ?? null,
@@ -454,6 +626,7 @@ export class PresencePool {
         job.exitedAt ??= Date.now();
         job.exitCode = result.exitCode ?? job.exitCode ?? null;
       }
+      this.#syncJob(job);
     }
     return {
       host,
@@ -469,10 +642,11 @@ export class PresencePool {
   }
 
   /**
-   * Drop local bookkeeping for jobs that are finished and past their welcome:
-   * older than `maxAgeMs`, or beyond the newest `maxEntries`. Only EXITED jobs
-   * are dropped — a running (or still unknown) job keeps its slot no matter
-   * how old it is, and the detached process on the machine is never touched.
+   * Drop bookkeeping for jobs that are finished and past their welcome: older
+   * than `maxAgeMs`, or beyond the newest `maxEntries`. Only EXITED jobs are
+   * dropped — a running (or still unknown) job keeps its slot no matter how old
+   * it is, and the detached process on the machine is never touched. The
+   * persisted store is bounded by the same limits.
    * Returns the ids that were forgotten.
    */
   async pruneJobs({ maxAgeMs = JOB_PRUNE_MAX_AGE_MS, maxEntries = JOB_PRUNE_MAX_ENTRIES } = {}) {
@@ -492,15 +666,42 @@ export class PresencePool {
       if (job.state === 'exited') dropped.add(job.id);
     }
     for (const id of dropped) this.#jobs.delete(id);
+    if (this.#jobStore) {
+      // The store applies the same age/count bounds; a record it drops beyond
+      // the local view is forgotten here too, so the two cannot drift apart.
+      let pruned = { removed: 0, ids: [] };
+      try {
+        pruned = this.#jobStore.prune({ maxAgeMs: age, maxEntries: limit }) ?? pruned;
+      } catch {
+        /* persistence is an optimization; a broken store must not fail a prune */
+      }
+      for (const id of Array.isArray(pruned.ids) ? pruned.ids : []) {
+        this.#jobs.delete(id);
+        dropped.add(id);
+      }
+      // Only records the store's own bound did not already drop need a second
+      // write; normally this set is empty (one save, not one per dropped job).
+      const prunedIds = new Set(Array.isArray(pruned.ids) ? pruned.ids : []);
+      for (const id of [...dropped]) {
+        if (prunedIds.has(id)) continue;
+        try {
+          this.#jobStore.remove(id);
+        } catch {
+          /* already gone */
+        }
+      }
+    }
     this.#lastPruneAt = now;
     return { removed: dropped.size, ids: [...dropped] };
   }
 
   /**
-   * Jobs this pool started, optionally narrowed to one session, with liveness
-   * read from the machines connected right now. `state` is 'unknown' when the
-   * host cannot be asked: a detached job outlives its SSH connection, so no
-   * answer is not the same as a dead job.
+   * Jobs this pool started, plus records inherited from the persisted store,
+   * optionally narrowed to one session, with liveness read from the machines
+   * connected right now. A record loaded from disk is `unknown` until a pid
+   * probe says otherwise: a detached job outlives its SSH connection, so no
+   * answer is not the same as a dead job. A record says `persisted: true` when
+   * it came from the JSON store rather than from this process.
    */
   async listJobs({ sessionId = undefined } = {}) {
     // Local bookkeeping only, so no round trip; at most once a minute so a
@@ -508,6 +709,9 @@ export class PresencePool {
     if (Date.now() - this.#lastPruneAt >= JOB_PRUNE_INTERVAL_MS) {
       await this.pruneJobs().catch(() => {});
     }
+    // A record written by another process (or by a previous run) joins the view
+    // as `unknown`; the per-host liveness pass below may resolve it.
+    this.#mergePersistedJobs();
     const jobs = [...this.#jobs.values()].filter((job) => sessionId === undefined || job.sessionId === sessionId);
     if (jobs.length === 0) return [];
 
@@ -516,90 +720,228 @@ export class PresencePool {
       if (!byHost.has(job.host)) byHost.set(job.host, []);
       byHost.get(job.host).push(job);
     }
+    await Promise.all([...byHost.entries()].map(([host, hostJobs]) => this.#probeHostJobs(host, hostJobs)));
+    return jobs.map((job) => this.#jobRow(job));
+  }
+
+  // ------------------------------------------------- job registry internals
 
+  /**
+   * Ask one host about a group of jobs: seed its process registry, read the
+   * whole process table once and every status file once. Live jobs keep their
+   * state; a record loaded from disk is only promoted to `running` when `ps`
+   * actually finds its process group.
+   */
+  async #probeHostJobs(host, hostJobs) {
+    const connection = this.#connections.get(host);
+    if (!connection || connection.closed) {
+      for (const job of hostJobs) if (job.state !== 'exited') job.state = 'unknown';
+      for (const job of hostJobs) this.#syncJob(job);
+      return;
+    }
+    // A job started before a reconnect is not in this connection's registry;
+    // its process group still exists on the machine, so seed it back in.
+    for (const job of hostJobs) {
+      if (job.stopped || job.state === 'exited' || job.pid === null || job.pid === undefined) continue;
+      connection.trackBackground({
+        pid: job.pid,
+        command: job.command,
+        cwd: job.cwd,
+        logPath: job.logPath,
+        statusPath: job.statusPath,
+        startedAt: job.startedAt,
+      });
+    }
+    let running;
+    try {
+      running = new Set((await connection.listBackground()).map((entry) => entry.pid));
+    } catch {
+      // Liveness is a convenience; a failed `ps` must not hide the job list.
+      running = null;
+    }
+    if (!running) {
+      for (const job of hostJobs) if (job.state !== 'exited') job.state = 'unknown';
+      for (const job of hostJobs) this.#syncJob(job);
+      return;
+    }
+    // One round trip for every status file on this host: the command may have
+    // finished while a child it spawned kept the group alive.
+    let statuses = null;
+    try {
+      statuses = await connection.backgroundStatuses(hostJobs.map((job) => job.statusPath ?? ''));
+    } catch {
+      // Liveness from `ps` is still worth reporting.
+    }
     await Promise.all(
-      [...byHost.entries()].map(async ([host, hostJobs]) => {
-        const connection = this.#connections.get(host);
-        if (!connection || connection.closed) {
-          for (const job of hostJobs) if (job.state !== 'exited') job.state = 'unknown';
-          return;
-        }
-        // A job started before a reconnect is not in this connection's registry;
-        // its process group still exists on the machine, so seed it back in.
-        for (const job of hostJobs) {
-          if (job.stopped || job.state === 'exited') continue;
-          connection.trackBackground({
-            pid: job.pid,
-            command: job.command,
-            cwd: job.cwd,
-            logPath: job.logPath,
-            statusPath: job.statusPath,
-            startedAt: job.startedAt,
-          });
-        }
-        let running;
-        try {
-          running = new Set((await connection.listBackground()).map((entry) => entry.pid));
-        } catch {
-          // Liveness is a convenience; a failed `ps` must not hide the job list.
-          running = null;
+      hostJobs.map(async (job, index) => {
+        const status = statuses?.[index] ?? null;
+        if (job.stopped) {
+          job.state = 'exited';
+          job.exitedAt ??= Date.now();
+        } else if (status?.found) {
+          job.state = 'exited';
+          job.exitCode = status.exitCode;
+          job.exitedAt ??= Date.now();
+        } else if (job.state === 'exited') {
+          // Already reported finished; a recycled pgid must not resurrect it.
+          job.exitedAt ??= Date.now();
+        } else if (job.pid !== null && job.pid !== undefined && running.has(job.pid)) {
+          job.state = 'running';
+        } else {
+          // No wrapper left to write a status: it was killed.
+          job.state = 'exited';
+          job.exitCode = null;
+          job.exitedAt ??= Date.now();
         }
-        if (!running) {
-          for (const job of hostJobs) if (job.state !== 'exited') job.state = 'unknown';
-          return;
-        }
-        // One round trip for every status file on this host: the command may
-        // have finished while a child it spawned kept the group alive.
-        let statuses = null;
-        try {
-          statuses = await connection.backgroundStatuses(hostJobs.map((job) => job.statusPath));
-        } catch {
-          // Liveness from `ps` is still worth reporting.
-        }
-        await Promise.all(
-          hostJobs.map(async (job, index) => {
-            const status = statuses?.[index] ?? null;
-            if (job.stopped) {
-              job.state = 'exited';
-              job.exitedAt ??= Date.now();
-            } else if (status?.found) {
-              job.state = 'exited';
-              job.exitCode = status.exitCode;
-              job.exitedAt ??= Date.now();
-            } else if (job.state === 'exited') {
-              // Already reported finished; a recycled pgid must not resurrect it.
-              job.exitedAt ??= Date.now();
-            } else if (running.has(job.pid)) {
-              job.state = 'running';
-            } else {
-              // No wrapper left to write a status: it was killed.
-              job.state = 'exited';
-              job.exitCode = null;
-              job.exitedAt ??= Date.now();
-            }
-            const info = await connection.stat(job.logPath).catch(() => null);
-            if (info?.exists) job.logSize = info.size;
-          }),
-        );
+        const info = job.logPath ? await connection.stat(job.logPath).catch(() => null) : null;
+        if (info?.exists) job.logSize = info.size;
+        this.#syncJob(job);
       }),
     );
+  }
 
-    return jobs.map((job) => ({
+  /** One public job row (live view, no remote probe). */
+  #jobRow(job) {
+    return {
       id: job.id,
       host: job.host,
-      pid: job.pid,
+      pid: job.pid ?? null,
+      pgid: job.pgid ?? job.pid ?? null,
       state: job.state ?? 'unknown',
       exitCode: job.exitCode ?? null,
       exitSignal: job.exitSignal ?? null,
+      exitedAt: job.exitedAt ?? null,
       label: job.label ?? null,
       sessionId: job.sessionId ?? null,
-      startedAt: job.startedAt,
-      logPath: job.logPath,
+      startedAt: job.startedAt ?? null,
+      logPath: job.logPath ?? null,
+      commandLogPath: job.commandLogPath ?? null,
       logSize: job.logSize ?? null,
+      persisted: job.persisted === true,
       // Kept for the existing jobs view, which still prints the command line.
-      command: job.command,
-      cwd: job.cwd,
-    }));
+      command: job.command ?? null,
+      cwd: job.cwd ?? null,
+    };
+  }
+
+  /**
+   * Next job id. The plain `jN` counter keeps ids readable; when a store is
+   * attached a short random suffix keeps two Harness windows that share the
+   * same jobs.json from overwriting each other's records.
+   */
+  #nextJobId() {
+    this.#counter += 1;
+    const base = `j${this.#counter}`;
+    return this.#jobStore ? `${base}-${randomUUID().slice(0, 4)}` : base;
+  }
+
+  #numericSuffix(id) {
+    const match = /^j(\d+)/u.exec(String(id ?? ''));
+    return match ? Number(match[1]) : 0;
+  }
+
+  /** Turn one stored record into an in-memory job; never assumes it is running. */
+  #adoptPersistedRecord(record) {
+    if (!record || typeof record !== 'object' || !record.id || this.#jobs.has(record.id)) return null;
+    const pid = Number.isFinite(Number(record.pid)) ? Number(record.pid) : null;
+    const pgid = Number.isFinite(Number(record.pgid)) ? Number(record.pgid) : pid;
+    const job = {
+      id: record.id,
+      host: record.host,
+      pid,
+      pgid,
+      logPath: record.logPath ?? null,
+      commandLogPath: record.commandLogPath ?? null,
+      statusPath: record.statusPath ?? null,
+      command: record.command ?? null,
+      cwd: record.cwd ?? null,
+      label: record.label ?? null,
+      sessionId: record.sessionId ?? null,
+      startedAt: record.startedAt ?? null,
+      // A finished record stays finished; anything else is `unknown` until a
+      // liveness probe proves the process group still exists.
+      state: record.state === 'exited' ? 'exited' : 'unknown',
+      exitCode: record.exitCode ?? null,
+      exitSignal: record.exitSignal ?? null,
+      exitedAt: record.exitedAt ?? null,
+      logSize: null,
+      logTail: '',
+      stopped: false,
+      persisted: true,
+      lastPersistedJson: null,
+    };
+    this.#jobs.set(job.id, job);
+    this.#counter = Math.max(this.#counter, this.#numericSuffix(job.id));
+    return job;
+  }
+
+  #storeRecords() {
+    const store = this.#jobStore;
+    if (!store || typeof store.list !== 'function') return [];
+    try {
+      const records = store.list();
+      return Array.isArray(records) ? records : [];
+    } catch {
+      return [];
+    }
+  }
+
+  /** Load the registry at startup; records are visible but not trusted as live. */
+  #loadPersistedJobs() {
+    for (const record of this.#storeRecords()) this.#adoptPersistedRecord(record);
+  }
+
+  /** Pick up records a second process (or an earlier prune) wrote meanwhile. */
+  #mergePersistedJobs() {
+    for (const record of this.#storeRecords()) this.#adoptPersistedRecord(record);
+  }
+
+  /**
+   * First connection to a host after a restart: resolve that machine's
+   * persisted jobs instead of leaving them `unknown` until the next listing.
+   * Called fire-and-forget from the connection handler; never affects dialling.
+   */
+  async #refreshPersistedJobs(host) {
+    const pending = [...this.#jobs.values()].filter(
+      (job) => job.host === host && job.persisted === true && job.state !== 'exited',
+    );
+    if (pending.length === 0) return;
+    await this.#probeHostJobs(host, pending);
+  }
+
+  /**
+   * Write one job through to the JSON store, skipping unchanged records: the
+   * state poll would otherwise rewrite the file every couple of seconds.
+   */
+  #syncJob(job) {
+    const store = this.#jobStore;
+    if (!store || typeof store.upsert !== 'function') return;
+    const record = {
+      id: job.id,
+      host: job.host,
+      label: job.label ?? null,
+      command: job.command ?? null,
+      sessionId: job.sessionId ?? null,
+      pid: job.pid ?? null,
+      pgid: job.pgid ?? job.pid ?? null,
+      logPath: job.logPath ?? null,
+      commandLogPath: job.commandLogPath ?? null,
+      statusPath: job.statusPath ?? null,
+      cwd: job.cwd ?? null,
+      startedAt: job.startedAt ?? null,
+      exitedAt: job.exitedAt ?? null,
+      exitCode: job.exitCode ?? null,
+      exitSignal: job.exitSignal ?? null,
+      state: job.state ?? 'unknown',
+    };
+    const json = JSON.stringify(record);
+    if (json === job.lastPersistedJson) return;
+    try {
+      store.upsert(record);
+      job.lastPersistedJson = json;
+    } catch {
+      /* a read-only home must never fail a tool; the live job still works */
+    }
   }
 
   /**
@@ -632,7 +974,11 @@ export class PresencePool {
       id: job.id,
       host: job.host,
       pid: job.pid,
+      pgid: job.pgid ?? job.pid ?? null,
+      // The wrapper log (exit status) and, when the kernel exposes one, the
+      // command's own stdout/stderr log. Older kernels only have `logPath`.
       logPath: job.logPath,
+      commandLogPath: job.commandLogPath ?? null,
       statusPath: job.statusPath,
       state,
       exitCode,
@@ -814,5 +1160,6 @@ export class PresencePool {
     this.#jobs.clear();
     for (const connection of this.#connections.values()) connection.end();
     this.#connections.clear();
+    this.#hostStates.clear();
   }
 }