@tekyzinc/gsd-t 3.15.10 → 3.16.12

package/bin/gsd-t-unattended-heartbeat.cjs

@@ -0,0 +1,188 @@
+/**
+ * gsd-t-unattended-heartbeat.cjs
+ *
+ * Liveness heartbeat watchdog for the unattended supervisor.
+ *
+ * Supersedes the pre-M43 `workerTimeoutMs` wall-clock guillotine as the
+ * PRIMARY stuck-worker detector. The guillotine remains as an absolute
+ * backstop (raised to 1 hour by default) for pathological cases where a
+ * child never writes a single event.
+ *
+ * How it works
+ * ────────────
+ * The supervisor polls `.gsd-t/events/YYYY-MM-DD.jsonl` mtime every 60 s
+ * during a worker iteration. If the mtime has not advanced for at least
+ * `staleHeartbeatMs` (default 300_000 = 5 min), the worker is considered
+ * stuck and SIGTERM'd. Healthy workers producing events run indefinitely
+ * under the 1-hour absolute cap.
+ *
+ * This module is pure and side-effect-free by default. `checkHeartbeat()`
+ * accepts injected `now` and `fsShim` so the entire watchdog can be
+ * unit-tested with a fake clock and fake filesystem.
+ *
+ * Zero external dependencies — Node built-ins only.
+ *
+ * Contract: .gsd-t/contracts/unattended-supervisor-contract.md v1.1.0
+ *   §"Heartbeat Watchdog"
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+
+/**
+ * Build the events JSONL path for a given date.
+ *
+ * @param {string} projectDir
+ * @param {Date|number} when  Date, or ms since epoch. Defaults to now when
+ *   omitted at the call site.
+ * @returns {string}
+ */
+function eventsPathFor(projectDir, when) {
+  const d = when instanceof Date ? when : new Date(when || Date.now());
+  const y = d.getUTCFullYear();
+  const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+  const day = String(d.getUTCDate()).padStart(2, "0");
+  return path.join(projectDir, ".gsd-t", "events", `${y}-${m}-${day}.jsonl`);
+}
+
+/**
+ * Check whether the worker's event stream is stale.
+ *
+ * A worker is "stale" when the relevant events JSONL file's mtime has not
+ * advanced within `staleHeartbeatMs` of the given `now`. The relevant file
+ * is the one matching the date of `now` — if the loop crosses a UTC day
+ * boundary mid-iteration, the new day's file is checked.
+ *
+ * Fresh worker grace: if the events file does not exist yet AND
+ * `(now - workerStartedAt) < staleHeartbeatMs`, the worker is considered
+ * healthy (still booting). After the grace window with no file, the worker
+ * is stale.
+ *
+ * @param {object} params
+ * @param {string} params.projectDir
+ * @param {number} params.workerStartedAt  ms since epoch
+ * @param {number} params.staleHeartbeatMs
+ * @param {number} [params.now]            ms since epoch (defaults to Date.now())
+ * @param {object} [params.fsShim]         { existsSync, statSync } — test hook
+ * @returns {{stale: boolean, reason: string, lastEventMs: (number|null), ageMs: (number|null), eventsPath: string}}
+ */
+function checkHeartbeat({
+  projectDir,
+  workerStartedAt,
+  staleHeartbeatMs,
+  now,
+  fsShim,
+}) {
+  if (typeof projectDir !== "string" || projectDir.length === 0) {
+    throw new Error("checkHeartbeat: projectDir is required");
+  }
+  if (typeof workerStartedAt !== "number" || !Number.isFinite(workerStartedAt)) {
+    throw new Error("checkHeartbeat: workerStartedAt must be a finite number");
+  }
+  if (
+    typeof staleHeartbeatMs !== "number" ||
+    !Number.isFinite(staleHeartbeatMs) ||
+    staleHeartbeatMs <= 0
+  ) {
+    throw new Error("checkHeartbeat: staleHeartbeatMs must be a positive number");
+  }
+  const nowMs = typeof now === "number" ? now : Date.now();
+  const shim = fsShim || fs;
+
+  const eventsPath = eventsPathFor(projectDir, nowMs);
+
+  let exists = false;
+  try {
+    exists = !!shim.existsSync(eventsPath);
+  } catch (_) {
+    exists = false;
+  }
+
+  if (!exists) {
+    const sinceStart = nowMs - workerStartedAt;
+    if (sinceStart < staleHeartbeatMs) {
+      return {
+        stale: false,
+        reason: `events file not yet created (grace: ${sinceStart}ms < ${staleHeartbeatMs}ms)`,
+        lastEventMs: null,
+        ageMs: null,
+        eventsPath,
+      };
+    }
+    return {
+      stale: true,
+      reason: `events file ${eventsPath} absent for ${sinceStart}ms since worker start (threshold ${staleHeartbeatMs}ms)`,
+      lastEventMs: null,
+      ageMs: sinceStart,
+      eventsPath,
+    };
+  }
+
+  let stat;
+  try {
+    stat = shim.statSync(eventsPath);
+  } catch (err) {
+    // File existed at existsSync but stat failed — treat as stale only if
+    // we are past the grace window. Under the grace window, assume transient.
+    const sinceStart = nowMs - workerStartedAt;
+    if (sinceStart < staleHeartbeatMs) {
+      return {
+        stale: false,
+        reason: `events stat transient failure (grace): ${err.message}`,
+        lastEventMs: null,
+        ageMs: null,
+        eventsPath,
+      };
+    }
+    return {
+      stale: true,
+      reason: `events stat failed past grace: ${err.message}`,
+      lastEventMs: null,
+      ageMs: sinceStart,
+      eventsPath,
+    };
+  }
+
+  const mtimeMs =
+    typeof stat.mtimeMs === "number"
+      ? stat.mtimeMs
+      : stat.mtime instanceof Date
+        ? stat.mtime.getTime()
+        : 0;
+
+  // Reference point for staleness: max(mtime, workerStartedAt). This handles
+  // the bootstrap case where the events file already existed from a prior
+  // iteration — we don't want to kill the worker on its first 60s poll just
+  // because it hasn't emitted yet. The worker gets at least staleHeartbeatMs
+  // from its own start to produce the first event.
+  const ref = Math.max(mtimeMs, workerStartedAt);
+  const ageMs = nowMs - ref;
+
+  if (ageMs >= staleHeartbeatMs) {
+    return {
+      stale: true,
+      reason: `last event ${ageMs}ms ago (threshold ${staleHeartbeatMs}ms)`,
+      lastEventMs: mtimeMs,
+      ageMs,
+      eventsPath,
+    };
+  }
+  return {
+    stale: false,
+    reason: `fresh — last event ${ageMs}ms ago`,
+    lastEventMs: mtimeMs,
+    ageMs,
+    eventsPath,
+  };
+}
+
+module.exports = {
+  checkHeartbeat,
+  eventsPathFor,
+  // Default heartbeat poll cadence — exported so tests and the supervisor
+  // can reference a single source of truth.
+  DEFAULT_HEARTBEAT_POLL_MS: 60 * 1000,
+  DEFAULT_STALE_HEARTBEAT_MS: 5 * 60 * 1000,
+};

package/bin/gsd-t-unattended-platform.cjs

@@ -14,7 +14,8 @@
  * Task 1 of m36-cross-platform delivers:
  *   - resolveClaudePath()
  *   - isAlive(pid)
- *   - spawnWorker(projectDir, timeoutMs)
+ *   - spawnWorker(projectDir, timeoutMs, opts)
+ *       opts.onHeartbeatCheck? — async liveness watchdog (M43 D?)
  *
  * Cross-platform notes:
  *   - darwin / linux paths are runtime-tested.
@@ -89,46 +90,85 @@ function isAlive(pid) {
 // ─── spawnWorker ─────────────────────────────────────────────────────────────
 
 /**
- * Spawn a synchronous `claude -p '/gsd-t-resume'` worker iteration for the
- * unattended supervisor.
+ * Spawn a `claude -p '/gsd-t-resume'` worker iteration for the unattended
+ * supervisor.
  *
  * Returns a normalized result object: `{ status, stdout, stderr, signal,
- * timedOut, error }`. Never throws — spawn errors are returned in `error`.
+ * timedOut, staleHeartbeat, error }`. Never throws — spawn errors are
+ * returned in `error`.
  *
- * Timeout semantics: when `spawnSync`'s `timeout` fires, the child is sent
- * SIGTERM (or the equivalent on win32), `status` is `null`, and `signal` is
- * non-null. We surface this as `timedOut: true` so callers can map to exit
- * code 3 per contract §5.
+ * Two kill paths:
+ *   1. Heartbeat watchdog (M43 primary) — when `opts.onHeartbeatCheck` is
+ *      provided, the function polls every `opts.heartbeatPollMs` (default
+ *      60_000). If the callback returns `{stale: true, ...}`, the child is
+ *      SIGTERM'd and `staleHeartbeat: true` is set on the result.
+ *   2. Wall-clock timeout (absolute backstop) — `timeoutMs` is the hard cap
+ *      regardless of heartbeat. On expiry the child is SIGTERM'd and
+ *      `timedOut: true`. Default raised to 1 h in supervisor-core so a
+ *      healthy long-running worker is not cut.
  *
- * Spawn recipe (uniform across platforms):
- *   - `shell: false`  → no shell quoting hazards
- *   - `windowsHide: true`  → no flashed window on win32
- *   - explicit `claude.cmd` filename on win32 (see resolveClaudePath JSDoc)
+ * Exactly one path is used per iteration:
+ *   - opts.onHeartbeatCheck present → heartbeat path (async event loop)
+ *   - opts.onHeartbeatCheck absent  → legacy spawnSync path (blocking)
+ * The legacy path is preserved so callers that have no meaningful liveness
+ * signal (test stubs, dry-run) keep the original semantics.
  *
- * @todo Spike C: verify `claude.cmd -p "/gsd-t-resume"` dispatches correctly
- *       under PowerShell + cmd.exe + Git Bash. See
- *       `docs/unattended-windows-caveats.md` (Task 3 of m36-cross-platform).
+ * Cross-platform:
+ *   - darwin / linux: runtime-tested.
+ *   - win32: implementation-complete; documented in
+ *     `docs/unattended-windows-caveats.md` (Task 3).
  *
  * @param {string} projectDir   Absolute path to the project directory (cwd).
- * @param {number} timeoutMs    Wall-clock cap per worker iteration in ms.
+ * @param {number} timeoutMs    Wall-clock backstop per worker iteration in ms.
  * @param {object} [opts]       Optional overrides (test-mode hooks).
  * @param {string} [opts.bin]   Override the resolved binary (test-mode only).
  * @param {string[]} [opts.args] Override args (defaults to `['-p', '/gsd-t-resume']`).
  * @param {object} [opts.env]   Override env (defaults to `process.env`).
+ * @param {Function} [opts.onHeartbeatCheck]  Called every heartbeatPollMs
+ *   with no args; must return `{stale: boolean, reason?: string}` or a
+ *   Promise thereof. When stale, the child is SIGTERM'd.
+ * @param {number} [opts.heartbeatPollMs]  Poll cadence in ms. Default 60_000.
+ * @param {Function} [opts.onHeartbeatSample]  Optional observer; receives the
+ *   raw callback result each poll for logging.
+ * @param {Function} [opts.onStdoutLine]  Optional live stdout line callback.
+ *   When provided (heartbeat path only), invoked once per `\n`-terminated line
+ *   as stdout streams in. Trailing partial line is flushed on close. Errors
+ *   are swallowed (best-effort tee). Used by the supervisor to write each
+ *   worker line into the M42 transcript file in real time, instead of
+ *   appending the entire stdout buffer post-hoc at child exit.
  * @returns {{
  *   status: number|null,
  *   stdout: string,
  *   stderr: string,
  *   signal: string|null,
  *   timedOut: boolean,
+ *   staleHeartbeat: boolean,
+ *   heartbeatReason: string|null,
  *   error: Error|null
- * }}
+ * }|Promise<...>}
+ *   Returns a Promise when `opts.onHeartbeatCheck` is provided; otherwise a
+ *   synchronous result (legacy path).
  */
 function spawnWorker(projectDir, timeoutMs, opts = {}) {
   const bin = opts.bin || resolveClaudePath();
   const args = opts.args || ["-p", "/gsd-t-resume"];
   const env = opts.env || process.env;
 
+  if (typeof opts.onHeartbeatCheck === "function") {
+    return _spawnWorkerAsyncHeartbeat({
+      bin,
+      args,
+      env,
+      cwd: projectDir,
+      timeoutMs,
+      onHeartbeatCheck: opts.onHeartbeatCheck,
+      heartbeatPollMs: opts.heartbeatPollMs || 60 * 1000,
+      onHeartbeatSample: opts.onHeartbeatSample,
+      onStdoutLine: opts.onStdoutLine,
+      spawnImpl: opts._spawnImpl || spawn,
+    });
+  }
+
   const result = spawnSync(bin, args, {
     cwd: projectDir,
     encoding: "utf8",
@@ -139,19 +179,11 @@ function spawnWorker(projectDir, timeoutMs, opts = {}) {
     windowsHide: true,
   });
 
-  // Normalize. spawnSync may return error if the binary cannot be launched
-  // (ENOENT etc.) — surface it instead of throwing.
   const stdout = typeof result.stdout === "string" ? result.stdout : "";
   const stderr = typeof result.stderr === "string" ? result.stderr : "";
   const signal = result.signal || null;
   const status = typeof result.status === "number" ? result.status : null;
 
-  // Timeout detection: when spawnSync's `timeout` option fires it sets
-  //   - status === null
-  //   - signal !== null  (SIGTERM on POSIX, equivalent on win32)
-  //   - error.code === 'ETIMEDOUT'  (Node surfaces it as a synthetic Error)
-  // The ETIMEDOUT code is the authoritative signal — checking it
-  // discriminates a genuine timeout from an ENOENT/spawn failure.
   const errCode = result.error && result.error.code;
   const timedOut =
     errCode === "ETIMEDOUT" || (status === null && signal !== null && !result.error);
@@ -162,12 +194,144 @@ function spawnWorker(projectDir, timeoutMs, opts = {}) {
     stderr,
     signal,
     timedOut,
-    // Suppress the synthetic ETIMEDOUT error so callers can rely on
-    // `timedOut` for the timeout case and `error` for genuine spawn failures.
+    staleHeartbeat: false,
+    heartbeatReason: null,
     error: errCode === "ETIMEDOUT" ? null : result.error || null,
   };
 }
 
+/**
+ * Async spawn path used when a heartbeat callback is provided.
+ *
+ * Returns a Promise resolving to the same result shape as spawnWorker's
+ * legacy path, plus `staleHeartbeat` and `heartbeatReason`.
+ *
+ * Kill precedence when both fire in the same tick: heartbeat wins (it is
+ * the more specific signal). The loser is suppressed on the result.
+ *
+ * @private
+ */
+function _spawnWorkerAsyncHeartbeat({
+  bin,
+  args,
+  env,
+  cwd,
+  timeoutMs,
+  onHeartbeatCheck,
+  heartbeatPollMs,
+  onHeartbeatSample,
+  onStdoutLine,
+  spawnImpl,
+}) {
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawnImpl(bin, args, {
+        cwd,
+        env,
+        stdio: ["ignore", "pipe", "pipe"],
+        shell: false,
+        windowsHide: true,
+      });
+    } catch (err) {
+      resolve({
+        status: null,
+        stdout: "",
+        stderr: "",
+        signal: null,
+        timedOut: false,
+        staleHeartbeat: false,
+        heartbeatReason: null,
+        error: err,
+      });
+      return;
+    }
+
+    const chunksOut = [];
+    const chunksErr = [];
+    let lineBuf = "";
+    const emitLine = (line) => {
+      if (typeof onStdoutLine !== "function" || line.length === 0) return;
+      try { onStdoutLine(line); } catch (_) { /* tee is best-effort */ }
+    };
+    if (child.stdout) child.stdout.on("data", (d) => {
+      chunksOut.push(d);
+      if (typeof onStdoutLine !== "function") return;
+      lineBuf += d.toString("utf8");
+      let nl;
+      while ((nl = lineBuf.indexOf("\n")) >= 0) {
+        emitLine(lineBuf.slice(0, nl));
+        lineBuf = lineBuf.slice(nl + 1);
+      }
+    });
+    if (child.stderr) child.stderr.on("data", (d) => chunksErr.push(d));
+
+    let resolved = false;
+    let timedOut = false;
+    let staleHeartbeat = false;
+    let heartbeatReason = null;
+    let spawnErr = null;
+
+    const heartbeatTimer = setInterval(async () => {
+      if (resolved) return;
+      let sample;
+      try {
+        sample = await onHeartbeatCheck();
+      } catch (err) {
+        // Heartbeat check must not kill the worker on its own failures.
+        // Surface via sample observer when provided and continue.
+        sample = { stale: false, reason: `heartbeat check threw: ${err.message}` };
+      }
+      if (typeof onHeartbeatSample === "function") {
+        try { onHeartbeatSample(sample); } catch (_) { /* observer best-effort */ }
+      }
+      if (sample && sample.stale) {
+        staleHeartbeat = true;
+        heartbeatReason = sample.reason || "stale heartbeat";
+        try { child.kill("SIGTERM"); } catch (_) { /* child may already be gone */ }
+      }
+    }, heartbeatPollMs);
+    if (typeof heartbeatTimer.unref === "function") heartbeatTimer.unref();
+
+    const absoluteTimer = setTimeout(() => {
+      if (resolved || staleHeartbeat) return;
+      timedOut = true;
+      try { child.kill("SIGTERM"); } catch (_) { /* child may already be gone */ }
+    }, timeoutMs);
+    if (typeof absoluteTimer.unref === "function") absoluteTimer.unref();
+
+    child.on("error", (err) => {
+      spawnErr = err;
+    });
+
+    child.on("close", (status, signal) => {
+      if (resolved) return;
+      resolved = true;
+      clearInterval(heartbeatTimer);
+      clearTimeout(absoluteTimer);
+
+      if (lineBuf.length > 0) {
+        emitLine(lineBuf);
+        lineBuf = "";
+      }
+
+      const stdout = Buffer.concat(chunksOut).toString("utf8");
+      const stderr = Buffer.concat(chunksErr).toString("utf8");
+
+      resolve({
+        status: typeof status === "number" ? status : null,
+        stdout,
+        stderr,
+        signal: signal || null,
+        timedOut,
+        staleHeartbeat,
+        heartbeatReason,
+        error: spawnErr,
+      });
+    });
+  });
+}
+
 // ─── spawnSupervisor ─────────────────────────────────────────────────────────
 
 /**

package/bin/gsd-t-unattended-safety.cjs

@@ -55,7 +55,13 @@ const DEFAULTS = Object.freeze({
   maxIterations: 200,
   hours: 24,
   gutterNoProgressIters: 5,
-  workerTimeoutMs: 270000,
+  // Absolute backstop — raised from 270_000 to 3_600_000 in contract v1.1.0
+  // now that the heartbeat watchdog is the primary stuck-worker detector.
+  workerTimeoutMs: 3600000,
+  // Heartbeat watchdog threshold — events JSONL mtime must advance within
+  // this many ms or the worker is SIGTERM'd. Contract v1.1.0 §"Heartbeat
+  // Watchdog".
+  staleHeartbeatMs: 300000,
 });
 
 // ── Glob → regex helper ─────────────────────────────────────────────────────
@@ -110,6 +116,7 @@ function cloneDefaults() {
     hours: DEFAULTS.hours,
     gutterNoProgressIters: DEFAULTS.gutterNoProgressIters,
     workerTimeoutMs: DEFAULTS.workerTimeoutMs,
+    staleHeartbeatMs: DEFAULTS.staleHeartbeatMs,
   };
 }