agent-nuvira 3.5.1 → 3.5.2

package/dist/web-dashboard/task-runner.d.ts

@@ -11,14 +11,40 @@
  *  - get(id) / list()                     → serializable task records
  *  - onEvent(cb)                          → 'log'/'status' events (SSE fan-out)
  *
- * Timeouts SIGTERM the child, then SIGKILL after a grace period. Logs are
- * ring-buffered (capped) so a chatty command can't exhaust memory. All state
- * is in-memory (P4: persist task history + rerun).
+ * An EXPLICIT timeout SIGTERMs the child, then SIGKILLs after a grace period.
+ * With no timeout (the default) nothing is killed on a clock — instead a
+ * LIVENESS WATCHDOG samples the task's process tree and distinguishes a command
+ * that is genuinely HUNG (no output AND no CPU: a prompt with no stdin, a lock,
+ * a socket that never answers) from one that is merely slow (silent but still
+ * burning CPU). A confirmed hang is flagged on the record and in the log, and —
+ * unless the command is a long-lived server/watch command, which is SUPPOSED to
+ * be silent and idle — the runner stops it after a further grace, so a hang is
+ * never left running forever. Logs are ring-buffered (capped) so a chatty
+ * command can't exhaust memory.
+ *
+ * STATE OUTLIVES THE PROCESS (see `task-registry.ts`). A task is written to a
+ * durable record the moment it starts — pid, args, cwd, start time, the OS's
+ * start token for that pid — and its output to a bounded log window. A new
+ * dashboard process HYDRATES from those: the list and the log pane come back,
+ * a task whose process is still alive is re-attached to the liveness watchdog,
+ * and one whose pid is gone is settled honestly (no exit code, because none was
+ * ever observed). The bug this fixes was measured: restarting the dashboard left
+ * the child running and the console empty, so work kept burning CPU that nothing
+ * could see, collect or stop.
+ *
+ * A recovered process is stopped ONLY once its identity is re-verified (the
+ * command line AND the OS start token — see `processIdentityVerdict`); a refusal
+ * is logged on the task with its pid so it can be stopped by hand.
+ *
+ * ONE OWNER PER PROFILE, like the monitored-run registry: two dashboards sharing
+ * a config dir write the same records and the last write wins. That is a
+ * limitation, not a claim.
  *
  * The spawn target is injectable (`execPath`/`cliEntry`) so unit tests can
  * point at fixture scripts instead of the real CLI.
  */
-export type TaskStatus = 'running' | 'done' | 'failed' | 'cancelled' | 'timeout' | 'error';
+import { type Liveness } from '../tools/process-liveness.js';
+export type TaskStatus = 'running' | 'done' | 'failed' | 'cancelled' | 'timeout' | 'error' | 'hung';
 export type TaskLogStream = 'stdout' | 'stderr' | 'system';
 export interface TaskLogLine {
     stream: TaskLogStream;
@@ -39,8 +65,36 @@ export interface TaskRecord {
     durationMs: number | null;
     timeoutMs: number;
     logs: TaskLogLine[];
+    /** When the last output arrived — the silence half of the liveness check. */
+    lastOutputAt: number;
+    /**
+     * The watchdog's last liveness verdict and the evidence behind it. Present
+     * once a run has been sampled; this is what makes "still running" different
+     * from "running, and here is why I believe it is stuck".
+     */
+    liveness?: {
+        verdict: Liveness;
+        why: string;
+        at: number;
+    };
+    /** When a CONFIRMED hang began (cleared the moment it speaks or computes). */
+    hungSince?: number | null;
+    /** Why the task stopped, when it was stopped rather than finishing on its own. */
+    stopReason?: string;
     /** Spawn-level errors (ENOENT, entry missing, …). */
     error?: string;
+    /** The child's pid — what a RESTART needs in order to find the work again. */
+    pid?: number | null;
+    /**
+     * Set only on a task recovered from a previous dashboard process. `verified`
+     * is the identity verdict (`null` = this platform could not tell), and `why`
+     * is the evidence — a recovered task that cannot be verified is still SHOWN,
+     * but never stopped.
+     */
+    recovered?: {
+        verified: boolean | null;
+        why: string;
+    };
 }
 export interface TaskRunnerOptions {
     /** Executable to spawn. Defaults to process.execPath (node). */
@@ -50,6 +104,16 @@ export interface TaskRunnerOptions {
     /** Working directory for spawned tasks. Defaults to process.cwd(). */
     cwd?: string;
 }
+/** True when this command is a long-lived server/watcher (never reaped for a hang). */
+export declare function isLongLivedCommand(args: string[]): boolean;
+/**
+ * How long a CONFIRMED hang runs before the runner stops it (0 = never — flag it
+ * and leave the decision to the operator, who can always cancel).
+ *
+ * Ten minutes by default, and the grace only starts once a hang has been
+ * ESTABLISHED from evidence, so a slow-but-working command is never touched.
+ */
+export declare function taskHangReapMs(): number;
 export interface TaskEventPayload {
     kind: 'log' | 'status';
     /** New log line (log events only). */
@@ -60,6 +124,8 @@ export interface TaskEventPayload {
         exitCode: number | null;
         durationMs: number | null;
         error?: string;
+        /** Set when the task was stopped rather than finishing on its own. */
+        stopReason?: string;
     };
 }
 export declare class TaskRunner {
@@ -69,8 +135,63 @@ export declare class TaskRunner {
     private children;
     private cancelled;
     private timedOut;
+    /** Tasks the liveness watchdog stopped (distinct from a user's cancel). */
+    private hung;
+    /** The last CPU sample per task — the delta is what proves a hang. */
+    private cpuSamples;
+    private watchers;
     private listeners;
+    /** The bounded on-disk log window per task (what a restart reads back). */
+    private logs;
+    /**
+     * Recovered RUNNING tasks: their pid, and whether it could be re-verified as
+     * the process this dashboard started. A recovered task has no child handle, so
+     * this is what the watchdog samples and what a cancel is gated on.
+     */
+    private recoveredRuns;
+    /**
+     * The identity of each task's process (OS start token + live command line at
+     * spawn), remembered so every later write of the record keeps it — a record
+     * that lost its token could no longer be re-verified after a restart.
+     */
+    private identities;
     constructor(opts?: TaskRunnerOptions);
+    /**
+     * Restore the previous process's task list, logs, and live pids.
+     *
+     * This is the whole point of the registry: without it, a dashboard restart
+     * orphaned a child that kept running while the console forgot it existed.
+     *
+     * A recovered task whose pid is GONE is settled honestly — with no exit code,
+     * because none was ever observed — rather than being left "running" forever or
+     * quietly deleted. A recovered task whose pid may belong to someone else is
+     * followed and reported, and only stopped once its identity is re-verified.
+     */
+    private hydrate;
+    /** The restored transcript of a task, capped like the live ring buffer. */
+    private restoreLogs;
+    /** Write one task's record to the registry (best-effort — never throws). */
+    private persist;
+    /**
+     * Capture the identity of a task's process, for a restart that has to tell it
+     * apart from whatever holds that pid later.
+     *
+     * Read NOW, at spawn, because that is the last moment it is certainly ours. If
+     * the read misses (a freshly forked child can take a few milliseconds to appear
+     * in the table) it is retried once off the request path; a task that still has
+     * no token is followable after a restart but not stoppable, and it SAYS so in
+     * its own log rather than failing quietly.
+     */
+    private captureIdentity;
+    /** Start the liveness watchdog for one task (spawned or recovered). */
+    private startWatching;
+    /**
+     * Settle a RECOVERED task — one with no child handle, so nothing will ever
+     * fire an exit event for it. The status is the best evidence available, and the
+     * exit code is `null` because none was observed: inventing one would be the
+     * exact lie this feature exists to avoid.
+     */
+    private settleRecovered;
     /** Subscribe to task events ('log' + 'status'). Returns an unsubscribe fn. */
     onEvent(cb: (id: string, payload: TaskEventPayload) => void): () => void;
     private emit;
@@ -92,7 +213,36 @@ export declare class TaskRunner {
         task?: TaskRecord;
         error?: string;
     };
-    /** Cancel a running task (SIGTERM). True when a task existed. */
+    private stopWatching;
+    /**
+     * One liveness sample for a running task: classify it, flag a hang on the
+     * record (and in the log, live), and — only for a task that is not a
+     * long-lived server, and only after the grace — stop it.
+     *
+     * Exported behaviour is reached through the watchdog interval; this method is
+     * also the seam a test drives directly.
+     */
+    private sampleLiveness;
+    /** Stop a task the watchdog has established is hung, and say why. */
+    private stopHungTask;
+    /**
+     * The same, for a task recovered from a previous dashboard process.
+     *
+     * Gated on the identity verdict. A pid that outlived the process table it came
+     * from may now belong to something else, and this runner only ever signals work
+     * IT started — so an unverifiable "hang" is reported and left running, and the
+     * refusal says why. That is the honest trade: a hung command that keeps running
+     * is bad, and stopping a stranger's process is worse.
+     */
+    private stopRecoveredHung;
+    /**
+     * Cancel a running task (SIGTERM). True when a task existed and was signalled.
+     *
+     * A task recovered from a previous dashboard process has no child handle, so it
+     * is stoppable only once its pid was re-verified as the process this dashboard
+     * started. A refusal is logged on the task itself (not just returned) so the
+     * operator sees it in the pane, with the pid for a manual stop.
+     */
     cancel(id: string): boolean;
 }
 //# sourceMappingURL=task-runner.d.ts.map

package/dist/web-dashboard/task-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"task-runner.d.ts","sourceRoot":"","sources":["../../src/web-dashboard/task-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAQH,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC;AAE3F,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE3D,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,aAAa,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,EAAE,CAAC;IACpB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAuBD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,GAAG,QAAQ,CAAC;IACvB,sCAAsC;IACtC,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,4CAA4C;IAC5C,MAAM,CAAC,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACrG;AAED,qBAAa,UAAU;IAQT,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPjC,OAAO,CAAC,KAAK,CAAiC;IAC9C,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,SAAS,CAAqB;IACtC,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,SAAS,CAA8D;gBAElD,IAAI,GAAE,iBAAsB;IAEzD,8EAA8E;IAC9E,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,KAAK,IAAI,GAAG,MAAM,IAAI;IAKxE,OAAO,CAAC,IAAI;IAUZ,wEAAwE;IACxE,QAAQ,IAAI,MAAM;IAMlB,0EAA0E;IAC1E,IAAI,IAAI,UAAU,EAAE;IAIpB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIvC,OAAO,CAAC,SAAS;IAajB;;;OAGG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,UAAU,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAmI1H,iEAAiE;IACjE,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;CAW5B"}
+{"version":3,"file":"task-runner.d.ts","sourceRoot":"","sources":["../../src/web-dashboard/task-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAOH,OAAO,EAUL,KAAK,QAAQ,EAEd,MAAM,8BAA8B,CAAC;AAYtC,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AAEpG,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE3D,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,aAAa,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,+DAA+D;AAC/D,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,WAAW,EAAE,CAAC;IACpB,6EAA6E;IAC7E,YAAY,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,QAAQ,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8EAA8E;IAC9E,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;;;;OAKG;IACH,SAAS,CAAC,EAAE;QAAE,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;CACvD;AAED,MAAM,WAAW,iBAAiB;IAChC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAgDD,uFAAuF;AACvF,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAE1D;AAOD;;;;;;GAMG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAEvC;AAQD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,GAAG,QAAQ,CAAC;IACvB,sCAAsC;IACtC,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,4CAA4C;IAC5C,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,UAAU,CAAC;QACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,sEAAsE;QACtE,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,qBAAa,UAAU;IA2BT,OAAO,CAAC,QAAQ,CAAC,IAAI;IA1BjC,OAAO,CAAC,KAAK,CAAiC;IAC9C,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,SAAS,CAAqB;IACtC,OAAO,CAAC,QAAQ,CAAqB;IACrC,2EAA2E;IAC3E,OAAO,CAAC,IAAI,CAAqB;IACjC,sEAAsE;IACtE,OAAO,CAAC,UAAU,CAA2D;IAC7E,OAAO,CAAC,QAAQ,CAAqD;IACrE,OAAO,CAAC,SAAS,CAA8D;IAC/E,2EAA2E;IAC3E,OAAO,CAAC,IAAI,CAAkC;IAC9C;;;;OAIG;IACH,OAAO,CAAC,aAAa,CAA6C;IAClE;;;;OAIG;IACH,OAAO,CAAC,UAAU,CAAmE;gBAExD,IAAI,GAAE,iBAAsB;IAMzD;;;;;;;;;;OAUG;IACH,OAAO,CAAC,OAAO;IA0Ef,2EAA2E;IAC3E,OAAO,CAAC,WAAW;IAOnB,4EAA4E;IAC5E,OAAO,CAAC,OAAO;IAuBf;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IA8BvB,uEAAuE;IACvE,OAAO,CAAC,aAAa;IAQrB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IA0BvB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,KAAK,IAAI,GAAG,MAAM,IAAI;IAKxE,OAAO,CAAC,IAAI;IAUZ,wEAAwE;IACxE,QAAQ,IAAI,MAAM;IAMlB,0EAA0E;IAC1E,IAAI,IAAI,UAAU,EAAE;IAIpB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIvC,OAAO,CAAC,SAAS;IA0BjB;;;OAGG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,UAAU,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IA+K1H,OAAO,CAAC,YAAY;IAMpB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IA6EtB,qEAAqE;IACrE,OAAO,CAAC,YAAY;IAUpB;;;;;;;;OAQG;IACH,OAAO,CAAC,iBAAiB;IAuBzB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;CA0C5B"}