@ai-dossier/sched 0.69.0 → 0.70.0-next.714.1

package/README.md

@@ -388,7 +388,7 @@ what makes it come back by itself, so no session ever has to start, stop or baby
 
 ```bash
 ai-dossier sched service install [--alert-issue <n>]   # systemd user unit; cron watchdog if no systemd
-ai-dossier sched service status                         # supervised? engine live? heartbeat?
+ai-dossier sched service status                         # supervised? engine live? heartbeat? gates in flight (#1233)?
 ai-dossier sched service uninstall
 ```
 
@@ -490,15 +490,24 @@ adds a second line with its stack):
 | `stop-timeout` | a requested stop was still running at the deadline; forced exit | released |
 | `uncaught-exception` / `unhandled-rejection` | crash, stack in `detail`, exit code 70 | **kept** as the crash marker |
 | `process-exit` | a bare `process.exit()` nothing else logged | released |
-| *(no line)* | SIGKILL / OOM killer — unobservable | left stale |
+| *(no line)* | SIGKILL — the OOM killer, or a host process reaper (cron jobs killing long-lived TTY-less `node` processes); check `journalctl -k`, allowlist `ai-dossier sched` / `ai-dossier cap run` in reapers (#1233) — unobservable | left stale |
 
 The exit logger is installed immediately after the lease is acquired, so no later startup
 step can exit unlogged. A tick that is blocked in a synchronous call cannot run signal
 handlers, so every synchronous exec on the tick path is timeout-bounded (`defaultExec`: 120 s;
 the engine's own execs have explicit budgets) and the stop is therefore bounded too.
+Since #1233 the continuous `sched start` loop runs each tick in a worker thread
+(`DOSSIER_SCHED_INLINE_TICK=1` restores the inline tick; `--once` is always inline), so
+SIGTERM/SIGINT are handled and journaled at once even during a 25–55 min batch gate. The
+graceful stop still waits for the tick, so a stop mid-gate ends as `stop-timeout` after 120 s
+(or the supervisor's own stop timeout); the gate runs in its own process group, keeps
+running, and the next engine adopts it from `<state>/in-flight/` instead of launching a
+duplicate.
 
 The lease's `updated_at` is a heartbeat rewritten (compare-and-swap on the lease id) every
-tick; `sched status` shows it. A live pid whose heartbeat is older than the hung threshold
+tick and, since #1233, every 30 s while an off-thread tick is waiting on a live gate this
+engine launched or adopted, within that gate's own deadline (`waitingOnLiveGate`; a hard
+4 h ceiling backs it up) — a tick stuck anywhere else still goes stale; `sched status` shows it. A live pid whose heartbeat is older than the hung threshold
 (10 x the reconcile interval, never under 5 min; `$DOSSIER_SCHED_HUNG_INTERVALS` overrides
 the multiplier) with unfinished work is an `engine-hung` warning/alert. Visible alerts are
 journaled, printed to stderr, and posted as a comment when a tracking issue is given
@@ -511,6 +520,9 @@ journaled, printed to stderr, and posted as a comment when a tracking issue is g
   hung (`reason=engine-hung`) while work is unfinished. Once per episode (an `O_EXCL` marker
   file, so racing watchers cannot double-alert); raised by a watcher — `sched status --alert`
   (cron-able). An unwritable store never makes the watcher throw.
+- Since #1233 these alerts name the SIGKILL suspects (`SIGKILL_SUSPECTS_HINT`: oom-kill,
+  host process reapers) and list the gates recorded in `<state>/in-flight/`
+  (`describeInFlight`), which the next engine adopts instead of relaunching.
 
 **#920's silent exit:** the log ended in `nothing to do`, which is just an idle tick's line.
 Only SIGINT had a handler, so a SIGTERM/SIGHUP (another session's restart, a closing
@@ -1339,6 +1351,14 @@ import {
   tick,                  // one engine cycle: reconcile + verify + refill + spawn,
                          //   and since #468: park-watch, teardown, report dispatch
   runLoop,               // the sched start loop (tick, sleep, repeat)
+  type RunLoopOptions,   // #1233: runTick (off-thread tick) / heartbeatMs / maxTickHeartbeatMs / shouldHeartbeat
+  writeInFlight, readInFlight, removeInFlight, listInFlight, // #1233: <state>/in-flight/ gate registry
+  inFlightAlive, inFlightFinished, inFlightDir, ensureInFlightDir, describeInFlight,
+  waitingOnLiveGate, pidExists, IN_FLIGHT_RUN_FILES, IN_FLIGHT_RUN_DIR_PREFIX,
+  type InFlightCapRun, type InFlightListing,
+  SIGKILL_SUSPECTS_HINT, // #1233: alert text naming oom-kill / host reapers
+  releaseOwnStateLock,   // #1233: free a state lock a dead tick worker left naming this pid
+  procStatFields,        // /proc/<pid>/stat post-comm fields (state, pgrp, starttime)
   type TickResult,       // what one tick did (spawned/parked/merge-accepted/stale-reconciled/
                          //   dependents-unblocked/report-dispatched/teardown/completed/
                          //   redispatched/failed/blocked, and since #544

package/dist/batch-dispatch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAWH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAeL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAGpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AAMpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAW5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EActB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EA2BL,kBAAkB,EAKnB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAGV,UAAU,EAEV,oBAAoB,EAKpB,kBAAkB,EAClB,cAAc,EAId,WAAW,EACX,UAAU,EAEX,MAAM,SAAS,CAAC;AAajB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAI1C,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,iBAAiB,KAAK,WAAW,CAAC;IACrE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;CACvB;AA+sBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AAuuDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe;AACvB,kGAAkG;AAClG,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,UAAU,EACxC,UAAU,CAAC,EAAE,MAAM,IAAI,GACtB,IAAI,CAiBN;AAoWD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA+KA;AAugCD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAgoB1C;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,CAuB3F;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,iBAAiB,EACvB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,EACvB,SAAS,EAAE,iBAAiB,GAAG,SAAS,GACvC,IAAI,CAmMN;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAuG/E;AAED;;;;;;GAMG;AACH,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,gBAAgB,CAAC;IACvB,uEAAuE;IACvE,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC;CAC5B;AAoTD;;;;;;;;;;;GAWG;AACH,oCAAoC;AACpC,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,kBAAkB,CAQpB;AAgnBD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,IAAI,CAAC;AA8UvC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAySjB"}
+{"version":3,"file":"batch-dispatch.d.ts","sourceRoot":"","sources":["../src/batch-dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AAWH,OAAO,EAIL,KAAK,YAAY,EAGjB,KAAK,gBAAgB,EAGtB,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAeL,KAAK,gBAAgB,EAGrB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAGpB,OAAO,EAGL,KAAK,WAAW,EAcjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,OAAO,EAAa,MAAM,WAAW,CAAC;AAMpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAW5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EACL,KAAK,iBAAiB,EActB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EA2BL,kBAAkB,EAKnB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAGV,UAAU,EAEV,oBAAoB,EAKpB,kBAAkB,EAClB,cAAc,EAId,WAAW,EACX,UAAU,EAEX,MAAM,SAAS,CAAC;AAajB,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAI1C,OAAO,EAAE,kBAAkB,EAAE,CAAC;AAE9B,8DAA8D;AAC9D,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,iBAAiB,KAAK,WAAW,CAAC;IACrE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,aAAa,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,oBAAoB,CAAC;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,uFAAuF;IACvF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,8FAA8F;IAC9F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;;;OAIG;IACH,IAAI,CAAC,EAAE,cAAc,CAAC;CACvB;AAgtBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3F;AAuuDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,EACtB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe;AACvB,kGAAkG;AAClG,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,UAAU,EACxC,UAAU,CAAC,EAAE,MAAM,IAAI,GACtB,IAAI,CAiBN;AAoWD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,EAC1B,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,GACR;IACD,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,cAAc,GAAG,qBAAqB,CAAC;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CA+KA;AAugCD;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,IAAI,CAAC;AAgoB1C;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,CAuB3F;AA6HD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,iBAAiB,EACvB,GAAG,EAAE,IAAI,EACT,MAAM,EAAE,eAAe,EACvB,SAAS,EAAE,iBAAiB,GAAG,SAAS,GACvC,IAAI,CAmMN;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,GAAG,IAAI,CAuG/E;AAED;;;;;;GAMG;AACH,UAAU,iBAAiB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,gBAAgB,CAAC;IACvB,uEAAuE;IACvE,SAAS,EAAE,gBAAgB,CAAC;IAC5B,YAAY,EAAE,YAAY,CAAC;CAC5B;AAoTD;;;;;;;;;;;GAWG;AACH,oCAAoC;AACpC,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,kBAAkB,CAQpB;AAgnBD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,IAAI,CAAC;AA8UvC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,EACvB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,GACzB,eAAe,CAySjB"}

package/dist/batch-dispatch.js

@@ -364,7 +364,8 @@ function describeInconclusive(gate) {
  * this repo's own machinery, at exactly two sites: `cli/src/capability.ts`'s
  * `classifySpawnResult` (the capability entry's own `timeout_ms`) and
  * `cli/src/commands/sched.ts`'s `createBatchCapabilityRunner` (the runner's
- * own `spawnSync` timeout, `BATCH_SUITE_TIMEOUT_MS`). The gate classifies on
+ * own timeout, `BATCH_SUITE_TIMEOUT_MS` — `spawnSync`'s or, for engine gates
+ * since #1233, the tracked launcher's synthetic ETIMEDOUT). The gate classifies on
  * it to tell "the harness needed more time than it was given" (#681 — a
  * statement about DURATION, not about the harness's reliability) apart from a
  * genuine machinery failure (missing tool, broken manifest, evidence-free