harness-dispatch 0.9.0 → 0.10.0

package/dist/jobs/supervisor.d.ts

@@ -0,0 +1,151 @@
+/**
+ * Admission control and the supervisor pool: who runs, when, and in which
+ * process.
+ *
+ * The concurrency cap here exists because of a measured OOM, and the pool
+ * exists because a runner process per job costs ~76 MB of wrapper. Both are
+ * load-bearing; see the comments on DEFAULT_MAX_CONCURRENT_RUNS.
+ */
+import type { RouterConfig } from "../types.js";
+import type { JobDeps, JobStatus } from "./types.js";
+/**
+ * The cap, or `null` for "no cap".
+ *
+ * `null` rather than `0`, and rather than `Infinity`, because both of those
+ * were wrong in a way that mattered. `0` used to short-circuit the whole slot
+ * queue and supervisor pool, so `max_concurrent_runs: 0` — documented as
+ * lifting a limit — silently gave every job its own runner process at ~76 MB,
+ * which is the per-job cost the pool exists to remove, on the memory-bound
+ * machine the cap exists for. And `Infinity` divides badly: the pool sizes
+ * itself with `outstanding / jobsPerSupervisor(limit)`, so an infinite limit
+ * asked for ZERO supervisors. An explicit `null` makes each site say what it
+ * means about the unbounded case.
+ */
+export declare function maxConcurrentRuns(config: RouterConfig | undefined): number | null;
+/**
+ * Occupied slots: jobs actually executing right now. Counts `running` (and
+ * plain `queued` — a runner spawned but not yet started) only while the
+ * heartbeat is fresh, so a crashed runner's slot is reclaimed by the same
+ * ORPHAN_THRESHOLD_MS rule that already frees its status. Slot-queued jobs
+ * are waiting for a slot, not holding one.
+ */
+/**
+ * What one run of a route costs against the concurrency budget.
+ *
+ * Unknown routes count as a full 1.0 on purpose. A job that has not been
+ * routed yet (no forced `service`) has no weight to look up, and this bound
+ * exists because a measured burst of 13 concurrent CLIs exhausted memory —
+ * so the safe assumption for "might be anything" is "might be heavy".
+ */
+export declare function resourceWeightFor(status: JobStatus, config: RouterConfig | undefined): number;
+/**
+ * Capacity currently in use, as a weighted sum rather than a job count.
+ *
+ * With every weight at 1.0 this is exactly the old count, so an existing
+ * `max_concurrent_runs` keeps its previous meaning.
+ */
+export declare function activeCapacity(statuses: Array<{
+    status: JobStatus;
+}>, config: RouterConfig | undefined): number;
+/**
+ * How many supervisor PROCESSES may exist, regardless of how many jobs run.
+ *
+ * Previously every job got its own detached Node process. Measured on Windows
+ * with Node 24: a bare node process is 52 MB RSS and one that has bootstrapped
+ * a runtime is 65 MB, against ~54 MB for the agent CLI it exists to supervise.
+ * So more than half the memory of a concurrent run was wrapper, and it scaled
+ * linearly — 13 concurrent jobs meant 845 MB of supervision before any agent
+ * had read a file. That is the concurrency ceiling.
+ *
+ * A supervisor is almost entirely idle: it waits on a child process and writes
+ * the result. One can watch several at once for the cost of async I/O, so
+ * wrapper memory becomes O(1) in the number of jobs instead of O(N), capped
+ * here at ~260 MB.
+ *
+ * Four rather than one purely to bound blast radius: a supervisor crash strands
+ * only the jobs it held. Those are recoverable anyway — the job directory is
+ * the source of truth and the heartbeat check already marks stranded jobs
+ * orphaned — but losing a quarter of in-flight work beats losing all of it.
+ */
+export declare const SUPERVISOR_POOL_SIZE = 4;
+/**
+ * Take exclusive ownership of a job directory.
+ *
+ * `wx` fails if the file exists, atomically, on both Windows and POSIX — which
+ * is what stops two supervisors racing onto the same job. A claim left behind
+ * by a crashed supervisor is reclaimed once that job's heartbeat has gone
+ * stale, by the same ORPHAN_THRESHOLD_MS rule used everywhere else.
+ *
+ * Exported for tests: the one-winner property under concurrent reclaim is the
+ * invariant, and it is only checkable by calling this directly.
+ */
+export declare function claimJobDir(jobDir: string, status: JobStatus): Promise<boolean>;
+/**
+ * Supervisor main loop: claim work, run several jobs at once, exit when idle.
+ *
+ * Exiting on idle keeps the no-jobs steady state at zero processes, same as
+ * before — the pool is a way to share supervision cost while work exists, not
+ * a daemon.
+ */
+export declare function runSupervisor(deps: JobDeps, supervisorId?: string): Promise<void>;
+/**
+ * Start slot-queued jobs, oldest first, until the machine is at its limit.
+ *
+ * Deliberately has no daemon behind it: this runs on every new dispatch and
+ * again as each runner exits, which between them covers every moment a slot
+ * can free. The cost of that choice is that if every runner dies while jobs
+ * are queued, the queue resumes on the next dispatch rather than immediately.
+ * Bounded waiting was the explicit alternative and was not chosen — a queued
+ * job keeps its jobId and its artifacts either way, so nothing is lost.
+ *
+ * NOT called at server start, which was tried and reverted: it silently ran
+ * jobs abandoned by a dead session. `orphanStrandedSlotQueue` runs there
+ * instead and reports them. See its comment for why reporting beats resuming.
+ */
+/**
+ * Mark jobs stranded in the slot queue by a server that is gone.
+ *
+ * Called once at server start, where the reasoning holds unconditionally: this
+ * process has not queued anything yet, so anything still slot-queued was
+ * queued by a session that no longer exists and nothing will ever drain it —
+ * a new dispatch would, but the caller is asking about THIS job, and until
+ * they happen to send unrelated work it reads `queued` forever.
+ *
+ * Deliberately reports rather than runs. Resuming was tried and is worse: a
+ * job queued days ago would execute at the next server start, in its original
+ * workingDir, at up to `full_auto`, with nobody watching. The job keeps its id
+ * and artifacts, so `retry_job` re-runs it as a decision rather than a side
+ * effect of opening an editor.
+ *
+ * The one status this writes back. Orphan detection elsewhere is
+ * compute-on-read and never persists its verdict, because the owner might
+ * still be alive; here the owner is definitionally gone.
+ */
+export declare function orphanStrandedSlotQueue(): Promise<number>;
+export declare function drainSlotQueue(config: RouterConfig | undefined, configPath: string | undefined): Promise<void>;
+/**
+ * Supervisors currently alive, counted from their heartbeat files.
+ *
+ * Approximate on purpose: over-counting briefly means the pool runs one short
+ * until the next drain, and under-counting means one extra supervisor that
+ * finds no work and exits within SUPERVISOR_IDLE_EXIT_MS. Neither warrants a
+ * lock, and both self-correct.
+ */
+declare function countLiveSupervisors(): Promise<number>;
+/**
+ * Exported for the cleanup test, which must exercise the REAL sweep rather
+ * than a copy of its logic — the bug being pinned is that a stale heartbeat
+ * was never removed, and a reimplementation in the test would pin nothing.
+ */
+export declare const countLiveSupervisorsForTest: typeof countLiveSupervisors;
+/**
+ * Why a detached runner would fail to bootstrap from this config path, if it
+ * would. `undefined` means the file loads (or there is none, which is the
+ * auto-detect case and always fine).
+ *
+ * Deliberately re-reads rather than trusting the server's in-memory config:
+ * the two disagreeing is exactly the condition being detected.
+ */
+export declare function configLoadError(configPath: string | undefined): Promise<string | undefined>;
+export {};
+//# sourceMappingURL=supervisor.d.ts.map

package/dist/jobs/supervisor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"supervisor.d.ts","sourceRoot":"","sources":["../../src/jobs/supervisor.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAWH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAWhD,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AA6BrD;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAMjF;AAoBD;;;;;;GAMG;AACH;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,CAQ7F;AAeD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,SAAS,CAAA;CAAE,CAAC,EACtC,MAAM,EAAE,YAAY,GAAG,SAAS,GAC/B,MAAM,CAUR;AAOD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAoBtC;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAuCrF;AAwBD;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgGvF;AAED;;;;;;;;;;;;;GAaG;AACH;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,MAAM,CAAC,CAqC/D;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,GAAG,SAAS,EAChC,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAkGD;;;;;;;GAOG;AACH,iBAAe,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CA2CrD;AAED;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,6BAAuB,CAAC;AA0ChE;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAcjG"}