@kici-dev/orchestrator 0.7.0 → 0.9.0

package/dist/stale-detector/all-terminal-runs.d.ts

@@ -0,0 +1,31 @@
+import { type Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+/**
+ * Runs whose every job row is terminal while the run row is not.
+ *
+ * Every coordinator finalizes a run from its own in-memory view when the
+ * last job it hears about finishes, and defers while a sibling still has a
+ * registration window open on the run. So one shape reaches no finalizer at
+ * all: the coordinator that saw the last job deferred to a live window holder,
+ * and that holder then died before releasing it — its stale id reads as no
+ * window, but nothing re-drives the check on the coordinator that deferred.
+ * The same shape appears when a coordinator dies between its jobs going
+ * terminal and its own finalization.
+ *
+ * This sweep is the backstop: every job row terminal, the run row still
+ * `pending` or `running`, the last completion older than the stale threshold
+ * (so a normal completion in flight is not raced), and no LIVE registration
+ * window holder. The caller finishes each candidate through the tracker's
+ * DB-fallback completion, which is clobber-guarded and computes the status
+ * from the rows — so a run a coordinator finishes in the meantime is left as
+ * that coordinator wrote it.
+ */
+export declare function selectAllTerminalRunCandidates(db: Kysely<Database>, opts: {
+    /** Runs whose last job completed before this instant qualify. */
+    threshold: Date;
+    /** Runs started before this instant are out of scope (ancient history). */
+    timeBound: Date;
+    /** How stale a `cluster_instances` heartbeat may be and still read as live. */
+    livenessGraceMs: number;
+}): Promise<string[]>;
+//# sourceMappingURL=all-terminal-runs.d.ts.map

package/dist/stale-detector/stale-run-detector.d.ts

@@ -12,6 +12,9 @@
  * - Optimistic concurrency: WHERE status='running' prevents race with agent completion
  * - Does NOT call executionTracker.onJobStatus() to avoid redundant DB writes
  * - Updates in-memory state and calls completeRunIfAllJobsTerminal for run completion
+ * - Finishes a run whose every job is terminal but whose finalizer never ran
+ *   (a coordinator that died mid-way, or a registration-window holder that
+ *   died after a sibling deferred to it)
  * - Force-terminates stale agents via scaler manager
  * - Updates GitHub check runs with timed_out conclusion
  */
@@ -49,6 +52,12 @@ export interface StaleRunDetectorDeps {
     staleThresholdMs: number;
     /** How often to scan for stale jobs in ms. Default: 60_000 */
     scanIntervalMs: number;
+    /**
+     * How stale a `cluster_instances` heartbeat may be and still read as live,
+     * for the all-terminal sweep's registration-window holder check. Defaults to
+     * {@link instanceLivenessGraceMs} at the default recovery grace period.
+     */
+    instanceLivenessGraceMs?: number;
     /** Held run store for expiring overdue held runs. Optional -- if not set, held run cleanup is skipped. */
     heldRunStore?: HeldRunStore;
     /** Step-approval bridge — notified when a step-scoped hold expires so the waiting agent fails the step. Optional. */
@@ -100,6 +109,7 @@ export declare class StaleRunDetector {
     private readonly rerouteFlapGraceFallbackMs;
     private readonly staleThresholdMs;
     private readonly scanIntervalMs;
+    private readonly instanceLivenessGraceMs;
     private readonly heldRunStore?;
     private readonly stepApprovalBridge?;
     private readonly failRun?;
@@ -188,6 +198,16 @@ export declare class StaleRunDetector {
      * so recovering jobs are naturally excluded.
      */
     scan(): Promise<void>;
+    /**
+     * Sub-scan F: runs left non-terminal after every job row went terminal.
+     *
+     * The coordinator that saw the last job finish defers finalization while a
+     * live sibling still has a registration window open on the run; a holder
+     * that then dies leaves nothing to re-drive that check. A coordinator that
+     * dies between its jobs going terminal and its own finalization leaves the
+     * same shape. Both are finished here from the rows.
+     */
+    private scanAllTerminalRuns;
     /**
      * Sub-scan A: Find running jobs with stale heartbeats.
      */