@kici-dev/orchestrator 0.1.27 → 0.2.0

package/dist/queue/job-queue.d.ts

@@ -1,6 +1,7 @@
 import { type Kysely } from 'kysely';
 import { type LabelMatcher, type ResourceRequest } from '@kici-dev/engine';
 import type { Database } from '../db/types.js';
+import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 /** Info about an expired dispatch_queue entry, returned by markExpired(). */
 export interface ExpiredJobInfo {
     /** dispatch_queue row ID */
@@ -9,6 +10,16 @@ export interface ExpiredJobInfo {
     jobName: string;
     /** Most recent scaler spawn-failure detail, if any was recorded. */
     lastProvisioningError: string | null;
+    /**
+     * The job's routing selectors, carried so the expiry sweep can ask whether
+     * any agent could ever have run it. A job that expires with NO matching agent
+     * is `unroutable` (a fleet/label problem); one whose agent existed but never
+     * freed up is `timed_out_stale` (a capacity problem).
+     */
+    runsOnLabels: string[];
+    runsOnPatterns: LabelMatcher[];
+    excludeLabels: string[];
+    excludePatterns: LabelMatcher[];
 }
 /**
  * Point-in-time breakdown of dispatch_queue depth used for Prometheus gauges
@@ -167,8 +178,12 @@ export interface QueuedJob {
  */
 export declare class JobQueue {
     private readonly db;
-    private readonly maxDepth;
+    /** Cluster-wide fallback for queue_max_depth when cluster_settings is null. */
+    private readonly defaultMaxDepth;
     private readonly defaultTimeoutMs;
+    private readonly clusterSettings?;
+    /** Per-job (per-org) queue-timeout resolver; falls back to defaultTimeoutMs. */
+    private readonly getQueueTimeoutMs?;
     /** 1-second TTL cache for pending depth count to avoid extra SELECT COUNT per enqueue. */
     private depthCache;
     /**
@@ -181,9 +196,16 @@ export declare class JobQueue {
     constructor(db: Kysely<Database>, options: {
         maxDepth: number;
         defaultTimeoutMs: number;
+        clusterSettings?: ClusterSettingsReader;
+        getQueueTimeoutMs?: (job: {
+            jobConfig?: Record<string, unknown>;
+        }) => Promise<number>;
     });
     /**
-     * Enqueue a job. Checks depth first, rejects with 'queue full' if >= maxDepth.
+     * Enqueue a job. Checks depth first, rejects with 'queue full' if >= the
+     * fleet-wide `queue_max_depth` (cluster_settings, falling back to the config
+     * default). The per-job timeout resolves through the per-org
+     * `queue_timeout_ms` override.
      * @returns The generated job ID.
      */
     enqueue(job: QueuedJobInput): Promise<string>;
@@ -269,17 +291,38 @@ export declare class JobQueue {
     /**
      * Insert a job directly with status='dispatched' (bypasses the queue).
      * Used when an agent is immediately available and the job doesn't need to wait.
-     * @returns The generated job ID.
+     *
+     * `agentId` is the durable owner, written here for the same reason
+     * {@link markDispatched} writes it on the queue-drain path: this row is
+     * dispatched the moment it is inserted, so it never passes through
+     * `markDispatched` and would otherwise carry a NULL owner for its whole life.
+     * A coordinator that never saw the dispatch resolves ownership from this
+     * column alone, so omitting it here would make {@link hasAgentOwnedJob} answer
+     * "not owned" for every directly-dispatched job after a failover.
+     *
+     * Idempotent on the primary key: a reroute re-dispatch reuses a preassigned
+     * jobId, so a concurrent reroute from a sibling coordinator (or a duplicate
+     * delivery) may target a row this instance already wrote. ON CONFLICT (id) DO
+     * NOTHING makes that a no-op instead of a dispatch_queue_pkey error; the
+     * returned `inserted` flag tells the caller whether a fresh row was created so
+     * it can avoid double-dispatching an already-present job. The conflicting row
+     * keeps the owner the winning writer recorded.
+     *
+     * @returns The job ID and whether a new row was inserted (false = row already existed).
      */
-    insertDispatched(job: QueuedJobInput): Promise<string>;
+    insertDispatched(job: QueuedJobInput, agentId: string): Promise<{
+        id: string;
+        inserted: boolean;
+    }>;
     /**
-     * Mark a job as dispatched.
+     * Mark a job as dispatched and record the agent it went to.
      *
-     * Note: agentId is not persisted in the dispatch_queue table.
-     * The dispatcher tracks agent-to-job mappings in memory (agentJobs Map).
-     * The dispatch_queue is a transient routing table, not the execution record.
+     * `agent_id` is the durable owner: the dispatcher also tracks agent-to-job
+     * mappings in memory (agentJobs Map), but that map is per-coordinator, so a
+     * coordinator that never saw the dispatch has to read the owner back from the
+     * row to answer an ownership question.
      */
-    markDispatched(jobId: string, _agentId: string): Promise<void>;
+    markDispatched(jobId: string, agentId: string): Promise<void>;
     /**
      * Mark a job as failed.
      *
@@ -329,6 +372,23 @@ export declare class JobQueue {
      * @returns Number of affected rows.
      */
     cancelByRunId(runId: string): Promise<number>;
+    /**
+     * Delete terminal dispatch_queue rows older than `retentionDays`.
+     *
+     * `dispatch_queue` is operational dispatch state; the durable run history
+     * lives in the cold-stored execution_runs/jobs/steps tables, so a terminal
+     * ({@link DispatchQueueStatus.Completed}/{@link DispatchQueueStatus.Failed}/
+     * {@link DispatchQueueStatus.Expired}) row has no archival value once it ages
+     * out. Non-terminal rows ({@link DispatchQueueStatus.Pending}/
+     * {@link DispatchQueueStatus.Dispatched}/{@link DispatchQueueStatus.Recovering})
+     * are never pruned — a still-active run keeps every one of its rows. Both the
+     * terminal-status filter and the age cutoff must hold for a row to be deleted.
+     *
+     * `retentionDays <= 0` disables pruning (returns 0 without a query).
+     *
+     * @returns Number of rows deleted.
+     */
+    pruneTerminalDispatchRows(retentionDays: number): Promise<number>;
     /**
      * Get the current number of pending jobs.
      * Uses a 1-second TTL cache to avoid extra SELECT COUNT per enqueue.
@@ -455,11 +515,12 @@ export declare class JobQueue {
      */
     getFullJobById(jobId: string): Promise<QueuedJob | null>;
     /**
-     * Mark a job as dispatched only if it is still in 'recovering' state.
+     * Mark a job as dispatched only if it is still in 'recovering' state, and
+     * record `agentId` as the durable owner of the reclaimed job.
      * Used when an agent reconnects and claims a recovering job.
      * @returns true if the update affected a row (job was still recovering).
      */
-    markDispatchedIfRecovering(jobId: string): Promise<boolean>;
+    markDispatchedIfRecovering(jobId: string, agentId: string): Promise<boolean>;
     /**
      * Get a single job by ID.
      * Used to look up runId during recovery timer setup.
@@ -471,10 +532,11 @@ export declare class JobQueue {
     } | null>;
     /**
      * HA-safe ownership check. Returns true if the DB shows that
-     * `agentId` previously held `jobId` according to any of:
+     * `agentId` holds or previously held `jobId` according to any of:
      *
-     *   - `status='dispatched'` AND the registry-managed bookkeeping
-     *     records the agent assignment (caller-side `agentJobs` map),
+     *   - `status='dispatched'` AND `agent_id = <agent>` — the live
+     *     owner of an in-flight job, readable by any coordinator
+     *     including one that never saw the dispatch,
      *   - `status='recovering'` AND `recovery_agent_id = <agent>` (so a
      *     replacement coord still recognises in-flight chunks), OR
      *   - the row is already terminal (`completed` / `failed` /
@@ -495,11 +557,29 @@ export declare class JobQueue {
         id: string;
         runId: string;
         status: DispatchQueueStatus;
+        agentId: string | null;
     }>>;
+    /**
+     * Shared builder for the non-expired pending rows, oldest-first
+     * (`created_at ASC` — the same FIFO ordering as `dequeueForLabels` /
+     * `markExpired`). Read-only: no `FOR UPDATE`, no claim. Used by both the
+     * unbounded `getPendingJobs` drain and the capped `listPending` re-drive.
+     */
+    private pendingOldestFirstQuery;
     /**
      * Get all pending jobs in FIFO order (for queue drain on agent connect).
      */
     getPendingJobs(): Promise<QueuedJob[]>;
+    /**
+     * Read-only oldest-first listing of pending jobs, capped at `limit`.
+     *
+     * Powers the scaler capacity-freed re-drive (`Dispatcher.retryPendingScaleRequests`):
+     * when a scaler agent frees capacity, the oldest jobs that previously got an
+     * `at-capacity` verdict are the ones re-offered to `requestScale`. Unlike
+     * `dequeueForLabels`, this neither claims nor locks rows — it is a pure read;
+     * the re-drive re-runs the normal scale path, which reserves capacity itself.
+     */
+    listPending(limit: number): Promise<QueuedJob[]>;
     /**
      * Convert a DB row to a QueuedJob object.
      * Handles both auto-parsed JSONB arrays (from pg driver) and JSON strings (from tests).

package/dist/queue/scheduled-job.d.ts

@@ -25,10 +25,10 @@ import type { Database } from '../db/types.js';
  */
 export declare const OrchestratorScheduledJobName: z.ZodEnum<{
     cleanup: "cleanup";
-    "orphan-secret-cleanup": "orphan-secret-cleanup";
-    "token-cleanup": "token-cleanup";
     "cold-store-archive": "cold-store-archive";
     "cold-store-purge": "cold-store-purge";
+    "orphan-secret-cleanup": "orphan-secret-cleanup";
+    "token-cleanup": "token-cleanup";
 }>;
 export type OrchestratorScheduledJobName = z.infer<typeof OrchestratorScheduledJobName>;
 /** Access-log action for a manually triggered off-cadence tick. */

package/dist/registration/registration-index.d.ts

@@ -151,7 +151,7 @@ export declare class RegistrationIndex {
      *
      * Event type -> trigger type mapping:
      * - 'kici_event' -> 'kici_event'
-     * - 'workflow_complete' -> 'workflow_complete' or 'lifecycle'
+     * - 'workflow_complete' -> 'workflow_complete', 'lifecycle', or 'workflows_failed_batch'
      * - 'job_complete' -> 'job_complete' or 'lifecycle'
      * - 'generic_webhook' -> 'generic_webhook'
      * - 'schedule' -> 'schedule'

package/dist/reporting/agent-failure-category.d.ts

@@ -13,7 +13,10 @@ export interface FailureSignals {
     hasInitFailure: boolean;
     /** The init-failure category (`InitFailureCategory`), if any. */
     initFailureCategory: string | null;
-    /** Any job ended in `timed_out_stale` (or a workflow/job timeout fired). */
+    /**
+     * Any job ended in an infra-class verdict that means it never really ran —
+     * `timed_out_stale` or `unroutable` — or a workflow/job timeout fired.
+     */
     timedOut: boolean;
     /** Any step recorded a non-zero exit code. */
     anyStepNonZeroExit: boolean;

package/dist/reporting/check-run-reporter.d.ts

@@ -30,7 +30,7 @@ import type { ProviderRegistry } from '../provider-registry.js';
 import type { StepLogBuffer } from './step-log-buffer.js';
 import { type SourceLocationData } from './check-run-summary.js';
 import type { CheckRunTrackingStore } from './check-run-tracking-store.js';
-import { ExecutionJobStatus } from '@kici-dev/engine';
+import { type TerminalJobStatus } from '@kici-dev/engine';
 /**
  * Dependencies for the CheckRunReporter.
  */
@@ -122,7 +122,7 @@ interface SetBuildCompleteOptions {
     repo: string;
     sha: string;
     workflowName: string;
-    status: Extract<ExecutionJobStatus, 'success' | 'failed' | 'cancelled' | 'timed_out_stale'>;
+    status: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -142,7 +142,7 @@ interface UpdateJobStatusOptions {
     sha: string;
     workflowName: string;
     jobName: string;
-    state: Extract<ExecutionJobStatus, 'success' | 'failed' | 'cancelled' | 'timed_out_stale'>;
+    state: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -167,7 +167,7 @@ interface UpdateWorkflowStatusOptions {
     repo: string;
     sha: string;
     workflowName: string;
-    overallStatus: Extract<ExecutionJobStatus, 'success' | 'failed' | 'cancelled' | 'timed_out_stale'>;
+    overallStatus: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -238,6 +238,19 @@ export declare class CheckRunReporter {
     private readonly progressTimers;
     /** L1 cache: first in-progress sent flag (synced to `check_run_tracking.in_progress_sent_at`). */
     private readonly inProgressSent;
+    /**
+     * Keys whose job check run has already been completed.
+     *
+     * Makes the check run's status monotonic: once a `completed` update is
+     * issued, no later step-progress update may push it back to `in_progress`.
+     * Cancelling the pending debounce timer at completion time is not sufficient
+     * on its own — a step status that arrives after the completion schedules a
+     * FRESH timer, which then fires and leaves the check run showing
+     * `status: in_progress` with a terminal `conclusion` already attached. That
+     * is the permanently-unresolved state check-run completion exists to prevent,
+     * so the guard is on the write itself rather than on the timer.
+     */
+    private readonly terminalSent;
     /**
      * L1 cache: runId → set of check-run composite keys. Synced to the
      * indexed `check_run_tracking.run_id` column so a replacement coord can
@@ -334,21 +347,28 @@ export declare class CheckRunReporter {
      */
     setBuildComplete(opts: SetBuildCompleteOptions): void;
     /**
-     * Clean up step-progress entries, debounce timers, and DB rows for a
-     * completed run. Called when the execution tracker prunes the run.
+     * Clean up step-progress entries and debounce timers for a completed run.
+     * Called when the execution tracker prunes the run.
      *
-     * In-memory cleanup is synchronous; the DB cleanup is fire-and-forget
-     * because the caller (run-pruning hook) is on the response-shaping path
-     * and shouldn't block on a network round-trip. Failure logs but does
-     * not propagate.
+     * In-memory only. Database rows are owned by the retention sweep in
+     * `queue/cleanup.ts`, which deletes on inactivity age rather than on run
+     * completion. Deleting here would strand a late terminal update: a check-run
+     * status PATCH that arrives after the prune resolves its check-run ID by
+     * loading through to this row, and a deleted row makes that lookup fail.
      */
     cleanupRun(runId: string): void;
     /**
-     * Hydrate the L1 caches from the DB after a leader switch (or any
-     * boot-time recovery). Called once on coord become-leader so the
-     * runIdToKeys reverse map is populated for any future cleanupRun calls
-     * without requiring a DB round-trip per cleanup. If no store is wired,
+     * Mark the reporter as DB-backed after a leader switch (or any boot-time
+     * recovery). Called once on coord become-leader. If no store is wired,
      * this is a no-op.
+     *
+     * Nothing is hydrated up front — the table can be large across many shas —
+     * so check-run IDs load through on demand inside `resolveCheckRunId`. The
+     * `runIdToKeys` reverse map is rebuilt only from this coord's own writes,
+     * so a run whose keys all predate the switch leaves `cleanupRun` nothing to
+     * evict. That is harmless: the L1 caches it clears are equally empty on a
+     * fresh coord, and the DB rows belong to the retention sweep rather than to
+     * run prune.
      */
     recoverState(): Promise<void>;
     /** Track a check run key associated with a runId for later cleanup. */

package/dist/reporting/check-run-tracking-store.d.ts

@@ -29,7 +29,16 @@ export interface CheckRunTrackingState {
     stepProgress: StepProgressEntry[];
     /** Timestamp the first running-step transition was sent to GitHub. */
     inProgressSentAt?: Date;
-    /** KiCI run this check-run belongs to. Used by `cleanupRun`. */
+    /**
+     * Timestamp the terminal (`completed`) update was accepted by the provider.
+     * Undefined means we have no record of sending it — not proof it failed.
+     */
+    terminalSentAt?: Date;
+    /**
+     * KiCI run this check-run belongs to. Written at create time so the row
+     * records its owning run; it is an attribution/lookup key, not a retention
+     * key — the sweep prunes on `updated_at` age alone.
+     */
     runId?: string;
     /** Last persisted update time; powers debounce-after-failover recovery. */
     updatedAt?: Date;
@@ -64,8 +73,13 @@ export declare class CheckRunTrackingStore {
      * Performed as an upsert so a re-issued setPending after a coord
      * failover replaces the prior ID rather than silently leaving a row
      * mismatched with the GitHub-side state.
+     *
+     * `runId` is written alongside so the row records which run owns it from
+     * the moment it is created. A row created without it carries a NULL
+     * `run_id`, which makes `listKeysByRunId` under-report and leaves the
+     * column useless to an operator asking which run posted a given check.
      */
-    setCheckRunId(key: CheckRunTrackingKey, checkRunId: number): Promise<void>;
+    setCheckRunId(key: CheckRunTrackingKey, checkRunId: number, runId?: string): Promise<void>;
     /**
      * Lookup the check-run ID for a key. Returns undefined if no row exists
      * yet OR the row exists but the GitHub create has not finished
@@ -93,6 +107,14 @@ export declare class CheckRunTrackingStore {
      * cluster-wide.
      */
     markInProgressSent(key: CheckRunTrackingKey, runId?: string): Promise<void>;
+    /**
+     * Stamp that the terminal (`completed`) provider update was sent.
+     *
+     * Mirrors `markInProgressSent`. Best-effort like every write on this table —
+     * the caller swallows failures, because a tracking write must never break
+     * check-run reporting — so a null column is "no record", not "never sent".
+     */
+    markTerminalSent(key: CheckRunTrackingKey, runId?: string): Promise<void>;
     /**
      * Get the full state snapshot for a key. Used by the L1 cache to
      * hydrate on miss and by tests to verify the on-disk layout. Returns
@@ -104,17 +126,56 @@ export declare class CheckRunTrackingStore {
      */
     deleteRow(key: CheckRunTrackingKey): Promise<boolean>;
     /**
-     * List every key currently tracked for a runId. Used by `cleanupRun`
-     * to reproduce the runId → keys reverse index that the in-memory map
-     * provided. Index `idx_check_run_tracking_run_id` keeps this O(matches).
+     * List every key currently tracked for a runId — the runId → keys reverse
+     * index that `CheckRunReporter` otherwise holds only in memory, so a
+     * replacement coord that never saw the run can still recover it. Index
+     * `idx_check_run_tracking_run_id` keeps this O(matches).
+     *
+     * No caller in tree: run-prune stopped consulting the store once row
+     * lifetime moved to {@link pruneStale}. Kept as the read half of the
+     * `run_id` column, beside {@link deleteByRunId}.
      */
     listKeysByRunId(runId: string): Promise<CheckRunTrackingKey[]>;
     /**
-     * Delete every row for a runId. Mirrors the bulk-cleanup semantics of
-     * `cleanupRun` so a single call from execution-tracker prune releases
-     * all rows for the run.
+     * Delete every row for a runId.
+     *
+     * Not on the run-prune path: `cleanupRun` evicts only its in-memory state,
+     * because a row deleted at prune time strands a late terminal PATCH that
+     * still has to resolve its check-run ID. Routine reclamation is
+     * {@link pruneStale}. This remains as the targeted escape hatch for
+     * discarding one run's rows deliberately.
      */
     deleteByRunId(runId: string): Promise<number>;
+    /**
+     * Delete every row untouched for longer than `retentionDays`, and return
+     * how many were removed. `retentionDays <= 0` disables the sweep, matching
+     * the `JobQueue.pruneTerminalDispatchRows` convention.
+     *
+     * `updated_at` is the staleness signal because the table carries no status
+     * column, and the rows that most need reaping are precisely the ones with no
+     * `run_id` to join on — so the predicate is age alone, deliberately
+     * unfiltered by `run_id`. Every upsert bumps `updated_at`, so a run that is
+     * still emitting check-run traffic keeps its rows fresh; a workflow-level
+     * row, written once at create, ages from creation.
+     *
+     * Inactivity age rather than run completion is what keeps a reader safe: a
+     * row a diagnostic is polling for cannot disappear underneath it, because
+     * the window is days and the row is only removed once nothing has touched
+     * it for that long.
+     *
+     * The bound this trades for that safety: age is measured on check-run
+     * writes, not on run liveness, so a run that stays live while emitting no
+     * check-run traffic for longer than the window — a run parked on a manual
+     * approval gate, whose hold statuses go through `CheckStatusPoster` and
+     * never touch this table — has its rows swept while it is still running.
+     * The reporter's L1 cache still holds the ID, so its terminal update lands
+     * anyway on a coord that stayed up; it takes a restart or failover across
+     * that window to leave the update with no row to load through and no
+     * check-run ID to resolve.
+     *
+     * Called from the hourly cleanup tick.
+     */
+    pruneStale(retentionDays: number): Promise<number>;
     private selectRow;
     private upsertRow;
 }
@@ -127,6 +188,7 @@ export declare function rowToState(row: {
     build_creation_state: string | null;
     step_progress_json: unknown;
     in_progress_sent_at: Date | null;
+    terminal_sent_at: Date | null;
     run_id: string | null;
     updated_at: Date;
 }): CheckRunTrackingState;