@kici-dev/orchestrator 0.9.1 → 0.11.0

package/dist/__test-helpers__/test-db.d.ts

@@ -2,9 +2,16 @@
  * Teardown helper for tests that create their own PostgreSQL database.
  */
 import type pg from 'pg';
+/** Backends still hold the database after they were terminated and waited on. */
+export declare class TestDbBackendsRemainError extends Error {
+    readonly dbName: string;
+    readonly remaining: number;
+    constructor(dbName: string, remaining: number);
+}
 /**
  * Terminate whatever backends still hold `dbName`, after first waiting for
- * the connections the test itself closed to finish closing.
+ * the connections the test itself closed to finish closing, then wait for the
+ * terminated backends to exit.
  *
  * `pool.end()` — and Kysely's `destroy()`, which calls it — resolves as soon
  * as every client has SENT its Terminate message, not when its socket has
@@ -14,8 +21,14 @@ import type pg from 'pg';
  * test file after every test in it passed. So this polls `pg_stat_activity`
  * until the database has no backend left (bounded by `timeoutMs`), and
  * terminates only what is still there — a connection the test never closed.
+ *
+ * A terminated backend stays in `pg_stat_activity` until it has exited, and
+ * `DROP DATABASE` fails while it is listed. So after the terminate this polls
+ * again until the count is zero, bounded by `exitTimeoutMs`, and throws
+ * {@link TestDbBackendsRemainError} if the bound passes.
  */
 export declare function terminateTestDbBackends(admin: pg.Pool, dbName: string, opts?: {
     timeoutMs?: number;
+    exitTimeoutMs?: number;
 }): Promise<void>;
 //# sourceMappingURL=test-db.d.ts.map

package/dist/agent/dispatcher.d.ts

@@ -2,6 +2,7 @@ import type { AgentRegistry } from './registry.js';
 import { type JobQueue, type QueuedJob, type QueuedJobInput } from '../queue/job-queue.js';
 import type { ScaleResult, ScalerRedispatchTrigger } from '../scaler/types.js';
 import type { ResolvedContainerSpawn } from '../scaler/types.js';
+import { type ScalerAgentView } from '../scaler/agent-fit.js';
 import type { ResourceRequest } from '@kici-dev/engine';
 /**
  * Why a freed agent's queue drain claimed nothing.
@@ -63,6 +64,10 @@ export interface HostRosterRebootStore {
  * released and onDispatch is NOT called). The reroute caller should treat the
  * job as already handled. Only reachable on a reroute re-dispatch — first-
  * dispatch paths use a fresh UUID that can never collide.
+ *
+ * `queued` also answers a direct dispatch refused because the job's run has
+ * stopped: the queue row is recorded `expired` and the job cancelled, the
+ * state a cancel leaves a queued job of its run in.
  */
 type DispatchResult = {
     status: 'dispatched';
@@ -89,6 +94,16 @@ export declare function containerSpawnFor(jobConfig: Record<string, unknown> | u
         serveraddress: string;
     } | undefined;
 } | undefined;
+/**
+ * What an `onDispatch` callback returns when it refuses to put a job on the
+ * wire. The dispatcher fails the job permanently with `refused` as the reason
+ * instead of arming an ack deadline for a message that was never sent.
+ */
+export interface DispatchRefusal {
+    refused: string;
+}
+/** The callback that sends a dispatched job to its agent. */
+export type OnDispatch = (agentId: string, job: QueuedJob) => void | DispatchRefusal | Promise<void | DispatchRefusal>;
 export declare class Dispatcher {
     private readonly registry;
     private readonly queue;
@@ -96,9 +111,7 @@ export declare class Dispatcher {
     private readonly onDispatch;
     private readonly onNoMatchingAgent?;
     /** See the constructor dep of the same name. */
-    private readonly canPrespawnedAgentServe?;
-    /** See the constructor dep of the same name. */
-    private readonly isPrespawnedAgent?;
+    private readonly scalerAgentView?;
     /**
      * Single-flight guard for `retryPendingScaleRequests`. The capacity-freed
      * hook and the leader-gated sweep can both fire concurrently; this prevents
@@ -189,11 +202,13 @@ export declare class Dispatcher {
     /** Coordinator drain predicate; when true, new jobs queue Pending instead of
      *  dispatching. Defaults to never-draining. */
     private readonly isDraining;
+    /** Whether another coordinator is live. Defaults to none (a single node). */
+    private readonly hasPeerCoordinators;
     constructor(deps: {
         registry: AgentRegistry;
         queue: JobQueue;
         metrics: DispatchMetrics;
-        onDispatch: (agentId: string, job: QueuedJob) => void | Promise<void>;
+        onDispatch: OnDispatch;
         /** Optional hook called when no agent matches the job's labels.
          *  When set, receives the per-job `resources` so the scaler can apply
          *  per-scaler / per-orchestrator / per-machine caps before spawning. */
@@ -205,26 +220,14 @@ export declare class Dispatcher {
          */
         containerSpawn?: ResolvedContainerSpawn) => Promise<ScaleResult>;
         /**
-         * Optional scaler predicate: may this pre-spawned (warm) agent serve this
-         * job? A warm agent is generic — started before the job existed, at the
-         * pool's shape and running the pool's image, both of which are fixed when
-         * it starts. When the predicate returns false the agent is skipped and the
-         * job falls through to `onNoMatchingAgent`, which spawns one that fits.
-         * Absent (no scaler) means every agent is eligible.
-         */
-        canPrespawnedAgentServe?: (agentId: string, job: {
-            resources?: ResourceRequest;
-            hasOwnContainerImage: boolean;
-        }) => boolean;
-        /**
-         * Whether this scaler pre-spawned the agent, i.e. whether
-         * `canPrespawnedAgentServe` can ever answer false for it. The queue drain
-         * (agent asks for work, rather than job looks for an agent) uses it to
-         * decide whether the suitability predicate is worth carrying into the
-         * claim: for every ordinary agent it is not, and the drain keeps its
-         * single-statement fast path.
+         * What the scaler knows about an agent it started
+         * (`ScalerManager.agentView`), or undefined for one it did not. It feeds
+         * `canAgentRunJob`: the job the agent was started for, whether it runs in
+         * that job's image, and a warm agent's fixed shape. Absent when no scaler
+         * is configured; the predicate then judges every agent on its own
+         * registration.
          */
-        isPrespawnedAgent?: (agentId: string) => boolean;
+        scalerAgentView?: (agentId: string) => ScalerAgentView | undefined;
         /** Max reconnection delay from agent config (default 60s). Used to derive grace period. */
         maxReconnectDelayMs?: number;
         /** Callback fired when a job is permanently failed before/outside agent execution. */
@@ -247,6 +250,13 @@ export declare class Dispatcher {
          *  Pending and onAgentAvailable() claims nothing — jobs already on agents
          *  finish. Defaults to never-draining. */
         isDraining?: () => boolean;
+        /**
+         * Whether another coordinator is live: connected, or disconnected for less
+         * than the peer flap grace. A job whose sealed secrets this coordinator
+         * cannot open is put back for such a peer; with none, no coordinator can
+         * open it, so it fails at once. Defaults to none.
+         */
+        hasPeerCoordinators?: () => boolean | Promise<boolean>;
     });
     /**
      * Dispatch a job to a matching agent, or queue it if none available.
@@ -273,9 +283,20 @@ export declare class Dispatcher {
      * overlapping hook+sweep cannot double-offer the same jobs. Bounded by
      * `maxJobs` so one free event cannot storm the scaler.
      *
+     * A job whose sealed secrets this coordinator cannot open is never scaled
+     * for: it is settled instead ({@link settleUnopenablePending}).
+     *
      * @returns the number of jobs that entered `spawning`.
      */
     retryPendingScaleRequests(maxJobs?: number, trigger?: ScalerRedispatchTrigger): Promise<number>;
+    /**
+     * Release the slot claimed for a direct dispatch the queue refused because
+     * the job's run has stopped. Nothing is sent to the agent. The queue already
+     * recorded the row `expired` and cancelled the job, so the job reads as a
+     * queued job of the run that the cancel expired, and the caller tracks it
+     * under `queued` exactly as it would that one.
+     */
+    private refuseStoppedRunDispatch;
     /**
      * Dispatch a host-fanout pinned child. The job targets exactly
      * `job.pinnedAgentId`: if that agent is locally connected, satisfies the
@@ -295,11 +316,16 @@ export declare class Dispatcher {
      * still pending and the agent's labels still satisfy it), marks it
      * dispatched, and sends it to the agent.
      *
-     * Returns true if dispatched, false if the bound job was already claimed
-     * elsewhere, expired, or the agent isn't registered. The caller falls
-     * back to the generic onAgentAvailable() drain in either case.
+     * Returns true only when the job was sent to the agent. False when nothing
+     * was claimed (the bound job was already claimed elsewhere, expired, or the
+     * agent isn't registered, is full or the coordinator is draining), and also
+     * when the claimed job was not sent: its dispatch was refused and it failed,
+     * or this coordinator could not open its sealed secrets. The caller falls
+     * back to the generic onAgentAvailable() drain in every false case.
      */
     dispatchBoundJob(agentId: string, jobId: string): Promise<boolean>;
+    /** {@link dispatchBoundJob}, reporting whether the job was claimed apart from whether it was sent. */
+    private claimBoundJob;
     /**
      * Try draining the queue for a specific agent.
      * Called when an agent registers or completes a job.
@@ -314,19 +340,24 @@ export declare class Dispatcher {
      */
     private filterRebootPending;
     /**
-     * Drop pre-spawned (warm) agents that cannot serve this job. Sibling of
+     * Drop the agents that cannot run this job (`canAgentRunJob`). Sibling of
      * {@link filterRebootPending}: both remove candidates `findAvailable` matched
      * on labels but that are unusable for a reason labels cannot express.
      *
-     * A warm agent's cpu, memory and container image are all set when it starts
-     * and cannot change afterwards, so a job asking for something else has to get
-     * an agent of its own. Dropping the candidate here is what sends it down the
-     * ordinary `onNoMatchingAgent` scale path. No-op when no scaler is wired.
+     * The reasons: an agent started inside another job's image, a container job
+     * on an agent that reports no container runtime, a warm agent of another
+     * shape. Dropping the candidate here is what sends the job down the ordinary
+     * `onNoMatchingAgent` scale path.
+     *
+     * `jobId` is the job's queue id when it has one: it is how the agent started
+     * for the job recognizes it.
      */
-    private filterUnsuitablePrespawned;
+    private filterUnservableAgents;
+    /** The registration facts, and the scaler's record, `canAgentRunJob` judges an agent on. */
+    private fitAgentFor;
     /**
-     * The fit question for one job: the shape it declares, and whether it brings
-     * its own container image.
+     * The fit question for one job: its queue id, the shape it declares, and
+     * what its `container:` asks of the agent's host.
      *
      * The shape comes from `jobConfig.resources` when the typed `resources`
      * mirror is absent, because the mirror is optional and several dispatch paths
@@ -336,16 +367,16 @@ export declare class Dispatcher {
      * declares nothing, which admits it onto a pre-spawned agent of some other
      * size — the exact mismatch this gate exists to refuse.
      */
-    private prespawnedFitFor;
+    private agentFitFor;
     /**
-     * The queue-drain half of {@link filterUnsuitablePrespawned}: a per-job
+     * The queue-drain half of {@link filterUnservableAgents}: a per-job
      * predicate for one agent, or undefined when this agent needs no check.
      *
-     * Undefined is the common answer — an ordinary agent is not pre-spawned, so
-     * the predicate could only ever say yes, and returning one would cost the
-     * drain its single-statement fast path for nothing.
+     * Undefined is the common answer for an agent that runs any job: the
+     * predicate could only ever say yes, and returning one would cost the drain
+     * its single-statement fast path for nothing.
      */
-    private prespawnedFitFilterFor;
+    private agentFitFilterFor;
     /**
      * Release a reboot-pending host on its real reconnect (down-then-up). Clears
      * the persisted flag so the very next `onAgentAvailable` drain dispatches the
@@ -361,8 +392,69 @@ export declare class Dispatcher {
      * Returns the reason nothing was dispatched, or null when a job was.
      */
     private drainForAgent;
+    /** {@link deliver} inside the job's own request context, restored for a queue-drained job. */
+    private deliverInRequestContext;
+    /**
+     * Hand a tracked job to `onDispatch`. Returns false when the job was not sent:
+     * either the callback refused it, and the job is failed permanently since the
+     * refusal is a fact about the job and another agent would refuse it too; or
+     * this coordinator cannot open its sealed secrets, and the job is put back so
+     * a coordinator holding the key can take it, or failed when none can
+     * ({@link releaseUnopenableDispatch}).
+     */
+    private deliver;
+    /**
+     * Handle a claimed job whose sealed secrets this coordinator cannot open: it
+     * was sealed with a master key this coordinator does not hold.
+     *
+     * With another coordinator live this is not a permanent refusal: during a
+     * rolling master-key rotation that peer may already hold the new key, and if
+     * it does, its own drains can take the job. The row returns to pending,
+     * spending one dispatch attempt and recording the reason as the job's
+     * provisioning error (what an expiry reports), and this coordinator stops
+     * offering the job for the sealed-secrets back-off. Once the back-off lapses
+     * its drains and periodic re-drives offer the job again like any other, so a
+     * job no coordinator can open is claimed again and fails with the reason once
+     * {@link MAX_DISPATCH_ATTEMPTS} is reached. With no other coordinator live
+     * nobody can open it, so it fails at once.
+     *
+     * `agentId` is the agent the job was claimed for, or undefined when a re-drive
+     * claimed it for no agent ({@link settleUnopenablePending}).
+     */
+    private releaseUnopenableDispatch;
+    /**
+     * Settle a pending job whose sealed secrets this coordinator cannot open,
+     * found by a re-drive that would otherwise scale for it.
+     *
+     * Scaling for the job is wrong twice over. The registry credentials of a
+     * private image are sealed with the job's other secrets, so a spawn of that
+     * image pulls without them and fails before any agent registers: the job is
+     * never claimed, never spends an attempt, and every re-drive spawns again.
+     * And any agent a spawn does start is one this coordinator would hand the job
+     * back from. So the job is claimed by id for no agent and gets the decision a
+     * drain that claimed it would give it ({@link releaseUnopenableDispatch}). A
+     * claim lost to another coordinator, or refused inside the back-off, leaves
+     * the job alone.
+     */
+    private settleUnopenablePending;
+    /**
+     * Undo the dispatch accounting of a refused job and fail it with the refusal
+     * reason. `agentId` is undefined for a job claimed for no agent.
+     */
+    private failRefusedDispatch;
+    /** Free the slot and tracking a claim took on its agent; a claim for no agent took none. */
+    private releaseClaimAccounting;
     /** Record that a job began executing on its agent. */
     markJobStarted(jobId: string): void;
+    /**
+     * The agent accepted `jobId` (a `job.ack`, or the `running` status that stands
+     * in for a lost one). Drop a busy hold that already ran out: a job accepted
+     * while it sits on the entry was dispatched after it expired, into a free
+     * slot, so the agent's job count is not stuck and its next teardown-window
+     * rejection must not spend an attempt. A hold still in force is kept — see
+     * {@link AgentRegistry.clearExpiredBusyHold}.
+     */
+    private noteJobAccepted;
     /**
      * Arm the ack deadline for a just-sent dispatch (timer + persisted row).
      *
@@ -400,7 +492,8 @@ export declare class Dispatcher {
     /**
      * Requeue a dispatched job for re-delivery, or fail it permanently when
      * its attempt budget is exhausted. Returns the outcome so disconnect
-     * triage can surface failed job IDs to the caller.
+     * triage can surface failed job IDs to the caller. `countAttempt: false`
+     * requeues without spending the budget (a busy rejection).
      */
     private requeueOrFail;
     /**
@@ -443,6 +536,9 @@ export declare class Dispatcher {
      * It never consults the scaler (that is `retryPendingScaleRequests`) and never
      * spawns: it only places jobs onto agents already connected here.
      *
+     * Returns the number of jobs sent. A job it claims but does not send (its
+     * dispatch refused, or its sealed secrets unopenable here) is not counted.
+     *
      * A failure propagates to the caller rather than being swallowed here — the
      * per-coord interval wrapper is the single error-log site, matching the
      * sibling recovery/ack sweeps that share its cadence. The `finally` only

package/dist/agent/registry.d.ts

@@ -9,7 +9,28 @@
  * No persistence needed -- agents re-register on reconnect.
  */
 import { type CanonicalLabel, type LabelMatcher } from '@kici-dev/engine';
-import type { WsLike } from '@kici-dev/engine';
+import type { AgentCapabilities, WsLike } from '@kici-dev/engine';
+/**
+ * The longest a busy rejection holds an agent out of routing when no evidence
+ * of a free slot lifts the hold first: an `agent.status` capacity report, or a
+ * job on the agent reaching a final status.
+ */
+export declare const BUSY_HOLD_MAX_MS = 30000;
+/** What {@link AgentRegistry.markBusyHeld} found on the agent. */
+export declare enum BusyHoldOutcome {
+    /** No hold was on the agent: it last reported an empty slot, or never rejected. */
+    Held = "held",
+    /** A hold that had run out was still on the agent: it never reported an empty slot. */
+    ReheldAfterExpiry = "reheld-after-expiry",
+    /** The agent is not in the registry. */
+    NotRegistered = "not-registered"
+}
+/**
+ * Whether an agent can take one more job now: a free slot, and no busy hold
+ * in force ({@link AgentRegistry.markBusyHeld}). Draining is checked by the
+ * callers that route around it.
+ */
+export declare function agentHasFreeSlot(entry: AgentEntry, now?: number): boolean;
 export type { WsLike } from '@kici-dev/engine';
 /**
  * A single tracked agent connection.
@@ -68,6 +89,11 @@ export interface AgentEntry {
      * WebSocket close that removes the entry — there is nothing to outlive.
      */
     draining: boolean;
+    /**
+     * Epoch ms until which the agent is held out of routing after it rejected a
+     * dispatch as busy. Unset when no hold is in force. See {@link AgentRegistry.markBusyHeld}.
+     */
+    busyHeldUntil?: number;
     /** Number of jobs currently executing on this agent. */
     activeJobs: number;
     /** Maximum concurrent jobs this agent can handle (default 1). */
@@ -82,6 +108,11 @@ export interface AgentEntry {
     registeredAt: number;
     /** Self-reported agent version (e.g. "0.0.1"). Null if not reported. */
     version: string | null;
+    /**
+     * Optional behaviours the agent advertised on `agent.register`. Null for an
+     * agent that advertised none, which is read as supporting none.
+     */
+    capabilities: AgentCapabilities | null;
     /**
      * The `agent_tokens.id` row used to authenticate this connection, or
      * `null` when auth mode is `none`. Indexed by `tokenIdIndex` so a
@@ -149,6 +180,8 @@ interface AgentMetadata {
      * any operator-declared properties). Undefined ⇒ the agent reported none.
      */
     properties?: Record<string, string | number | boolean>;
+    /** Optional behaviours the agent advertised on `agent.register`. */
+    capabilities?: AgentCapabilities;
 }
 /**
  * Subset of `HostRosterStore` the registry reconciles into. Optional — when no
@@ -324,7 +357,13 @@ export declare class AgentRegistry {
      * an off-gate gated agent as "matching but busy" and skip the peer
      * reroute path even though the local agent can never accept the job.
      */
-    hasMatchingAgent(requiredLabels: string[], requiredPatterns?: LabelMatcher[], excludeLabels?: string[], excludePatterns?: LabelMatcher[]): boolean;
+    hasMatchingAgent(requiredLabels: string[], requiredPatterns?: LabelMatcher[], excludeLabels?: string[], excludePatterns?: LabelMatcher[], 
+    /**
+     * An extra test a label-matching agent must pass: whether it can run the
+     * job at all, which its labels cannot express. An agent that matches but
+     * never could is not a reason to keep the job here rather than reroute it.
+     */
+    canRun?: (entry: AgentEntry) => boolean): boolean;
     /**
      * Internal helper: intersect the per-label candidate sets to find agentIds
      * that satisfy `requiredLabels`. Returns `null` when any required label
@@ -387,6 +426,44 @@ export declare class AgentRegistry {
      * @returns the agent's active-job count, or `undefined` when it is not registered
      */
     markDraining(agentId: string): number | undefined;
+    /**
+     * Hold an agent out of routing after it rejected a dispatch as busy.
+     *
+     * A busy rejection means the agent still holds the slot of the job it is
+     * tearing down, while the registry already counts that slot free (the
+     * job reported completion first). Without the hold, the requeue re-picks the
+     * same agent at once and each rejection repeats within milliseconds. The hold
+     * lifts on evidence of a free slot ({@link clearBusyHeld}): the agent reports
+     * an empty slot on `agent.status`, or a job tracked to it reaches a final
+     * status. An expired hold is also dropped when the agent accepts a job
+     * ({@link clearExpiredBusyHold}). Otherwise the hold ends after
+     * {@link BUSY_HOLD_MAX_MS}, so a lost status report cannot strand the agent.
+     *
+     * An expired hold keeps its `busyHeldUntil` until that evidence arrives. So a
+     * hold still on the entry here means the agent was re-picked because its last
+     * hold ran out, and has neither reported a free slot nor accepted or finished
+     * a job since — an agent whose job count is stuck. The caller spends an
+     * attempt for that, which keeps such an agent from holding a job pending
+     * forever.
+     */
+    markBusyHeld(agentId: string, now?: number): BusyHoldOutcome;
+    /**
+     * Lift a {@link markBusyHeld} hold: the agent reported an empty slot, or a
+     * job tracked to it reached a final status.
+     */
+    clearBusyHeld(agentId: string): void;
+    /**
+     * Drop a {@link markBusyHeld} hold that has already run out: the agent
+     * accepted a job since.
+     *
+     * Every dispatch path skips an agent whose hold is in force, so a job the
+     * agent accepts while an expired hold is on its entry was dispatched after
+     * the hold ran out, into a slot that was free. That refutes the stuck job
+     * count the leftover hold stands for. A hold still in force is kept: the
+     * accepted job occupies the slot, so the acceptance says nothing about a
+     * free slot now.
+     */
+    clearExpiredBusyHold(agentId: string, now?: number): void;
     /** Undo {@link markDraining} — a destroy that was rejected leaves the agent alive. */
     clearDraining(agentId: string): void;
     get(agentId: string): AgentEntry | undefined;

package/dist/app.d.ts

@@ -208,6 +208,8 @@ export interface AppDependencies {
         issuer: string;
         resolveSigner: () => Promise<import('./oidc/signer.js').Signer | null>;
         repo: import('./db/repos/signing-keys-repo.js').OrchestratorSigningKeyRepo;
+        /** Whether this node is the Raft leader; named when a mint defers for want of a key. */
+        isLeader?: () => boolean;
     };
     /**
      * Dashboard-encryption (X25519) key custody. Present whenever KICI_SECRET_KEY

package/dist/cancel/cancel-held-run.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Cancel a run that sits `held` behind an approval request.
+ *
+ * A held run has no job and no dispatch yet: what it has is a pending
+ * `held_runs` row, the pending `KiCI Security` check a security hold posted,
+ * and the stored dispatch context an approve would replay. Cancelling it has to
+ * settle all three, or a later approve still finds its hold pending, replays
+ * the stored dispatch, and closes the check as approved for a run that was
+ * cancelled. So a cancel withdraws the approval request through the path a
+ * reject takes: each pending hold of the run is flipped `rejected` with the
+ * cancel reason, then handed to the workflow-hold rejection
+ * (`rejectWorkflow`), which cancels the run, completes the checks the held
+ * dispatch posted, settles the security check, and drops the stored context.
+ */
+import type { Kysely } from 'kysely';
+import type { Database, HeldRun } from '../db/types.js';
+import type { ExecutionTracker } from '../reporting/execution-tracker.js';
+/** How a cancel of a held run's approval requests ended. */
+export declare enum HeldRunWithdrawal {
+    /** The cancel rejected at least one pending hold itself. */
+    Withdrawn = "withdrawn",
+    /** An approve released the run first, so it is resuming. */
+    Approved = "approved",
+    /** A reject decided the hold first, and that rejection ends the run. */
+    Rejected = "rejected",
+    /** The hold expired first, and that expiry ends the run. */
+    Expired = "expired"
+}
+/** A decision on a held run that reached its holds before a cancel did. */
+export type HeldRunDecidedFirst = Exclude<HeldRunWithdrawal, HeldRunWithdrawal.Withdrawn>;
+/** Options for a workflow-hold rejection reached through a cancel. */
+export interface RejectHeldWorkflowOptions {
+    /**
+     * False when the run is not `held`: the rejection then settles the checks
+     * and drops the stored context but leaves the run row to the job
+     * cancellation that follows. Defaults to true.
+     */
+    runHeld?: boolean;
+}
+/** Options for {@link cancelHeldRunWithReason}. */
+export interface CancelHeldRunOptions {
+    /**
+     * False when the caller ends the run itself — a cancel of a run that is not
+     * `held`, whose jobs are cancelled next. The holds are still rejected and
+     * the checks settled, but no held-run terminal write is attempted: it would
+     * match no row and only report a missing held run. Defaults to true.
+     */
+    terminalWrite?: boolean;
+}
+export interface CancelHeldRunDeps {
+    db: Kysely<Database>;
+    executionTracker: ExecutionTracker;
+    /**
+     * The workflow-hold rejection (`rejectWorkflow` bound to live processing
+     * deps). Undefined where no provider wiring exists: the run is then cancelled
+     * and its holds rejected, but no provider check is settled.
+     */
+    rejectHeldWorkflow?: ((hold: HeldRun, reason: string, opts?: RejectHeldWorkflowOptions) => Promise<boolean>) | undefined;
+}
+/**
+ * Withdraw every pending approval request of a held run and cancel the run.
+ * Returns {@link HeldRunWithdrawal.Withdrawn} when this call withdrew the run.
+ * Otherwise a decision reached every hold first — an approve, a reject or an
+ * expiry, which the result names — the call writes nothing, and the caller
+ * refuses the cancel.
+ */
+export declare function cancelHeldRunWithReason(deps: CancelHeldRunDeps, runId: string, reason: string, opts?: CancelHeldRunOptions): Promise<HeldRunWithdrawal>;
+//# sourceMappingURL=cancel-held-run.d.ts.map

package/dist/cancel/cancel-route-answer.d.ts

@@ -0,0 +1,21 @@
+/**
+ * The admin cancel route's answer for a cancel that did not find the run
+ * already terminal. Kept apart from the route so the mapping is testable
+ * without assembling the whole app.
+ */
+import { ExecutionRunStatus } from '@kici-dev/engine';
+import { type CancelRunResult } from './cancel-run.js';
+export interface CancelRouteAnswer {
+    httpStatus: 200 | 409;
+    body: {
+        status: ExecutionRunStatus;
+        cancelledJobs: number;
+    } | {
+        error: string;
+        status: ExecutionRunStatus;
+    };
+    /** Note recorded on the access-log row, when the answer is not a plain success. */
+    accessNote?: string;
+}
+export declare function cancelRouteAnswer(result: CancelRunResult): CancelRouteAnswer;
+//# sourceMappingURL=cancel-route-answer.d.ts.map

package/dist/cancel/cancel-run.d.ts

@@ -20,6 +20,8 @@ import type { Database } from '../db/types.js';
 import type { JobQueue } from '../queue/job-queue.js';
 import type { AgentRegistry } from '../agent/registry.js';
 import type { ExecutionTracker } from '../reporting/execution-tracker.js';
+import type { HeldRun } from '../db/types.js';
+import { type HeldRunDecidedFirst, type RejectHeldWorkflowOptions } from './cancel-held-run.js';
 export interface CancelRunDeps {
     db: Kysely<Database>;
     jobQueue: JobQueue;
@@ -38,6 +40,13 @@ export interface CancelRunDeps {
     cancelJobOnPeer?: (peerId: string, runId: string, jobId: string, reason: string) => boolean;
     /** How stale a `cluster_instances` heartbeat may be and still read as live. */
     ownershipGraceMs?: number;
+    /**
+     * Reject a held run's workflow-scoped hold: `rejectWorkflow` bound to the live
+     * processing deps. A cancel of a `held` run goes through it so the approval
+     * request is withdrawn with the run. Undefined where no provider wiring
+     * exists; the held run is then still cancelled and its holds rejected.
+     */
+    rejectHeldWorkflow?: (hold: HeldRun, reason: string, opts?: RejectHeldWorkflowOptions) => Promise<boolean>;
 }
 export interface CancelRunOptions {
     /** Force immediate SIGKILL on the agent (skip graceful hooks). */
@@ -64,6 +73,29 @@ export interface CancelRunResult {
      * no-op: no agent was notified and no row was written.
      */
     alreadyTerminal: boolean;
+    /**
+     * Set when the run was `held` and a decision reached its holds first — an
+     * approve, a reject or an expiry — so the cancel wrote nothing. Entry points
+     * refuse the cancel with {@link heldRunCancelRefusal}.
+     */
+    decidedBeforeCancel?: HeldRunDecidedFirst;
+}
+/** Why a cancel of a held run whose hold an approve already released is refused. */
+export declare const RUN_RESUMING_AFTER_APPROVAL_MESSAGE = "Run is resuming after approval; cancel it again once it is running";
+/** Why a cancel of a held run whose hold a reject already decided is refused. */
+export declare const RUN_REJECTED_BEFORE_CANCEL_MESSAGE = "Run was rejected before the cancel reached it; the rejection ends it";
+/** Why a cancel of a held run whose hold already expired is refused. */
+export declare const RUN_EXPIRED_BEFORE_CANCEL_MESSAGE = "Run approval request expired before the cancel reached it; the expiry ends it";
+/** The message a refused cancel of a held run answers with, naming the decision that won. */
+export declare function heldRunCancelRefusal(decision: HeldRunDecidedFirst): string;
+/**
+ * Thrown by an entry point that refuses a cancel because a decision reached
+ * the held run first. An expected race, not a failure: callers answer it as a
+ * refusal rather than logging an error.
+ */
+export declare class HeldRunCancelRefusedError extends Error {
+    readonly decision: HeldRunDecidedFirst;
+    constructor(decision: HeldRunDecidedFirst);
 }
 /**
  * Cancel a whole run, recording `reason` on its cancelled jobs and (if not

package/dist/cli/api-client.d.ts

@@ -158,6 +158,25 @@ export declare class AdminApiClient {
     }>;
     setSecret(orgId: string, scope: string, key: string, value: string): Promise<void>;
     deleteSecret(orgId: string, scope: string, key: string): Promise<void>;
+    /**
+     * Create an empty secret scope. A `<backend>:` qualifier selects the
+     * backend; an unqualified scope targets the PG backend. Creating a scope
+     * that already exists succeeds and changes nothing.
+     */
+    createScope(orgId: string, scope: string): Promise<{
+        created: boolean;
+    }>;
+    /**
+     * Rename a secret scope inside its backend. The route refuses a move between
+     * backends and a rename onto a scope that already exists.
+     */
+    renameScope(orgId: string, oldScope: string, newScope: string): Promise<{
+        renamed: boolean;
+    }>;
+    /** Delete a secret scope and every secret in it. */
+    deleteScope(orgId: string, scope: string): Promise<{
+        deleted: boolean;
+    }>;
     /**
      * Create (or upsert) a deployment context. `allowLocalExecution` lets
      * CLI-initiated / local runs resolve secrets through this context.
@@ -204,6 +223,9 @@ export declare class AdminApiClient {
         skippedEphemeralKeys: number;
         reEncryptedSecretOutputs: number;
         skippedSecretOutputs: number;
+        /** Absent from an orchestrator that predates sealed job secrets. */
+        reEncryptedJobSecrets?: number;
+        skippedJobSecrets?: number;
     }>;
     createGenericSource(data: {
         orgId: string;
@@ -270,7 +292,7 @@ export declare class AdminApiClient {
         to?: string;
         limit?: number;
         offset?: number;
-        /** Phase D opt-in for cold-store read-through. */
+        /** Opt-in cold-store read-through. */
         includeArchived?: boolean;
     }): Promise<any[]>;
     createToken(data: {