@kici-dev/orchestrator 0.9.0 → 0.10.0

package/dist/pipeline/processor.d.ts

@@ -59,6 +59,7 @@ import { type Context as EngineContext } from '@kici-dev/engine';
 import type { OrchestratorMode, WorkflowDecision } from '@kici-dev/engine';
 import type { TrustPolicyStore } from '../security/trust-policy-store.js';
 import type { TrustDirectoryStore } from '../security/trust-directory-store.js';
+import { type ContextDataDeps, type DeferredContextResolution } from './held-context-data.js';
 /**
  * Pending dispatch context for jobs gated by the needs scheduler.
  * Keyed by `${runId}:${jobName}`. Populated at processWebhook time for all
@@ -76,6 +77,19 @@ interface PendingJobContext {
      * to an agent.
      */
     invoke?: InvokeGateParams;
+    /**
+     * Set for a job bound to a context: what `dispatchReadyJob` resolves the
+     * job's context variables and secrets from when the job leaves its pending
+     * context, since `jobInput` is stored without them. Absent for a job with no
+     * bound context, which dispatches its stored input unchanged.
+     */
+    contextResolution?: DeferredContextResolution;
+    /**
+     * Set when the stored row's sealed secret fields could not be decrypted:
+     * `dispatchReadyJob` fails the job with this reason instead of dispatching
+     * it without them.
+     */
+    secretsUnavailable?: string;
 }
 /**
  * Register an eval gate and return a promise that resolves when the scheduler
@@ -580,6 +594,12 @@ export interface ReadyDispatchGateDeps {
     accessLogWriter?: Pick<AccessLogWriter, 'record'>;
     /** The routing key the audit row is attributed to, when the call site knows it. */
     routingKey?: string | null;
+    /**
+     * The stores a released held job's context variables and secrets resolve
+     * against. Called at release time, so a call site whose stores are built after
+     * it is wired can still supply them.
+     */
+    contextData?: () => ContextDataDeps;
 }
 /** What {@link resolveRunConcurrency} needs to evaluate the concurrency gate. */
 export interface RunConcurrency {

package/dist/pipeline/rerun-global.d.ts

@@ -0,0 +1,28 @@
+import type { OriginalRunRow, RerunDeps } from './rerun.js';
+/** True when the run executed a workflow defined in another repository. */
+export declare function isCrossRepoGlobalRun(originalRun: OriginalRunRow): boolean;
+/**
+ * Re-run an organization-wide run against the commits it recorded: the source
+ * repository at `sha`, and the workflow repository at `workflow_sha`.
+ *
+ * The workflow is read from the workflow repository's lock file at the recorded
+ * commit, through that repository's registration — its provider bundle and
+ * provider context — never from the registration's current lock entry, which
+ * may have moved on. The run goes through the same pipeline a first global run
+ * does, so contexts, holds, approval gates, and the workflow repository's
+ * credentials apply exactly as they did.
+ *
+ * Refuses, naming the workflow repository, when the run recorded no workflow
+ * commit or the workflow is no longer registered there. Nothing falls back to a
+ * current lock file.
+ */
+export declare function rerunCrossRepoGlobalRun(opts: {
+    originalRun: OriginalRunRow;
+    triggeredBy: string | null;
+    triggeredByAgentLabel: string | null;
+    deps: RerunDeps;
+    requestId: string;
+}): Promise<{
+    newRunId: string;
+}>;
+//# sourceMappingURL=rerun-global.d.ts.map

package/dist/pipeline/rerun-inputs.d.ts

@@ -0,0 +1,88 @@
+import type { SimulatedEvent } from '@kici-dev/engine';
+import type { ProviderRegistry } from '../provider-registry.js';
+import type { TrustPolicyOutcome } from '../security/trust-policy-gate.js';
+import type { WebhookInfo } from '../webhook/handler.js';
+import type { OriginalRunRow, RerunDeps } from './rerun.js';
+/**
+ * Load the original webhook payload from object storage. Returns
+ * null for cron/schedule runs (no payload was stored) or when the payload
+ * cannot be parsed.
+ */
+export declare function loadWebhookPayload(originalRunId: string, deps: Pick<RerunDeps, 'logStorage'>): Promise<Record<string, unknown> | null>;
+/** The provider bundle + context a re-run resolves out of the original run's row. */
+export interface RerunProviderBinding {
+    providerBundle: NonNullable<ReturnType<ProviderRegistry['getByRoutingKey']>>;
+    providerContext: Record<string, unknown>;
+    routingKey: string;
+}
+/**
+ * Resolve the provider bundle and stored provider context for a run being
+ * re-run.
+ *
+ * Shared by the workflow re-run and the evaluation-round re-run so the two
+ * cannot disagree about which source a run belongs to, or about how its
+ * `provider_context` column is parsed.
+ */
+export declare function resolveRerunProviderBinding(originalRun: OriginalRunRow, deps: Pick<RerunDeps, 'providerRegistry'>): RerunProviderBinding;
+/** How a refusal to rebuild a re-run's inputs names the run and the way forward. */
+export interface RerunRefusal {
+    /** The sentence's opening, naming the run: `Cannot re-run …`. */
+    subject: string;
+    /** How the run relates to its event: `was deciding` for a round, `ran for` for a run. */
+    eventVerb: string;
+    /** What the caller can do instead. */
+    remedy: string;
+}
+/** The refusal wording for a global evaluation round. */
+export declare function roundRefusal(originalRun: OriginalRunRow): RerunRefusal;
+/**
+ * The provider event name + action the original delivery carried.
+ *
+ * `execution_runs` records neither: the event name arrives in a provider header,
+ * not in the payload, so it cannot be recovered from the stored payload either.
+ * The delivery's own `event_log` row is where it lives, and a round's run row
+ * carries the delivery id that addresses it. That row is written at the end of
+ * the delivery, after the round's failure is recorded, so it is present for
+ * every re-run an operator can actually reach.
+ *
+ * Addressed by `(org_id, delivery_id)` — the table's own uniqueness — never by
+ * the delivery id alone. A generic source's delivery id is taken verbatim from
+ * a sender-supplied header, so one tenant can choose an id another tenant's
+ * round already carries; a lookup by id alone would then re-evaluate one org's
+ * global workflows against another org's event shape, and with no `ORDER BY`
+ * the row it picked would not even be stable.
+ */
+export declare function loadDeliveryEventName(originalRun: OriginalRunRow, deps: Pick<RerunDeps, 'db'>, refusal?: RerunRefusal): Promise<{
+    event: string;
+    action: string | null;
+}>;
+/** The verdict an approved security hold stands for. */
+export declare const RELEASED_DECISION: TrustPolicyOutcome;
+/**
+ * The normalized form of the delivery the round was deciding.
+ *
+ * Read-only, and it throws — so it runs before the requestId claim, never
+ * inside the re-evaluation.
+ */
+export declare function normalizeRoundEvent(originalRun: OriginalRunRow, providerBundle: NonNullable<ReturnType<ProviderRegistry['getByRoutingKey']>>, delivery: {
+    event: string;
+    action: string | null;
+}, payload: Record<string, unknown>, refusal?: RerunRefusal): SimulatedEvent;
+/**
+ * Stamp the re-evaluated event with the source repository's changed files.
+ *
+ * Unconditional, unlike the delivery path's fetch: that path skips the fetch
+ * when no trigger in the source repo's lock file uses path patterns, and a
+ * scoped re-evaluation has no such lock file to read. An error carries
+ * `unavailable`, which every path filter downstream already treats
+ * conservatively.
+ */
+export declare function withChangedFiles(opts: {
+    event: SimulatedEvent;
+    bundle: NonNullable<ReturnType<ProviderRegistry['getByRoutingKey']>>;
+    info: WebhookInfo;
+    payload: Record<string, unknown>;
+    credentials: Record<string, unknown>;
+    repoIdentifier: string;
+}): Promise<SimulatedEvent>;
+//# sourceMappingURL=rerun-inputs.d.ts.map

package/dist/pipeline/rerun.d.ts

@@ -10,8 +10,8 @@
  * It goes directly to: lock file parse -> job expansion -> dispatch.
  */
 import { type ColdStore } from '@kici-dev/shared';
-import type { Kysely } from 'kysely';
-import type { Database } from '../db/types.js';
+import type { Kysely, Selectable } from 'kysely';
+import type { Database, ExecutionRunTable } from '../db/types.js';
 import type { LogStorage } from '../reporting/log-storage.js';
 import type { ProviderRegistry } from '../provider-registry.js';
 import type { Dispatcher } from '../agent/dispatcher.js';
@@ -78,6 +78,12 @@ export interface RerunDeps {
      */
     processingDeps?: (() => ProcessingDeps) | null;
 }
+/**
+ * Original run row from execution_runs (selectAll).
+ * Aliased to the Kysely Selectable so all column names stay typed
+ * as the underlying schema.
+ */
+export type OriginalRunRow = Selectable<ExecutionRunTable>;
 export declare function handleRerun(originalRunId: string, triggeredBy: string | null, triggeredByAgentLabel: string | null, deps: RerunDeps, 
 /**
  * Platform-minted `requestId` for this rerun. Stable across an HA relay
@@ -104,4 +110,21 @@ routingKeyHint?: string): Promise<{
  * deterministically instead of racing the microtask queue.
  */
 export declare function settlePendingRoundReevaluations(): Promise<void>;
+/**
+ * Release a held global evaluation round: re-evaluate the round of its workflow
+ * repository from the stored webhook payload with the approved event treated as
+ * passing, dispatch what it admits, and settle the held run.
+ *
+ * Reached from the security queue's workflow release (`resumeWorkflow`), which
+ * recognises the held round by its run row. The held row stores no dispatch
+ * context: everything is rebuilt from the row, the payload and the delivery's
+ * event log entry — the same reconstruction a re-run of a failed round uses —
+ * so the release does not depend on how a workflow dispatch context would be
+ * replayed.
+ *
+ * Never throws. A round that cannot be rebuilt or decided fails the held run
+ * with the reason, which is the record an approver can read. A release that
+ * cannot claim the held row — another release already did — does nothing.
+ */
+export declare function releaseHeldGlobalEvalRound(runId: string, deps: ProcessingDeps): Promise<void>;
 //# sourceMappingURL=rerun.d.ts.map

package/dist/pipeline/resume-workflow.d.ts

@@ -1,16 +1,42 @@
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 import type { ProcessingDeps } from './processor.js';
+import type { ProviderBundle } from '../provider-registry.js';
 import type { ReleaseSignal } from '../contexts/held-runs.js';
 import { type WorkflowDispatchContext } from './dispatch-matched-workflow.js';
 import { type SerializableWorkflowDispatchInputs } from './pending-workflow-context.js';
+import type { GlobalDispatchIdentity } from './global-dispatch-identity.js';
 import { type SecurityCheckHold } from './security-hold-check.js';
+/**
+ * A global run's identity rebuilt from its stored hold, before the workflow
+ * repository's credentials are minted. It has no `workflowCredentials`, so a
+ * context carrying it is not a {@link WorkflowDispatchContext}.
+ */
+export type UnmintedGlobalIdentity = Omit<GlobalDispatchIdentity, 'workflowCredentials' | 'workflowBundle'> & {
+    readonly workflowBundle: ProviderBundle;
+};
+/**
+ * A dispatch context rebuilt from a stored hold. `dispatchMatchedWorkflow`
+ * does not accept it: only {@link withWorkflowRepoCredentials} turns it into a
+ * {@link WorkflowDispatchContext}, so a resume cannot dispatch a global run
+ * without a freshly minted clone token.
+ */
+export type RebuiltWorkflowDispatchContext = Omit<WorkflowDispatchContext, 'global'> & {
+    global?: UnmintedGlobalIdentity;
+};
 /**
  * Rebuild a live `WorkflowDispatchContext` from the persisted serializable
  * inputs by re-attaching the orchestrator's live `deps` and reconstructing the
  * provider `bundle` from the live registry (keyed by the stored routing key).
  * Returns null when the provider bundle can no longer be resolved.
  *
+ * A global run's identity is rebuilt from what was stored at hold time — the
+ * workflow commit, branch and provider context it was held at — never from the
+ * live registration, which may have moved to a newer commit while the run
+ * waited. Its workflow bundle is looked up by the stored workflow routing key,
+ * and it has no credentials until {@link withWorkflowRepoCredentials} mints
+ * them. Returns null when the workflow bundle can no longer be resolved either.
+ *
  * The key it looks the bundle up by is `effectiveRoutingKey ?? info.routingKey`
  * — the post-overlay key, the same one `setupDispatchContext` builds
  * `setup.info` from and the same one the run row records. `info.routingKey` is
@@ -19,7 +45,13 @@ import { type SecurityCheckHold } from './security-hold-check.js';
  * than the source the workflow belongs to: the resumed run would get the wrong
  * app, the wrong credentials, and a check poster writing to the wrong place.
  */
-export declare function rebuildWorkflowDispatchContext(inputs: SerializableWorkflowDispatchInputs, deps: ProcessingDeps): WorkflowDispatchContext | null;
+export declare function rebuildWorkflowDispatchContext(inputs: SerializableWorkflowDispatchInputs, deps: ProcessingDeps): RebuiltWorkflowDispatchContext | null;
+/**
+ * The rebuilt context with fresh credentials for a global run's workflow
+ * repository, minted by its bundle the way the first dispatch minted them. A
+ * same-repo context gets no global identity. Throws when the mint fails.
+ */
+export declare function withWorkflowRepoCredentials(rebuilt: RebuiltWorkflowDispatchContext): Promise<WorkflowDispatchContext>;
 /**
  * Resume a released workflow-scoped hold. Loads the pending context, rebuilds
  * the dispatch context, and re-dispatches. On a lost pending context (or
@@ -55,5 +87,7 @@ export declare function resumeWorkflow(signal: ReleaseSignal, deps: ProcessingDe
  * a rejection this declines to report (an install gate, or a commit whose other
  * holds are still pending) leaves the caller free to decide for itself.
  */
-export declare function rejectWorkflow(hold: SecurityCheckHold, deps: ProcessingDeps, db: Kysely<Database> | undefined, reason: string): Promise<boolean>;
+export declare function rejectWorkflow(hold: SecurityCheckHold, deps: ProcessingDeps, db: Kysely<Database> | undefined, reason: string, opts?: {
+    runHeld?: boolean;
+}): Promise<boolean>;
 //# sourceMappingURL=resume-workflow.d.ts.map

package/dist/pipeline/unbound-job-context.d.ts

@@ -0,0 +1,34 @@
+/**
+ * The dispatch-time warning for a job context that has no scope binding.
+ *
+ * A job receives a context's secrets only through the context's scope
+ * bindings, so a context with none delivers no secret, and nothing fails until
+ * a step reads one. This names each such context in the orchestrator log when
+ * the job's context data resolves. Advisory only: it runs after resolution, a
+ * failed lookup logs nothing, and the dispatch proceeds unchanged.
+ */
+import type { Context } from '@kici-dev/engine';
+import type { SecretResolverApi } from '../secrets/secret-resolver.js';
+/** The warning line's message; the structured fields carry the identities. */
+export declare const UNBOUND_JOB_CONTEXT_MESSAGE = "Job binds a context that has no scope binding; it receives no secrets from it";
+/**
+ * Warn once for every context in `entries` that resolved no secret and has no
+ * scope binding. A context that resolved at least one secret is skipped without
+ * a lookup, so a bound context costs nothing here.
+ */
+export declare function warnUnboundJobContexts(args: {
+    secretResolver: SecretResolverApi | undefined;
+    orgId: string;
+    entries: ReadonlyArray<{
+        name: string;
+        env: Context;
+    }>;
+    /** Per-context resolved secrets, keyed by the declared name; empty maps are absent. */
+    resolvedByName: Readonly<Record<string, Record<string, string>>> | undefined;
+    log: {
+        runId: string;
+        workflow: string;
+        job: string;
+    };
+}): Promise<void>;
+//# sourceMappingURL=unbound-job-context.d.ts.map

package/dist/pipeline/workflow-lock-at-sha.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Load a workflow repository's lock entries as they were at a recorded commit.
+ *
+ * An organization-wide run records the commit of the repository that defines
+ * its workflow (`execution_runs.workflow_sha`). Two paths act on such a run
+ * later, after that repository may have moved on: the re-run of a finished
+ * global run, and the release of a held evaluation round. Both must run the
+ * workflow version the run was created against, never the registration's
+ * current one. This module is the one place that resolves it.
+ */
+import type { LockFile, LockWorkflow } from '@kici-dev/engine';
+import type { ProviderRegistry } from '../provider-registry.js';
+import type { RegisteredWorkflow } from '../registration/registration-index.js';
+import { type DepCacheKey } from '@kici-dev/shared';
+/**
+ * Whether a lock entry is an organization-wide workflow: a trigger carries
+ * `repos:`. The re-run and the held-round release both judge a workflow by its
+ * entry at the recorded commit through this one rule, never by the current
+ * registration.
+ */
+export declare function isOrganizationWideEntry(entry: LockWorkflow): boolean;
+/**
+ * The whole lock file of a registration's repository at `sha`, fetched through
+ * the registration's bundle. Throws, naming the repository, when it cannot be
+ * read.
+ */
+export declare function loadWorkflowLockFileAtSha(args: {
+    registration: WorkflowLockRegistration;
+    sha: string;
+    providerRegistry: Pick<ProviderRegistry, 'getByRoutingKey'>;
+}): Promise<LockFile>;
+/** The registration fields that address a workflow repository's lock file. */
+export type WorkflowLockRegistration = Pick<RegisteredWorkflow, 'repoIdentifier' | 'workflowName' | 'routingKey' | 'providerContext' | 'commitSha' | 'lockEntry' | 'lockfileHash' | 'siblingsDigest'>;
+/** One workflow's lock entry at a commit, with the dependency-cache key of the lock file it comes from. */
+export interface WorkflowLockEntryAtSha {
+    lockEntry: LockWorkflow;
+    depCacheKey: DepCacheKey;
+}
+/**
+ * The lock entries of `registrations` at `sha`, keyed by workflow name.
+ *
+ * Every registration must belong to the same workflow repository. When each
+ * registration is already at `sha`, its stored lock entry is that version and
+ * nothing is fetched. Otherwise the lock file is fetched at `sha` through the
+ * registration's own provider bundle, with the registration's provider context
+ * — the same fetcher a per-repository re-run reads its own lock file with. A
+ * workflow the lock file at `sha` does not define is absent from the result.
+ *
+ * Throws, naming the workflow repository, when the lock file cannot be read:
+ * the caller refuses rather than falling back to any other version.
+ */
+export declare function loadWorkflowLockEntriesAtSha(args: {
+    registrations: readonly WorkflowLockRegistration[];
+    sha: string;
+    providerRegistry: Pick<ProviderRegistry, 'getByRoutingKey'>;
+}): Promise<Map<string, LockWorkflow>>;
+/**
+ * The lock entry of one registered workflow at `sha`, with the dependency-cache
+ * key of the lock file it comes from. Throws, naming the workflow repository,
+ * when the lock file at `sha` does not define it.
+ */
+export declare function loadWorkflowLockEntryAtSha(args: {
+    registration: WorkflowLockRegistration;
+    sha: string;
+    providerRegistry: Pick<ProviderRegistry, 'getByRoutingKey'>;
+}): Promise<WorkflowLockEntryAtSha>;
+//# sourceMappingURL=workflow-lock-at-sha.d.ts.map

package/dist/provenance/backfill-run.d.ts

@@ -20,6 +20,11 @@ export interface BackfillRunRow {
     started_at: Date | null;
     completed_at: Date | null;
     duration_ms: number | null;
+    /**
+     * The run's status generation. Forwarded so the Platform mirror records the
+     * generation the backfilled status belongs to. Absent reads as 0.
+     */
+    status_epoch?: number | null;
 }
 /** The local `execution_jobs` fields the backfill needs. */
 export interface BackfillJobRow {

package/dist/providers/github/check-status-poster.d.ts

@@ -4,12 +4,11 @@
  * Posts check statuses to GitHub PRs using the Checks API for:
  * - Security holds (pending): "KiCI Security" — "Held for approval"
  * - Workflow modifications (neutral): "KiCI: Workflow changes"
- * - Org globals skipped by the trust policy (neutral): "KiCI: Organization workflows"
  * - Org global evaluation failed (failure): "KiCI: Organization workflow evaluation"
  * - Approved runs (success) / Rejected/expired runs (failure)
  *
- * `postWorkflowModificationCheck`, `postGlobalWorkflowsSkippedCheck`, and
- * `postGlobalEvalFailedCheck` are interface-backed (the `CheckStatusPoster`
+ * `postWorkflowModificationCheck` and `postGlobalEvalFailedCheck` are
+ * interface-backed (the `CheckStatusPoster`
  * contract in `@kici-dev/engine`) and post on their own check names so none of
  * them overwrites the security-hold check.
  *
@@ -38,23 +37,12 @@ export declare class GitHubCheckStatusPoster implements CheckStatusPoster {
      * conflict with the security hold check. Always posted as neutral/completed.
      */
     postWorkflowModificationCheck(repoIdentifier: string, commitSha: string, modifications: WorkflowModification[], credentials: unknown): Promise<void>;
-    /**
-     * Post the informational check recording that the org's global workflows were
-     * skipped because the trust policy held or rejected the event.
-     *
-     * Uses a separate check name ("KiCI: Organization workflows") so it never
-     * overwrites the security-hold check: that one is a single named run per
-     * commit which the hold posts as pending and approve / reject later complete,
-     * so writing this notice onto it would resolve the hold's check to neutral
-     * while the run is still held. Always posted as neutral/completed.
-     */
-    postGlobalWorkflowsSkippedCheck(repoIdentifier: string, commitSha: string, summary: string, credentials: unknown): Promise<void>;
     /**
      * Post the check recording that the pre-run evaluation of the organization's
      * global workflows failed, so none of the workflows it was deciding on ran.
      *
-     * Its own check name ("KiCI: Organization workflow evaluation") for the same
-     * reason as the notice above: the security-hold check is a single named run
+     * Its own check name ("KiCI: Organization workflow evaluation") because the
+     * security-hold check is a single named run
      * per commit which a hold posts as pending and approve / reject later
      * complete, so writing this through `postCheckStatus` would resolve a
      * still-held run's check and unblock a branch protection rule that requires

package/dist/queue/job-queue.d.ts

@@ -2,6 +2,7 @@ import { type Kysely, type SqlBool } from 'kysely';
 import { type CanonicalLabel, type CanonicalMatcher, type LabelMatcher, type ResourceRequest } from '@kici-dev/engine';
 import type { Database } from '../db/types.js';
 import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
+import { JobContainerNeed } from '../scaler/agent-fit.js';
 /**
  * The ownership predicate, in its two forms. Both come from here so they cannot
  * drift apart, because they answer the same question from opposite directions.
@@ -35,6 +36,8 @@ export interface OwnershipPredicateOptions {
      */
     livePeerIds?: readonly string[];
 }
+/** Recorded on a job whose direct dispatch was refused because its run stopped. */
+export declare const STOPPED_RUN_DISPATCH_REASON = "the run stopped before this job was dispatched";
 /**
  * Strict selection: the rows a cluster-wide sweep may recover. Excludes a NULL
  * owner and excludes any row whose owner is still live.
@@ -67,6 +70,12 @@ export interface ExpiredJobInfo {
     runsOnPatterns: CanonicalMatcher[];
     excludeLabels: CanonicalLabel[];
     excludePatterns: CanonicalMatcher[];
+    /**
+     * What the job's `container:` asks of the agent that runs it. A container job
+     * reaches only an agent that can start one, so an agent that merely matches
+     * its labels may still be no route at all. Absent reads as no container.
+     */
+    container?: JobContainerNeed;
 }
 /**
  * A pending job as the unroutable probe sees it: the same routing facts the
@@ -109,11 +118,23 @@ export declare enum DispatchQueueStatus {
     Expired = "expired",
     Recovering = "recovering"
 }
+/**
+ * The `dispatch_queue` statuses a row never leaves. `cancelled` is not a
+ * {@link DispatchQueueStatus}, but existing databases hold rows the test-run
+ * cancel stamped with it; those rows are terminal too, so the prune and the
+ * sealed-secrets scrub cover them.
+ */
+export declare const TERMINAL_DISPATCH_STATUSES: string[];
 /**
  * Maximum delivery attempts for a single dispatch_queue job. A job whose
  * `dispatch_attempts` reaches this value is failed permanently instead of
  * being requeued again. Bounds requeue loops from repeated job.reject /
- * pre-start agent loss; `expires_at` is the time-based backstop.
+ * pre-start agent loss. A busy job.reject is not counted while the agent keeps
+ * reporting an empty slot between rejections, but is counted once the agent is
+ * re-picked only because its busy hold expired, so this cap still bounds an
+ * agent whose job count is stuck. `expires_at` is no backstop for this: a
+ * job dispatched straight to an agent carries none, and requeue does not set
+ * one.
  */
 export declare const MAX_DISPATCH_ATTEMPTS = 5;
 /**
@@ -231,6 +252,12 @@ export interface QueuedJob {
     resources?: ResourceRequest;
     /** For a runsOnAll host-fanout child: the agent this job is pinned to. */
     pinnedAgentId?: string;
+    /**
+     * Set when the row's sealed secret fields could not be decrypted. The
+     * dispatcher never sends such a job: it puts the job back so a coordinator
+     * holding the key can take it, or fails it with this reason when none can.
+     */
+    secretsUnavailable?: string;
 }
 /**
  * The resource shape a job declares, read from the one location that always
@@ -272,6 +299,19 @@ export declare class JobQueue {
      * gauge callback never issues an extra COUNT per scrape.
      */
     private breakdownCache;
+    /**
+     * Jobs this coordinator put back because it could not open their sealed
+     * secrets (sealed with a master key it does not hold), keyed to the time it
+     * may claim them again. Until then every claim path and the re-drive readers
+     * skip them, so during a rolling key rotation a coordinator that holds the
+     * key can take the job in that window. After the back-off this coordinator
+     * can claim the job again, and each claim spends another attempt, so a job
+     * no coordinator can open still reaches {@link MAX_DISPATCH_ATTEMPTS} and
+     * fails. Bounded; an evicted entry only ends its back-off early.
+     */
+    private readonly deferredUnopenable;
+    /** The configured default for `sealed_secrets_retry_backoff_ms`. */
+    private readonly sealedSecretsRetryBackoffMs;
     constructor(db: Kysely<Database>, options: {
         maxDepth: number;
         defaultTimeoutMs: number;
@@ -293,6 +333,11 @@ export declare class JobQueue {
          * recovery grace period.
          */
         ownershipGraceMs?: number;
+        /**
+         * Default back-off before this coordinator claims again a job whose
+         * sealed secrets it could not open; `cluster_settings` overrides it.
+         */
+        sealedSecretsRetryBackoffMs?: number;
     });
     /** Ownership predicate inputs, in the shape both predicate builders take. */
     private ownershipOptions;
@@ -324,12 +369,14 @@ export declare class JobQueue {
      * @param agentMandatoryLabels Mandatory labels the spawning scaler declared
      *   (empty for static / non-scaler agents).
      * @param canServe Optional extra predicate applied to every candidate row
-     *   before it is claimed. Labels cannot express whether a pre-spawned agent's
-     *   fixed cpu / memory / image fit a job, so a caller that has to answer that
-     *   supplies it here — the row must never be claimed and put back, because a
-     *   claim-then-release both strands it as Dispatched for a window and burns a
-     *   dispatch attempt. Supplying it opts out of the single-statement fast
-     *   path, so pass it only for an agent that actually needs the check.
+     *   before it is claimed. Labels cannot express whether an agent may run a
+     *   job — a pre-spawned agent's fixed shape, a container job on an agent with
+     *   no runtime, an agent started inside another job's image — so a caller
+     *   that has to answer that supplies it here. The row must never be claimed
+     *   and put back, because a claim-then-release both strands it as Dispatched
+     *   for a window and burns a dispatch attempt. Supplying it opts out of the
+     *   single-statement fast path, so pass it only for an agent that actually
+     *   needs the check.
      * @returns The matching job, or null if none found.
      */
     dequeueForLabels(agentLabels: string[], agentMandatoryLabels?: string[], agentId?: string, canServe?: (job: QueuedJob) => boolean): Promise<QueuedJob | null>;
@@ -491,12 +538,38 @@ export declare class JobQueue {
      * it can avoid double-dispatching an already-present job. The conflicting row
      * keeps the owner the winning writer recorded.
      *
-     * @returns The job ID and whether a new row was inserted (false = row already existed).
+     * A job whose run has already stopped is refused, with the same stopped-run
+     * guard the queue drain applies (`notTerminalRunSql`). The row is still
+     * written, then settled `expired` in a guarded update — the state a cancel
+     * leaves a queued job of its run in — and the job's `execution_jobs` row, if
+     * one exists yet, is cancelled. `runStopped` tells the caller not to send it.
+     * Checking after the insert rather than before closes the race with a cancel:
+     * a cancel that lands after the check sees the dispatched row and cancels it
+     * through the agent, and one that lands before is seen by the check.
+     *
+     * @returns The job ID, whether a new row was inserted (false = row already
+     *   existed), and whether the dispatch was refused because the run stopped.
      */
     insertDispatched(job: QueuedJobInput, agentId: string): Promise<{
         id: string;
         inserted: boolean;
+        runStopped?: boolean;
     }>;
+    /**
+     * Settle a claimed row whose run stopped after the claim: expire it and
+     * cancel its pending job row, the way the cancel path ends a stopped run's
+     * work. {@link requeue} refuses a stopped run's row, so a caller that claimed
+     * a job to put it back uses this when that requeue returns null; otherwise
+     * the row stays dispatched to no agent until a sweep finds it. Returns
+     * whether it settled the row: false when the row is no longer dispatched or
+     * its run is still going.
+     */
+    settleClaimOfStoppedRun(id: string, runId: string): Promise<boolean>;
+    /**
+     * Settle a dispatched row `expired` when its run has stopped, and cancel the
+     * job's `execution_jobs` row. Returns whether it did.
+     */
+    private expireDispatchOfStoppedRun;
     /**
      * Mark a job as dispatched and record the agent it went to.
      *
@@ -555,6 +628,15 @@ export declare class JobQueue {
      * @returns Number of affected rows.
      */
     cancelByRunId(runId: string): Promise<number>;
+    /**
+     * Drop the sealed secret fields of every terminal row. A completed, failed or
+     * expired row is never dispatched again (a requeue moves only a dispatched
+     * row), so its secrets have no reader left; the row stays for its retention
+     * window without them.
+     *
+     * @returns Number of rows scrubbed.
+     */
+    scrubTerminalSealedSecrets(): Promise<number>;
     /**
      * Delete terminal dispatch_queue rows older than `retentionDays`.
      *
@@ -739,10 +821,17 @@ export declare class JobQueue {
      * job started. Only flips rows still in 'dispatched' — a job that was
      * concurrently completed / failed / cancelled is left untouched.
      *
-     * @returns the post-increment dispatch_attempts, or null when the row
+     * `countAttempt: false` leaves the counter alone — used for a busy rejection
+     * from an agent still tearing down its previous job (see
+     * {@link MAX_DISPATCH_ATTEMPTS} for when a busy rejection does count).
+     *
+     * @returns the dispatch_attempts after the requeue, or null when the row
      *   was not in 'dispatched' state (nothing requeued).
      */
-    requeue(jobId: string): Promise<number | null>;
+    requeue(jobId: string, opts?: {
+        countAttempt?: boolean;
+        provisioningError?: string;
+    }): Promise<number | null>;
     /**
      * Requeue a dispatched job **only** when its ack genuinely never landed.
      *
@@ -876,6 +965,34 @@ export declare class JobQueue {
      *   dispatched, cancelled, expired, or claimed by another coordinator.
      */
     claimUnroutable(id: string): Promise<boolean>;
+    /**
+     * Leave a job alone for the sealed-secrets back-off: the dispatcher calls
+     * this when it puts back a job whose sealed secrets this coordinator could not
+     * open. See {@link deferredUnopenable}.
+     */
+    deferUnopenable(jobId: string): Promise<void>;
+    /** Whether `jobId` is still inside its sealed-secrets back-off on this coordinator. */
+    isDeferredUnopenable(jobId: string): boolean;
+    /**
+     * Claim a specific pending job for no agent: Pending -> Dispatched, with no
+     * `agent_id`. The dispatcher calls this for a job whose sealed secrets this
+     * coordinator cannot open, when a re-drive would otherwise scale for it, so
+     * the job gets the put-back or fail decision a drain claim would give it.
+     *
+     * The guards of the other claim paths apply: the row is still pending and
+     * unexpired, its run has not stopped, and it is outside this coordinator's
+     * sealed-secrets back-off. The `status = Pending` guard is the arbiter, so a
+     * concurrent claimant makes this return false.
+     *
+     * @returns whether this call claimed the row.
+     */
+    claimUnopenableById(jobId: string): Promise<boolean>;
+    /**
+     * The query condition that skips jobs still inside their sealed-secrets
+     * back-off, or undefined when none is (the common case adds no predicate).
+     * Expired entries are dropped here, which is what lets a job be claimed again.
+     */
+    private deferredFilter;
     /**
      * Convert a DB row to a QueuedJob object.
      * Handles both auto-parsed JSONB arrays (from pg driver) and JSON strings (from tests).