@kici-dev/orchestrator 0.6.1 → 0.8.0

package/dist/agent/dispatcher.d.ts

@@ -27,7 +27,7 @@ export declare enum AgentDrainDecline {
  * Default per-pass cap on how many pending jobs a capacity-freed re-drive
  * re-offers to the scaler. Bounds the burst so a single free event cannot storm
  * the scaler; the per-backend spawn semaphore bounds actual provisioning, and a
- * job that gets `at-capacity` again simply stays pending for the next tick.
+ * job that gets `at-capacity` again stays pending for the next tick.
  */
 export declare const DEFAULT_REDRIVE_BATCH = 10;
 /**
@@ -403,6 +403,15 @@ export declare class Dispatcher {
      * triage can surface failed job IDs to the caller.
      */
     private requeueOrFail;
+    /**
+     * The half of {@link requeueOrFail} that runs once the row has been flipped:
+     * fail the job when its attempt budget is exhausted, otherwise re-dispatch.
+     *
+     * Split out so the ack-expiry paths can drive it from the attempt count their
+     * own guarded requeue (`requeueIfAwaitingAck`) returned, instead of flipping
+     * the row a second time.
+     */
+    private finishRequeue;
     /**
      * Try to deliver a requeued pending job: prefer an idle matching agent
      * (atomic claim via dispatchBoundJob), otherwise consult the scaler so a

package/dist/agent/host-roster.d.ts

@@ -1,5 +1,5 @@
 import { type Kysely } from 'kysely';
-import { type HostInventoryEntry, type InventorySelector, type LabelMatcher } from '@kici-dev/engine';
+import { type CanonicalLabel, type HostInventoryEntry, type InventorySelector, type LabelMatcher } from '@kici-dev/engine';
 import type { Database, HostRosterRow } from '../db/types.js';
 /** Typed host-vars bag carried by roster rows (`string | number | boolean`). */
 export type HostProperties = Record<string, string | number | boolean>;
@@ -61,9 +61,14 @@ export type LifecycleClass = 'static' | 'ephemeral';
  */
 export interface MatchedHost {
     agentId: string;
-    /** Hostname (falls back to agentId). */
+    /** Canonical hostname; falls back to the raw agentId when the row has none. */
     host: string;
-    labels: string[];
+    /**
+     * The row's labels, folded on read. A row written before labels were
+     * canonicalized still carries the case the operator typed, and this is the
+     * only place that case would otherwise reach a matcher.
+     */
+    labels: CanonicalLabel[];
     lifecycleClass: LifecycleClass;
     /** Which orchestrator holds the live WS (null = disconnected). For the cross-cluster pin. */
     connectedInstanceId: string | null;

package/dist/agent/registry.d.ts

@@ -8,7 +8,7 @@
  *
  * No persistence needed -- agents re-register on reconnect.
  */
-import { type LabelMatcher } from '@kici-dev/engine';
+import { type CanonicalLabel, type LabelMatcher } from '@kici-dev/engine';
 import type { WsLike } from '@kici-dev/engine';
 export type { WsLike } from '@kici-dev/engine';
 /**
@@ -19,8 +19,12 @@ export interface AgentEntry {
     agentId: string;
     /** The underlying WebSocket (or mock). */
     ws: WsLike;
-    /** Set of labels this agent provides (e.g., "linux", "gpu", "docker"). */
-    labels: Set<string>;
+    /**
+     * Set of labels this agent provides (e.g., "linux", "gpu", "docker"), folded
+     * to canonical form by `register()`. The label reverse index is keyed the
+     * same way, so a lookup must fold its query before it reaches either.
+     */
+    labels: Set<CanonicalLabel>;
     /**
      * Kubernetes-taint-style mandatory labels inherited from the scaler that
      * spawned this agent. A job is only dispatched to this agent when every
@@ -35,8 +39,11 @@ export interface AgentEntry {
      * (`onAgentAvailable` → `dequeueForLabels`) sees only the agent's labels
      * and would otherwise pull a queued job whose `runsOn` is a subset of the
      * agent's labels but does not satisfy the gate).
+     *
+     * Folded to canonical form by `register()`, so a scaler that declared `GPU`
+     * gates the same jobs a scaler that declared `gpu` does.
      */
-    mandatoryLabels: Set<string>;
+    mandatoryLabels: Set<CanonicalLabel>;
     /**
      * True when this agent was spawned by a scaler backend. Scaler-managed
      * agents are single-use: the scaler destroys them on disconnect, so they
@@ -45,6 +52,22 @@ export interface AgentEntry {
      * recovery window that cannot succeed.
      */
     scalerManaged: boolean;
+    /**
+     * True while this agent is being torn down and must receive no new work.
+     *
+     * The warm pool decides an agent is idle, marks it, then awaits a destroy
+     * that takes seconds — a container `stop({ t: 5 })`, or a Firecracker
+     * `SendCtrlAltDel` plus its grace. The registry entry survives until the
+     * agent's WebSocket actually closes, so without this flag `findAvailable`
+     * kept returning the agent throughout: a job arriving in that window was
+     * dispatched onto a container that was already exiting, and failed as a
+     * spurious infrastructure error on a healthy system.
+     *
+     * In memory rather than DB-backed, unlike reboot-pending: a warm-pool
+     * destroy is issued by this process, lasts seconds, and ends with the
+     * WebSocket close that removes the entry — there is nothing to outlive.
+     */
+    draining: boolean;
     /** Number of jobs currently executing on this agent. */
     activeJobs: number;
     /** Maximum concurrent jobs this agent can handle (default 1). */
@@ -211,8 +234,16 @@ export declare class AgentRegistry {
     /**
      * Register an agent. If the agentId already exists, updates the existing
      * entry (agent reconnection scenario).
+     *
+     * @returns the socket the previous entry was bound to, when this register
+     *   replaced one bound to a *different* socket. The caller owns that socket's
+     *   per-connection state and must retire it: a half-open connection can
+     *   outlive the reconnect, and its later close carries the same agent id, so
+     *   a caller that keeps treating it as live tears down the registration that
+     *   replaced it. `undefined` for a first register or a re-register on the
+     *   same socket.
      */
-    register(agentId: string, ws: WsLike, labels: string[], platform?: string, arch?: string, version?: string, maxConcurrency?: number, metadata?: AgentMetadata): void;
+    register(agentId: string, ws: WsLike, labels: string[], platform?: string, arch?: string, version?: string, maxConcurrency?: number, metadata?: AgentMetadata): WsLike | undefined;
     /**
      * Unregister an agent by ID. Removes from all indexes.
      * @returns The removed AgentEntry, or undefined if not found.
@@ -300,6 +331,9 @@ export declare class AgentRegistry {
      * matches zero agents (so callers can short-circuit). Returns an empty
      * Set on `requiredLabels.length === 0` since the caller decides the
      * walk strategy in that case.
+     *
+     * Takes canonical labels: the index it reads is keyed on the canonical form,
+     * so a caller folds before it gets here.
      */
     private intersectLabelCandidates;
     /**
@@ -334,6 +368,27 @@ export declare class AgentRegistry {
     /**
      * Get a single agent entry by ID.
      */
+    /**
+     * Whether one agent id is currently registered.
+     *
+     * Distinct from `findAvailable`, which answers "which agents could take this
+     * job" — the container orphan sweep needs to know only whether a running
+     * container's agent is present at all, including one that is busy or gated.
+     */
+    isRegistered(agentId: string): boolean;
+    /**
+     * Condemn an agent: it stays registered and keeps finishing its current work,
+     * but takes no new dispatches.
+     *
+     * Called before a warm-pool destroy is issued. Returns the agent's current
+     * `activeJobs` so the caller can see a job that was claimed between its idle
+     * check and this mark, and back out.
+     *
+     * @returns the agent's active-job count, or `undefined` when it is not registered
+     */
+    markDraining(agentId: string): number | undefined;
+    /** Undo {@link markDraining} — a destroy that was rejected leaves the agent alive. */
+    clearDraining(agentId: string): void;
     get(agentId: string): AgentEntry | undefined;
     /**
      * Get an agent entry by its WebSocket reference.

package/dist/app.d.ts

@@ -57,6 +57,7 @@ import type { OwnershipTracker } from './agent/ownership-tracker.js';
 import type { ObserverRegistry } from './ws/observer-registry.js';
 import type { FleetAgentCollector } from './ws/fleet-agent-collector.js';
 import type { FleetTopology } from './diagnostics/fleet-topology.js';
+import type { ResolvedMasterKeys } from './secrets/config.js';
 import type { TokenManager } from './secrets/token-manager.js';
 import type { SecretResolver } from './secrets/secret-resolver.js';
 import type { VerifyInboundDeps } from './webhook/verify-inbound.js';
@@ -75,7 +76,6 @@ import { type OverflowDelivery } from './webhook/ingest-overflow-types.js';
 import type { IngestAdmissionController } from './webhook/ingest-admission.js';
 import type { OrgIngestCapReader } from './webhook/org-ingest-cap-reader.js';
 import type { SandboxAllowListReader } from './pipeline/sandbox-allowlist-reader.js';
-import type { ConcurrencyGroupTracker } from './concurrency/group-tracker.js';
 import type { ConcurrencyQueueManager } from './concurrency/queue-manager.js';
 import type { EventRouter } from './events/event-router.js';
 import type { EventStore } from './events/event-store.js';
@@ -85,7 +85,7 @@ import type { EventLogWriter } from './webhook/event-log.js';
 import type { AccessLogWriter } from './audit/access-log.js';
 import type { GenericSourceManager } from './webhook/generic-sources.js';
 import type { TrustStore } from './events/trust-store.js';
-import type { ContextStore } from './contexts/context-store.js';
+import { ContextStore } from './contexts/context-store.js';
 import type { VariableStore } from './contexts/variable-store.js';
 import type { HeldRunStore, ReleaseSignal } from './contexts/held-runs.js';
 import type { StepApprovalBridge } from './approvals/step-approval-bridge.js';
@@ -97,6 +97,13 @@ export interface AppDependencies {
     config: AppConfig;
     db: Kysely<Database>;
     pool: pg.Pool;
+    /**
+     * The orchestrator master key in both forms, plus the previous generation
+     * during a rotation grace window. Null when no master key is configured.
+     * Resolved once at boot so every master-key-wrapped store agrees, and so
+     * `KICI_SECRET_KEY_FILE` reaches all of them.
+     */
+    masterKeys?: ResolvedMasterKeys | null;
     /** Fleet-wide settings reader; threads the live lock-file cap into warm-added universal-git sources. */
     clusterSettings?: ClusterSettingsReader;
     registry: AgentRegistry;
@@ -187,7 +194,7 @@ export interface AppDependencies {
      * the composition root / server hooks. Type-only reference — the runtime
      * module never enters the shipped bundle.
      */
-    faultInjection?: import('./testing/fault-injection.js').OrchestratorFaultInjection;
+    faultInjection?: import('./fault-injection-types.js').OrchestratorFaultInjection;
     /**
      * Orchestrator-owned provenance signing (Phase 1 root of trust). Present when
      * `KICI_ORCHESTRATOR_PROVENANCE_ISSUER` is configured: the orchestrator mints +
@@ -336,8 +343,6 @@ export interface AppDependencies {
     coldStore?: ColdStore | null;
     /** Read + mutation attribution log writer. Optional -- if not set, access_log rows are not written. */
     accessLogWriter?: AccessLogWriter;
-    /** In-memory concurrency group tracker. Optional -- if not set, concurrency is disabled. */
-    concurrencyTracker?: ConcurrencyGroupTracker;
     /** DB-backed concurrency queue manager. Optional -- if not set, concurrency is disabled. */
     concurrencyQueueManager?: ConcurrencyQueueManager;
     /** Optional callback when agent sends encrypted secret outputs on job success. */

package/dist/cache/dep-cache.d.ts

@@ -27,14 +27,22 @@ import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.j
 export declare function depTarballKey(depsHash: string, platform: string, arch: string): string;
 /**
  * Cache key for the pointer that resolves a lockfile to the content hash of the
- * tarball built from it: `deps/{platform}-{arch}/{lockfileHash}.hash`.
+ * tarball built from it: `deps/{platform}-{arch}/{lockfileHash}.hash`, or
+ * `…/{lockfileHash}-{siblingsDigest}.hash` when the lock carries one.
+ *
+ * `siblingsDigest` is part of the key because the deps tarball carries in-repo
+ * `workspace:` sibling directories WITH their built output, and editing a
+ * sibling's source moves no package-manager lock file — so a `lockfileHash`-only
+ * key restored the sibling's stale build over the fresh clone on every run. A
+ * lock with no in-repo sibling carries no digest and keys exactly as before, so
+ * existing pointers stay live.
  *
  * This is the one mutable object in the scheme. A concurrent write replaces a
  * pointer wholesale — there is no window in which it is half-written — so the
  * worst a racing pair of builders can do is leave whichever pointer landed last,
  * and both tarballs remain valid and fetchable.
  */
-export declare function depPointerKey(lockfileHash: string, platform: string, arch: string): string;
+export declare function depPointerKey(lockfileHash: string, platform: string, arch: string, siblingsDigest?: string): string;
 export declare class DepCache {
     private readonly storage;
     private readonly maxTarballBytes;
@@ -67,19 +75,19 @@ export declare class DepCache {
      * Both halves must be present: a pointer whose tarball has aged out is a miss,
      * not a hit, or the caller skips a rebuild and dispatches a URL that 404s.
      */
-    has(lockfileHash: string, platform: string, arch: string): Promise<boolean>;
+    has(lockfileHash: string, platform: string, arch: string, siblingsDigest?: string): Promise<boolean>;
     /**
      * Get a pre-signed download URL for the dep tarball (for agent delivery).
      * Refreshes TTL on hit.
      */
-    getUrl(lockfileHash: string, platform: string, arch: string): Promise<string | null>;
+    getUrl(lockfileHash: string, platform: string, arch: string, siblingsDigest?: string): Promise<string | null>;
     /**
      * Get a pre-signed download URL and the tarball content hash.
      *
      * Returns null on cache miss — including an entry with no pointer, which is
      * unverifiable and so is deliberately not served.
      */
-    getUrlAndHash(lockfileHash: string, platform: string, arch: string): Promise<{
+    getUrlAndHash(lockfileHash: string, platform: string, arch: string, siblingsDigest?: string): Promise<{
         url: string;
         hash?: string;
     } | null>;
@@ -93,7 +101,7 @@ export declare class DepCache {
      * Called only after the agent confirms its upload completed. Publishing before
      * the bytes land would let a reader follow the pointer to a missing object.
      */
-    publishPointer(lockfileHash: string, platform: string, arch: string, depsHash: string): Promise<void>;
+    publishPointer(lockfileHash: string, platform: string, arch: string, depsHash: string, siblingsDigest?: string): Promise<void>;
     /**
      * Store a dep tarball in cache.
      * Throws if tarball exceeds max size (per user decision).

package/dist/cache/global-eval-round-cache.d.ts

@@ -42,7 +42,7 @@ import type { GlobalEvalRoundResult } from '@kici-dev/engine';
  * than a wrong verdict.
  *
  * Returns `null` when the inputs cannot be serialized. A round that cannot be
- * keyed is simply not cached — never keyed on a partial input, which is how a
+ * keyed is not cached — never keyed on a partial input, which is how a
  * cache starts answering questions it was not asked.
  */
 export declare function globalEvalRoundCacheKey(args: {

package/dist/cache/index.d.ts

@@ -2,7 +2,7 @@
  * Cache module for customer-workflow source tarballs and dependency tarballs.
  *
  * SourceCache: content-hash-keyed storage for `.kici/` source tarballs
- *   (`source/{contentHash}.tar.gz`).
+ *   (`source/v2/{orgId}/{sourceTarDigest}.tar.gz`).
  * BuildCoordinator: deduplicates concurrent build requests.
  * DepCache: lockfileHash-keyed storage for dependency tarballs.
  */

package/dist/cache/source-cache.d.ts

@@ -1,30 +1,107 @@
 /**
  * Source-tarball cache layer wrapping CacheStorage.
  *
- * Provides content-hash-keyed operations for customer-workflow source tarballs
- * (the `.kici/` directory minus `node_modules/`, packed by the agent's build
- * job via `source-packer.ts`). Tarballs are platform-independent: raw TypeScript
- * source files are identical across architectures. Refreshes TTL on reads
+ * Stores customer-workflow source tarballs — the `.kici/` directory minus
+ * `node_modules/`, packed by the agent's build job via `source-packer.ts` —
+ * under the tarball's OWN content hash, with a small pointer resolving an org's
+ * workflow `contentHash` to that hash. Tarballs are platform-independent: raw
+ * TypeScript source is identical across architectures. Refreshes TTL on reads
  * (touch-on-read) so actively used sources stay in cache longer.
  *
- * Cache key format: source/{contentHash}.tar.gz
+ * Keys: `source/v2/{orgId}/{sourceTarDigest}.tar.gz` (immutable) and
+ * `source/v2/{orgId}/{contentHash}.hash` (the pointer) — the shape `dep-cache.ts`
+ * already uses, for the same reasons — content addressing and org scoping.
  *
- * Replaces the former `BundleCache` / `bundles/{contentHash}.js` layout that
- * stored Rolldown-bundled `.compiled.mjs` artifacts — the new execution path
- * downloads the source tarball, extracts it, and imports the workflow entry
- * via the shared oxc-transform ESM loader hook.
+ * **Content addressing** makes a mismatched pair unrepresentable. At
+ * `source/{contentHash}.tar.gz` the object's key was derived from something
+ * other than its bytes, so nothing a reader fetched could be verified against
+ * anything, and the agent restored whatever the signed URL returned.
+ *
+ * **Org scoping** makes the key a tenant's key. `contentHash` alone is a
+ * property of a `.kici/` tree, so two repositories scaffolded from the same
+ * `kici init` template — or a fork and its upstream — produced one object, and
+ * whichever built first won for the life of that object. On an orchestrator
+ * serving more than one org that is a cross-tenant read of `.kici/` contents.
+ * The org is the tenant boundary and the segment the user cache already scopes
+ * by; inside one org, sharing a key now means the two trees are byte-identical,
+ * so a monorepo or a same-org fork keeps its warm cache and pays nothing.
+ *
+ * Legacy `source/{contentHash}.tar.gz` objects are **not read**. Only a
+ * `compileSchemaVersion: 5` lock can name one, and that `contentHash` covers
+ * the workflow entry file alone rather than the tree the tarball holds — the
+ * defect this layout closes — so serving it would serve the bug. For a v6 lock
+ * the fallback is unreachable anyway, since the schema bump moves every hash.
+ * Old objects are orphaned and aged out by the normal cache TTL.
  */
 import type { CacheStorage } from '../storage/types.js';
+/**
+ * Cache key for a source tarball, addressed by the tarball's OWN content hash:
+ * `source/v2/{orgId}/{sourceTarDigest}.tar.gz`.
+ *
+ * Naming the object by its own hash means every pair a reader can observe is
+ * self-consistent, and the bytes at a given key can never change after a URL is
+ * signed for it.
+ */
+export declare function sourceTarballKey(orgId: string, sourceTarDigest: string): string;
+/**
+ * Cache key for the pointer that resolves a workflow `contentHash` to the
+ * content hash of the tarball built from it: `source/v2/{orgId}/{contentHash}.hash`.
+ *
+ * This is the one mutable object in the scheme. A concurrent write replaces a
+ * pointer wholesale — there is no half-written window — so the worst a racing
+ * pair of builders can do is leave whichever pointer landed last, and both
+ * tarballs stay valid and fetchable.
+ */
+export declare function sourcePointerKey(orgId: string, contentHash: string): string;
 export declare class SourceCache {
     private readonly storage;
     constructor(options: {
         storage: CacheStorage;
     });
-    has(contentHash: string): Promise<boolean>;
-    get(contentHash: string): Promise<Buffer | null>;
-    getUrl(contentHash: string): Promise<string | null>;
-    getUploadUrl(contentHash: string): Promise<string>;
-    store(contentHash: string, tarball: Buffer | string): Promise<void>;
-    remove(contentHash: string): Promise<boolean>;
+    /** The single place the pointer indirection is read. */
+    private resolvePointer;
+    /**
+     * Whether a source tarball for this org's `contentHash` exists.
+     *
+     * Both halves must be present: a pointer whose tarball has aged out is a
+     * miss, not a hit, or the caller skips a rebuild and dispatches a URL that
+     * 404s on the agent.
+     */
+    has(orgId: string, contentHash: string): Promise<boolean>;
+    get(orgId: string, contentHash: string): Promise<Buffer | null>;
+    /**
+     * A pre-signed download URL and the tarball's own digest.
+     *
+     * The pointer is resolved FIRST: it names the content hash, which is also the
+     * tarball's key, so the url and the digest returned here are read from a
+     * single source of truth and cannot disagree. Returns null on a miss —
+     * including a pointer with no tarball behind it, which is unverifiable and so
+     * is deliberately not served.
+     */
+    getUrlAndDigest(orgId: string, contentHash: string): Promise<{
+        url: string;
+        digest: string;
+    } | null>;
+    getUrl(orgId: string, contentHash: string): Promise<string | null>;
+    /**
+     * A pre-signed upload URL for direct agent-to-storage upload.
+     *
+     * Takes the tarball's OWN digest, not the workflow `contentHash`: the agent
+     * computes it before asking for a URL, and the object is named by it.
+     */
+    getUploadUrl(orgId: string, sourceTarDigest: string): Promise<string>;
+    /**
+     * Publish the pointer that makes an uploaded tarball discoverable by
+     * `contentHash`. Called only after the agent confirms its upload completed —
+     * publishing before the bytes land would let a reader follow the pointer to a
+     * missing object.
+     */
+    publishPointer(orgId: string, contentHash: string, sourceTarDigest: string): Promise<void>;
+    /** Store a tarball and the pointer that names it. */
+    store(orgId: string, contentHash: string, tarball: Buffer | string): Promise<void>;
+    /** SHA-256 of a tarball's bytes. The agent computes the same value. */
+    static computeDigest(data: Buffer): string;
+    /** Remove a source tarball and its pointer. */
+    remove(orgId: string, contentHash: string): Promise<boolean>;
 }
 //# sourceMappingURL=source-cache.d.ts.map

package/dist/cancel/cancel-run.d.ts

@@ -18,15 +18,26 @@
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 import type { JobQueue } from '../queue/job-queue.js';
-import type { Dispatcher } from '../agent/dispatcher.js';
 import type { AgentRegistry } from '../agent/registry.js';
 import type { ExecutionTracker } from '../reporting/execution-tracker.js';
 export interface CancelRunDeps {
     db: Kysely<Database>;
     jobQueue: JobQueue;
-    dispatcher: Dispatcher;
     registry: AgentRegistry;
     executionTracker: ExecutionTracker;
+    /**
+     * This coordinator's instance id. Undefined ⇒ no cluster identity, so every
+     * job resolves to "mine" and the behaviour is the single-coordinator one.
+     */
+    instanceId?: string;
+    /**
+     * Forward `job.cancel` to the coordinator holding the agent's socket.
+     * Returns whether the send left this process. Undefined ⇒ no peer transport,
+     * so a job owned by a sibling is unreachable rather than orphaned.
+     */
+    cancelJobOnPeer?: (peerId: string, runId: string, jobId: string, reason: string) => boolean;
+    /** How stale a `cluster_instances` heartbeat may be and still read as live. */
+    ownershipGraceMs?: number;
 }
 export interface CancelRunOptions {
     /** Force immediate SIGKILL on the agent (skip graceful hooks). */
@@ -39,6 +50,13 @@ export interface CancelRunOptions {
 export interface CancelRunResult {
     /** Number of `job.cancel` messages dispatched to agents (running jobs). */
     agentsNotified: number;
+    /**
+     * Jobs whose owning coordinator is alive but could not be reached — the peer
+     * send failed, or there is no peer transport. They are NOT notified and they
+     * are NOT orphaned: the job may well still be running, so the run stays
+     * `cancelling` and the re-drive sweep tries again.
+     */
+    unreachable: number;
     /** Number of pending/queued execution_jobs rows marked cancelled. */
     pendingCancelled: number;
     /**

package/dist/cancel/sweep-stuck-cancelling.d.ts

@@ -0,0 +1,30 @@
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+export interface SweepStuckCancellingDeps {
+    db: Kysely<Database>;
+    /** The canonical cancel path, bound to this orchestrator's deps. */
+    cancelRun: (runId: string, reason: string) => Promise<unknown>;
+    /**
+     * How long a run may sit in `cancelling` before it is re-driven. Twice the
+     * recovery grace period: long enough that a graceful agent teardown finishes
+     * on its own, short enough that a dropped cancel does not sit for an hour.
+     */
+    stuckAfterMs: number;
+}
+/**
+ * Re-drive the cancel for runs left in `cancelling`.
+ *
+ * A cancel reaches its target over the peer channel, and a peer blip drops it.
+ * The run then sits in `cancelling` with its jobs still executing, and nothing
+ * retries: the workflow-deadline detector re-scans `cancelling` runs, but only
+ * ones that HAVE a workflow timeout, so a run without one waits forever. This
+ * extends the same coverage to every run.
+ *
+ * Leader-gated by its caller, and idempotent by construction — re-driving a
+ * cancel that already landed re-sends `job.cancel` to an agent that is already
+ * unwinding, and re-driving one whose jobs are all terminal completes the run.
+ *
+ * @returns how many runs were re-driven.
+ */
+export declare function sweepStuckCancelling(deps: SweepStuckCancellingDeps): Promise<number>;
+//# sourceMappingURL=sweep-stuck-cancelling.d.ts.map

package/dist/cli/api-client.d.ts

@@ -199,6 +199,14 @@ export declare class AdminApiClient {
         skippedConfigs: number;
         reEncryptedBackends: number;
         skippedBackends: number;
+        reEncryptedSigningKeys: number;
+        skippedSigningKeys: number;
+        reEncryptedDashboardKeys: number;
+        skippedDashboardKeys: number;
+        reEncryptedEphemeralKeys: number;
+        skippedEphemeralKeys: number;
+        reEncryptedSecretOutputs: number;
+        skippedSecretOutputs: number;
     }>;
     createGenericSource(data: {
         orgId: string;
@@ -274,6 +282,8 @@ export declare class AdminApiClient {
         routingKey?: string;
         /** Optional absolute expiry as an ISO datetime string. */
         expiresAt?: string;
+        /** Optional intended holder (an OIDC `sub` or an email). Advisory only. */
+        subject?: string;
     }): Promise<{
         token: string;
         id: string;

package/dist/cli/commands/agent-service/logs.d.ts

@@ -5,7 +5,7 @@
  * and output format. Delegates to the platform-specific service
  * manager's logs() method.
  *
- * Target resolution goes through resolveInstance — the same priority chain
+ * Target resolution goes through resolveInstanceTarget — the same priority chain
  * every lifecycle command uses (--instance-dir > --name > CWD manifest >
  * refusal with candidate list).
  */

package/dist/cli/commands/agent-service/restart.d.ts

@@ -2,7 +2,7 @@
  * `kici-admin agent restart` command.
  *
  * Restarts the agent service. Target resolution goes through
- * resolveInstance — the same priority chain every lifecycle command uses
+ * resolveInstanceTarget — the same priority chain every lifecycle command uses
  * (--instance-dir > --name > CWD manifest > refusal with candidate list).
  * `--name` has no default; every invocation must resolve via one of the
  * above paths.

package/dist/cli/commands/agent-service/start.d.ts

@@ -2,7 +2,7 @@
  * `kici-admin agent start` command.
  *
  * Starts the agent service. Target resolution goes through
- * resolveInstance — the same priority chain every lifecycle command uses
+ * resolveInstanceTarget — the same priority chain every lifecycle command uses
  * (--instance-dir > --name > CWD manifest > refusal with candidate list).
  * `--name` has no default; every invocation must resolve via one of the
  * above paths.

package/dist/cli/commands/agent-service/status.d.ts

@@ -5,7 +5,7 @@
  * the running agent's health API for agent-specific info (labels,
  * orchestrator connection, current job).
  *
- * Target resolution goes through resolveInstance — the same priority chain
+ * Target resolution goes through resolveInstanceTarget — the same priority chain
  * every lifecycle command uses (--instance-dir > --name > CWD manifest >
  * refusal with candidate list).
  */

package/dist/cli/commands/agent-service/stop.d.ts

@@ -2,7 +2,7 @@
  * `kici-admin agent stop` command.
  *
  * Stops the agent service. Target resolution goes through
- * resolveInstance — the same priority chain every lifecycle command uses
+ * resolveInstanceTarget — the same priority chain every lifecycle command uses
  * (--instance-dir > --name > CWD manifest > refusal with candidate list).
  * `--name` has no default; every invocation must resolve via one of the
  * above paths.

package/dist/cli/commands/agent-service/uninstall.d.ts

@@ -2,7 +2,7 @@
  * `kici-admin agent uninstall` command.
  *
  * Removes the agent service registration. Target resolution goes
- * through resolveInstance — the same priority chain every lifecycle command
+ * through resolveInstanceTarget — the same priority chain every lifecycle command
  * uses (--instance-dir > --name > CWD manifest > refusal with candidate list).
  * The manifest stays on disk so subsequent commands can still reference the
  * deploy folder; the host-wide index entry is dropped unconditionally so a

package/dist/cli/commands/agent.d.ts

@@ -6,5 +6,30 @@
  */
 import type { Command } from 'commander';
 import type { AdminApiClient } from '../api-client.js';
+/**
+ * The authorized label scope a static agent token is minted with.
+ *
+ * Beyond the operator's own `--labels` set, the authorized scope carries:
+ *
+ * - A mandatory (taint) label must also be advertisable (the selector side),
+ *   so union it into the authorized labels — a taint the agent can't even
+ *   advertise would be incoherent.
+ * - Every real agent derives its `kici:role:*` labels from its own host and
+ *   advertises them at registration, so a scope omitting them can never admit
+ *   the agent that presents it: the orchestrator's register-time scope gate
+ *   closes the connection with `Agent labels exceed token-bound scope`. A role
+ *   the agent can't even advertise is incoherent for the same reason a taint
+ *   is. The mint site cannot know which roles the target host will derive — the
+ *   agent discovers them from its own capabilities — so the whole role
+ *   namespace is authorized and the agent advertises the subset it has.
+ *
+ * The operator's own labels still bound everything else: a token minted with
+ * `--labels linux,x64` still refuses an agent advertising `gpu`. An agent that
+ * must carry no roles at all sets `KICI_ROLES=` on its own side.
+ *
+ * Returns undefined when the operator supplied no labels and no taint — an
+ * unscoped token, which the register-time gate skips entirely.
+ */
+export declare function authorizedAgentTokenLabels(baseLabels: readonly string[], mandatoryLabels: readonly string[] | undefined): string[] | undefined;
 export declare function registerAgentCommands(program: Command, getClient: () => AdminApiClient): void;
 //# sourceMappingURL=agent.d.ts.map

package/dist/cli/commands/config.d.ts

@@ -12,5 +12,21 @@
  */
 import type { Command } from 'commander';
 import type { AdminApiClient } from '../api-client.js';
+/**
+ * Whether a shared-config path is one a joining orchestrator receives.
+ *
+ * True for the consumed paths themselves and for anything below them, so
+ * `storage.bucket` counts and `storageQuota` does not.
+ */
+export declare function isConsumedSharedConfigPath(path: string): boolean;
+/**
+ * Warning text for a `config set` on a path nothing reads back, or null when
+ * the path is consumed.
+ *
+ * The write itself succeeds and is versioned — the operator is told which
+ * mechanism actually changes a running orchestrator, so a stored-but-inert
+ * value is not mistaken for an applied one.
+ */
+export declare function sharedConfigSetWarning(path: string): string | null;
 export declare function registerConfigCommands(program: Command, getClient: () => AdminApiClient): void;
 //# sourceMappingURL=config.d.ts.map