@kici-dev/orchestrator 0.5.0 → 0.6.0

package/dist/cluster/cluster-identity.d.ts

@@ -23,9 +23,9 @@ export interface ClusterIdentityDeps {
      */
     storagePrefix?: string;
     /**
-     * E2E escape hatch — when true, validateS3Sentinel logs a warning and
-     * returns early. Mirrors the orchestrator's
-     * `config.skipS3SentinelValidation` (KICI_SKIP_S3_SENTINEL_VALIDATION).
+     * Test-only escape hatch — when true, validateS3Sentinel logs a warning and
+     * returns early. Supplied only by the build-time test double's injected
+     * fault-injection policy; the shipped orchestrator always validates.
      */
     skipSentinelValidation?: boolean;
 }

package/dist/cluster/cluster-settings-reader.d.ts

@@ -1,7 +1,9 @@
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 /** Numeric columns on cluster_settings readable via {@link ClusterSettingsReader}. */
-export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms' | 'contributor_cache_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'check_run_tracking_ttl_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms';
+export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms'
+/** @deprecated Readable, but no call site reads it. Removed at v1.0.0. */
+ | 'contributor_cache_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'check_run_tracking_ttl_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
 /** Text columns on cluster_settings readable via {@link ClusterSettingsReader}. */
 export type ClusterStringColumn = 'dashboard_verified_issuer';
 /** Boolean columns on cluster_settings readable via {@link ClusterSettingsReader}. */
@@ -39,6 +41,10 @@ export declare const CACHE_MAX_ENTRIES_CEILING = 100000;
  * A value that is not a usable positive count (NaN, non-finite, below 1) falls
  * back to the configured default rather than clamping to 1: a 1-entry cache
  * thrashes silently, which is harder to diagnose than simply ignoring garbage.
+ * The fallback is floored and bounded by the ceiling on the same path as the
+ * stored value, so neither a fractional nor an above-ceiling configured default
+ * can reach the LRU constructor as a non-integer or unclamped max — the
+ * constructor rejects both, and either would prevent boot.
  */
 export declare function clampCacheMaxEntries(value: number, fallback: number): number;
 /**

package/dist/cluster/coordinator.d.ts

@@ -90,6 +90,8 @@ export interface RunCoordinatorDeps {
     sendAndWaitAckViaHandler?: (targetInstanceId: string, msg: JobReroute, timeoutMs: number) => Promise<boolean>;
     /** Fallback: send a message via server-side (incoming) peer connection (fire-and-forget). */
     sendToPeerViaHandler?: (targetInstanceId: string, msg: PeerToPeerMessage) => boolean;
+    /** Close a peer's incoming socket by instanceId (used by worker eviction). */
+    closePeerViaHandler?: (targetInstanceId: string, code: number, reason: string) => void;
     ackTimeoutMs?: number;
     /** Stale peer timeout in ms. Default: 60000 (60s). */
     staleTimeoutMs?: number;
@@ -137,7 +139,23 @@ export declare class RunCoordinator {
     private readonly nakTracker;
     /** Stale eviction timer handle. */
     private staleEvictionTimer;
+    /** Plan-ceiling worker eviction (drain-then-disconnect). */
+    private readonly workerEviction;
+    /** Per-peer "no in-flight rerouted jobs" callbacks, fired by worker drain. */
+    private readonly evictionIdleCallbacks;
+    private readonly closePeerViaHandler?;
     constructor(deps: RunCoordinatorDeps);
+    /** In-flight rerouted jobs currently attributed to a peer. */
+    private runningJobCountForPeer;
+    /** Fire a registered drain idle-callback once the peer has no in-flight jobs. */
+    private fireIdleIfDrained;
+    /**
+     * Reconcile this coordinator's workers against the pushed ceiling. Invoked on
+     * every `plan.headroom` frame: when `evictExcess` is set, the newest workers
+     * past `ceiling` are drained; otherwise any in-flight drain is cancelled and
+     * the worker restored, so a raised ceiling rescues it.
+     */
+    reconcileWorkerEviction(ceiling: number, evictExcess: boolean): void;
     /**
      * Route jobs for a webhook run. Claims jobs that local agents can handle,
      * reroutes the rest to peers with matching capacity.
@@ -268,6 +286,32 @@ export declare class RunCoordinator {
      * its worker is connected.
      */
     private trackReroutedJob;
+    /** Arm a spawn-window / reap timer that fires {@link handleRerouteSpawnTimeout}. */
+    private armRerouteWindow;
+    /**
+     * Whether the reroute target has actually started the job, read from the
+     * shared `execution_jobs` row.
+     *
+     * A read failure answers "not running" so a transient database fault cannot
+     * silently disable the backstop — the re-dispatch it then performs is the
+     * behavior that shipped before this check existed, and it is idempotent.
+     * Only the FIRST observation goes through this path: once `peerStarted`
+     * latches, a read fault can no longer bounce a running job.
+     */
+    private rerouteTargetIsRunning;
+    /**
+     * The peer owns the job, so the backstop must not re-dispatch it. Release the
+     * tracking entry once the shared row is terminal, otherwise re-arm the window
+     * as a reap poll.
+     *
+     * The entry cannot simply be dropped here: cancel propagation still needs it
+     * while the job runs. It also cannot simply be kept, because a peer
+     * COORDINATOR relays no terminal `job.progress` — the shared row IS its
+     * report — so nothing else would ever release it and `reroutedJobs` would
+     * grow for the lifetime of the process. A read fault re-arms (never releases),
+     * so a transient fault costs one more poll rather than tracking.
+     */
+    private reapOrDeferRerouteTracking;
     /**
      * Post-ACK spawn-window backstop. Fires when a rerouted job's peer accepted
      * but produced no progress within the window (Layer A), or immediately when a

package/dist/cluster/index.d.ts

@@ -21,6 +21,10 @@ export type { RaftPersistentState } from './raft-state.js';
 export { OrphanRecovery } from './orphan-recovery.js';
 export { RunCoordinator } from './coordinator.js';
 export type { RunContext, JobToRoute, RouteResult } from './coordinator.js';
+export { PlanHeadroomStore } from './plan-headroom-store.js';
+export type { StoredPlanHeadroom } from './plan-headroom-store.js';
+export { WorkerEviction, selectEvictionTargets, WORKER_DRAIN_TIMEOUT_MS, } from './worker-eviction.js';
+export type { EvictionHooks } from './worker-eviction.js';
 export { createClusterHealthRoutes } from './health-api.js';
 export type { ClusterHealthRoutesDeps } from './health-api.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/cluster/peer-handler.d.ts

@@ -33,6 +33,14 @@ export interface PeerHandlerDeps {
     peerRegistry: PeerRegistry;
     /** Callback to get this orchestrator's local agent inventory for heartbeats. */
     getLocalInventory: () => Omit<PeerHeartbeat, 'type'>;
+    /**
+     * The Platform-pushed worker ceiling this coordinator enforces, or `null`
+     * when none was ever received (admit worker joins freely). Read at each
+     * worker-peer admission. Served from the persisted PlanHeadroomStore, so it
+     * survives a Platform disconnect — the coordinator keeps enforcing the last
+     * known ceiling rather than resetting to unlimited.
+     */
+    getWorkerCeiling?: () => Promise<number | null>;
     /** Heartbeat interval in ms. Default: 30000 (30s). */
     heartbeatIntervalMs?: number;
     /** Auth timeout in ms. Default: 15000 (15s). */
@@ -103,9 +111,20 @@ export interface PeerHandlerDeps {
  * Returns `handleConnection(ws, remoteIp?)` which should be called when a new
  * WebSocket connection is upgraded on the peer endpoint.
  */
+/**
+ * Whether a joining peer may be admitted against the plan ceiling.
+ *
+ * Only workers are gated: a coordinator peer holds its own Platform connection
+ * and is counted there, and a peer advertising no role registers as a
+ * coordinator (see PeerRegistry.addPeer), so neither is gated here. A `null`
+ * ceiling — none ever received — admits freely. The ceiling is ABSOLUTE, so the
+ * join is admitted only while the currently-connected worker count is below it.
+ */
+export declare function shouldAdmitWorker(role: string | undefined, ceiling: number | null, connectedWorkerCount: number): boolean;
 export declare function createPeerHandler(deps: PeerHandlerDeps): {
     handleConnection: (ws: PeerWsLike, remoteIp?: string) => void;
     sendToPeer: (targetInstanceId: string, msg: PeerToPeerMessage) => boolean;
+    closePeer: (targetInstanceId: string, code: number, reason: string) => void;
     sendAndWaitAck: (targetInstanceId: string, msg: JobReroute, timeoutMs?: number) => Promise<boolean>;
     sendConfigReloadAndWait: (targetInstanceId: string, msg: PeerConfigReload, timeoutMs?: number) => Promise<PeerConfigReloadResponse | null>;
     sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;

package/dist/cluster/peer-registry.d.ts

@@ -35,6 +35,8 @@ export interface PeerInfo {
     routingKeys: string[];
     connected: boolean;
     lastHeartbeatAt: number;
+    /** When this peer first connected. Distinct from `lastHeartbeatAt`, which moves; this does not. */
+    connectedAt: number;
     agents: PeerAgentInfo[];
     draining: boolean;
     capabilities: PeerCapabilities;
@@ -71,6 +73,12 @@ export interface PeerRegistryOptions {
     onClusterSettingsVersionBehind?: (peerVersion: number) => void;
     /** Called when a peer transitions to disconnected state. */
     onPeerDisconnected?: (instanceId: string) => void;
+    /**
+     * Called when the connected-peer set changes (a fresh add or a
+     * markDisconnected), so a coordinator can re-report its `cluster.membership`
+     * snapshot to the Platform without waiting for the periodic interval.
+     */
+    onMembershipChange?: () => void;
 }
 export declare class PeerRegistry {
     private readonly peers;
@@ -81,6 +89,7 @@ export declare class PeerRegistry {
     private readonly onRegistryVersionBehind?;
     private readonly onClusterSettingsVersionBehind?;
     private readonly onPeerDisconnected?;
+    private readonly onMembershipChange?;
     constructor(options?: PeerRegistryOptions);
     /**
      * Update the local config version for comparison with peer heartbeats.
@@ -196,6 +205,14 @@ export declare class PeerRegistry {
      * Get all peers with role=worker.
      */
     getWorkerPeers(): PeerInfo[];
+    /**
+     * Currently-connected worker peers.
+     *
+     * A peer that advertises no role registers as a coordinator (see `addPeer`),
+     * so a worker predating the role field is not counted here. That under-counts,
+     * which errs permissive, and it self-heals as workers upgrade.
+     */
+    getConnectedWorkerPeers(): PeerInfo[];
     /**
      * Get all peers with role=coordinator.
      */

package/dist/cluster/plan-headroom-store.d.ts

@@ -0,0 +1,26 @@
+import type { Kysely } from 'kysely';
+import type { PlanHeadroom } from '@kici-dev/engine';
+import type { Database } from '../db/types.js';
+/** The ceiling as last pushed by the Platform. */
+export interface StoredPlanHeadroom {
+    maxWorkerPeers: number;
+    orgLimit: number;
+    orgTotal: number;
+    evictExcess: boolean;
+    updatedAt: Date;
+}
+/**
+ * The orchestrator's cache of the Platform-owned worker ceiling.
+ *
+ * Persisted so a coordinator restarting during a Platform outage keeps
+ * enforcing the last known ceiling instead of resetting to unlimited. Same
+ * ownership shape as `org_trust_policy`: the Platform writes it, the
+ * orchestrator only reads it back.
+ */
+export declare class PlanHeadroomStore {
+    private readonly db;
+    constructor(db: Kysely<Database>);
+    read(): Promise<StoredPlanHeadroom | null>;
+    write(headroom: PlanHeadroom): Promise<void>;
+}
+//# sourceMappingURL=plan-headroom-store.d.ts.map

package/dist/cluster/worker-eviction.d.ts

@@ -0,0 +1,69 @@
+import type { PeerInfo, PeerRegistry } from './peer-registry.js';
+/** How long a draining worker may hold in-flight jobs before it is disconnected anyway. */
+export declare const WORKER_DRAIN_TIMEOUT_MS: number;
+/**
+ * The coordinator-side operations eviction needs.
+ *
+ * Declared here rather than reached for on the coordinator directly: the
+ * coordinator owns dispatch and job attribution, but exposes no ineligibility
+ * or idle-notification surface, and widening its public API for one caller
+ * would couple eviction to it. A fake implementation makes this class unit
+ * testable without a running cluster.
+ */
+export interface EvictionHooks {
+    /** Stop rerouting new jobs to this peer. */
+    markIneligible(instanceId: string): void;
+    /**
+     * Restore this peer to dispatch eligibility and drop any idle-notification
+     * registration. Called when a drain is cancelled (the ceiling rose so the
+     * peer is no longer excess) and after a drained peer is disconnected, so a
+     * later re-admission of the same instance starts dispatch-eligible.
+     */
+    clearIneligible(instanceId: string): void;
+    /** In-flight rerouted jobs currently attributed to this peer. */
+    runningJobCount(instanceId: string): number;
+    /** Invoke the callback once this peer has no in-flight rerouted jobs. */
+    onPeerIdle(instanceId: string, callback: () => void): void;
+    /** Close this peer's connection with the given code and reason. */
+    closePeer(instanceId: string, code: number, reason: string): void;
+}
+/**
+ * The workers to drain to bring this coordinator down to its ceiling.
+ *
+ * Newest-first by connect time: the joins that crossed the line are the ones
+ * that go, so long-running capacity is not shuffled arbitrarily.
+ */
+export declare function selectEvictionTargets(workers: readonly PeerInfo[], ceiling: number): PeerInfo[];
+/**
+ * Drain-then-disconnect this coordinator's excess workers.
+ *
+ * A target is first marked ineligible for new dispatch, then closed once its
+ * in-flight rerouted jobs reach terminal status — which the worker's durable
+ * outbox and `job.progress.ack` already guarantee — or after
+ * `WORKER_DRAIN_TIMEOUT_MS`, after which the coordinator's existing
+ * orphan-recovery guard handles whatever remains.
+ */
+export declare class WorkerEviction {
+    private readonly peerRegistry;
+    private readonly hooks;
+    private readonly logger;
+    private readonly draining;
+    constructor(peerRegistry: PeerRegistry, hooks: EvictionHooks, logger: {
+        warn: (msg: string, meta?: unknown) => void;
+    });
+    /**
+     * Reconcile the connected worker set against the ceiling.
+     *
+     * When `evictExcess` is true, drain the newest workers past `ceiling`;
+     * otherwise the target set is empty. Any drain in flight for a worker that is
+     * no longer a target is CANCELLED — its timer cleared and its dispatch
+     * eligibility restored — so a raised ceiling (or the Platform clearing
+     * `evictExcess` once the org drops back under) rescues a worker instead of
+     * disconnecting it at the drain timeout.
+     */
+    reconcile(ceiling: number, evictExcess: boolean): void;
+    /** Stop an in-flight drain and restore the worker to dispatch eligibility. */
+    private cancelDrain;
+    private startDrain;
+}
+//# sourceMappingURL=worker-eviction.d.ts.map

package/dist/cold-store/load-event-log-range.d.ts

@@ -13,10 +13,17 @@
  * dashboard's payload-detail view continues to resolve archived
  * deliveries identically to hot ones.
  */
-import { type Kysely, type Selectable } from 'kysely';
+import { type Kysely } from 'kysely';
 import { type ColdStore } from '@kici-dev/shared';
-import type { Database, EventLogTable } from '../db/types.js';
-export type EventLogColdStoreRow = Selectable<EventLogTable>;
+import type { Database } from '../db/types.js';
+import type { EventLogColdStoreRow } from './tables/event-log.js';
+/**
+ * Re-exported, not re-declared: the alias is defined beside the cold-store
+ * adapter that owns the table, and a second identical declaration here is what
+ * a homonym sweep flags. Kept as an export so this module's public surface is
+ * unchanged for its consumers.
+ */
+export type { EventLogColdStoreRow };
 export interface LoadOrchEventLogRangeArgs {
     db: Kysely<Database>;
     coldStore: ColdStore | undefined;

package/dist/cold-store/load-secret-audit-log-range.d.ts

@@ -15,10 +15,11 @@
  * rows after streaming. This is correct because cold chunks are
  * partitioned by `(routing_key, day)`, not by the secondary filters.
  */
-import { type Kysely, type Selectable } from 'kysely';
+import { type Kysely } from 'kysely';
 import { type ColdStore } from '@kici-dev/shared';
-import type { Database, SecretAuditLogTable } from '../db/types.js';
-export type SecretAuditLogRow = Selectable<SecretAuditLogTable>;
+import type { Database, SecretAuditLogRow } from '../db/types.js';
+/** Re-exported rather than re-declared — see the note in `load-event-log-range`. */
+export type { SecretAuditLogRow };
 export interface LoadSecretAuditLogRangeArgs {
     db: Kysely<Database>;
     coldStore: ColdStore | undefined;

package/dist/cold-store/tables/secret-audit-log.d.ts

@@ -13,10 +13,11 @@
  * synthetic prefix `__orchestrator__`. Mirrors the same pattern already
  * used for `event_log` payload keys when the routing key is missing.
  */
-import { type Kysely, type Selectable } from 'kysely';
+import { type Kysely } from 'kysely';
 import { type ChunkCommitMetadata, type ColdRetention, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter } from '@kici-dev/shared';
-import type { Database, SecretAuditLogTable } from '../../db/types.js';
-export type SecretAuditLogRow = Selectable<SecretAuditLogTable>;
+import type { Database, SecretAuditLogRow } from '../../db/types.js';
+/** Re-exported rather than re-declared — see the note in `load-event-log-range`. */
+export type { SecretAuditLogRow };
 export interface SecretAuditLogAdapterOptions {
     overrides?: Partial<ColdStoreTableConfig>;
 }

package/dist/config.d.ts

@@ -97,6 +97,13 @@ declare const configSchema: z.ZodObject<{
     scalerSpawnTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     backupStalenessWarnHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     scalerPendingSweepIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerReapIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerReapStrandedTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerReapReattemptIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerClaimRetentionMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerProvisionBackoffBaseMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerProvisionBackoffMaxMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerProvisionMaxConsecutiveFailures: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheStorageType: z.ZodOptional<z.ZodEnum<{
         filesystem: "filesystem";
         s3: "s3";
@@ -177,12 +184,6 @@ declare const configSchema: z.ZodObject<{
     eventRouterRetryBaseBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRetryMaxBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRetryScanIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
-    testMode: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
-    testEventFailFirstN: z.ZodOptional<z.ZodString>;
-    testMintDeferAudience: z.ZodOptional<z.ZodString>;
-    testMintRejectAudience: z.ZodOptional<z.ZodString>;
-    testRerunDelayMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
-    testOmitDashboardRequestTypes: z.ZodOptional<z.ZodString>;
     eventLogMaxPayloadBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     maxGithubPayloadBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     lockFileMaxBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -202,7 +203,6 @@ declare const configSchema: z.ZodObject<{
     }>>;
     autoMigrate: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     agentMaxReconnectDelayMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
-    skipS3SentinelValidation: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     otelExporterOtlpEndpoint: z.ZodOptional<z.ZodString>;
     clusterName: z.ZodOptional<z.ZodString>;
     cluster: z.ZodPrefault<z.ZodObject<{
@@ -300,6 +300,13 @@ export declare const packagingConfigSchema: z.ZodObject<{
     scalerSpawnTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     backupStalenessWarnHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     scalerPendingSweepIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerReapIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerReapStrandedTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerReapReattemptIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerClaimRetentionMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerProvisionBackoffBaseMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerProvisionBackoffMaxMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    scalerProvisionMaxConsecutiveFailures: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheStorageType: z.ZodOptional<z.ZodEnum<{
         filesystem: "filesystem";
         s3: "s3";
@@ -380,12 +387,6 @@ export declare const packagingConfigSchema: z.ZodObject<{
     eventRouterRetryBaseBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRetryMaxBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRetryScanIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
-    testMode: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
-    testEventFailFirstN: z.ZodOptional<z.ZodString>;
-    testMintDeferAudience: z.ZodOptional<z.ZodString>;
-    testMintRejectAudience: z.ZodOptional<z.ZodString>;
-    testRerunDelayMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
-    testOmitDashboardRequestTypes: z.ZodOptional<z.ZodString>;
     eventLogMaxPayloadBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     maxGithubPayloadBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     lockFileMaxBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -405,7 +406,6 @@ export declare const packagingConfigSchema: z.ZodObject<{
     }>>;
     autoMigrate: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     agentMaxReconnectDelayMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
-    skipS3SentinelValidation: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     otelExporterOtlpEndpoint: z.ZodOptional<z.ZodString>;
     clusterName: z.ZodOptional<z.ZodString>;
     cluster: z.ZodPrefault<z.ZodObject<{
@@ -533,6 +533,13 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     scalerSpawnTimeoutMs: number;
     backupStalenessWarnHours: number;
     scalerPendingSweepIntervalMs: number;
+    scalerReapIntervalMs: number;
+    scalerReapStrandedTimeoutMs: number;
+    scalerReapReattemptIntervalMs: number;
+    scalerClaimRetentionMs: number;
+    scalerProvisionBackoffBaseMs: number;
+    scalerProvisionBackoffMaxMs: number;
+    scalerProvisionMaxConsecutiveFailures: number;
     cacheStorageType?: "filesystem" | "s3" | undefined;
     cacheStoragePath?: string | undefined;
     cacheStorageS3Bucket?: string | undefined;
@@ -595,12 +602,6 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     eventRouterRetryBaseBackoffMs: number;
     eventRouterRetryMaxBackoffMs: number;
     eventRouterRetryScanIntervalMs: number;
-    testMode: boolean;
-    testEventFailFirstN?: string | undefined;
-    testMintDeferAudience?: string | undefined;
-    testMintRejectAudience?: string | undefined;
-    testRerunDelayMs?: number | undefined;
-    testOmitDashboardRequestTypes?: string | undefined;
     eventLogMaxPayloadBytes: number;
     maxGithubPayloadBytes: number;
     lockFileMaxBytes: number;
@@ -611,7 +612,6 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     nodeEnv: "development" | "production" | "test";
     autoMigrate: boolean;
     agentMaxReconnectDelayMs: number;
-    skipS3SentinelValidation: boolean;
     otelExporterOtlpEndpoint?: string | undefined;
     clusterName?: string | undefined;
     cluster: {

package/dist/contexts/held-runs.d.ts

@@ -3,10 +3,12 @@
  *
  * Manages the lifecycle: pending -> approved/rejected/expired.
  */
-import { Kysely } from 'kysely';
+import { Kysely, type Transaction } from 'kysely';
 import { z } from 'zod';
 import { type ApprovalRequirement, type ApproverClause, type StepApprovalPayload, ApprovalDecision, HoldScope, TriggerSource } from '@kici-dev/engine';
 import type { Database, HeldRun, HeldRunApproval } from '../db/types.js';
+/** A Kysely root handle or an in-flight transaction — query builders accept either. */
+type Executor = Kysely<Database> | Transaction<Database>;
 /** Status values for held runs (held_runs table). */
 export declare enum HeldRunStatus {
     Pending = "pending",
@@ -17,7 +19,25 @@ export declare enum HeldRunStatus {
 }
 /**
  * Reason a run was held in the security queue. Persisted verbatim in
- * `held_runs.reason` and switched on by `buildSecurityHoldSummary`.
+ * `held_runs.reason` and whose vocabulary `buildSecurityHoldSummary` switches
+ * on.
+ *
+ * Each value names why a run sits in the queue. The enum stays whole because
+ * compiling code still names all four: `buildSecurityHoldSummary` renders one
+ * branch per value. Dropping a member breaks that, and breaks the parity test
+ * that pins `SECURITY_HOLD_JOB_IDS`' three policy-reason keys to this enum's
+ * options — the engine constant cannot import the enum, so that coupling lives
+ * only in the orchestrator's test. Nothing validates `held_runs.reason` against
+ * this enum, so a stored value is not rejected anywhere.
+ *
+ * - `fork_pr` — the org trust policy's fork switch held the run.
+ * - `context_trust` — a context's minimum-trust gate held the run.
+ * - `workflow_modification` — deprecated; no longer raised. Modifications to
+ *   `.kici/` are surfaced on their own informational check and no longer feed
+ *   a policy arm. Removed at v1.0.0.
+ * - `unknown_contributor` — deprecated; no longer raised. The policy turns on
+ *   whether the pull request came from a fork, not on who opened it. Removed
+ *   at v1.0.0.
  */
 export declare const SecurityHoldReason: z.ZodEnum<{
     context_trust: "context_trust";
@@ -37,6 +57,20 @@ export interface CreateHeldRunData {
     expiresAt: Date;
     /** Queue type: 'context' (default) or 'security'. */
     queueType?: 'context' | 'security';
+    /**
+     * Granularity of the held element. Omit to leave the column at its `'job'`
+     * default. The org trust policy's PR-wide hold passes `'workflow'`: it fires
+     * before any job is materialized and resumes by rebuilding the whole workflow
+     * dispatch, so `routeRelease` must send it to the workflow resume path.
+     */
+    scope?: HoldScope;
+    /**
+     * What triggered the hold. Omit to leave the column at its `'context'`
+     * default. Written explicitly by the trust-policy hold so its release signal
+     * carries the pair `routeRelease` discriminates on rather than relying on a
+     * column default to supply half of it.
+     */
+    triggerSource?: TriggerSource;
 }
 /**
  * Data required to create a generalized approval hold. Unlike the legacy
@@ -109,14 +143,57 @@ export interface ListHeldRunsOptions {
 export declare class HeldRunStore {
     private readonly db;
     constructor(db: Kysely<Database>);
-    /** Create a new held run with pending status. */
-    create(orgId: string, data: CreateHeldRunData): Promise<HeldRun>;
+    /**
+     * Create a new held run with pending status.
+     *
+     * `exec` is the executor the INSERT runs through, defaulting to the store's
+     * own connection. A caller that writes something the row cannot exist without
+     * — the job's pending dispatch context, without which the hold can never be
+     * resumed — passes its enclosing transaction, so the two land or roll back
+     * together. Handed in rather than taken from an ambient scope: Kysely has no
+     * such scope, so a `this.db` insert inside a `db.transaction()` callback runs
+     * on a different connection and commits on its own.
+     */
+    create(orgId: string, data: CreateHeldRunData, exec?: Executor): Promise<HeldRun>;
     /**
      * Create a generalized approval hold (workflow/job/step scope, explicit or
      * context trigger) carrying a normalized `ApprovalRequirement`. Returns
      * the created row.
+     *
+     * `exec` carries the same meaning it does on {@link create}.
+     */
+    createHold(orgId: string, data: CreateHoldData, exec?: Executor): Promise<HeldRun>;
+    /**
+     * Record that the pending `KiCI Security` check reached the provider, for
+     * every hold that gates it, so the settle that ends one knows it has a check
+     * to terminalize.
+     *
+     * Written AFTER the post returns, never before. The two orders fail
+     * differently and the failures are not equivalent: recording first and dying
+     * before the post leaves a row claiming a check the commit does not have, and
+     * the settle then CREATES one — a completed `KiCI Security` run appearing on a
+     * commit nothing ever held. Recording second leaves the opposite residue, a
+     * real pending check the settle declines to close, which is the same stuck
+     * check the fire-and-forget post could already produce. A fabricated failing
+     * check on a pull request is worse than a stuck one, so the write goes last.
+     *
+     * **One statement for all of them, not one per hold.** A commit's check run is
+     * shared by every hold on it, and the settle asks the contention query which
+     * of them still owns it. Marking them in a loop admits a PARTIAL mark: mark
+     * the reviewer hold, fail on the security hold, and the security row keeps
+     * `posted_pending_check: false` — so it is not counted as a contender, and
+     * approving the reviewer hold terminalizes the shared check `success` while
+     * the trust hold still gates the job. That is a fabricated PASSING check,
+     * which is the worse direction, reached without any process dying: a
+     * deadlock, a statement timeout or a lost connection between the two UPDATEs
+     * is enough. A single `WHERE id IN (…)` either marks every hold or none.
+     *
+     * The residual window is therefore one statement issued immediately after the
+     * provider call returns — narrow, and not only reachable by a process death,
+     * which is why the caller logs its failure rather than treating it as
+     * impossible.
      */
-    createHold(orgId: string, data: CreateHoldData): Promise<HeldRun>;
+    markPendingCheckPosted(orgId: string, heldRunIds: readonly string[]): Promise<void>;
     /** INSERT one decision row using the given executor (root or transaction). */
     private insertDecisionRow;
     /** Flip a pending hold to 'approved' using the given executor. Undefined if not pending. */
@@ -190,20 +267,49 @@ export declare class HeldRunStore {
      */
     listOverdue(): Promise<HeldRun[]>;
     /**
-     * Release overdue workflow timer holds. The install-gate wait action pauses
-     * the workflow as a held run; on timer expiry it must RESUME (not fail like
-     * a reviewer-hold expiry). Flips each overdue pending `hold_type` timer,
-     * `hold_scope='workflow'` row to `released` and returns a `ReleaseSignal`
-     * per row so the caller can resume the workflow. Runs BEFORE
-     * `expireOverdue()` so these rows leave the pending pool before the
-     * expire-and-fail sweep sees them.
+     * Release overdue timer holds at ANY scope. A wait action pauses its element
+     * as a held run; on timer expiry it must RESUME (not fail like a reviewer-hold
+     * expiry). Flips each overdue pending timer row to `released` and returns a
+     * `ReleaseSignal` per row, carrying the row's own scope so the caller can
+     * route it — `routeRelease` sends a workflow-scoped one to the install-gate
+     * rebuild and a job-scoped one to the job re-dispatch path.
+     *
+     * Runs BEFORE `expireOverdue()` so these rows leave the pending pool before
+     * the expire-and-fail sweep sees them. That ordering is load-bearing, not
+     * incidental: `expireOverdue` is not scope-filtered, so a released-but-not-yet-
+     * resumed row would otherwise be expired out from under its resume.
      *
-     * The filter matches every persisted spelling of the timer hold type, so a
-     * row an un-upgraded orchestrator wrote as `wait_timer` still resumes rather
-     * than falling through to the expire-and-fail sweep. `hold_scope` is what
-     * keeps job-scoped dispatch-gate timer holds out of this sweep.
+     * The filter matches every persisted spelling of the timer hold type, so a row
+     * an un-upgraded orchestrator wrote as `wait_timer` still resumes rather than
+     * falling through to the expire-and-fail sweep. It deliberately does NOT
+     * filter on `hold_scope`: a job-scoped timer hold used to be excluded here,
+     * which left it with no release path at all — created, never released,
+     * eventually expired, its job never dispatched.
      */
     releaseDueWaitHolds(): Promise<ReleaseSignal[]>;
+    /**
+     * Every pending queued (concurrency) hold, with the org and concurrency group
+     * it belongs to — the input to the periodic release sweep, which needs to know
+     * WHICH groups have someone waiting before it looks up any limits.
+     *
+     * Returns the pair rather than full rows: the sweep only groups by it, and
+     * `listQueuedHoldsForContext` fetches the rows it actually releases.
+     */
+    listAllQueuedHolds(): Promise<Array<{
+        orgId: string | null;
+        concurrencyGroup: string | null;
+    }>>;
+    /**
+     * List the pending queued (concurrency) holds for one context's concurrency
+     * group, oldest first.
+     *
+     * Joined to `execution_runs` and filtered by `customer_id` as well as
+     * `context`: a context NAME is not unique across tenants, so without the org
+     * predicate one org's completing job could release another org's queued hold.
+     * Oldest-first is the release order — a queue that released newest-first would
+     * starve whoever waited longest.
+     */
+    listQueuedHoldsForContext(orgId: string, concurrencyGroup: string): Promise<HeldRun[]>;
     /**
      * Expire overdue pending runs. Called by the stale detector.
      * Sets status to 'expired' and resolved_at to now() for all
@@ -224,4 +330,5 @@ export declare function createHeldRunStoreFromUrl(databaseUrl: string, opts?: {
     store: HeldRunStore;
     dispose: () => Promise<void>;
 };
+export {};
 //# sourceMappingURL=held-runs.d.ts.map