@kici-dev/orchestrator 0.6.0 → 0.7.0

package/dist/cluster/cluster-settings-reader.d.ts

@@ -3,7 +3,7 @@ import type { Database } from '../db/types.js';
 /** Numeric columns on cluster_settings readable via {@link ClusterSettingsReader}. */
 export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms'
 /** @deprecated Readable, but no call site reads it. Removed at v1.0.0. */
- | 'contributor_cache_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'check_run_tracking_ttl_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
+ | 'contributor_cache_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'check_run_tracking_ttl_days' | 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
 /** Text columns on cluster_settings readable via {@link ClusterSettingsReader}. */
 export type ClusterStringColumn = 'dashboard_verified_issuer';
 /** Boolean columns on cluster_settings readable via {@link ClusterSettingsReader}. */
@@ -40,7 +40,7 @@ export declare const CACHE_MAX_ENTRIES_CEILING = 100000;
  *
  * A value that is not a usable positive count (NaN, non-finite, below 1) falls
  * back to the configured default rather than clamping to 1: a 1-entry cache
- * thrashes silently, which is harder to diagnose than simply ignoring garbage.
+ * thrashes silently, which is harder to diagnose than ignoring garbage.
  * The fallback is floored and bounded by the ceiling on the same path as the
  * stored value, so neither a fractional nor an above-ceiling configured default
  * can reach the LRU constructor as a non-integer or unclamped max — the

package/dist/cluster/coordinator.d.ts

@@ -47,6 +47,7 @@ export interface JobToRoute {
     sha: string;
     sourceTarUrl?: string;
     sourceTarHash?: string;
+    sourceTarDigest?: string;
     depsUrl?: string;
     depsHash?: string;
     /** Labels that the dispatched agent must NOT have. */
@@ -325,6 +326,16 @@ export declare class RunCoordinator {
     private handleRerouteSpawnTimeout;
     /** Best-effort cancel a single job on a peer (double-execution guard). */
     private cancelPeerJob;
+    /**
+     * Forward `job.cancel` for ONE job to the coordinator that holds its agent's
+     * socket, reporting whether the message left this process.
+     *
+     * The run-cancel path needs this for a job this coordinator never routed:
+     * the Platform picks any pool member to receive `run.cancel.request`, and only
+     * the coordinator holding the agent's socket can put a cancel frame on it.
+     * {@link RunCoordinator.cancelRun} covers rerouted jobs and is unchanged.
+     */
+    cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string): boolean;
     /**
      * Local fallback for a re-dispatch: dispatch the job on this coordinator under
      * the existing jobId. Returns true when the local dispatcher accepts it.

package/dist/cluster/instance-heartbeat.d.ts

@@ -0,0 +1,96 @@
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+/** Default interval between `cluster_instances` heartbeat writes. */
+export declare const DEFAULT_INSTANCE_HEARTBEAT_MS = 10000;
+/**
+ * How many missed ticks a coordinator gets before it reads as dead. Six ticks
+ * (60 s at the default interval) is wide enough that a GC pause, a slow DB or a
+ * brief network stall does not evict a healthy coordinator — the cost of a
+ * false "dead" verdict is another coordinator recovering jobs it is watching.
+ */
+export declare const HEARTBEAT_MISSES_BEFORE_DEAD = 6;
+/**
+ * The dispatcher's recovery grace period at the default config: twice
+ * `agentMaxReconnectDelayMs` (60 s). Used as the fallback floor when a caller
+ * has no configured grace period to hand in.
+ */
+export declare const DEFAULT_RECOVERY_GRACE_MS = 120000;
+/**
+ * The window inside which a `cluster_instances` row counts as live.
+ *
+ * Deliberately the larger of the recovery grace period and six heartbeat ticks:
+ * the ownership predicate must never call a coordinator dead sooner than the
+ * recovery machinery would have given it to reconnect.
+ */
+export declare function instanceLivenessGraceMs(gracePeriodMs: number, heartbeatMs?: number): number;
+export interface InstanceHeartbeatOptions {
+    db: Kysely<Database>;
+    instanceId: string;
+    /** Interval between writes. Defaults to {@link DEFAULT_INSTANCE_HEARTBEAT_MS}. */
+    intervalMs?: number;
+    /** Orchestrator version, recorded for operator visibility. */
+    version?: string;
+    /** Reads the current Raft role, when the node runs Raft. */
+    getRole?: () => string | null;
+}
+/**
+ * Writes this coordinator's liveness into `cluster_instances` on a fixed tick.
+ *
+ * Every ownership predicate in the dispatch plane asks "is the coordinator that
+ * owns this row still alive?", and the answer has to come from the database.
+ * The in-memory peer registry cannot answer it: it is empty at exactly the
+ * moment startup recovery runs, because a freshly booted process has not
+ * handshaken with any peer yet, so it reads every live sibling as dead — which
+ * is the boot-time bug the predicate exists to fix. Raft membership is worse:
+ * a partitioned coordinator self-elects and reads the other half as gone.
+ *
+ * A DB heartbeat needs no peer connectivity, so it behaves the same in a
+ * Raft cluster, a plain multi-coordinator deployment and a standalone one.
+ * `host_roster` already derives agent liveness this way; this is the same
+ * mechanism pointed at coordinators.
+ */
+export declare class InstanceHeartbeat {
+    private readonly db;
+    private readonly instanceId;
+    private readonly intervalMs;
+    private readonly version;
+    private readonly getRole?;
+    private timer;
+    constructor(opts: InstanceHeartbeatOptions);
+    /**
+     * Write one heartbeat immediately, then every `intervalMs`.
+     *
+     * The immediate write matters: startup recovery runs moments after boot and
+     * reads this table, so a coordinator that had not yet written a row would be
+     * invisible to its own siblings' predicates.
+     */
+    start(): Promise<void>;
+    stop(): void;
+    /**
+     * Remove this instance's row on a clean shutdown, so a coordinator that
+     * stopped on purpose is dead to its siblings immediately rather than after
+     * the grace window. Best-effort: a crash leaves the row to expire.
+     */
+    retire(): Promise<void>;
+    /**
+     * One heartbeat write. Best-effort: a failed tick is retried on the next.
+     *
+     * `processStart` is set for the write `start()` makes, and is the only one
+     * that rewrites `started_at` — the column records when THIS process came up,
+     * so a restart under the same instance id must move it while an ordinary tick
+     * must not.
+     */
+    beat(opts?: {
+        processStart?: boolean;
+    }): Promise<void>;
+}
+/**
+ * Which of the given coordinator instance ids are alive right now.
+ *
+ * Reads `cluster_instances` directly rather than the peer registry, for the
+ * reason the class above documents: the peer registry is empty during boot and
+ * blind under a partition, and a wrong "dead" verdict here means acting on a
+ * row a working coordinator owns.
+ */
+export declare function liveInstanceIds(db: Kysely<Database>, instanceIds: readonly string[], graceMs: number): Promise<Set<string>>;
+//# sourceMappingURL=instance-heartbeat.d.ts.map

package/dist/cluster/join-client.d.ts

@@ -6,7 +6,7 @@
  * 2. Send join.request with the join token
  * 3. Receive join.response with encrypted config bundle
  * 4. Decrypt bundle using token-derived AES-256-GCM key
- * 5. Write decrypted config as local YAML file
+ * 5. Write the decrypted config as an env file the orchestrator boots from
  *
  * The token carries routing info (cleartext for Platform relay) and a secret
  * (used for HKDF key derivation). Only the joiner and the token creator
@@ -22,12 +22,23 @@ interface JoinClientOptions {
     peerUrl?: string;
     /** API key for Platform authentication (required for --platform mode) */
     apiKey?: string;
-    /** Path to write the resulting local config YAML */
+    /**
+     * Path to write the resulting local config YAML.
+     *
+     * @deprecated The orchestrator boots from its environment, so nothing reads
+     * this file. Leave it unset and use {@link JoinClientOptions.envFilePath}.
+     */
     configPath?: string;
+    /** Path to write the env file `orchestrator install --env-file` consumes. */
+    envFilePath?: string;
 }
+/** Default path the join writes its env file to. */
+export declare const DEFAULT_JOIN_ENV_FILE = "./kici-orchestrator.env";
 /**
- * Local config structure written by the join client.
- * Compatible with loadLocalConfig() from config/loader.ts.
+ * Local config structure written by the deprecated `--config` path.
+ *
+ * @deprecated `loadLocalConfig()` strips `storage` and `secrets`, and the boot
+ * path never reads the file at all.
  */
 interface JoinLocalConfig {
     database: {
@@ -45,10 +56,48 @@ export declare function decryptAndParseBundle(encryptedB64: string, encryptionKe
 /**
  * Build a JoinLocalConfig from a decrypted ConfigBundle.
  * Maps bundle fields to the structure expected by loadLocalConfig().
+ *
+ * @deprecated Feeds the deprecated `--config` artifact. Use
+ * {@link buildEnvFile}.
  */
 export declare function buildLocalConfig(bundle: ConfigBundle): JoinLocalConfig;
+/**
+ * Startup environment variable each `ConfigBundle.storage` field is delivered
+ * through.
+ *
+ * The `satisfies` clause makes a bundle storage field with no entry here a
+ * compile error, so the projection cannot silently drop one — which is the
+ * whole defect this artifact exists to avoid. The companion test pins the
+ * other half: every field `sharedConfigSchema.storage` accepts, and every
+ * variable named here, resolves in the startup env definition.
+ */
+export declare const STORAGE_ENV_VARS: {
+    readonly type: 'KICI_STORAGE_TYPE';
+    readonly bucket: 'KICI_STORAGE_BUCKET';
+    readonly prefix: 'KICI_STORAGE_PREFIX';
+    readonly region: 'KICI_STORAGE_REGION';
+    readonly endpoint: 'KICI_STORAGE_ENDPOINT';
+    readonly externalEndpoint: 'KICI_STORAGE_EXTERNAL_ENDPOINT';
+    readonly forcePathStyle: 'KICI_STORAGE_FORCE_PATH_STYLE';
+    readonly logBucket: 'KICI_STORAGE_LOG_BUCKET';
+};
+/**
+ * Project a decrypted `ConfigBundle` onto the environment variables the
+ * orchestrator boot path reads.
+ *
+ * A value carrying a newline would silently truncate the file, so it is
+ * rejected by name rather than written.
+ */
+export declare function buildEnvFile(bundle: ConfigBundle): string;
+/**
+ * Write the env file the orchestrator boots from.
+ */
+export declare function writeEnvFile(path: string, bundle: ConfigBundle): Promise<void>;
 /**
  * Write a local config object to a YAML file.
+ *
+ * @deprecated The orchestrator boot path is environment-only, so nothing reads
+ * this file. Use {@link writeEnvFile}.
  */
 export declare function writeConfigFile(path: string, config: Record<string, unknown>): Promise<void>;
 export declare class JoinClient {
@@ -59,7 +108,8 @@ export declare class JoinClient {
      * 1. Send join.request with token to Platform relay or direct peer
      * 2. Receive join.response with encrypted config bundle
      * 3. Decrypt bundle using token-derived key
-     * 4. Write local config YAML
+     * 4. Write the env file the orchestrator boots from (and, when `--config`
+     *    names one, the deprecated local YAML)
      */
     join(): Promise<void>;
     /**

package/dist/cluster/join-handler.d.ts

@@ -33,6 +33,7 @@ export interface ConfigBundle {
         prefix?: string;
         region?: string;
         endpoint?: string;
+        externalEndpoint?: string;
         forcePathStyle?: boolean;
         logBucket?: string;
     };

package/dist/cluster/orphan-recovery.d.ts

@@ -60,6 +60,22 @@ export declare class OrphanRecovery {
      * Only runs when this node is the Raft leader.
      */
     scanForOrphans(): Promise<void>;
+    /**
+     * Release concurrency slots still marked `active` for a run that has already
+     * finished. Run once per leader transition (`start()`), which covers a boot
+     * and a failover.
+     *
+     * A slot outlives its run whenever the release path did not run — a
+     * coordinator crash between "the run finished" and "the slot was released".
+     * On a `max: 1` deploy gate a single leaked slot blocks every later deploy
+     * forever, and nothing else would ever notice: the DB says the slot is held
+     * and the run that held it is gone.
+     *
+     * `getActiveRuns` reconciles the same way inside its own transaction on every
+     * acquire, so this is the sweep for a scope nobody happens to ask about —
+     * which is why once per leader transition is enough.
+     */
+    releaseLeakedConcurrencySlots(): Promise<void>;
     /**
      * Attempt to recover a single potentially orphaned run.
      */

package/dist/cluster/peer-registry.d.ts

@@ -127,6 +127,16 @@ export declare class PeerRegistry {
      * Update peer state from a heartbeat message.
      * Updates agent inventory, draining status, capabilities, term, leaderId,
      * and lastHeartbeatAt timestamp.
+     *
+     * A heartbeat is an ingress into the label-matching domain, so every label
+     * it carries is folded here rather than at each comparison site. A peer
+     * running an older build sends the case its operator typed — `Docker` — and
+     * every consumer of `PeerInfo` compares against folded job selectors, so a
+     * consumer that forgets to fold silently stops counting that peer's agents.
+     * Folding on arrival means a consumer cannot forget.
+     *
+     * Identifiers are deliberately NOT folded: `agentId`, `scalerName`, and the
+     * scaler's `name` / `type` are opaque and case-sensitive.
      */
     updateHeartbeat(instanceId: string, heartbeat: PeerHeartbeat): void;
     /**

package/dist/concurrency/dispatch-next-queued.d.ts

@@ -12,27 +12,15 @@
  * get permanently stuck on a phantom holder.
  */
 import type { AgentRegistry } from '../agent/registry.js';
-import type { ConcurrencyGroupTracker } from './group-tracker.js';
 import type { ConcurrencyQueueManager } from './queue-manager.js';
 import type { ConcurrencyWaiters } from './waiters.js';
 export interface DispatchNextQueuedDeps {
-    /** In-memory tracker — kept in sync with the DB on dequeue. */
-    tracker: ConcurrencyGroupTracker;
-    /** DB-backed queue manager. */
+    /** The DB arbiter. */
     queueManager: ConcurrencyQueueManager;
     /** Agent registry for finding the waiter's WS. */
     registry: AgentRegistry;
     /** In-memory waiter map. */
     waiters: ConcurrencyWaiters;
-    /**
-     * Default max for `acquireSlot` when re-syncing the in-memory tracker after
-     * a dequeue. The runtime concurrency config (per-workflow `max`) is not
-     * available at slot-release time; the dequeued waiter has already been
-     * accounted for under that policy when it was originally enqueued, so any
-     * value `>= 1` is safe here. We pass 1 because the v1 contract documents
-     * single-slot semantics (max>1 is out of scope).
-     */
-    trackerMaxFallback?: number;
 }
 /**
  * Try to dequeue the next queued entry for `(group, routingKey)` and notify

package/dist/concurrency/index.d.ts

@@ -1,3 +1,2 @@
-export { ConcurrencyGroupTracker, type HydrateRecord } from './group-tracker.js';
-export { ConcurrencyQueueManager, type QueuedJob, type EnqueueOptions } from './queue-manager.js';
+export { ConcurrencyQueueManager, ConcurrencySlotStatus, type QueuedJob, type EnqueueOptions, } from './queue-manager.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/concurrency/queue-manager.d.ts

@@ -1,12 +1,23 @@
 /**
- * DB-backed concurrency queue manager.
+ * The arbiter of `concurrency: { group, max }`.
  *
- * Persists queued jobs to the concurrency_groups table and dequeues
- * the oldest queued job when a slot opens. Works with ConcurrencyGroupTracker
- * for slot-based coordination.
+ * Every decision is a transaction against `concurrency_groups`, which is what
+ * makes `max` mean the same thing across a restart and across every coordinator
+ * in a cluster. A process-local map cannot: a restart came back with an empty
+ * map and let a second run deploy alongside the one already deploying, and two
+ * coordinators each held their own map so `max: 1` was never cluster-wide.
+ * `concurrency: { group, max: 1 }` is the guard customers use to serialise
+ * deploys and migrations, so "usually right" is not a useful guarantee.
  */
-import type { Kysely } from 'kysely';
+import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
+/** `concurrency_groups.status` values. */
+export declare enum ConcurrencySlotStatus {
+    Active = "active",
+    Queued = "queued",
+    Completed = "completed",
+    Cancelled = "cancelled"
+}
 /** A queued job returned from dequeue operations. */
 export interface QueuedJob {
     id: string;
@@ -25,20 +36,63 @@ export interface EnqueueOptions {
 export declare class ConcurrencyQueueManager {
     private readonly db;
     constructor(db: Kysely<Database>);
+    /**
+     * Serialise every arbitration for one (routingKey, groupKey) scope.
+     *
+     * A transaction-scoped advisory lock, so it is released on commit or
+     * rollback with no cleanup path to get wrong. Counting active rows and
+     * inserting one are two statements, and without the lock two coordinators
+     * both read `count < max` and both insert — which is exactly the
+     * double-deploy the cap exists to prevent.
+     */
+    private static lockScope;
+    /**
+     * Try to take a slot in a concurrency group.
+     *
+     * Idempotent: a run that already holds an active slot re-acquires it without
+     * consuming a second one, which the partial unique index on
+     * `(routing_key, group_key, run_id) WHERE status = 'active'` also enforces at
+     * the DB level.
+     *
+     * @returns whether this run holds a slot afterwards.
+     */
+    acquireSlot(groupKey: string, routingKey: string, runId: string, jobId: string, opts: {
+        max: number;
+    }): Promise<boolean>;
+    /** Release this run's slot in a group. No-op when it holds none. */
+    releaseSlot(groupKey: string, routingKey: string, runId: string): Promise<void>;
+    /**
+     * The runs holding an active slot, oldest first.
+     *
+     * A run whose `execution_runs` row is already terminal is excluded and its
+     * slot is released in the same transaction. A slot outlives its run whenever
+     * the release path did not run — a coordinator crash between "the run
+     * finished" and "the slot was released" — and a leaked slot on a `max: 1`
+     * deploy gate blocks every later deploy forever.
+     */
+    getActiveRuns(groupKey: string, routingKey: string): Promise<string[]>;
+    /** The oldest active run, which `cancelInProgress` supersedes. */
+    getOldestRun(groupKey: string, routingKey: string): Promise<string | null>;
+    /**
+     * Active run ids for a scope, reconciled against `execution_runs`. Must be
+     * called with the scope lock already held.
+     */
+    private static activeRunIds;
     /**
      * Enqueue a job that is waiting for a concurrency slot.
      * Inserts a row with status='queued' into concurrency_groups.
      */
     enqueue(opts: EnqueueOptions): Promise<void>;
-    /**
-     * Record an active slot in the DB for persistence across restarts.
-     */
-    recordActive(opts: EnqueueOptions): Promise<void>;
     /**
      * Dequeue the oldest queued job for a concurrency group.
      * Returns null if no queued jobs exist.
      */
     dequeueNext(groupKey: string, routingKey: string): Promise<QueuedJob | null>;
+    /**
+     * Promote the oldest queued waiter to `active`. Must be called with the scope
+     * lock already held.
+     */
+    private static promoteOldestQueued;
     /**
      * Cancel all queued entries for a run.
      * Marks them as 'cancelled' with a completion timestamp.
@@ -55,14 +109,5 @@ export declare class ConcurrencyQueueManager {
      * queued job should be dispatched.
      */
     onJobComplete(groupKey: string, routingKey: string, runId: string): Promise<QueuedJob | null>;
-    /**
-     * Load all active entries from DB for hydrating the in-memory tracker.
-     * Called on orchestrator startup.
-     */
-    loadActiveEntries(): Promise<Array<{
-        groupKey: string;
-        routingKey: string;
-        runId: string;
-    }>>;
 }
 //# sourceMappingURL=queue-manager.d.ts.map

package/dist/config/env-overlay.d.ts

@@ -12,9 +12,10 @@
  * Convert a KICI_ env var key to a config path array.
  * Returns null for non-KICI_ keys or unknown mappings.
  *
- * Handles two patterns:
+ * Handles three patterns:
  * 1. Direct mappings (KICI_DATABASE_URL -> ['database', 'url'])
- * 2. Multi-app GitHub provider (KICI_PROVIDERS_GITHUB_<NAME>_<FIELD> ->
+ * 2. Startup names whose overlay spelling differs (KICI_PORT -> ['server', 'port'])
+ * 3. Multi-app GitHub provider (KICI_PROVIDERS_GITHUB_<NAME>_<FIELD> ->
  *    ['providers', 'github', '<name-lowered>', '<camelField>'])
  */
 export declare function envKeyToConfigPath(key: string): string[] | null;

package/dist/config.d.ts

@@ -34,6 +34,7 @@ declare const configSchema: z.ZodObject<{
         platform: "platform";
     }>>;
     port: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    host: z.ZodDefault<z.ZodString>;
     basePath: z.ZodDefault<z.ZodString>;
     tlsCertPath: z.ZodOptional<z.ZodString>;
     platformUrl: z.ZodOptional<z.ZodString>;
@@ -42,6 +43,7 @@ declare const configSchema: z.ZodObject<{
     provenanceIssuer: z.ZodOptional<z.ZodString>;
     provenanceSigningIssuer: z.ZodOptional<z.ZodString>;
     provenanceSignerKind: z.ZodOptional<z.ZodString>;
+    oidcLegacyPrSub: z.ZodPipe<z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>, z.ZodTransform<boolean, string | boolean>>;
     provenanceKmsKeyArn: z.ZodOptional<z.ZodString>;
     provenanceKmsRegion: z.ZodOptional<z.ZodString>;
     provenanceKmsAccessKeyId: z.ZodOptional<z.ZodString>;
@@ -89,6 +91,7 @@ declare const configSchema: z.ZodObject<{
     ingestOverflowReplayIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     ingestOverflowReplayBatch: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     ingestOverflowMaxAttempts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    ingestOverflowMaxAgeMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     ingestOverflowClaimTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteSpawnWindowMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -129,6 +132,10 @@ declare const configSchema: z.ZodObject<{
     dispatchQueueTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     stepLogTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     checkRunTrackingTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    runRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    auditRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    provenanceRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    heldRunRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -203,6 +210,7 @@ declare const configSchema: z.ZodObject<{
     }>>;
     autoMigrate: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     agentMaxReconnectDelayMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    clusterInstanceHeartbeatMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     otelExporterOtlpEndpoint: z.ZodOptional<z.ZodString>;
     clusterName: z.ZodOptional<z.ZodString>;
     cluster: z.ZodPrefault<z.ZodObject<{
@@ -237,6 +245,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
         platform: "platform";
     }>>;
     port: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    host: z.ZodDefault<z.ZodString>;
     basePath: z.ZodDefault<z.ZodString>;
     tlsCertPath: z.ZodOptional<z.ZodString>;
     platformUrl: z.ZodOptional<z.ZodString>;
@@ -245,6 +254,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
     provenanceIssuer: z.ZodOptional<z.ZodString>;
     provenanceSigningIssuer: z.ZodOptional<z.ZodString>;
     provenanceSignerKind: z.ZodOptional<z.ZodString>;
+    oidcLegacyPrSub: z.ZodPipe<z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>, z.ZodTransform<boolean, string | boolean>>;
     provenanceKmsKeyArn: z.ZodOptional<z.ZodString>;
     provenanceKmsRegion: z.ZodOptional<z.ZodString>;
     provenanceKmsAccessKeyId: z.ZodOptional<z.ZodString>;
@@ -292,6 +302,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
     ingestOverflowReplayIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     ingestOverflowReplayBatch: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     ingestOverflowMaxAttempts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    ingestOverflowMaxAgeMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     ingestOverflowClaimTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteSpawnWindowMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -332,6 +343,10 @@ export declare const packagingConfigSchema: z.ZodObject<{
     dispatchQueueTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     stepLogTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     checkRunTrackingTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    runRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    auditRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    provenanceRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    heldRunRetentionDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -406,6 +421,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
     }>>;
     autoMigrate: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     agentMaxReconnectDelayMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    clusterInstanceHeartbeatMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     otelExporterOtlpEndpoint: z.ZodOptional<z.ZodString>;
     clusterName: z.ZodOptional<z.ZodString>;
     cluster: z.ZodPrefault<z.ZodObject<{
@@ -473,6 +489,7 @@ export type AppConfig = z.infer<typeof configSchema> & {
 export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     mode: "hybrid" | "independent" | "observed" | "platform";
     port: number;
+    host: string;
     basePath: string;
     tlsCertPath?: string | undefined;
     platformUrl?: string | undefined;
@@ -481,6 +498,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     provenanceIssuer?: string | undefined;
     provenanceSigningIssuer?: string | undefined;
     provenanceSignerKind?: string | undefined;
+    oidcLegacyPrSub: boolean;
     provenanceKmsKeyArn?: string | undefined;
     provenanceKmsRegion?: string | undefined;
     provenanceKmsAccessKeyId?: string | undefined;
@@ -525,6 +543,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     ingestOverflowReplayIntervalMs: number;
     ingestOverflowReplayBatch: number;
     ingestOverflowMaxAttempts: number;
+    ingestOverflowMaxAgeMs: number;
     ingestOverflowClaimTimeoutMs: number;
     rerouteSpawnWindowMs: number;
     rerouteAckTimeoutMs: number;
@@ -559,6 +578,10 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     dispatchQueueTtlDays: number;
     stepLogTtlDays: number;
     checkRunTrackingTtlDays: number;
+    runRetentionDays: number;
+    auditRetentionDays: number;
+    provenanceRetentionDays: number;
+    heldRunRetentionDays: number;
     reconnectReplayWindowHours: number;
     cacheBuildTimeoutMs: number;
     cacheMaxTarballBytes: number;
@@ -612,6 +635,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     nodeEnv: "development" | "production" | "test";
     autoMigrate: boolean;
     agentMaxReconnectDelayMs: number;
+    clusterInstanceHeartbeatMs: number;
     otelExporterOtlpEndpoint?: string | undefined;
     clusterName?: string | undefined;
     cluster: {

package/dist/contexts/release-queued-holds.d.ts

@@ -12,6 +12,13 @@ export interface ReleaseQueuedHoldsArgs {
     /** Resume a released hold by re-dispatching its job. */
     onJobRelease: (signal: ReleaseSignal) => Promise<void>;
 }
+/** Jobs of one run the caller already accounts for itself. */
+export interface OccupyingJobExclusion {
+    /** The run the excluded names belong to. */
+    runId: string;
+    /** `execution_jobs.job_name` values to leave out of the count. */
+    jobNames: readonly string[];
+}
 /**
  * Count the jobs OCCUPYING a slot in a concurrency group, scoped to one org.
  *
@@ -23,10 +30,25 @@ export interface ReleaseQueuedHoldsArgs {
  * one: the release path re-checks the limit precisely so the bound holds
  * whenever the sweep happens to run, and an undercount defeats that re-check.
  *
+ * A PLACEHOLDER row is excluded, and that exclusion is what keeps the count
+ * from deadlocking the queue it gates. A job the dispatch pass held back — on a
+ * `needs` edge, a rolling-wave slot, an invoke-gate summon, or a protection-rule
+ * hold — is registered under a `needs-pending-` id at status `pending` so the
+ * run stays open for it. It has reached no agent, so it occupies nothing; and
+ * counting it meant a queued hold counted against the very limit that gates its
+ * own release. One held job in a limit-1 group then read as `free = 1 - 1 = 0`
+ * and was never released at all, and the hold sat until the expiry sweep failed
+ * it. Its real row, with the dispatcher's own job id, replaces the placeholder
+ * the moment it dispatches — so the slot is counted from exactly then.
+ *
  * The `customer_id` predicate is not optional: a context name shared across
  * tenants would otherwise leak concurrency between them.
+ *
+ * `exclude` lets a caller that carries its own in-memory reservation for some
+ * of these jobs keep the two terms disjoint. The dispatch gate is the one such
+ * caller; the release sweep passes nothing and counts everything.
  */
-export declare function countOccupyingJobs(db: Kysely<Database>, orgId: string, concurrencyGroup: string): Promise<number>;
+export declare function countOccupyingJobs(db: Kysely<Database>, orgId: string, concurrencyGroup: string, exclude?: OccupyingJobExclusion): Promise<number>;
 /**
  * Release up to `limit - running` queued holds for a group, oldest first.
  * Returns how many actually resumed.

package/dist/dashboard/attestation-filters.d.ts

@@ -21,6 +21,7 @@ export declare function baseAttestationsQuery(db: Kysely<Database>): import("kys
     batch_accumulation_items: import("../db/types.js").BatchAccumulationItemsTable;
     batch_accumulation_windows: import("../db/types.js").BatchAccumulationWindowsTable;
     check_run_tracking: import("../db/types.js").CheckRunTrackingTable;
+    cluster_instances: import("../db/types.js").ClusterInstancesTable;
     cluster_meta: import("../db/types.js").ClusterMetaTable;
     cluster_settings: import("../db/types.js").ClusterSettingsTable;
     cold_store_chunk_counts: import("../db/types.js").ColdStoreChunkCountsTable;
@@ -96,6 +97,7 @@ export declare function basePendingAttestationsQuery(db: Kysely<Database>): impo
     batch_accumulation_items: import("../db/types.js").BatchAccumulationItemsTable;
     batch_accumulation_windows: import("../db/types.js").BatchAccumulationWindowsTable;
     check_run_tracking: import("../db/types.js").CheckRunTrackingTable;
+    cluster_instances: import("../db/types.js").ClusterInstancesTable;
     cluster_meta: import("../db/types.js").ClusterMetaTable;
     cluster_settings: import("../db/types.js").ClusterSettingsTable;
     cold_store_chunk_counts: import("../db/types.js").ColdStoreChunkCountsTable;