@kici-dev/orchestrator 0.12.0 → 0.14.0

package/dist/storage/s3.d.ts

@@ -47,6 +47,11 @@ export declare class S3CacheStorage implements CacheStorage {
     private readonly bucket;
     private readonly prefix;
     private readonly ttlMs;
+    /**
+     * False once the backend answered a conditional self-copy with
+     * `501 NotImplemented`. From then on `updateMeta` copies unconditionally.
+     */
+    private conditionalCopySupported;
     constructor(options: S3CacheStorageOptions);
     /** Build the full S3 object key from a cache key. */
     private objectKey;
@@ -57,11 +62,42 @@ export declare class S3CacheStorage implements CacheStorage {
      * Returns null if the object doesn't exist (NoSuchKey/NotFound).
      */
     private readMeta;
+    /**
+     * Read an object's TTL metadata together with the ETag of the version it
+     * belongs to. Returns null if the object doesn't exist (NoSuchKey/NotFound).
+     */
+    private readHead;
     /**
      * Update metadata on an S3 object by copying it to itself.
      * This is the standard S3 pattern for updating metadata without re-uploading data.
+     *
+     * The copy is conditional on `etag`, the version whose metadata the caller
+     * read. A self-copy is a whole-object write: without the condition, one that
+     * races a `put()` of the same key can land after it and write the older body
+     * back. Cache pointers are rewritten in place while builds publish them, so a
+     * touch-on-read racing a publish reverted the pointer to a tarball that no
+     * longer exists. `CopySourceIfMatch` pins the bytes copied to that version and
+     * `IfMatch` refuses the write unless the key still holds it.
+     *
+     * Returns false when the object was replaced since the read (`412`), or is
+     * being replaced right now (`409 ConditionalRequestConflict`): the newer
+     * write stamps metadata of its own, so there is nothing left to refresh.
+     *
+     * An S3-compatible backend that does not implement conditional copies
+     * answers `501 NotImplemented`. The copy then runs again without the
+     * condition, and later copies skip it, as on a backend that ignores the
+     * headers.
      */
     private updateMeta;
+    /** Copy an object onto itself with new TTL metadata, conditional on `etag` when given. */
+    private copyOntoItself;
+    /** Check if an error is a failed conditional request (PreconditionFailed, 412). */
+    private isPreconditionFailedError;
+    /** Check if an error is a conditional write that raced another write (ConditionalRequestConflict, 409). */
+    private isConditionalConflictError;
+    /** Check if an error is a request the backend does not implement (NotImplemented, 501). */
+    private isNotImplementedError;
+    private hasErrorShape;
     /** Check if an error is a "not found" error (NoSuchKey, NotFound, 404). */
     private isNotFoundError;
     put(key: string, data: Buffer | string): Promise<void>;

package/dist/worker/in-memory-execution-tracker.d.ts

@@ -80,6 +80,11 @@ export declare class InMemoryExecutionTracker {
      * Update step status. Forwards to coordinator callback if configured.
      */
     onStepStatus(runId: string, jobId: string, stepIndex: number, stepName: string, status: string, timestamp: number, data?: Record<string, unknown>): Promise<void>;
+    /**
+     * Forget a job this worker gave up or was told to stop before it ran. Nothing is
+     * forwarded: the coordinator that owns the run already decided what happens to it.
+     */
+    dropJob(runId: string, jobId: string): void;
     /**
      * Get the current status of a run.
      */

package/dist/worker/in-memory-job-queue.d.ts

@@ -16,6 +16,25 @@ export declare class InMemoryJobQueue {
     private readonly dispatched;
     /** Per-job re-dispatch counter (parity with dispatch_queue.dispatch_attempts). */
     private readonly attempts;
+    /**
+     * Jobs the owning coordinator stopped (a cancel), or that this worker gave back
+     * after its spawn retries ran out. A stopped job is never claimed, listed or
+     * re-pended, which is this queue's counterpart of the database queue's
+     * stopped-run guard. The mark goes with the job's entry.
+     */
+    private readonly stopped;
+    /** Stop-mark every job of `runId` (or just `jobId`), pending or dispatched. Synchronous. */
+    stop(runId: string, jobId?: string): {
+        pending: string[];
+        dispatched: string[];
+    };
+    /** True while the job waits in this queue and has not been stopped. */
+    isPending(jobId: string): boolean;
+    /**
+     * Parity with the database queue, whose dispatcher calls this after a `null`
+     * requeue. `requeue` already dropped a stopped job, so nothing is left to settle.
+     */
+    settleClaimOfStoppedRun(_jobId: string, _runId: string): Promise<boolean>;
     /** Enqueue a job in memory and return its ID. */
     enqueue(input: QueuedJobInput): Promise<string>;
     /**
@@ -98,7 +117,8 @@ export declare class InMemoryJobQueue {
      * Return a dispatched job to pending for re-dispatch, bumping its attempt
      * counter unless `countAttempt` is false. Mirrors the DB-backed
      * `JobQueue.requeue`. Returns the attempt count, or null when the job is
-     * not currently dispatched.
+     * not currently dispatched. A stopped job is dropped instead of re-pended, and
+     * also returns null.
      */
     requeue(jobId: string, opts?: {
         countAttempt?: boolean;

package/dist/worker/queued-reroute-release.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Remove a worker's queued rerouted jobs and stop-mark its dispatched ones.
+ *
+ * Two triggers reach here: the owning coordinator's `peer.job.cancel`, and this
+ * worker exhausting a job's spawn-retry budget. Either way the coordinator owns
+ * the job's verdict — it already failed, cancelled or re-dispatched the job — so
+ * this reports **nothing** back to it. A worker-side `failed` or `cancelled`
+ * terminal would reach the coordinator as a superseded peer's terminal at best,
+ * and as a second verdict at worst.
+ *
+ * A dispatched job keeps its agent, projection and ownership: the agent is
+ * running it, and the cancel receiver delivers its `job.cancel`. The stop mark
+ * only makes a later requeue drop the job instead of re-pending it.
+ */
+import type { Dispatcher } from '../agent/dispatcher.js';
+import type { InMemoryJobQueue } from './in-memory-job-queue.js';
+import type { InMemoryExecutionTracker } from './in-memory-execution-tracker.js';
+import type { SpawnRetryBudget } from './spawn-retry-budget.js';
+export interface ReleaseDeps {
+    queue: Pick<InMemoryJobQueue, 'stop'>;
+    dispatcher: Pick<Dispatcher, 'cancelQueuedJob'>;
+    executionTracker: Pick<InMemoryExecutionTracker, 'dropJob'>;
+    /** jobId → owning coordinator URL. */
+    jobOwnership: Map<string, string>;
+    budget: Pick<SpawnRetryBudget, 'release'>;
+}
+/**
+ * Stop every matching job of `runId` (or just `jobId`) and remove the pending
+ * ones. The stop mark is set synchronously, before the first `await`, so a
+ * requeue racing this call drops the job.
+ *
+ * @returns the ids of the pending jobs removed.
+ */
+export declare function releaseQueuedRerouteJobs(deps: ReleaseDeps, target: {
+    runId: string;
+    jobId?: string;
+    reason: string;
+}): Promise<string[]>;
+//# sourceMappingURL=queued-reroute-release.d.ts.map

package/dist/worker/reroute-spawn-control.d.ts

@@ -0,0 +1,58 @@
+/**
+ * The worker-side half of the reroute spawn-retry agreement.
+ *
+ * A coordinator reroutes a job to this worker with a spawn-retry budget
+ * (`job.reroute.spawnRetry`). This module applies it:
+ *
+ * - every scaler request for the job passes a per-job attempt gate, so one spawn
+ *   is in flight at a time and none starts during the backoff;
+ * - each `scaler.failed` is relayed to the owning coordinator with the budget's
+ *   verdict (`final`), and the last one releases the job from this worker's queue;
+ * - a coordinator's cancel releases a queued job and stop-marks a dispatched one.
+ *
+ * Assembled here so `bootstrapWorker` only wires it.
+ */
+import { type JobReroute, type PeerScalerEvent, type RerouteSpawnRetry } from '@kici-dev/engine';
+import type { Dispatcher } from '../agent/dispatcher.js';
+import type { ScalerManager } from '../scaler/manager.js';
+import type { ScaleResult, ScalerEvent } from '../scaler/types.js';
+import type { InMemoryJobQueue } from './in-memory-job-queue.js';
+import type { InMemoryExecutionTracker } from './in-memory-execution-tracker.js';
+/** The scaler request the dispatcher's `onNoMatchingAgent` makes. */
+export type ScaleRequest = (...args: Parameters<ScalerManager['requestScale']>) => Promise<ScaleResult>;
+export interface RerouteSpawnControlDeps {
+    queue: InMemoryJobQueue;
+    /** Late-bound: the scaler that emits events is built before the dispatcher. */
+    getDispatcher: () => Pick<Dispatcher, 'cancelQueuedJob' | 'redrivePendingJob'>;
+    executionTracker: InMemoryExecutionTracker;
+    /** jobId → owning coordinator URL. */
+    jobOwnership: Map<string, string>;
+    /** Relay a message to the coordinator that owns the job. */
+    sendToOwningCoord: (jobId: string, msg: PeerScalerEvent) => void;
+    /** The worker's configured budget, used when the coordinator sent none. */
+    defaults: RerouteSpawnRetry;
+    logger: {
+        debug(message: string, meta?: Record<string, unknown>): unknown;
+        info(message: string, meta?: Record<string, unknown>): unknown;
+        warn(message: string, meta?: Record<string, unknown>): unknown;
+        error(message: string, meta?: Record<string, unknown>): unknown;
+    };
+}
+export interface RerouteSpawnControl {
+    /** Start the job's budget. Call before the job is dispatched. */
+    registerReroute(msg: JobReroute): void;
+    /** Drop the budget of a reroute this worker did not accept. */
+    forgetReroute(jobId: string): void;
+    /** Gate a scaler request through the job's budget. */
+    wrapScaleRequest(fn: ScaleRequest): ScaleRequest;
+    /** Relay a scaler event to the owning coordinator, with the budget's verdict. */
+    onScalerEvent(runId: string, jobId: string, ev: ScalerEvent): void;
+    /** An agent received the job. */
+    onDelivered(jobId: string): void;
+    /** The job reached a terminal state. */
+    onJobTerminal(jobId: string): void;
+    /** The owning coordinator cancelled the run, or one job of it. */
+    releaseQueued(runId: string, jobId: string | undefined): Promise<string[]>;
+}
+export declare function createRerouteSpawnControl(deps: RerouteSpawnControlDeps): RerouteSpawnControl;
+//# sourceMappingURL=reroute-spawn-control.d.ts.map

package/dist/worker/spawn-retry-budget.d.ts

@@ -0,0 +1,87 @@
+/**
+ * The spawn-retry budget a worker applies to each job a coordinator rerouted to it.
+ *
+ * A worker whose agent spawn fails retries, but only `maxAttempts` times and with
+ * `backoffMs` between attempts, so the coordinator that owns the run can tell
+ * "still trying" from "gave up". The coordinator sends the budget on `job.reroute`,
+ * and the verdict for each failure rides back on the relayed `scaler.failed` as
+ * `final`. One entry per rerouted job, created before the job is dispatched and
+ * released when the job leaves this worker's queue.
+ */
+import { z } from 'zod';
+import type { RerouteSpawnRetry } from '@kici-dev/engine';
+/** The answer to "may this job start a spawn now?". */
+export declare const SpawnAttemptGate: z.ZodEnum<{
+    backoff: "backoff";
+    begun: "begun";
+    "in-flight": "in-flight";
+    "not-queued": "not-queued";
+    untracked: "untracked";
+}>;
+export type SpawnAttemptGate = z.infer<typeof SpawnAttemptGate>;
+/** The budget's verdict on one spawn failure. */
+export type FailureVerdict = {
+    tracked: false;
+} | {
+    tracked: true;
+    final: boolean;
+    failures: number;
+    maxAttempts: number;
+    duplicate: boolean;
+};
+export interface SpawnRetryBudgetDeps {
+    /** True while the job waits in the worker's queue (not claimed, not stopped). */
+    isQueued: (jobId: string) => boolean;
+    /** Offer one pending job to the scaler again. */
+    redrive: (jobId: string) => Promise<unknown>;
+    logger: {
+        debug(message: string, meta?: object): unknown;
+        info(message: string, meta?: object): unknown;
+        warn(message: string, meta?: object): unknown;
+    };
+    now?: () => number;
+}
+export declare class SpawnRetryBudget {
+    private readonly deps;
+    private readonly entries;
+    private readonly now;
+    constructor(deps: SpawnRetryBudgetDeps);
+    /** Start tracking a rerouted job. A second call for the same job keeps the first entry. */
+    register(jobId: string, runId: string, policy: RerouteSpawnRetry): void;
+    /** Claim the right to start one spawn. Synchronous, so two callers cannot both win. */
+    beginAttempt(jobId: string): SpawnAttemptGate;
+    /**
+     * Record the agent the claimed attempt is spawning. Ignored when the attempt is
+     * no longer in flight, or when that agent's failure already arrived.
+     */
+    setHolder(jobId: string, agentId: string): void;
+    /** The claimed attempt started no spawn (at capacity, no backend, skipped). */
+    abortAttempt(jobId: string): void;
+    /**
+     * The claimed attempt started no spawn because the scaler is deferring for
+     * `delayMs`. Nothing is charged; the job is offered to the scaler again when
+     * the deferral ends. Without this a worker, which has no periodic pending
+     * sweep, would ask once during the deferral and never again, and the
+     * coordinator's spawn window would run out on an attempt the budget still
+     * holds.
+     */
+    deferAttempt(jobId: string, delayMs: number): void;
+    /** An agent received the job. The budget stays in place in case the job is requeued. */
+    onDelivered(jobId: string): void;
+    /**
+     * Count one failed spawn. A failure for a job no longer queued is not a spawn
+     * failure (an agent already took the job), so it is untracked. Only the spawn
+     * holding the attempt gate is charged: a failure from any other agent (a
+     * superseded spawn failing late) is untracked and leaves the gate held. While
+     * the scaler has not yet named the in-flight agent, the in-flight attempt is
+     * the only one that can fail, so its failure is charged. On a non-final
+     * failure the backoff starts, and its timer offers the job to the scaler again.
+     */
+    recordFailure(jobId: string, agentId: string): FailureVerdict;
+    /** The job left the queue: terminal, stopped, or given back. */
+    release(jobId: string): void;
+    private armBackoff;
+    /** Gate new attempts for `delayMs`, then offer the job to the scaler again. */
+    private scheduleRedrive;
+}
+//# sourceMappingURL=spawn-retry-budget.d.ts.map

package/dist/worker-core.d.ts

@@ -64,13 +64,22 @@ interface WorkerSubsystems {
 }
 /**
  * Resolve the ephemeral agent-token TTL a worker-spawned agent should use:
- * the fleet-wide `agent_token_ttl_ms` pulled from the leader when a snapshot has
- * landed, otherwise the boot-time config default. This is the single seam the
+ * the fleet-wide `agent_token_ttl_ms` pulled from a coordinator when a snapshot
+ * has landed, otherwise the boot-time config default. This is the single seam the
  * worker's three scaler backends read at spawn.
  */
 export declare function resolveWorkerAgentTokenTtlMs(pushed: {
     settings: WorkerClusterSettings;
 } | null, fallback: number): number;
+/**
+ * Resolve the Firecracker API-socket wait a worker-spawned VM should use: the
+ * fleet-wide `firecracker_api_socket_wait_ms` pulled from a coordinator,
+ * otherwise the boot-time config default. A snapshot from a coordinator that
+ * predates the knob carries no value, and the default applies.
+ */
+export declare function resolveWorkerFirecrackerApiSocketWaitMs(pushed: {
+    settings: WorkerClusterSettings;
+} | null, fallback: number): number;
 /**
  * Bootstrap the orchestrator in worker mode.
  *

package/dist/ws/agent-handler.d.ts

@@ -20,7 +20,7 @@
  * token, and the agent then registers as normal.
  */
 import type { WSContext, WSEvents, WSMessageReceive } from 'hono/ws';
-import { LogStream } from '@kici-dev/engine';
+import { LogStream, ScaleDownReason } from '@kici-dev/engine';
 import type { RateLimiterConfig, AttestationVerifyStatus } from '@kici-dev/engine';
 import type { ProvenanceTrustRoot } from '../provenance/trust-root.js';
 import type { AgentRegistry } from '../agent/registry.js';
@@ -36,6 +36,7 @@ import type { PendingBuildTracker } from '../cache/pending-builds.js';
 import type { PendingInitTracker } from '../cache/pending-inits.js';
 import type { PendingDynamicTracker } from '../cache/pending-dynamics.js';
 import type { PendingGlobalEvalTracker } from '../cache/pending-global-evals.js';
+import { type CacheUploadSettle } from '../cache/cache-upload-settle.js';
 import type { CacheStorage } from '../storage/types.js';
 import type { AgentMetricsAggregator } from '../metrics/agent-metrics-aggregator.js';
 import { type AgentApiRegistry } from './agent-api-registry.js';
@@ -94,6 +95,8 @@ export declare function frameIdentity(data: unknown): FrameIdentity;
 export interface AgentWsEvents extends Omit<WSEvents, 'onMessage'> {
     onMessage(evt: MessageEvent<WSMessageReceive>, ws: WSContext): Promise<void>;
 }
+/** `error` frame code sent when a registration is refused because another agent token holds the id. */
+export declare const AGENT_ID_IN_USE = "AGENT_ID_IN_USE";
 export interface AgentWsHandlerDeps {
     registry: AgentRegistry;
     dispatcher: Dispatcher;
@@ -161,8 +164,11 @@ export interface AgentWsHandlerDeps {
         boundJobId?: string;
         mandatoryLabels: string[];
     } | null>;
-    /** Optional callback when an agent disconnects (for scaler lifecycle). */
-    onScalerAgentDisconnected?: (agentId: string) => void;
+    /**
+     * Optional callback when an agent disconnects (for scaler lifecycle). `reason`
+     * is the scale-down reason an event scaler emits for it.
+     */
+    onScalerAgentDisconnected?: (agentId: string, reason?: ScaleDownReason) => void;
     /** Optional callback when an agent completes a job (for scaler lifecycle). */
     onScalerJobComplete?: (agentId: string) => void;
     /** Optional callback when agent sends per-job heartbeats. */
@@ -256,6 +262,13 @@ export interface AgentWsHandlerDeps {
     artifactStore?: ArtifactStore;
     /** Cache storage for setting metadata after upload completion. */
     cacheStorage?: CacheStorage;
+    /**
+     * In-flight build-cache publishes and the per-org bound a build's success
+     * waits for them. Built once per orchestrator, not per connection, so a
+     * success replayed on a reconnected socket still sees a publish started on
+     * the old one. Absent: no registration and no wait.
+     */
+    cacheUploadSettle?: CacheUploadSettle;
     /**
      * Storage used to mint presigned PUT URLs for provenance bundles. Distinct
      * dep so a deployment can decide whether attestations share the cache bucket.
@@ -424,6 +437,14 @@ export interface AgentLogChunkInput {
     /** Absent when the agent does not report a stream; read as `stdout`. */
     stream?: LogStream;
 }
+/**
+ * Release the in-memory waiters of jobs a disconnect triage failed: pending
+ * builds, inits, dynamic evaluations and global eval rounds, and the user-cache
+ * refs of the dispatch. The dispatcher runs it inside `onAgentDisconnect`, so
+ * every path that drops an agent reaches it, whether or not it waits for the
+ * socket to close.
+ */
+export declare function releaseFailedJobWaiters(deps: Pick<AgentWsHandlerDeps, 'pendingBuilds' | 'pendingInits' | 'pendingDynamics' | 'pendingGlobalEvals' | 'dispatchCacheRefs'>, failedJobIds: string[]): void;
 /**
  * Create a Hono WS event handler for agent connections.
  *

package/dist/ws/agent-heartbeat.d.ts

@@ -23,8 +23,8 @@ interface AgentHeartbeatMonitorDeps {
 /**
  * Periodically inspects all registered agent connections and:
  * - Marks agents as unhealthy after 90s of silence (log only).
- * - Closes and unregisters agents after 180s of silence.
- * - Triggers dispatcher.onAgentDisconnect for stale agents.
+ * - Triggers dispatcher.onAgentDisconnect for an agent silent for 180s, then
+ *   unregisters it and closes its socket.
  */
 export declare class AgentHeartbeatMonitor {
     private readonly registry;

package/installer-image-digests.json

@@ -1,7 +1,7 @@
 {
-  "version": "0.12.0",
+  "version": "0.14.0",
   "images": {
-    "kici-agent": "sha256:eaa1d2270da370f150b149dbf51569b51759d7b62607fe6a1cee8b01babac2e9",
-    "kici-orchestrator": "sha256:a1a73e458b0cda327e534b74ba02406fb64170d61a3de34bfbfa8c42ab75d087"
+    "kici-agent": "sha256:c5b2ae281f683b43568c238560aa8bb82d4014a3e3b9e86b857496812eabb493",
+    "kici-orchestrator": "sha256:6ae78b978f52e80b2ffafae46cf3d33d1dcae3b48b93ab91ea2bc7dd28412d26"
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kici-dev/orchestrator",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "Customer-deployable orchestrator for the KiCI CI/CD stack. Receives webhook events (direct or via Platform relay), matches triggers against the lock file, and dispatches jobs to connected agents.",
   "keywords": [
     "ci",
@@ -24,7 +24,7 @@
   },
   "license": "AGPL-3.0-only",
   "engines": {
-    "node": ">=24"
+    "node": ">=24.5.0"
   },
   "type": "module",
   "files": [
@@ -90,8 +90,8 @@
     "ws": "^8.21.3",
     "yaml": "^2.9.1",
     "zod": "^4.6.5",
-    "@kici-dev/engine": "0.12.0",
-    "@kici-dev/shared": "0.12.0"
+    "@kici-dev/engine": "0.14.0",
+    "@kici-dev/shared": "0.14.0"
   },
   "kici": {
     "metrics": {
@@ -106,7 +106,7 @@
     "@types/dockerode": "^4.0.1",
     "@types/ws": "^8.18.1",
     "kysely-ctl": "^0.21.0",
-    "@kici-dev/agent": "0.12.0"
+    "@kici-dev/agent": "0.14.0"
   },
   "scripts": {
     "build": "node ../../scripts/build-service.mjs && tsc --emitDeclarationOnly",