@kici-dev/orchestrator 0.12.0 → 0.13.0

package/dist/agent/dispatcher.d.ts

@@ -258,6 +258,17 @@ export declare class Dispatcher {
          */
         hasPeerCoordinators?: () => boolean | Promise<boolean>;
     });
+    /** Runs with the ids of the jobs each disconnect triage failed; see {@link setDisconnectCleanup}. */
+    private disconnectCleanup?;
+    /**
+     * Install what runs with the ids of the jobs a disconnect triage failed. The
+     * agent WebSocket handler installs the release of their in-memory waiters
+     * (pending builds, inits, dynamic evaluations, global eval rounds). It runs
+     * inside {@link onAgentDisconnect}, so a path that drops an agent without
+     * waiting for its socket (heartbeat timeout, ack timeout, token revocation)
+     * reaches it too.
+     */
+    setDisconnectCleanup(cleanup: (failedJobIds: string[]) => void): void;
     /**
      * Dispatch a job to a matching agent, or queue it if none available.
      *
@@ -508,6 +519,15 @@ export declare class Dispatcher {
      * triage finds nothing to double-handle.
      */
     private handleAckExpiry;
+    /**
+     * Drop an agent that did not acknowledge a dispatch. Its other in-flight jobs
+     * (a multi-slot agent may run more than the one that timed out) go through
+     * {@link onAgentDisconnect}, which reads the registration before its first
+     * await to tell a scaler-managed agent from a static one, so the triage
+     * starts first. The unregister right after removes the agent from routing at
+     * once, so a redispatch cannot re-select it while the triage awaits.
+     */
+    private dropUnacknowledgingAgent;
     /**
      * Handle an explicit dispatch rejection (`job.reject`) from an agent.
      * Undoes the dispatch accounting and requeues the job for another agent.
@@ -634,6 +654,8 @@ export declare class Dispatcher {
      * Used by the cancel-run API. Returns null if the job is not tracked (e.g. already completed).
      */
     getAgentIdForJob(jobId: string): string | null;
+    /** The jobs of `runId` this dispatcher currently tracks to an agent. */
+    getTrackedJobIdsForRun(runId: string): string[];
     /**
      * Start periodic cleanup of expired grace window entries.
      * Should be called during server startup.

package/dist/agent/registry.d.ts

@@ -263,6 +263,37 @@ export declare class AgentRegistry {
      * registry behaves exactly as before.
      */
     constructor(deps?: AgentRegistryDeps);
+    /**
+     * Triages the in-flight jobs of an agent this registry drops on its own (a
+     * revoked or expired token): fails or requeues a scaler-managed agent's
+     * jobs, starts recovery for a static agent's. Installed by the dispatcher,
+     * which is built after the registry.
+     */
+    private disconnectTriage?;
+    /** Install the job triage {@link disconnectByTokenId} runs for each agent it drops. */
+    setDisconnectTriage(triage: (agentId: string) => Promise<unknown>): void;
+    /**
+     * Sockets whose `agent.register` is in flight (awaiting the scaler lookup),
+     * per agent id, with the id of the agent token each authenticated with
+     * (`null` without token auth). Kept here rather than in the WebSocket
+     * handler because the coordinator builds one handler per connection: a
+     * socket's close, and the check a registration makes against another agent
+     * token, must see a registration in flight on another connection.
+     */
+    private readonly registeringSockets;
+    /** Record that `ws`, authenticated with `tokenId`, has started registering `agentId`. */
+    beginRegistering(agentId: string, ws: WsLike, tokenId: string | null): void;
+    /** Record that `ws` has finished registering `agentId`, however it ended. */
+    endRegistering(agentId: string, ws: WsLike): void;
+    /** Whether a socket other than `ws` is registering `agentId` right now. */
+    isRegisteringElsewhere(agentId: string, ws: WsLike): boolean;
+    /**
+     * Whether another connection holds `agentId` under an agent token other than
+     * `tokenId`: the live registration, or a registration in flight on another
+     * socket. A registration without token auth (`null`) holds no token identity
+     * and conflicts with nothing.
+     */
+    isAgentIdHeldByOtherToken(agentId: string, tokenId: string, ws: WsLike): boolean;
     /**
      * Register an agent. If the agentId already exists, updates the existing
      * entry (agent reconnection scenario).

package/dist/app.d.ts

@@ -41,6 +41,7 @@ import type { PendingInitTracker } from './cache/index.js';
 import type { PendingDynamicTracker, PendingGlobalEvalTracker } from './cache/index.js';
 import type { GlobalEvalRoundCache } from './cache/index.js';
 import type { CacheStorage } from './storage/types.js';
+import type { CacheUploadSettle } from './cache/cache-upload-settle.js';
 import type { ProvenanceTrustRoot } from './provenance/trust-root.js';
 import type { LocalSigner } from './oidc/local-dev-signer.js';
 import type { CheckRunReporter } from './reporting/check-run-reporter.js';
@@ -178,6 +179,11 @@ export interface AppDependencies {
     dispatchCacheRefs?: DispatchCacheRefTracker;
     /** Cache storage backend (S3) for metadata operations on upload completion. */
     cacheStorage?: CacheStorage;
+    /**
+     * In-flight build-cache publishes plus the per-org settle bound. Constructed
+     * once by orchestrator-core: the agent-WS factory below runs per connection.
+     */
+    cacheUploadSettle?: CacheUploadSettle;
     /** Provenance trust root used to verify build-provenance bundles at ingest. */
     provenanceTrustRoot?: ProvenanceTrustRoot;
     /**

package/dist/cache/cache-upload-settle.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Holding a build job's success until the cache publish its agent reported
+ * just before it has finished.
+ *
+ * A build agent sends `cache.upload.complete` and then, moments later, the
+ * build's terminal `job.status success`. The agent WebSocket handler runs
+ * frames concurrently, so without this record the success can release the
+ * jobs that wait on the build while the pointer naming the uploaded tarball is
+ * still being written — and they dispatch without the cache URL.
+ *
+ * The upload frame registers here before its first `await`. Frames enter the
+ * handler in wire order, so the registration is in place by the time the
+ * success frame is handled. The success then waits, bounded by the org's
+ * `cache_upload_settle_timeout_ms` (cluster default
+ * `KICI_CACHE_UPLOAD_SETTLE_TIMEOUT_MS`).
+ */
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+/** How a build success's wait for its cache publish ended. */
+export declare enum CacheUploadSettleOutcome {
+    /** No publish was in flight for the job. */
+    None = "none",
+    /** The effective timeout is 0: the wait is turned off. */
+    Disabled = "disabled",
+    /** Every in-flight publish finished, successfully or not. */
+    Settled = "settled",
+    /** The bound passed first. The publish may still finish afterwards. */
+    TimedOut = "timed-out"
+}
+/**
+ * The cache publishes in flight, per job. A full build registers two (the
+ * dependency tarball and the source tarball); a publish always settles —
+ * success or failure — so a waiter never sees a rejection.
+ */
+export declare class InFlightCacheUploads {
+    private readonly inFlight;
+    /** Register one publish for `jobId`; the returned function settles it (idempotent). */
+    begin(jobId: string): () => void;
+    has(jobId: string): boolean;
+    /** Number of jobs with a publish in flight. */
+    get size(): number;
+    /**
+     * Wait for every publish registered for `jobId` at call time, bounded by
+     * `timeoutMs`. A timeout leaves the records in place: they clear when their
+     * publishes settle.
+     */
+    waitFor(jobId: string, timeoutMs: number): Promise<CacheUploadSettleOutcome>;
+}
+/** What the agent WebSocket handler needs to hold a build's success. */
+export interface CacheUploadSettle {
+    uploads: InFlightCacheUploads;
+    /** The effective bound for the job's org (org override, else cluster default). */
+    timeoutMsFor: (orgId: string | undefined) => Promise<number>;
+}
+export interface CacheUploadSettleResult {
+    outcome: CacheUploadSettleOutcome;
+    timeoutMs?: number;
+    waitedMs?: number;
+}
+/**
+ * Wait for a job's in-flight cache publishes. Reads the setting only when a
+ * publish is in flight, so an ordinary job's success costs no query.
+ */
+export declare function awaitCacheUploadSettle(settle: CacheUploadSettle | undefined, job: {
+    jobId: string;
+    orgId: string | undefined;
+}): Promise<CacheUploadSettleResult>;
+/**
+ * Resolve the settle bound for an org: `org_settings.cache_upload_settle_timeout_ms`
+ * when set (0 included), otherwise `fallbackMs`. A missing DB, org, row or
+ * value, or a failed query, falls back — the caller is a frame handler that
+ * must not reject.
+ */
+export declare function createCacheUploadSettleTimeoutReader(db: Kysely<Database> | undefined, fallbackMs: number): (orgId: string | undefined) => Promise<number>;
+//# sourceMappingURL=cache-upload-settle.d.ts.map

package/dist/cache/pending-inits.d.ts

@@ -28,6 +28,15 @@ export interface InitResult {
      * explicit `false` suppresses.
      */
     filterPassed?: boolean;
+    /**
+     * Verdict on the workflow's deferred `paths`, reported only when the init job
+     * was asked to decide them. `false` means the agent's clone diff matched none
+     * of them and the job must not be dispatched.
+     *
+     * Optional on purpose: an agent that predates deferred paths never sends it,
+     * so absence means "no verdict" and dispatch proceeds.
+     */
+    pathsPassed?: boolean;
 }
 export declare class PendingInitTracker extends PendingTracker<InitResult> {
     constructor();

package/dist/cancel/cancel-run.d.ts

@@ -33,11 +33,12 @@ export interface CancelRunDeps {
      */
     instanceId?: string;
     /**
-     * Forward `job.cancel` to the coordinator holding the agent's socket.
-     * Returns whether the send left this process. Undefined ⇒ no peer transport,
-     * so a job owned by a sibling is unreachable rather than orphaned.
+     * Forward `job.cancel` to the coordinator holding the agent's socket, with
+     * `force` when the cancel is forced. Returns whether the send left this
+     * process. Undefined ⇒ no peer transport, so a job owned by a sibling is
+     * unreachable rather than orphaned.
      */
-    cancelJobOnPeer?: (peerId: string, runId: string, jobId: string, reason: string) => boolean;
+    cancelJobOnPeer?: (peerId: string, runId: string, jobId: string, reason: string, force?: boolean) => boolean;
     /** How stale a `cluster_instances` heartbeat may be and still read as live. */
     ownershipGraceMs?: number;
     /**
@@ -112,4 +113,21 @@ export declare class HeldRunCancelRefusedError extends Error {
  * returns a structured already-terminal result).
  */
 export declare function cancelRunWithReason(deps: CancelRunDeps, runId: string, reason: string, options?: CancelRunOptions): Promise<CancelRunResult>;
+/**
+ * Send `job.cancel` down this coordinator's own socket for the agent.
+ *
+ * Only an OPEN socket can receive it. A socket mid-close is still in the
+ * registry until its close handler unregisters it, and sending to it throws, so
+ * a non-OPEN socket leaves the job un-notified and the orphan sweep takes it.
+ * The send is wrapped so one bad socket cannot abort the whole cancellation and
+ * strand the run in `cancelling`.
+ */
+export declare function sendLocalCancel(args: {
+    registry: Pick<AgentRegistry, 'get'>;
+    runId: string;
+    jobId: string;
+    agentId: string | null;
+    reason: string;
+    force: boolean;
+}): boolean;
 //# sourceMappingURL=cancel-run.d.ts.map

package/dist/cancel/peer-job-cancel.d.ts

@@ -0,0 +1,50 @@
+/**
+ * The receiving end of `peer.job.cancel`.
+ *
+ * A sibling forwards a cancel to the coordinator its agent is connected to,
+ * which it reads from the host roster. That coordinator's dispatcher tracks the
+ * job only when it dispatched or re-adopted it: an agent that reconnected here
+ * from a sibling, or one not yet back after this process restarted, is known
+ * only to `dispatch_queue.agent_id`. So the agent is resolved from the
+ * dispatcher first and from the dispatch record second, and a cancel that
+ * cannot be delivered is logged with the reason.
+ */
+import type { Kysely } from 'kysely';
+import { z } from 'zod';
+import type { PeerJobCancel } from '@kici-dev/engine';
+import type { Database } from '../db/types.js';
+import type { AgentRegistry } from '../agent/registry.js';
+import type { Dispatcher } from '../agent/dispatcher.js';
+export declare const PeerCancelOutcome: z.ZodEnum<{
+    "agent-not-connected": "agent-not-connected";
+    delivered: "delivered";
+    "lookup-failed": "lookup-failed";
+    "not-tracked": "not-tracked";
+}>;
+export type PeerCancelOutcome = z.infer<typeof PeerCancelOutcome>;
+/** A live dispatch row: the job and the agent it was handed to. */
+export interface DispatchedAgent {
+    jobId: string;
+    agentId: string | null;
+}
+export interface PeerCancelDelivery extends DispatchedAgent {
+    outcome: PeerCancelOutcome;
+}
+export interface PeerJobCancelDeps {
+    dispatcher: Pick<Dispatcher, 'getAgentIdForJob' | 'getTrackedJobIdsForRun'>;
+    registry: Pick<AgentRegistry, 'get'>;
+    /** The durable dispatch record. Absent on a worker, which has no database. */
+    lookupDispatched?: (runId: string, jobId: string | undefined) => Promise<DispatchedAgent[]>;
+    logger: {
+        info(message: string, meta?: object): unknown;
+        warn(message: string, meta?: object): unknown;
+    };
+}
+/** Read the live dispatch rows of a run, or of one job of it. */
+export declare function readDispatchedAgents(db: Kysely<Database>, runId: string, jobId: string | undefined): Promise<DispatchedAgent[]>;
+/**
+ * Deliver a `peer.job.cancel` to the agents connected here. Resolves; never
+ * rejects: every failure is an outcome and a warn line.
+ */
+export declare function deliverPeerJobCancel(deps: PeerJobCancelDeps, msg: PeerJobCancel): Promise<PeerCancelDelivery[]>;
+//# sourceMappingURL=peer-job-cancel.d.ts.map

package/dist/cancel/sweep-stuck-cancelling.d.ts

@@ -3,7 +3,9 @@ import type { Database } from '../db/types.js';
 export interface SweepStuckCancellingDeps {
     db: Kysely<Database>;
     /** The canonical cancel path, bound to this orchestrator's deps. */
-    cancelRun: (runId: string, reason: string) => Promise<unknown>;
+    cancelRun: (runId: string, reason: string, options: {
+        force: boolean;
+    }) => Promise<unknown>;
     /**
      * How long a run may sit in `cancelling` before it is re-driven. Twice the
      * recovery grace period: long enough that a graceful agent teardown finishes

package/dist/cli/commands/local-hook.d.ts

@@ -1,7 +1,8 @@
 /**
  * Render the post-receive hook script. For each pushed ref the hook calls
- * `kici-admin source trigger-local <id> --ref <ref> --sha <new> --base-url <url>`,
- * so a push to any branch dispatches a run for that ref.
+ * `kici-admin source trigger-local <id> --ref <ref> --before <old> --sha <new> --base-url <url>`,
+ * so a push to any branch dispatches a run for that ref, and its changed files
+ * are the commits the push added (`<old>..<new>`).
  */
 export declare function renderPostReceiveHook(opts: {
     sourceId: string;

package/dist/cli/commands/local-trigger.d.ts

@@ -5,6 +5,11 @@ export interface LocalTriggerInput {
     event: 'push' | 'pull_request';
     ref: string;
     sha: string;
+    /**
+     * The commit before the push. With it the changed files are `before..sha`;
+     * without it the push has no range and path filters match conservatively.
+     */
+    before?: string;
     defaultBranch: string;
 }
 export interface LocalTriggerRequest {
@@ -19,6 +24,8 @@ export declare function readRepoHead(repoPath: string): {
     ref: string;
     sha: string;
 };
+/** The parent of `sha` in a local repo, or undefined for a root commit or an unreadable repo. */
+export declare function readParentSha(repoPath: string, sha: string): string | undefined;
 /** POST the trigger request to the orchestrator base URL. Returns the HTTP status. */
 export declare function sendLocalTrigger(baseUrl: string, req: LocalTriggerRequest): Promise<number>;
 //# sourceMappingURL=local-trigger.d.ts.map