@kici-dev/orchestrator 0.4.0 → 0.5.0

package/dist/webhook/ingest-accept.d.ts

@@ -0,0 +1,70 @@
+import type { WebhookInfo } from './handler.js';
+import { WebhookIngestOutcome } from '../pipeline/process-webhook.js';
+/**
+ * Seams the accept path needs. Each is a narrow function rather than the
+ * concrete collaborator so the sequencing below can be driven in a unit test
+ * without a database, an admission controller, or a pipeline.
+ */
+export interface IngestAcceptDeps {
+    /**
+     * Reserve an admission slot. Resolves to a release function when admitted, to
+     * `null` when the controller shed, or to `undefined` when no controller is
+     * wired (tests / minimal wirings — no gate).
+     */
+    admit: () => Promise<(() => void) | null | undefined>;
+    /**
+     * Advisory duplicate probe. Non-claiming: the atomic claim stays inside the
+     * pipeline, where it has always been. See {@link acceptWebhookDelivery}.
+     */
+    isKnownDelivery: (deliveryId: string) => Promise<boolean>;
+    /** Durably store the delivery; resolves to the row id, or null at the cap. */
+    enqueue: (info: WebhookInfo) => Promise<number | null>;
+    /** Take the row's claim (`buffered` → `replaying`). False when someone else has it. */
+    claimRow: (rowId: number) => Promise<boolean>;
+    /** Mark the row done so the drain pass sweeps it. */
+    markProcessed: (rowId: number) => Promise<void>;
+    /** Hand the claim back on failure so the drain pass retries the delivery. */
+    releaseClaim: (rowId: number, reason: string) => Promise<boolean>;
+    /** Run the match-and-dispatch pipeline. */
+    runPipeline: (info: WebhookInfo) => Promise<WebhookIngestOutcome>;
+    /**
+     * Schedule the post-acknowledgement work. Defaults to a detached microtask;
+     * a test passes a collector so it can await the work deterministically.
+     */
+    schedule?: (work: () => Promise<void>) => void;
+}
+/**
+ * Accept an inbound delivery and run its pipeline afterwards.
+ *
+ * The acknowledgement a caller receives means **the delivery is durably
+ * queued**, not that it was matched or dispatched. That is the whole point: a
+ * provider's delivery attempt times out in seconds (GitHub's is 10), while one
+ * matched workflow's build phase alone may legitimately take ten minutes, so a
+ * response that waited for the pipeline turned a slow build into a failed
+ * delivery for work that usually succeeded.
+ *
+ * Ordering is load-bearing and each step earns its place:
+ *
+ * 1. **Admit.** A shed delivery never reaches the queue, so admission stays the
+ *    first gate and a saturated orchestrator still answers 429 immediately.
+ * 2. **Probe for a duplicate.** Advisory only — a redelivery the orchestrator
+ *    already knows about is reported as one without a row being written. The
+ *    *authoritative* claim is still the pipeline's atomic
+ *    `INSERT … ON CONFLICT`, so a genuine cross-instance race is arbitrated
+ *    exactly where it always was; the loser simply learns after acknowledging.
+ * 3. **Enqueue durably.** Nothing is acknowledged before this row exists. At the
+ *    row cap the delivery is shed rather than acknowledged — answering 202 for
+ *    a delivery we did not store would turn a hang into silent data loss, which
+ *    is strictly worse than the hang this change removes.
+ * 4. **Claim the row**, then acknowledge, then run the pipeline detached. The
+ *    claim is what stops the drain pass re-injecting a delivery already in
+ *    flight; releasing it on failure is what makes the row a retry rather than
+ *    a tombstone. The admission slot is held across the pipeline, so in-flight
+ *    pipeline concurrency stays bounded exactly as it was when the response
+ *    waited for it.
+ *
+ * Returns `queued` on the acknowledge path, or `duplicate` / `shed` for the two
+ * pre-queue exits.
+ */
+export declare function acceptWebhookDelivery(info: WebhookInfo, deps: IngestAcceptDeps): Promise<WebhookIngestOutcome>;
+//# sourceMappingURL=ingest-accept.d.ts.map

package/dist/webhook/ingest-overflow-buffer.d.ts

@@ -7,10 +7,22 @@ export interface IngestOverflowBufferDeps {
     maxRows: number;
 }
 /**
- * Capture side of the durable overflow buffer. On a shed, the ingest path calls
- * {@link IngestOverflowBuffer.capture} additively (the 429 still stands). At the
- * row cap, capture drops the delivery from the buffer (never unbounded rows) —
- * the lossy fallback a retrying sender still covers.
+ * Capture side of the durable ingest queue. Two feeders write rows here.
+ *
+ * The **accept** path (HTTP direct ingress) enqueues every admitted delivery
+ * before the route answers 202: the row is what makes "durably queued" true, so
+ * a worker that dies mid-pipeline leaves recoverable work behind rather than a
+ * delivery the sender believes was accepted.
+ *
+ * The **shed** path enqueues additively when the admission controller rejects a
+ * delivery (the 429 still stands), so capacity recovering is enough to get the
+ * delivery processed without the sender redelivering.
+ *
+ * At the row cap, an enqueue fails rather than growing the table without bound.
+ * The two feeders differ in what that means: a shed delivery is dropped (the
+ * lossy fallback a retrying sender still covers), while the accept path must
+ * NOT answer 202 for a delivery it did not store — it sheds instead, so the
+ * sender learns the delivery was refused.
  */
 export declare class IngestOverflowBuffer {
     private readonly db;
@@ -18,10 +30,29 @@ export declare class IngestOverflowBuffer {
     constructor(deps: IngestOverflowBufferDeps);
     /** Current buffered-row depth. */
     currentDepth(): Promise<number>;
+    /**
+     * Persist a delivery as `buffered` and return its row id, or null when the
+     * buffer is at cap. Callers that must not acknowledge an unstored delivery
+     * check for null; the shed path treats null as a drop.
+     */
+    enqueue(d: OverflowDelivery): Promise<number | null>;
     /**
      * Persist a shed delivery. Returns true when a `buffered` row was inserted,
      * false when the buffer is at cap (delivery dropped, `cap_full` metric bumped).
      */
     capture(d: OverflowDelivery): Promise<boolean>;
+    /**
+     * Claim a specific row for processing: `buffered` → `replaying`, stamping the
+     * claim clock. Returns false when someone else already owns it — a drain pass
+     * on another instance, or a reclaim that ran while this caller was scheduling.
+     * The conditional update is the arbiter, so two workers can never both own a
+     * row and dispatch its delivery twice.
+     */
+    claimRow(id: number): Promise<boolean>;
+    /**
+     * Mark a claimed row done. The row is swept by the drain pass rather than
+     * deleted here, so the sweep stays the single deletion site.
+     */
+    markProcessed(id: number): Promise<void>;
 }
 //# sourceMappingURL=ingest-overflow-buffer.d.ts.map

package/dist/webhook/ingest-overflow-replayer.d.ts

@@ -1,5 +1,6 @@
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
+import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 import { WebhookIngestOutcome } from '../pipeline/process-webhook.js';
 import { type OverflowDelivery } from './ingest-overflow-types.js';
 export type ReinjectFn = (d: OverflowDelivery) => Promise<WebhookIngestOutcome>;
@@ -11,14 +12,29 @@ export interface IngestOverflowReplayerDeps {
     intervalMs: number;
     batchSize: number;
     maxAttempts: number;
+    /**
+     * How long a `replaying` claim may stand before it is reclaimed. Resolved per
+     * pass so a fleet-wide `cluster_settings` override takes effect without a
+     * restart; the number passed here is the configured cluster default.
+     */
+    claimTimeoutMs: number;
+    /** Fleet-wide override reader for {@link IngestOverflowReplayerDeps.claimTimeoutMs}. */
+    clusterSettings?: ClusterSettingsReader;
 }
 /**
- * Background drain for the durable overflow buffer. A pass runs only while the
- * admission controller is NOT shedding (replaying into a still-overloaded
- * orchestrator just re-sheds), claims the oldest `buffered` rows FIFO up to a
- * bounded batch, and re-injects each through the admission-gated ingest path.
- * A re-shed or error reverts the row to `buffered` (never lost); past the
- * max-attempts ceiling it goes `failed`. Successful rows are swept.
+ * Background drain for the durable ingest queue. Each pass first reclaims rows
+ * whose `replaying` claim went stale — a worker killed mid-pipeline releases
+ * nothing, so without this its delivery would sit claimed forever and the
+ * durable row it acknowledged would never be worth anything. It then claims the
+ * oldest `buffered` rows FIFO up to a bounded batch and re-injects each through
+ * the admission-gated ingest path. A re-shed or error reverts the row to
+ * `buffered` (never lost); past the max-attempts ceiling it goes `failed`.
+ * Successful rows are swept.
+ *
+ * Re-injection runs only while the admission controller is NOT shedding
+ * (replaying into a still-overloaded orchestrator just re-sheds). Reclaiming is
+ * not gated that way: a stranded claim is stranded regardless of load, and the
+ * row it frees simply waits in `buffered` until capacity returns.
  */
 export declare class IngestOverflowReplayer {
     private readonly db;
@@ -26,6 +42,8 @@ export declare class IngestOverflowReplayer {
     private readonly intervalMs;
     private readonly batchSize;
     private readonly maxAttempts;
+    private readonly claimTimeoutMs;
+    private readonly clusterSettings;
     private reinjectDirect;
     private reinjectRelay;
     private timer;
@@ -38,10 +56,36 @@ export declare class IngestOverflowReplayer {
     stop(): void;
     /** One drain pass. Test-drivable. */
     runPass(): Promise<void>;
+    /**
+     * Revert `replaying` rows whose claim went stale to `buffered`, counting the
+     * abandoned attempt so a row that keeps stranding eventually goes `failed`
+     * instead of looping forever.
+     *
+     * A row claimed before the claim clock shipped has a null `claimed_at`; it is
+     * aged off `captured_at` instead, which is a safe over-estimate of how long
+     * it has been claimed.
+     */
+    private reclaimStaleClaims;
+    /**
+     * Release a claim: revert the row to `buffered` so the drain retries it, or
+     * mark it `failed` once the attempt ceiling is hit. Returns false when the
+     * row is gone or is no longer `replaying` — another worker owns it now, so
+     * this caller must not touch it.
+     *
+     * Public because the accept path's background worker holds a claim it did not
+     * take through {@link claimBatch} and must release it the same way on failure.
+     */
+    releaseClaim(id: number, reason: string): Promise<boolean>;
     /** Select the oldest buffered rows and claim each via a conditional update. */
     private claimBatch;
     private toDelivery;
     private replayOne;
+    /**
+     * Move a claimed row back to `buffered` (or to `failed` at the attempt
+     * ceiling). Every update is conditional on the row still being `replaying`,
+     * so a caller whose claim was reclaimed underneath it cannot yank a row a
+     * different worker now owns. Returns whether this call moved the row.
+     */
     private revertOrFail;
     private sweepReplayed;
     private refreshDepthGauge;

package/dist/ws/agent-handler.d.ts

@@ -31,6 +31,7 @@ import type { DispatchCacheRefTracker } from '../cache/dispatch-cache-ref-tracke
 import type { PendingBuildTracker } from '../cache/pending-builds.js';
 import type { PendingInitTracker } from '../cache/pending-inits.js';
 import type { PendingDynamicTracker } from '../cache/pending-dynamics.js';
+import type { PendingGlobalEvalTracker } from '../cache/pending-global-evals.js';
 import type { CacheStorage } from '../storage/types.js';
 import type { AgentMetricsAggregator } from '../metrics/agent-metrics-aggregator.js';
 import { type AgentApiRegistry } from './agent-api-registry.js';
@@ -267,6 +268,8 @@ export interface AgentWsHandlerDeps {
     pendingInits?: PendingInitTracker;
     /** Optional pending dynamic tracker for cleanup on agent disconnect. */
     pendingDynamics?: PendingDynamicTracker;
+    /** Optional pending global-eval-round tracker for cleanup on agent disconnect. */
+    pendingGlobalEvals?: PendingGlobalEvalTracker;
     /** Optional callback when agent sends encrypted secret outputs on job success. */
     onSecretOutputs?: (runId: string, jobId: string, secretOutputs: Record<string, {
         agentPublicKey: string;

package/dist/ws/dashboard-global-workflows-handler.d.ts

@@ -2,10 +2,12 @@
  * Dashboard global-workflows handler for the orchestrator.
  *
  * Responds to `dashboard.global-workflows.*` WS messages from Platform by
- * reading or upserting the `org_settings` row keyed by `customer_id`. The
- * row holds the org-level global-workflow policy: master enable, plus the
- * three repo-pattern lists (allow / deny / elevate). Each list entry is a
- * `{routingKey?, pattern}` object that may optionally pin to one source.
+ * reading or upserting the `org_settings` row keyed by `customer_id`. The row
+ * holds the three per-org repo-pattern lists (allow / deny / elevate); each
+ * entry is a `{routingKey?, pattern}` object that may optionally pin to one
+ * source. The master enable switch is fleet-wide
+ * (`cluster_settings.global_workflows_enabled`) and read-only here — projected
+ * onto the response as `enabled`, never written from this handler.
  *
  * The handler binds a single `customerId` at construction time (via the
  * sources / generic_webhook_sources lookup at server startup) and updates
@@ -15,12 +17,17 @@ import { type Kysely } from 'kysely';
 import type { GlobalWorkflowsGetRequest, GlobalWorkflowsUpdateRequest, GlobalWorkflowSettings, RepoPatternEntry } from '@kici-dev/engine/protocol/dashboard-global-workflows';
 import type { Database, OrgSettings } from '../db/types.js';
 import type { AccessLogWriter } from '../audit/access-log.js';
+import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 interface DashboardGlobalWorkflowsHandlerDeps {
     /** Customer / org identifier — primary key on org_settings. */
     customerId: string;
     /** Send a response message back to Platform over the WS connection. */
     send: (msg: unknown) => void;
     db: Kysely<Database>;
+    /** Reads the fleet-wide master switch for the read-only `enabled` projection. */
+    clusterSettings: ClusterSettingsReader;
+    /** Applies when the cluster column is NULL — `config.globalWorkflowsEnabled`. */
+    globalWorkflowsEnabledDefault: boolean;
     /** Access log writer — records one row per read / mutation with actor attribution. */
     accessLog?: AccessLogWriter;
 }
@@ -53,11 +60,24 @@ export declare class DashboardGlobalWorkflowsHandler {
     private handleGet;
     private handleUpdate;
     private readRow;
+    /**
+     * The effective fleet-wide master switch, for the read-only projection the
+     * dashboard renders as a status badge.
+     *
+     * Uses the shared 10s-cached reader — unlike the admin route, which reads
+     * uncached because an operator does set-then-show. A dashboard badge that is
+     * up to 10s stale is fine; a second uncached read on every dashboard poll is
+     * not.
+     *
+     * An unreadable row resolves to the configured default here rather than
+     * failing closed. This value only decides what the UI displays; the actual
+     * gate is `GlobalWorkflowPolicy`, which does fail closed.
+     */
+    private effectiveEnabled;
     private upsertRow;
     private sendError;
 }
 interface NormalizedPatch {
-    enabled: boolean;
     allowedRepos: RepoPatternEntry[] | null;
     deniedRepos: RepoPatternEntry[] | null;
     elevatedRepos: RepoPatternEntry[] | null;
@@ -69,10 +89,11 @@ interface NormalizedPatch {
  */
 export declare function buildPatch(existing: OrgSettings | undefined, msg: GlobalWorkflowsUpdateRequest): NormalizedPatch;
 /**
- * Project a row into the public settings shape. Returns a defaulted "disabled"
- * settings object when the row does not yet exist, so callers always get a
- * renderable state.
+ * Project a row into the public settings shape. `enabled` is the effective
+ * fleet-wide master switch, supplied by the caller (read-only here). When the
+ * row does not yet exist, the three per-org lists project as null so callers
+ * always get a renderable state.
  */
-export declare function rowToSettings(customerId: string, row: OrgSettings | undefined): GlobalWorkflowSettings;
+export declare function rowToSettings(customerId: string, row: OrgSettings | undefined, enabled: boolean): GlobalWorkflowSettings;
 export {};
 //# sourceMappingURL=dashboard-global-workflows-handler.d.ts.map

package/dist/ws/execution-status-frame.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Build the `execution.status` frame the orchestrator forwards to the Platform.
+ *
+ * Extracted from `server.ts`'s `onExecutionStatusChange` hook so the projection
+ * from an {@link ExecutionContext} onto the wire message is a pure function with
+ * its own tests. The hook itself stays a thin adapter: it supplies the message
+ * id and the clock and hands the result to the Platform client.
+ *
+ * Every field here is conditional on the context actually carrying it. That is
+ * load-bearing for `workflowRepoIdentifier` in particular: the tracker records
+ * it only when the repository that DEFINES the workflow differs from the
+ * repository the run acted on, so the frame must stay silent for a per-repository
+ * run or "present" stops marking a cross-repository global run downstream.
+ */
+import type { ExecutionContext } from '../reporting/execution-tracker.js';
+import type { ExecutionRunStatus, ExecutionStatus, InitFailure } from '@kici-dev/engine';
+export interface ExecutionStatusFrameArgs {
+    messageId: string;
+    runId: string;
+    status: ExecutionRunStatus;
+    context: ExecutionContext;
+    jobCount: number;
+    startedAt: number;
+    timestamp: number;
+    completedAt?: number;
+    durationMs?: number;
+    failureReason?: string;
+    logBytes?: number;
+    initFailure?: InitFailure;
+}
+export declare function buildExecutionStatusFrame(args: ExecutionStatusFrameArgs): ExecutionStatus;
+//# sourceMappingURL=execution-status-frame.d.ts.map

package/dist/ws/platform-client.d.ts

@@ -36,6 +36,20 @@ export type ReplaySendResult = {
  */
 export type { ProviderSource } from '../entry-helpers.js';
 import type { ProviderSource } from '../entry-helpers.js';
+/**
+ * Hard ceiling on how long an admitted fire-and-forget relay pipeline may hold
+ * its admission slot before it is force-released. A pipeline running longer than
+ * this is treated as hung; releasing its slot (idempotently) prevents a
+ * permanent slot leak that would otherwise shrink capacity for every future
+ * webhook. Well above any legitimate ingest-pipeline duration.
+ *
+ * Exported because it is also the hard bound on anything a webhook waits for
+ * inline: the global eval round caps its raised wait ceiling at this value,
+ * since a ceiling past the point the relay force-releases the pipeline buys
+ * latency and no verdict. Two copies of "5 minutes" with nothing coupling them
+ * is exactly the pair that drifts.
+ */
+export declare const ADMITTED_PIPELINE_LIFETIME_MS: number;
 export interface PlatformClientOptions {
     /** WebSocket URL of the Platform relay. */
     url: string;

package/dist/ws/test-relay-handlers.d.ts

@@ -24,15 +24,6 @@ export interface TestRelayHandlerDeps extends ProcessingDeps {
     agentRegistry: NonNullable<ProcessingDeps['agentRegistry']>;
     cacheStorage?: CacheStorage;
     logStorage?: LogStorage;
-    /**
-     * Log writer that owns the in-flight append tracking. The logs cursor
-     * handler drains its pending appends for a terminal run before computing the
-     * `done` flag, so the final (fire-and-forget) log chunk can't be lost to a
-     * race with the run-status transition.
-     */
-    logWriter?: {
-        drain(runId: string): Promise<void>;
-    };
     accessLog?: AccessLogWriter;
     /** Canonical org id this orchestrator is bound to (for access_log attribution). */
     orgId?: string | null;
@@ -79,7 +70,41 @@ export interface TestRunStatusPayload {
     }>;
     done: boolean;
 }
-/** Snapshot a run's status + per-job status. `done` is true at a terminal run state. */
+/**
+ * Snapshot a run's status + per-job status. `done` is true at a terminal run state.
+ *
+ * ## Why this does not check ownership, even though the run now records an owner
+ *
+ * `handleTestTrigger` records the initiator on `execution_runs.triggered_by`,
+ * so comparing `msg.actor` against it here looks like free defence in depth. It
+ * is not, for three reasons that are worth writing down because the idea
+ * recurs.
+ *
+ * **The orchestrator cannot authenticate the actor.** It is asserted by the
+ * Platform over the authenticated control-plane connection; nothing here can
+ * verify it. A Platform that is lying can name any principal and pass the
+ * check, so enforcing would defend only against a Platform that reports the
+ * actor faithfully and forgets to check ownership — not against a hostile one.
+ *
+ * **The two gates would disagree.** The Platform deliberately admits a caller
+ * with unrestricted repository scope alongside the creator, because such a
+ * caller can already read these runs through the dashboard run plane. Repo
+ * patterns do not exist on this tier — only the actor arrives — so an
+ * organization owner reading a developer's run would be allowed there and
+ * refused here. Closing that would mean the Platform asserting "this caller is
+ * unrestricted", which is the first problem again with extra steps.
+ *
+ * **The version skew runs the wrong way.** This component is customer-deployed
+ * and versions independently of the Platform, so an orchestrator upgraded to
+ * enforce would start refusing legitimate reads against a Platform that has not
+ * changed — a break the customer triggers and we cannot roll back. Fail-closed
+ * is right for a security control and wrong for a compatibility surface, and
+ * this is both.
+ *
+ * So the actor is recorded, not enforced. Enforcing here needs the caller's
+ * effective authorization scope on the wire (so one policy is evaluated rather
+ * than two), and an actor this tier can establish independently.
+ */
 export declare function handleTestRunStatus(msg: TestRelayRunStatusRequest, deps: TestRelayHandlerDeps): Promise<TestRunStatusPayload | {
     error: string;
 }>;

package/installer-image-digests.json

@@ -1,7 +1,7 @@
 {
-  "version": "0.4.0",
+  "version": "0.5.0",
   "images": {
-    "kici-agent": "sha256:60f463973e2fa02c4e1d9311c9214e679269fa6ead3135b4c826789ca0278cbf",
-    "kici-orchestrator": "sha256:75312fc8bee960384aa2c57ff72cf2fe65bcd52d222278632c79a5f14ccd3e27"
+    "kici-agent": "sha256:f5c3557be41a50900a5f7045ac769079e219766f01ef290d33b11c479d4dcb9c",
+    "kici-orchestrator": "sha256:596989e4804e6111f4263168e7fd41e0be9dc50d796c8ad0952af3e6e6d8e930"
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kici-dev/orchestrator",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Customer-deployable orchestrator for the KiCI CI/CD stack. Receives webhook events (direct or via Platform relay), matches triggers against the lock file, and dispatches jobs to connected agents.",
   "keywords": [
     "ci",
@@ -84,8 +84,8 @@
     "ws": "^8.21.1",
     "yaml": "^2.9.0",
     "zod": "^4.4.3",
-    "@kici-dev/engine": "0.4.0",
-    "@kici-dev/shared": "0.4.0"
+    "@kici-dev/engine": "0.5.0",
+    "@kici-dev/shared": "0.5.0"
   },
   "kici": {
     "metrics": {
@@ -100,7 +100,7 @@
     "@types/dockerode": "^4.0.1",
     "@types/ws": "^8.18.1",
     "kysely-ctl": "^0.21.0",
-    "@kici-dev/agent": "0.4.0"
+    "@kici-dev/agent": "0.5.0"
   },
   "scripts": {
     "build": "node ../../scripts/build-service.mjs && tsgo --emitDeclarationOnly",