@kici-dev/orchestrator 0.3.0 → 0.5.0

package/dist/__test-helpers__/mock-db.d.ts

@@ -72,6 +72,10 @@ export interface MockDbMocks {
     insertExecute: ReturnType<typeof vi.fn>;
     insertExecuteTakeFirstOrThrow: ReturnType<typeof vi.fn>;
     onConflict: ReturnType<typeof vi.fn>;
+    /** The upsert's ON CONFLICT DO UPDATE SET payload — shared across both the
+     *  column() and columns() shapes, so a test can assert what an existing row
+     *  is actually updated with. */
+    doUpdateSet: ReturnType<typeof vi.fn>;
     updateTable: ReturnType<typeof vi.fn>;
     updateSet: ReturnType<typeof vi.fn>;
     updateWhere: ReturnType<typeof vi.fn>;

package/dist/agent/agent-version.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Comparing a self-reported agent version against a feature's minimum.
+ *
+ * An agent reports its own package version in `agent.register`
+ * (`agentRegisterSchema.version`), and the field is optional — an agent old
+ * enough to omit it predates every feature this module gates on. The
+ * orchestrator and the agent are deployed and upgraded independently
+ * (`.claude/rules/compatibility.md`), so a feature the orchestrator ships is
+ * routinely dispatched to a fleet that cannot run it, and the version is the
+ * only fact the orchestrator has about what the fleet understands.
+ */
+/** A version's release triple, with any prerelease suffix dropped. */
+type VersionBase = [number, number, number];
+/**
+ * Parse `MAJOR.MINOR.PATCH`, ignoring any `-prerelease` suffix, or `null` when
+ * the string is not a version at all.
+ *
+ * The prerelease suffix is dropped rather than ordered because the suffixes in
+ * play are build counters from the dev registry (`0.5.0-9159`), not semver
+ * release candidates. Strict semver orders those BELOW `0.5.0`, which would
+ * read every staging agent as too old for a feature it in fact carries.
+ */
+export declare function parseVersionBase(version: string): VersionBase | null;
+/**
+ * True when `version` is at least `minimum`, comparing release bases only.
+ *
+ * An absent or unparseable version returns `false`: the orchestrator learns
+ * nothing from it, and a feature that needs a specific agent build must not
+ * treat "we cannot tell" as "yes". Callers decide what an all-unknown fleet
+ * means for them — this function only reports what the version proves.
+ */
+export declare function agentVersionAtLeast(version: string | null | undefined, minimum: string): boolean;
+export {};
+//# sourceMappingURL=agent-version.d.ts.map

package/dist/agent/dispatcher.d.ts

@@ -2,6 +2,26 @@ import type { AgentRegistry } from './registry.js';
 import { type JobQueue, type QueuedJob, type QueuedJobInput } from '../queue/job-queue.js';
 import type { ScaleResult, ScalerRedispatchTrigger } from '../scaler/types.js';
 import type { ResourceRequest } from '@kici-dev/engine';
+/**
+ * Why a freed agent's queue drain claimed nothing.
+ *
+ * Named rather than inlined because the value travels: it is the `reason`
+ * field of the drain-declined log line and the assertion subject of the tests
+ * that pin each exit. The five members are the five ways
+ * `Dispatcher.drainForAgent` can return without a dispatch.
+ */
+export declare enum AgentDrainDecline {
+    /** The agent id is not in the in-memory registry (already disconnected). */
+    NotRegistered = "agent-not-registered",
+    /** This coordinator is draining and must not claim new work. */
+    CoordinatorDraining = "coordinator-draining",
+    /** The agent is already at `maxConcurrency`. */
+    NoCapacity = "no-capacity",
+    /** The host is flagged reboot-pending, so anything dispatched would be lost. */
+    RebootPending = "reboot-pending",
+    /** Nothing in the queue matched this agent's labels or pin. */
+    NoMatchingJob = "no-matching-job"
+}
 /**
  * Default per-pass cap on how many pending jobs a capacity-freed re-drive
  * re-offers to the scaler. Bounds the burst so a single free event cannot storm
@@ -85,6 +105,12 @@ export declare class Dispatcher {
      * the same tick.
      */
     private redriveInFlight;
+    /**
+     * Single-flight guard for `redrivePendingToConnectedAgents`. The per-coord
+     * safety-net tick must not overlap itself, so a slow re-drive can't be
+     * re-entered by the next interval fire and double-scan the same pending rows.
+     */
+    private pendingRedriveInFlight;
     /**
      * Tracks which jobs are dispatched to which agents.
      * Used for marking jobs as failed on agent disconnect.
@@ -268,6 +294,12 @@ export declare class Dispatcher {
      */
     releaseRebootPending(agentId: string): Promise<void>;
     onAgentAvailable(agentId: string): Promise<void>;
+    /**
+     * Claim at most one queued job for a freed agent.
+     *
+     * Returns the reason nothing was dispatched, or null when a job was.
+     */
+    private drainForAgent;
     /** Record that a job began executing on its agent. */
     markJobStarted(jobId: string): void;
     /**
@@ -316,6 +348,44 @@ export declare class Dispatcher {
      * fresh agent is spawned bound to this job.
      */
     private redispatch;
+    /**
+     * Safety-net re-drive: deliver every pending job that a currently-connected,
+     * idle, matching agent could take, through the same atomic claim the drain
+     * uses (`dispatchBoundJob` → `dequeueById`).
+     *
+     * Runs per-coordinator and is NOT leader-gated: each coordinator drains the
+     * shared queue onto its OWN connected agents, and the atomic claim guarantees
+     * at most one agent (on any coordinator) wins each job, so concurrent ticks
+     * cannot double-dispatch.
+     *
+     * It closes the requeue re-drive gap. A job requeued by `handleAckExpiry` /
+     * `onJobRejected` / the leader ack sweep gets exactly one delivery attempt —
+     * `redispatch`'s single `findAvailable` + `dispatchBoundJob`. An idle matching
+     * agent whose own drain trigger (registration / completion / status) already
+     * fired before the requeue has no further trigger, so if that one attempt
+     * transiently misses the agent — the agent is momentarily at capacity while an
+     * in-flight drain holds its eagerly-claimed slot, or the agent is connected to
+     * a different coordinator than the one that ran the expiry — the requeued
+     * pending job would otherwise sit undelivered until it expired. This tick
+     * re-attempts delivery onto connected agents so the miss recovers on the next
+     * sweep.
+     *
+     * It never consults the scaler (that is `retryPendingScaleRequests`) and never
+     * spawns: it only places jobs onto agents already connected here.
+     *
+     * A failure propagates to the caller rather than being swallowed here — the
+     * per-coord interval wrapper is the single error-log site, matching the
+     * sibling recovery/ack sweeps that share its cadence. The `finally` only
+     * releases the single-flight guard.
+     */
+    redrivePendingToConnectedAgents(maxJobs?: number): Promise<number>;
+    /**
+     * Pick a connected agent that may take a pending job, or null. A pinned
+     * host-fanout child may run ONLY on its pinned agent — and `dequeueById`
+     * ignores the pin — so a pinned job is routed to its own agent and never
+     * offered to `findAvailable`, which would mis-deliver it to any label match.
+     */
+    private selectConnectedTargetForPending;
     /**
      * Handle agent disconnect. Starts per-job recovery timers instead of
      * immediately failing jobs, giving the agent a grace period to reconnect.

package/dist/app.d.ts

@@ -26,6 +26,7 @@ import type { PeerRegistry } from './cluster/peer-registry.js';
 import type { JobQueue } from './queue/job-queue.js';
 import type { DedupCache } from './webhook/dedup.js';
 import type { LockFileCache } from './lockfile-cache.js';
+import type { ContentRequirementsCache } from './content-requirements-cache.js';
 import type { PlatformClient } from './ws/platform-client.js';
 import type { ScalerManager } from './scaler/manager.js';
 import type { ProviderRegistry } from './provider-registry.js';
@@ -37,7 +38,8 @@ import type { ArtifactStore } from './artifacts/artifact-store.js';
 import type { DispatchCacheRefTracker } from './cache/index.js';
 import type { PendingBuildTracker } from './cache/index.js';
 import type { PendingInitTracker } from './cache/index.js';
-import type { PendingDynamicTracker } from './cache/index.js';
+import type { PendingDynamicTracker, PendingGlobalEvalTracker } from './cache/index.js';
+import type { GlobalEvalRoundCache } from './cache/index.js';
 import type { CacheStorage } from './storage/types.js';
 import type { ProvenanceTrustRoot } from './provenance/trust-root.js';
 import type { LocalSigner } from './oidc/local-dev-signer.js';
@@ -66,6 +68,7 @@ import type { CronScheduler } from './cron/cron-scheduler.js';
 import { type ConfigRouteDeps } from './routes/admin-config.js';
 import { WebhookIngestOutcome } from './pipeline/process-webhook.js';
 import type { IngestOverflowBuffer } from './webhook/ingest-overflow-buffer.js';
+import type { IngestOverflowReplayer } from './webhook/ingest-overflow-replayer.js';
 import { type OverflowDelivery } from './webhook/ingest-overflow-types.js';
 import type { IngestAdmissionController } from './webhook/ingest-admission.js';
 import type { OrgIngestCapReader } from './webhook/org-ingest-cap-reader.js';
@@ -108,6 +111,7 @@ export interface AppDependencies {
     jobQueue: JobQueue;
     dedup: DedupCache;
     lockFileCache: LockFileCache;
+    contentRequirementsCache: ContentRequirementsCache;
     providerRegistry: ProviderRegistry;
     platformClient?: PlatformClient;
     /**
@@ -126,6 +130,13 @@ export interface AppDependencies {
      * for later replay; absent → capture disabled.
      */
     ingestOverflowBuffer?: IngestOverflowBuffer;
+    /**
+     * Drain loop for the durable ingest queue. The accept path borrows its
+     * claim-release so a failed pipeline hands the row back for retry through the
+     * same attempt ceiling the drain uses; absent → the accept path degrades to
+     * inline ingestion.
+     */
+    ingestOverflowReplayer?: IngestOverflowReplayer;
     /**
      * Fulfil deferred attestations on demand (mints in this process, which owns
      * the Platform WS). Backs `POST /api/v1/admin/attestations/retry`. Wired only
@@ -208,6 +219,8 @@ export interface AppDependencies {
     pendingInits?: PendingInitTracker;
     /** Pending dynamic tracker for DynamicJobFn evaluation coordination. */
     pendingDynamics?: PendingDynamicTracker;
+    pendingGlobalEvals?: PendingGlobalEvalTracker;
+    globalEvalCache?: GlobalEvalRoundCache;
     /** Commit status reporter for setting pending/success/failure/error on commits. Optional. */
     checkRunReporter?: CheckRunReporter;
     /** Execution tracker for DB persistence of execution state. Optional — requires database. */
@@ -250,6 +263,15 @@ export interface AppDependencies {
     eventEmitter?: EventEmitter;
     /** Generic webhook source manager. Optional -- if not set, generic webhooks are disabled. */
     genericSourceManager?: GenericSourceManager;
+    /**
+     * Re-register a generic source's provider bundle from its database row,
+     * resolving to true when a per-routing-key bundle is now registered.
+     *
+     * Built in orchestrator-core so it shares the exact dependency bag the
+     * startup enumeration and the LISTEN/NOTIFY drain already register with.
+     * Optional -- wirings without one keep the previous behaviour.
+     */
+    ensureProviderBundle?: (routingKey: string) => Promise<boolean>;
     /** GitHub direct-ingress route deps (hybrid/independent only). Optional -- if not set, the direct GitHub ingress route is not mounted. */
     githubSourceStore?: SourceStore;
     githubVerifyDeps?: VerifyInboundDeps;

package/dist/cache/global-eval-round-cache.d.ts

@@ -0,0 +1,88 @@
+/**
+ * LRU for completed Tier-2 global eval round results.
+ *
+ * **This cache only helps webhook redelivery and same-input re-deliveries.** Its
+ * key pins the source commit and a digest of the whole round input, and every
+ * real push carries a new source SHA — so a developer pushing twice gets two
+ * rounds, by design. Read it as a duplicate-delivery guard, never as a
+ * steady-state optimization: sizing it up buys nothing, and a hit rate near zero
+ * on a healthy cluster is the expected shape, not a defect.
+ *
+ * There is deliberately no TTL. A round verdict is a pure function of its
+ * inputs, and {@link globalEvalRoundCacheKey} covers all of them, so an entry
+ * cannot go stale within its own key — the same content-addressable argument the
+ * lock-file and content caches make for their SHA-keyed entries, without their
+ * branch-name escape hatch.
+ */
+import type { GlobalEvalRoundResult } from '@kici-dev/engine';
+/**
+ * Key one round over **every input its verdict depends on**.
+ *
+ * The three SHAs are not enough, and the gap is a wrong-answer bug rather than a
+ * missed-hit one. `createFilterContext` and `buildGeneratorContext` both receive
+ * the whole event, so a filter can branch on `event.type`, `targetBranch`,
+ * `changedFiles`, or anything in the raw payload. A push to `main` at commit X
+ * and a pull-request synchronize whose head is commit X share all three SHAs and
+ * genuinely deserve different verdicts: their events differ, and their
+ * changed-file sets are computed over different ranges. Two branches pointing at
+ * one commit collide the same way. `workflowRoutingKey` is in the key for the
+ * reason `groupCandidates` keeps it in the group key — it selects the provider
+ * bundle that mints the clone credentials.
+ *
+ * The candidate list and the event are folded in as a SHA-256 digest of their
+ * JSON, so the key stays a bounded string no matter how large a payload is. Both
+ * are rebuilt by the same code from the same delivery, so a genuine redelivery
+ * reproduces the digest; anything else is a miss, which costs a round rather
+ * than a wrong verdict.
+ *
+ * Returns `null` when the inputs cannot be serialized. A round that cannot be
+ * keyed is simply not cached — never keyed on a partial input, which is how a
+ * cache starts answering questions it was not asked.
+ */
+export declare function globalEvalRoundCacheKey(args: {
+    workflowRepoIdentifier: string;
+    workflowSha: string;
+    workflowRoutingKey: string;
+    sourceSha: string;
+    /** The exact per-candidate payload the round job carries. */
+    candidates: unknown;
+    /** The exact event the round job carries. */
+    event: unknown;
+}): string | null;
+/**
+ * A round that left ANY candidate undecided is not a result worth replaying.
+ *
+ * An agent-side budget breach reports `success` with the affected candidates
+ * marked indeterminate — the whole round on a round-budget breach, one workflow
+ * on a candidate-budget breach. Caching either turns one slow round into a
+ * permanently replayed failure for the whole key: a webhook redelivery, which
+ * is exactly when an operator is retrying, would be served the stored failure
+ * instead of running again, and the round's own retry sits behind this cache
+ * read so a stored breach short-circuits that too.
+ *
+ * Requiring EVERY candidate to be decided costs almost no hit rate — an
+ * indeterminate verdict is the exceptional path — and buys a guarantee that is
+ * easy to state: nothing a redelivery replays was ever undecided.
+ *
+ * An empty candidate list is not cacheable either. For a non-empty group it
+ * means every candidate is about to be recorded as "no verdict", which is the
+ * same failure by another route.
+ */
+export declare function isCacheableRoundResult(result: GlobalEvalRoundResult): boolean;
+export declare class GlobalEvalRoundCache {
+    private readonly cache;
+    private hits;
+    private misses;
+    constructor(options: {
+        max: number;
+    });
+    get(key: string): GlobalEvalRoundResult | undefined;
+    set(key: string, value: GlobalEvalRoundResult): void;
+    /** Hit/miss counters for metrics and debugging. */
+    stats(): {
+        hits: number;
+        misses: number;
+        size: number;
+    };
+}
+//# sourceMappingURL=global-eval-round-cache.d.ts.map

package/dist/cache/index.d.ts

@@ -15,4 +15,7 @@ export { PendingBuildTracker } from './pending-builds.js';
 export { PendingInitTracker } from './pending-inits.js';
 export type { InitResult } from './pending-inits.js';
 export { PendingDynamicTracker } from './pending-dynamics.js';
+export { PendingGlobalEvalTracker } from './pending-global-evals.js';
+export { GlobalEvalRoundCache, globalEvalRoundCacheKey, isCacheableRoundResult, } from './global-eval-round-cache.js';
+export { parseGlobalEvalResult } from './pending-global-evals.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/cache/pending-global-evals.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Pending tracker for the Tier-2 global eval round.
+ *
+ * The orchestrator dispatches one round job per (event × workflow repo) and
+ * waits for the agent to return a verdict per candidate workflow. The
+ * underlying tracker logic lives in `PendingTracker<GlobalEvalRoundResult>`;
+ * this subclass wires the round-specific logger prefix and disconnect error.
+ *
+ * Unlike the dynamic-eval and init trackers, this round runs BEFORE any run row
+ * exists: its whole purpose is to decide which global workflows produce a run at
+ * all, so creating one up-front would defeat it.
+ */
+import { type GlobalEvalRoundResult } from '@kici-dev/engine';
+import { PendingTracker } from './pending-tracker.js';
+/**
+ * Validate the round result carried on a `job.status` message before it reaches
+ * the tracker.
+ *
+ * **This is the one place the round's wire payload is validated.** The status
+ * message's `data` field is `z.record(z.string(), z.unknown())`, so nothing
+ * upstream checks `globalEvalResult` against its schema — a cast at the message
+ * handler would be a type assertion over an arbitrary agent-supplied value, and
+ * any agent that owns the round job can send one. Parsing here keeps the
+ * "typed but unvalidated" boundary at a single named function instead of spread
+ * across every consumer.
+ *
+ * The parsed value is widened back to {@link GlobalEvalRoundResult}: the schema
+ * is deliberately loose about `jobs` (the lock-job shape has no Zod mirror), and
+ * the dispatch path re-checks every generated job before it builds anything from
+ * one.
+ */
+export declare function parseGlobalEvalResult(raw: unknown): {
+    ok: true;
+    value: GlobalEvalRoundResult;
+} | {
+    ok: false;
+    error: string;
+};
+export declare class PendingGlobalEvalTracker extends PendingTracker<GlobalEvalRoundResult> {
+    constructor();
+}
+//# sourceMappingURL=pending-global-evals.d.ts.map

package/dist/cache/pending-inits.d.ts

@@ -18,6 +18,16 @@ export interface InitResult {
      * function. The dispatch path re-materializes these into N execution jobs.
      */
     matrixValues?: Array<Record<string, string | undefined>>;
+    /**
+     * Verdict of the workflow-level `filter`, reported only when the init job was
+     * asked to evaluate one. `false` means the workflow does not apply to this
+     * event and its job must not be dispatched.
+     *
+     * Optional on purpose: an agent that predates the filter never sends it, so
+     * absence means "no verdict was reported" and dispatch proceeds — only an
+     * explicit `false` suppresses.
+     */
+    filterPassed?: boolean;
 }
 export declare class PendingInitTracker extends PendingTracker<InitResult> {
     constructor();

package/dist/cli/commands/cluster-settings.d.ts

@@ -25,14 +25,52 @@ interface StringKnobSpec {
     expects: string;
     validate: (value: string) => boolean;
 }
+/**
+ * One boolean knob. A third spec type for the same reason {@link StringKnobSpec}
+ * is separate from {@link KnobSpec}: the value parser, the PATCH body type, and
+ * the `show` formatter all differ. A boolean also has no bounds to validate —
+ * only two accepted spellings.
+ */
+interface BooleanKnobSpec {
+    field: string;
+    flag: string;
+    label: string;
+}
 /** camelCase → value (null clears the override). */
-type PatchBody = Record<string, number | string | null>;
-/** Parse an integer CLI flag with a minimum, exiting with an error on failure. */
-export declare function parseKnobValue(flag: string, value: string, min: number): number;
+type PatchBody = Record<string, number | string | boolean | null>;
+/**
+ * Parse an integer CLI flag against its knob's bounds, exiting on failure.
+ *
+ * `max` is optional because most knobs have no meaningful ceiling; the ones
+ * that do carry a hard boot-safety bound (see {@link KnobSpec.max}).
+ */
+export declare function parseKnobValue(flag: string, value: string, min: number, max?: number): number;
 /** Validate a text CLI flag against its knob's shape, exiting on failure. */
 export declare function parseStringKnobValue(knob: StringKnobSpec, value: string): string;
+/**
+ * Parse a boolean CLI flag, exiting on failure.
+ *
+ * Accepts only the exact strings `true` and `false`. A permissive parser (any
+ * non-empty string is true) would read `--global-workflows-enabled no` as an
+ * enable, which for this knob turns a typo into a security posture change.
+ */
+export declare function parseBooleanKnobValue(knob: BooleanKnobSpec, value: string): boolean;
 /** Assemble the PATCH body from provided `set` flags (exits if none given). */
 export declare function buildClusterPatch(opts: Record<string, string | undefined>): PatchBody;
+/**
+ * Warn when only one of the two ordered global-eval timeouts is being set.
+ *
+ * The orchestrator's wait ceiling must exceed the agent's round budget — the
+ * agent's budget starts only once the round job is RUNNING, so a lower ceiling
+ * fires on every round that merely waited for a free agent, failing every round
+ * silently. The server rejects the pair outright when BOTH effective values are
+ * stored, but it cannot compare against a column left NULL: NULL means "the
+ * orchestrator's configured default applies", and the route does not know that
+ * number. Setting one alone is exactly that blind spot, so warn here.
+ *
+ * Returns the lines rather than printing them so the check is unit-testable.
+ */
+export declare function unpairedEvalTimeoutWarnings(patch: PatchBody): string[];
 /** Build the reset PATCH body: all knobs → null, or just the flagged ones. */
 export declare function buildClusterReset(opts: Record<string, boolean | undefined>): PatchBody;
 /**

package/dist/cli/commands/runs.d.ts

@@ -9,6 +9,7 @@
  *   runs list                          List execution runs (table or JSON)
  *   runs show <runId>                  Show run detail with jobs and steps
  *   runs jobs <runId>                  List jobs for a run (optional steps)
+ *   runs logs <runId> --job <jobId>    Print a page of one step's log lines
  *   runs ephemeral-key <runId>         Show scrub status of the run's key
  *   runs secret-outputs <runId>        List secret outputs (masked / reveal)
  *