@kici-dev/orchestrator 0.4.0 → 0.6.0

package/dist/__test-helpers__/fake-scaler-state-store.d.ts

@@ -0,0 +1,24 @@
+/**
+ * In-memory stand-in for the `scaler_pending_claims` half of `ScalerStateStore`.
+ *
+ * It reproduces the three properties the real table enforces in SQL — a claim
+ * is consumable exactly once, a consume past `expires_at` finds nothing, and a
+ * per-agent invalidation removes every row for that agent — against an injected
+ * clock. That lets a test drive the whole register → redeem round trip without a
+ * database, so the single-use and TTL properties are executed rather than
+ * hand-fed through a stubbed answer.
+ *
+ * Only the pending-claim methods are implemented. Reaching any other
+ * `ScalerStateStore` method is a `TypeError` at the call site, not a silent
+ * no-op.
+ */
+import type { ScalerStateStore } from '../scaler/scaler-state-store.js';
+/**
+ * Build an in-memory pending-claim store.
+ *
+ * @param now Clock (epoch ms) the TTL is evaluated against. Pass the same clock
+ *   the `ClaimStore` under test uses, or its `expiresAt` lands in a different
+ *   era than this store's `now()` and every redeem reads as expired.
+ */
+export declare function makeFakeScalerStateStore(now?: () => number): ScalerStateStore;
+//# sourceMappingURL=fake-scaler-state-store.d.ts.map

package/dist/__test-helpers__/mock-db-query.d.ts

@@ -0,0 +1,130 @@
+/**
+ * Predicate + projection evaluation for the shared `createMockDb` harness.
+ *
+ * Kept BYTE-IDENTICAL in two places —
+ * `packages/orchestrator/src/__test-helpers__/mock-db-query.ts` and
+ * `packages/platform/src/__test-helpers__/mock-db-query.ts`. Each package owns an
+ * independent `createMockDb` builder and neither package may import the other, so
+ * the evaluator is duplicated rather than shared. `mock-db-mirror.test.ts` in both
+ * packages fails the moment the copies diverge.
+ *
+ * `createMockDb` used to return its configured rows verbatim, whatever the
+ * query asked for: a `.where(...)` or a `.select([...])` was recorded on a spy
+ * and then discarded. A test therefore observed only the rows it had itself
+ * configured, so dropping a filter or a projected column from the code under
+ * test changed nothing the test could see. This module is the evaluator that
+ * closes that: the mock now *applies* the query it was handed.
+ *
+ * Scope, stated precisely because the guarantee is partial by construction:
+ *
+ * - **Binary predicates are evaluated.** `.where(column, op, value)` for the
+ *   operators in {@link MockDbOperator}. That is the overwhelming majority of
+ *   the predicates this repo issues.
+ * - **Anything else is opaque and is NOT evaluated.** An expression-builder
+ *   callback (`.where((eb) => …)`) and a raw `sql` fragment carry their meaning
+ *   in code the mock cannot read, so they are recorded and skipped. A filter
+ *   written in one of those shapes is still unpinned.
+ * - **A predicate on a column the fixture row does not declare is NOT
+ *   evaluated.** The fixture, not the schema, is the source of truth here: a
+ *   test row is a partial literal, and a missing key means "this test did not
+ *   model that column", not "this column is NULL". Treating absence as NULL
+ *   would fail nearly every fixture in the repo for saying too little, while
+ *   catching nothing a fixture that *does* declare the column would not
+ *   already catch. The consequence is real and worth naming: to pin a filter,
+ *   a fixture row must mention the column that filter reads.
+ *
+ * Everything here is pure — no vitest, no chain state — so it is unit-tested
+ * directly in `mock-db-query.test.ts` rather than only through the mock.
+ */
+/** SQL comparison operators the mock evaluates. Anything else is opaque. */
+export declare enum MockDbOperator {
+    eq = "=",
+    neq = "!=",
+    neqAnsi = "<>",
+    lt = "<",
+    lte = "<=",
+    gt = ">",
+    gte = ">=",
+    in = "in",
+    notIn = "not in",
+    is = "is",
+    isNot = "is not",
+    like = "like",
+    notLike = "not like",
+    ilike = "ilike",
+    notILike = "not ilike"
+}
+/** A `.where(column, op, value)` predicate the mock can evaluate. */
+export interface BinaryPredicate {
+    kind: 'binary';
+    /** Column name with any `table.` qualifier stripped. */
+    column: string;
+    op: MockDbOperator;
+    value: unknown;
+}
+/** A predicate shape the mock cannot read (eb callback, raw sql, unknown op). */
+export interface OpaquePredicate {
+    kind: 'opaque';
+    /** Why it could not be parsed — surfaced in `explainSkippedPredicates`. */
+    reason: string;
+}
+export type MockDbPredicate = BinaryPredicate | OpaquePredicate;
+/** Drop a `table.` / `"table".` qualifier so a predicate keys into a flat row. */
+export declare function unqualify(column: string): string;
+/**
+ * Parse the arguments of one `.where(...)` call into a predicate.
+ *
+ * Only the 3-argument `(column, op, value)` form with a string column and a
+ * recognized operator is evaluable; everything else is opaque.
+ */
+export declare function parseWhereArgs(args: readonly unknown[]): MockDbPredicate;
+/**
+ * Evaluate one binary predicate against a row.
+ *
+ * Returns `undefined` when the predicate cannot be decided — the row does not
+ * declare the column, or the values are not comparable. An undecided predicate
+ * does not exclude the row (see the module docblock).
+ */
+export declare function evaluatePredicate(row: Record<string, unknown>, predicate: BinaryPredicate): boolean | undefined;
+/** True when the row satisfies every predicate the mock could decide. */
+export declare function rowMatches(row: unknown, predicates: readonly MockDbPredicate[]): boolean;
+/** Keep only the rows satisfying every decidable predicate. */
+export declare function filterRows<T>(rows: readonly T[], predicates: readonly MockDbPredicate[]): T[];
+/** One projected column: the row key read, and the key it is emitted under. */
+export interface ProjectedColumn {
+    source: string;
+    alias: string;
+}
+/**
+ * Parse `.select(...)` arguments into a projection.
+ *
+ * Returns `undefined` when the projection cannot be modelled — an aggregate
+ * expression, an `eb` callback, a raw `sql` fragment. An unmodelled projection
+ * leaves rows untouched rather than guessing at their shape.
+ */
+export declare function parseSelectArgs(args: readonly unknown[]): ProjectedColumn[] | undefined;
+/**
+ * Narrow a row to the projected columns.
+ *
+ * A projected column the fixture row does not declare is omitted rather than
+ * emitted as `undefined`, so `'key' in row` keeps meaning "the fixture modelled
+ * this" — the same convention {@link evaluatePredicate} reads.
+ *
+ * A column is read under its `source` name and, failing that, under its
+ * `alias`. Fixtures in this repo are written both ways — some model the raw
+ * table row (`storage_key`), others the row the aliased query returns
+ * (`storageKey`) — and the projection has to narrow both. Reading either name
+ * costs nothing: a column dropped from the projection disappears from the
+ * output under both conventions, which is the regression this exists to catch.
+ */
+export declare function projectRow<T>(row: T, columns: readonly ProjectedColumn[]): T;
+/** Narrow every row to the projected columns. */
+export declare function projectRows<T>(rows: readonly T[], columns: readonly ProjectedColumn[]): T[];
+/**
+ * Describe the predicates the mock recorded but could not evaluate.
+ *
+ * Exposed so a test that cares whether its filter was actually pinned can
+ * assert the mock understood it, instead of assuming it did.
+ */
+export declare function explainSkippedPredicates(predicates: readonly MockDbPredicate[]): string[];
+//# sourceMappingURL=mock-db-query.d.ts.map

package/dist/__test-helpers__/mock-db.d.ts

@@ -15,6 +15,29 @@
  *
  * Each test can configure return values via options and access the
  * underlying vi.fn() mocks for assertions via the `mocks` property.
+ *
+ * ## The select chain APPLIES the query (contract)
+ *
+ * A select chain does not return `selectRows` / `selectFirstRow` verbatim. It
+ * evaluates the `.where(...)` predicates and the `.select([...])` projection the
+ * code under test issued, so a test observes the query instead of restating it:
+ * dropping a filter or a projected column changes what the test sees.
+ *
+ * - **Per query, not per test.** Every `db.selectFrom(...)` builds a fresh chain
+ *   with its own predicate list, so two queries in one test cannot inherit each
+ *   other's filters. The `mocks.select*` spies stay shared across the whole
+ *   test, so existing clause assertions are unaffected.
+ * - **Only the shapes the evaluator reads.** `.where(column, op, value)` is
+ *   evaluated; an `eb` callback or a raw `sql` fragment is recorded and skipped.
+ *   `.select()` is applied only when every argument is a plain column string, so
+ *   an aggregate or `eb` projection leaves rows untouched.
+ * - **A predicate on a column the fixture omits does not exclude the row.** To
+ *   pin a filter, the fixture row must declare the column that filter reads.
+ * - **`executeTakeFirstOrThrow` yields `{}`** when the configured row does not
+ *   satisfy the query, rather than throwing as Kysely would.
+ *
+ * `mock-db-query.ts` holds the evaluator and states the same contract with its
+ * reasoning; `mock-db-query.test.ts` tests it directly.
  */
 import { vi } from 'vitest';
 export interface MockDbOptions {
@@ -72,6 +95,10 @@ export interface MockDbMocks {
     insertExecute: ReturnType<typeof vi.fn>;
     insertExecuteTakeFirstOrThrow: ReturnType<typeof vi.fn>;
     onConflict: ReturnType<typeof vi.fn>;
+    /** The upsert's ON CONFLICT DO UPDATE SET payload — shared across both the
+     *  column() and columns() shapes, so a test can assert what an existing row
+     *  is actually updated with. */
+    doUpdateSet: ReturnType<typeof vi.fn>;
     updateTable: ReturnType<typeof vi.fn>;
     updateSet: ReturnType<typeof vi.fn>;
     updateWhere: ReturnType<typeof vi.fn>;

package/dist/agent/agent-version.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Comparing a self-reported agent version against a feature's minimum.
+ *
+ * An agent reports its own package version in `agent.register`
+ * (`agentRegisterSchema.version`), and the field is optional — an agent old
+ * enough to omit it predates every feature this module gates on. The
+ * orchestrator and the agent are deployed and upgraded independently
+ * (`.claude/rules/compatibility.md`), so a feature the orchestrator ships is
+ * routinely dispatched to a fleet that cannot run it, and the version is the
+ * only fact the orchestrator has about what the fleet understands.
+ */
+/** A version's release triple, with any prerelease suffix dropped. */
+type VersionBase = [number, number, number];
+/**
+ * Parse `MAJOR.MINOR.PATCH`, ignoring any `-prerelease` suffix, or `null` when
+ * the string is not a version at all.
+ *
+ * The prerelease suffix is dropped rather than ordered because the suffixes in
+ * play are build counters from the dev registry (`0.5.0-9159`), not semver
+ * release candidates. Strict semver orders those BELOW `0.5.0`, which would
+ * read every staging agent as too old for a feature it in fact carries.
+ */
+export declare function parseVersionBase(version: string): VersionBase | null;
+/**
+ * True when `version` is at least `minimum`, comparing release bases only.
+ *
+ * An absent or unparseable version returns `false`: the orchestrator learns
+ * nothing from it, and a feature that needs a specific agent build must not
+ * treat "we cannot tell" as "yes". Callers decide what an all-unknown fleet
+ * means for them — this function only reports what the version proves.
+ */
+export declare function agentVersionAtLeast(version: string | null | undefined, minimum: string): boolean;
+export {};
+//# sourceMappingURL=agent-version.d.ts.map

package/dist/agent/dispatcher.d.ts

@@ -1,7 +1,28 @@
 import type { AgentRegistry } from './registry.js';
 import { type JobQueue, type QueuedJob, type QueuedJobInput } from '../queue/job-queue.js';
 import type { ScaleResult, ScalerRedispatchTrigger } from '../scaler/types.js';
+import type { ResolvedContainerSpawn } from '../scaler/types.js';
 import type { ResourceRequest } from '@kici-dev/engine';
+/**
+ * Why a freed agent's queue drain claimed nothing.
+ *
+ * Named rather than inlined because the value travels: it is the `reason`
+ * field of the drain-declined log line and the assertion subject of the tests
+ * that pin each exit. The five members are the five ways
+ * `Dispatcher.drainForAgent` can return without a dispatch.
+ */
+export declare enum AgentDrainDecline {
+    /** The agent id is not in the in-memory registry (already disconnected). */
+    NotRegistered = "agent-not-registered",
+    /** This coordinator is draining and must not claim new work. */
+    CoordinatorDraining = "coordinator-draining",
+    /** The agent is already at `maxConcurrency`. */
+    NoCapacity = "no-capacity",
+    /** The host is flagged reboot-pending, so anything dispatched would be lost. */
+    RebootPending = "reboot-pending",
+    /** Nothing in the queue matched this agent's labels or pin. */
+    NoMatchingJob = "no-matching-job"
+}
 /**
  * Default per-pass cap on how many pending jobs a capacity-freed re-drive
  * re-offers to the scaler. Bounds the burst so a single free event cannot storm
@@ -60,24 +81,24 @@ type DispatchResult = {
     status: 'rejected';
     reason: string;
 };
-/**
- * Dispatch coordinator connecting the agent registry and job queue.
- *
- * Routes incoming jobs to matching agents based on label requirements,
- * falling back to the persistent queue when no agent is available.
- * Handles agent lifecycle events (connect, disconnect, job completion).
- *
- * The dispatcher does NOT directly access WebSocket connections --
- * it uses the onDispatch callback provided at construction. The caller
- * (app.ts or server.ts) provides an onDispatch that looks up the agent's
- * WS from the registry and sends the job.dispatch message.
- */
+export declare function containerSpawnFor(jobConfig: Record<string, unknown> | undefined): {
+    image: string;
+    authconfig?: {
+        username: string;
+        password: string;
+        serveraddress: string;
+    } | undefined;
+} | undefined;
 export declare class Dispatcher {
     private readonly registry;
     private readonly queue;
     private readonly metrics;
     private readonly onDispatch;
     private readonly onNoMatchingAgent?;
+    /** See the constructor dep of the same name. */
+    private readonly canPrespawnedAgentServe?;
+    /** See the constructor dep of the same name. */
+    private readonly isPrespawnedAgent?;
     /**
      * Single-flight guard for `retryPendingScaleRequests`. The capacity-freed
      * hook and the leader-gated sweep can both fire concurrently; this prevents
@@ -85,6 +106,12 @@ export declare class Dispatcher {
      * the same tick.
      */
     private redriveInFlight;
+    /**
+     * Single-flight guard for `redrivePendingToConnectedAgents`. The per-coord
+     * safety-net tick must not overlap itself, so a slow re-drive can't be
+     * re-entered by the next interval fire and double-scan the same pending rows.
+     */
+    private pendingRedriveInFlight;
     /**
      * Tracks which jobs are dispatched to which agents.
      * Used for marking jobs as failed on agent disconnect.
@@ -170,7 +197,34 @@ export declare class Dispatcher {
         /** Optional hook called when no agent matches the job's labels.
          *  When set, receives the per-job `resources` so the scaler can apply
          *  per-scaler / per-orchestrator / per-machine caps before spawning. */
-        onNoMatchingAgent?: (labels: string[], jobId: string, runId: string, excludeLabels: string[], resources?: ResourceRequest, orgId?: string) => Promise<ScaleResult>;
+        onNoMatchingAgent?: (labels: string[], jobId: string, runId: string, excludeLabels: string[], resources?: ResourceRequest, orgId?: string, 
+        /**
+         * The job's own container image plus registry credentials, already
+         * resolved at dispatch. Present means the backend spawns THAT image with
+         * the KiCI runtime injected instead of the pool's fixed agent image.
+         */
+        containerSpawn?: ResolvedContainerSpawn) => Promise<ScaleResult>;
+        /**
+         * Optional scaler predicate: may this pre-spawned (warm) agent serve this
+         * job? A warm agent is generic — started before the job existed, at the
+         * pool's shape and running the pool's image, both of which are fixed when
+         * it starts. When the predicate returns false the agent is skipped and the
+         * job falls through to `onNoMatchingAgent`, which spawns one that fits.
+         * Absent (no scaler) means every agent is eligible.
+         */
+        canPrespawnedAgentServe?: (agentId: string, job: {
+            resources?: ResourceRequest;
+            hasOwnContainerImage: boolean;
+        }) => boolean;
+        /**
+         * Whether this scaler pre-spawned the agent, i.e. whether
+         * `canPrespawnedAgentServe` can ever answer false for it. The queue drain
+         * (agent asks for work, rather than job looks for an agent) uses it to
+         * decide whether the suitability predicate is worth carrying into the
+         * claim: for every ordinary agent it is not, and the drain keeps its
+         * single-statement fast path.
+         */
+        isPrespawnedAgent?: (agentId: string) => boolean;
         /** Max reconnection delay from agent config (default 60s). Used to derive grace period. */
         maxReconnectDelayMs?: number;
         /** Callback fired when a job is permanently failed before/outside agent execution. */
@@ -259,6 +313,39 @@ export declare class Dispatcher {
      * that is about to reboot. No-op when no roster store is wired.
      */
     private filterRebootPending;
+    /**
+     * Drop pre-spawned (warm) agents that cannot serve this job. Sibling of
+     * {@link filterRebootPending}: both remove candidates `findAvailable` matched
+     * on labels but that are unusable for a reason labels cannot express.
+     *
+     * A warm agent's cpu, memory and container image are all set when it starts
+     * and cannot change afterwards, so a job asking for something else has to get
+     * an agent of its own. Dropping the candidate here is what sends it down the
+     * ordinary `onNoMatchingAgent` scale path. No-op when no scaler is wired.
+     */
+    private filterUnsuitablePrespawned;
+    /**
+     * The fit question for one job: the shape it declares, and whether it brings
+     * its own container image.
+     *
+     * The shape comes from `jobConfig.resources` when the typed `resources`
+     * mirror is absent, because the mirror is optional and several dispatch paths
+     * never fill it — a job reaching {@link dispatch} straight off the webhook
+     * pipeline or off a worker's reroute handler carries its declaration in
+     * `jobConfig` alone. Reading only the mirror there reports a job that
+     * declares nothing, which admits it onto a pre-spawned agent of some other
+     * size — the exact mismatch this gate exists to refuse.
+     */
+    private prespawnedFitFor;
+    /**
+     * The queue-drain half of {@link filterUnsuitablePrespawned}: a per-job
+     * predicate for one agent, or undefined when this agent needs no check.
+     *
+     * Undefined is the common answer — an ordinary agent is not pre-spawned, so
+     * the predicate could only ever say yes, and returning one would cost the
+     * drain its single-statement fast path for nothing.
+     */
+    private prespawnedFitFilterFor;
     /**
      * Release a reboot-pending host on its real reconnect (down-then-up). Clears
      * the persisted flag so the very next `onAgentAvailable` drain dispatches the
@@ -268,6 +355,12 @@ export declare class Dispatcher {
      */
     releaseRebootPending(agentId: string): Promise<void>;
     onAgentAvailable(agentId: string): Promise<void>;
+    /**
+     * Claim at most one queued job for a freed agent.
+     *
+     * Returns the reason nothing was dispatched, or null when a job was.
+     */
+    private drainForAgent;
     /** Record that a job began executing on its agent. */
     markJobStarted(jobId: string): void;
     /**
@@ -316,6 +409,51 @@ export declare class Dispatcher {
      * fresh agent is spawned bound to this job.
      */
     private redispatch;
+    /**
+     * Safety-net re-drive: deliver every pending job that a currently-connected,
+     * idle, matching agent could take, through the same atomic claim the drain
+     * uses (`dispatchBoundJob` → `dequeueById`).
+     *
+     * Runs per-coordinator and is NOT leader-gated: each coordinator drains the
+     * shared queue onto its OWN connected agents, and the atomic claim guarantees
+     * at most one agent (on any coordinator) wins each job, so concurrent ticks
+     * cannot double-dispatch.
+     *
+     * It closes the requeue re-drive gap. A job requeued by `handleAckExpiry` /
+     * `onJobRejected` / the leader ack sweep gets exactly one delivery attempt —
+     * `redispatch`'s single `findAvailable` + `dispatchBoundJob`. An idle matching
+     * agent whose own drain trigger (registration / completion / status) already
+     * fired before the requeue has no further trigger, so if that one attempt
+     * transiently misses the agent — the agent is momentarily at capacity while an
+     * in-flight drain holds its eagerly-claimed slot, or the agent is connected to
+     * a different coordinator than the one that ran the expiry — the requeued
+     * pending job would otherwise sit undelivered until it expired. This tick
+     * re-attempts delivery onto connected agents so the miss recovers on the next
+     * sweep.
+     *
+     * It never consults the scaler (that is `retryPendingScaleRequests`) and never
+     * spawns: it only places jobs onto agents already connected here.
+     *
+     * A failure propagates to the caller rather than being swallowed here — the
+     * per-coord interval wrapper is the single error-log site, matching the
+     * sibling recovery/ack sweeps that share its cadence. The `finally` only
+     * releases the single-flight guard.
+     */
+    redrivePendingToConnectedAgents(maxJobs?: number): Promise<number>;
+    /**
+     * Pick a connected agent that may take a pending job, or null. A pinned
+     * host-fanout child may run ONLY on its pinned agent — and `dequeueById`
+     * ignores the pin — so a pinned job is routed to its own agent and never
+     * offered to `findAvailable`, which would mis-deliver it to any label match.
+     *
+     * Reboot-pending gate: a host whose `restart` job just completed is still
+     * connected but about to reboot, so its held post-restart job must NOT be
+     * re-driven into the about-to-die box. This is the safety-net re-drive's
+     * counterpart of the same gate in `dispatch()` / `dispatchPinned` /
+     * `drainForAgent`; without it this path re-drives the held job onto the
+     * reboot-pending host and defeats the hold.
+     */
+    private selectConnectedTargetForPending;
     /**
      * Handle agent disconnect. Starts per-job recovery timers instead of
      * immediately failing jobs, giving the agent a grace period to reconnect.

package/dist/agent/host-roster.d.ts

@@ -77,6 +77,13 @@ export interface UpsertHostInput {
     agentId: string;
     tokenId: string | null;
     lifecycleClass: LifecycleClass;
+    /**
+     * True when an auto-scaler backend spawned this agent, so `runsOnAll`
+     * fan-out skips it. Derived from the scaler manager's registration lookup,
+     * never from `lifecycleClass` (which is `ephemeral` for every agent when the
+     * auth mode is `none`). Omitted ⇒ false.
+     */
+    scalerManaged?: boolean;
     labels: string[];
     hostname: string | null;
     platform: string;
@@ -176,13 +183,38 @@ export declare class HostRosterStore {
     getStagedVersion(agentId: string): Promise<string | null>;
     listAll(): Promise<HostRosterRow[]>;
     /**
-     * Resolve every roster host matching a `runsOnAll` predicate (OR-of-AND
-     * include groups, minus exclude labels), tagged with its derived status. This
-     * is the host-fanout resolver: it returns declared-but-absent static hosts
-     * (status `unreachable`) so the caller can apply `onUnreachable` — the live
-     * registry alone cannot name an expected-but-absent host.
+     * Resolve EVERY roster host matching a label predicate (OR-of-AND include
+     * groups, minus exclude labels), tagged with its derived status. Returns
+     * declared-but-absent static hosts (status `unreachable`) so a caller can
+     * apply `onUnreachable` — the live registry alone cannot name an
+     * expected-but-absent host.
+     *
+     * This is the INVENTORY query: it includes auto-scaler-spawned hosts, whose
+     * `lifecycleClass` is what the SDK's `ctx.kici.inventory` exposes to tell
+     * them apart. `runsOnAll` fan-out uses {@link findFanoutTargets} instead.
      */
     findMatching(include: readonly (readonly LabelMatcher[])[], exclude: readonly LabelMatcher[], graceMs: number): Promise<MatchedHost[]>;
+    /**
+     * Resolve the `runsOnAll` fan-out target set: {@link findMatching}, minus
+     * every auto-scaler-spawned host.
+     *
+     * Fan-out targets DECLARED fleet members. An auto-scaler agent is spawned to
+     * a pool's fixed shape, so a child pinned to one runs at that shape rather
+     * than its own — the same defect the dispatcher's warm-pool suitability gate
+     * removes, on the path a pin deliberately leaves ungated.
+     *
+     * Separate from `findMatching` on purpose: `findMatching` also backs the
+     * SDK's `inventory.query`, where a host the caller asked for must never go
+     * missing (a selector returning FEWER hosts than no selector at all).
+     */
+    findFanoutTargets(include: readonly (readonly LabelMatcher[])[], exclude: readonly LabelMatcher[], graceMs: number): Promise<MatchedHost[]>;
+    /**
+     * Shared row loop behind {@link findMatching} and {@link findFanoutTargets}.
+     * `excludeScalerManaged` stays private to the store — each public method
+     * names its own semantics, so a call site cannot silently pick the wrong
+     * population by passing a bare boolean.
+     */
+    private matchRows;
     /**
      * Map a roster row to the canonical {@link HostInventoryEntry} — the queryable
      * shape returned by the `inventory.query`/`inventory.get` RPC and typed on the

package/dist/agent/registry.d.ts

@@ -137,6 +137,8 @@ export interface RosterReconciler {
         agentId: string;
         tokenId: string | null;
         lifecycleClass: 'static' | 'ephemeral';
+        /** True when a scaler backend spawned this agent (kept out of `runsOnAll`). */
+        scalerManaged?: boolean;
         labels: string[];
         hostname: string | null;
         platform: string;

package/dist/app.d.ts

@@ -26,6 +26,7 @@ import type { PeerRegistry } from './cluster/peer-registry.js';
 import type { JobQueue } from './queue/job-queue.js';
 import type { DedupCache } from './webhook/dedup.js';
 import type { LockFileCache } from './lockfile-cache.js';
+import type { ContentRequirementsCache } from './content-requirements-cache.js';
 import type { PlatformClient } from './ws/platform-client.js';
 import type { ScalerManager } from './scaler/manager.js';
 import type { ProviderRegistry } from './provider-registry.js';
@@ -37,7 +38,8 @@ import type { ArtifactStore } from './artifacts/artifact-store.js';
 import type { DispatchCacheRefTracker } from './cache/index.js';
 import type { PendingBuildTracker } from './cache/index.js';
 import type { PendingInitTracker } from './cache/index.js';
-import type { PendingDynamicTracker } from './cache/index.js';
+import type { PendingDynamicTracker, PendingGlobalEvalTracker } from './cache/index.js';
+import type { GlobalEvalRoundCache } from './cache/index.js';
 import type { CacheStorage } from './storage/types.js';
 import type { ProvenanceTrustRoot } from './provenance/trust-root.js';
 import type { LocalSigner } from './oidc/local-dev-signer.js';
@@ -64,8 +66,11 @@ import type { RegistrationStore } from './registration/registration-store.js';
 import type { RegistrationIndex } from './registration/registration-index.js';
 import type { CronScheduler } from './cron/cron-scheduler.js';
 import { type ConfigRouteDeps } from './routes/admin-config.js';
+import type { ProcessingDeps } from './pipeline/processor.js';
+import type { InvokeGateDeps } from './pipeline/invoke-gate.js';
 import { WebhookIngestOutcome } from './pipeline/process-webhook.js';
 import type { IngestOverflowBuffer } from './webhook/ingest-overflow-buffer.js';
+import type { IngestOverflowReplayer } from './webhook/ingest-overflow-replayer.js';
 import { type OverflowDelivery } from './webhook/ingest-overflow-types.js';
 import type { IngestAdmissionController } from './webhook/ingest-admission.js';
 import type { OrgIngestCapReader } from './webhook/org-ingest-cap-reader.js';
@@ -82,9 +87,8 @@ import type { GenericSourceManager } from './webhook/generic-sources.js';
 import type { TrustStore } from './events/trust-store.js';
 import type { ContextStore } from './contexts/context-store.js';
 import type { VariableStore } from './contexts/variable-store.js';
-import type { HeldRunStore } from './contexts/held-runs.js';
+import type { HeldRunStore, ReleaseSignal } from './contexts/held-runs.js';
 import type { StepApprovalBridge } from './approvals/step-approval-bridge.js';
-import type { ContributorCache } from './security/contributor-cache.js';
 import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
 /**
  * All dependencies needed to create the orchestrator Hono app.
@@ -108,6 +112,7 @@ export interface AppDependencies {
     jobQueue: JobQueue;
     dedup: DedupCache;
     lockFileCache: LockFileCache;
+    contentRequirementsCache: ContentRequirementsCache;
     providerRegistry: ProviderRegistry;
     platformClient?: PlatformClient;
     /**
@@ -126,6 +131,13 @@ export interface AppDependencies {
      * for later replay; absent → capture disabled.
      */
     ingestOverflowBuffer?: IngestOverflowBuffer;
+    /**
+     * Drain loop for the durable ingest queue. The accept path borrows its
+     * claim-release so a failed pipeline hands the row back for retry through the
+     * same attempt ceiling the drain uses; absent → the accept path degrades to
+     * inline ingestion.
+     */
+    ingestOverflowReplayer?: IngestOverflowReplayer;
     /**
      * Fulfil deferred attestations on demand (mints in this process, which owns
      * the Platform WS). Backs `POST /api/v1/admin/attestations/retry`. Wired only
@@ -168,6 +180,14 @@ export interface AppDependencies {
      * is registered so `ctx.kici.oidc.token()` mints a `kici-local` dev token.
      */
     localOidcSigner?: LocalSigner;
+    /**
+     * Test-only fault-injection policy, threaded from a test-only entrypoint via
+     * `bootstrapOrchestrator`. Undefined in production. Consumed here only for the
+     * initial-mint provenance seam (`initialMintFault`); the other seams live in
+     * the composition root / server hooks. Type-only reference — the runtime
+     * module never enters the shipped bundle.
+     */
+    faultInjection?: import('./testing/fault-injection.js').OrchestratorFaultInjection;
     /**
      * Orchestrator-owned provenance signing (Phase 1 root of trust). Present when
      * `KICI_ORCHESTRATOR_PROVENANCE_ISSUER` is configured: the orchestrator mints +
@@ -208,6 +228,8 @@ export interface AppDependencies {
     pendingInits?: PendingInitTracker;
     /** Pending dynamic tracker for DynamicJobFn evaluation coordination. */
     pendingDynamics?: PendingDynamicTracker;
+    pendingGlobalEvals?: PendingGlobalEvalTracker;
+    globalEvalCache?: GlobalEvalRoundCache;
     /** Commit status reporter for setting pending/success/failure/error on commits. Optional. */
     checkRunReporter?: CheckRunReporter;
     /** Execution tracker for DB persistence of execution state. Optional — requires database. */
@@ -244,12 +266,33 @@ export interface AppDependencies {
     configRouteDeps?: ConfigRouteDeps;
     /** Event router for internal event delivery. Optional -- if not set, event routing is inactive. */
     eventRouter?: EventRouter;
+    /** Invoke-gate dependencies (summon callback + chain-depth bound). Optional -- if not set, invoke gates fail loudly. */
+    invokeGateDeps?: InvokeGateDeps;
+    /**
+     * Late-binding handle the internal-event dispatch adapter reads to obtain the
+     * live `ProcessingDeps` bag. `createApp` populates it, because the bag can
+     * only be assembled here while the event router that dispatches internal
+     * events is constructed earlier in the bootstrap. Left unset by callers that
+     * never route internal events (tests constructing a bare app).
+     */
+    processingDepsRef?: {
+        current: (() => ProcessingDeps) | null;
+    };
     /** Event store. Optional -- mounted when admin DLQ admin route should be available. */
     eventStore?: EventStore;
     /** Event emitter for system events (workflow/job complete). Optional -- if not set, system events are skipped. */
     eventEmitter?: EventEmitter;
     /** Generic webhook source manager. Optional -- if not set, generic webhooks are disabled. */
     genericSourceManager?: GenericSourceManager;
+    /**
+     * Re-register a generic source's provider bundle from its database row,
+     * resolving to true when a per-routing-key bundle is now registered.
+     *
+     * Built in orchestrator-core so it shares the exact dependency bag the
+     * startup enumeration and the LISTEN/NOTIFY drain already register with.
+     * Optional -- wirings without one keep the previous behaviour.
+     */
+    ensureProviderBundle?: (routingKey: string) => Promise<boolean>;
     /** GitHub direct-ingress route deps (hybrid/independent only). Optional -- if not set, the direct GitHub ingress route is not mounted. */
     githubSourceStore?: SourceStore;
     githubVerifyDeps?: VerifyInboundDeps;
@@ -275,6 +318,16 @@ export interface AppDependencies {
     heldRunStore?: HeldRunStore;
     /** Step-approval bridge — opens step-scoped holds and relays their resolution back to the waiting agent. Optional. */
     stepApprovalBridge?: StepApprovalBridge;
+    /**
+     * Re-dispatch a job whose hold was released. Supplied by both mode hooks
+     * through `appDepsExtras`; declared here so the admin held-run routes can
+     * read it type-safely rather than off an untyped spread. Optional — an app
+     * assembled without it mounts no local held-run decision surface, because a
+     * release that cannot dispatch is a row flip pretending to be a release.
+     */
+    onJobRelease?: (signal: ReleaseSignal) => Promise<void>;
+    /** Replay the stored dispatch context of a released workflow-scoped hold. Same provenance as `onJobRelease`. */
+    onWorkflowRelease?: (signal: ReleaseSignal) => Promise<void>;
     /** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are dispatched without permission checks. */
     globalWorkflowPolicy?: GlobalWorkflowPolicy;
     /** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
@@ -308,10 +361,6 @@ export interface AppDependencies {
     coordinator?: RunCoordinator;
     /** Peer registry for aggregating infrastructure across cluster. Optional. */
     peerRegistry?: PeerRegistry;
-    /** Contributor permission cache. Optional -- threaded through from server.ts
-     *  so membership-related webhooks invalidate matching entries immediately
-     *  (instead of waiting for the 15-minute TTL). */
-    contributorCache?: ContributorCache;
     /**
      * Shared aggregator for agent-pushed metrics. When omitted, app.ts
      * constructs its own — kept optional so existing tests still work