@kici-dev/orchestrator 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/dist/__test-helpers__/fake-scaler-state-store.d.ts +24 -0
  2. package/dist/__test-helpers__/mock-db-query.d.ts +130 -0
  3. package/dist/__test-helpers__/mock-db.d.ts +23 -0
  4. package/dist/agent/dispatcher.d.ts +81 -13
  5. package/dist/agent/host-roster.d.ts +37 -5
  6. package/dist/agent/registry.d.ts +2 -0
  7. package/dist/app.d.ts +33 -6
  8. package/dist/approvals/apply-decision.d.ts +67 -19
  9. package/dist/approvals/held-run-release-wiring.d.ts +45 -0
  10. package/dist/approvals/independent-wiring.d.ts +55 -0
  11. package/dist/approvals/resume-router.d.ts +9 -0
  12. package/dist/approvals/triggerer-subject.d.ts +37 -0
  13. package/dist/cache/dep-cache.d.ts +54 -10
  14. package/dist/cache/global-eval-round-cache.d.ts +7 -0
  15. package/dist/cli/api-client.d.ts +18 -0
  16. package/dist/cli/commands/cluster-settings.d.ts +11 -0
  17. package/dist/cli/commands/diagnose.d.ts +6 -0
  18. package/dist/cli/commands/held-run.d.ts +58 -0
  19. package/dist/cli/commands/trust-policy.d.ts +117 -12
  20. package/dist/cli/kici-admin.d.ts +10 -0
  21. package/dist/cli.js +2272 -1025
  22. package/dist/cluster/cluster-identity.d.ts +3 -3
  23. package/dist/cluster/cluster-settings-reader.d.ts +7 -1
  24. package/dist/cluster/coordinator.d.ts +44 -0
  25. package/dist/cluster/index.d.ts +4 -0
  26. package/dist/cluster/peer-handler.d.ts +19 -0
  27. package/dist/cluster/peer-registry.d.ts +17 -0
  28. package/dist/cluster/plan-headroom-store.d.ts +26 -0
  29. package/dist/cluster/worker-eviction.d.ts +69 -0
  30. package/dist/cold-store/load-event-log-range.d.ts +10 -3
  31. package/dist/cold-store/load-secret-audit-log-range.d.ts +4 -3
  32. package/dist/cold-store/tables/secret-audit-log.d.ts +4 -3
  33. package/dist/config.d.ts +21 -21
  34. package/dist/contexts/held-runs.d.ts +123 -16
  35. package/dist/contexts/protection/aggregate.d.ts +3 -2
  36. package/dist/contexts/protection/branch-gate.d.ts +15 -0
  37. package/dist/contexts/protection/concurrency-gate.d.ts +8 -2
  38. package/dist/contexts/protection/pipeline.d.ts +18 -0
  39. package/dist/contexts/release-queued-holds.d.ts +35 -0
  40. package/dist/dashboard/attestation-filters.d.ts +8 -0
  41. package/dist/dashboard/handler.d.ts +6 -24
  42. package/dist/db/migrations/116_org_plan_headroom.d.ts +4 -0
  43. package/dist/db/migrations/117_invoke_gate_proxy_jobs.d.ts +4 -0
  44. package/dist/db/migrations/118_invoke_gate_scheduling.d.ts +4 -0
  45. package/dist/db/migrations/119_scaler_ha_ownership.d.ts +4 -0
  46. package/dist/db/migrations/120_cluster_settings_scaler_reaper.d.ts +4 -0
  47. package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +19 -0
  48. package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +25 -0
  49. package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +24 -0
  50. package/dist/db/migrations/124_cluster_settings_provision_backoff.d.ts +4 -0
  51. package/dist/db/migrations/125_org_trust_directory.d.ts +30 -0
  52. package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +28 -0
  53. package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +26 -0
  54. package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +23 -0
  55. package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +24 -0
  56. package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +32 -0
  57. package/dist/db/types.d.ts +368 -10
  58. package/dist/entry-helpers.d.ts +28 -0
  59. package/dist/events/event-emitter.d.ts +22 -0
  60. package/dist/events/event-router.d.ts +87 -1
  61. package/dist/events/types.d.ts +2 -23
  62. package/dist/git/credential-broker.d.ts +94 -0
  63. package/dist/git/inherited-secret.d.ts +24 -0
  64. package/dist/git/installation-token.d.ts +38 -0
  65. package/dist/git/job-context.d.ts +13 -0
  66. package/dist/index.js +324 -84
  67. package/dist/metrics/agent-metrics-aggregator.d.ts +2 -2
  68. package/dist/metrics/prometheus.d.ts +111 -3
  69. package/dist/oidc/oidc-mint-registration.d.ts +7 -3
  70. package/dist/orchestrator-core.d.ts +79 -23
  71. package/dist/pipeline/direct-ingress-deps.d.ts +100 -0
  72. package/dist/pipeline/dispatch-matched-workflow.d.ts +302 -31
  73. package/dist/pipeline/global-eval-round.d.ts +14 -0
  74. package/dist/pipeline/internal-event-pipeline.d.ts +140 -0
  75. package/dist/pipeline/invoke-gate.d.ts +164 -0
  76. package/dist/pipeline/needs-scheduler.d.ts +13 -0
  77. package/dist/pipeline/process-webhook.d.ts +119 -31
  78. package/dist/pipeline/processor.d.ts +226 -21
  79. package/dist/pipeline/rerun.d.ts +21 -0
  80. package/dist/pipeline/resume-workflow.d.ts +40 -7
  81. package/dist/pipeline/route-or-dispatch-jobs.d.ts +9 -0
  82. package/dist/pipeline/security-hold-check.d.ts +217 -0
  83. package/dist/pipeline/undispatched-hold-checks.d.ts +23 -0
  84. package/dist/policy/dashboard-write-policy-listener.d.ts +62 -0
  85. package/dist/policy/dashboard-write-policy.d.ts +14 -0
  86. package/dist/provider-registry.d.ts +10 -2
  87. package/dist/providers/github/check-status-poster.d.ts +11 -0
  88. package/dist/providers/github/index.d.ts +0 -2
  89. package/dist/providers/github/normalizer.d.ts +3 -2
  90. package/dist/providers/local/normalizer.d.ts +3 -5
  91. package/dist/providers/universal-git/index.d.ts +11 -2
  92. package/dist/queue/job-queue.d.ts +43 -1
  93. package/dist/registration/extractor.d.ts +3 -3
  94. package/dist/registration/registration-index.d.ts +7 -0
  95. package/dist/registration/registration-store.d.ts +22 -0
  96. package/dist/reporting/check-run-reporter.d.ts +94 -1
  97. package/dist/reporting/check-run-summary.d.ts +25 -1
  98. package/dist/reporting/execution-tracker.d.ts +207 -2
  99. package/dist/reporting/run-aggregator.d.ts +4 -14
  100. package/dist/reporting/step-display-order.d.ts +43 -0
  101. package/dist/routes/admin-held-runs.d.ts +142 -0
  102. package/dist/routes/admin-trust-policy.d.ts +34 -6
  103. package/dist/routes/admin.d.ts +9 -0
  104. package/dist/scaler/backend-factory.d.ts +55 -0
  105. package/dist/scaler/bare-metal-backend.d.ts +43 -5
  106. package/dist/scaler/claim-store.d.ts +119 -0
  107. package/dist/scaler/config.d.ts +4 -0
  108. package/dist/scaler/container-backend.d.ts +18 -4
  109. package/dist/scaler/container-routing.d.ts +23 -0
  110. package/dist/scaler/container-spawn.d.ts +28 -0
  111. package/dist/scaler/event-backend.d.ts +124 -0
  112. package/dist/scaler/event-provision-reaper.d.ts +350 -0
  113. package/dist/scaler/failure-tracker.d.ts +1 -1
  114. package/dist/scaler/firecracker-backend.d.ts +33 -6
  115. package/dist/scaler/index.d.ts +11 -1
  116. package/dist/scaler/label-matcher.d.ts +4 -3
  117. package/dist/scaler/manager.d.ts +899 -72
  118. package/dist/scaler/resolve-container-auth.d.ts +45 -0
  119. package/dist/scaler/scaler-events.d.ts +11 -0
  120. package/dist/scaler/scaler-state-store.d.ts +294 -10
  121. package/dist/scaler/types.d.ts +131 -13
  122. package/dist/scaler/warm-pool.d.ts +132 -29
  123. package/dist/security/comment-handler.d.ts +50 -11
  124. package/dist/security/identity-link.d.ts +60 -0
  125. package/dist/security/lock-source.d.ts +11 -13
  126. package/dist/security/reduced-privilege-note.d.ts +59 -0
  127. package/dist/security/trust-directory-store.d.ts +233 -0
  128. package/dist/security/trust-policy-gate.d.ts +94 -59
  129. package/dist/security/trust-policy-store.d.ts +3 -0
  130. package/dist/security/trust-resolver.d.ts +14 -98
  131. package/dist/server.d.ts +11 -1
  132. package/dist/server.js +17671 -7350
  133. package/dist/stale-detector/gate-deadline-detector.d.ts +52 -0
  134. package/dist/stale-detector/stale-run-detector.d.ts +61 -3
  135. package/dist/standalone.js +24238 -13822
  136. package/dist/worker/in-memory-job-queue.d.ts +1 -1
  137. package/dist/ws/agent-handler.d.ts +28 -4
  138. package/dist/ws/dashboard-context-handler.d.ts +28 -4
  139. package/dist/ws/dashboard-global-workflows-handler.d.ts +6 -0
  140. package/dist/ws/failure-messages.d.ts +2 -0
  141. package/dist/ws/git-credential-relay.d.ts +39 -0
  142. package/dist/ws/oidc-token-relay.d.ts +6 -11
  143. package/dist/ws/platform-client.d.ts +54 -9
  144. package/installer-image-digests.json +3 -3
  145. package/package.json +24 -19
  146. package/sbom.spdx.json +864 -822
  147. package/dist/approvals/team-membership-lookup.d.ts +0 -13
  148. package/dist/providers/github/contributor-resolver.d.ts +0 -30
  149. package/dist/security/contributor-cache.d.ts +0 -83
@@ -0,0 +1,24 @@
1
+ /**
2
+ * In-memory stand-in for the `scaler_pending_claims` half of `ScalerStateStore`.
3
+ *
4
+ * It reproduces the three properties the real table enforces in SQL — a claim
5
+ * is consumable exactly once, a consume past `expires_at` finds nothing, and a
6
+ * per-agent invalidation removes every row for that agent — against an injected
7
+ * clock. That lets a test drive the whole register → redeem round trip without a
8
+ * database, so the single-use and TTL properties are executed rather than
9
+ * hand-fed through a stubbed answer.
10
+ *
11
+ * Only the pending-claim methods are implemented. Reaching any other
12
+ * `ScalerStateStore` method is a `TypeError` at the call site, not a silent
13
+ * no-op.
14
+ */
15
+ import type { ScalerStateStore } from '../scaler/scaler-state-store.js';
16
+ /**
17
+ * Build an in-memory pending-claim store.
18
+ *
19
+ * @param now Clock (epoch ms) the TTL is evaluated against. Pass the same clock
20
+ * the `ClaimStore` under test uses, or its `expiresAt` lands in a different
21
+ * era than this store's `now()` and every redeem reads as expired.
22
+ */
23
+ export declare function makeFakeScalerStateStore(now?: () => number): ScalerStateStore;
24
+ //# sourceMappingURL=fake-scaler-state-store.d.ts.map
@@ -0,0 +1,130 @@
1
+ /**
2
+ * Predicate + projection evaluation for the shared `createMockDb` harness.
3
+ *
4
+ * Kept BYTE-IDENTICAL in two places —
5
+ * `packages/orchestrator/src/__test-helpers__/mock-db-query.ts` and
6
+ * `packages/platform/src/__test-helpers__/mock-db-query.ts`. Each package owns an
7
+ * independent `createMockDb` builder and neither package may import the other, so
8
+ * the evaluator is duplicated rather than shared. `mock-db-mirror.test.ts` in both
9
+ * packages fails the moment the copies diverge.
10
+ *
11
+ * `createMockDb` used to return its configured rows verbatim, whatever the
12
+ * query asked for: a `.where(...)` or a `.select([...])` was recorded on a spy
13
+ * and then discarded. A test therefore observed only the rows it had itself
14
+ * configured, so dropping a filter or a projected column from the code under
15
+ * test changed nothing the test could see. This module is the evaluator that
16
+ * closes that: the mock now *applies* the query it was handed.
17
+ *
18
+ * Scope, stated precisely because the guarantee is partial by construction:
19
+ *
20
+ * - **Binary predicates are evaluated.** `.where(column, op, value)` for the
21
+ * operators in {@link MockDbOperator}. That is the overwhelming majority of
22
+ * the predicates this repo issues.
23
+ * - **Anything else is opaque and is NOT evaluated.** An expression-builder
24
+ * callback (`.where((eb) => …)`) and a raw `sql` fragment carry their meaning
25
+ * in code the mock cannot read, so they are recorded and skipped. A filter
26
+ * written in one of those shapes is still unpinned.
27
+ * - **A predicate on a column the fixture row does not declare is NOT
28
+ * evaluated.** The fixture, not the schema, is the source of truth here: a
29
+ * test row is a partial literal, and a missing key means "this test did not
30
+ * model that column", not "this column is NULL". Treating absence as NULL
31
+ * would fail nearly every fixture in the repo for saying too little, while
32
+ * catching nothing a fixture that *does* declare the column would not
33
+ * already catch. The consequence is real and worth naming: to pin a filter,
34
+ * a fixture row must mention the column that filter reads.
35
+ *
36
+ * Everything here is pure — no vitest, no chain state — so it is unit-tested
37
+ * directly in `mock-db-query.test.ts` rather than only through the mock.
38
+ */
39
+ /** SQL comparison operators the mock evaluates. Anything else is opaque. */
40
+ export declare enum MockDbOperator {
41
+ eq = "=",
42
+ neq = "!=",
43
+ neqAnsi = "<>",
44
+ lt = "<",
45
+ lte = "<=",
46
+ gt = ">",
47
+ gte = ">=",
48
+ in = "in",
49
+ notIn = "not in",
50
+ is = "is",
51
+ isNot = "is not",
52
+ like = "like",
53
+ notLike = "not like",
54
+ ilike = "ilike",
55
+ notILike = "not ilike"
56
+ }
57
+ /** A `.where(column, op, value)` predicate the mock can evaluate. */
58
+ export interface BinaryPredicate {
59
+ kind: 'binary';
60
+ /** Column name with any `table.` qualifier stripped. */
61
+ column: string;
62
+ op: MockDbOperator;
63
+ value: unknown;
64
+ }
65
+ /** A predicate shape the mock cannot read (eb callback, raw sql, unknown op). */
66
+ export interface OpaquePredicate {
67
+ kind: 'opaque';
68
+ /** Why it could not be parsed — surfaced in `explainSkippedPredicates`. */
69
+ reason: string;
70
+ }
71
+ export type MockDbPredicate = BinaryPredicate | OpaquePredicate;
72
+ /** Drop a `table.` / `"table".` qualifier so a predicate keys into a flat row. */
73
+ export declare function unqualify(column: string): string;
74
+ /**
75
+ * Parse the arguments of one `.where(...)` call into a predicate.
76
+ *
77
+ * Only the 3-argument `(column, op, value)` form with a string column and a
78
+ * recognized operator is evaluable; everything else is opaque.
79
+ */
80
+ export declare function parseWhereArgs(args: readonly unknown[]): MockDbPredicate;
81
+ /**
82
+ * Evaluate one binary predicate against a row.
83
+ *
84
+ * Returns `undefined` when the predicate cannot be decided — the row does not
85
+ * declare the column, or the values are not comparable. An undecided predicate
86
+ * does not exclude the row (see the module docblock).
87
+ */
88
+ export declare function evaluatePredicate(row: Record<string, unknown>, predicate: BinaryPredicate): boolean | undefined;
89
+ /** True when the row satisfies every predicate the mock could decide. */
90
+ export declare function rowMatches(row: unknown, predicates: readonly MockDbPredicate[]): boolean;
91
+ /** Keep only the rows satisfying every decidable predicate. */
92
+ export declare function filterRows<T>(rows: readonly T[], predicates: readonly MockDbPredicate[]): T[];
93
+ /** One projected column: the row key read, and the key it is emitted under. */
94
+ export interface ProjectedColumn {
95
+ source: string;
96
+ alias: string;
97
+ }
98
+ /**
99
+ * Parse `.select(...)` arguments into a projection.
100
+ *
101
+ * Returns `undefined` when the projection cannot be modelled — an aggregate
102
+ * expression, an `eb` callback, a raw `sql` fragment. An unmodelled projection
103
+ * leaves rows untouched rather than guessing at their shape.
104
+ */
105
+ export declare function parseSelectArgs(args: readonly unknown[]): ProjectedColumn[] | undefined;
106
+ /**
107
+ * Narrow a row to the projected columns.
108
+ *
109
+ * A projected column the fixture row does not declare is omitted rather than
110
+ * emitted as `undefined`, so `'key' in row` keeps meaning "the fixture modelled
111
+ * this" — the same convention {@link evaluatePredicate} reads.
112
+ *
113
+ * A column is read under its `source` name and, failing that, under its
114
+ * `alias`. Fixtures in this repo are written both ways — some model the raw
115
+ * table row (`storage_key`), others the row the aliased query returns
116
+ * (`storageKey`) — and the projection has to narrow both. Reading either name
117
+ * costs nothing: a column dropped from the projection disappears from the
118
+ * output under both conventions, which is the regression this exists to catch.
119
+ */
120
+ export declare function projectRow<T>(row: T, columns: readonly ProjectedColumn[]): T;
121
+ /** Narrow every row to the projected columns. */
122
+ export declare function projectRows<T>(rows: readonly T[], columns: readonly ProjectedColumn[]): T[];
123
+ /**
124
+ * Describe the predicates the mock recorded but could not evaluate.
125
+ *
126
+ * Exposed so a test that cares whether its filter was actually pinned can
127
+ * assert the mock understood it, instead of assuming it did.
128
+ */
129
+ export declare function explainSkippedPredicates(predicates: readonly MockDbPredicate[]): string[];
130
+ //# sourceMappingURL=mock-db-query.d.ts.map
@@ -15,6 +15,29 @@
15
15
  *
16
16
  * Each test can configure return values via options and access the
17
17
  * underlying vi.fn() mocks for assertions via the `mocks` property.
18
+ *
19
+ * ## The select chain APPLIES the query (contract)
20
+ *
21
+ * A select chain does not return `selectRows` / `selectFirstRow` verbatim. It
22
+ * evaluates the `.where(...)` predicates and the `.select([...])` projection the
23
+ * code under test issued, so a test observes the query instead of restating it:
24
+ * dropping a filter or a projected column changes what the test sees.
25
+ *
26
+ * - **Per query, not per test.** Every `db.selectFrom(...)` builds a fresh chain
27
+ * with its own predicate list, so two queries in one test cannot inherit each
28
+ * other's filters. The `mocks.select*` spies stay shared across the whole
29
+ * test, so existing clause assertions are unaffected.
30
+ * - **Only the shapes the evaluator reads.** `.where(column, op, value)` is
31
+ * evaluated; an `eb` callback or a raw `sql` fragment is recorded and skipped.
32
+ * `.select()` is applied only when every argument is a plain column string, so
33
+ * an aggregate or `eb` projection leaves rows untouched.
34
+ * - **A predicate on a column the fixture omits does not exclude the row.** To
35
+ * pin a filter, the fixture row must declare the column that filter reads.
36
+ * - **`executeTakeFirstOrThrow` yields `{}`** when the configured row does not
37
+ * satisfy the query, rather than throwing as Kysely would.
38
+ *
39
+ * `mock-db-query.ts` holds the evaluator and states the same contract with its
40
+ * reasoning; `mock-db-query.test.ts` tests it directly.
18
41
  */
19
42
  import { vi } from 'vitest';
20
43
  export interface MockDbOptions {
@@ -1,6 +1,7 @@
1
1
  import type { AgentRegistry } from './registry.js';
2
2
  import { type JobQueue, type QueuedJob, type QueuedJobInput } from '../queue/job-queue.js';
3
3
  import type { ScaleResult, ScalerRedispatchTrigger } from '../scaler/types.js';
4
+ import type { ResolvedContainerSpawn } from '../scaler/types.js';
4
5
  import type { ResourceRequest } from '@kici-dev/engine';
5
6
  /**
6
7
  * Why a freed agent's queue drain claimed nothing.
@@ -80,24 +81,24 @@ type DispatchResult = {
80
81
  status: 'rejected';
81
82
  reason: string;
82
83
  };
83
- /**
84
- * Dispatch coordinator connecting the agent registry and job queue.
85
- *
86
- * Routes incoming jobs to matching agents based on label requirements,
87
- * falling back to the persistent queue when no agent is available.
88
- * Handles agent lifecycle events (connect, disconnect, job completion).
89
- *
90
- * The dispatcher does NOT directly access WebSocket connections --
91
- * it uses the onDispatch callback provided at construction. The caller
92
- * (app.ts or server.ts) provides an onDispatch that looks up the agent's
93
- * WS from the registry and sends the job.dispatch message.
94
- */
84
+ export declare function containerSpawnFor(jobConfig: Record<string, unknown> | undefined): {
85
+ image: string;
86
+ authconfig?: {
87
+ username: string;
88
+ password: string;
89
+ serveraddress: string;
90
+ } | undefined;
91
+ } | undefined;
95
92
  export declare class Dispatcher {
96
93
  private readonly registry;
97
94
  private readonly queue;
98
95
  private readonly metrics;
99
96
  private readonly onDispatch;
100
97
  private readonly onNoMatchingAgent?;
98
+ /** See the constructor dep of the same name. */
99
+ private readonly canPrespawnedAgentServe?;
100
+ /** See the constructor dep of the same name. */
101
+ private readonly isPrespawnedAgent?;
101
102
  /**
102
103
  * Single-flight guard for `retryPendingScaleRequests`. The capacity-freed
103
104
  * hook and the leader-gated sweep can both fire concurrently; this prevents
@@ -196,7 +197,34 @@ export declare class Dispatcher {
196
197
  /** Optional hook called when no agent matches the job's labels.
197
198
  * When set, receives the per-job `resources` so the scaler can apply
198
199
  * per-scaler / per-orchestrator / per-machine caps before spawning. */
199
- onNoMatchingAgent?: (labels: string[], jobId: string, runId: string, excludeLabels: string[], resources?: ResourceRequest, orgId?: string) => Promise<ScaleResult>;
200
+ onNoMatchingAgent?: (labels: string[], jobId: string, runId: string, excludeLabels: string[], resources?: ResourceRequest, orgId?: string,
201
+ /**
202
+ * The job's own container image plus registry credentials, already
203
+ * resolved at dispatch. Present means the backend spawns THAT image with
204
+ * the KiCI runtime injected instead of the pool's fixed agent image.
205
+ */
206
+ containerSpawn?: ResolvedContainerSpawn) => Promise<ScaleResult>;
207
+ /**
208
+ * Optional scaler predicate: may this pre-spawned (warm) agent serve this
209
+ * job? A warm agent is generic — started before the job existed, at the
210
+ * pool's shape and running the pool's image, both of which are fixed when
211
+ * it starts. When the predicate returns false the agent is skipped and the
212
+ * job falls through to `onNoMatchingAgent`, which spawns one that fits.
213
+ * Absent (no scaler) means every agent is eligible.
214
+ */
215
+ canPrespawnedAgentServe?: (agentId: string, job: {
216
+ resources?: ResourceRequest;
217
+ hasOwnContainerImage: boolean;
218
+ }) => boolean;
219
+ /**
220
+ * Whether this scaler pre-spawned the agent, i.e. whether
221
+ * `canPrespawnedAgentServe` can ever answer false for it. The queue drain
222
+ * (agent asks for work, rather than job looks for an agent) uses it to
223
+ * decide whether the suitability predicate is worth carrying into the
224
+ * claim: for every ordinary agent it is not, and the drain keeps its
225
+ * single-statement fast path.
226
+ */
227
+ isPrespawnedAgent?: (agentId: string) => boolean;
200
228
  /** Max reconnection delay from agent config (default 60s). Used to derive grace period. */
201
229
  maxReconnectDelayMs?: number;
202
230
  /** Callback fired when a job is permanently failed before/outside agent execution. */
@@ -285,6 +313,39 @@ export declare class Dispatcher {
285
313
  * that is about to reboot. No-op when no roster store is wired.
286
314
  */
287
315
  private filterRebootPending;
316
+ /**
317
+ * Drop pre-spawned (warm) agents that cannot serve this job. Sibling of
318
+ * {@link filterRebootPending}: both remove candidates `findAvailable` matched
319
+ * on labels but that are unusable for a reason labels cannot express.
320
+ *
321
+ * A warm agent's cpu, memory and container image are all set when it starts
322
+ * and cannot change afterwards, so a job asking for something else has to get
323
+ * an agent of its own. Dropping the candidate here is what sends it down the
324
+ * ordinary `onNoMatchingAgent` scale path. No-op when no scaler is wired.
325
+ */
326
+ private filterUnsuitablePrespawned;
327
+ /**
328
+ * The fit question for one job: the shape it declares, and whether it brings
329
+ * its own container image.
330
+ *
331
+ * The shape comes from `jobConfig.resources` when the typed `resources`
332
+ * mirror is absent, because the mirror is optional and several dispatch paths
333
+ * never fill it — a job reaching {@link dispatch} straight off the webhook
334
+ * pipeline or off a worker's reroute handler carries its declaration in
335
+ * `jobConfig` alone. Reading only the mirror there reports a job that
336
+ * declares nothing, which admits it onto a pre-spawned agent of some other
337
+ * size — the exact mismatch this gate exists to refuse.
338
+ */
339
+ private prespawnedFitFor;
340
+ /**
341
+ * The queue-drain half of {@link filterUnsuitablePrespawned}: a per-job
342
+ * predicate for one agent, or undefined when this agent needs no check.
343
+ *
344
+ * Undefined is the common answer — an ordinary agent is not pre-spawned, so
345
+ * the predicate could only ever say yes, and returning one would cost the
346
+ * drain its single-statement fast path for nothing.
347
+ */
348
+ private prespawnedFitFilterFor;
288
349
  /**
289
350
  * Release a reboot-pending host on its real reconnect (down-then-up). Clears
290
351
  * the persisted flag so the very next `onAgentAvailable` drain dispatches the
@@ -384,6 +445,13 @@ export declare class Dispatcher {
384
445
  * host-fanout child may run ONLY on its pinned agent — and `dequeueById`
385
446
  * ignores the pin — so a pinned job is routed to its own agent and never
386
447
  * offered to `findAvailable`, which would mis-deliver it to any label match.
448
+ *
449
+ * Reboot-pending gate: a host whose `restart` job just completed is still
450
+ * connected but about to reboot, so its held post-restart job must NOT be
451
+ * re-driven into the about-to-die box. This is the safety-net re-drive's
452
+ * counterpart of the same gate in `dispatch()` / `dispatchPinned` /
453
+ * `drainForAgent`; without it this path re-drives the held job onto the
454
+ * reboot-pending host and defeats the hold.
387
455
  */
388
456
  private selectConnectedTargetForPending;
389
457
  /**
@@ -77,6 +77,13 @@ export interface UpsertHostInput {
77
77
  agentId: string;
78
78
  tokenId: string | null;
79
79
  lifecycleClass: LifecycleClass;
80
+ /**
81
+ * True when an auto-scaler backend spawned this agent, so `runsOnAll`
82
+ * fan-out skips it. Derived from the scaler manager's registration lookup,
83
+ * never from `lifecycleClass` (which is `ephemeral` for every agent when the
84
+ * auth mode is `none`). Omitted ⇒ false.
85
+ */
86
+ scalerManaged?: boolean;
80
87
  labels: string[];
81
88
  hostname: string | null;
82
89
  platform: string;
@@ -176,13 +183,38 @@ export declare class HostRosterStore {
176
183
  getStagedVersion(agentId: string): Promise<string | null>;
177
184
  listAll(): Promise<HostRosterRow[]>;
178
185
  /**
179
- * Resolve every roster host matching a `runsOnAll` predicate (OR-of-AND
180
- * include groups, minus exclude labels), tagged with its derived status. This
181
- * is the host-fanout resolver: it returns declared-but-absent static hosts
182
- * (status `unreachable`) so the caller can apply `onUnreachable` the live
183
- * registry alone cannot name an expected-but-absent host.
186
+ * Resolve EVERY roster host matching a label predicate (OR-of-AND include
187
+ * groups, minus exclude labels), tagged with its derived status. Returns
188
+ * declared-but-absent static hosts (status `unreachable`) so a caller can
189
+ * apply `onUnreachable` the live registry alone cannot name an
190
+ * expected-but-absent host.
191
+ *
192
+ * This is the INVENTORY query: it includes auto-scaler-spawned hosts, whose
193
+ * `lifecycleClass` is what the SDK's `ctx.kici.inventory` exposes to tell
194
+ * them apart. `runsOnAll` fan-out uses {@link findFanoutTargets} instead.
184
195
  */
185
196
  findMatching(include: readonly (readonly LabelMatcher[])[], exclude: readonly LabelMatcher[], graceMs: number): Promise<MatchedHost[]>;
197
+ /**
198
+ * Resolve the `runsOnAll` fan-out target set: {@link findMatching}, minus
199
+ * every auto-scaler-spawned host.
200
+ *
201
+ * Fan-out targets DECLARED fleet members. An auto-scaler agent is spawned to
202
+ * a pool's fixed shape, so a child pinned to one runs at that shape rather
203
+ * than its own — the same defect the dispatcher's warm-pool suitability gate
204
+ * removes, on the path a pin deliberately leaves ungated.
205
+ *
206
+ * Separate from `findMatching` on purpose: `findMatching` also backs the
207
+ * SDK's `inventory.query`, where a host the caller asked for must never go
208
+ * missing (a selector returning FEWER hosts than no selector at all).
209
+ */
210
+ findFanoutTargets(include: readonly (readonly LabelMatcher[])[], exclude: readonly LabelMatcher[], graceMs: number): Promise<MatchedHost[]>;
211
+ /**
212
+ * Shared row loop behind {@link findMatching} and {@link findFanoutTargets}.
213
+ * `excludeScalerManaged` stays private to the store — each public method
214
+ * names its own semantics, so a call site cannot silently pick the wrong
215
+ * population by passing a bare boolean.
216
+ */
217
+ private matchRows;
186
218
  /**
187
219
  * Map a roster row to the canonical {@link HostInventoryEntry} — the queryable
188
220
  * shape returned by the `inventory.query`/`inventory.get` RPC and typed on the
@@ -137,6 +137,8 @@ export interface RosterReconciler {
137
137
  agentId: string;
138
138
  tokenId: string | null;
139
139
  lifecycleClass: 'static' | 'ephemeral';
140
+ /** True when a scaler backend spawned this agent (kept out of `runsOnAll`). */
141
+ scalerManaged?: boolean;
140
142
  labels: string[];
141
143
  hostname: string | null;
142
144
  platform: string;
package/dist/app.d.ts CHANGED
@@ -66,6 +66,8 @@ import type { RegistrationStore } from './registration/registration-store.js';
66
66
  import type { RegistrationIndex } from './registration/registration-index.js';
67
67
  import type { CronScheduler } from './cron/cron-scheduler.js';
68
68
  import { type ConfigRouteDeps } from './routes/admin-config.js';
69
+ import type { ProcessingDeps } from './pipeline/processor.js';
70
+ import type { InvokeGateDeps } from './pipeline/invoke-gate.js';
69
71
  import { WebhookIngestOutcome } from './pipeline/process-webhook.js';
70
72
  import type { IngestOverflowBuffer } from './webhook/ingest-overflow-buffer.js';
71
73
  import type { IngestOverflowReplayer } from './webhook/ingest-overflow-replayer.js';
@@ -85,9 +87,8 @@ import type { GenericSourceManager } from './webhook/generic-sources.js';
85
87
  import type { TrustStore } from './events/trust-store.js';
86
88
  import type { ContextStore } from './contexts/context-store.js';
87
89
  import type { VariableStore } from './contexts/variable-store.js';
88
- import type { HeldRunStore } from './contexts/held-runs.js';
90
+ import type { HeldRunStore, ReleaseSignal } from './contexts/held-runs.js';
89
91
  import type { StepApprovalBridge } from './approvals/step-approval-bridge.js';
90
- import type { ContributorCache } from './security/contributor-cache.js';
91
92
  import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
92
93
  /**
93
94
  * All dependencies needed to create the orchestrator Hono app.
@@ -179,6 +180,14 @@ export interface AppDependencies {
179
180
  * is registered so `ctx.kici.oidc.token()` mints a `kici-local` dev token.
180
181
  */
181
182
  localOidcSigner?: LocalSigner;
183
+ /**
184
+ * Test-only fault-injection policy, threaded from a test-only entrypoint via
185
+ * `bootstrapOrchestrator`. Undefined in production. Consumed here only for the
186
+ * initial-mint provenance seam (`initialMintFault`); the other seams live in
187
+ * the composition root / server hooks. Type-only reference — the runtime
188
+ * module never enters the shipped bundle.
189
+ */
190
+ faultInjection?: import('./testing/fault-injection.js').OrchestratorFaultInjection;
182
191
  /**
183
192
  * Orchestrator-owned provenance signing (Phase 1 root of trust). Present when
184
193
  * `KICI_ORCHESTRATOR_PROVENANCE_ISSUER` is configured: the orchestrator mints +
@@ -257,6 +266,18 @@ export interface AppDependencies {
257
266
  configRouteDeps?: ConfigRouteDeps;
258
267
  /** Event router for internal event delivery. Optional -- if not set, event routing is inactive. */
259
268
  eventRouter?: EventRouter;
269
+ /** Invoke-gate dependencies (summon callback + chain-depth bound). Optional -- if not set, invoke gates fail loudly. */
270
+ invokeGateDeps?: InvokeGateDeps;
271
+ /**
272
+ * Late-binding handle the internal-event dispatch adapter reads to obtain the
273
+ * live `ProcessingDeps` bag. `createApp` populates it, because the bag can
274
+ * only be assembled here while the event router that dispatches internal
275
+ * events is constructed earlier in the bootstrap. Left unset by callers that
276
+ * never route internal events (tests constructing a bare app).
277
+ */
278
+ processingDepsRef?: {
279
+ current: (() => ProcessingDeps) | null;
280
+ };
260
281
  /** Event store. Optional -- mounted when admin DLQ admin route should be available. */
261
282
  eventStore?: EventStore;
262
283
  /** Event emitter for system events (workflow/job complete). Optional -- if not set, system events are skipped. */
@@ -297,6 +318,16 @@ export interface AppDependencies {
297
318
  heldRunStore?: HeldRunStore;
298
319
  /** Step-approval bridge — opens step-scoped holds and relays their resolution back to the waiting agent. Optional. */
299
320
  stepApprovalBridge?: StepApprovalBridge;
321
+ /**
322
+ * Re-dispatch a job whose hold was released. Supplied by both mode hooks
323
+ * through `appDepsExtras`; declared here so the admin held-run routes can
324
+ * read it type-safely rather than off an untyped spread. Optional — an app
325
+ * assembled without it mounts no local held-run decision surface, because a
326
+ * release that cannot dispatch is a row flip pretending to be a release.
327
+ */
328
+ onJobRelease?: (signal: ReleaseSignal) => Promise<void>;
329
+ /** Replay the stored dispatch context of a released workflow-scoped hold. Same provenance as `onJobRelease`. */
330
+ onWorkflowRelease?: (signal: ReleaseSignal) => Promise<void>;
300
331
  /** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are dispatched without permission checks. */
301
332
  globalWorkflowPolicy?: GlobalWorkflowPolicy;
302
333
  /** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
@@ -330,10 +361,6 @@ export interface AppDependencies {
330
361
  coordinator?: RunCoordinator;
331
362
  /** Peer registry for aggregating infrastructure across cluster. Optional. */
332
363
  peerRegistry?: PeerRegistry;
333
- /** Contributor permission cache. Optional -- threaded through from server.ts
334
- * so membership-related webhooks invalidate matching entries immediately
335
- * (instead of waiting for the 15-minute TTL). */
336
- contributorCache?: ContributorCache;
337
364
  /**
338
365
  * Shared aggregator for agent-pushed metrics. When omitted, app.ts
339
366
  * constructs its own — kept optional so existing tests still work
@@ -1,22 +1,15 @@
1
- /**
2
- * Shared approve/reject applier — the single code path behind both the
3
- * dashboard WS handler and the `kici` CLI HTTP route, so the authorization +
4
- * resume story is identical regardless of the surface.
5
- *
6
- * Flow for an approve:
7
- * 1. Load the hold + its recorded decisions.
8
- * 2. Build the team-membership lookup and run `canApprove` (eligibility +
9
- * self-approval gate). Ineligible actors are rejected with a clear reason.
10
- * 3. Record the decision in `held_run_approvals`.
11
- * 4. Re-evaluate. If all clauses are satisfied, `release()` the hold and
12
- * resume the element (re-dispatch for job/workflow; the agent bridge for
13
- * step scope — wired by the caller via `onStepRelease`).
14
- *
15
- * A reject records the decision and `reject()`s the hold (failing the element).
16
- */
17
1
  import { ApprovalDecision } from '@kici-dev/engine';
2
+ import type { HeldRun } from '../db/types.js';
3
+ import { HoldOutcome } from '../pipeline/security-hold-check.js';
18
4
  import type { HeldRunStore, ReleaseSignal } from '../contexts/held-runs.js';
19
5
  import { type TeamMembershipLookup } from './approval-resolver.js';
6
+ /** Whether a decision's post-commit consequence ran to completion. */
7
+ export interface DecisionConsequenceResult {
8
+ /** True when the whole consequence chain ran; false when a step of it threw. */
9
+ ok: boolean;
10
+ /** The failure message, when `ok` is false. */
11
+ error?: string;
12
+ }
20
13
  /** Outcome of applying a decision. */
21
14
  export interface ApplyDecisionResult {
22
15
  /** Whether the decision was accepted (recorded). */
@@ -29,6 +22,29 @@ export interface ApplyDecisionResult {
29
22
  reason?: string;
30
23
  /** The release signal, when the element was released (for the caller to resume). */
31
24
  release?: ReleaseSignal;
25
+ /**
26
+ * The decision's post-commit consequence — already running, deliberately not
27
+ * awaited by the applier. Present on `released` and `rejected`, absent on
28
+ * every outcome that changed nothing (`pending`, `ineligible`, `not-found`).
29
+ *
30
+ * It NEVER rejects: a throw anywhere in the chain is logged and surfaces as
31
+ * `{ ok: false, error }`, so a caller may drop it on the floor without
32
+ * risking an unhandled rejection. A caller that answers a waiting client
33
+ * answers without awaiting this, and attaches its audit record to it instead
34
+ * — the record then carries the consequence's real outcome rather than a
35
+ * guess. A caller that needs the consequence finished before it continues
36
+ * (a test, a sequential sweep) awaits it.
37
+ *
38
+ * The consequence is where an approve was spending nine-plus seconds behind a
39
+ * ten-second relay budget, answering 504 on an approval that had already
40
+ * landed. It is not a stronger answer for having been awaited: the resume it
41
+ * runs is the same dispatch the original webhook delivery ran without any
42
+ * client waiting on it, `dispatchMatchedWorkflow` returning says only that
43
+ * jobs were routed rather than that any of them ran, and `resumeWorkflow`
44
+ * already reports its own failures on the run row and on the provider's check
45
+ * runs rather than through this return value.
46
+ */
47
+ consequence?: Promise<DecisionConsequenceResult>;
32
48
  }
33
49
  /** Dependencies injected into `applyDecision` (testable in isolation). */
34
50
  export interface ApplyDecisionDeps {
@@ -55,10 +71,42 @@ export interface ApplyDecisionDeps {
55
71
  */
56
72
  onStepReject?: (heldRunId: string, reason?: string) => Promise<void> | void;
57
73
  /**
58
- * Called when a workflow-scoped hold is rejected (install gate rejected)
59
- * cancel the run and drop the pending workflow context. Carries the runId.
74
+ * Called when a workflow-scoped hold is rejected cancel the run and drop
75
+ * the pending workflow context. Carries the whole hold row and the rejecter's
76
+ * own reason, which becomes the run's cancellation reason: the callers serve
77
+ * the install gate, the org trust policy's PR-wide hold and the SDK's
78
+ * workflow-level `requireApproval` alike, so a reason invented at the wiring
79
+ * site would misattribute one of them.
80
+ *
81
+ * The row is what tells those apart — only some of them posted a pending
82
+ * `KiCI Security` check, and only those have one to complete. Resolves to
83
+ * whether a terminal security check was WRITTEN, so this applier suppresses
84
+ * its own post only when one actually landed.
85
+ */
86
+ onWorkflowReject?: (hold: HeldRun, reason?: string) => Promise<boolean>;
87
+ /**
88
+ * Terminalize the `KiCI Security` check a just-ended hold posted, for every
89
+ * hold this applier ends that `onWorkflowReject` did not already report.
90
+ *
91
+ * Without it a job-scoped security hold answered from the dashboard strands
92
+ * its check forever: the row leaves `pending`, so the stale detector's
93
+ * approval-window sweep — which only ever sees pending rows — can no longer
94
+ * reach it. Optional so an orchestrator with no provider wiring degrades to
95
+ * flip-and-resume rather than failing the decision.
96
+ *
97
+ * `actorSub` is the deciding actor's opaque subject id, which is what this
98
+ * applier has and NOT something a contributor can read. Resolving it to a
99
+ * display name — or dropping the attribution when it resolves to none — is
100
+ * the wiring site's job, because only that site holds the org's identity
101
+ * directory. Omitted entirely where the summary must match one a delegate
102
+ * built without it; see the reject arm below.
60
103
  */
61
- onWorkflowReject?: (runId: string) => Promise<void>;
104
+ settleSecurityCheck?: (args: {
105
+ hold: HeldRun;
106
+ outcome: HoldOutcome;
107
+ actorSub?: string | undefined;
108
+ reason?: string | undefined;
109
+ }) => Promise<boolean>;
62
110
  }
63
111
  /** Apply a single approve/reject decision to a hold. */
64
112
  export declare function applyDecision(deps: ApplyDecisionDeps, args: {
@@ -0,0 +1,45 @@
1
+ /**
2
+ * The release wiring the local (`kici-admin`) held-run decision surface runs on.
3
+ *
4
+ * `createApp` composes this for every mode. It lives here rather than inline in
5
+ * that composition root because the defect it guards against is a MISSING
6
+ * field, not a wrong algorithm: a release path that flips the `held_runs` row
7
+ * without dispatching looks like it works and does nothing. A factory can be
8
+ * constructed by a test and its fields asserted; an object literal buried in
9
+ * `createApp` cannot.
10
+ *
11
+ * It returns `undefined` when there is no job-release callback, so an app
12
+ * assembled without one mounts no decision route at all rather than one whose
13
+ * approves are silent no-ops.
14
+ *
15
+ * Every callback reads the LIVE processing-deps bag at call time rather than
16
+ * capturing one: the provider registry is swapped whenever sources reload, and
17
+ * a captured bag would resolve a check poster from a registry that no longer
18
+ * serves the routing key.
19
+ */
20
+ import type { ReleaseSignal } from '../contexts/held-runs.js';
21
+ import type { ProcessingDeps } from '../pipeline/processor.js';
22
+ import type { HeldRunReleaseWiring } from '../routes/admin-held-runs.js';
23
+ /** The subset of `createApp`'s deps this factory reads. */
24
+ export interface HeldRunReleaseInputs {
25
+ /**
26
+ * Re-dispatch a released job-scoped hold. Supplied by both mode hooks through
27
+ * `appDepsExtras`; its absence is what makes the whole surface absent.
28
+ */
29
+ onJobRelease?: ((signal: ReleaseSignal) => Promise<void>) | undefined;
30
+ /** Replay the stored dispatch context of a released workflow-scoped hold. */
31
+ onWorkflowRelease?: ((signal: ReleaseSignal) => Promise<void>) | undefined;
32
+ /** The live processing-deps bag, assembled per call. */
33
+ buildProcessingDeps: () => ProcessingDeps;
34
+ }
35
+ /**
36
+ * The reason a workflow-scoped hold rejected through this surface records on the
37
+ * run it cancels, when the operator supplied none.
38
+ *
39
+ * Scope-neutral on purpose: the install gate, the org trust policy's PR-wide
40
+ * hold and an SDK workflow-level `requireApproval` all arrive here, so a
41
+ * reason naming one of them would mislabel the other two.
42
+ */
43
+ export declare const DEFAULT_LOCAL_REJECT_REASON = "Workflow hold rejected by the orchestrator operator";
44
+ export declare function buildHeldRunRelease(inputs: HeldRunReleaseInputs): HeldRunReleaseWiring | undefined;
45
+ //# sourceMappingURL=held-run-release-wiring.d.ts.map