@kici-dev/orchestrator 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/dist/__test-helpers__/fake-scaler-state-store.d.ts +24 -0
  2. package/dist/__test-helpers__/mock-db-query.d.ts +130 -0
  3. package/dist/__test-helpers__/mock-db.d.ts +27 -0
  4. package/dist/agent/agent-version.d.ts +34 -0
  5. package/dist/agent/dispatcher.d.ts +151 -13
  6. package/dist/agent/host-roster.d.ts +37 -5
  7. package/dist/agent/registry.d.ts +2 -0
  8. package/dist/app.d.ts +56 -7
  9. package/dist/approvals/apply-decision.d.ts +67 -19
  10. package/dist/approvals/held-run-release-wiring.d.ts +45 -0
  11. package/dist/approvals/independent-wiring.d.ts +55 -0
  12. package/dist/approvals/resume-router.d.ts +9 -0
  13. package/dist/approvals/triggerer-subject.d.ts +37 -0
  14. package/dist/cache/dep-cache.d.ts +54 -10
  15. package/dist/cache/global-eval-round-cache.d.ts +95 -0
  16. package/dist/cache/index.d.ts +3 -0
  17. package/dist/cache/pending-global-evals.d.ts +42 -0
  18. package/dist/cache/pending-inits.d.ts +10 -0
  19. package/dist/cli/api-client.d.ts +18 -0
  20. package/dist/cli/commands/cluster-settings.d.ts +52 -3
  21. package/dist/cli/commands/diagnose.d.ts +6 -0
  22. package/dist/cli/commands/held-run.d.ts +58 -0
  23. package/dist/cli/commands/runs.d.ts +1 -0
  24. package/dist/cli/commands/trust-policy.d.ts +117 -12
  25. package/dist/cli/kici-admin.d.ts +10 -0
  26. package/dist/cli.js +2770 -938
  27. package/dist/cluster/cluster-identity.d.ts +3 -3
  28. package/dist/cluster/cluster-settings-reader.d.ts +59 -1
  29. package/dist/cluster/coordinator.d.ts +44 -0
  30. package/dist/cluster/index.d.ts +4 -0
  31. package/dist/cluster/peer-handler.d.ts +19 -0
  32. package/dist/cluster/peer-registry.d.ts +17 -0
  33. package/dist/cluster/plan-headroom-store.d.ts +26 -0
  34. package/dist/cluster/worker-eviction.d.ts +69 -0
  35. package/dist/cold-store/load-event-log-range.d.ts +10 -3
  36. package/dist/cold-store/load-secret-audit-log-range.d.ts +4 -3
  37. package/dist/cold-store/tables/secret-audit-log.d.ts +4 -3
  38. package/dist/config.d.ts +48 -21
  39. package/dist/content-requirements-cache.d.ts +55 -0
  40. package/dist/contexts/held-runs.d.ts +123 -16
  41. package/dist/contexts/protection/aggregate.d.ts +3 -2
  42. package/dist/contexts/protection/branch-gate.d.ts +15 -0
  43. package/dist/contexts/protection/concurrency-gate.d.ts +8 -2
  44. package/dist/contexts/protection/pipeline.d.ts +18 -0
  45. package/dist/contexts/release-queued-holds.d.ts +35 -0
  46. package/dist/dashboard/attestation-filters.d.ts +8 -0
  47. package/dist/dashboard/handler.d.ts +6 -24
  48. package/dist/db/migrations/109_cluster_settings_cache_knobs.d.ts +4 -0
  49. package/dist/db/migrations/110_cluster_settings_global_eval_knobs.d.ts +4 -0
  50. package/dist/db/migrations/111_cluster_settings_global_eval_wait.d.ts +4 -0
  51. package/dist/db/migrations/112_execution_runs_workflow_repo.d.ts +4 -0
  52. package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +30 -0
  53. package/dist/db/migrations/114_ingest_queue_claim.d.ts +4 -0
  54. package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +9 -0
  55. package/dist/db/migrations/116_org_plan_headroom.d.ts +4 -0
  56. package/dist/db/migrations/117_invoke_gate_proxy_jobs.d.ts +4 -0
  57. package/dist/db/migrations/118_invoke_gate_scheduling.d.ts +4 -0
  58. package/dist/db/migrations/119_scaler_ha_ownership.d.ts +4 -0
  59. package/dist/db/migrations/120_cluster_settings_scaler_reaper.d.ts +4 -0
  60. package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +19 -0
  61. package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +25 -0
  62. package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +24 -0
  63. package/dist/db/migrations/124_cluster_settings_provision_backoff.d.ts +4 -0
  64. package/dist/db/migrations/125_org_trust_directory.d.ts +30 -0
  65. package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +28 -0
  66. package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +26 -0
  67. package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +23 -0
  68. package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +24 -0
  69. package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +32 -0
  70. package/dist/db/types.d.ts +428 -12
  71. package/dist/entry-helpers.d.ts +28 -0
  72. package/dist/events/event-emitter.d.ts +22 -0
  73. package/dist/events/event-router.d.ts +87 -1
  74. package/dist/events/types.d.ts +2 -23
  75. package/dist/git/credential-broker.d.ts +94 -0
  76. package/dist/git/inherited-secret.d.ts +24 -0
  77. package/dist/git/installation-token.d.ts +38 -0
  78. package/dist/git/job-context.d.ts +13 -0
  79. package/dist/index.js +324 -84
  80. package/dist/metrics/agent-metrics-aggregator.d.ts +4 -4
  81. package/dist/metrics/prometheus.d.ts +170 -3
  82. package/dist/oidc/oidc-mint-registration.d.ts +7 -3
  83. package/dist/orchestrator-core.d.ts +91 -24
  84. package/dist/pipeline/content-filter.d.ts +71 -0
  85. package/dist/pipeline/direct-ingress-deps.d.ts +100 -0
  86. package/dist/pipeline/dispatch-matched-workflow.d.ts +521 -11
  87. package/dist/pipeline/global-eval-round.d.ts +307 -0
  88. package/dist/pipeline/internal-event-pipeline.d.ts +140 -0
  89. package/dist/pipeline/invoke-gate.d.ts +164 -0
  90. package/dist/pipeline/job-contexts.d.ts +16 -17
  91. package/dist/pipeline/needs-scheduler.d.ts +13 -0
  92. package/dist/pipeline/process-webhook.d.ts +126 -31
  93. package/dist/pipeline/processor.d.ts +282 -23
  94. package/dist/pipeline/rerun.d.ts +21 -0
  95. package/dist/pipeline/resume-workflow.d.ts +40 -7
  96. package/dist/pipeline/route-or-dispatch-jobs.d.ts +15 -0
  97. package/dist/pipeline/security-hold-check.d.ts +217 -0
  98. package/dist/pipeline/test-pipeline.d.ts +12 -0
  99. package/dist/pipeline/undispatched-hold-checks.d.ts +23 -0
  100. package/dist/pipeline/webhook-payload-store.d.ts +20 -0
  101. package/dist/policy/dashboard-write-policy-listener.d.ts +62 -0
  102. package/dist/policy/dashboard-write-policy.d.ts +14 -0
  103. package/dist/provenance/backfill-run.d.ts +10 -1
  104. package/dist/provider-registry.d.ts +47 -4
  105. package/dist/providers/github/check-status-poster.d.ts +33 -3
  106. package/dist/providers/github/commit-message.d.ts +20 -0
  107. package/dist/providers/github/file-contents.d.ts +40 -0
  108. package/dist/providers/github/index.d.ts +2 -2
  109. package/dist/providers/github/normalizer.d.ts +3 -2
  110. package/dist/providers/local/normalizer.d.ts +3 -5
  111. package/dist/providers/universal-git/config.d.ts +2 -0
  112. package/dist/providers/universal-git/index.d.ts +11 -2
  113. package/dist/providers/universal-git/normalizer.d.ts +10 -0
  114. package/dist/queue/cleanup.d.ts +7 -1
  115. package/dist/queue/job-queue.d.ts +112 -7
  116. package/dist/queue/terminalize-unroutable.d.ts +13 -0
  117. package/dist/registration/extractor.d.ts +3 -3
  118. package/dist/registration/registration-index.d.ts +7 -0
  119. package/dist/registration/registration-run-match.d.ts +47 -0
  120. package/dist/registration/registration-store.d.ts +22 -0
  121. package/dist/reporting/check-run-reporter.d.ts +146 -2
  122. package/dist/reporting/check-run-summary.d.ts +25 -1
  123. package/dist/reporting/execution-tracker.d.ts +324 -9
  124. package/dist/reporting/log-chunk-sink.d.ts +8 -5
  125. package/dist/reporting/run-aggregator.d.ts +4 -14
  126. package/dist/reporting/step-display-order.d.ts +43 -0
  127. package/dist/routes/admin-held-runs.d.ts +142 -0
  128. package/dist/routes/admin-org-settings.d.ts +5 -0
  129. package/dist/routes/admin-trust-policy.d.ts +34 -6
  130. package/dist/routes/admin.d.ts +15 -0
  131. package/dist/scaler/backend-factory.d.ts +55 -0
  132. package/dist/scaler/bare-metal-backend.d.ts +43 -5
  133. package/dist/scaler/claim-store.d.ts +119 -0
  134. package/dist/scaler/config.d.ts +4 -0
  135. package/dist/scaler/container-backend.d.ts +18 -4
  136. package/dist/scaler/container-routing.d.ts +23 -0
  137. package/dist/scaler/container-spawn.d.ts +28 -0
  138. package/dist/scaler/event-backend.d.ts +124 -0
  139. package/dist/scaler/event-provision-reaper.d.ts +350 -0
  140. package/dist/scaler/failure-tracker.d.ts +1 -1
  141. package/dist/scaler/firecracker-backend.d.ts +33 -6
  142. package/dist/scaler/index.d.ts +11 -1
  143. package/dist/scaler/label-matcher.d.ts +4 -3
  144. package/dist/scaler/manager.d.ts +905 -76
  145. package/dist/scaler/resolve-container-auth.d.ts +45 -0
  146. package/dist/scaler/scaler-events.d.ts +11 -0
  147. package/dist/scaler/scaler-state-store.d.ts +294 -10
  148. package/dist/scaler/types.d.ts +131 -13
  149. package/dist/scaler/warm-pool.d.ts +132 -29
  150. package/dist/security/comment-handler.d.ts +50 -11
  151. package/dist/security/global-workflow-policy.d.ts +52 -12
  152. package/dist/security/identity-link.d.ts +60 -0
  153. package/dist/security/lock-source.d.ts +11 -13
  154. package/dist/security/reduced-privilege-note.d.ts +59 -0
  155. package/dist/security/trust-directory-store.d.ts +233 -0
  156. package/dist/security/trust-policy-gate.d.ts +94 -59
  157. package/dist/security/trust-policy-store.d.ts +3 -0
  158. package/dist/security/trust-resolver.d.ts +14 -98
  159. package/dist/server.d.ts +11 -1
  160. package/dist/server.js +56329 -42234
  161. package/dist/stale-detector/gate-deadline-detector.d.ts +52 -0
  162. package/dist/stale-detector/stale-run-detector.d.ts +61 -3
  163. package/dist/standalone.js +37619 -23570
  164. package/dist/webhook/ingest-accept.d.ts +70 -0
  165. package/dist/webhook/ingest-overflow-buffer.d.ts +35 -4
  166. package/dist/webhook/ingest-overflow-replayer.d.ts +50 -6
  167. package/dist/worker/in-memory-job-queue.d.ts +1 -1
  168. package/dist/ws/agent-handler.d.ts +31 -4
  169. package/dist/ws/dashboard-context-handler.d.ts +28 -4
  170. package/dist/ws/dashboard-global-workflows-handler.d.ts +36 -9
  171. package/dist/ws/execution-status-frame.d.ts +32 -0
  172. package/dist/ws/failure-messages.d.ts +2 -0
  173. package/dist/ws/git-credential-relay.d.ts +39 -0
  174. package/dist/ws/oidc-token-relay.d.ts +6 -11
  175. package/dist/ws/platform-client.d.ts +68 -9
  176. package/dist/ws/test-relay-handlers.d.ts +35 -10
  177. package/installer-image-digests.json +3 -3
  178. package/package.json +24 -19
  179. package/sbom.spdx.json +868 -821
  180. package/dist/approvals/team-membership-lookup.d.ts +0 -13
  181. package/dist/pipeline/inline-eval.d.ts +0 -44
  182. package/dist/providers/github/contributor-resolver.d.ts +0 -30
  183. package/dist/security/contributor-cache.d.ts +0 -83
@@ -0,0 +1,142 @@
1
+ /**
2
+ * Admin API routes for reading and answering held runs.
3
+ *
4
+ * Exposes GET `/api/v1/admin/held-runs` and POST
5
+ * `/api/v1/admin/held-runs/decision` so `kici-admin held-run list|approve|reject`
6
+ * can release a hold locally, without a Platform.
7
+ *
8
+ * **Independent orchestrators only.** Wherever a Platform is attached it owns
9
+ * who may answer a hold: the Platform's held-run trust gate resolves the hold's
10
+ * type and requires `ci_trust:write` for a security hold and `contexts:write` /
11
+ * `contexts:admin` for the rest, against the acting member's org RBAC. This
12
+ * route authenticates with an orchestrator admin token, which carries
13
+ * orchestrator RBAC and no Platform membership at all — so answering a hold here
14
+ * on a Platform-attached deployment would land a release the Platform's own gate
15
+ * never authorized. Both write and read therefore refuse with 409 in
16
+ * `platform` / `hybrid` / `observed`, mirroring the trust-policy write verbs.
17
+ *
18
+ * On an independent orchestrator there is no upstream authority, and until this
19
+ * route existed an approval-queue hold there had no answer at all: `/kici
20
+ * approve` releases the `security` queue only, and the dashboard, `kici
21
+ * approve` and the MCP tools all reach the applier over the Platform relay.
22
+ *
23
+ * Answering goes through the shared `applyDecision` applier, not through a
24
+ * direct row flip, so an approve here obeys the hold's own
25
+ * `approval_requirement` clauses, records a `held_run_approvals` row, and — this
26
+ * is the part a row flip silently skips — resumes the held element through
27
+ * `routeRelease`, which re-dispatches the job or replays the workflow dispatch.
28
+ *
29
+ * The resume runs AFTER this route answers. The decision is durable at the
30
+ * applier's own transaction, and the resume it triggers is unbounded work whose
31
+ * latency has no business being the operator's — see
32
+ * `ApplyDecisionResult.consequence`. The access-log entry is written when that
33
+ * consequence settles, so a resume that failed is recorded as `error` with its
34
+ * message rather than being lost behind a successful-looking answer.
35
+ *
36
+ * Releasing a hold lets the held work RUN; it does not make its contributor
37
+ * trusted. The resumed dispatch replays the trust resolution the hold was
38
+ * created under, so an untrusted fork PR still runs with the base-branch lock
39
+ * file, no install/registry secrets, and an isolated cache write scope.
40
+ */
41
+ import { Hono } from 'hono';
42
+ import type { Kysely } from 'kysely';
43
+ import type { OrchestratorMode } from '@kici-dev/engine';
44
+ import { HeldRunStore, type ReleaseSignal } from '../contexts/held-runs.js';
45
+ import { TrustDirectoryStore } from '../security/trust-directory-store.js';
46
+ import { adminActorSub, triggererSubjectFor } from '../approvals/triggerer-subject.js';
47
+ import { type ResolveCheckStatusPoster } from '../pipeline/security-hold-check.js';
48
+ import type { RbacEnforcer, Role } from '../secrets/rbac.js';
49
+ import type { AccessLogWriter } from '../audit/access-log.js';
50
+ import type { Database, HeldRun } from '../db/types.js';
51
+ /**
52
+ * Wording for the Platform-managed refusal, surfaced verbatim by the CLI.
53
+ *
54
+ * It names the surfaces that DO answer a hold on a Platform-attached
55
+ * orchestrator, because "not here" without "there instead" is the refusal an
56
+ * operator reads as a broken build.
57
+ */
58
+ export declare const PLATFORM_MANAGED_HELD_RUN_MESSAGE: string;
59
+ /**
60
+ * Wording for the step-scope refusal.
61
+ *
62
+ * A step-scoped hold is answered by notifying the waiting agent through the
63
+ * step-approval bridge, which an independent orchestrator does not wire — so
64
+ * this route has no way to tell the agent. Flipping the row anyway would be
65
+ * strictly worse than refusing: the agent would keep waiting while the row left
66
+ * `pending`, putting it beyond the stale detector's expiry sweep, so the step
67
+ * would block with nothing left to release OR expire it.
68
+ */
69
+ export declare const STEP_SCOPE_UNSUPPORTED_MESSAGE: string;
70
+ /** The release wiring the applier needs to actually resume a released element. */
71
+ export interface HeldRunReleaseWiring {
72
+ /**
73
+ * Re-dispatch a job-scoped release.
74
+ *
75
+ * REQUIRED, and the mount is conditional on it. A release path that flips the
76
+ * row without dispatching looks like it works and does nothing — the hold
77
+ * reads `approved`, the job never starts, and no sweep will ever look at the
78
+ * row again.
79
+ */
80
+ onJobRelease: (signal: ReleaseSignal) => Promise<void>;
81
+ /** Replay the stored dispatch context of a released workflow-scoped hold. */
82
+ onWorkflowRelease?: (signal: ReleaseSignal) => Promise<void>;
83
+ /**
84
+ * Cancel the run and drop the pending dispatch context of a REJECTED
85
+ * workflow-scoped hold, resolving to whether it wrote the hold's terminal
86
+ * security check itself.
87
+ */
88
+ onWorkflowReject?: (hold: HeldRun, reason?: string) => Promise<boolean>;
89
+ /** Resolve the check poster of the provider bundle serving a routing key. */
90
+ resolveCheckStatusPoster?: ResolveCheckStatusPoster;
91
+ }
92
+ export interface HeldRunRouteDeps {
93
+ /** The held-run store the decision is applied through. */
94
+ store: HeldRunStore;
95
+ /**
96
+ * The org approval directory — the only place an independent orchestrator
97
+ * holds team membership, and therefore the only thing that can satisfy a
98
+ * `{team}` clause on a hold's requirement.
99
+ */
100
+ directory: TrustDirectoryStore;
101
+ /**
102
+ * Orchestrator database, for the two per-decision reads the applier needs
103
+ * that no store owns: `org_settings.allow_self_approval` and the run's
104
+ * triggerer.
105
+ */
106
+ db: Kysely<Database>;
107
+ rbac: RbacEnforcer;
108
+ /** This orchestrator's mode; decides whether either verb is permitted at all. */
109
+ mode: OrchestratorMode;
110
+ /**
111
+ * Audit sink, REQUIRED for the same reason the trust-policy route's is: the
112
+ * guarantee this route makes is that a locally-answered hold is always
113
+ * attributable, and an optional sink would leave that resting on every
114
+ * construction site remembering to pass one.
115
+ */
116
+ accessLog: AccessLogWriter;
117
+ release: HeldRunReleaseWiring;
118
+ }
119
+ type AdminEnv = {
120
+ Variables: {
121
+ role: Role;
122
+ userId: string;
123
+ routingKey: string | null;
124
+ };
125
+ };
126
+ /**
127
+ * The approval subject for a decision taken through an admin token.
128
+ *
129
+ * Namespaced `service:<token user id>`, matching how the dashboard applier
130
+ * derives a subject for a service-account actor. The namespace is what keeps it
131
+ * from ever colliding with a Keycloak sub named by an `approvers:` clause: an
132
+ * operator token answers as itself, never as a person it typed the name of.
133
+ * `held_run_approvals.approver_user_id` is the record of who approved, and a
134
+ * subject the operator merely asserted would make that record false.
135
+ *
136
+ * Re-exported from the shared subject module so this route and the dashboard
137
+ * handler cannot drift into two `service:` namespaces.
138
+ */
139
+ export { adminActorSub };
140
+ export { triggererSubjectFor };
141
+ export declare function createHeldRunRoutes(deps: HeldRunRouteDeps): Hono<AdminEnv>;
142
+ //# sourceMappingURL=admin-held-runs.d.ts.map
@@ -18,6 +18,11 @@ import type { AccessLogWriter } from '../audit/access-log.js';
18
18
  interface OrgSettingsRouteDeps {
19
19
  db: Kysely<Database>;
20
20
  rbac: RbacEnforcer;
21
+ /**
22
+ * Applies when `cluster_settings.global_workflows_enabled` is NULL —
23
+ * `config.globalWorkflowsEnabled`.
24
+ */
25
+ globalWorkflowsEnabledDefault: boolean;
21
26
  /**
22
27
  * Optional — when wired, each `dashboard_write_policy` flip emits one
23
28
  * `access_log` row (`org_settings.dashboard_write_policy.update`)
@@ -1,17 +1,28 @@
1
1
  /**
2
- * Admin API routes for the org trust policy.
2
+ * Admin API routes for the org trust policy and the approval directory it
3
+ * arrives with.
3
4
  *
4
5
  * Exposes GET / PATCH `/api/v1/admin/trust-policy` so `kici-admin` can read the
5
- * enforced policy and, on an independent orchestrator, set it.
6
+ * enforced policy and, on an independent orchestrator, set it, plus
7
+ * GET / PATCH / DELETE `/api/v1/admin/trust-policy/directory` for the approval
8
+ * directory that arrives with it.
6
9
  *
7
- * The policy is Platform-owned wherever a Platform is attached: the Platform
8
- * pushes it on `trust_policy.update` and the next push would clobber a local
9
- * write. PATCH therefore refuses on any Platform-attached orchestrator, and the
10
- * refusal is server-side so it cannot be bypassed by calling the API directly.
10
+ * Both are Platform-owned wherever a Platform is attached: the Platform pushes
11
+ * them together on `trust_policy.update` and the next push would clobber a
12
+ * local write. Every write verb therefore refuses on any Platform-attached
13
+ * orchestrator, and the refusal is server-side so it cannot be bypassed by
14
+ * calling the API directly. On an independent orchestrator there is no upstream
15
+ * authority at all, so the operator is the only possible one — which is what
16
+ * the directory's PATCH and DELETE are for. Without them nobody can ever be
17
+ * registered as an approver there, and `/kici approve` can release no hold.
18
+ *
19
+ * GET stays available in every mode: reading the cache is what lets an operator
20
+ * tell a stale directory from an absent one.
11
21
  */
12
22
  import { Hono } from 'hono';
13
23
  import type { OrchestratorMode } from '@kici-dev/engine';
14
24
  import type { TrustPolicyStore } from '../security/trust-policy-store.js';
25
+ import type { TrustDirectoryStore } from '../security/trust-directory-store.js';
15
26
  import type { RbacEnforcer, Role } from '../secrets/rbac.js';
16
27
  import type { AccessLogWriter } from '../audit/access-log.js';
17
28
  /**
@@ -19,8 +30,25 @@ import type { AccessLogWriter } from '../audit/access-log.js';
19
30
  * response body — the CLI surfaces this verbatim rather than inventing its own.
20
31
  */
21
32
  export declare const PLATFORM_MANAGED_MESSAGE: string;
33
+ /**
34
+ * Wording for the directory's own Platform-managed refusal. Separate from
35
+ * {@link PLATFORM_MANAGED_MESSAGE} because it names a different thing to change
36
+ * and a different place to change it: identity links and member CI trust levels
37
+ * are org membership, not a policy switch.
38
+ */
39
+ export declare const PLATFORM_MANAGED_DIRECTORY_MESSAGE: string;
22
40
  interface TrustPolicyRouteDeps {
23
41
  store: TrustPolicyStore;
42
+ /**
43
+ * The org approval directory — the Platform's cache where one is attached,
44
+ * the operator's own registrations where none is.
45
+ *
46
+ * REQUIRED for the same reason `accessLog` is: an optional store would let an
47
+ * orchestrator mount a `directory` endpoint that reports "nothing cached" on
48
+ * every call, which reads identically to a genuinely empty cache and is
49
+ * exactly the wrong answer to give an operator debugging a refused approval.
50
+ */
51
+ directory: TrustDirectoryStore;
24
52
  rbac: RbacEnforcer;
25
53
  /** This orchestrator's mode; decides whether PATCH is permitted at all. */
26
54
  mode: OrchestratorMode;
@@ -16,6 +16,7 @@ import type { AuditLogger } from '../secrets/audit-logger.js';
16
16
  import type { AgentTokenStore } from '../agent/token-store.js';
17
17
  import type { AgentRegistry } from '../agent/registry.js';
18
18
  import type { SharedConfigStore } from '../config/shared-store.js';
19
+ import { type HeldRunReleaseWiring } from './admin-held-runs.js';
19
20
  import type { DrainController } from '../drain/drain-controller.js';
20
21
  import type { SourceStore } from '../sources/source-store.js';
21
22
  import type { JoinTokenManager } from '../cluster/join-token.js';
@@ -40,6 +41,12 @@ export interface AdminRouteDeps {
40
41
  rbac: RbacEnforcer;
41
42
  secretStore: PgSecretStore;
42
43
  auditLogger: AuditLogger;
44
+ /**
45
+ * Fleet-wide default for the global-workflows master switch, applied when the
46
+ * cluster column is NULL. Optional so WS-only / test admins can omit it;
47
+ * omitting it means the secure default (disabled).
48
+ */
49
+ globalWorkflowsEnabledDefault?: boolean;
43
50
  /**
44
51
  * Optional -- the coordinator drain controller. When provided, the
45
52
  * `POST`/`GET /api/v1/admin/orchestrator/drain` routes are mounted (backing
@@ -141,6 +148,14 @@ export interface AdminRouteDeps {
141
148
  stillPending: number;
142
149
  rejected: number;
143
150
  }>;
151
+ /**
152
+ * Optional -- the resume wiring behind `kici-admin held-run approve|reject`.
153
+ * When set (alongside `db` + `accessLog`), the local held-run read and
154
+ * decision routes are mounted; they refuse with 409 on a Platform-attached
155
+ * orchestrator. Unset leaves them unmounted, which is the honest state for an
156
+ * app that cannot dispatch a released hold.
157
+ */
158
+ heldRunRelease?: HeldRunReleaseWiring;
144
159
  }
145
160
  /** Hono env type for admin routes with context variables. */
146
161
  type AdminEnv = {
@@ -0,0 +1,55 @@
1
+ /**
2
+ * The single construction path for scaler backends.
3
+ *
4
+ * Both startup hosts (`orchestrator-core`, `worker-core`) and
5
+ * `ScalerManager.reload` call this, so a backend built by a reload is built
6
+ * exactly the way startup builds one. Host differences (DB-backed vs in-memory
7
+ * IP allocation, event support) arrive as context fields rather than forked
8
+ * code.
9
+ */
10
+ import type { createLogger, ToolRequirement } from '@kici-dev/shared';
11
+ import type { AgentTokenStore } from '../agent/token-store.js';
12
+ import type { ScalerEventEmitterLike } from './event-backend.js';
13
+ import type { ScalerStateStore } from './scaler-state-store.js';
14
+ import type { IpAllocator } from './ip-allocator.js';
15
+ import type { ScalerBackend, ScalerConfig, ScalerEntry } from './types.js';
16
+ /** Network parameters an IP allocator needs, resolved from `scalerConfig.firecracker`. */
17
+ export interface IpAllocatorParams {
18
+ cidr: string;
19
+ gateway: string;
20
+ netmask: string;
21
+ }
22
+ export interface BackendFactoryContext {
23
+ /** The whole config — supplies `defaults.resources` and the firecracker network block. */
24
+ scalerConfig: ScalerConfig;
25
+ /**
26
+ * Always supplied when available; the event backend's claim store needs it
27
+ * regardless of the auth mode.
28
+ */
29
+ tokenStore?: AgentTokenStore;
30
+ /** True when `config.agentAuth === 'token'`; gates token injection for the three local backends. */
31
+ injectAgentToken: boolean;
32
+ tokenTtlMs?: number;
33
+ /** Live per-spawn resolver for the fleet-wide agent-token TTL. */
34
+ tokenTtlProvider: () => Promise<number>;
35
+ /** DB-backed on the leader, in-memory on the worker. */
36
+ ipAllocator: (params: IpAllocatorParams) => IpAllocator;
37
+ /**
38
+ * The already-constructed scaler state store. The event backend's claim
39
+ * store writes its pending-claim rows through it. Absent on the worker,
40
+ * which has no database — and so no event backend either.
41
+ */
42
+ stateStore?: ScalerStateStore;
43
+ /** Absent ⇒ the `event` type is unsupported on this host (worker mode). */
44
+ eventEmitterProvider?: () => ScalerEventEmitterLike;
45
+ logger: ReturnType<typeof createLogger>;
46
+ }
47
+ /** Tool requirements for a set of entries, for `validateRequiredTools` from `@kici-dev/shared`. */
48
+ export declare function requiredToolsFor(entries: ScalerEntry[]): ToolRequirement[];
49
+ /**
50
+ * Construct one backend. Returns null (with a warning) when the type is not
51
+ * supported on this host — the worker has no event emitter, and `kubernetes`
52
+ * is not implemented.
53
+ */
54
+ export declare function createScalerBackend(s: ScalerEntry, ctx: BackendFactoryContext): Promise<ScalerBackend | null>;
55
+ //# sourceMappingURL=backend-factory.d.ts.map
@@ -7,7 +7,7 @@
7
7
  */
8
8
  import { type ToolRequirement } from '@kici-dev/shared';
9
9
  import type { AgentTokenStore } from '../agent/token-store.js';
10
- import type { ScalerBackend, ScalerEntry, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
10
+ import type { ScalerBackend, ScalerDestroyContext, ScalerEntry, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
11
11
  export interface BareMetalScalerBackendOptions {
12
12
  /** Human-readable name for this scaler */
13
13
  name: string;
@@ -47,7 +47,7 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
47
47
  readonly type: "bare-metal";
48
48
  readonly spawnsOnLocalHost = true;
49
49
  readonly logsSource = "bare-metal";
50
- readonly maxAgents: number;
50
+ maxAgents: number;
51
51
  private _labelSets;
52
52
  private readonly name;
53
53
  private readonly defaultResources?;
@@ -79,15 +79,53 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
79
79
  static getRequiredTools(entry: ScalerEntry): ToolRequirement[];
80
80
  get labelSets(): LabelSetConfig[];
81
81
  getActiveCount(): number;
82
- spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, _spawnContext?: SpawnContext, _signal?: AbortSignal): Promise<ManagedAgent>;
82
+ spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
83
+ /**
84
+ * Launch the agent inside the job's own container image.
85
+ *
86
+ * The bare-metal backend historically only ran a local process, which meant a
87
+ * job naming its own image could not use this pool at all. With the KiCI
88
+ * runtime injected, the image needs neither Node nor git, so an arbitrary
89
+ * customer image can host the agent.
90
+ *
91
+ * Split out of `spawn()` because that function is already near the 200-line
92
+ * ceiling and the two launches share nothing past the env.
93
+ */
94
+ private spawnContainerAgent;
83
95
  getScalerContext(agentId: string): Record<string, unknown> | undefined;
84
- destroy(managedId: string): Promise<void>;
96
+ destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
97
+ /**
98
+ * Force-reclaim a container-mode agent this backend no longer tracks.
99
+ *
100
+ * `destroy()` opens with an in-memory map lookup, so an orchestrator restart
101
+ * makes it a silent no-op while the container keeps running. The container
102
+ * backend's own startup sweep does not cover this one either: it removes every
103
+ * `kici-managed` container, but it only runs when a container scaler is
104
+ * configured, and a bare-metal-only deployment has none.
105
+ *
106
+ * The host-local evidence is the container itself: `spawnContainerAgent` stamps
107
+ * `kici-agent-id` and `kici-scaler-name` onto it, and a container listing only
108
+ * ever names containers on this host. So a coordinator cannot reach a peer's
109
+ * compute through this path — a wrongly-routed agent id simply matches nothing.
110
+ *
111
+ * A process-mode agent is NOT reclaimed here. Its PID lives in the in-memory
112
+ * entry alone, so a restart leaves no durable, host-local artifact to key off,
113
+ * and reclaiming it would need a persistence mechanism this backend does not
114
+ * have. Scanning the process table instead would risk signalling a recycled
115
+ * PID.
116
+ *
117
+ * @returns `true` when a container for `managedId` was found and removed.
118
+ */
119
+ reapUnowned(managedId: string): Promise<boolean>;
85
120
  /**
86
121
  * Get the LogCapture for a managed agent (used by ScalerManager for log forwarding).
87
122
  */
88
123
  getLogCapture(managedId: string): LogCapture | undefined;
89
124
  shutdownAll(): Promise<void>;
90
- reload(labelSets: LabelSetConfig[]): ValidationResult;
125
+ reload(labelSets: LabelSetConfig[], opts?: {
126
+ maxAgents?: number;
127
+ entry?: ScalerEntry;
128
+ }): ValidationResult;
91
129
  /**
92
130
  * Build the (command, args) tuple used to spawn an agent process.
93
131
  *
@@ -0,0 +1,119 @@
1
+ /**
2
+ * Pending-claim store for the event scaler backend.
3
+ *
4
+ * When the event backend spawns, it registers a pending claim and emits a
5
+ * scale-up event carrying a single-use `claimCode` (never the token itself).
6
+ * The provisioning workflow forwards that code to the instance it boots, and the
7
+ * agent exchanges it — over the `scaler.claim-credentials` RPC — for a freshly
8
+ * minted ephemeral agent token it then registers with. The claim code is the
9
+ * authorization for that RPC, so the agent can redeem it before it authenticates
10
+ * or registers; a workflow can also redeem the code itself and deliver the token.
11
+ * The token is therefore minted lazily (only when a real provision claims it) and
12
+ * is single-use: a leaked or replayed claim code mints nothing.
13
+ *
14
+ * The claim is persisted in `scaler_pending_claims` rather than held in a
15
+ * per-process map, so a code minted by one coordinator can be redeemed by any
16
+ * other coordinator behind the same shared endpoint. Only the sha256 of the
17
+ * code is stored, so a DB read can never hand back a redeemable secret.
18
+ */
19
+ import type { ScalerStateStore } from './scaler-state-store.js';
20
+ /**
21
+ * TTL (seconds) a pending claim code stays redeemable when the scaler entry
22
+ * does not set `claimTtlSeconds`. Long enough for a provisioning workflow to
23
+ * boot an instance, short enough that a leaked code ages out quickly.
24
+ */
25
+ export declare const DEFAULT_CLAIM_TTL_SECONDS = 300;
26
+ /**
27
+ * Function that mints an ephemeral agent token bound to an agent id + labels.
28
+ * Modelled on `AgentTokenStore.createEphemeral(agentId, labels, ttlMs)`.
29
+ */
30
+ export type CreateEphemeralToken = (agentId: string, labels: string[], ttlMs: number) => Promise<string>;
31
+ /** What the event backend registers when it emits a scale-up. */
32
+ export interface ClaimSpec {
33
+ /** Agent id the provisioned instance must register with. */
34
+ agentId: string;
35
+ /** Exact label set the ephemeral token is authorized for. */
36
+ labels: string[];
37
+ /** Mandatory (taint) labels the pool gates on, carried for the workflow. */
38
+ mandatoryLabels: string[];
39
+ /** TTL of the ephemeral agent token minted on claim (seconds). */
40
+ agentTokenTtlSeconds: number;
41
+ /** Orchestrator WS URL the provisioned agent connects back to. */
42
+ orchestratorUrl: string;
43
+ }
44
+ /** What a successful claim returns to the provisioning workflow. */
45
+ export interface ClaimedCredentials {
46
+ /** The freshly minted, single-use-per-claim ephemeral agent token. */
47
+ agentToken: string;
48
+ /** Agent id the instance must register with. */
49
+ agentId: string;
50
+ /** Orchestrator WS URL to connect back to. */
51
+ orchestratorUrl: string;
52
+ /** Labels the token authorizes. */
53
+ labels: string[];
54
+ }
55
+ export interface ClaimStoreOptions {
56
+ /** Mints the ephemeral agent token when a claim is honored. */
57
+ createEphemeral: CreateEphemeralToken;
58
+ /** Shared persistence, so any coordinator can redeem a code. */
59
+ stateStore: ScalerStateStore;
60
+ /**
61
+ * Scaler this store belongs to; recorded on each claim row. Required for
62
+ * `register`, and omitted on a redemption-only store (the manager's), which
63
+ * never writes a row.
64
+ */
65
+ scalerName?: string;
66
+ /** Injected clock (epoch ms) — no ambient `Date.now()` so TTLs are testable. */
67
+ now?: () => number;
68
+ /** Default TTL of a pending claim code (seconds) before it expires. */
69
+ ttlDefaultSec: number;
70
+ }
71
+ /**
72
+ * Shared store of pending provisioning claims, backed by `scaler_pending_claims`.
73
+ */
74
+ export declare class ClaimStore {
75
+ private readonly createEphemeral;
76
+ private readonly stateStore;
77
+ private readonly scalerName;
78
+ private readonly now;
79
+ private ttlDefaultMs;
80
+ constructor(opts: ClaimStoreOptions);
81
+ /**
82
+ * Replace the default pending-claim TTL. Called when a config reload changes
83
+ * the owning scaler's `claimTtlSeconds`, so the new value applies without an
84
+ * orchestrator restart.
85
+ */
86
+ setDefaultTtlSeconds(seconds: number): void;
87
+ /**
88
+ * Register a pending claim and return its single-use code. The code is a
89
+ * 256-bit cryptographically-random token — unguessable, so it can travel in
90
+ * the (persisted) scale-up event without leaking the agent token. Only the
91
+ * sha256 of the code is persisted, so a DB read can never hand back a
92
+ * redeemable secret.
93
+ */
94
+ register(spec: ClaimSpec): Promise<string>;
95
+ /**
96
+ * Exchange a claim code for freshly minted credentials. Single-use and
97
+ * TTL-bounded, enforced by a conditional UPDATE that commits the consumption
98
+ * BEFORE the mint is attempted — so two concurrent claims of one code, on any
99
+ * two instances, can never both mint. Fail-closed: a failed mint does not
100
+ * reopen the code.
101
+ *
102
+ * The redeem is one round trip on the happy path; `describeClaim` runs only
103
+ * when the redeem found nothing, to say why (unknown / consumed / expired).
104
+ */
105
+ claim(code: string): Promise<ClaimedCredentials>;
106
+ /**
107
+ * Drop every claim for an agent id, so a torn-down provision's code can no
108
+ * longer be redeemed on any instance.
109
+ */
110
+ invalidate(agentId: string): Promise<void>;
111
+ /**
112
+ * A redemption-only store never writes a claim row, so it is constructed
113
+ * without a scaler name. Reaching `register` without one is a wiring bug, not
114
+ * a runtime condition — fail loudly rather than persisting a row nothing can
115
+ * attribute.
116
+ */
117
+ private requireScalerName;
118
+ }
119
+ //# sourceMappingURL=claim-store.d.ts.map
@@ -170,6 +170,7 @@ export declare const scalerFileSchema: z.ZodObject<{
170
170
  type: z.ZodEnum<{
171
171
  "bare-metal": "bare-metal";
172
172
  container: "container";
173
+ event: "event";
173
174
  firecracker: "firecracker";
174
175
  }>;
175
176
  maxAgents: z.ZodNumber;
@@ -287,6 +288,9 @@ export declare const scalerFileSchema: z.ZodObject<{
287
288
  vcpuCount: z.ZodDefault<z.ZodNumber>;
288
289
  memSizeMib: z.ZodDefault<z.ZodNumber>;
289
290
  requireSudo: z.ZodDefault<z.ZodBoolean>;
291
+ provisioningTargets: z.ZodOptional<z.ZodArray<z.ZodString>>;
292
+ claimTtlSeconds: z.ZodDefault<z.ZodNumber>;
293
+ agentTokenTtlSeconds: z.ZodDefault<z.ZodNumber>;
290
294
  }, z.core.$strict>>;
291
295
  firecracker: z.ZodOptional<z.ZodObject<{
292
296
  cidr: z.ZodDefault<z.ZodString>;
@@ -8,7 +8,7 @@
8
8
  */
9
9
  import { type ToolRequirement } from '@kici-dev/shared';
10
10
  import type { AgentTokenStore } from '../agent/token-store.js';
11
- import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
11
+ import type { ScalerBackend, ScalerDestroyContext, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
12
12
  /**
13
13
  * Result of runtime detection.
14
14
  */
@@ -56,7 +56,7 @@ export interface ContainerScalerBackendOptions {
56
56
  export declare class ContainerScalerBackend implements ScalerBackend {
57
57
  readonly type: "container";
58
58
  readonly spawnsOnLocalHost: boolean;
59
- readonly maxAgents: number;
59
+ maxAgents: number;
60
60
  private _labelSets;
61
61
  private readonly name;
62
62
  private readonly docker;
@@ -121,15 +121,29 @@ export declare class ContainerScalerBackend implements ScalerBackend {
121
121
  get logsSource(): string;
122
122
  get labelSets(): LabelSetConfig[];
123
123
  getActiveCount(): number;
124
+ /**
125
+ * The full label set the agent will present, and the ephemeral token bound to
126
+ * exactly that set.
127
+ *
128
+ * The binding is the point: the agent's register-time labels must not trip the
129
+ * scope gate, and the only labels it may add on top are the self-reported
130
+ * os/arch/host facts the gate exempts. The pool's platform taints are NOT such
131
+ * a fact — they are a routing grant, so the manager asserts them here via
132
+ * `spawnContext` rather than letting the agent claim them.
133
+ */
134
+ private mintAgentIdentity;
124
135
  spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
125
136
  getScalerContext(agentId: string): Record<string, unknown> | undefined;
126
- destroy(managedId: string): Promise<void>;
137
+ destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
127
138
  /**
128
139
  * Get the LogCapture for a managed agent (used by ScalerManager for log forwarding).
129
140
  */
130
141
  getLogCapture(managedId: string): LogCapture | undefined;
131
142
  shutdownAll(): Promise<void>;
132
- reload(labelSets: LabelSetConfig[]): ValidationResult;
143
+ reload(labelSets: LabelSetConfig[], opts?: {
144
+ maxAgents?: number;
145
+ entry?: ScalerEntry;
146
+ }): ValidationResult;
133
147
  /**
134
148
  * Clean up orphaned kici-managed containers on startup.
135
149
  * Finds containers with the `kici-managed=true` label and removes them.
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Capability label for hosts that can run containers.
3
+ *
4
+ * An operator may put this on a pool's label sets and on a container job's
5
+ * `runsOn` to steer container work at hosts with a runtime. KiCI does NOT add
6
+ * it automatically, and does not gate routing on it.
7
+ *
8
+ * **Why it is not automatic.** A container job needs a runtime on the host that
9
+ * ends up running it, and the orchestrator does not know that: an agent runs on
10
+ * its own machine, and whether that machine has docker or podman is the agent's
11
+ * fact, not the orchestrator's. Probing the orchestrator's own filesystem
12
+ * answers a different question — it was tried, and it stranded container jobs
13
+ * that had been running fine, because the probe and the job ran in different
14
+ * places.
15
+ *
16
+ * Doing this properly means the AGENT reporting the capability at registration,
17
+ * alongside the `kici:os:*` / `kici:arch:*` facts it already self-reports. Until
18
+ * then a mis-routed container job fails at the image preflight or a backend's
19
+ * fail-fast, both of which name what is missing — a clear late error, rather
20
+ * than a job that silently matches nothing and never runs.
21
+ */
22
+ export declare const KICI_RUNTIME_DOCKER_LABEL = "kici:runtime:docker";
23
+ //# sourceMappingURL=container-routing.d.ts.map
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Container-spawn primitives specific to the orchestrator's scaler backends.
3
+ *
4
+ * The runtime-injection half of this module (`pullImageIfMissing`,
5
+ * `ensureRuntimeVolume`, `runtimeInjectBind`, `injectedAgentCommand`,
6
+ * `runtimeVolumeName`, `RUNTIME_MOUNT`) now lives in
7
+ * `@kici-dev/shared/container-runtime`, because the AGENT needs exactly the
8
+ * same behaviour when it nests a job container. What stays here is the piece
9
+ * only a scaler backend does: streaming a host workspace into a container it
10
+ * just created.
11
+ */
12
+ /**
13
+ * Stream a host directory into a container path.
14
+ *
15
+ * Used to populate a container-owned `/workspace` volume, which is a volume
16
+ * rather than a host bind precisely so the host-uid vs container-uid conflict
17
+ * disappears once `CapDrop: ALL` removes CAP_DAC_OVERRIDE — so the tree has to
18
+ * be copied in rather than mounted.
19
+ *
20
+ * A failure is fatal to the spawn: swallowing it starts the job against an
21
+ * empty workspace, which surfaces as a baffling "file not found" later.
22
+ */
23
+ export declare function copyTreeIntoContainer(container: {
24
+ putArchive(stream: unknown, opts: {
25
+ path: string;
26
+ }): Promise<unknown>;
27
+ }, hostDir: string, containerPath: string): Promise<void>;
28
+ //# sourceMappingURL=container-spawn.d.ts.map