@kici-dev/orchestrator 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/__test-helpers__/mock-db.d.ts +2 -0
- package/dist/agent/dispatcher.d.ts +110 -6
- package/dist/agent/registry.d.ts +14 -0
- package/dist/app.d.ts +36 -0
- package/dist/approvals/apply-decision.d.ts +60 -0
- package/dist/approvals/approval-resolver.d.ts +66 -0
- package/dist/approvals/step-approval-bridge.d.ts +72 -0
- package/dist/approvals/team-membership-lookup.d.ts +13 -0
- package/dist/cache/agent-job-failed-error.d.ts +13 -0
- package/dist/cache/dispatch-cache-ref-tracker.d.ts +45 -0
- package/dist/cache/index.d.ts +2 -0
- package/dist/cache/user-cache.d.ts +116 -0
- package/dist/cancel/cancel-run.d.ts +56 -0
- package/dist/cli/api-client.d.ts +24 -0
- package/dist/cli/commands/environment.d.ts +1 -0
- package/dist/cli/commands/firecracker/index.d.ts +11 -0
- package/dist/cli/commands/firecracker/provision.d.ts +13 -0
- package/dist/cli/commands/firecracker/teardown.d.ts +3 -0
- package/dist/cli/commands/firecracker/verify.d.ts +3 -0
- package/dist/cli/commands/scaler.d.ts +18 -0
- package/dist/cli/kici-admin.d.ts +10 -1
- package/dist/cli/service/image-digests.d.ts +21 -0
- package/dist/cli.js +4148 -455
- package/dist/cluster/peer-client.d.ts +23 -1
- package/dist/cluster/peer-handler.d.ts +9 -1
- package/dist/cluster/peer-registry.d.ts +6 -0
- package/dist/config/schema.d.ts +4 -0
- package/dist/config.d.ts +13 -0
- package/dist/dashboard/handler.d.ts +92 -1
- package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts +11 -0
- package/dist/db/migrations/027_workflow_timeout.d.ts +20 -0
- package/dist/db/migrations/028_org_settings_user_cache.d.ts +4 -0
- package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +16 -0
- package/dist/db/migrations/030_held_runs_env_set_null.d.ts +13 -0
- package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +19 -0
- package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts +14 -0
- package/dist/db/migrations/033_org_settings_approval.d.ts +19 -0
- package/dist/db/migrations/034_held_runs_generalize.d.ts +24 -0
- package/dist/db/types.d.ts +89 -3
- package/dist/diagnostics/bundle-writer.d.ts +1 -17
- package/dist/diagnostics/checks/firecracker-network.d.ts +13 -0
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/fleet-collector.d.ts +52 -0
- package/dist/diagnostics/fleet-constants.d.ts +8 -0
- package/dist/diagnostics/fleet-selection.d.ts +15 -0
- package/dist/diagnostics/fleet-topology.d.ts +47 -0
- package/dist/diagnostics/fleet-wiring.d.ts +60 -0
- package/dist/environments/environment-store.d.ts +14 -1
- package/dist/environments/held-runs.d.ts +69 -1
- package/dist/firecracker/host-network.d.ts +83 -0
- package/dist/firecracker/persist.d.ts +14 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +380 -40
- package/dist/lockfile-cache.d.ts +1 -1
- package/dist/metrics/prometheus.d.ts +8 -0
- package/dist/orchestrator-core.d.ts +13 -2
- package/dist/pipeline/dispatch-matched-workflow.d.ts +9 -0
- package/dist/pipeline/inline-eval.d.ts +17 -2
- package/dist/pipeline/process-webhook.d.ts +19 -0
- package/dist/pipeline/processor.d.ts +28 -1
- package/dist/pipeline/test-pipeline.d.ts +10 -0
- package/dist/providers/github/lock-file.d.ts +1 -1
- package/dist/providers/internal/lock-file-fetcher.d.ts +3 -2
- package/dist/queue/job-queue.d.ts +53 -1
- package/dist/reporting/execution-tracker.d.ts +3 -1
- package/dist/routes/admin-environments.d.ts +1 -0
- package/dist/routes/fleet.d.ts +20 -0
- package/dist/scaler/bare-metal-backend.d.ts +1 -0
- package/dist/scaler/config.d.ts +4 -2
- package/dist/scaler/container-backend.d.ts +3 -2
- package/dist/scaler/disk-guard.d.ts +27 -0
- package/dist/scaler/firecracker-backend.d.ts +35 -0
- package/dist/scaler/manager.d.ts +8 -0
- package/dist/scaler/nftables.d.ts +29 -3
- package/dist/scaler/reap-orphans.d.ts +27 -0
- package/dist/scaler/types.d.ts +28 -1
- package/dist/server.js +6138 -1587
- package/dist/stale-detector/stale-run-detector.d.ts +18 -0
- package/dist/stale-detector/workflow-deadline-detector.d.ts +49 -0
- package/dist/standalone.js +18499 -14925
- package/dist/storage/filesystem.d.ts +14 -4
- package/dist/storage/s3.d.ts +19 -5
- package/dist/storage/types.d.ts +32 -5
- package/dist/worker/in-memory-job-queue.d.ts +40 -7
- package/dist/ws/agent-handler.d.ts +45 -0
- package/dist/ws/dashboard-env-handler.d.ts +36 -0
- package/dist/ws/fleet-agent-collector.d.ts +23 -0
- package/dist/ws/platform-client.d.ts +10 -1
- package/installer-image-digests.json +7 -0
- package/package.json +15 -11
- package/sbom.spdx.json +62 -57
package/README.md
CHANGED
|
@@ -1 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
# @kici-dev/orchestrator
|
|
2
|
+
|
|
3
|
+
Customer-deployable orchestrator for the KiCI CI/CD stack. Receives webhook events (directly or via the hosted relay), matches triggers against the lock file, and dispatches jobs to connected agents — including auto-scaled ephemeral agents (containers, bare-metal, micro-VMs).
|
|
4
|
+
|
|
5
|
+
You normally don't install this package directly: deploy the orchestrator with the [`kici-admin`](https://www.npmjs.com/package/kici-admin) CLI (`kici-admin orchestrator install`) or run the published container image `quay.io/kici-dev/kici-orchestrator`.
|
|
6
|
+
|
|
7
|
+
Part of [KiCI](https://kici.dev) — CI/CD workflows as TypeScript code: author them with full language power, dry-run them locally, and run them on your own infrastructure.
|
|
8
|
+
|
|
9
|
+
## Links
|
|
10
|
+
|
|
11
|
+
- Documentation: <https://docs.kici.dev/operator/orchestrator/getting-started/>
|
|
12
|
+
- Source: <https://github.com/kici-dev/kici-public/tree/main/packages/orchestrator>
|
|
13
|
+
- License: AGPL-3.0-only
|
|
@@ -28,6 +28,8 @@ export interface MockDbOptions {
|
|
|
28
28
|
insertReturning?: unknown;
|
|
29
29
|
/** Row returned by update chains ending in .returningAll().executeTakeFirstOrThrow() or executeTakeFirst(). Default: undefined */
|
|
30
30
|
updatedRow?: unknown;
|
|
31
|
+
/** Row returned by update chains ending in .returning(...).executeTakeFirst(). Default: undefined */
|
|
32
|
+
updateReturning?: unknown;
|
|
31
33
|
/** Result for update .execute() calls. Default: { numUpdatedRows: 0n } */
|
|
32
34
|
updateResult?: {
|
|
33
35
|
numUpdatedRows: bigint;
|
|
@@ -71,10 +71,40 @@ export declare class Dispatcher {
|
|
|
71
71
|
private graceCleanupTimer;
|
|
72
72
|
/** Max reconnection delay from agent config, used to derive grace period. */
|
|
73
73
|
private readonly maxReconnectDelayMs;
|
|
74
|
-
/**
|
|
75
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Callback fired when a job reaches a permanent dispatch-side failure:
|
|
76
|
+
* recovery window expired, re-dispatch attempts exhausted, or a
|
|
77
|
+
* scaler-managed agent was destroyed mid-execution. The receiver marks
|
|
78
|
+
* the execution-tracker job failed with the given reason.
|
|
79
|
+
*/
|
|
80
|
+
private readonly onJobFailedPermanently?;
|
|
76
81
|
/** Callback when a job enters recovery (starts timer). */
|
|
77
82
|
private readonly onRecoveryStarted?;
|
|
83
|
+
/** Fallback ack deadline when no getAckTimeoutMs dep is wired (tests). */
|
|
84
|
+
private static readonly DEFAULT_ACK_TIMEOUT_MS;
|
|
85
|
+
/**
|
|
86
|
+
* Pending dispatch acknowledgments: armed when a job.dispatch is sent,
|
|
87
|
+
* resolved by job.ack / job.reject / job.status running / disconnect
|
|
88
|
+
* triage. On expiry the dispatch is treated as lost: requeue + disconnect.
|
|
89
|
+
*/
|
|
90
|
+
private readonly pendingAcks;
|
|
91
|
+
/**
|
|
92
|
+
* Acks that arrived before their pending-ack entry was armed (the agent
|
|
93
|
+
* answered faster than the orchestrator could resolve the timeout + stamp
|
|
94
|
+
* the deadline). Keyed by jobId -> the agent that acked. Consumed by
|
|
95
|
+
* `armAckDeadline`, which then skips arming.
|
|
96
|
+
*/
|
|
97
|
+
private readonly earlyAcks;
|
|
98
|
+
/** Resolve the per-job ack deadline (per-org override / cluster default). */
|
|
99
|
+
private readonly getAckTimeoutMs;
|
|
100
|
+
/** Cancel + disconnect an agent whose dispatch ack deadline expired. */
|
|
101
|
+
private readonly onAckTimeout?;
|
|
102
|
+
/**
|
|
103
|
+
* Jobs that have reached agent-side execution (a `job.status: running`
|
|
104
|
+
* arrived). Disconnect triage treats never-started jobs as safely
|
|
105
|
+
* re-dispatchable; started jobs on destroyed agents fail fast.
|
|
106
|
+
*/
|
|
107
|
+
private readonly startedJobs;
|
|
78
108
|
/**
|
|
79
109
|
* Per-job recovery timers for jobs in 'recovering' state.
|
|
80
110
|
* When an agent disconnects with in-flight jobs, each job gets a timer.
|
|
@@ -94,10 +124,14 @@ export declare class Dispatcher {
|
|
|
94
124
|
onNoMatchingAgent?: (labels: string[], jobId: string, runId: string, excludeLabels: string[], resources?: ResourceRequest) => Promise<ScaleResult>;
|
|
95
125
|
/** Max reconnection delay from agent config (default 60s). Used to derive grace period. */
|
|
96
126
|
maxReconnectDelayMs?: number;
|
|
97
|
-
/** Callback when a
|
|
98
|
-
|
|
127
|
+
/** Callback fired when a job is permanently failed before/outside agent execution. */
|
|
128
|
+
onJobFailedPermanently?: (agentId: string, jobId: string, runId: string, reason: string) => void;
|
|
99
129
|
/** Callback when a job enters recovery (starts timer). */
|
|
100
130
|
onRecoveryStarted?: (agentId: string, jobId: string) => void;
|
|
131
|
+
/** Resolve the per-job dispatch-ack deadline (ms). Defaults to 10s. */
|
|
132
|
+
getAckTimeoutMs?: (job: QueuedJob) => Promise<number>;
|
|
133
|
+
/** Cancel + disconnect an agent whose dispatch ack deadline expired. */
|
|
134
|
+
onAckTimeout?: (agentId: string, jobId: string, runId: string) => void;
|
|
101
135
|
});
|
|
102
136
|
/**
|
|
103
137
|
* Dispatch a job to a matching agent, or queue it if none available.
|
|
@@ -132,6 +166,54 @@ export declare class Dispatcher {
|
|
|
132
166
|
* calling onDispatch for each.
|
|
133
167
|
*/
|
|
134
168
|
onAgentAvailable(agentId: string): Promise<void>;
|
|
169
|
+
/** Record that a job began executing on its agent. */
|
|
170
|
+
markJobStarted(jobId: string): void;
|
|
171
|
+
/**
|
|
172
|
+
* Arm the ack deadline for a just-sent dispatch (timer + persisted row).
|
|
173
|
+
*
|
|
174
|
+
* Resolving the timeout and persisting the deadline both touch the DB, so
|
|
175
|
+
* a fast agent's job.ack can arrive before this method registers its
|
|
176
|
+
* in-memory entry. `earlyAcks` records such an ack so this method skips
|
|
177
|
+
* arming entirely instead of starting a timer that has nothing to resolve
|
|
178
|
+
* it (which would expire a dispatch the agent actually accepted).
|
|
179
|
+
*/
|
|
180
|
+
private armAckDeadline;
|
|
181
|
+
/**
|
|
182
|
+
* Consume a recorded early ack for `jobId` from `agentId`. Returns true if
|
|
183
|
+
* one was pending (and removes it).
|
|
184
|
+
*/
|
|
185
|
+
private consumeEarlyAck;
|
|
186
|
+
/** Clear a pending ack (answered or otherwise settled). Idempotent. */
|
|
187
|
+
private resolvePendingAck;
|
|
188
|
+
/** Handle an explicit dispatch acknowledgment (`job.ack`) from an agent. */
|
|
189
|
+
onJobAcked(agentId: string, jobId: string): void;
|
|
190
|
+
/**
|
|
191
|
+
* The ack deadline expired: the dispatch is lost. Untrack the job, then
|
|
192
|
+
* remove the unresponsive agent from the registry BEFORE requeueing, so the
|
|
193
|
+
* redispatch cannot land the job back on the very agent we are tearing down.
|
|
194
|
+
* The job is already untracked when the WS close fires, so the disconnect
|
|
195
|
+
* triage finds nothing to double-handle.
|
|
196
|
+
*/
|
|
197
|
+
private handleAckExpiry;
|
|
198
|
+
/**
|
|
199
|
+
* Handle an explicit dispatch rejection (`job.reject`) from an agent.
|
|
200
|
+
* Undoes the dispatch accounting and requeues the job for another agent.
|
|
201
|
+
*/
|
|
202
|
+
onJobRejected(agentId: string, jobId: string, reason: string): Promise<void>;
|
|
203
|
+
/** Remove a job from all per-agent in-memory tracking. */
|
|
204
|
+
private untrackJob;
|
|
205
|
+
/**
|
|
206
|
+
* Requeue a dispatched job for re-delivery, or fail it permanently when
|
|
207
|
+
* its attempt budget is exhausted. Returns the outcome so disconnect
|
|
208
|
+
* triage can surface failed job IDs to the caller.
|
|
209
|
+
*/
|
|
210
|
+
private requeueOrFail;
|
|
211
|
+
/**
|
|
212
|
+
* Try to deliver a requeued pending job: prefer an idle matching agent
|
|
213
|
+
* (atomic claim via dispatchBoundJob), otherwise consult the scaler so a
|
|
214
|
+
* fresh agent is spawned bound to this job.
|
|
215
|
+
*/
|
|
216
|
+
private redispatch;
|
|
135
217
|
/**
|
|
136
218
|
* Handle agent disconnect. Starts per-job recovery timers instead of
|
|
137
219
|
* immediately failing jobs, giving the agent a grace period to reconnect.
|
|
@@ -141,6 +223,19 @@ export declare class Dispatcher {
|
|
|
141
223
|
* asynchronously when recovery timers expire).
|
|
142
224
|
*/
|
|
143
225
|
onAgentDisconnect(agentId: string): Promise<string[]>;
|
|
226
|
+
/** Drop grace-window entries owned by an agent. */
|
|
227
|
+
private cleanupGraceEntriesForAgent;
|
|
228
|
+
/**
|
|
229
|
+
* Scaler-managed agent disconnected: requeue never-started jobs, fail
|
|
230
|
+
* started ones. Returns the permanently failed job IDs.
|
|
231
|
+
*/
|
|
232
|
+
private triageScalerManagedDisconnect;
|
|
233
|
+
/**
|
|
234
|
+
* Static agent disconnected: start per-job recovery timers (the agent
|
|
235
|
+
* may reconnect and reclaim). Always returns [] — failures happen
|
|
236
|
+
* asynchronously when recovery timers expire.
|
|
237
|
+
*/
|
|
238
|
+
private startRecoveryForDisconnect;
|
|
144
239
|
/**
|
|
145
240
|
* Called when an agent completes a job (success or failure).
|
|
146
241
|
* Decrements active jobs and moves the job to the grace window
|
|
@@ -197,7 +292,9 @@ export declare class Dispatcher {
|
|
|
197
292
|
getRecoveringJobsForAgent(agentId: string): string[];
|
|
198
293
|
/**
|
|
199
294
|
* Restore a reconciled job into the dispatcher's in-memory tracking.
|
|
200
|
-
* Called when an agent reconnects and claims a recovering job.
|
|
295
|
+
* Called when an agent reconnects and claims a recovering job. A
|
|
296
|
+
* reconnecting agent reporting the job in-flight means it had started,
|
|
297
|
+
* so mark it started for disconnect triage.
|
|
201
298
|
*/
|
|
202
299
|
restoreJobForAgent(agentId: string, jobId: string): void;
|
|
203
300
|
/**
|
|
@@ -221,10 +318,17 @@ export declare class Dispatcher {
|
|
|
221
318
|
* fast-failed by the sweep on its first run.
|
|
222
319
|
*/
|
|
223
320
|
recoverState(): Promise<void>;
|
|
321
|
+
/**
|
|
322
|
+
* Leader-gated sweep: requeue every `dispatched` row whose ack deadline
|
|
323
|
+
* passed while no coord was watching (owning coord crashed before its
|
|
324
|
+
* in-memory timer fired). Requeue is atomic (WHERE status='dispatched'),
|
|
325
|
+
* so racing coords cannot double-requeue.
|
|
326
|
+
*/
|
|
327
|
+
sweepExpiredAckDeadlines(): Promise<number>;
|
|
224
328
|
/**
|
|
225
329
|
* Leader-gated sweep: mark every `recovering` row whose
|
|
226
330
|
* `recovery_deadline < now` as `failed` and fire the per-job
|
|
227
|
-
* `
|
|
331
|
+
* `onJobFailedPermanently` hook. Intended to run on the Raft leader at a
|
|
228
332
|
* fixed interval so jobs whose owning coord crashed mid-recovery
|
|
229
333
|
* still reach a terminal state.
|
|
230
334
|
*
|
package/dist/agent/registry.d.ts
CHANGED
|
@@ -36,6 +36,14 @@ export interface AgentEntry {
|
|
|
36
36
|
* agent's labels but does not satisfy the gate).
|
|
37
37
|
*/
|
|
38
38
|
mandatoryLabels: Set<string>;
|
|
39
|
+
/**
|
|
40
|
+
* True when this agent was spawned by a scaler backend. Scaler-managed
|
|
41
|
+
* agents are single-use: the scaler destroys them on disconnect, so they
|
|
42
|
+
* can never reconnect to reclaim recovering jobs. The dispatcher uses
|
|
43
|
+
* this to triage disconnects (requeue / fail-fast) instead of starting a
|
|
44
|
+
* recovery window that cannot succeed.
|
|
45
|
+
*/
|
|
46
|
+
scalerManaged: boolean;
|
|
39
47
|
/** Number of jobs currently executing on this agent. */
|
|
40
48
|
activeJobs: number;
|
|
41
49
|
/** Maximum concurrent jobs this agent can handle (default 1). */
|
|
@@ -92,6 +100,12 @@ interface AgentMetadata {
|
|
|
92
100
|
* `onAgentRegistered(...)` return value.
|
|
93
101
|
*/
|
|
94
102
|
mandatoryLabels?: string[];
|
|
103
|
+
/**
|
|
104
|
+
* True when a scaler backend spawned this agent (single-use; destroyed
|
|
105
|
+
* on disconnect). Set by `agent-handler.ts` from the presence of the
|
|
106
|
+
* scaler registration metadata.
|
|
107
|
+
*/
|
|
108
|
+
scalerManaged?: boolean;
|
|
95
109
|
}
|
|
96
110
|
export declare class AgentRegistry {
|
|
97
111
|
/** Primary: agentId -> AgentEntry */
|
package/dist/app.d.ts
CHANGED
|
@@ -30,6 +30,8 @@ import type { ProviderRegistry } from './provider-registry.js';
|
|
|
30
30
|
import type { SourceCache } from './cache/index.js';
|
|
31
31
|
import type { BuildCoordinator } from './cache/index.js';
|
|
32
32
|
import type { DepCache } from './cache/index.js';
|
|
33
|
+
import type { UserCache } from './cache/index.js';
|
|
34
|
+
import type { DispatchCacheRefTracker } from './cache/index.js';
|
|
33
35
|
import type { PendingBuildTracker } from './cache/index.js';
|
|
34
36
|
import type { PendingInitTracker } from './cache/index.js';
|
|
35
37
|
import type { PendingDynamicTracker } from './cache/index.js';
|
|
@@ -46,6 +48,8 @@ import type { PeerToPeerMessage } from '@kici-dev/engine';
|
|
|
46
48
|
import type { AgentTokenStore } from './agent/token-store.js';
|
|
47
49
|
import type { OwnershipTracker } from './agent/ownership-tracker.js';
|
|
48
50
|
import type { ObserverRegistry } from './ws/observer-registry.js';
|
|
51
|
+
import type { FleetAgentCollector } from './ws/fleet-agent-collector.js';
|
|
52
|
+
import type { FleetTopology } from './diagnostics/fleet-topology.js';
|
|
49
53
|
import type { TokenManager } from './secrets/token-manager.js';
|
|
50
54
|
import type { SecretResolver } from './secrets/secret-resolver.js';
|
|
51
55
|
import { type AdminRouteDeps } from './routes/admin.js';
|
|
@@ -66,6 +70,7 @@ import type { TrustStore } from './events/trust-store.js';
|
|
|
66
70
|
import type { EnvironmentStore } from './environments/environment-store.js';
|
|
67
71
|
import type { VariableStore } from './environments/variable-store.js';
|
|
68
72
|
import type { HeldRunStore } from './environments/held-runs.js';
|
|
73
|
+
import type { StepApprovalBridge } from './approvals/step-approval-bridge.js';
|
|
69
74
|
import type { ContributorCache } from './security/contributor-cache.js';
|
|
70
75
|
import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
|
|
71
76
|
/**
|
|
@@ -89,6 +94,15 @@ export interface AppDependencies {
|
|
|
89
94
|
buildCoordinator?: BuildCoordinator;
|
|
90
95
|
/** Dep cache for dependency tarballs. Optional — requires S3 storage. */
|
|
91
96
|
depCache?: DepCache;
|
|
97
|
+
/** User-facing cache (ctx.cache / declarative job-step cache). Optional — requires cache storage. */
|
|
98
|
+
userCache?: UserCache;
|
|
99
|
+
/**
|
|
100
|
+
* Server-side jobId -> user-cache-namespace store. Written at dispatch time
|
|
101
|
+
* (orchestrator-core's buildOnDispatch); read by the agent-WS handler to
|
|
102
|
+
* resolve the cache ref for `cache.user.*` requests WITHOUT trusting the wire
|
|
103
|
+
* message. Optional — absent for modes that never serve the user cache.
|
|
104
|
+
*/
|
|
105
|
+
dispatchCacheRefs?: DispatchCacheRefTracker;
|
|
92
106
|
/** Cache storage backend (S3) for metadata operations on upload completion. */
|
|
93
107
|
cacheStorage?: CacheStorage;
|
|
94
108
|
/**
|
|
@@ -121,6 +135,11 @@ export interface AppDependencies {
|
|
|
121
135
|
tokenStore?: AgentTokenStore;
|
|
122
136
|
/** Job ownership tracker for validating agent messages. Optional — requires database. */
|
|
123
137
|
ownershipTracker?: OwnershipTracker;
|
|
138
|
+
/**
|
|
139
|
+
* Orchestrator-scoped collector correlating fleet.logs.request with agents'
|
|
140
|
+
* chunked bundle responses. Shared with the fleet fan-out in orchestrator-core.
|
|
141
|
+
*/
|
|
142
|
+
fleetAgentCollector?: FleetAgentCollector;
|
|
124
143
|
/** Admin API route dependencies. Optional -- only mounted when secrets management is configured. */
|
|
125
144
|
adminDeps?: AdminRouteDeps;
|
|
126
145
|
/**
|
|
@@ -159,6 +178,8 @@ export interface AppDependencies {
|
|
|
159
178
|
variableStore?: VariableStore;
|
|
160
179
|
/** Held run store for persisting protection rule holds. Optional -- if not set, holds are not persisted. */
|
|
161
180
|
heldRunStore?: HeldRunStore;
|
|
181
|
+
/** Step-approval bridge — opens step-scoped holds and relays their resolution back to the waiting agent. Optional. */
|
|
182
|
+
stepApprovalBridge?: StepApprovalBridge;
|
|
162
183
|
/** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are dispatched without permission checks. */
|
|
163
184
|
globalWorkflowPolicy?: GlobalWorkflowPolicy;
|
|
164
185
|
/** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
|
|
@@ -204,6 +225,21 @@ export interface AppDependencies {
|
|
|
204
225
|
* can read the same store and Mimir gets agent metrics per-org.
|
|
205
226
|
*/
|
|
206
227
|
agentMetricsAggregator?: AgentMetricsAggregator;
|
|
228
|
+
/**
|
|
229
|
+
* Fleet log-collection route backing. `getTopology` enumerates the cluster
|
|
230
|
+
* (no fan-out); `collectBundle` drives the recursive fan-out and returns the
|
|
231
|
+
* assembled ZIP. Mounted at /admin/fleet-topology + /admin/fleet-bundle.
|
|
232
|
+
*/
|
|
233
|
+
fleetRoutes?: FleetRoutesDeps;
|
|
234
|
+
}
|
|
235
|
+
/** Backing for the /admin/fleet-* routes. */
|
|
236
|
+
export interface FleetRoutesDeps {
|
|
237
|
+
getTopology: () => FleetTopology;
|
|
238
|
+
collectBundle: (opts: {
|
|
239
|
+
selectors: string[];
|
|
240
|
+
logWindowHours?: number;
|
|
241
|
+
timeoutSeconds?: number;
|
|
242
|
+
}) => Promise<Buffer>;
|
|
207
243
|
}
|
|
208
244
|
/**
|
|
209
245
|
* In-memory cache for step source locations extracted from lock files.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared approve/reject applier — the single code path behind both the
|
|
3
|
+
* dashboard WS handler and the `kici` CLI HTTP route, so the authorization +
|
|
4
|
+
* resume story is identical regardless of the surface.
|
|
5
|
+
*
|
|
6
|
+
* Flow for an approve:
|
|
7
|
+
* 1. Load the hold + its recorded decisions.
|
|
8
|
+
* 2. Build the team-membership lookup and run `canApprove` (eligibility +
|
|
9
|
+
* self-approval gate). Ineligible actors are rejected with a clear reason.
|
|
10
|
+
* 3. Record the decision in `held_run_approvals`.
|
|
11
|
+
* 4. Re-evaluate. If all clauses are satisfied, `release()` the hold and
|
|
12
|
+
* resume the element (re-dispatch for job/workflow; the agent bridge for
|
|
13
|
+
* step scope — wired by the caller via `onStepRelease`).
|
|
14
|
+
*
|
|
15
|
+
* A reject records the decision and `reject()`s the hold (failing the element).
|
|
16
|
+
*/
|
|
17
|
+
import { ApprovalDecision } from '@kici-dev/engine';
|
|
18
|
+
import type { HeldRunStore, ReleaseSignal } from '../environments/held-runs.js';
|
|
19
|
+
import { type TeamMembershipLookup } from './approval-resolver.js';
|
|
20
|
+
/** Outcome of applying a decision. */
|
|
21
|
+
export interface ApplyDecisionResult {
|
|
22
|
+
/** Whether the decision was accepted (recorded). */
|
|
23
|
+
accepted: boolean;
|
|
24
|
+
/** Human-readable status: 'released' | 'rejected' | 'pending' | 'ineligible' | 'not-found'. */
|
|
25
|
+
status: 'released' | 'rejected' | 'pending' | 'ineligible' | 'not-found';
|
|
26
|
+
/** When still pending, how many clauses remain unsatisfied. */
|
|
27
|
+
remainingClauses?: number;
|
|
28
|
+
/** When ineligible/not-found, a reason for the caller to surface. */
|
|
29
|
+
reason?: string;
|
|
30
|
+
/** The release signal, when the element was released (for the caller to resume). */
|
|
31
|
+
release?: ReleaseSignal;
|
|
32
|
+
}
|
|
33
|
+
/** Dependencies injected into `applyDecision` (testable in isolation). */
|
|
34
|
+
export interface ApplyDecisionDeps {
|
|
35
|
+
orgId: string;
|
|
36
|
+
store: HeldRunStore;
|
|
37
|
+
/** Team name → member user ids (from the Plan-1 trust-policy cache). */
|
|
38
|
+
teamMembershipLookup: TeamMembershipLookup;
|
|
39
|
+
/** Whether the run triggerer may self-approve (org_settings.allow_self_approval). */
|
|
40
|
+
allowSelfApproval: boolean;
|
|
41
|
+
/** Resolve the Keycloak sub of the user who triggered a run (for the self-approval gate). */
|
|
42
|
+
resolveTriggererSub: (runId: string) => Promise<string | undefined>;
|
|
43
|
+
/** Called when a job/workflow hold is released — re-dispatch the element. */
|
|
44
|
+
onJobRelease: (signal: ReleaseSignal) => Promise<void>;
|
|
45
|
+
/** Called when a step hold is released — notify the waiting agent (approved). */
|
|
46
|
+
onStepRelease?: (signal: ReleaseSignal) => Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Called when a step-scoped hold is rejected — notify the waiting agent so it
|
|
49
|
+
* fails the step instead of blocking until expiry. Carries the holdId.
|
|
50
|
+
*/
|
|
51
|
+
onStepReject?: (heldRunId: string, reason?: string) => Promise<void> | void;
|
|
52
|
+
}
|
|
53
|
+
/** Apply a single approve/reject decision to a hold. */
|
|
54
|
+
export declare function applyDecision(deps: ApplyDecisionDeps, args: {
|
|
55
|
+
heldRunId: string;
|
|
56
|
+
actorSub: string;
|
|
57
|
+
decision: ApprovalDecision;
|
|
58
|
+
reason?: string;
|
|
59
|
+
}): Promise<ApplyDecisionResult>;
|
|
60
|
+
//# sourceMappingURL=apply-decision.d.ts.map
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure approval-clause evaluator. No DB access, no I/O — given a requirement,
|
|
3
|
+
* a team-membership lookup, the recorded decisions, and (for eligibility) the
|
|
4
|
+
* run's triggerer + the self-approval policy, it answers:
|
|
5
|
+
*
|
|
6
|
+
* - `evaluate` — is the requirement satisfied (all clauses ANDed), and was it
|
|
7
|
+
* rejected (any single reject decision)?
|
|
8
|
+
* - `canApprove` — may a given actor cast an approve decision right now?
|
|
9
|
+
*
|
|
10
|
+
* Both the dashboard handler and the CLI HTTP route funnel through these so the
|
|
11
|
+
* authorization story is identical regardless of the surface.
|
|
12
|
+
*/
|
|
13
|
+
import type { ApprovalRequirement, ApproverClause } from '@kici-dev/engine';
|
|
14
|
+
/** A team name → set of member user ids (Keycloak subs). */
|
|
15
|
+
export type TeamMembershipLookup = (team: string) => Set<string>;
|
|
16
|
+
/** One recorded decision row (subset of `held_run_approvals`). */
|
|
17
|
+
export interface RecordedDecision {
|
|
18
|
+
approver_user_id: string;
|
|
19
|
+
decision: 'approve' | 'reject';
|
|
20
|
+
}
|
|
21
|
+
/** Per-clause satisfaction detail for attribution + dashboard progress. */
|
|
22
|
+
export interface PerClauseResult {
|
|
23
|
+
clause: ApproverClause;
|
|
24
|
+
satisfied: boolean;
|
|
25
|
+
/** The approver who satisfied this clause, if any. */
|
|
26
|
+
by?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface EvaluationResult {
|
|
29
|
+
/** All clauses satisfied (AND). For empty clauses: any single approve. */
|
|
30
|
+
satisfied: boolean;
|
|
31
|
+
/** Any reject decision present → the element is rejected. */
|
|
32
|
+
rejected: boolean;
|
|
33
|
+
perClause: PerClauseResult[];
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Is a single clause satisfied by any approve decision whose approver is
|
|
37
|
+
* eligible for the clause? Reject decisions never satisfy.
|
|
38
|
+
*/
|
|
39
|
+
export declare function isClauseSatisfied(clause: ApproverClause, decisions: RecordedDecision[], lookup: TeamMembershipLookup): boolean;
|
|
40
|
+
/** Does an actor satisfy/qualify for a clause (team membership or exact user)? */
|
|
41
|
+
export declare function isActorEligibleForClause(actorSub: string, clause: ApproverClause, lookup: TeamMembershipLookup): boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Evaluate a requirement against the recorded decisions.
|
|
44
|
+
*
|
|
45
|
+
* - A single reject decision sets `rejected` (the caller fails the element).
|
|
46
|
+
* - Empty clauses ⇒ satisfied by ANY single approve decision.
|
|
47
|
+
* - Non-empty clauses ⇒ satisfied iff every clause is satisfied (AND).
|
|
48
|
+
*/
|
|
49
|
+
export declare function evaluate(requirement: ApprovalRequirement, decisions: RecordedDecision[], lookup: TeamMembershipLookup): EvaluationResult;
|
|
50
|
+
export interface CanApproveContext {
|
|
51
|
+
/** The Keycloak sub of the user who triggered the run. */
|
|
52
|
+
triggererSub: string;
|
|
53
|
+
/** Whether the triggerer may self-approve their own held elements. */
|
|
54
|
+
allowSelfApproval: boolean;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* May `actorSub` cast an approve decision on this requirement right now?
|
|
58
|
+
*
|
|
59
|
+
* - Self-approval gate: when `allowSelfApproval` is false and the actor is the
|
|
60
|
+
* run's triggerer, they may never approve.
|
|
61
|
+
* - Eligibility: the actor must qualify for at least one still-UNSATISFIED
|
|
62
|
+
* clause. (Approving a clause that is already satisfied adds nothing.) For an
|
|
63
|
+
* empty-clause requirement, any non-triggerer-blocked actor qualifies.
|
|
64
|
+
*/
|
|
65
|
+
export declare function canApprove(actorSub: string, requirement: ApprovalRequirement, decisions: RecordedDecision[], lookup: TeamMembershipLookup, ctx: CanApproveContext): boolean;
|
|
66
|
+
//# sourceMappingURL=approval-resolver.d.ts.map
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { HeldRunStore } from '../environments/held-runs.js';
|
|
2
|
+
import type { AccessLogWriter } from '../audit/access-log.js';
|
|
3
|
+
/** Outcome relayed back to the waiting agent. */
|
|
4
|
+
export type StepApprovalOutcome = 'approved' | 'rejected' | 'expired';
|
|
5
|
+
/** Arguments to open a step-scoped approval hold. */
|
|
6
|
+
export interface StepApprovalRequest {
|
|
7
|
+
agentId: string;
|
|
8
|
+
runId: string;
|
|
9
|
+
jobId: string;
|
|
10
|
+
stepIndex: number;
|
|
11
|
+
stepName: string;
|
|
12
|
+
clauses: Array<{
|
|
13
|
+
team: string;
|
|
14
|
+
} | {
|
|
15
|
+
user: string;
|
|
16
|
+
}>;
|
|
17
|
+
reason: string;
|
|
18
|
+
/** Per-gate timeout (seconds); falls back to the org default. */
|
|
19
|
+
timeoutSeconds?: number;
|
|
20
|
+
}
|
|
21
|
+
/** Dependencies injected into the bridge. */
|
|
22
|
+
export interface StepApprovalBridgeDeps {
|
|
23
|
+
store: HeldRunStore;
|
|
24
|
+
/**
|
|
25
|
+
* Resolve the orchestrator's tenant org id (same value the dashboard handler
|
|
26
|
+
* uses to read held_runs). A function because the org is resolved post-auth
|
|
27
|
+
* for sourceless orchestrators.
|
|
28
|
+
*/
|
|
29
|
+
resolveOrgId: () => string;
|
|
30
|
+
/** Resolve the authoritative approval expiry (seconds) for an org. */
|
|
31
|
+
resolveExpirySeconds: (orgId: string) => Promise<number>;
|
|
32
|
+
/**
|
|
33
|
+
* Access-log writer for the orchestrator audit stream. Optional -- if not
|
|
34
|
+
* set, the step-hold creation audit row (`held_run.request`) is skipped.
|
|
35
|
+
*/
|
|
36
|
+
accessLogWriter?: AccessLogWriter;
|
|
37
|
+
/** Routing key recorded on the audit row. Optional. */
|
|
38
|
+
routingKey?: string | null;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Owns the step-scoped held_runs rows and the map from holdId → the waiting
|
|
42
|
+
* agent's resolver. The only place that opens a step hold and the only place
|
|
43
|
+
* that settles a step-scoped wait.
|
|
44
|
+
*/
|
|
45
|
+
export declare class StepApprovalBridge {
|
|
46
|
+
private readonly deps;
|
|
47
|
+
private readonly pending;
|
|
48
|
+
constructor(deps: StepApprovalBridgeDeps);
|
|
49
|
+
/**
|
|
50
|
+
* Create a step-scoped hold and return a promise that settles when the hold
|
|
51
|
+
* is approved / rejected / expired (via {@link resolve}).
|
|
52
|
+
*/
|
|
53
|
+
request(req: StepApprovalRequest): Promise<{
|
|
54
|
+
outcome: StepApprovalOutcome;
|
|
55
|
+
reason?: string;
|
|
56
|
+
}>;
|
|
57
|
+
/**
|
|
58
|
+
* Settle a step-scoped wait. Called by the approve/reject applier
|
|
59
|
+
* (`onStepRelease`) and by the stale detector (`expired`). A no-op when no
|
|
60
|
+
* agent is waiting on the hold (e.g. the agent already disconnected).
|
|
61
|
+
*/
|
|
62
|
+
resolve(holdId: string, outcome: StepApprovalOutcome, reason?: string): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Reject every pending wait for a disconnected agent. The held_runs rows are
|
|
65
|
+
* left as-is (the stale detector will expire them) — only the in-memory
|
|
66
|
+
* resolver is dropped so the handler's relayed `ws.send` is skipped.
|
|
67
|
+
*/
|
|
68
|
+
failAgent(agentId: string): void;
|
|
69
|
+
/** Number of in-flight step-approval waits (test/diagnostics). */
|
|
70
|
+
size(): number;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=step-approval-bridge.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only lookup over the team memberships pushed from the Platform via
|
|
3
|
+
* `trust_policy.update`. The orchestrator has no identity store, so this is the
|
|
4
|
+
* only source of "who is in team X" — the approval resolver uses it to satisfy
|
|
5
|
+
* `{team}` approver clauses.
|
|
6
|
+
*/
|
|
7
|
+
export interface TeamMembershipLookup {
|
|
8
|
+
/** Returns the set of member user ids for a team name (empty if unknown). */
|
|
9
|
+
getTeamMembers(name: string): Set<string>;
|
|
10
|
+
}
|
|
11
|
+
/** A lookup backed by no teams — the default before any push arrives. */
|
|
12
|
+
export declare const EMPTY_TEAM_MEMBERSHIP_LOOKUP: TeamMembershipLookup;
|
|
13
|
+
//# sourceMappingURL=team-membership-lookup.d.ts.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { InitFailure } from '@kici-dev/engine';
|
|
2
|
+
/**
|
|
3
|
+
* Reject reason for a pending init / dynamic-eval job the agent reported as
|
|
4
|
+
* failed. Carries the optional structured initFailure the agent attached to
|
|
5
|
+
* job.status.data so the orchestrator catch can record the correct category
|
|
6
|
+
* instead of collapsing everything to a string.
|
|
7
|
+
*/
|
|
8
|
+
export declare class AgentJobFailedError extends Error {
|
|
9
|
+
readonly initFailure?: InitFailure | undefined;
|
|
10
|
+
readonly name = "AgentJobFailedError";
|
|
11
|
+
constructor(message: string, initFailure?: InitFailure | undefined);
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=agent-job-failed-error.d.ts.map
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-side record of the user-cache namespacing for each dispatched job.
|
|
3
|
+
*
|
|
4
|
+
* The user-facing cache is namespaced by `{orgId, repoId, cacheRefScope, runId}`.
|
|
5
|
+
* Those values are decided by the orchestrator at dispatch time (lifted onto the
|
|
6
|
+
* `job.dispatch` message). The agent later sends `cache.user.*` requests that
|
|
7
|
+
* carry ONLY a `jobId` + `key` — never the namespacing — so the WS handler must
|
|
8
|
+
* resolve the namespace from a trusted server-side store keyed by jobId. This
|
|
9
|
+
* tracker is that store: a wire `cache.user.*` message can name a `jobId`, but it
|
|
10
|
+
* can never influence the org/repo/scope the orchestrator resolves for it.
|
|
11
|
+
*
|
|
12
|
+
* The map is written when a job is dispatched, read on every `cache.user.*`
|
|
13
|
+
* request, and deleted when the job completes or its agent disconnects (mirroring
|
|
14
|
+
* the dispatcher's own per-job cleanup lifecycle) so it cannot leak.
|
|
15
|
+
*/
|
|
16
|
+
import type { CacheRefScope } from '@kici-dev/engine';
|
|
17
|
+
/** Cache namespacing recorded for a dispatched job. */
|
|
18
|
+
export interface DispatchCacheRef {
|
|
19
|
+
/** Org that owns the run — the per-tenant cache isolation boundary. Absent for sourceless deploys. */
|
|
20
|
+
orgId?: string;
|
|
21
|
+
/** Repo identifier (e.g. "owner/repo") — second namespacing level. Absent for sourceless deploys. */
|
|
22
|
+
repoId?: string;
|
|
23
|
+
/** Write scope: `shared` (trusted ref) or `isolated` (untrusted ref, per-run scope). */
|
|
24
|
+
cacheRefScope?: CacheRefScope;
|
|
25
|
+
/** Run id — the per-run isolation namespace for `isolated`-scope writes. */
|
|
26
|
+
runId: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* In-memory `jobId -> DispatchCacheRef` map populated at dispatch time and
|
|
30
|
+
* consumed by the agent WS handler to resolve a `UserCacheRef` server-side.
|
|
31
|
+
*/
|
|
32
|
+
export declare class DispatchCacheRefTracker {
|
|
33
|
+
private readonly refs;
|
|
34
|
+
/** Record the cache namespacing for a dispatched job. */
|
|
35
|
+
record(jobId: string, ref: DispatchCacheRef): void;
|
|
36
|
+
/** Resolve the cache namespacing for a job, or `undefined` if it was never dispatched / already cleaned up. */
|
|
37
|
+
get(jobId: string): DispatchCacheRef | undefined;
|
|
38
|
+
/** Drop a job's recorded ref (on completion or agent disconnect). */
|
|
39
|
+
delete(jobId: string): void;
|
|
40
|
+
/** Drop every recorded ref (test/teardown helper). */
|
|
41
|
+
clear(): void;
|
|
42
|
+
/** Number of currently-tracked dispatches (leak assertions in tests). */
|
|
43
|
+
get size(): number;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=dispatch-cache-ref-tracker.d.ts.map
|
package/dist/cache/index.d.ts
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
export { SourceCache } from './source-cache.js';
|
|
10
10
|
export { BuildCoordinator } from './build-coordinator.js';
|
|
11
11
|
export { DepCache } from './dep-cache.js';
|
|
12
|
+
export { UserCache, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, type UserCacheRef, type UserCacheRestoreResult, type UserCacheBeginSaveResult, type UserCacheOrgLimits, type UserCacheOrgLimitsReader, } from './user-cache.js';
|
|
13
|
+
export { DispatchCacheRefTracker, type DispatchCacheRef } from './dispatch-cache-ref-tracker.js';
|
|
12
14
|
export { PendingBuildTracker } from './pending-builds.js';
|
|
13
15
|
export { PendingInitTracker } from './pending-inits.js';
|
|
14
16
|
export type { InitResult } from './pending-inits.js';
|