@kici-dev/orchestrator 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +10 -1
- package/dist/agent/host-roster.d.ts +8 -3
- package/dist/agent/registry.d.ts +60 -5
- package/dist/app.d.ts +10 -5
- package/dist/cache/dep-cache.d.ts +14 -6
- package/dist/cache/global-eval-round-cache.d.ts +1 -1
- package/dist/cache/index.d.ts +1 -1
- package/dist/cache/source-cache.d.ts +92 -15
- package/dist/cancel/cancel-run.d.ts +20 -2
- package/dist/cancel/sweep-stuck-cancelling.d.ts +30 -0
- package/dist/cli/api-client.d.ts +10 -0
- package/dist/cli/commands/agent-service/logs.d.ts +1 -1
- package/dist/cli/commands/agent-service/restart.d.ts +1 -1
- package/dist/cli/commands/agent-service/start.d.ts +1 -1
- package/dist/cli/commands/agent-service/status.d.ts +1 -1
- package/dist/cli/commands/agent-service/stop.d.ts +1 -1
- package/dist/cli/commands/agent-service/uninstall.d.ts +1 -1
- package/dist/cli/commands/agent.d.ts +25 -0
- package/dist/cli/commands/config.d.ts +16 -0
- package/dist/cli/commands/db-backup.d.ts +46 -0
- package/dist/cli/commands/held-run.d.ts +9 -0
- package/dist/cli/commands/orchestrator-service/drain.d.ts +2 -19
- package/dist/cli/commands/orchestrator-service/index.d.ts +9 -1
- package/dist/cli/commands/orchestrator-service/install-env.d.ts +43 -2
- package/dist/cli/commands/orchestrator-service/logs.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/restart.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/start.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/status.d.ts +17 -1
- package/dist/cli/commands/orchestrator-service/stop.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/uninstall.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +2 -1
- package/dist/cli/commands/rotate.d.ts +1 -1
- package/dist/cli/commands/shared/env-file-mode.d.ts +16 -0
- package/dist/cli/commands/shared/upgrade-hooks.d.ts +32 -0
- package/dist/cli/commands/shared/upgrade-safety.d.ts +81 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +67 -12
- package/dist/cli/commands/source.d.ts +1 -0
- package/dist/cli/commands/token.d.ts +4 -0
- package/dist/cli/commands/trust-policy.d.ts +57 -1
- package/dist/cli/join.d.ts +1 -1
- package/dist/cli/service/backup-timer.d.ts +108 -0
- package/dist/cli/service/compose-path.d.ts +11 -0
- package/dist/cli/service/compose.d.ts +22 -0
- package/dist/cli/service/deploy-env.d.ts +10 -11
- package/dist/cli/service/env-assignment.d.ts +24 -0
- package/dist/cli/service/index.d.ts +2 -2
- package/dist/cli/service/instance/resolve.d.ts +71 -7
- package/dist/cli/service/instance/types.d.ts +15 -0
- package/dist/cli/service/launchd.d.ts +7 -0
- package/dist/cli/service/systemd.d.ts +1 -0
- package/dist/cli/service/types.d.ts +36 -0
- package/dist/cli/service/windows.d.ts +16 -0
- package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -3
- package/dist/cli/wizard/prompts.d.ts +7 -0
- package/dist/cli.js +12525 -9298
- package/dist/cluster/cluster-settings-reader.d.ts +2 -2
- package/dist/cluster/coordinator.d.ts +11 -0
- package/dist/cluster/instance-heartbeat.d.ts +96 -0
- package/dist/cluster/join-client.d.ts +55 -5
- package/dist/cluster/join-handler.d.ts +1 -0
- package/dist/cluster/orphan-recovery.d.ts +16 -0
- package/dist/cluster/peer-registry.d.ts +10 -0
- package/dist/concurrency/dispatch-next-queued.d.ts +1 -13
- package/dist/concurrency/index.d.ts +1 -2
- package/dist/concurrency/queue-manager.d.ts +63 -18
- package/dist/config/env-overlay.d.ts +3 -2
- package/dist/config.d.ts +24 -0
- package/dist/contexts/release-queued-holds.d.ts +23 -1
- package/dist/dashboard/attestation-filters.d.ts +2 -0
- package/dist/dashboard/handler.d.ts +22 -2
- package/dist/db/deferred-indexes.d.ts +72 -0
- package/dist/db/migration-test-harness.d.ts +22 -0
- package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +30 -0
- package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +23 -0
- package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +23 -0
- package/dist/db/migrations/134_dispatch_queue_ownership.d.ts +4 -0
- package/dist/db/migrations/135_cluster_settings_retention_days.d.ts +4 -0
- package/dist/db/migrations/136_master_key_wrapped_tables_key_version.d.ts +4 -0
- package/dist/db/migrations/137_execution_runs_pr_head_context.d.ts +4 -0
- package/dist/db/migrations/138_execution_runs_subject_trigger_event.d.ts +4 -0
- package/dist/db/migrations/139_admin_token_subject.d.ts +4 -0
- package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +54 -0
- package/dist/db/migrations/141_event_log_shed.d.ts +15 -0
- package/dist/db/migrator.d.ts +47 -0
- package/dist/db/notify-listener.d.ts +60 -0
- package/dist/db/synthetic-job-ids.d.ts +24 -0
- package/dist/db/types.d.ts +169 -4
- package/dist/deployment/deployment-identity.d.ts +31 -1
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/checks/retention.d.ts +24 -0
- package/dist/diagnostics/types.d.ts +7 -0
- package/dist/events/event-router.d.ts +1 -1
- package/dist/fault-injection-types.d.ts +53 -0
- package/dist/firecracker/host-network.d.ts +63 -8
- package/dist/git/credential-broker.d.ts +35 -2
- package/dist/git/inherited-secret.d.ts +7 -2
- package/dist/git/job-context.d.ts +7 -4
- package/dist/helpers/secure-write.d.ts +35 -0
- package/dist/index.js +196 -67
- package/dist/metrics/prometheus.d.ts +116 -1
- package/dist/oidc/db-signer.d.ts +27 -5
- package/dist/oidc/id-token-claims.d.ts +33 -2
- package/dist/oidc/oidc-mint-registration.d.ts +6 -0
- package/dist/oidc/orchestrator-mint.d.ts +8 -0
- package/dist/oidc/reconcile-signing-key.d.ts +37 -0
- package/dist/orchestrator-core.d.ts +9 -3
- package/dist/pipeline/decorating-secret-resolver.d.ts +1 -1
- package/dist/pipeline/dispatch-matched-workflow.d.ts +65 -14
- package/dist/pipeline/manual-schedule.d.ts +1 -0
- package/dist/pipeline/process-webhook.d.ts +20 -2
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +7 -0
- package/dist/pipeline/run-event-context.d.ts +39 -0
- package/dist/policy/dashboard-write-policy-listener.d.ts +1 -1
- package/dist/policy/dashboard-write-policy.d.ts +47 -1
- package/dist/provenance/dispatch-context.d.ts +46 -0
- package/dist/provenance/verify-deferred-capture.d.ts +39 -0
- package/dist/providers/github/auth.d.ts +9 -0
- package/dist/providers/github/deliveries.d.ts +118 -0
- package/dist/providers/universal-git/config.d.ts +1 -1
- package/dist/queue/cleanup.d.ts +13 -0
- package/dist/queue/job-queue.d.ts +162 -14
- package/dist/queue/retention.d.ts +105 -0
- package/dist/reporting/execution-tracker.d.ts +87 -36
- package/dist/routes/admin-db.d.ts +5 -0
- package/dist/routes/admin-events.d.ts +13 -0
- package/dist/routes/admin-held-runs.d.ts +11 -1
- package/dist/routes/admin-org-settings.d.ts +9 -0
- package/dist/routes/admin-runs.d.ts +7 -4
- package/dist/routes/admin-sources.d.ts +15 -0
- package/dist/routes/admin-trust-policy.d.ts +11 -0
- package/dist/routes/admin.d.ts +25 -0
- package/dist/scaler/backend-factory.d.ts +16 -0
- package/dist/scaler/bare-metal-backend.d.ts +22 -4
- package/dist/scaler/config.d.ts +6 -3
- package/dist/scaler/container-backend.d.ts +102 -3
- package/dist/scaler/container-hostconfig.d.ts +60 -0
- package/dist/scaler/file-tail.d.ts +25 -2
- package/dist/scaler/firecracker-api.d.ts +9 -1
- package/dist/scaler/firecracker-backend.d.ts +135 -2
- package/dist/scaler/host-access.d.ts +59 -0
- package/dist/scaler/index.d.ts +1 -0
- package/dist/scaler/manager.d.ts +33 -0
- package/dist/scaler/scaler-state-store.d.ts +6 -0
- package/dist/scaler/types.d.ts +5 -7
- package/dist/secrets/config.d.ts +48 -0
- package/dist/secrets/dashboard-encryption-key.d.ts +19 -0
- package/dist/secrets/ephemeral-keys.d.ts +9 -3
- package/dist/secrets/index.d.ts +3 -1
- package/dist/secrets/job-secret-gate.d.ts +76 -0
- package/dist/secrets/master-key-rotation.d.ts +65 -0
- package/dist/secrets/secret-output-crypto.d.ts +17 -0
- package/dist/secrets/secret-resolver.d.ts +23 -6
- package/dist/secrets/source-credentials.d.ts +1 -1
- package/dist/secrets/token-manager.d.ts +4 -1
- package/dist/security/reduced-privilege-note.d.ts +5 -0
- package/dist/security/trust-policy-gate.d.ts +70 -6
- package/dist/security/trust-tier.d.ts +31 -0
- package/dist/server.d.ts +1 -1
- package/dist/server.js +10661 -5282
- package/dist/sources/source-manager.d.ts +1 -1
- package/dist/stale-detector/stale-dispatch-candidates.d.ts +43 -0
- package/dist/standalone.js +9258 -4122
- package/dist/storage/loopback-guard.d.ts +53 -6
- package/dist/webhook/event-log.d.ts +14 -1
- package/dist/webhook/generic-sources-listener.d.ts +1 -1
- package/dist/webhook/ingest-admission.d.ts +24 -0
- package/dist/webhook/ingest-overflow-replayer.d.ts +78 -14
- package/dist/webhook/ingest-overflow-types.d.ts +13 -1
- package/dist/webhook/register-source-bundle.d.ts +23 -0
- package/dist/webhook/relay-reinject.d.ts +11 -12
- package/dist/webhook/shed-breadcrumb.d.ts +15 -0
- package/dist/ws/git-credential-relay.d.ts +41 -12
- package/dist/ws/platform-client.d.ts +28 -5
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +53 -53
- package/dist/concurrency/group-tracker.d.ts +0 -51
- package/dist/scaler/nftables.d.ts +0 -111
|
@@ -233,6 +233,43 @@ export interface ExecutionTrackerDeps {
|
|
|
233
233
|
*/
|
|
234
234
|
resolveOrgId?: (routingKey: string) => Promise<string>;
|
|
235
235
|
}
|
|
236
|
+
/**
|
|
237
|
+
* The per-job fields the tracker persists to an execution_jobs row.
|
|
238
|
+
*
|
|
239
|
+
* Exported and shared by every registration entry point (`onExecutionStarted`,
|
|
240
|
+
* `addJobsToRun`) rather than restated inline at each one: three structural
|
|
241
|
+
* copies had already drifted apart on which optional fields they carried, so a
|
|
242
|
+
* new column had to be added in three places or silently go unwritten from one
|
|
243
|
+
* of them.
|
|
244
|
+
*/
|
|
245
|
+
export interface TrackedJobRow {
|
|
246
|
+
jobId: string;
|
|
247
|
+
jobName: string;
|
|
248
|
+
matrixValues?: Record<string, unknown>;
|
|
249
|
+
runsOnLabels?: string[];
|
|
250
|
+
baseJobName?: string;
|
|
251
|
+
variantKind?: string;
|
|
252
|
+
variantLabel?: string;
|
|
253
|
+
waveGated?: boolean;
|
|
254
|
+
waveMaxParallel?: number;
|
|
255
|
+
waveFailFast?: boolean;
|
|
256
|
+
contexts?: string[];
|
|
257
|
+
/**
|
|
258
|
+
* The `gitCredentials` map this job's lock entry declared, verbatim.
|
|
259
|
+
* Persisted so the credential relay can check a workflow-supplied ref against
|
|
260
|
+
* server truth without re-parsing a lock file on a hot path — so it must
|
|
261
|
+
* match the lock exactly, including a `<name>Value` the lock declared.
|
|
262
|
+
*/
|
|
263
|
+
gitCredentials?: Readonly<Record<string, Readonly<Record<string, string>>>>;
|
|
264
|
+
skippedContexts?: string[];
|
|
265
|
+
envWarning?: string;
|
|
266
|
+
/** `gate` for an invoke gate, `proxy` for a summoned-run mirror. Defaults to `standard`. */
|
|
267
|
+
jobKind?: JobKind;
|
|
268
|
+
/** For a proxy job, the summoned run it mirrors. */
|
|
269
|
+
summonedRunId?: string;
|
|
270
|
+
/** For a gate job, its wall-clock timeout in ms (orchestrator-swept). */
|
|
271
|
+
timeoutMs?: number;
|
|
272
|
+
}
|
|
236
273
|
export declare class ExecutionTracker {
|
|
237
274
|
private readonly db;
|
|
238
275
|
private readonly observerRegistry?;
|
|
@@ -301,21 +338,7 @@ export declare class ExecutionTracker {
|
|
|
301
338
|
* Inserts execution_runs and execution_jobs rows in the DB and
|
|
302
339
|
* sets up in-memory tracking state.
|
|
303
340
|
*/
|
|
304
|
-
onExecutionStarted(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, triggerDecision: Record<string, unknown> | null, jobs:
|
|
305
|
-
jobId: string;
|
|
306
|
-
jobName: string;
|
|
307
|
-
matrixValues?: Record<string, unknown>;
|
|
308
|
-
runsOnLabels?: string[];
|
|
309
|
-
baseJobName?: string;
|
|
310
|
-
variantKind?: string;
|
|
311
|
-
variantLabel?: string;
|
|
312
|
-
waveGated?: boolean;
|
|
313
|
-
waveMaxParallel?: number;
|
|
314
|
-
waveFailFast?: boolean;
|
|
315
|
-
contexts?: string[];
|
|
316
|
-
skippedContexts?: string[];
|
|
317
|
-
envWarning?: string;
|
|
318
|
-
}>, routingKey?: string,
|
|
341
|
+
onExecutionStarted(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, triggerDecision: Record<string, unknown> | null, jobs: TrackedJobRow[], routingKey?: string,
|
|
319
342
|
/** Secret context names dispatched with jobs (for context-disable job lookup). */
|
|
320
343
|
dispatchedContexts?: string[],
|
|
321
344
|
/** Trigger event type (e.g. "push", "pr:open") for dashboard display. */
|
|
@@ -353,7 +376,34 @@ export declare class ExecutionTracker {
|
|
|
353
376
|
* repository and dispatched against another. Omitted/null for every
|
|
354
377
|
* per-repository run, where the two are the same repository.
|
|
355
378
|
*/
|
|
356
|
-
workflowRepoIdentifier?: string | null
|
|
379
|
+
workflowRepoIdentifier?: string | null,
|
|
380
|
+
/**
|
|
381
|
+
* Pull-request head context, resolved from the normalized event.
|
|
382
|
+
*
|
|
383
|
+
* Written HERE, on the authoritative insert, and never in the
|
|
384
|
+
* fire-and-forget post-start update `trust_tier` uses. A lost write would
|
|
385
|
+
* leave `is_fork` NULL, which every claim renders `'unresolved'` — correct
|
|
386
|
+
* for a legacy row, and a silent policy failure for a live one.
|
|
387
|
+
*/
|
|
388
|
+
eventContext?: {
|
|
389
|
+
headRef?: string | null;
|
|
390
|
+
headRepository?: string | null;
|
|
391
|
+
/**
|
|
392
|
+
* NULL only when the event genuinely does not resolve one: a PR payload
|
|
393
|
+
* carrying no head or base repository. A non-PR event resolves to
|
|
394
|
+
* `false`, because a push has no fork dimension — leaving it unresolved
|
|
395
|
+
* would make every `is_fork = false` trust policy reject legitimate
|
|
396
|
+
* pushes, and the customer's fix for that would re-open the hole.
|
|
397
|
+
*/
|
|
398
|
+
isFork?: boolean | null;
|
|
399
|
+
/**
|
|
400
|
+
* The event the OIDC subject is derived from, when `triggerEvent` is not
|
|
401
|
+
* it. Set only by the re-run path, which records `'rerun'` as its trigger
|
|
402
|
+
* event and carries the original run's event here so a re-run of a pull
|
|
403
|
+
* request keeps the pull-request subject.
|
|
404
|
+
*/
|
|
405
|
+
subjectTriggerEvent?: string | null;
|
|
406
|
+
}): Promise<void>;
|
|
357
407
|
/**
|
|
358
408
|
* Upsert one execution_jobs row per dispatched job (idempotent on
|
|
359
409
|
* (run_id, job_id) to tolerate a race with an early `onJobStatus`). The
|
|
@@ -414,6 +464,18 @@ export declare class ExecutionTracker {
|
|
|
414
464
|
* never a finished one.
|
|
415
465
|
*/
|
|
416
466
|
isJobTerminal(runId: string, jobId: string): Promise<boolean>;
|
|
467
|
+
/**
|
|
468
|
+
* The status `execution_jobs` currently records for a job, or undefined when
|
|
469
|
+
* no row exists. Read after a guarded write is rejected, to tell a row that
|
|
470
|
+
* contradicts the frame from one that already agrees with it.
|
|
471
|
+
*/
|
|
472
|
+
private readJobStatus;
|
|
473
|
+
/**
|
|
474
|
+
* The status `execution_runs` currently records for a run, or undefined when
|
|
475
|
+
* no row exists. Read on the agreeing-frame path to tell a run that still
|
|
476
|
+
* needs driving from one that was already rolled up.
|
|
477
|
+
*/
|
|
478
|
+
private readRunStatus;
|
|
417
479
|
/**
|
|
418
480
|
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
419
481
|
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
@@ -430,21 +492,7 @@ export declare class ExecutionTracker {
|
|
|
430
492
|
* cross-run lock-ordering deadlock.
|
|
431
493
|
*/
|
|
432
494
|
private withRunLock;
|
|
433
|
-
addJobsToRun(runId: string, jobs:
|
|
434
|
-
jobId: string;
|
|
435
|
-
jobName: string;
|
|
436
|
-
matrixValues?: Record<string, unknown>;
|
|
437
|
-
runsOnLabels?: string[];
|
|
438
|
-
baseJobName?: string;
|
|
439
|
-
variantKind?: string;
|
|
440
|
-
variantLabel?: string;
|
|
441
|
-
contexts?: string[];
|
|
442
|
-
skippedContexts?: string[];
|
|
443
|
-
envWarning?: string;
|
|
444
|
-
jobKind?: JobKind;
|
|
445
|
-
summonedRunId?: string;
|
|
446
|
-
timeoutMs?: number;
|
|
447
|
-
}>, dispatchedContexts?: string[],
|
|
495
|
+
addJobsToRun(runId: string, jobs: TrackedJobRow[], dispatchedContexts?: string[],
|
|
448
496
|
/** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
|
|
449
497
|
replaceSyntheticId?: string): Promise<void>;
|
|
450
498
|
private addJobsToRunImpl;
|
|
@@ -479,8 +527,11 @@ export declare class ExecutionTracker {
|
|
|
479
527
|
private recoverJobFromDispatchQueue;
|
|
480
528
|
/**
|
|
481
529
|
* Phase 2: build the upsert payload for execution_jobs and execute it.
|
|
482
|
-
*
|
|
483
|
-
*
|
|
530
|
+
*
|
|
531
|
+
* The conflict update is guarded so it cannot rewrite a row that already
|
|
532
|
+
* reached a terminal state. `applied` reports the guard's verdict; the log
|
|
533
|
+
* byte total is the per-job value when the job reached a terminal state (used
|
|
534
|
+
* downstream by the run-completion phase to populate run.log_bytes).
|
|
484
535
|
*/
|
|
485
536
|
private persistJobStatusUpdate;
|
|
486
537
|
/**
|
|
@@ -969,9 +1020,9 @@ export declare class ExecutionTracker {
|
|
|
969
1020
|
* A run this tracker does not know is a no-op: a missing run means the status
|
|
970
1021
|
* updates that would read the fields are not coming either.
|
|
971
1022
|
*
|
|
972
|
-
* Keep this synchronous.
|
|
973
|
-
*
|
|
974
|
-
*
|
|
1023
|
+
* Keep this synchronous. Each caller wraps it in `try`/`catch`, which catches
|
|
1024
|
+
* a throw but not a rejected promise, so an async body would put the failure
|
|
1025
|
+
* back outside the guard.
|
|
975
1026
|
*/
|
|
976
1027
|
setRunTrustContext(runId: string, trustTier: string, lockFileSource?: string): void;
|
|
977
1028
|
/**
|
|
@@ -15,6 +15,11 @@ import type { Role } from '../secrets/rbac.js';
|
|
|
15
15
|
interface DbRouteDeps {
|
|
16
16
|
db: Kysely<any>;
|
|
17
17
|
pool: pg.Pool;
|
|
18
|
+
/**
|
|
19
|
+
* Connection string for the timeout-free pool that schema work runs on.
|
|
20
|
+
* Absent in unit tests, which fall back to the shared pool.
|
|
21
|
+
*/
|
|
22
|
+
databaseUrl?: string;
|
|
18
23
|
}
|
|
19
24
|
type AdminDbEnv = {
|
|
20
25
|
Variables: {
|
|
@@ -41,6 +41,19 @@ interface AdminEventRouteDeps {
|
|
|
41
41
|
secretResolver: SecretResolver | null;
|
|
42
42
|
/** Fleet-wide settings reader; threads the live lock-file cap into newly-registered universal-git sources. */
|
|
43
43
|
clusterSettings?: ClusterSettingsReader;
|
|
44
|
+
/**
|
|
45
|
+
* The canonical Platform organization id this orchestrator authenticated
|
|
46
|
+
* as, or `undefined` when it is not Platform-connected (independent mode)
|
|
47
|
+
* or has not authenticated yet.
|
|
48
|
+
*
|
|
49
|
+
* A generic source mints the routing key `generic:<orgId>:<id>`, and the
|
|
50
|
+
* Platform refuses to register a key naming an organization other than the
|
|
51
|
+
* authenticated one. So a source created under a mismatched org would
|
|
52
|
+
* register, be rejected, and silently never deliver — and its published
|
|
53
|
+
* webhook URL, which the Platform composes from the Platform org id, would
|
|
54
|
+
* 404. The create handler refuses the mismatch up front instead.
|
|
55
|
+
*/
|
|
56
|
+
getPlatformOrgId?: () => string | undefined;
|
|
44
57
|
/**
|
|
45
58
|
* Optional — when provided, the `POST /api/v1/admin/events/emit` route is
|
|
46
59
|
* mounted so operators can INSERT into `kici_events` + `pg_notify` via HTTP.
|
|
@@ -42,7 +42,7 @@ import { Hono } from 'hono';
|
|
|
42
42
|
import type { Kysely } from 'kysely';
|
|
43
43
|
import type { OrchestratorMode } from '@kici-dev/engine';
|
|
44
44
|
import { HeldRunStore, type ReleaseSignal } from '../contexts/held-runs.js';
|
|
45
|
-
import { TrustDirectoryStore } from '../security/trust-directory-store.js';
|
|
45
|
+
import type { TrustDirectoryStore } from '../security/trust-directory-store.js';
|
|
46
46
|
import { adminActorSub, triggererSubjectFor } from '../approvals/triggerer-subject.js';
|
|
47
47
|
import { type ResolveCheckStatusPoster } from '../pipeline/security-hold-check.js';
|
|
48
48
|
import type { RbacEnforcer, Role } from '../secrets/rbac.js';
|
|
@@ -67,6 +67,16 @@ export declare const PLATFORM_MANAGED_HELD_RUN_MESSAGE: string;
|
|
|
67
67
|
* would block with nothing left to release OR expire it.
|
|
68
68
|
*/
|
|
69
69
|
export declare const STEP_SCOPE_UNSUPPORTED_MESSAGE: string;
|
|
70
|
+
/**
|
|
71
|
+
* Wording for an `--as` id the approval directory does not hold.
|
|
72
|
+
*
|
|
73
|
+
* The directory is the enumerable set of subjects this route will answer as. An
|
|
74
|
+
* id it does not carry is refused rather than passed through, because an
|
|
75
|
+
* unvalidated approver subject is free text: it would let an operator token
|
|
76
|
+
* write any string it liked into `held_run_approvals.approver_user_id` and
|
|
77
|
+
* satisfy any `{user}` clause by naming it.
|
|
78
|
+
*/
|
|
79
|
+
export declare function unregisteredApproverMessage(userId: string): string;
|
|
70
80
|
/** The release wiring the applier needs to actually resume a released element. */
|
|
71
81
|
export interface HeldRunReleaseWiring {
|
|
72
82
|
/**
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { Hono } from 'hono';
|
|
13
13
|
import { type Kysely } from 'kysely';
|
|
14
|
+
import { type OrchestratorMode } from '@kici-dev/engine';
|
|
14
15
|
import { DashboardWriteOperation } from '@kici-dev/engine/protocol/dashboard-write-operations';
|
|
15
16
|
import type { Database } from '../db/types.js';
|
|
16
17
|
import type { RbacEnforcer, Role } from '../secrets/rbac.js';
|
|
@@ -23,6 +24,14 @@ interface OrgSettingsRouteDeps {
|
|
|
23
24
|
* `config.globalWorkflowsEnabled`.
|
|
24
25
|
*/
|
|
25
26
|
globalWorkflowsEnabledDefault: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* This orchestrator's mode.
|
|
29
|
+
*
|
|
30
|
+
* The dashboard-write policy needs it: disabling `held_runs.approve` /
|
|
31
|
+
* `held_runs.reject` is coherent only where `kici-admin held-run` can still
|
|
32
|
+
* answer a hold, which is independent mode alone.
|
|
33
|
+
*/
|
|
34
|
+
mode: OrchestratorMode;
|
|
26
35
|
/**
|
|
27
36
|
* Optional — when wired, each `dashboard_write_policy` flip emits one
|
|
28
37
|
* `access_log` row (`org_settings.dashboard_write_policy.update`)
|
|
@@ -24,13 +24,14 @@ import type { Database } from '../db/types.js';
|
|
|
24
24
|
import type { TokenManager } from '../secrets/token-manager.js';
|
|
25
25
|
import type { RbacEnforcer, Role } from '../secrets/rbac.js';
|
|
26
26
|
import type { AuditLogger } from '../secrets/audit-logger.js';
|
|
27
|
+
import type { ResolvedMasterKeys } from '../secrets/config.js';
|
|
27
28
|
import type { LogStorage } from '../reporting/log-storage.js';
|
|
28
29
|
/**
|
|
29
30
|
* Statuses accepted by the `?status=` filter.
|
|
30
31
|
*
|
|
31
32
|
* The full canonical union rather than just `ExecutionRunStatus.options`: only
|
|
32
33
|
* run statuses can appear in `execution_runs.status`, but the job-only members
|
|
33
|
-
* were accepted before and
|
|
34
|
+
* were accepted before and match no row, so keeping them avoids turning
|
|
34
35
|
* an empty result into a rejected request.
|
|
35
36
|
*/
|
|
36
37
|
export declare const RUN_STATUSES: ReadonlySet<string>;
|
|
@@ -44,10 +45,12 @@ export interface AdminRunRoutesDeps {
|
|
|
44
45
|
/** Required to reveal secret-output plaintext via ?reveal=true. */
|
|
45
46
|
auditLogger?: AuditLogger;
|
|
46
47
|
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
48
|
+
* The orchestrator master key, plus the previous generation during a
|
|
49
|
+
* rotation grace window. Required to reveal secret-output plaintext; a row
|
|
50
|
+
* still sealed under the old key reveals until `rotate-key` has swept the
|
|
51
|
+
* table.
|
|
49
52
|
*/
|
|
50
|
-
|
|
53
|
+
masterKeys?: ResolvedMasterKeys | null;
|
|
51
54
|
/**
|
|
52
55
|
* Log storage backend. Required only by the agent step-logs endpoint
|
|
53
56
|
* (`/runs/:runId/jobs/:jobId/steps/:stepIndex/logs`); when absent, that
|
|
@@ -12,6 +12,7 @@ import { Hono } from 'hono';
|
|
|
12
12
|
import { OrchestratorMode } from '@kici-dev/engine';
|
|
13
13
|
import type { SourceStore } from '../sources/source-store.js';
|
|
14
14
|
import type { Role } from '../secrets/rbac.js';
|
|
15
|
+
import { type RedeliverWindowResult } from '../providers/github/deliveries.js';
|
|
15
16
|
import { type FetchGithubAppIdentity } from '../github-app-name-refresher/github-app-name-refresher.js';
|
|
16
17
|
interface SourceRouteDeps {
|
|
17
18
|
sourceStore: SourceStore;
|
|
@@ -47,6 +48,11 @@ interface SourceRouteDeps {
|
|
|
47
48
|
* `source refresh` route.
|
|
48
49
|
*/
|
|
49
50
|
fetchAppIdentity?: FetchGithubAppIdentity;
|
|
51
|
+
/**
|
|
52
|
+
* Replay a window of GitHub webhook deliveries for one App. Injectable for
|
|
53
|
+
* tests; defaults to the real `redeliverWindow`.
|
|
54
|
+
*/
|
|
55
|
+
redeliverDeliveries?: RedeliverDeliveries;
|
|
50
56
|
/**
|
|
51
57
|
* Orchestrator operating mode. `observed` refuses GitHub-App source creation:
|
|
52
58
|
* those sources are ingested through the Platform relay, and an observed
|
|
@@ -54,6 +60,15 @@ interface SourceRouteDeps {
|
|
|
54
60
|
*/
|
|
55
61
|
mode?: OrchestratorMode;
|
|
56
62
|
}
|
|
63
|
+
/** Replay a delivery window for a GitHub App. Matches `redeliverWindow`. */
|
|
64
|
+
export type RedeliverDeliveries = (app: {
|
|
65
|
+
appId: string;
|
|
66
|
+
privateKey: string;
|
|
67
|
+
}, opts: {
|
|
68
|
+
since: Date;
|
|
69
|
+
until: Date;
|
|
70
|
+
dryRun?: boolean;
|
|
71
|
+
}) => Promise<RedeliverWindowResult>;
|
|
57
72
|
type AdminSourcesEnv = {
|
|
58
73
|
Variables: {
|
|
59
74
|
role: Role;
|
|
@@ -52,6 +52,17 @@ interface TrustPolicyRouteDeps {
|
|
|
52
52
|
rbac: RbacEnforcer;
|
|
53
53
|
/** This orchestrator's mode; decides whether PATCH is permitted at all. */
|
|
54
54
|
mode: OrchestratorMode;
|
|
55
|
+
/**
|
|
56
|
+
* Whether the Platform connection is up right now.
|
|
57
|
+
*
|
|
58
|
+
* `mode` alone cannot answer this: it says a Platform is configured, not that
|
|
59
|
+
* one is reachable, and the two differ during exactly the outage that leaves
|
|
60
|
+
* the directory below frozen. Read per request rather than captured once, so
|
|
61
|
+
* a reconnect is reflected without re-mounting the route.
|
|
62
|
+
*
|
|
63
|
+
* Absent on an independent orchestrator, which has no connection to report.
|
|
64
|
+
*/
|
|
65
|
+
platformConnected?: () => boolean;
|
|
55
66
|
/**
|
|
56
67
|
* Audit sink for the PATCH, written inside the policy transaction.
|
|
57
68
|
*
|
package/dist/routes/admin.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ import type { AuditLogger } from '../secrets/audit-logger.js';
|
|
|
16
16
|
import type { AgentTokenStore } from '../agent/token-store.js';
|
|
17
17
|
import type { AgentRegistry } from '../agent/registry.js';
|
|
18
18
|
import type { SharedConfigStore } from '../config/shared-store.js';
|
|
19
|
+
import type { ResolvedMasterKeys } from '../secrets/config.js';
|
|
19
20
|
import { type HeldRunReleaseWiring } from './admin-held-runs.js';
|
|
20
21
|
import type { DrainController } from '../drain/drain-controller.js';
|
|
21
22
|
import type { SourceStore } from '../sources/source-store.js';
|
|
@@ -31,6 +32,12 @@ import type { AccessLogWriter } from '../audit/access-log.js';
|
|
|
31
32
|
*/
|
|
32
33
|
export interface AdminRouteDeps {
|
|
33
34
|
tokenManager: TokenManager;
|
|
35
|
+
/**
|
|
36
|
+
* Connection string handed to the DB migration routes, which run schema work
|
|
37
|
+
* on their own timeout-free pool. Optional so WS-only / test admins can omit
|
|
38
|
+
* it; omitting it makes those routes fall back to the shared pool.
|
|
39
|
+
*/
|
|
40
|
+
databaseUrl?: string;
|
|
34
41
|
/**
|
|
35
42
|
* Orchestrator operating mode. Gates mode-specific admin behavior — today the
|
|
36
43
|
* `observed`-mode refusal of GitHub-App source creation (those sources are
|
|
@@ -38,6 +45,14 @@ export interface AdminRouteDeps {
|
|
|
38
45
|
* relay). Optional so WS-only / test admins can omit it.
|
|
39
46
|
*/
|
|
40
47
|
mode?: OrchestratorMode;
|
|
48
|
+
/**
|
|
49
|
+
* Whether the Platform connection is up right now, read per request.
|
|
50
|
+
*
|
|
51
|
+
* Reported by the trust-policy directory route so an operator can tell a
|
|
52
|
+
* directory that is merely old from one that cannot refresh at all. Absent on
|
|
53
|
+
* an independent orchestrator and on a test admin, which have no connection.
|
|
54
|
+
*/
|
|
55
|
+
platformConnected?: () => boolean;
|
|
41
56
|
rbac: RbacEnforcer;
|
|
42
57
|
secretStore: PgSecretStore;
|
|
43
58
|
auditLogger: AuditLogger;
|
|
@@ -127,6 +142,16 @@ export interface AdminRouteDeps {
|
|
|
127
142
|
* response reports `reEncryptedConfigs: 0`.
|
|
128
143
|
*/
|
|
129
144
|
sharedStore?: SharedConfigStore;
|
|
145
|
+
/**
|
|
146
|
+
* Optional -- the resolved master key, for `POST /api/v1/admin/rotate-key`.
|
|
147
|
+
*
|
|
148
|
+
* When set, `rotate-key` additionally sweeps the four remaining
|
|
149
|
+
* master-key-wrapped stores: `orchestrator_signing_keys`,
|
|
150
|
+
* `dashboard_encryption_keys`, `run_ephemeral_keys` and `run_secret_outputs`.
|
|
151
|
+
* When unset the secrets subsystem is off and there is nothing wrapped to
|
|
152
|
+
* sweep, so those counts report 0.
|
|
153
|
+
*/
|
|
154
|
+
masterKeys?: ResolvedMasterKeys | null;
|
|
130
155
|
/**
|
|
131
156
|
* Optional -- attribution writer for routes that emit an `access_log`
|
|
132
157
|
* row directly (today: org-settings dashboard-write policy flips). When
|
|
@@ -29,6 +29,12 @@ export interface BackendFactoryContext {
|
|
|
29
29
|
tokenStore?: AgentTokenStore;
|
|
30
30
|
/** True when `config.agentAuth === 'token'`; gates token injection for the three local backends. */
|
|
31
31
|
injectAgentToken: boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Host services the orchestrator itself directed agents at, as `hostAccess`
|
|
34
|
+
* entries — today its object-storage endpoint. Resolved by the startup host,
|
|
35
|
+
* which is where `AppConfig` lives.
|
|
36
|
+
*/
|
|
37
|
+
hostServices?: string[];
|
|
32
38
|
tokenTtlMs?: number;
|
|
33
39
|
/** Live per-spawn resolver for the fleet-wide agent-token TTL. */
|
|
34
40
|
tokenTtlProvider: () => Promise<number>;
|
|
@@ -42,6 +48,16 @@ export interface BackendFactoryContext {
|
|
|
42
48
|
stateStore?: ScalerStateStore;
|
|
43
49
|
/** Absent ⇒ the `event` type is unsupported on this host (worker mode). */
|
|
44
50
|
eventEmitterProvider?: () => ScalerEventEmitterLike;
|
|
51
|
+
/**
|
|
52
|
+
* Whether an agent id is currently registered on this host.
|
|
53
|
+
*
|
|
54
|
+
* The container backend's orphan sweep uses it to tell a running agent that
|
|
55
|
+
* is doing work from one whose agent never registered. Absent ⇒ the sweep
|
|
56
|
+
* assumes every running container is registered, which is the fail-safe
|
|
57
|
+
* direction: it leaves an orphan for the next pass rather than killing a
|
|
58
|
+
* live agent.
|
|
59
|
+
*/
|
|
60
|
+
isAgentRegistered?: (agentId: string) => boolean;
|
|
45
61
|
logger: ReturnType<typeof createLogger>;
|
|
46
62
|
}
|
|
47
63
|
/** Tool requirements for a set of entries, for `validateRequiredTools` from `@kici-dev/shared`. */
|
|
@@ -17,6 +17,12 @@ export interface BareMetalScalerBackendOptions {
|
|
|
17
17
|
maxAgents: number;
|
|
18
18
|
/** Default resource limits applied when label set has none */
|
|
19
19
|
defaultResources?: ResourceRequest;
|
|
20
|
+
/**
|
|
21
|
+
* Extra `hostAccess` entries the orchestrator itself directed agents at —
|
|
22
|
+
* today its object-storage endpoint. Folded into the agent default for the
|
|
23
|
+
* container mode, which is the only mode with a network boundary.
|
|
24
|
+
*/
|
|
25
|
+
hostServices?: string[];
|
|
20
26
|
/** Token store for creating ephemeral agent auth tokens. Optional -- when undefined, no token is injected. */
|
|
21
27
|
tokenStore?: AgentTokenStore;
|
|
22
28
|
/** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
|
|
@@ -56,11 +62,23 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
|
|
|
56
62
|
private readonly tokenTtlProvider?;
|
|
57
63
|
private readonly roles;
|
|
58
64
|
private readonly enforceCgroups;
|
|
65
|
+
/** Host services the orchestrator directed agents at, as `hostAccess` entries. */
|
|
66
|
+
private readonly hostServices?;
|
|
59
67
|
/** Tracks all managed agent processes */
|
|
60
68
|
private readonly agents;
|
|
61
69
|
/** LogCapture instances for each managed agent (keyed by ManagedAgent.id) */
|
|
62
70
|
private readonly logCaptures;
|
|
71
|
+
/** Container-mode agents' IPs on the isolated network, for nftables cleanup. */
|
|
72
|
+
private readonly containerIps;
|
|
63
73
|
constructor(options: BareMetalScalerBackendOptions);
|
|
74
|
+
/**
|
|
75
|
+
* Warn about label sets whose `networkPolicy` this backend cannot enforce.
|
|
76
|
+
*
|
|
77
|
+
* Only process-mode label sets qualify. A label set that declares an `image`
|
|
78
|
+
* and no `binaryPath` spawns its agent as a container on the isolated agent
|
|
79
|
+
* network, where the policy IS applied — warning about it tells an operator
|
|
80
|
+
* their working configuration does nothing, which invites them to remove it.
|
|
81
|
+
*/
|
|
64
82
|
private warnNetworkPolicy;
|
|
65
83
|
/**
|
|
66
84
|
* Declare required tools for a bare-metal scaler entry.
|
|
@@ -99,14 +117,14 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
|
|
|
99
117
|
*
|
|
100
118
|
* `destroy()` opens with an in-memory map lookup, so an orchestrator restart
|
|
101
119
|
* makes it a silent no-op while the container keeps running. The container
|
|
102
|
-
* backend's own startup sweep does not cover this one either: it
|
|
103
|
-
* `kici-
|
|
104
|
-
* configured,
|
|
120
|
+
* backend's own startup sweep does not cover this one either: it reaps only
|
|
121
|
+
* containers stamped with its OWN `kici-scaler-name`, and it runs only when a
|
|
122
|
+
* container scaler is configured, which a bare-metal-only deployment has not.
|
|
105
123
|
*
|
|
106
124
|
* The host-local evidence is the container itself: `spawnContainerAgent` stamps
|
|
107
125
|
* `kici-agent-id` and `kici-scaler-name` onto it, and a container listing only
|
|
108
126
|
* ever names containers on this host. So a coordinator cannot reach a peer's
|
|
109
|
-
* compute through this path — a wrongly-routed agent id
|
|
127
|
+
* compute through this path — a wrongly-routed agent id matches nothing.
|
|
110
128
|
*
|
|
111
129
|
* A process-mode agent is NOT reclaimed here. Its PID lives in the in-memory
|
|
112
130
|
* entry alone, so a restart leaves no durable, host-local artifact to key off,
|
package/dist/scaler/config.d.ts
CHANGED
|
@@ -23,12 +23,13 @@ export declare const DEFAULT_MAX_CONCURRENT_SPAWNS = 8;
|
|
|
23
23
|
export declare const networkPolicySchema: z.ZodOptional<z.ZodObject<{
|
|
24
24
|
allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
25
25
|
denyAll: z.ZodDefault<z.ZodBoolean>;
|
|
26
|
+
hostAccess: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
26
27
|
}, z.core.$strip>>;
|
|
27
28
|
/**
|
|
28
29
|
* Zod schema for a label-set configuration entry.
|
|
29
30
|
*/
|
|
30
31
|
export declare const labelSetConfigSchema: z.ZodObject<{
|
|
31
|
-
labels: z.ZodArray<z.ZodString
|
|
32
|
+
labels: z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<import("@kici-dev/engine").CanonicalLabel[], string[]>>;
|
|
32
33
|
image: z.ZodOptional<z.ZodString>;
|
|
33
34
|
imagePullPolicy: z.ZodDefault<z.ZodEnum<{
|
|
34
35
|
Always: "Always";
|
|
@@ -80,6 +81,7 @@ export declare const labelSetConfigSchema: z.ZodObject<{
|
|
|
80
81
|
networkPolicy: z.ZodOptional<z.ZodObject<{
|
|
81
82
|
allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
82
83
|
denyAll: z.ZodDefault<z.ZodBoolean>;
|
|
84
|
+
hostAccess: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
83
85
|
}, z.core.$strip>>;
|
|
84
86
|
rootfsPath: z.ZodOptional<z.ZodString>;
|
|
85
87
|
kernelPath: z.ZodOptional<z.ZodString>;
|
|
@@ -176,7 +178,7 @@ export declare const scalerFileSchema: z.ZodObject<{
|
|
|
176
178
|
maxAgents: z.ZodNumber;
|
|
177
179
|
maxConcurrentSpawns: z.ZodDefault<z.ZodNumber>;
|
|
178
180
|
labelSets: z.ZodArray<z.ZodObject<{
|
|
179
|
-
labels: z.ZodArray<z.ZodString
|
|
181
|
+
labels: z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<import("@kici-dev/engine").CanonicalLabel[], string[]>>;
|
|
180
182
|
image: z.ZodOptional<z.ZodString>;
|
|
181
183
|
imagePullPolicy: z.ZodDefault<z.ZodEnum<{
|
|
182
184
|
Always: "Always";
|
|
@@ -228,6 +230,7 @@ export declare const scalerFileSchema: z.ZodObject<{
|
|
|
228
230
|
networkPolicy: z.ZodOptional<z.ZodObject<{
|
|
229
231
|
allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
230
232
|
denyAll: z.ZodDefault<z.ZodBoolean>;
|
|
233
|
+
hostAccess: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
231
234
|
}, z.core.$strip>>;
|
|
232
235
|
rootfsPath: z.ZodOptional<z.ZodString>;
|
|
233
236
|
kernelPath: z.ZodOptional<z.ZodString>;
|
|
@@ -250,7 +253,7 @@ export declare const scalerFileSchema: z.ZodObject<{
|
|
|
250
253
|
size: z.ZodDefault<z.ZodNumber>;
|
|
251
254
|
idleTimeoutSeconds: z.ZodDefault<z.ZodNumber>;
|
|
252
255
|
}, z.core.$strip>>;
|
|
253
|
-
mandatoryLabels: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
256
|
+
mandatoryLabels: z.ZodPipe<z.ZodDefault<z.ZodArray<z.ZodString>>, z.ZodTransform<import("@kici-dev/engine").CanonicalLabel[], string[]>>;
|
|
254
257
|
platform: z.ZodOptional<z.ZodObject<{
|
|
255
258
|
os: z.ZodEnum<{
|
|
256
259
|
linux: "linux";
|