@kici-dev/orchestrator 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/dist/__test-helpers__/fake-scaler-state-store.d.ts +24 -0
  2. package/dist/__test-helpers__/mock-db-query.d.ts +130 -0
  3. package/dist/__test-helpers__/mock-db.d.ts +27 -0
  4. package/dist/agent/agent-version.d.ts +34 -0
  5. package/dist/agent/dispatcher.d.ts +151 -13
  6. package/dist/agent/host-roster.d.ts +37 -5
  7. package/dist/agent/registry.d.ts +2 -0
  8. package/dist/app.d.ts +56 -7
  9. package/dist/approvals/apply-decision.d.ts +67 -19
  10. package/dist/approvals/held-run-release-wiring.d.ts +45 -0
  11. package/dist/approvals/independent-wiring.d.ts +55 -0
  12. package/dist/approvals/resume-router.d.ts +9 -0
  13. package/dist/approvals/triggerer-subject.d.ts +37 -0
  14. package/dist/cache/dep-cache.d.ts +54 -10
  15. package/dist/cache/global-eval-round-cache.d.ts +95 -0
  16. package/dist/cache/index.d.ts +3 -0
  17. package/dist/cache/pending-global-evals.d.ts +42 -0
  18. package/dist/cache/pending-inits.d.ts +10 -0
  19. package/dist/cli/api-client.d.ts +18 -0
  20. package/dist/cli/commands/cluster-settings.d.ts +52 -3
  21. package/dist/cli/commands/diagnose.d.ts +6 -0
  22. package/dist/cli/commands/held-run.d.ts +58 -0
  23. package/dist/cli/commands/runs.d.ts +1 -0
  24. package/dist/cli/commands/trust-policy.d.ts +117 -12
  25. package/dist/cli/kici-admin.d.ts +10 -0
  26. package/dist/cli.js +2770 -938
  27. package/dist/cluster/cluster-identity.d.ts +3 -3
  28. package/dist/cluster/cluster-settings-reader.d.ts +59 -1
  29. package/dist/cluster/coordinator.d.ts +44 -0
  30. package/dist/cluster/index.d.ts +4 -0
  31. package/dist/cluster/peer-handler.d.ts +19 -0
  32. package/dist/cluster/peer-registry.d.ts +17 -0
  33. package/dist/cluster/plan-headroom-store.d.ts +26 -0
  34. package/dist/cluster/worker-eviction.d.ts +69 -0
  35. package/dist/cold-store/load-event-log-range.d.ts +10 -3
  36. package/dist/cold-store/load-secret-audit-log-range.d.ts +4 -3
  37. package/dist/cold-store/tables/secret-audit-log.d.ts +4 -3
  38. package/dist/config.d.ts +48 -21
  39. package/dist/content-requirements-cache.d.ts +55 -0
  40. package/dist/contexts/held-runs.d.ts +123 -16
  41. package/dist/contexts/protection/aggregate.d.ts +3 -2
  42. package/dist/contexts/protection/branch-gate.d.ts +15 -0
  43. package/dist/contexts/protection/concurrency-gate.d.ts +8 -2
  44. package/dist/contexts/protection/pipeline.d.ts +18 -0
  45. package/dist/contexts/release-queued-holds.d.ts +35 -0
  46. package/dist/dashboard/attestation-filters.d.ts +8 -0
  47. package/dist/dashboard/handler.d.ts +6 -24
  48. package/dist/db/migrations/109_cluster_settings_cache_knobs.d.ts +4 -0
  49. package/dist/db/migrations/110_cluster_settings_global_eval_knobs.d.ts +4 -0
  50. package/dist/db/migrations/111_cluster_settings_global_eval_wait.d.ts +4 -0
  51. package/dist/db/migrations/112_execution_runs_workflow_repo.d.ts +4 -0
  52. package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +30 -0
  53. package/dist/db/migrations/114_ingest_queue_claim.d.ts +4 -0
  54. package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +9 -0
  55. package/dist/db/migrations/116_org_plan_headroom.d.ts +4 -0
  56. package/dist/db/migrations/117_invoke_gate_proxy_jobs.d.ts +4 -0
  57. package/dist/db/migrations/118_invoke_gate_scheduling.d.ts +4 -0
  58. package/dist/db/migrations/119_scaler_ha_ownership.d.ts +4 -0
  59. package/dist/db/migrations/120_cluster_settings_scaler_reaper.d.ts +4 -0
  60. package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +19 -0
  61. package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +25 -0
  62. package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +24 -0
  63. package/dist/db/migrations/124_cluster_settings_provision_backoff.d.ts +4 -0
  64. package/dist/db/migrations/125_org_trust_directory.d.ts +30 -0
  65. package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +28 -0
  66. package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +26 -0
  67. package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +23 -0
  68. package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +24 -0
  69. package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +32 -0
  70. package/dist/db/types.d.ts +428 -12
  71. package/dist/entry-helpers.d.ts +28 -0
  72. package/dist/events/event-emitter.d.ts +22 -0
  73. package/dist/events/event-router.d.ts +87 -1
  74. package/dist/events/types.d.ts +2 -23
  75. package/dist/git/credential-broker.d.ts +94 -0
  76. package/dist/git/inherited-secret.d.ts +24 -0
  77. package/dist/git/installation-token.d.ts +38 -0
  78. package/dist/git/job-context.d.ts +13 -0
  79. package/dist/index.js +324 -84
  80. package/dist/metrics/agent-metrics-aggregator.d.ts +4 -4
  81. package/dist/metrics/prometheus.d.ts +170 -3
  82. package/dist/oidc/oidc-mint-registration.d.ts +7 -3
  83. package/dist/orchestrator-core.d.ts +91 -24
  84. package/dist/pipeline/content-filter.d.ts +71 -0
  85. package/dist/pipeline/direct-ingress-deps.d.ts +100 -0
  86. package/dist/pipeline/dispatch-matched-workflow.d.ts +521 -11
  87. package/dist/pipeline/global-eval-round.d.ts +307 -0
  88. package/dist/pipeline/internal-event-pipeline.d.ts +140 -0
  89. package/dist/pipeline/invoke-gate.d.ts +164 -0
  90. package/dist/pipeline/job-contexts.d.ts +16 -17
  91. package/dist/pipeline/needs-scheduler.d.ts +13 -0
  92. package/dist/pipeline/process-webhook.d.ts +126 -31
  93. package/dist/pipeline/processor.d.ts +282 -23
  94. package/dist/pipeline/rerun.d.ts +21 -0
  95. package/dist/pipeline/resume-workflow.d.ts +40 -7
  96. package/dist/pipeline/route-or-dispatch-jobs.d.ts +15 -0
  97. package/dist/pipeline/security-hold-check.d.ts +217 -0
  98. package/dist/pipeline/test-pipeline.d.ts +12 -0
  99. package/dist/pipeline/undispatched-hold-checks.d.ts +23 -0
  100. package/dist/pipeline/webhook-payload-store.d.ts +20 -0
  101. package/dist/policy/dashboard-write-policy-listener.d.ts +62 -0
  102. package/dist/policy/dashboard-write-policy.d.ts +14 -0
  103. package/dist/provenance/backfill-run.d.ts +10 -1
  104. package/dist/provider-registry.d.ts +47 -4
  105. package/dist/providers/github/check-status-poster.d.ts +33 -3
  106. package/dist/providers/github/commit-message.d.ts +20 -0
  107. package/dist/providers/github/file-contents.d.ts +40 -0
  108. package/dist/providers/github/index.d.ts +2 -2
  109. package/dist/providers/github/normalizer.d.ts +3 -2
  110. package/dist/providers/local/normalizer.d.ts +3 -5
  111. package/dist/providers/universal-git/config.d.ts +2 -0
  112. package/dist/providers/universal-git/index.d.ts +11 -2
  113. package/dist/providers/universal-git/normalizer.d.ts +10 -0
  114. package/dist/queue/cleanup.d.ts +7 -1
  115. package/dist/queue/job-queue.d.ts +112 -7
  116. package/dist/queue/terminalize-unroutable.d.ts +13 -0
  117. package/dist/registration/extractor.d.ts +3 -3
  118. package/dist/registration/registration-index.d.ts +7 -0
  119. package/dist/registration/registration-run-match.d.ts +47 -0
  120. package/dist/registration/registration-store.d.ts +22 -0
  121. package/dist/reporting/check-run-reporter.d.ts +146 -2
  122. package/dist/reporting/check-run-summary.d.ts +25 -1
  123. package/dist/reporting/execution-tracker.d.ts +324 -9
  124. package/dist/reporting/log-chunk-sink.d.ts +8 -5
  125. package/dist/reporting/run-aggregator.d.ts +4 -14
  126. package/dist/reporting/step-display-order.d.ts +43 -0
  127. package/dist/routes/admin-held-runs.d.ts +142 -0
  128. package/dist/routes/admin-org-settings.d.ts +5 -0
  129. package/dist/routes/admin-trust-policy.d.ts +34 -6
  130. package/dist/routes/admin.d.ts +15 -0
  131. package/dist/scaler/backend-factory.d.ts +55 -0
  132. package/dist/scaler/bare-metal-backend.d.ts +43 -5
  133. package/dist/scaler/claim-store.d.ts +119 -0
  134. package/dist/scaler/config.d.ts +4 -0
  135. package/dist/scaler/container-backend.d.ts +18 -4
  136. package/dist/scaler/container-routing.d.ts +23 -0
  137. package/dist/scaler/container-spawn.d.ts +28 -0
  138. package/dist/scaler/event-backend.d.ts +124 -0
  139. package/dist/scaler/event-provision-reaper.d.ts +350 -0
  140. package/dist/scaler/failure-tracker.d.ts +1 -1
  141. package/dist/scaler/firecracker-backend.d.ts +33 -6
  142. package/dist/scaler/index.d.ts +11 -1
  143. package/dist/scaler/label-matcher.d.ts +4 -3
  144. package/dist/scaler/manager.d.ts +905 -76
  145. package/dist/scaler/resolve-container-auth.d.ts +45 -0
  146. package/dist/scaler/scaler-events.d.ts +11 -0
  147. package/dist/scaler/scaler-state-store.d.ts +294 -10
  148. package/dist/scaler/types.d.ts +131 -13
  149. package/dist/scaler/warm-pool.d.ts +132 -29
  150. package/dist/security/comment-handler.d.ts +50 -11
  151. package/dist/security/global-workflow-policy.d.ts +52 -12
  152. package/dist/security/identity-link.d.ts +60 -0
  153. package/dist/security/lock-source.d.ts +11 -13
  154. package/dist/security/reduced-privilege-note.d.ts +59 -0
  155. package/dist/security/trust-directory-store.d.ts +233 -0
  156. package/dist/security/trust-policy-gate.d.ts +94 -59
  157. package/dist/security/trust-policy-store.d.ts +3 -0
  158. package/dist/security/trust-resolver.d.ts +14 -98
  159. package/dist/server.d.ts +11 -1
  160. package/dist/server.js +56329 -42234
  161. package/dist/stale-detector/gate-deadline-detector.d.ts +52 -0
  162. package/dist/stale-detector/stale-run-detector.d.ts +61 -3
  163. package/dist/standalone.js +37619 -23570
  164. package/dist/webhook/ingest-accept.d.ts +70 -0
  165. package/dist/webhook/ingest-overflow-buffer.d.ts +35 -4
  166. package/dist/webhook/ingest-overflow-replayer.d.ts +50 -6
  167. package/dist/worker/in-memory-job-queue.d.ts +1 -1
  168. package/dist/ws/agent-handler.d.ts +31 -4
  169. package/dist/ws/dashboard-context-handler.d.ts +28 -4
  170. package/dist/ws/dashboard-global-workflows-handler.d.ts +36 -9
  171. package/dist/ws/execution-status-frame.d.ts +32 -0
  172. package/dist/ws/failure-messages.d.ts +2 -0
  173. package/dist/ws/git-credential-relay.d.ts +39 -0
  174. package/dist/ws/oidc-token-relay.d.ts +6 -11
  175. package/dist/ws/platform-client.d.ts +68 -9
  176. package/dist/ws/test-relay-handlers.d.ts +35 -10
  177. package/installer-image-digests.json +3 -3
  178. package/package.json +24 -19
  179. package/sbom.spdx.json +868 -821
  180. package/dist/approvals/team-membership-lookup.d.ts +0 -13
  181. package/dist/pipeline/inline-eval.d.ts +0 -44
  182. package/dist/providers/github/contributor-resolver.d.ts +0 -30
  183. package/dist/security/contributor-cache.d.ts +0 -83
@@ -0,0 +1,70 @@
1
+ import type { WebhookInfo } from './handler.js';
2
+ import { WebhookIngestOutcome } from '../pipeline/process-webhook.js';
3
+ /**
4
+ * Seams the accept path needs. Each is a narrow function rather than the
5
+ * concrete collaborator so the sequencing below can be driven in a unit test
6
+ * without a database, an admission controller, or a pipeline.
7
+ */
8
+ export interface IngestAcceptDeps {
9
+ /**
10
+ * Reserve an admission slot. Resolves to a release function when admitted, to
11
+ * `null` when the controller shed, or to `undefined` when no controller is
12
+ * wired (tests / minimal wirings — no gate).
13
+ */
14
+ admit: () => Promise<(() => void) | null | undefined>;
15
+ /**
16
+ * Advisory duplicate probe. Non-claiming: the atomic claim stays inside the
17
+ * pipeline, where it has always been. See {@link acceptWebhookDelivery}.
18
+ */
19
+ isKnownDelivery: (deliveryId: string) => Promise<boolean>;
20
+ /** Durably store the delivery; resolves to the row id, or null at the cap. */
21
+ enqueue: (info: WebhookInfo) => Promise<number | null>;
22
+ /** Take the row's claim (`buffered` → `replaying`). False when someone else has it. */
23
+ claimRow: (rowId: number) => Promise<boolean>;
24
+ /** Mark the row done so the drain pass sweeps it. */
25
+ markProcessed: (rowId: number) => Promise<void>;
26
+ /** Hand the claim back on failure so the drain pass retries the delivery. */
27
+ releaseClaim: (rowId: number, reason: string) => Promise<boolean>;
28
+ /** Run the match-and-dispatch pipeline. */
29
+ runPipeline: (info: WebhookInfo) => Promise<WebhookIngestOutcome>;
30
+ /**
31
+ * Schedule the post-acknowledgement work. Defaults to a detached microtask;
32
+ * a test passes a collector so it can await the work deterministically.
33
+ */
34
+ schedule?: (work: () => Promise<void>) => void;
35
+ }
36
+ /**
37
+ * Accept an inbound delivery and run its pipeline afterwards.
38
+ *
39
+ * The acknowledgement a caller receives means **the delivery is durably
40
+ * queued**, not that it was matched or dispatched. That is the whole point: a
41
+ * provider's delivery attempt times out in seconds (GitHub's is 10), while one
42
+ * matched workflow's build phase alone may legitimately take ten minutes, so a
43
+ * response that waited for the pipeline turned a slow build into a failed
44
+ * delivery for work that usually succeeded.
45
+ *
46
+ * Ordering is load-bearing and each step earns its place:
47
+ *
48
+ * 1. **Admit.** A shed delivery never reaches the queue, so admission stays the
49
+ * first gate and a saturated orchestrator still answers 429 immediately.
50
+ * 2. **Probe for a duplicate.** Advisory only — a redelivery the orchestrator
51
+ * already knows about is reported as one without a row being written. The
52
+ * *authoritative* claim is still the pipeline's atomic
53
+ * `INSERT … ON CONFLICT`, so a genuine cross-instance race is arbitrated
54
+ * exactly where it always was; the loser simply learns after acknowledging.
55
+ * 3. **Enqueue durably.** Nothing is acknowledged before this row exists. At the
56
+ * row cap the delivery is shed rather than acknowledged — answering 202 for
57
+ * a delivery we did not store would turn a hang into silent data loss, which
58
+ * is strictly worse than the hang this change removes.
59
+ * 4. **Claim the row**, then acknowledge, then run the pipeline detached. The
60
+ * claim is what stops the drain pass re-injecting a delivery already in
61
+ * flight; releasing it on failure is what makes the row a retry rather than
62
+ * a tombstone. The admission slot is held across the pipeline, so in-flight
63
+ * pipeline concurrency stays bounded exactly as it was when the response
64
+ * waited for it.
65
+ *
66
+ * Returns `queued` on the acknowledge path, or `duplicate` / `shed` for the two
67
+ * pre-queue exits.
68
+ */
69
+ export declare function acceptWebhookDelivery(info: WebhookInfo, deps: IngestAcceptDeps): Promise<WebhookIngestOutcome>;
70
+ //# sourceMappingURL=ingest-accept.d.ts.map
@@ -7,10 +7,22 @@ export interface IngestOverflowBufferDeps {
7
7
  maxRows: number;
8
8
  }
9
9
  /**
10
- * Capture side of the durable overflow buffer. On a shed, the ingest path calls
11
- * {@link IngestOverflowBuffer.capture} additively (the 429 still stands). At the
12
- * row cap, capture drops the delivery from the buffer (never unbounded rows) —
13
- * the lossy fallback a retrying sender still covers.
10
+ * Capture side of the durable ingest queue. Two feeders write rows here.
11
+ *
12
+ * The **accept** path (HTTP direct ingress) enqueues every admitted delivery
13
+ * before the route answers 202: the row is what makes "durably queued" true, so
14
+ * a worker that dies mid-pipeline leaves recoverable work behind rather than a
15
+ * delivery the sender believes was accepted.
16
+ *
17
+ * The **shed** path enqueues additively when the admission controller rejects a
18
+ * delivery (the 429 still stands), so capacity recovering is enough to get the
19
+ * delivery processed without the sender redelivering.
20
+ *
21
+ * At the row cap, an enqueue fails rather than growing the table without bound.
22
+ * The two feeders differ in what that means: a shed delivery is dropped (the
23
+ * lossy fallback a retrying sender still covers), while the accept path must
24
+ * NOT answer 202 for a delivery it did not store — it sheds instead, so the
25
+ * sender learns the delivery was refused.
14
26
  */
15
27
  export declare class IngestOverflowBuffer {
16
28
  private readonly db;
@@ -18,10 +30,29 @@ export declare class IngestOverflowBuffer {
18
30
  constructor(deps: IngestOverflowBufferDeps);
19
31
  /** Current buffered-row depth. */
20
32
  currentDepth(): Promise<number>;
33
+ /**
34
+ * Persist a delivery as `buffered` and return its row id, or null when the
35
+ * buffer is at cap. Callers that must not acknowledge an unstored delivery
36
+ * check for null; the shed path treats null as a drop.
37
+ */
38
+ enqueue(d: OverflowDelivery): Promise<number | null>;
21
39
  /**
22
40
  * Persist a shed delivery. Returns true when a `buffered` row was inserted,
23
41
  * false when the buffer is at cap (delivery dropped, `cap_full` metric bumped).
24
42
  */
25
43
  capture(d: OverflowDelivery): Promise<boolean>;
44
+ /**
45
+ * Claim a specific row for processing: `buffered` → `replaying`, stamping the
46
+ * claim clock. Returns false when someone else already owns it — a drain pass
47
+ * on another instance, or a reclaim that ran while this caller was scheduling.
48
+ * The conditional update is the arbiter, so two workers can never both own a
49
+ * row and dispatch its delivery twice.
50
+ */
51
+ claimRow(id: number): Promise<boolean>;
52
+ /**
53
+ * Mark a claimed row done. The row is swept by the drain pass rather than
54
+ * deleted here, so the sweep stays the single deletion site.
55
+ */
56
+ markProcessed(id: number): Promise<void>;
26
57
  }
27
58
  //# sourceMappingURL=ingest-overflow-buffer.d.ts.map
@@ -1,5 +1,6 @@
1
1
  import type { Kysely } from 'kysely';
2
2
  import type { Database } from '../db/types.js';
3
+ import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
3
4
  import { WebhookIngestOutcome } from '../pipeline/process-webhook.js';
4
5
  import { type OverflowDelivery } from './ingest-overflow-types.js';
5
6
  export type ReinjectFn = (d: OverflowDelivery) => Promise<WebhookIngestOutcome>;
@@ -11,14 +12,29 @@ export interface IngestOverflowReplayerDeps {
11
12
  intervalMs: number;
12
13
  batchSize: number;
13
14
  maxAttempts: number;
15
+ /**
16
+ * How long a `replaying` claim may stand before it is reclaimed. Resolved per
17
+ * pass so a fleet-wide `cluster_settings` override takes effect without a
18
+ * restart; the number passed here is the configured cluster default.
19
+ */
20
+ claimTimeoutMs: number;
21
+ /** Fleet-wide override reader for {@link IngestOverflowReplayerDeps.claimTimeoutMs}. */
22
+ clusterSettings?: ClusterSettingsReader;
14
23
  }
15
24
  /**
16
- * Background drain for the durable overflow buffer. A pass runs only while the
17
- * admission controller is NOT shedding (replaying into a still-overloaded
18
- * orchestrator just re-sheds), claims the oldest `buffered` rows FIFO up to a
19
- * bounded batch, and re-injects each through the admission-gated ingest path.
20
- * A re-shed or error reverts the row to `buffered` (never lost); past the
21
- * max-attempts ceiling it goes `failed`. Successful rows are swept.
25
+ * Background drain for the durable ingest queue. Each pass first reclaims rows
26
+ * whose `replaying` claim went stale a worker killed mid-pipeline releases
27
+ * nothing, so without this its delivery would sit claimed forever and the
28
+ * durable row it acknowledged would never be worth anything. It then claims the
29
+ * oldest `buffered` rows FIFO up to a bounded batch and re-injects each through
30
+ * the admission-gated ingest path. A re-shed or error reverts the row to
31
+ * `buffered` (never lost); past the max-attempts ceiling it goes `failed`.
32
+ * Successful rows are swept.
33
+ *
34
+ * Re-injection runs only while the admission controller is NOT shedding
35
+ * (replaying into a still-overloaded orchestrator just re-sheds). Reclaiming is
36
+ * not gated that way: a stranded claim is stranded regardless of load, and the
37
+ * row it frees simply waits in `buffered` until capacity returns.
22
38
  */
23
39
  export declare class IngestOverflowReplayer {
24
40
  private readonly db;
@@ -26,6 +42,8 @@ export declare class IngestOverflowReplayer {
26
42
  private readonly intervalMs;
27
43
  private readonly batchSize;
28
44
  private readonly maxAttempts;
45
+ private readonly claimTimeoutMs;
46
+ private readonly clusterSettings;
29
47
  private reinjectDirect;
30
48
  private reinjectRelay;
31
49
  private timer;
@@ -38,10 +56,36 @@ export declare class IngestOverflowReplayer {
38
56
  stop(): void;
39
57
  /** One drain pass. Test-drivable. */
40
58
  runPass(): Promise<void>;
59
+ /**
60
+ * Revert `replaying` rows whose claim went stale to `buffered`, counting the
61
+ * abandoned attempt so a row that keeps stranding eventually goes `failed`
62
+ * instead of looping forever.
63
+ *
64
+ * A row claimed before the claim clock shipped has a null `claimed_at`; it is
65
+ * aged off `captured_at` instead, which is a safe over-estimate of how long
66
+ * it has been claimed.
67
+ */
68
+ private reclaimStaleClaims;
69
+ /**
70
+ * Release a claim: revert the row to `buffered` so the drain retries it, or
71
+ * mark it `failed` once the attempt ceiling is hit. Returns false when the
72
+ * row is gone or is no longer `replaying` — another worker owns it now, so
73
+ * this caller must not touch it.
74
+ *
75
+ * Public because the accept path's background worker holds a claim it did not
76
+ * take through {@link claimBatch} and must release it the same way on failure.
77
+ */
78
+ releaseClaim(id: number, reason: string): Promise<boolean>;
41
79
  /** Select the oldest buffered rows and claim each via a conditional update. */
42
80
  private claimBatch;
43
81
  private toDelivery;
44
82
  private replayOne;
83
+ /**
84
+ * Move a claimed row back to `buffered` (or to `failed` at the attempt
85
+ * ceiling). Every update is conditional on the row still being `replaying`,
86
+ * so a caller whose claim was reclaimed underneath it cannot yank a row a
87
+ * different worker now owns. Returns whether this call moved the row.
88
+ */
45
89
  private revertOrFail;
46
90
  private sweepReplayed;
47
91
  private refreshDepthGauge;
@@ -35,7 +35,7 @@ export declare class InMemoryJobQueue {
35
35
  * label in the gate set. Empty (default) is a no-op for static / non-scaler
36
36
  * agents — the gate trivially passes.
37
37
  */
38
- dequeueForLabels(agentLabels: string[], agentMandatoryLabels?: string[]): Promise<QueuedJob | null>;
38
+ dequeueForLabels(agentLabels: string[], agentMandatoryLabels?: string[], _agentId?: string, canServe?: (job: QueuedJob) => boolean): Promise<QueuedJob | null>;
39
39
  /**
40
40
  * Dequeue a specific pending job by id, validating that the agent's labels
41
41
  * still satisfy the job's runsOn / excludeLabels and the agent's
@@ -13,7 +13,11 @@
13
13
  * 2. pendingRegistration (10s): After auth.success, agent must send agent.register.
14
14
  * 3. registered: Agent is fully connected and can exchange messages.
15
15
  *
16
- * When agentAuthMode === 'none', the auth phase is skipped (legacy behavior).
16
+ * When agentAuthMode === 'none', the auth phase is skipped and the connection
17
+ * starts in pendingRegistration. An agent that happens to carry a token still
18
+ * opens with auth.request — nothing on the wire tells it the mode — so that
19
+ * frame is accepted there and answered with auth.success without reading the
20
+ * token, and the agent then registers as normal.
17
21
  */
18
22
  import type { WSEvents } from 'hono/ws';
19
23
  import { LogStream } from '@kici-dev/engine';
@@ -31,6 +35,7 @@ import type { DispatchCacheRefTracker } from '../cache/dispatch-cache-ref-tracke
31
35
  import type { PendingBuildTracker } from '../cache/pending-builds.js';
32
36
  import type { PendingInitTracker } from '../cache/pending-inits.js';
33
37
  import type { PendingDynamicTracker } from '../cache/pending-dynamics.js';
38
+ import type { PendingGlobalEvalTracker } from '../cache/pending-global-evals.js';
34
39
  import type { CacheStorage } from '../storage/types.js';
35
40
  import type { AgentMetricsAggregator } from '../metrics/agent-metrics-aggregator.js';
36
41
  import { type AgentApiRegistry } from './agent-api-registry.js';
@@ -123,12 +128,16 @@ export interface AgentWsHandlerDeps {
123
128
  * AgentRegistry so subsequent label matches (queue drain + eager
124
129
  * dispatch) apply the same gate the scaler-side selector applied.
125
130
  *
126
- * Returns `null` for static agents not managed by any scaler.
131
+ * Resolves to `null` for static agents not managed by any scaler.
132
+ *
133
+ * Asynchronous because the agent may reach an instance that did not spawn
134
+ * it: that instance adopts the agent from the shared spawn record, which is
135
+ * a database round trip.
127
136
  */
128
- onScalerAgentRegistered?: (agentId: string, labels: string[]) => {
137
+ onScalerAgentRegistered?: (agentId: string, labels: string[]) => Promise<{
129
138
  boundJobId?: string;
130
139
  mandatoryLabels: string[];
131
- } | null;
140
+ } | null>;
132
141
  /** Optional callback when an agent disconnects (for scaler lifecycle). */
133
142
  onScalerAgentDisconnected?: (agentId: string) => void;
134
143
  /** Optional callback when an agent completes a job (for scaler lifecycle). */
@@ -169,6 +178,22 @@ export interface AgentWsHandlerDeps {
169
178
  deliveryId?: string;
170
179
  error?: string;
171
180
  }>;
181
+ /**
182
+ * Optional callback: a provisioning workflow's agent exchanges a single-use
183
+ * claim code (from a `kici.scaler.scale-up` event) for freshly minted
184
+ * ephemeral agent credentials. Delegates to the event scaler backend's claim
185
+ * store. Returns the credentials or an author-actionable error; the token is
186
+ * carried only in the response, never logged.
187
+ */
188
+ onClaimCredentials?: (agentId: string, claimCode: string) => Promise<{
189
+ credentials?: {
190
+ agentToken: string;
191
+ agentId: string;
192
+ orchestratorUrl: string;
193
+ labels: string[];
194
+ };
195
+ error?: string;
196
+ }>;
172
197
  /** Optional callback when agent sends a run.event for infrastructure lifecycle tracking. */
173
198
  onRunEvent?: (agentId: string, msg: {
174
199
  runId: string;
@@ -267,6 +292,8 @@ export interface AgentWsHandlerDeps {
267
292
  pendingInits?: PendingInitTracker;
268
293
  /** Optional pending dynamic tracker for cleanup on agent disconnect. */
269
294
  pendingDynamics?: PendingDynamicTracker;
295
+ /** Optional pending global-eval-round tracker for cleanup on agent disconnect. */
296
+ pendingGlobalEvals?: PendingGlobalEvalTracker;
270
297
  /** Optional callback when agent sends encrypted secret outputs on job success. */
271
298
  onSecretOutputs?: (runId: string, jobId: string, secretOutputs: Record<string, {
272
299
  agentPublicKey: string;
@@ -13,11 +13,12 @@
13
13
  import { type Kysely } from 'kysely';
14
14
  import type { HeldRunStore, ReleaseSignal } from '../contexts/held-runs.js';
15
15
  import type { TeamMembershipLookup } from '../approvals/approval-resolver.js';
16
+ import { type ApplyDecisionDeps } from '../approvals/apply-decision.js';
16
17
  import type { DashboardPlatformToOrchMessage } from '@kici-dev/engine';
17
18
  import type { ContextStore } from '../contexts/context-store.js';
18
19
  import type { VariableStore } from '../contexts/variable-store.js';
19
20
  import type { BindingStore } from '../contexts/binding-store.js';
20
- import type { Database } from '../db/types.js';
21
+ import type { Database, HeldRun } from '../db/types.js';
21
22
  import type { AccessLogWriter } from '../audit/access-log.js';
22
23
  import { type ScopedSecretStore } from '../secrets/scope-routing.js';
23
24
  import type { ResolvedDashboardEncryptionKey } from '../secrets/dashboard-encryption-key.js';
@@ -74,10 +75,22 @@ export interface ApprovalHandlerDeps {
74
75
  resumeStep?: (signal: ReleaseSignal) => Promise<void>;
75
76
  /** Notify a waiting agent that a step hold was rejected (step scope). */
76
77
  rejectStep?: (heldRunId: string, reason?: string) => Promise<void> | void;
77
- /** Resume a released workflow install-gate hold (workflow scope). */
78
+ /** Resume a released workflow-scoped hold (install gate or trust policy). */
78
79
  resumeWorkflow?: (signal: ReleaseSignal) => Promise<void>;
79
- /** Cancel a rejected workflow install-gate hold (workflow scope). */
80
- rejectWorkflow?: (runId: string) => Promise<void>;
80
+ /**
81
+ * Cancel a rejected workflow-scoped hold (install gate, trust policy, or an
82
+ * SDK workflow-level `requireApproval`), recording the rejecter's own reason
83
+ * as the run's cancellation reason. The hold row tells those apart, and the
84
+ * resolved boolean says whether a terminal `KiCI Security` check was written
85
+ * — see `onWorkflowReject` on `ApplyDecisionDeps`.
86
+ */
87
+ rejectWorkflow?: (hold: HeldRun, reason?: string) => Promise<boolean>;
88
+ /**
89
+ * Terminalize the `KiCI Security` check of any hold this handler ends that
90
+ * `rejectWorkflow` did not report on — see `settleSecurityCheck` on
91
+ * `ApplyDecisionDeps`.
92
+ */
93
+ settleSecurityCheck?: ApplyDecisionDeps['settleSecurityCheck'];
81
94
  }
82
95
  /**
83
96
  * Handler for all dashboard context and held run WS messages.
@@ -162,6 +175,12 @@ export declare class DashboardContextHandler {
162
175
  /**
163
176
  * Resolve the actor's Keycloak sub for approval attribution. Only `user` and
164
177
  * `platform_operator` actors carry a `sub`; others fall back to a stable id.
178
+ *
179
+ * The `service:` namespace comes from {@link adminActorSub} rather than a
180
+ * literal here: `applyDecision`'s self-approval gate compares this value
181
+ * against {@link triggererSubjectFor}'s output, so the two renderings of one
182
+ * principal have to be produced by the same code or the gate compares strings
183
+ * that can never be equal.
165
184
  */
166
185
  private actorSub;
167
186
  /** Read org_settings.allow_self_approval (default true). */
@@ -170,6 +189,11 @@ export declare class DashboardContextHandler {
170
189
  * Route an approve/reject through the shared `applyDecision` applier:
171
190
  * eligibility check, real attribution, multi-clause accumulation, and the
172
191
  * resume-after-approval re-dispatch. Sends the matching response message.
192
+ *
193
+ * The response goes out as soon as the decision is durable, not when its
194
+ * consequence finishes — see `ApplyDecisionResult.consequence`. The audit
195
+ * entry is attached to that consequence instead, so there is still exactly one
196
+ * per decision and it still carries the real outcome.
173
197
  */
174
198
  private applyApprovalDecision;
175
199
  private handleHeldRunApprove;
@@ -2,10 +2,12 @@
2
2
  * Dashboard global-workflows handler for the orchestrator.
3
3
  *
4
4
  * Responds to `dashboard.global-workflows.*` WS messages from Platform by
5
- * reading or upserting the `org_settings` row keyed by `customer_id`. The
6
- * row holds the org-level global-workflow policy: master enable, plus the
7
- * three repo-pattern lists (allow / deny / elevate). Each list entry is a
8
- * `{routingKey?, pattern}` object that may optionally pin to one source.
5
+ * reading or upserting the `org_settings` row keyed by `customer_id`. The row
6
+ * holds the three per-org repo-pattern lists (allow / deny / elevate); each
7
+ * entry is a `{routingKey?, pattern}` object that may optionally pin to one
8
+ * source. The master enable switch is fleet-wide
9
+ * (`cluster_settings.global_workflows_enabled`) and read-only here — projected
10
+ * onto the response as `enabled`, never written from this handler.
9
11
  *
10
12
  * The handler binds a single `customerId` at construction time (via the
11
13
  * sources / generic_webhook_sources lookup at server startup) and updates
@@ -15,12 +17,17 @@ import { type Kysely } from 'kysely';
15
17
  import type { GlobalWorkflowsGetRequest, GlobalWorkflowsUpdateRequest, GlobalWorkflowSettings, RepoPatternEntry } from '@kici-dev/engine/protocol/dashboard-global-workflows';
16
18
  import type { Database, OrgSettings } from '../db/types.js';
17
19
  import type { AccessLogWriter } from '../audit/access-log.js';
20
+ import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
18
21
  interface DashboardGlobalWorkflowsHandlerDeps {
19
22
  /** Customer / org identifier — primary key on org_settings. */
20
23
  customerId: string;
21
24
  /** Send a response message back to Platform over the WS connection. */
22
25
  send: (msg: unknown) => void;
23
26
  db: Kysely<Database>;
27
+ /** Reads the fleet-wide master switch for the read-only `enabled` projection. */
28
+ clusterSettings: ClusterSettingsReader;
29
+ /** Applies when the cluster column is NULL — `config.globalWorkflowsEnabled`. */
30
+ globalWorkflowsEnabledDefault: boolean;
24
31
  /** Access log writer — records one row per read / mutation with actor attribution. */
25
32
  accessLog?: AccessLogWriter;
26
33
  }
@@ -53,11 +60,30 @@ export declare class DashboardGlobalWorkflowsHandler {
53
60
  private handleGet;
54
61
  private handleUpdate;
55
62
  private readRow;
63
+ /**
64
+ * The effective fleet-wide master switch, for the read-only projection the
65
+ * dashboard renders as a status badge.
66
+ *
67
+ * Uses the shared 10s-cached reader — unlike the admin route, which reads
68
+ * uncached because an operator does set-then-show. A dashboard badge that is
69
+ * up to 10s stale is fine; a second uncached read on every dashboard poll is
70
+ * not.
71
+ *
72
+ * An unreadable row resolves to the configured default here rather than
73
+ * failing closed. This value only decides what the UI displays; the actual
74
+ * gate is `GlobalWorkflowPolicy`, which does fail closed.
75
+ */
76
+ private effectiveEnabled;
56
77
  private upsertRow;
78
+ /**
79
+ * Same envelope as `sendError`, for a request the client got wrong. Logged at
80
+ * warn level: bad input is not a server fault, and error-level rows here would
81
+ * pollute the error dashboards an operator watches.
82
+ */
83
+ private sendClientError;
57
84
  private sendError;
58
85
  }
59
86
  interface NormalizedPatch {
60
- enabled: boolean;
61
87
  allowedRepos: RepoPatternEntry[] | null;
62
88
  deniedRepos: RepoPatternEntry[] | null;
63
89
  elevatedRepos: RepoPatternEntry[] | null;
@@ -69,10 +95,11 @@ interface NormalizedPatch {
69
95
  */
70
96
  export declare function buildPatch(existing: OrgSettings | undefined, msg: GlobalWorkflowsUpdateRequest): NormalizedPatch;
71
97
  /**
72
- * Project a row into the public settings shape. Returns a defaulted "disabled"
73
- * settings object when the row does not yet exist, so callers always get a
74
- * renderable state.
98
+ * Project a row into the public settings shape. `enabled` is the effective
99
+ * fleet-wide master switch, supplied by the caller (read-only here). When the
100
+ * row does not yet exist, the three per-org lists project as null so callers
101
+ * always get a renderable state.
75
102
  */
76
- export declare function rowToSettings(customerId: string, row: OrgSettings | undefined): GlobalWorkflowSettings;
103
+ export declare function rowToSettings(customerId: string, row: OrgSettings | undefined, enabled: boolean): GlobalWorkflowSettings;
77
104
  export {};
78
105
  //# sourceMappingURL=dashboard-global-workflows-handler.d.ts.map
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Build the `execution.status` frame the orchestrator forwards to the Platform.
3
+ *
4
+ * Extracted from `server.ts`'s `onExecutionStatusChange` hook so the projection
5
+ * from an {@link ExecutionContext} onto the wire message is a pure function with
6
+ * its own tests. The hook itself stays a thin adapter: it supplies the message
7
+ * id and the clock and hands the result to the Platform client.
8
+ *
9
+ * Every field here is conditional on the context actually carrying it. That is
10
+ * load-bearing for `workflowRepoIdentifier` in particular: the tracker records
11
+ * it only when the repository that DEFINES the workflow differs from the
12
+ * repository the run acted on, so the frame must stay silent for a per-repository
13
+ * run or "present" stops marking a cross-repository global run downstream.
14
+ */
15
+ import type { ExecutionContext } from '../reporting/execution-tracker.js';
16
+ import type { ExecutionRunStatus, ExecutionStatus, InitFailure } from '@kici-dev/engine';
17
+ export interface ExecutionStatusFrameArgs {
18
+ messageId: string;
19
+ runId: string;
20
+ status: ExecutionRunStatus;
21
+ context: ExecutionContext;
22
+ jobCount: number;
23
+ startedAt: number;
24
+ timestamp: number;
25
+ completedAt?: number;
26
+ durationMs?: number;
27
+ failureReason?: string;
28
+ logBytes?: number;
29
+ initFailure?: InitFailure;
30
+ }
31
+ export declare function buildExecutionStatusFrame(args: ExecutionStatusFrameArgs): ExecutionStatus;
32
+ //# sourceMappingURL=execution-status-frame.d.ts.map
@@ -25,5 +25,7 @@ export declare const AgentWsInternalFailure: Readonly<{
25
25
  readonly eventEmitFailed: 'the orchestrator hit an internal error while emitting this event';
26
26
  /** An agent private-API handler threw for a reason other than its two deliberate rejections. */
27
27
  readonly agentApiFailed: 'the orchestrator hit an internal error while handling this API request';
28
+ /** The event-scaler claim store threw while minting ephemeral credentials. */
29
+ readonly scalerClaimFailed: 'the orchestrator hit an internal error while claiming scaler credentials';
28
30
  }>;
29
31
  //# sourceMappingURL=failure-messages.d.ts.map
@@ -0,0 +1,39 @@
1
+ /**
2
+ * `agent.api` handler for GIT_CREDENTIAL_REQUEST_METHOD.
3
+ *
4
+ * The agent's credential helper calls this on every git network operation, so
5
+ * this path is hot and must stay cheap: validate, authorize, delegate.
6
+ *
7
+ * Authorization has two layers. Job ownership comes first — an agent may only
8
+ * request credentials for a job it was dispatched, resolved from server truth
9
+ * via the dispatcher, never from the params. Then the repository fence, which
10
+ * applies to WRITE only: a read with the source credential may reach any
11
+ * repository inside the App installation's own selection (GitHub enforces that
12
+ * selection when the mint names the repository), so cloning a sibling repo
13
+ * needs no credential in workflow code. A write must additionally stay inside
14
+ * the organisation that owns the job's source repository.
15
+ *
16
+ * A workflow-supplied credential needs no fence of its own: it names secrets
17
+ * the job can already read with `ctx.secrets.get()`, so brokering it grants no
18
+ * capability the workflow did not already have — the secret ACL is the
19
+ * authorization.
20
+ */
21
+ import type { GitCredentialBroker } from '../git/credential-broker.js';
22
+ /** Per-job facts the handler needs. Resolved from server truth, never from params. */
23
+ export interface JobCredentialContext {
24
+ orgId: string;
25
+ /** The repository this job was dispatched for. */
26
+ sourceRepo: string;
27
+ }
28
+ export interface GitCredentialHandlerDeps {
29
+ broker: GitCredentialBroker;
30
+ /** The dispatcher's job-ownership resolver — the same one the OIDC relay uses. */
31
+ dispatcher: {
32
+ resolveOwnedJob(agentId: string, jobId: string): {
33
+ runId: string;
34
+ } | undefined;
35
+ };
36
+ jobContext: (runId: string, jobId: string) => Promise<JobCredentialContext | null>;
37
+ }
38
+ export declare function buildGitCredentialHandler(deps: GitCredentialHandlerDeps): (agentId: string, rawParams: Record<string, unknown>) => Promise<unknown>;
39
+ //# sourceMappingURL=git-credential-relay.d.ts.map
@@ -60,19 +60,14 @@ export interface OidcTokenHandlerDeps {
60
60
  };
61
61
  platformClient: MintPlatformClient;
62
62
  orchestratorId: string;
63
- /** Test-only fault-injection master switch (config.testMode). */
64
- testMode: boolean;
65
63
  /**
66
- * Test-only: force-defer the *initial* agent mint for any job requesting this
67
- * OIDC audience (config.testMintDeferAudience). Ignored unless `testMode`.
64
+ * Test-only fault-injection predicate over the requested OIDC `audience`,
65
+ * supplied only by the build-time test double. Returning `true` short-circuits
66
+ * the *initial* mint to a transient defer BEFORE the real Platform relay is
67
+ * contacted — it can only defer/reject, never forge or weaken a signature.
68
+ * Undefined (the shipped default) means no fault injection.
68
69
  */
69
- testMintDeferAudience?: string;
70
- /**
71
- * Test-only: force the *initial* agent mint to defer for this OIDC audience
72
- * (config.testMintRejectAudience) so a pending row is created, which the
73
- * retrier then terminally rejects. Ignored unless `testMode`.
74
- */
75
- testMintRejectAudience?: string;
70
+ initialMintFault?: (audience: string) => boolean;
76
71
  }
77
72
  /**
78
73
  * Build the agent.api handler for `OIDC_TOKEN_REQUEST_METHOD`. Validates the