@kici-dev/orchestrator 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/dist/agent/dispatcher.d.ts +10 -1
  2. package/dist/agent/host-roster.d.ts +8 -3
  3. package/dist/agent/registry.d.ts +60 -5
  4. package/dist/app.d.ts +10 -5
  5. package/dist/cache/dep-cache.d.ts +14 -6
  6. package/dist/cache/global-eval-round-cache.d.ts +1 -1
  7. package/dist/cache/index.d.ts +1 -1
  8. package/dist/cache/source-cache.d.ts +92 -15
  9. package/dist/cancel/cancel-run.d.ts +20 -2
  10. package/dist/cancel/sweep-stuck-cancelling.d.ts +30 -0
  11. package/dist/cli/api-client.d.ts +10 -0
  12. package/dist/cli/commands/agent-service/logs.d.ts +1 -1
  13. package/dist/cli/commands/agent-service/restart.d.ts +1 -1
  14. package/dist/cli/commands/agent-service/start.d.ts +1 -1
  15. package/dist/cli/commands/agent-service/status.d.ts +1 -1
  16. package/dist/cli/commands/agent-service/stop.d.ts +1 -1
  17. package/dist/cli/commands/agent-service/uninstall.d.ts +1 -1
  18. package/dist/cli/commands/agent.d.ts +25 -0
  19. package/dist/cli/commands/config.d.ts +16 -0
  20. package/dist/cli/commands/db-backup.d.ts +46 -0
  21. package/dist/cli/commands/held-run.d.ts +9 -0
  22. package/dist/cli/commands/orchestrator-service/drain.d.ts +2 -19
  23. package/dist/cli/commands/orchestrator-service/index.d.ts +9 -1
  24. package/dist/cli/commands/orchestrator-service/install-env.d.ts +43 -2
  25. package/dist/cli/commands/orchestrator-service/logs.d.ts +1 -1
  26. package/dist/cli/commands/orchestrator-service/restart.d.ts +1 -1
  27. package/dist/cli/commands/orchestrator-service/start.d.ts +1 -1
  28. package/dist/cli/commands/orchestrator-service/status.d.ts +17 -1
  29. package/dist/cli/commands/orchestrator-service/stop.d.ts +1 -1
  30. package/dist/cli/commands/orchestrator-service/uninstall.d.ts +1 -1
  31. package/dist/cli/commands/orchestrator-service/upgrade.d.ts +2 -1
  32. package/dist/cli/commands/rotate.d.ts +1 -1
  33. package/dist/cli/commands/shared/env-file-mode.d.ts +16 -0
  34. package/dist/cli/commands/shared/upgrade-hooks.d.ts +32 -0
  35. package/dist/cli/commands/shared/upgrade-safety.d.ts +81 -0
  36. package/dist/cli/commands/shared/versioned-upgrade.d.ts +67 -12
  37. package/dist/cli/commands/source.d.ts +1 -0
  38. package/dist/cli/commands/token.d.ts +4 -0
  39. package/dist/cli/commands/trust-policy.d.ts +57 -1
  40. package/dist/cli/join.d.ts +1 -1
  41. package/dist/cli/service/backup-timer.d.ts +108 -0
  42. package/dist/cli/service/compose-path.d.ts +11 -0
  43. package/dist/cli/service/compose.d.ts +22 -0
  44. package/dist/cli/service/deploy-env.d.ts +10 -11
  45. package/dist/cli/service/env-assignment.d.ts +24 -0
  46. package/dist/cli/service/index.d.ts +2 -2
  47. package/dist/cli/service/instance/resolve.d.ts +71 -7
  48. package/dist/cli/service/instance/types.d.ts +15 -0
  49. package/dist/cli/service/launchd.d.ts +7 -0
  50. package/dist/cli/service/systemd.d.ts +1 -0
  51. package/dist/cli/service/types.d.ts +36 -0
  52. package/dist/cli/service/windows.d.ts +16 -0
  53. package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -3
  54. package/dist/cli/wizard/prompts.d.ts +7 -0
  55. package/dist/cli.js +12525 -9298
  56. package/dist/cluster/cluster-settings-reader.d.ts +2 -2
  57. package/dist/cluster/coordinator.d.ts +11 -0
  58. package/dist/cluster/instance-heartbeat.d.ts +96 -0
  59. package/dist/cluster/join-client.d.ts +55 -5
  60. package/dist/cluster/join-handler.d.ts +1 -0
  61. package/dist/cluster/orphan-recovery.d.ts +16 -0
  62. package/dist/cluster/peer-registry.d.ts +10 -0
  63. package/dist/concurrency/dispatch-next-queued.d.ts +1 -13
  64. package/dist/concurrency/index.d.ts +1 -2
  65. package/dist/concurrency/queue-manager.d.ts +63 -18
  66. package/dist/config/env-overlay.d.ts +3 -2
  67. package/dist/config.d.ts +24 -0
  68. package/dist/contexts/release-queued-holds.d.ts +23 -1
  69. package/dist/dashboard/attestation-filters.d.ts +2 -0
  70. package/dist/dashboard/handler.d.ts +22 -2
  71. package/dist/db/deferred-indexes.d.ts +72 -0
  72. package/dist/db/migration-test-harness.d.ts +22 -0
  73. package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +30 -0
  74. package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +23 -0
  75. package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +23 -0
  76. package/dist/db/migrations/134_dispatch_queue_ownership.d.ts +4 -0
  77. package/dist/db/migrations/135_cluster_settings_retention_days.d.ts +4 -0
  78. package/dist/db/migrations/136_master_key_wrapped_tables_key_version.d.ts +4 -0
  79. package/dist/db/migrations/137_execution_runs_pr_head_context.d.ts +4 -0
  80. package/dist/db/migrations/138_execution_runs_subject_trigger_event.d.ts +4 -0
  81. package/dist/db/migrations/139_admin_token_subject.d.ts +4 -0
  82. package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +54 -0
  83. package/dist/db/migrations/141_event_log_shed.d.ts +15 -0
  84. package/dist/db/migrator.d.ts +47 -0
  85. package/dist/db/notify-listener.d.ts +60 -0
  86. package/dist/db/synthetic-job-ids.d.ts +24 -0
  87. package/dist/db/types.d.ts +169 -4
  88. package/dist/deployment/deployment-identity.d.ts +31 -1
  89. package/dist/diagnostics/checks/index.d.ts +2 -1
  90. package/dist/diagnostics/checks/retention.d.ts +24 -0
  91. package/dist/diagnostics/types.d.ts +7 -0
  92. package/dist/events/event-router.d.ts +1 -1
  93. package/dist/fault-injection-types.d.ts +53 -0
  94. package/dist/firecracker/host-network.d.ts +63 -8
  95. package/dist/git/credential-broker.d.ts +35 -2
  96. package/dist/git/inherited-secret.d.ts +7 -2
  97. package/dist/git/job-context.d.ts +7 -4
  98. package/dist/helpers/secure-write.d.ts +35 -0
  99. package/dist/index.js +196 -67
  100. package/dist/metrics/prometheus.d.ts +116 -1
  101. package/dist/oidc/db-signer.d.ts +27 -5
  102. package/dist/oidc/id-token-claims.d.ts +33 -2
  103. package/dist/oidc/oidc-mint-registration.d.ts +6 -0
  104. package/dist/oidc/orchestrator-mint.d.ts +8 -0
  105. package/dist/oidc/reconcile-signing-key.d.ts +37 -0
  106. package/dist/orchestrator-core.d.ts +9 -3
  107. package/dist/pipeline/decorating-secret-resolver.d.ts +1 -1
  108. package/dist/pipeline/dispatch-matched-workflow.d.ts +65 -14
  109. package/dist/pipeline/manual-schedule.d.ts +1 -0
  110. package/dist/pipeline/process-webhook.d.ts +20 -2
  111. package/dist/pipeline/route-or-dispatch-jobs.d.ts +7 -0
  112. package/dist/pipeline/run-event-context.d.ts +39 -0
  113. package/dist/policy/dashboard-write-policy-listener.d.ts +1 -1
  114. package/dist/policy/dashboard-write-policy.d.ts +47 -1
  115. package/dist/provenance/dispatch-context.d.ts +46 -0
  116. package/dist/provenance/verify-deferred-capture.d.ts +39 -0
  117. package/dist/providers/github/auth.d.ts +9 -0
  118. package/dist/providers/github/deliveries.d.ts +118 -0
  119. package/dist/providers/universal-git/config.d.ts +1 -1
  120. package/dist/queue/cleanup.d.ts +13 -0
  121. package/dist/queue/job-queue.d.ts +162 -14
  122. package/dist/queue/retention.d.ts +105 -0
  123. package/dist/reporting/execution-tracker.d.ts +87 -36
  124. package/dist/routes/admin-db.d.ts +5 -0
  125. package/dist/routes/admin-events.d.ts +13 -0
  126. package/dist/routes/admin-held-runs.d.ts +11 -1
  127. package/dist/routes/admin-org-settings.d.ts +9 -0
  128. package/dist/routes/admin-runs.d.ts +7 -4
  129. package/dist/routes/admin-sources.d.ts +15 -0
  130. package/dist/routes/admin-trust-policy.d.ts +11 -0
  131. package/dist/routes/admin.d.ts +25 -0
  132. package/dist/scaler/backend-factory.d.ts +16 -0
  133. package/dist/scaler/bare-metal-backend.d.ts +22 -4
  134. package/dist/scaler/config.d.ts +6 -3
  135. package/dist/scaler/container-backend.d.ts +102 -3
  136. package/dist/scaler/container-hostconfig.d.ts +60 -0
  137. package/dist/scaler/file-tail.d.ts +25 -2
  138. package/dist/scaler/firecracker-api.d.ts +9 -1
  139. package/dist/scaler/firecracker-backend.d.ts +135 -2
  140. package/dist/scaler/host-access.d.ts +59 -0
  141. package/dist/scaler/index.d.ts +1 -0
  142. package/dist/scaler/manager.d.ts +33 -0
  143. package/dist/scaler/scaler-state-store.d.ts +6 -0
  144. package/dist/scaler/types.d.ts +5 -7
  145. package/dist/secrets/config.d.ts +48 -0
  146. package/dist/secrets/dashboard-encryption-key.d.ts +19 -0
  147. package/dist/secrets/ephemeral-keys.d.ts +9 -3
  148. package/dist/secrets/index.d.ts +3 -1
  149. package/dist/secrets/job-secret-gate.d.ts +76 -0
  150. package/dist/secrets/master-key-rotation.d.ts +65 -0
  151. package/dist/secrets/secret-output-crypto.d.ts +17 -0
  152. package/dist/secrets/secret-resolver.d.ts +23 -6
  153. package/dist/secrets/source-credentials.d.ts +1 -1
  154. package/dist/secrets/token-manager.d.ts +4 -1
  155. package/dist/security/reduced-privilege-note.d.ts +5 -0
  156. package/dist/security/trust-policy-gate.d.ts +70 -6
  157. package/dist/security/trust-tier.d.ts +31 -0
  158. package/dist/server.d.ts +1 -1
  159. package/dist/server.js +10674 -5283
  160. package/dist/sources/source-manager.d.ts +1 -1
  161. package/dist/stale-detector/stale-dispatch-candidates.d.ts +43 -0
  162. package/dist/standalone.js +9258 -4122
  163. package/dist/storage/loopback-guard.d.ts +53 -6
  164. package/dist/webhook/event-log.d.ts +14 -1
  165. package/dist/webhook/generic-sources-listener.d.ts +1 -1
  166. package/dist/webhook/ingest-admission.d.ts +24 -0
  167. package/dist/webhook/ingest-overflow-replayer.d.ts +78 -14
  168. package/dist/webhook/ingest-overflow-types.d.ts +13 -1
  169. package/dist/webhook/register-source-bundle.d.ts +23 -0
  170. package/dist/webhook/relay-reinject.d.ts +11 -12
  171. package/dist/webhook/shed-breadcrumb.d.ts +15 -0
  172. package/dist/ws/git-credential-relay.d.ts +41 -12
  173. package/dist/ws/platform-client.d.ts +28 -5
  174. package/installer-image-digests.json +3 -3
  175. package/package.json +4 -4
  176. package/sbom.spdx.json +53 -53
  177. package/dist/concurrency/group-tracker.d.ts +0 -51
  178. package/dist/scaler/nftables.d.ts +0 -111
@@ -9,14 +9,33 @@ import { ScalerBackendType } from '@kici-dev/engine';
9
9
  import type { AppConfig } from '../config.js';
10
10
  import type { ScalerConfig } from '../scaler/types.js';
11
11
  /**
12
- * Every current scaler backend places the agent outside the orchestrator's
13
- * loopback (separate netns, microVM, or host). Only a future co-located/"local"
14
- * backend would be absent from this set. `kubernetes` is included defensively
15
- * even though `ScalerEntry.type` excludes it today.
12
+ * Backends that place the agent outside the orchestrator's loopback (separate
13
+ * netns, microVM, or host) whenever the entry does not say otherwise.
14
+ * `kubernetes` is included defensively even though `ScalerEntry.type` excludes
15
+ * it today.
16
+ *
17
+ * Backend type alone is an imprecise signal — a `bare-metal` scaler may spawn
18
+ * on this very machine or on a rack of remote hosts — so it is only the
19
+ * fallback. When an entry declares its own `orchestratorUrl`, that URL says
20
+ * where the agent will be, and `scalerIsColocated` reads it instead.
16
21
  */
17
22
  export declare const NON_COLOCATED_BACKENDS: ReadonlySet<ScalerBackendType>;
18
- /** True for any host an agent in another netns/VM/host cannot reach. */
23
+ /**
24
+ * Answers "could an agent in another netns/VM/host reach this address?" — the
25
+ * question a DESTINATION URL asks. A wildcard (`0.0.0.0`, `::`) is not a
26
+ * destination at all, so it is reported as loopback here.
27
+ *
28
+ * For a listener's bind address the question is the opposite one; use
29
+ * `isLoopbackBind`.
30
+ */
19
31
  export declare function isLoopbackHost(hostname: string): boolean;
32
+ /**
33
+ * Answers "does a listener on this address stay on this machine?" — the
34
+ * question a BIND address asks. A wildcard (`0.0.0.0`, `::`) accepts every
35
+ * interface, so it is NOT loopback here, which is the one case where this
36
+ * predicate and `isLoopbackHost` disagree.
37
+ */
38
+ export declare function isLoopbackBind(hostname: string): boolean;
20
39
  /**
21
40
  * Decide whether an agent-facing storage URL is a loopback address that a
22
41
  * non-co-located agent could not reach. Returns a remediation message, or null
@@ -26,7 +45,10 @@ export declare function checkLoopbackAgentEndpoint(input: {
26
45
  agentFacingUrl: string | null;
27
46
  endpointSource: string;
28
47
  fixEnvVar: string;
29
- scalerBackends: ScalerBackendType[];
48
+ scalers: {
49
+ type: ScalerBackendType;
50
+ orchestratorUrl?: string;
51
+ }[];
30
52
  }): {
31
53
  message: string;
32
54
  } | null;
@@ -46,4 +68,29 @@ export declare function resolveAgentFacingStorage(config: AppConfig): {
46
68
  * remediation (so it reaches `kici-admin orchestrator logs`) then throws.
47
69
  */
48
70
  export declare function assertAgentReachableStorage(config: AppConfig, scalerConfig: ScalerConfig | null): void;
71
+ /**
72
+ * Refuse to start when agent authentication is disabled and the listener is
73
+ * bound to something other than a loopback address.
74
+ *
75
+ * `KICI_AGENT_AUTH=none` answers `auth.success` to any `auth.request`
76
+ * (`ws/agent-handler.ts`), so a listener on a routable address hands anyone who
77
+ * can reach the port a registered agent: it receives dispatched jobs — with the
78
+ * org's resolved secrets in hybrid mode — and streams results back as the
79
+ * operator's own agent. No credential is involved anywhere in that sequence.
80
+ *
81
+ * `none` is a local-development affordance, and any bind other hosts can reach
82
+ * is never an intended pairing with it. The one-shot startup warning that used
83
+ * to be the only signal was demonstrably not enough — the local dev plane
84
+ * shipped in exactly this state — so this refuses, following the sibling guard
85
+ * above: an orchestrator misconfiguration that produces a confusing failure
86
+ * later must refuse to start with an actionable message.
87
+ *
88
+ * The bind question is `isLoopbackBind`, not `isLoopbackHost`. The default
89
+ * `KICI_HOST` is the wildcard `0.0.0.0`, which `isLoopbackHost` reports as
90
+ * loopback because a wildcard is not a reachable destination — true of a
91
+ * storage URL, and the opposite of what a listener on it does. So the guard
92
+ * covers the default bind, which is the configuration an operator reaches
93
+ * without setting `KICI_HOST` at all.
94
+ */
95
+ export declare function assertAgentAuthBindSafe(config: AppConfig): void;
49
96
  //# sourceMappingURL=loopback-guard.d.ts.map
@@ -53,6 +53,19 @@ export interface PayloadBytes {
53
53
  /** UTF-8 encoded raw body bytes (what Platform hashed). */
54
54
  raw: Buffer;
55
55
  }
56
+ /** Per-call write mode for {@link EventLogWriter.record}. */
57
+ export interface RecordOptions {
58
+ /**
59
+ * Insert only when `(org_id, delivery_id)` has no row yet, leaving an
60
+ * existing row untouched instead of updating its outcome fields.
61
+ *
62
+ * Used by breadcrumb writers that record a delivery was SEEN rather than how
63
+ * it ended — a shed under load, say. Those run before the pipeline, so with
64
+ * the default upsert a re-shed of an already-processed delivery would
65
+ * downgrade a terminal `processed` row back to `shed`.
66
+ */
67
+ onlyIfAbsent?: boolean;
68
+ }
56
69
  /**
57
70
  * Build a `PayloadBytes` from the parsed webhook payload object. Uses
58
71
  * `JSON.stringify` with no formatting — same shape Platform uses to compute
@@ -89,6 +102,6 @@ export declare class EventLogWriter {
89
102
  * payload_key / hash / size fields are NOT overwritten on update — those
90
103
  * come from the original delivery's body.
91
104
  */
92
- record(info: WebhookInfo, payload: PayloadBytes, outcome: EventLogOutcome): Promise<void>;
105
+ record(info: WebhookInfo, payload: PayloadBytes, outcome: EventLogOutcome, opts?: RecordOptions): Promise<void>;
93
106
  }
94
107
  //# sourceMappingURL=event-log.d.ts.map
@@ -72,7 +72,7 @@ export interface GenericSourcesChangeListenerOptions {
72
72
  }
73
73
  export declare class GenericSourcesChangeListener {
74
74
  private readonly opts;
75
- private client;
75
+ private listener;
76
76
  private readonly pending;
77
77
  private debounceTimer;
78
78
  private readonly debounceMs;
@@ -101,6 +101,30 @@ export declare class IngestAdmissionController {
101
101
  private incKey;
102
102
  private decKey;
103
103
  isShedding(): boolean;
104
+ /**
105
+ * Take an admission slot on behalf of the durable-overflow replayer, BEFORE
106
+ * it claims a row. The replayer holds the grant across the whole re-injection
107
+ * and releases it afterwards, so a capacity refusal leaves the buffered row
108
+ * untouched instead of costing it a replay attempt.
109
+ *
110
+ * This is NOT an exempt lane. It evaluates the same layer-1 loop-lag gate and
111
+ * the same layer-2 global + per-key caps {@link admit} does, in the same
112
+ * order, through the same {@link canAdmit} predicate — so a reservation can
113
+ * always be refused, and a granted one occupies a slot indistinguishable from
114
+ * a sender's. The two differences are deliberate and both narrow the grant
115
+ * rather than widening it:
116
+ *
117
+ * - It never queues (there is no caller waiting on an ack; the next pass
118
+ * retries in `intervalMs`), so it cannot occupy a CoDel queue slot a real
119
+ * sender needs.
120
+ * - A refusal does not fire the shed seam or the shed metric.
121
+ * `kici_orch_ingest_shed_total` means "a sender was refused"; a background
122
+ * drain deciding not to start is not that, and counting it there would make
123
+ * the shed rate unreadable during exactly the overload it measures. The
124
+ * replayer counts its own refusals on
125
+ * `kici_orch_ingest_overflow_replay_refused_total` instead.
126
+ */
127
+ reserve(key: string, orgCap: number): AdmitResult;
104
128
  snapshot(): IngestAdmissionSnapshot;
105
129
  private emit;
106
130
  /**
@@ -3,15 +3,50 @@ import type { Database } from '../db/types.js';
3
3
  import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
4
4
  import { WebhookIngestOutcome } from '../pipeline/process-webhook.js';
5
5
  import { type OverflowDelivery } from './ingest-overflow-types.js';
6
+ import type { AdmitResult } from './ingest-admission.js';
7
+ /**
8
+ * Re-inject one buffered delivery through the ingest pipeline.
9
+ *
10
+ * The caller (this replayer) already holds the admission grant, so an
11
+ * implementation MUST NOT admit again — that second admission is what used to
12
+ * charge a per-org capacity refusal against the delivery's failure budget.
13
+ */
6
14
  export type ReinjectFn = (d: OverflowDelivery) => Promise<WebhookIngestOutcome>;
15
+ /**
16
+ * The admission-controller surface the replayer needs: the pass-level
17
+ * short-circuit and the per-row reservation.
18
+ */
19
+ export interface ReplayAdmissionController {
20
+ isShedding(): boolean;
21
+ reserve(key: string, orgCap: number): AdmitResult;
22
+ }
23
+ /** Resolve a routing key to its fairness key + per-org concurrency cap. */
24
+ export type ResolveAdmissionKeyFn = (routingKey: string) => Promise<{
25
+ key: string;
26
+ orgCap: number;
27
+ }>;
7
28
  export interface IngestOverflowReplayerDeps {
8
29
  db: Kysely<Database>;
9
- controller: {
10
- isShedding(): boolean;
11
- };
30
+ controller: ReplayAdmissionController;
31
+ /**
32
+ * Resolve a buffered row's routing key to the same fairness key + per-org cap
33
+ * the live ingest paths admit on, so a reservation competes on exactly the
34
+ * terms a fresh delivery would.
35
+ */
36
+ resolveAdmissionKey: ResolveAdmissionKeyFn;
12
37
  intervalMs: number;
13
38
  batchSize: number;
14
39
  maxAttempts: number;
40
+ /**
41
+ * Wall-clock retention bound. A row that has sat `buffered` this long without
42
+ * ever winning a capacity grant is marked `failed` (`max_age`) rather than
43
+ * held forever. This is what keeps the buffer bounded now that a capacity
44
+ * refusal no longer consumes an attempt: without it, a permanently saturated
45
+ * org's rows would fill `ingestOverflowMax` and capture would start dropping
46
+ * FRESH deliveries instead — trading a bounded per-delivery loss for an
47
+ * unbounded new-delivery one.
48
+ */
49
+ maxAgeMs: number;
15
50
  /**
16
51
  * How long a `replaying` claim may stand before it is reclaimed. Resolved per
17
52
  * pass so a fleet-wide `cluster_settings` override takes effect without a
@@ -20,30 +55,45 @@ export interface IngestOverflowReplayerDeps {
20
55
  claimTimeoutMs: number;
21
56
  /** Fleet-wide override reader for {@link IngestOverflowReplayerDeps.claimTimeoutMs}. */
22
57
  clusterSettings?: ClusterSettingsReader;
58
+ /** Injectable clock (default `Date.now`); the retention bound reads it. */
59
+ now?: () => number;
23
60
  }
24
61
  /**
25
62
  * Background drain for the durable ingest queue. Each pass first reclaims rows
26
63
  * whose `replaying` claim went stale — a worker killed mid-pipeline releases
27
64
  * nothing, so without this its delivery would sit claimed forever and the
28
- * durable row it acknowledged would never be worth anything. It then claims the
29
- * oldest `buffered` rows FIFO up to a bounded batch and re-injects each through
30
- * the admission-gated ingest path. A re-shed or error reverts the row to
31
- * `buffered` (never lost); past the max-attempts ceiling it goes `failed`.
32
- * Successful rows are swept.
65
+ * durable row it acknowledged would never be worth anything. It then expires
66
+ * rows past the wall-clock retention bound, and finally drains the oldest
67
+ * `buffered` rows FIFO up to a bounded batch.
68
+ *
69
+ * **Reserve, then claim.** The drain takes an admission slot from the same
70
+ * controller the live ingest paths use BEFORE it claims a row, and holds that
71
+ * grant across the whole re-injection. A refusal therefore leaves the row
72
+ * untouched in `buffered` with its attempt counter unmoved — the delivery never
73
+ * entered the pipeline, so there was no delivery failure to charge. Ordering it
74
+ * the other way round is what used to spend a per-delivery failure budget on a
75
+ * transient per-org capacity refusal, killing a whole FIFO cohort in seconds.
76
+ *
77
+ * `replay_attempts` therefore counts genuine failures only: a verify throw, a
78
+ * pipeline throw, a stale claim. Retention is bounded by `maxAgeMs` instead.
33
79
  *
34
- * Re-injection runs only while the admission controller is NOT shedding
35
- * (replaying into a still-overloaded orchestrator just re-sheds). Reclaiming is
36
- * not gated that way: a stranded claim is stranded regardless of load, and the
37
- * row it frees simply waits in `buffered` until capacity returns.
80
+ * Refusal is per fairness key: a saturated org's rows are skipped for the rest
81
+ * of the pass while other orgs' rows keep draining, mirroring the fair skip the
82
+ * controller's own queue does. Reclaiming is not gated on load at all: a
83
+ * stranded claim is stranded regardless, and the row it frees simply waits in
84
+ * `buffered` until a reservation succeeds.
38
85
  */
39
86
  export declare class IngestOverflowReplayer {
40
87
  private readonly db;
41
88
  private readonly controller;
89
+ private readonly resolveAdmissionKey;
42
90
  private readonly intervalMs;
43
91
  private readonly batchSize;
44
92
  private readonly maxAttempts;
93
+ private readonly maxAgeMs;
45
94
  private readonly claimTimeoutMs;
46
95
  private readonly clusterSettings;
96
+ private readonly now;
47
97
  private reinjectDirect;
48
98
  private reinjectRelay;
49
99
  private timer;
@@ -76,8 +126,22 @@ export declare class IngestOverflowReplayer {
76
126
  * take through {@link claimBatch} and must release it the same way on failure.
77
127
  */
78
128
  releaseClaim(id: number, reason: string): Promise<boolean>;
79
- /** Select the oldest buffered rows and claim each via a conditional update. */
80
- private claimBatch;
129
+ /**
130
+ * Mark every `buffered` row past the retention bound `failed`. Only
131
+ * `buffered` rows are eligible, so a row a worker is actively holding is
132
+ * never expired out from under it — the stale-claim reclaim above hands that
133
+ * row back to `buffered` first, and the next pass expires it there. That is
134
+ * why the two bounds compose rather than race even when `maxAgeMs` equals
135
+ * `claimTimeoutMs`.
136
+ */
137
+ private expireAgedRows;
138
+ /**
139
+ * Reserve-then-claim drain. For each candidate, oldest first: resolve its
140
+ * fairness key, take an admission slot, and only then claim the row. A
141
+ * refusal skips that key for the rest of the pass and leaves the row exactly
142
+ * as it was — `buffered`, same attempt count, no `last_error`.
143
+ */
144
+ private drain;
81
145
  private toDelivery;
82
146
  private replayOne;
83
147
  /**
@@ -32,9 +32,21 @@ export declare const OverflowSourceKind: z.ZodEnum<{
32
32
  relay: "relay";
33
33
  }>;
34
34
  export type OverflowSourceKind = z.infer<typeof OverflowSourceKind>;
35
- /** Reasons a delivery is permanently dropped from the overflow buffer. */
35
+ /**
36
+ * Reasons a delivery is permanently dropped from the overflow buffer.
37
+ *
38
+ * - `cap_full` — the buffer was at `ingestOverflowMax` when capture ran.
39
+ * - `max_attempts` — replay failed `ingestOverflowMaxAttempts` times. Only a
40
+ * genuine failure counts (a verify throw, a pipeline throw, a stale claim);
41
+ * a capacity refusal never reaches a claim, so it cannot consume an attempt.
42
+ * - `max_age` — the row sat in `buffered` past `ingestOverflowMaxAgeMs`.
43
+ * This is the retention bound: without an attempt ceiling on the capacity
44
+ * path, age is the only thing that stops a permanently-saturated org's rows
45
+ * from filling the cap and dropping fresh captures instead.
46
+ */
36
47
  export declare const OverflowDropReason: z.ZodEnum<{
37
48
  cap_full: "cap_full";
49
+ max_age: "max_age";
38
50
  max_attempts: "max_attempts";
39
51
  }>;
40
52
  export type OverflowDropReason = z.infer<typeof OverflowDropReason>;
@@ -67,4 +67,27 @@ export interface RegisterSourceBundleDeps {
67
67
  * second call with the same routing key replaces the prior bundle.
68
68
  */
69
69
  export declare function registerProviderBundleForSource(row: GenericWebhookSource, deps: RegisterSourceBundleDeps): void;
70
+ /** What {@link registerAllGenericSourceBundles} needs beyond the per-row deps. */
71
+ export interface RegisterAllGenericSourceBundlesDeps extends RegisterSourceBundleDeps {
72
+ sourceManager: {
73
+ listLocalSources(): Promise<GenericWebhookSource[]>;
74
+ listUniversalGitSources(): Promise<GenericWebhookSource[]>;
75
+ };
76
+ }
77
+ /**
78
+ * Register a provider bundle for every generic source currently in the
79
+ * database.
80
+ *
81
+ * Two call sites share this: cold boot, and a `generic_sources_change`
82
+ * listener that has just reconnected. A reconnect missed an unknown number of
83
+ * NOTIFYs and Postgres keeps no durable log of them, so re-deriving the whole
84
+ * set from the database is the only correct catch-up — and it is exactly what
85
+ * boot already does.
86
+ *
87
+ * `ProviderRegistry.registerByRoutingKey` replaces a bundle at the same key,
88
+ * so re-running this is idempotent. Each list is fetched inside its own
89
+ * try/catch: one failing query must not cost the other its registrations.
90
+ * Per-row failures are handled inside `registerProviderBundleForSource`.
91
+ */
92
+ export declare function registerAllGenericSourceBundles(deps: RegisterAllGenericSourceBundlesDeps): Promise<void>;
70
93
  //# sourceMappingURL=register-source-bundle.d.ts.map
@@ -1,4 +1,3 @@
1
- import type { AdmitResult } from './ingest-admission.js';
2
1
  import type { WebhookInfo } from './handler.js';
3
2
  import { WebhookIngestOutcome } from '../pipeline/process-webhook.js';
4
3
  import type { OverflowDelivery } from './ingest-overflow-types.js';
@@ -13,12 +12,6 @@ export interface RelayVerifyResult {
13
12
  * store / provider registry. server.ts wires these to the real subsystems.
14
13
  */
15
14
  export interface RelayReinjectSeams {
16
- /**
17
- * Re-admit on the routing key (allowQueue:false — same non-queueing gate the
18
- * live relay path uses). A re-shed maps the reinject to `shed` so the replayer
19
- * reverts the row to `buffered`.
20
- */
21
- admit: (routingKey: string) => Promise<AdmitResult>;
22
15
  /** Re-verify the raw body against the stored headers. */
23
16
  verify: (d: OverflowDelivery, body: Buffer) => Promise<RelayVerifyResult>;
24
17
  /** Resolve the provider string for a routing key (bundle → prefix fallback). */
@@ -33,11 +26,17 @@ export interface RelayReinjectSeams {
33
26
  */
34
27
  export declare function parseRelayPayload(body: Buffer, headers: Record<string, string>): Record<string, unknown>;
35
28
  /**
36
- * Build the relay-origin overflow re-injector. Re-admits on the routing key,
37
- * re-verifies the stored raw body (relay deliveries are captured pre-verify),
38
- * normalizes it, and processes it through the admission-gated relay pipeline.
39
- * A re-shed maps to `shed` (revert); a verify miss maps to `skipped` (terminal
40
- * — the delivery is no longer routable, so it is not retried forever).
29
+ * Build the relay-origin overflow re-injector. Re-verifies the stored raw body
30
+ * (relay deliveries are captured pre-verify), normalizes it, and processes it
31
+ * through the relay pipeline. A verify miss maps to `skipped` (terminal — the
32
+ * delivery is no longer routable, so it is not retried forever).
33
+ *
34
+ * It does NOT admit. The replayer reserves an admission slot before it claims
35
+ * the row and holds that grant for the whole call, so admitting again here
36
+ * would be a second, redundant gate — and a refusal from it would be charged to
37
+ * the delivery as a failed replay attempt, which is the defect this ordering
38
+ * exists to remove. The slot is genuinely held: nothing here bypasses the
39
+ * controller, the decision was simply made one step earlier.
41
40
  */
42
41
  export declare function buildRelayReinject(seams: RelayReinjectSeams): ReinjectFn;
43
42
  //# sourceMappingURL=relay-reinject.d.ts.map
@@ -0,0 +1,15 @@
1
+ import type { EventLogWriter } from './event-log.js';
2
+ import type { RelayStartMeta } from './relay-buffer.js';
3
+ export interface ShedBreadcrumbDeps {
4
+ /** Writer for the orchestrator `event_log`. Absent in wirings with no log. */
5
+ eventLog: EventLogWriter | undefined;
6
+ /** Routing key → owning org id. Must not throw; returns the resolved tenant. */
7
+ resolveOrgId: (routingKey: string) => Promise<string>;
8
+ }
9
+ /**
10
+ * Record that a relay delivery was shed. Best-effort and never throws: a
11
+ * breadcrumb failure must not change the `shed_retry_later` ack the sender
12
+ * gets. Returns whether a write was attempted and completed.
13
+ */
14
+ export declare function recordShedBreadcrumb(deps: ShedBreadcrumbDeps, meta: RelayStartMeta, body: Buffer, reason: string): Promise<boolean>;
15
+ //# sourceMappingURL=shed-breadcrumb.d.ts.map
@@ -4,26 +4,55 @@
4
4
  * The agent's credential helper calls this on every git network operation, so
5
5
  * this path is hot and must stay cheap: validate, authorize, delegate.
6
6
  *
7
- * Authorization has two layers. Job ownership comes first an agent may only
8
- * request credentials for a job it was dispatched, resolved from server truth
9
- * via the dispatcher, never from the params. Then the repository fence, which
10
- * applies to WRITE only: a read with the source credential may reach any
11
- * repository inside the App installation's own selection (GitHub enforces that
12
- * selection when the mint names the repository), so cloning a sibling repo
13
- * needs no credential in workflow code. A write must additionally stay inside
14
- * the organisation that owns the job's source repository.
7
+ * Authorization is five layers, every one of them resolved from server truth
8
+ * rather than from the request:
15
9
  *
16
- * A workflow-supplied credential needs no fence of its own: it names secrets
17
- * the job can already read with `ctx.secrets.get()`, so brokering it grants no
18
- * capability the workflow did not already have the secret ACL is the
19
- * authorization.
10
+ * 1. job ownership an agent may only request credentials for a job it was
11
+ * dispatched, resolved through the dispatcher;
12
+ * 2. the repository fence, which applies to WRITE only: a read with the
13
+ * source credential may reach any repository inside the App installation's
14
+ * own selection (the forge enforces that selection when the mint names the
15
+ * repository), so cloning a sibling repo needs no credential in workflow
16
+ * code. A write must additionally stay inside the organisation that owns
17
+ * the job's source repository;
18
+ * 3. the LOCK DECLARATION — a workflow-supplied `ref` must equal one of the
19
+ * entries in the job's `gitCredentials` map, which the orchestrator itself
20
+ * wrote from the lock at dispatch (`git/job-context.ts` names the two rows
21
+ * that carry it and why it reads them in that order). This is the only
22
+ * check a ref faces, and it is deliberately the whole of it: a ref that
23
+ * carries inline `*Value` material is admitted exactly when the lock
24
+ * declared that material, and refused otherwise;
25
+ * 4. the named context's protection rules, run by
26
+ * `resolveJobQualifiedSecret` when the broker resolves a
27
+ * `<context>:<key>` reference;
28
+ * 5. the trust tier — an untrusted contributor gets no workflow-supplied
29
+ * credential at all, mirroring the install-secrets strip.
30
+ *
31
+ * The step process is the same process as the IPC sender, so job code can forge
32
+ * any request frame it likes. Layers 3 to 5 are what make that forgery
33
+ * worthless: a forged `ref` names a credential the job's lock never declared,
34
+ * and every check here reads what the orchestrator wrote rather than what the
35
+ * agent claimed.
20
36
  */
37
+ import type { TrustTier } from '@kici-dev/engine';
21
38
  import type { GitCredentialBroker } from '../git/credential-broker.js';
22
39
  /** Per-job facts the handler needs. Resolved from server truth, never from params. */
23
40
  export interface JobCredentialContext {
24
41
  orgId: string;
25
42
  /** The repository this job was dispatched for. */
26
43
  sourceRepo: string;
44
+ /**
45
+ * The `gitCredentials` map this job's lock entry declared, verbatim — a
46
+ * request's ref must equal one of these entries. An empty map means the job
47
+ * declared none, so every workflow-supplied ref is refused.
48
+ */
49
+ declaredCredentials: Readonly<Record<string, Readonly<Record<string, string>>>>;
50
+ /** The run's resolved contributor trust tier, or undefined when unresolved. */
51
+ trustTier: TrustTier | undefined;
52
+ /** The branch the run presents, for the named context's branch restrictions. */
53
+ branch: string;
54
+ /** The event type that started the run, for a context's trigger-type filters. */
55
+ triggerType: string;
27
56
  }
28
57
  export interface GitCredentialHandlerDeps {
29
58
  broker: GitCredentialBroker;
@@ -1,4 +1,4 @@
1
- import { type StateReplayRun, type OrchestratorToPlatformMessage, type WebhookRelay, type WebhookRelayResult, type TrustPolicyUpdate, type StaleCheckrunCleanup, type DashboardRunDetailRequest, type DashboardRunStructuredRequest, type DashboardRunStateRequest, type DashboardRunsListRequest, type DashboardRunsFiltersRequest, type DashboardSourcesListRequest, type DashboardStepLogsRequest, type DashboardAttestationsListRequest, type DashboardAttestationsListAllRequest, type DashboardAttestationGetRequest, type DashboardAttestationRetryRequest, type DashboardArtifactsListRequest, type DashboardOrchLogsRequest, type RunRerunRequest, type ManualScheduleRequest, type RunCancelRequest, type DashboardPayloadRequest, type DashboardPlatformToOrchMessage, type TestRelayRequest, type DashboardDiagnosticsRequest, type DashboardScalerCapacityRequest, type DashboardScalerAgentsRequest, type DashboardFleetHostsRequest, type DashboardFleetHostRequest, type DashboardFleetPreviewRequest, type DashboardFleetWorkflowsForHostRequest, type JoinRequest, type JoinResponse, type DeploymentIdentity, type OrchCapabilities, type OrchRole } from '@kici-dev/engine';
1
+ import { type StateReplayRun, type OrchestratorToPlatformMessage, type WebhookRelay, type WebhookRelayResult, type TrustPolicyUpdate, type StaleCheckrunCleanup, type DashboardRunDetailRequest, type DashboardRunStructuredRequest, type DashboardRunStateRequest, type DashboardRunsListRequest, type DashboardRunsFiltersRequest, type DashboardSourcesListRequest, type DashboardAdminTokensListRequest, type DashboardStepLogsRequest, type DashboardAttestationsListRequest, type DashboardAttestationsListAllRequest, type DashboardAttestationGetRequest, type DashboardAttestationRetryRequest, type DashboardArtifactsListRequest, type DashboardOrchLogsRequest, type RunRerunRequest, type ManualScheduleRequest, type RunCancelRequest, type DashboardPayloadRequest, type DashboardPlatformToOrchMessage, type TestRelayRequest, type DashboardDiagnosticsRequest, type DashboardScalerCapacityRequest, type DashboardScalerAgentsRequest, type DashboardFleetHostsRequest, type DashboardFleetHostRequest, type DashboardFleetPreviewRequest, type DashboardFleetWorkflowsForHostRequest, type JoinRequest, type JoinResponse, type DeploymentIdentity, type ConfigPaths, type OrchCapabilities, type OrchRole } from '@kici-dev/engine';
2
2
  import type { PlanHeadroomStore } from '../cluster/plan-headroom-store.js';
3
3
  import { RelayBufferRegistry, type RelayStartMeta } from '../webhook/relay-buffer.js';
4
4
  import type { AdmitResult } from '../webhook/ingest-admission.js';
@@ -86,6 +86,8 @@ export interface PlatformClientOptions {
86
86
  scalerBackends?: string[];
87
87
  /** How the orchestrator process was deployed. Sent in source.register so the dashboard can build the correct kici-admin invocation. */
88
88
  deployment?: DeploymentIdentity;
89
+ /** Where this orchestrator's own config files live on its host. Sent in source.register so the dashboard can point an operator at them. */
90
+ configPaths?: ConfigPaths;
89
91
  /** Whether this orchestrator has S3 log storage configured. Sent in source.register for pool validation. */
90
92
  s3LogAccess?: boolean;
91
93
  /** Queue timeout in ms. Sent in source.register for Platform safety-net GC. */
@@ -165,6 +167,8 @@ export interface PlatformClientOptions {
165
167
  onDashboardRunsFilters?: (msg: DashboardRunsFiltersRequest) => void;
166
168
  /** Optional callback for dashboard sources.list (operator console) requests from Platform. */
167
169
  onDashboardSourcesList?: (msg: DashboardSourcesListRequest) => void;
170
+ /** Optional callback for dashboard admin-tokens.list (RBAC drift report) requests from Platform. */
171
+ onDashboardAdminTokensList?: (msg: DashboardAdminTokensListRequest) => void;
168
172
  /** Optional callback for dashboard step logs requests from Platform. */
169
173
  onDashboardStepLogs?: (msg: DashboardStepLogsRequest) => void;
170
174
  /** Optional callback for dashboard attestations-list requests from Platform. */
@@ -247,11 +251,13 @@ export interface PlatformClientOptions {
247
251
  */
248
252
  onAdmit?: (routingKey: string) => Promise<AdmitResult>;
249
253
  /**
250
- * Optional durable-overflow capture seam. Invoked with the pre-verify relay
251
- * meta + assembled body on a shed, before the shed_retry_later ack. Best-effort
252
- * the client wraps it so a capture failure never blocks the ack.
254
+ * Optional shed-recording seam. Invoked with the pre-verify relay meta +
255
+ * assembled body + the admission controller's shed reason, before the
256
+ * shed_retry_later ack. The wiring records the delivery durably an
257
+ * `event_log` breadcrumb, and the overflow-buffer row when that queue is
258
+ * enabled. Best-effort — the client wraps it so a failure never blocks the ack.
253
259
  */
254
- onShedCapture?: (meta: RelayStartMeta, body: Buffer) => Promise<void>;
260
+ onShedCapture?: (meta: RelayStartMeta, body: Buffer, reason: string) => Promise<void>;
255
261
  }
256
262
  /** Error `*.response` frame shape for a dashboard request that failed validation. */
257
263
  export interface DashboardRequestErrorFrame {
@@ -333,6 +339,7 @@ export declare class PlatformClient {
333
339
  private readonly mode?;
334
340
  private readonly scalerBackends?;
335
341
  private readonly deployment?;
342
+ private readonly configPaths?;
336
343
  private readonly s3LogAccess?;
337
344
  private readonly queueTimeoutMs?;
338
345
  private readonly heartbeatIntervalMs;
@@ -351,6 +358,7 @@ export declare class PlatformClient {
351
358
  private readonly onDashboardRunsList?;
352
359
  private readonly onDashboardRunsFilters?;
353
360
  private readonly onDashboardSourcesList?;
361
+ private readonly onDashboardAdminTokensList?;
354
362
  private readonly onDashboardStepLogs?;
355
363
  private readonly onDashboardAttestationsList?;
356
364
  private readonly onDashboardAttestationsListAll?;
@@ -387,12 +395,27 @@ export declare class PlatformClient {
387
395
  * predates the alias plumbing, or before the first successful auth.
388
396
  */
389
397
  private _orgPublicAlias?;
398
+ /** Canonical Platform org id from `auth.success`; see {@link getOrgId}. */
399
+ private _orgId?;
390
400
  /**
391
401
  * Returns the cached public alias of the orchestrator's owning org,
392
402
  * or `undefined` if Platform has not supplied one yet. Read by
393
403
  * `check-run-reporter.ts` when building `details_url`.
394
404
  */
395
405
  getOrgPublicAlias(): string | undefined;
406
+ /**
407
+ * The canonical Platform organization id this orchestrator authenticated
408
+ * as, from `auth.success`. `undefined` before the first successful auth, or
409
+ * against a Platform that does not supply it.
410
+ *
411
+ * A generic source's routing key embeds the organization
412
+ * (`generic:<orgId>:<id>`), and the Platform now refuses to register a key
413
+ * naming a different one. So the source-create path reads this to refuse a
414
+ * mismatched `--org` up front, naming the correct value, instead of letting
415
+ * the operator create a source that registers, is rejected, and never
416
+ * delivers.
417
+ */
418
+ getOrgId(): string | undefined;
396
419
  constructor(options: PlatformClientOptions);
397
420
  /**
398
421
  * Merge `updates` into the stored orch capabilities and broadcast the
@@ -1,7 +1,7 @@
1
1
  {
2
- "version": "0.6.0",
2
+ "version": "0.7.0",
3
3
  "images": {
4
- "kici-agent": "sha256:6a6dad749ae8a520774139588b6b83d194dcddc5fec58ed46207e0654eb9bc27",
5
- "kici-orchestrator": "sha256:86cc2e5c316492bffce415ac55989681491de25085258f411928ed19ad7a36fb"
4
+ "kici-agent": "sha256:ad6028feb1eda81eb8c1b53490fad8530a0b9137650d436250a8cfd2ec699eed",
5
+ "kici-orchestrator": "sha256:f2706520bf3b8b196e2613deb8d0530ce4b50cc376261304b6eecee7f433385f"
6
6
  }
7
7
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kici-dev/orchestrator",
3
- "version": "0.6.0",
3
+ "version": "0.7.0",
4
4
  "description": "Customer-deployable orchestrator for the KiCI CI/CD stack. Receives webhook events (direct or via Platform relay), matches triggers against the lock file, and dispatches jobs to connected agents.",
5
5
  "keywords": [
6
6
  "ci",
@@ -89,8 +89,8 @@
89
89
  "ws": "^8.21.3",
90
90
  "yaml": "^2.9.0",
91
91
  "zod": "^4.4.3",
92
- "@kici-dev/shared": "0.6.0",
93
- "@kici-dev/engine": "0.6.0"
92
+ "@kici-dev/shared": "0.7.0",
93
+ "@kici-dev/engine": "0.7.0"
94
94
  },
95
95
  "kici": {
96
96
  "metrics": {
@@ -105,7 +105,7 @@
105
105
  "@types/dockerode": "^4.0.1",
106
106
  "@types/ws": "^8.18.1",
107
107
  "kysely-ctl": "^0.21.0",
108
- "@kici-dev/agent": "0.6.0"
108
+ "@kici-dev/agent": "0.7.0"
109
109
  },
110
110
  "scripts": {
111
111
  "build": "node ../../scripts/build-service.mjs && tsgo --emitDeclarationOnly",