@kici-dev/orchestrator 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/dist/__test-helpers__/mock-db-query.d.ts +4 -5
  2. package/dist/agent/agent-version.d.ts +5 -5
  3. package/dist/agent/dispatcher.d.ts +48 -0
  4. package/dist/agent/registry.d.ts +38 -9
  5. package/dist/agent-packaging/node-runtime-members.d.ts +14 -0
  6. package/dist/app.d.ts +9 -3
  7. package/dist/approvals/apply-decision.d.ts +1 -1
  8. package/dist/audit/access-log.d.ts +2 -3
  9. package/dist/cache/cache-upload-settle.d.ts +75 -0
  10. package/dist/cache/pending-inits.d.ts +9 -0
  11. package/dist/cancel/cancel-run.d.ts +22 -4
  12. package/dist/cancel/peer-job-cancel.d.ts +50 -0
  13. package/dist/cancel/sweep-stuck-cancelling.d.ts +3 -1
  14. package/dist/cli/commands/access-log.d.ts +3 -3
  15. package/dist/cli/commands/agent-service/agent-env.d.ts +20 -0
  16. package/dist/cli/commands/agent-service/status.d.ts +3 -3
  17. package/dist/cli/commands/attestations.d.ts +1 -1
  18. package/dist/cli/commands/context.d.ts +2 -2
  19. package/dist/cli/commands/event-dlq.d.ts +5 -6
  20. package/dist/cli/commands/event-log.d.ts +2 -3
  21. package/dist/cli/commands/execution.d.ts +1 -3
  22. package/dist/cli/commands/local-hook.d.ts +3 -2
  23. package/dist/cli/commands/local-trigger.d.ts +7 -0
  24. package/dist/cli/commands/orchestrator-service/status.d.ts +10 -3
  25. package/dist/cli/commands/queue.d.ts +1 -2
  26. package/dist/cli/commands/runs.d.ts +2 -3
  27. package/dist/cli/commands/service-health.d.ts +68 -0
  28. package/dist/cli/commands/shared/env-file-mode.d.ts +3 -2
  29. package/dist/cli/commands/shared/upgrade-safety.d.ts +11 -5
  30. package/dist/cli/commands/shared/versioned-upgrade.d.ts +119 -7
  31. package/dist/cli/commands/workflow.d.ts +3 -4
  32. package/dist/cli/open-browser.d.ts +1 -1
  33. package/dist/cli/service/entrypoint.d.ts +18 -0
  34. package/dist/cli/service/image-digests.d.ts +10 -0
  35. package/dist/cli/service/index.d.ts +4 -2
  36. package/dist/cli/service/instance/manifest.d.ts +39 -0
  37. package/dist/cli/service/launchd.d.ts +45 -10
  38. package/dist/cli/service/shutdown-grace.d.ts +35 -0
  39. package/dist/cli/service/types.d.ts +9 -0
  40. package/dist/cli/service/windows-acl.d.ts +40 -0
  41. package/dist/cli/service/windows-env-file.d.ts +45 -0
  42. package/dist/cli/service/windows.d.ts +13 -0
  43. package/dist/cli.js +2601 -1232
  44. package/dist/cluster/cluster-identity.d.ts +4 -4
  45. package/dist/cluster/coordinator-credential.d.ts +45 -0
  46. package/dist/cluster/coordinator.d.ts +18 -2
  47. package/dist/cluster/index.d.ts +1 -0
  48. package/dist/cluster/instance-heartbeat.d.ts +6 -1
  49. package/dist/cluster/join-token.d.ts +4 -2
  50. package/dist/cluster/leader-gated-scheduler.d.ts +7 -1
  51. package/dist/cluster/peer-auth-coordinator.d.ts +13 -0
  52. package/dist/cluster/peer-client.d.ts +4 -0
  53. package/dist/cluster/peer-credentials.d.ts +32 -1
  54. package/dist/cluster/raft-state.d.ts +6 -0
  55. package/dist/cluster/raft.d.ts +16 -2
  56. package/dist/cold-store/load-access-log-range.d.ts +3 -3
  57. package/dist/cold-store/load-event-log-range.d.ts +1 -1
  58. package/dist/cold-store/load-secret-audit-log-range.d.ts +2 -2
  59. package/dist/cold-store/orchestrator-cold-store.d.ts +9 -11
  60. package/dist/cold-store/tables/access-log.d.ts +2 -2
  61. package/dist/cold-store/tables/event-log.d.ts +4 -10
  62. package/dist/cold-store/tables/execution-jobs.d.ts +1 -1
  63. package/dist/cold-store/tables/execution-runs.d.ts +4 -4
  64. package/dist/cold-store/tables/execution-steps.d.ts +1 -1
  65. package/dist/cold-store/tables/secret-audit-log.d.ts +3 -3
  66. package/dist/config.d.ts +6 -1
  67. package/dist/contexts/held-runs.d.ts +2 -2
  68. package/dist/dashboard/attestation-filters.d.ts +1 -1
  69. package/dist/dashboard/handler.d.ts +8 -8
  70. package/dist/db/deferred-indexes.d.ts +2 -2
  71. package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +2 -4
  72. package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +4 -4
  73. package/dist/db/migrations/007_audit_logs_archived_at.d.ts +6 -9
  74. package/dist/db/migrations/008_event_log_archived_at.d.ts +4 -6
  75. package/dist/db/migrations/010_cold_store_chunks.d.ts +2 -2
  76. package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +2 -4
  77. package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +2 -2
  78. package/dist/db/migrations/045_host_properties.d.ts +1 -1
  79. package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +1 -1
  80. package/dist/db/migrations/048_host_reboot_pending.d.ts +1 -1
  81. package/dist/db/migrations/049_held_runs_payload.d.ts +1 -1
  82. package/dist/db/migrations/050_sources_slug.d.ts +1 -1
  83. package/dist/db/migrations/051_binding_host_pattern.d.ts +1 -1
  84. package/dist/db/migrations/052_host_reach_metadata.d.ts +1 -1
  85. package/dist/db/migrations/053_agent_token_single_use.d.ts +1 -1
  86. package/dist/db/migrations/054_local_working_tree.d.ts +1 -1
  87. package/dist/db/migrations/055_agent_token_mandatory_labels.d.ts +1 -1
  88. package/dist/db/migrations/056_execution_jobs_environments.d.ts +1 -1
  89. package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +1 -1
  90. package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +15 -0
  91. package/dist/db/migrations/153_execution_runs_cancel_force.d.ts +16 -0
  92. package/dist/db/types.d.ts +27 -13
  93. package/dist/events/types.d.ts +1 -1
  94. package/dist/fault-injection-types.d.ts +2 -2
  95. package/dist/firecracker/host-network.d.ts +21 -3
  96. package/dist/firecracker/net-interfaces.d.ts +1 -1
  97. package/dist/helpers/run-detached.d.ts +12 -0
  98. package/dist/index.js +242 -67
  99. package/dist/oidc/db-signer.d.ts +1 -1
  100. package/dist/pipeline/changed-files.d.ts +51 -0
  101. package/dist/pipeline/cross-source-tally.d.ts +21 -0
  102. package/dist/pipeline/degraded-reason.d.ts +8 -0
  103. package/dist/pipeline/dispatch-matched-workflow.d.ts +10 -0
  104. package/dist/pipeline/global-eval-round.d.ts +5 -4
  105. package/dist/pipeline/processor.d.ts +11 -3
  106. package/dist/pipeline/rerun-inputs.d.ts +16 -7
  107. package/dist/provenance/retrier-mint.d.ts +1 -1
  108. package/dist/provenance/trust-root.d.ts +3 -3
  109. package/dist/providers/github/changed-files.d.ts +14 -4
  110. package/dist/providers/universal-git/changed-files.d.ts +6 -0
  111. package/dist/providers/universal-git/clone-token.d.ts +9 -13
  112. package/dist/providers/universal-git/config.d.ts +4 -0
  113. package/dist/providers/universal-git/index.d.ts +1 -2
  114. package/dist/providers/universal-git/normalizer.d.ts +2 -3
  115. package/dist/providers/universal-git/ssh-auth.d.ts +4 -4
  116. package/dist/queue/cleanup.d.ts +2 -2
  117. package/dist/queue/job-queue.d.ts +6 -6
  118. package/dist/queue/scheduled-job.d.ts +5 -7
  119. package/dist/registration/extractor.d.ts +1 -1
  120. package/dist/registration/registration-index.d.ts +1 -1
  121. package/dist/reporting/execution-tracker.d.ts +3 -3
  122. package/dist/routes/admin-access-log.d.ts +1 -1
  123. package/dist/routes/admin-auth.d.ts +2 -2
  124. package/dist/routes/admin-event-log.d.ts +2 -2
  125. package/dist/routes/admin-runs.d.ts +3 -3
  126. package/dist/routes/admin-scheduled-jobs.d.ts +2 -3
  127. package/dist/routes/health.d.ts +9 -0
  128. package/dist/scaler/container-backend.d.ts +3 -1
  129. package/dist/scaler/container-runtime.d.ts +9 -0
  130. package/dist/scaler/firecracker-backend.d.ts +15 -10
  131. package/dist/scaler/firecracker-extra-hosts.d.ts +20 -0
  132. package/dist/scaler/machine-ledger.d.ts +3 -4
  133. package/dist/scaler/manager.d.ts +26 -2
  134. package/dist/scaler/scaler-state-store.d.ts +1 -1
  135. package/dist/scaler/types.d.ts +4 -1
  136. package/dist/secrets/audit-logger.d.ts +2 -2
  137. package/dist/security/reduced-privilege-note.d.ts +2 -3
  138. package/dist/security/trust-policy-store.d.ts +3 -3
  139. package/dist/server.d.ts +2 -1
  140. package/dist/server.js +4955 -3206
  141. package/dist/stale-detector/stale-run-detector.d.ts +1 -1
  142. package/dist/stale-detector/workflow-deadline-detector.d.ts +7 -1
  143. package/dist/standalone.d.ts +1 -1
  144. package/dist/standalone.js +12944 -11243
  145. package/dist/storage/filesystem.d.ts +1 -1
  146. package/dist/storage/index.d.ts +1 -1
  147. package/dist/storage/s3.d.ts +36 -0
  148. package/dist/storage/sign-url.d.ts +2 -1
  149. package/dist/storage/types.d.ts +2 -2
  150. package/dist/ws/agent-handler.d.ts +58 -6
  151. package/dist/ws/agent-heartbeat.d.ts +4 -5
  152. package/dist/ws/dashboard-context-handler.d.ts +1 -1
  153. package/dist/ws/platform-client.d.ts +44 -32
  154. package/dist/ws/server-options.d.ts +1 -1
  155. package/dist/ws/test-relay-handlers.d.ts +1 -1
  156. package/installer-image-digests.json +3 -3
  157. package/package.json +5 -5
  158. package/sbom.spdx.json +53 -53
@@ -34,10 +34,10 @@ export interface ClusterIdentityDeps {
34
34
  * `cacheStorageS3Prefix` default). Empty means the cache blobs and the
35
35
  * cluster-identity sentinel live at the bucket root — the bucket already scopes
36
36
  * the cluster. Every place that resolves a sentinel prefix (the orchestrator
37
- * runtime, the `kici-admin cluster reconcile-identity` CLI, and the staging
38
- * deploy's self-heal step) MUST fall back to THIS value when no explicit prefix
39
- * is set, otherwise they compute divergent sentinel keys and crash-loop the
40
- * orchestrator boot on a spurious "Cluster identity mismatch".
37
+ * runtime and the `kici-admin cluster reconcile-identity` CLI, including a
38
+ * deploy's pre-start self-heal run of it) MUST fall back to THIS value when no
39
+ * explicit prefix is set, otherwise they compute divergent sentinel keys and
40
+ * crash-loop the orchestrator boot on a spurious "Cluster identity mismatch".
41
41
  */
42
42
  export declare const DEFAULT_CACHE_STORAGE_S3_PREFIX = "";
43
43
  /**
@@ -0,0 +1,45 @@
1
+ import type { Kysely } from 'kysely';
2
+ import { type Logger } from '@kici-dev/shared';
3
+ import type { Database } from '../db/types.js';
4
+ import { PeerCredentialStore } from './peer-credentials.js';
5
+ /** What one reconciliation did. */
6
+ export declare enum CoordinatorCredentialOutcome {
7
+ Valid = "valid",
8
+ Issued = "issued",
9
+ Replaced = "replaced",
10
+ Revoked = "revoked",
11
+ Failed = "failed"
12
+ }
13
+ /** Why a new credential was issued, recorded on the issue log line. */
14
+ export declare enum CredentialIssueReason {
15
+ NoCredential = "no-credential",
16
+ Expired = "expired",
17
+ FileMissing = "credential-file-missing",
18
+ FileMismatch = "credential-file-mismatch"
19
+ }
20
+ export declare const COORDINATOR_CREDENTIAL_REVOKED_MESSAGE = "Peer credential for this coordinator was revoked; not issuing a new one";
21
+ export type CoordinatorCredentialStore = Pick<PeerCredentialStore, 'save' | 'findUnrevokedByInstanceId' | 'findLatestRevokedByInstanceId' | 'retireSelfIssued'>;
22
+ export interface EnsureCoordinatorCredentialOptions {
23
+ store: CoordinatorCredentialStore;
24
+ /** Absolute path of the credential file, with `~` already expanded. */
25
+ credentialFile: string;
26
+ instanceId: string;
27
+ /** Whether an instance has a live `cluster_instances` heartbeat. */
28
+ isInstanceLive: (instanceId: string) => Promise<boolean>;
29
+ logger?: Logger;
30
+ now?: () => Date;
31
+ }
32
+ /**
33
+ * Make sure this coordinator holds a credential its peers accept, issuing one
34
+ * when it has none. Never throws: a failure is logged and reported as `Failed`.
35
+ */
36
+ export declare function ensureCoordinatorCredential(opts: EnsureCoordinatorCredentialOptions): Promise<CoordinatorCredentialOutcome>;
37
+ /** The `selfIssue` callback the coordinator's `PeerAuthCoordinator` runs. */
38
+ export declare function coordinatorSelfIssuer(opts: {
39
+ db: Kysely<Database>;
40
+ credentialFile: string;
41
+ instanceId: string;
42
+ agentMaxReconnectDelayMs: number;
43
+ clusterInstanceHeartbeatMs: number;
44
+ }): () => Promise<CoordinatorCredentialOutcome>;
45
+ //# sourceMappingURL=coordinator-credential.d.ts.map
@@ -334,9 +334,25 @@ export declare class RunCoordinator {
334
334
  * The run-cancel path needs this for a job this coordinator never routed:
335
335
  * the Platform picks any pool member to receive `run.cancel.request`, and only
336
336
  * the coordinator holding the agent's socket can put a cancel frame on it.
337
- * {@link RunCoordinator.cancelRun} covers rerouted jobs and is unchanged.
337
+ * `force` rides on the message, so a force cancel stays forced on the peer.
338
+ * {@link RunCoordinator.cancelRun} covers rerouted jobs.
338
339
  */
339
- cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string): boolean;
340
+ cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string, force?: boolean): boolean;
341
+ /**
342
+ * Send a fire-and-forget message to a peer. The outbound client carries it
343
+ * when connected; otherwise the peer's own connection to this process does.
344
+ * The outbound client is down while it reconnects, and for good on a
345
+ * coordinator whose credential an operator revoked.
346
+ * `PeerClient.send` returns false only when it sent nothing, so the fallback
347
+ * never delivers a message twice.
348
+ */
349
+ private sendToPeer;
350
+ /**
351
+ * The link a reroute uses, chosen before sending. A reroute is never retried
352
+ * over the second link: `false` from a connected client is a NAK or an ack
353
+ * timeout, and resending would deliver the job twice.
354
+ */
355
+ private rerouteTransport;
340
356
  /**
341
357
  * Local fallback for a re-dispatch: dispatch the job on this coordinator under
342
358
  * the existing jobId. Returns true when the local dispatcher accepts it.
@@ -9,6 +9,7 @@ export type { PeerInfo, PeerAgentInfo, PeerRegistryOptions } from './peer-regist
9
9
  export { PeerClient } from './peer-client.js';
10
10
  export type { PeerClientOptions } from './peer-client.js';
11
11
  export { PeerAuthCoordinator } from './peer-auth-coordinator.js';
12
+ export { coordinatorSelfIssuer } from './coordinator-credential.js';
12
13
  export type { AuthDecision, RejectionAction } from './peer-auth-coordinator.js';
13
14
  export { createPeerHandler } from './peer-handler.js';
14
15
  export type { PeerHandlerDeps } from './peer-handler.js';
@@ -1,4 +1,4 @@
1
- import type { Kysely } from 'kysely';
1
+ import { type Kysely } from 'kysely';
2
2
  import type { Database } from '../db/types.js';
3
3
  /** Default interval between `cluster_instances` heartbeat writes. */
4
4
  export declare const DEFAULT_INSTANCE_HEARTBEAT_MS = 10000;
@@ -23,6 +23,11 @@ export declare const DEFAULT_RECOVERY_GRACE_MS = 120000;
23
23
  * recovery machinery would have given it to reconnect.
24
24
  */
25
25
  export declare function instanceLivenessGraceMs(gracePeriodMs: number, heartbeatMs?: number): number;
26
+ /**
27
+ * Whether `instanceId` has a `cluster_instances` heartbeat inside `graceMs`:
28
+ * the same liveness the dispatch plane's ownership predicate reads.
29
+ */
30
+ export declare function isInstanceLive(db: Kysely<Database>, instanceId: string, graceMs: number): Promise<boolean>;
26
31
  export interface InstanceHeartbeatOptions {
27
32
  db: Kysely<Database>;
28
33
  instanceId: string;
@@ -18,6 +18,8 @@ import { Kysely } from 'kysely';
18
18
  * and a stray log line would break the contract).
19
19
  */
20
20
  export declare function silenceJoinTokenLogger(): void;
21
+ export declare const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Create a new token with: kici-admin peer create-token";
22
+ export declare const TOKEN_EXPIRED_MESSAGE = "Join token has expired. Create a new token with: kici-admin peer create-token";
21
23
  interface TokenRouting {
22
24
  orgId: string;
23
25
  routingKey: string;
@@ -79,8 +81,8 @@ export declare class JoinTokenManager {
79
81
  /**
80
82
  * Build a JoinTokenManager backed by its own connection pool to the given
81
83
  * orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl`;
82
- * consumed by E2E tests that need to exercise token validation/reuse against
83
- * the real cluster DB.
84
+ * used by tests that need to exercise token validation/reuse against a real
85
+ * cluster DB.
84
86
  */
85
87
  export declare function createJoinTokenManagerFromUrl(databaseUrl: string, opts?: {
86
88
  maxConnections?: number;
@@ -23,7 +23,8 @@ export interface LeaderGatedSchedulerOptions {
23
23
  /**
24
24
  * Raft-leader-only periodic scheduler. Encapsulates the become -> (optional async setup) ->
25
25
  * start-interval / lose -> stop lifecycle that the orchestrator's leader-gated schedulers share.
26
- * The interval runs only while this node is the leader.
26
+ * The interval runs only while this node is the leader. `stop()` is final: no
27
+ * later leadership change starts another tenure.
27
28
  */
28
29
  export declare class LeaderGatedScheduler {
29
30
  private readonly opts;
@@ -39,6 +40,11 @@ export declare class LeaderGatedScheduler {
39
40
  * would leak a timer running tick() while no longer leader.
40
41
  */
41
42
  private generation;
43
+ /**
44
+ * Set by {@link stop}: the owning process is shutting down, so no later
45
+ * leadership change starts another tenure.
46
+ */
47
+ private stopped;
42
48
  constructor(opts: LeaderGatedSchedulerOptions);
43
49
  get isLeader(): boolean;
44
50
  onBecomeLeader(): Promise<void>;
@@ -1,4 +1,5 @@
1
1
  import { type CredentialFileData } from './peer-credentials.js';
2
+ import { CoordinatorCredentialOutcome } from './coordinator-credential.js';
2
3
  export type AuthDecision = {
3
4
  mode: 'credential';
4
5
  credential: CredentialFileData;
@@ -15,6 +16,11 @@ export declare class PeerAuthCoordinator {
15
16
  private readonly instanceId;
16
17
  private readonly joinToken?;
17
18
  private readonly joinWaitTimeoutMs;
19
+ private readonly selfIssue?;
20
+ /** The issuance in progress; concurrent sibling clients share it. */
21
+ private selfIssueInFlight;
22
+ /** An operator revoked this coordinator's credential: never issue again in this process. */
23
+ private selfIssueRevoked;
18
24
  /** Promise-chain mutex tail; every file op awaits the prior one. */
19
25
  private lock;
20
26
  /** Set while one peer-client is mid token-join; siblings await it. */
@@ -24,11 +30,18 @@ export declare class PeerAuthCoordinator {
24
30
  instanceId: string;
25
31
  joinToken?: string;
26
32
  joinWaitTimeoutMs?: number;
33
+ /**
34
+ * Issues this coordinator its own credential when a decision finds no
35
+ * credential and no join token. Only a coordinator wires it.
36
+ */
37
+ selfIssue?: () => Promise<CoordinatorCredentialOutcome>;
27
38
  });
28
39
  /** Run `fn` exclusively against the credential file. */
29
40
  private withLock;
30
41
  private readValidCredential;
31
42
  decideAuth(): Promise<AuthDecision>;
43
+ /** Run the issuer under the mutex, once for every concurrent caller. */
44
+ private runSelfIssue;
32
45
  private makeComplete;
33
46
  private awaitInFlightJoin;
34
47
  reportRejection(provedCredential: string | null, reason: string): Promise<RejectionAction>;
@@ -13,6 +13,8 @@
13
13
  import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
14
14
  import type { PeerRegistry } from './peer-registry.js';
15
15
  import type { PeerAuthCoordinator } from './peer-auth-coordinator.js';
16
+ /** Logged when this instance has neither a credential file nor a join token. */
17
+ export declare const NO_AUTH_METHOD_MESSAGE = "No peer auth method: this instance has no credential file and no join token";
16
18
  type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
17
19
  export interface PeerClientOptions {
18
20
  /** WebSocket URL of the remote peer orchestrator. */
@@ -108,6 +110,8 @@ export declare class PeerClient {
108
110
  private heartbeatTimer;
109
111
  private reconnectTimer;
110
112
  private reconnectAttempts;
113
+ /** Whether this client already warned that it has no auth method; re-armed by an accepted auth. */
114
+ private noAuthWarned;
111
115
  private intentionalDisconnect;
112
116
  private _targetInstanceId;
113
117
  private sessionKey;
@@ -1,4 +1,15 @@
1
1
  import { Kysely } from 'kysely';
2
+ /** How a credential row was issued, recorded in `peer_credentials.metadata.issuance`. */
3
+ export declare enum PeerCredentialIssuance {
4
+ /** A coordinator issued the credential to itself (`coordinator-credential.ts`). */
5
+ Self = "self"
6
+ }
7
+ /**
8
+ * `metadata` key set when a coordinator retires the self-issued credential its
9
+ * previous run left behind. An operator revoke never sets it, which is how a
10
+ * retirement is told apart from a revoke.
11
+ */
12
+ export declare const RETIRED_BY_INSTANCE_KEY = "retiredByInstance";
2
13
  /**
3
14
  * A peer credential record stored in the coordinator's database.
4
15
  */
@@ -14,6 +25,7 @@ export interface PeerCredential {
14
25
  lastValidatedBy: string | null;
15
26
  expiresAt: Date;
16
27
  revokedAt: Date | null;
28
+ metadata: Record<string, unknown>;
17
29
  }
18
30
  /**
19
31
  * Data persisted in this orchestrator's local credential file
@@ -55,6 +67,7 @@ export declare class PeerCredentialStore {
55
67
  routingKeys: string[];
56
68
  sourceTokenHash?: string;
57
69
  expiryDays?: number;
70
+ metadata?: Record<string, unknown>;
58
71
  }): Promise<{
59
72
  revokedCount: number;
60
73
  }>;
@@ -66,6 +79,24 @@ export declare class PeerCredentialStore {
66
79
  * Find a non-revoked, non-expired credential by instance ID.
67
80
  */
68
81
  findByInstanceId(instanceId: string): Promise<PeerCredential | null>;
82
+ /**
83
+ * The credential row for `instanceId` that nobody revoked, expired or not.
84
+ * `peer_credentials_active_uniq` allows at most one.
85
+ */
86
+ findUnrevokedByInstanceId(instanceId: string): Promise<PeerCredential | null>;
87
+ /** The most recently revoked credential row for `instanceId`, or null. */
88
+ findLatestRevokedByInstanceId(instanceId: string): Promise<PeerCredential | null>;
89
+ /**
90
+ * Revoke `instanceId`'s unrevoked, self-issued row whose hash is
91
+ * `credentialHash`, recording who retired it. Every condition sits in the
92
+ * WHERE clause, so a row that changed meanwhile is left alone.
93
+ * Returns whether a row was retired.
94
+ */
95
+ retireSelfIssued(opts: {
96
+ instanceId: string;
97
+ credentialHash: string;
98
+ retiredByInstance: string;
99
+ }): Promise<boolean>;
69
100
  /**
70
101
  * Update the last_seen_at timestamp for a credential.
71
102
  * Optionally records which coordinator validated it.
@@ -107,7 +138,7 @@ export declare function readCredentialFile(filePath: string): Promise<Credential
107
138
  * store plus a disposer that closes the underlying pool. Lets tests
108
139
  * exercise the real Kysely-backed store without importing pg/kysely.
109
140
  *
110
- * Intended for e2e coverage of the store's own CRUD methods — production
141
+ * Intended for tests of the store's own CRUD methods — production
111
142
  * code constructs the store with an already-allocated Kysely instance
112
143
  * shared with the orchestrator.
113
144
  */
@@ -28,6 +28,12 @@ export declare class RaftStateStore {
28
28
  * Upsert full Raft state into the DB.
29
29
  */
30
30
  save(state: RaftPersistentState): Promise<void>;
31
+ /**
32
+ * Upsert full Raft state unless the stored term is newer. A coordinator that
33
+ * leaves the cluster saves through this: the row is shared, and the remaining
34
+ * coordinators may already have elected a leader in a newer term.
35
+ */
36
+ saveUnlessNewerTerm(state: RaftPersistentState): Promise<void>;
31
37
  /**
32
38
  * Update just the leader and term fields (lightweight update for heartbeat acceptance).
33
39
  */
@@ -4,12 +4,16 @@
4
4
  * Implements only leader election (no log replication) for cluster coordination.
5
5
  * State machine: follower -> candidate -> leader. State is persisted to Postgres
6
6
  * via RaftStateStore for crash recovery. Election timeout is 5-10s with randomized
7
- * jitter (WAN-appropriate per research). Leader sends heartbeats every 2s.
7
+ * jitter (WAN-appropriate). Leader sends heartbeats every 2s.
8
8
  *
9
9
  * Dormant mode: When peerRegistry has 0 connected peers, the node self-elects
10
10
  * immediately (single-orchestrator deployment). If peers connect later, normal
11
11
  * election resumes.
12
12
  *
13
+ * `stop()` takes the node out of every election until the next `start()`: it
14
+ * ends the node's leadership and ignores every election input afterwards, so a
15
+ * process that is shutting down cannot be elected again.
16
+ *
13
17
  * Raft heartbeat (raft.append.entries every 2s) is separate from peer inventory
14
18
  * heartbeat (peer.heartbeat every 30s). Raft heartbeat is purely for leader
15
19
  * liveness detection.
@@ -47,6 +51,13 @@ export declare class RaftNode {
47
51
  private votedFor;
48
52
  private leaderId;
49
53
  private role;
54
+ /**
55
+ * Set by {@link stop}. A stopped node takes no further part in any election:
56
+ * it belongs to a process that is shutting down, and an election would restart
57
+ * the leader-only services the shutdown is stopping and write Raft state to a
58
+ * database that is being closed.
59
+ */
60
+ private stopped;
50
61
  private electionTimer;
51
62
  private leaderHeartbeatTimer;
52
63
  /** Votes received during the current election as candidate. */
@@ -65,7 +76,10 @@ export declare class RaftNode {
65
76
  */
66
77
  start(): Promise<void>;
67
78
  /**
68
- * Stop the Raft node. Clears all timers and persists state.
79
+ * Stop the Raft node. Clears its timers, ends its leadership (so the
80
+ * leader-only services stop with it), and persists its state. Every election
81
+ * input after this is ignored until the next {@link start}. A second call does
82
+ * nothing.
69
83
  */
70
84
  stop(): Promise<void>;
71
85
  /** Whether this node is the current leader. */
@@ -1,11 +1,11 @@
1
1
  /**
2
- * Read-through helper for `access_log` (Orchestrator) — Phase D.
2
+ * Read-through helper for `access_log` (Orchestrator).
3
3
  *
4
4
  * Unlike `audit_log` and `secret_audit_log`, the `access_log` callers
5
5
  * (dashboard "Data access" tab, admin HTTP route, `kici-admin
6
6
  * access-log list` CLI) all use the same cursor-based pagination
7
- * already implemented by `AccessLogWriter.query()`. Phase D's design
8
- * §7 row 11 classifies access_log as "paginated/transparent" — there
7
+ * already implemented by `AccessLogWriter.query()`. access_log is
8
+ * "paginated/transparent" — there
9
9
  * is NO `--include-archived` flag. The helper merges cold rows
10
10
  * automatically when pagination crosses the warm cutoff or the caller
11
11
  * supplies a `from` timestamp older than warm.
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Read-through helper for `event_log` (Orchestrator) — Phase E.
2
+ * Read-through helper for `event_log` (Orchestrator).
3
3
  *
4
4
  * Mirrors the Platform-side `load-event-log-range.ts`. Tenant column is
5
5
  * `routing_key` (NOT NULL on this side) so cold-store partitioning
@@ -1,7 +1,7 @@
1
1
  /**
2
- * Read-through helper for `secret_audit_log` (Orchestrator) — Phase D.
2
+ * Read-through helper for `secret_audit_log` (Orchestrator).
3
3
  *
4
- * Mirrors the Phase B/C reader pattern. Wraps `AuditLogger.query()`'s
4
+ * Mirrors the execution-table reader pattern. Wraps `AuditLogger.query()`'s
5
5
  * existing offset/limit shape; cold-store rows are fetched only when
6
6
  * `includeArchived === true`.
7
7
  *
@@ -1,10 +1,9 @@
1
1
  /**
2
2
  * Orchestrator-side cold-store.
3
3
  *
4
- * Extends BaseColdStore with orchestrator-specific table adapters.
5
- * Phase A registered no adapters; Phase C added execution_runs /
6
- * execution_jobs / execution_steps. Phase D adds secret_audit_log
7
- * and access_log.
4
+ * Extends BaseColdStore with orchestrator-specific table adapters:
5
+ * execution_runs / execution_jobs / execution_steps, secret_audit_log,
6
+ * access_log and event_log.
8
7
  */
9
8
  import { type Kysely } from 'kysely';
10
9
  import { BaseColdStore, type BaseColdStoreDeps, type ColdStoreConfig, type PurgeableChunk } from '@kici-dev/shared';
@@ -17,7 +16,7 @@ export declare class OrchestratorColdStore extends BaseColdStore {
17
16
  private readonly kdb;
18
17
  constructor(deps: OrchestratorColdStoreDeps);
19
18
  /**
20
- * Phase 2 — query the orchestrator's `cold_store_chunks` table for
19
+ * Query the orchestrator's `cold_store_chunks` table for
21
20
  * chunks past their per-row cold-retention horizon. See the matching
22
21
  * Platform-side method for the full rationale.
23
22
  */
@@ -27,7 +26,7 @@ export declare class OrchestratorColdStore extends BaseColdStore {
27
26
  limit: number;
28
27
  }): Promise<PurgeableChunk[]>;
29
28
  /**
30
- * Phase 2 — per-chunk advisory lock keyed on
29
+ * Per-chunk advisory lock keyed on
31
30
  * `hashtext('cold-store-purge|orchestrator|<table>|<chunkId>')`.
32
31
  */
33
32
  protected withPurgeLock<T>(args: {
@@ -37,10 +36,9 @@ export declare class OrchestratorColdStore extends BaseColdStore {
37
36
  }
38
37
  /**
39
38
  * Build the orchestrator cold-store config from env vars. Reads
40
- * `KICI_COLD_STORE_*` vars mirrored with the Platform side. YAML
41
- * integration (per the design doc) lands in a follow-up commit that
42
- * wires the `coldStore` section of the orchestrator config schema;
43
- * Phase A ships the env-var path only.
39
+ * `KICI_COLD_STORE_*` vars mirrored with the Platform side. The
40
+ * orchestrator config schema has no `coldStore` section, so env vars are
41
+ * the only input.
44
42
  */
45
43
  export declare function readOrchestratorColdStoreConfig(env?: NodeJS.ProcessEnv): ColdStoreConfig;
46
44
  /**
@@ -50,7 +48,7 @@ export declare function readOrchestratorColdStoreConfig(env?: NodeJS.ProcessEnv)
50
48
  */
51
49
  export declare function createColdStoreArchiveHandler(instanceId: string, kdb: Kysely<Database>): () => Promise<void>;
52
50
  /**
53
- * Phase 2 — build the cold-store-purge scheduled-job handler. Same
51
+ * Build the cold-store-purge scheduled-job handler. Same
54
52
  * env-var re-read pattern as the archive handler so SIGHUP picks up
55
53
  * config changes mid-run.
56
54
  */
@@ -5,7 +5,7 @@
5
5
  * Contract:
6
6
  * - tenant column: `org_id` (NULL → synthetic `__orchestrator__`)
7
7
  * - partition column: `created_at`
8
- * - warm TTL: 30 days (design §5 matrix row 11)
8
+ * - warm TTL: 30 days
9
9
  *
10
10
  * Replaces the previous 90-day `expires_at`-based hard delete (removed
11
11
  * by migration 007). Rows older than 30 days now live in S3
@@ -57,7 +57,7 @@ export declare class AccessLogAdapter implements TableAdapter<AccessLogColdStore
57
57
  rowId(row: AccessLogColdStoreRow): string | number;
58
58
  rowTimestamp(row: AccessLogColdStoreRow): Date | string;
59
59
  /**
60
- * Phase 2 — per-row cold-retention TTL. Mirrors the warm-side override
60
+ * Per-row cold-retention TTL. Mirrors the warm-side override
61
61
  * layering: outcome=denied/error → 730d (forensic), platform_operator →
62
62
  * `'forever'` (compliance), otherwise per-action map. The framework
63
63
  * groups rows by `coldDaysToBucket(coldTtlDays(row))` at archive time.
@@ -1,25 +1,19 @@
1
1
  /**
2
- * `event_log` cold-store adapter (Orchestrator side) —.
3
- *
4
- * §2 row 6
5
- * and §10.
2
+ * `event_log` cold-store adapter (Orchestrator side).
6
3
  *
7
4
  * Contract:
8
5
  * - tenant column: `routing_key` (NOT NULL on this side — no
9
6
  * synthetic-tenant fallback like access_log / secret_audit_log)
10
7
  * - partition column: `received_at`
11
- * - warm TTL: 30 days (replaces the previous 30-day hard delete in
12
- * `packages/orchestrator/src/webhook/event-log.ts:cleanup` +
13
- * `packages/orchestrator/src/queue/cleanup.ts:runCleanup` step 4)
8
+ * - warm TTL: 30 days (rows are archived, not hard-deleted)
14
9
  *
15
10
  * Per-row payload retention: rows on this table carry a `payload_key`
16
11
  * pointing at a gzipped webhook body in object storage (LogStorage).
17
12
  * Cold-store packages the row metadata — including `payload_key` —
18
13
  * but does NOT touch the body the key points to. The body remains in
19
14
  * S3 indefinitely so the dashboard delivery-detail page resolves
20
- * payload reads identically for hot and cold rows. The pre-Phase-E
21
- * cleanup that deleted both the row and the payload blob in
22
- * lock-step is retired.
15
+ * payload reads identically for hot and cold rows. Nothing deletes the
16
+ * row and the payload blob in lock-step.
23
17
  *
24
18
  * Recursive write: archiving event_log rows writes one access_log
25
19
  * audit row per chunk (the orchestrator's audit surface). The new
@@ -5,7 +5,7 @@
5
5
  * Contract:
6
6
  * - tenant column: `routing_key` (denormalized in migration 006)
7
7
  * - partition column: `created_at`
8
- * - warm TTL: 30 days (design §5 matrix row 8)
8
+ * - warm TTL: 30 days
9
9
  * - eligibility: terminal job status AND no live steps remain for this
10
10
  * (run, job)
11
11
  *
@@ -5,7 +5,7 @@
5
5
  * Contract:
6
6
  * - tenant column: `routing_key`
7
7
  * - partition column: `created_at`
8
- * - warm TTL: 30 days (design §5 matrix row 7)
8
+ * - warm TTL: 30 days
9
9
  * - eligibility: terminal run status AND no live jobs reference this run
10
10
  *
11
11
  * Order: registered LAST in the orchestrator cold-store. Within one
@@ -51,17 +51,17 @@ export declare class ExecutionRunsAdapter implements TableAdapter<ExecutionRunRo
51
51
  rowId(row: ExecutionRunRow): string | number;
52
52
  rowTimestamp(row: ExecutionRunRow): Date | string;
53
53
  /**
54
- * Phase F — natural-key extractor used by `replayRow` so callers
54
+ * Natural-key extractor used by `replayRow` so callers
55
55
  * (the rerun pipeline) can locate the chunk holding a given UUID
56
56
  * `run_id` without knowing the internal SERIAL `id` returned by
57
57
  * `rowId()`.
58
58
  */
59
59
  replayLookupKey(row: ExecutionRunRow): string;
60
60
  /**
61
- * Phase F — replay a chunk's `execution_runs` rows back into
61
+ * Replay a chunk's `execution_runs` rows back into
62
62
  * Orchestrator PG. Mirrors the Platform-side replayInsert; the
63
63
  * audit row goes to `access_log` (not `audit_log`) because the
64
- * orchestrator's audit surface is access_log per design §8.
64
+ * orchestrator's audit surface is access_log.
65
65
  *
66
66
  * Idempotent on duplicate replays via `ON CONFLICT (run_id) DO
67
67
  * NOTHING`. Decrements `cold_store_chunk_counts` (floored at 0).
@@ -5,7 +5,7 @@
5
5
  * Contract:
6
6
  * - tenant column: `routing_key` (denormalized in migration 006)
7
7
  * - partition column: `created_at`
8
- * - warm TTL: 30 days (design §5 matrix row 9)
8
+ * - warm TTL: 30 days
9
9
  * - eligibility: created_at < cutoff (steps are append-only and have no
10
10
  * terminal-status predicate — there is no UPDATE statement against
11
11
  * `execution_steps` in the orchestrator codebase, so age alone is
@@ -5,8 +5,8 @@
5
5
  * Contract:
6
6
  * - tenant column: `routing_key` (NULL → synthetic `__orchestrator__`)
7
7
  * - partition column: `timestamp`
8
- * - warm TTL: 90 days (longer window than other audit tables — design
9
- * §5 matrix row 10. Volume is low, forensic value is high)
8
+ * - warm TTL: 90 days (longer window than other audit tables — volume is
9
+ * low, forensic value is high)
10
10
  *
11
11
  * Synthetic tenant: rows without `routing_key` (e.g. orchestrator-level
12
12
  * key rotations done before any source is bound) collapse to a single
@@ -51,7 +51,7 @@ export declare class SecretAuditLogAdapter implements TableAdapter<SecretAuditLo
51
51
  rowId(row: SecretAuditLogRow): string | number;
52
52
  rowTimestamp(row: SecretAuditLogRow): Date | string;
53
53
  /**
54
- * Phase 2 — per-row cold-retention TTL. Sampled `resolve` /
54
+ * Per-row cold-retention TTL. Sampled `resolve` /
55
55
  * `resolve_named` rows get 180d cold; mutations get `'forever'`. The
56
56
  * denied-outcome override promotes to 730d (forensic).
57
57
  *
package/dist/config.d.ts CHANGED
@@ -15,6 +15,8 @@
15
15
  * and PgSecretStore, not through config. See SourceStore and SourceManager.
16
16
  */
17
17
  import { z } from 'zod';
18
+ /** How long the database pool waits for a connection when `KICI_DB_POOL_ACQUIRE_TIMEOUT_MS` is unset. */
19
+ export declare const DB_POOL_ACQUIRE_TIMEOUT_DEFAULT_MS = 5000;
18
20
  /**
19
21
  * Default for `sealed_secrets_retry_backoff_ms`: how long a coordinator stops
20
22
  * offering a queued job it put back because it could not open its seal. The
@@ -145,6 +147,7 @@ declare const configSchema: z.ZodObject<{
145
147
  reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
146
148
  cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
147
149
  cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
150
+ cacheUploadSettleTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
148
151
  userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
149
152
  userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
150
153
  artifactQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -355,6 +358,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
355
358
  reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
356
359
  cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
357
360
  cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
361
+ cacheUploadSettleTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
358
362
  userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
359
363
  userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
360
364
  artifactQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -484,7 +488,7 @@ export type AppConfig = z.infer<typeof configSchema> & {
484
488
  };
485
489
  /**
486
490
  * Env-var definition for the orchestrator. Exported so the docs generator and
487
- * the deploy-stg pre-validator can re-parse without going through process.env.
491
+ * config pre-validation can re-parse without going through process.env.
488
492
  *
489
493
  * Note: passes the inner `baseSchema` (a ZodObject) so describe() can walk
490
494
  * `.shape`, but uses the outer `configSchema` (with .superRefine) as the parser
@@ -589,6 +593,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
589
593
  reconnectReplayWindowHours: number;
590
594
  cacheBuildTimeoutMs: number;
591
595
  cacheMaxTarballBytes: number;
596
+ cacheUploadSettleTimeoutMs: number;
592
597
  userCacheQuotaBytes: number;
593
598
  userCacheTtlMs: number;
594
599
  artifactQuotaBytes: number;
@@ -321,8 +321,8 @@ export declare class HeldRunStore {
321
321
  /**
322
322
  * Build a HeldRunStore backed by its own connection pool to the given
323
323
  * orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl` /
324
- * `createJoinTokenManagerFromUrl`; consumed by E2E tests that exercise the
325
- * PR-scoped hold selection against the real deployed orchestrator DB.
324
+ * `createJoinTokenManagerFromUrl`; used by tests that exercise the PR-scoped
325
+ * hold selection against a real orchestrator DB.
326
326
  */
327
327
  export declare function createHeldRunStoreFromUrl(databaseUrl: string, opts?: {
328
328
  maxConnections?: number;
@@ -5,7 +5,7 @@ import type { Database } from '../db/types.js';
5
5
  * The org-wide attestations base query: joins `attestations` to `execution_jobs`
6
6
  * (for the job name) and `execution_runs` (for repository / workflow context).
7
7
  *
8
- * `attestations.run_id` / `job_id` are TEXT (P1.5 schema) while the
8
+ * `attestations.run_id` / `job_id` are TEXT while the
9
9
  * `execution_*` keys are `uuid`; Postgres won't compare `uuid = text` implicitly,
10
10
  * so the join casts the uuid side to text — mirroring `resolveAttestationsForRun`.
11
11
  * The `execution_runs` join is a LEFT join so a row with no matching run still