@kici-dev/orchestrator 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__test-helpers__/mock-db-query.d.ts +4 -5
- package/dist/agent/agent-version.d.ts +5 -5
- package/dist/agent/dispatcher.d.ts +26 -0
- package/dist/agent/registry.d.ts +7 -9
- package/dist/agent-packaging/node-runtime-members.d.ts +14 -0
- package/dist/app.d.ts +3 -3
- package/dist/approvals/apply-decision.d.ts +1 -1
- package/dist/audit/access-log.d.ts +2 -3
- package/dist/cli/commands/access-log.d.ts +3 -3
- package/dist/cli/commands/agent-service/agent-env.d.ts +20 -0
- package/dist/cli/commands/agent-service/status.d.ts +3 -3
- package/dist/cli/commands/attestations.d.ts +1 -1
- package/dist/cli/commands/context.d.ts +2 -2
- package/dist/cli/commands/event-dlq.d.ts +5 -6
- package/dist/cli/commands/event-log.d.ts +2 -3
- package/dist/cli/commands/execution.d.ts +1 -3
- package/dist/cli/commands/orchestrator-service/status.d.ts +10 -3
- package/dist/cli/commands/queue.d.ts +1 -2
- package/dist/cli/commands/runs.d.ts +2 -3
- package/dist/cli/commands/service-health.d.ts +68 -0
- package/dist/cli/commands/shared/env-file-mode.d.ts +3 -2
- package/dist/cli/commands/shared/upgrade-safety.d.ts +11 -5
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +119 -7
- package/dist/cli/commands/workflow.d.ts +3 -4
- package/dist/cli/open-browser.d.ts +1 -1
- package/dist/cli/service/entrypoint.d.ts +18 -0
- package/dist/cli/service/image-digests.d.ts +10 -0
- package/dist/cli/service/index.d.ts +4 -2
- package/dist/cli/service/instance/manifest.d.ts +39 -0
- package/dist/cli/service/launchd.d.ts +45 -10
- package/dist/cli/service/shutdown-grace.d.ts +35 -0
- package/dist/cli/service/types.d.ts +9 -0
- package/dist/cli/service/windows-acl.d.ts +40 -0
- package/dist/cli/service/windows-env-file.d.ts +45 -0
- package/dist/cli/service/windows.d.ts +13 -0
- package/dist/cli.js +1620 -551
- package/dist/cluster/cluster-identity.d.ts +4 -4
- package/dist/cluster/join-token.d.ts +2 -2
- package/dist/cluster/leader-gated-scheduler.d.ts +7 -1
- package/dist/cluster/peer-credentials.d.ts +1 -1
- package/dist/cluster/raft-state.d.ts +6 -0
- package/dist/cluster/raft.d.ts +16 -2
- package/dist/cold-store/load-access-log-range.d.ts +3 -3
- package/dist/cold-store/load-event-log-range.d.ts +1 -1
- package/dist/cold-store/load-secret-audit-log-range.d.ts +2 -2
- package/dist/cold-store/orchestrator-cold-store.d.ts +9 -11
- package/dist/cold-store/tables/access-log.d.ts +2 -2
- package/dist/cold-store/tables/event-log.d.ts +4 -10
- package/dist/cold-store/tables/execution-jobs.d.ts +1 -1
- package/dist/cold-store/tables/execution-runs.d.ts +4 -4
- package/dist/cold-store/tables/execution-steps.d.ts +1 -1
- package/dist/cold-store/tables/secret-audit-log.d.ts +3 -3
- package/dist/config.d.ts +3 -1
- package/dist/contexts/held-runs.d.ts +2 -2
- package/dist/dashboard/attestation-filters.d.ts +1 -1
- package/dist/dashboard/handler.d.ts +8 -8
- package/dist/db/deferred-indexes.d.ts +2 -2
- package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +2 -4
- package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +4 -4
- package/dist/db/migrations/007_audit_logs_archived_at.d.ts +6 -9
- package/dist/db/migrations/008_event_log_archived_at.d.ts +4 -6
- package/dist/db/migrations/010_cold_store_chunks.d.ts +2 -2
- package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +2 -4
- package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +2 -2
- package/dist/db/migrations/045_host_properties.d.ts +1 -1
- package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +1 -1
- package/dist/db/migrations/048_host_reboot_pending.d.ts +1 -1
- package/dist/db/migrations/049_held_runs_payload.d.ts +1 -1
- package/dist/db/migrations/050_sources_slug.d.ts +1 -1
- package/dist/db/migrations/051_binding_host_pattern.d.ts +1 -1
- package/dist/db/migrations/052_host_reach_metadata.d.ts +1 -1
- package/dist/db/migrations/053_agent_token_single_use.d.ts +1 -1
- package/dist/db/migrations/054_local_working_tree.d.ts +1 -1
- package/dist/db/migrations/055_agent_token_mandatory_labels.d.ts +1 -1
- package/dist/db/migrations/056_execution_jobs_environments.d.ts +1 -1
- package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +1 -1
- package/dist/db/types.d.ts +12 -13
- package/dist/events/types.d.ts +1 -1
- package/dist/fault-injection-types.d.ts +2 -2
- package/dist/firecracker/host-network.d.ts +21 -3
- package/dist/firecracker/net-interfaces.d.ts +1 -1
- package/dist/helpers/run-detached.d.ts +12 -0
- package/dist/index.js +50 -14
- package/dist/oidc/db-signer.d.ts +1 -1
- package/dist/pipeline/global-eval-round.d.ts +2 -2
- package/dist/pipeline/processor.d.ts +2 -2
- package/dist/provenance/retrier-mint.d.ts +1 -1
- package/dist/provenance/trust-root.d.ts +3 -3
- package/dist/providers/universal-git/clone-token.d.ts +9 -13
- package/dist/providers/universal-git/index.d.ts +1 -2
- package/dist/providers/universal-git/normalizer.d.ts +2 -3
- package/dist/providers/universal-git/ssh-auth.d.ts +4 -4
- package/dist/queue/cleanup.d.ts +2 -2
- package/dist/queue/job-queue.d.ts +6 -6
- package/dist/queue/scheduled-job.d.ts +5 -7
- package/dist/registration/extractor.d.ts +1 -1
- package/dist/registration/registration-index.d.ts +1 -1
- package/dist/reporting/execution-tracker.d.ts +3 -3
- package/dist/routes/admin-access-log.d.ts +1 -1
- package/dist/routes/admin-auth.d.ts +2 -2
- package/dist/routes/admin-event-log.d.ts +2 -2
- package/dist/routes/admin-runs.d.ts +3 -3
- package/dist/routes/admin-scheduled-jobs.d.ts +2 -3
- package/dist/routes/health.d.ts +9 -0
- package/dist/scaler/container-backend.d.ts +1 -1
- package/dist/scaler/firecracker-backend.d.ts +15 -10
- package/dist/scaler/firecracker-extra-hosts.d.ts +20 -0
- package/dist/scaler/machine-ledger.d.ts +3 -4
- package/dist/scaler/scaler-state-store.d.ts +1 -1
- package/dist/scaler/types.d.ts +4 -1
- package/dist/secrets/audit-logger.d.ts +2 -2
- package/dist/security/reduced-privilege-note.d.ts +2 -3
- package/dist/security/trust-policy-store.d.ts +3 -3
- package/dist/server.d.ts +2 -1
- package/dist/server.js +2736 -2211
- package/dist/stale-detector/stale-run-detector.d.ts +1 -1
- package/dist/standalone.d.ts +1 -1
- package/dist/standalone.js +2105 -1629
- package/dist/storage/filesystem.d.ts +1 -1
- package/dist/storage/index.d.ts +1 -1
- package/dist/storage/sign-url.d.ts +2 -1
- package/dist/storage/types.d.ts +2 -2
- package/dist/ws/agent-handler.d.ts +34 -3
- package/dist/ws/agent-heartbeat.d.ts +2 -3
- package/dist/ws/dashboard-context-handler.d.ts +1 -1
- package/dist/ws/platform-client.d.ts +44 -32
- package/dist/ws/server-options.d.ts +1 -1
- package/dist/ws/test-relay-handlers.d.ts +1 -1
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +53 -53
|
@@ -34,10 +34,10 @@ export interface ClusterIdentityDeps {
|
|
|
34
34
|
* `cacheStorageS3Prefix` default). Empty means the cache blobs and the
|
|
35
35
|
* cluster-identity sentinel live at the bucket root — the bucket already scopes
|
|
36
36
|
* the cluster. Every place that resolves a sentinel prefix (the orchestrator
|
|
37
|
-
* runtime
|
|
38
|
-
* deploy's self-heal
|
|
39
|
-
* is set, otherwise they compute divergent sentinel keys and
|
|
40
|
-
* orchestrator boot on a spurious "Cluster identity mismatch".
|
|
37
|
+
* runtime and the `kici-admin cluster reconcile-identity` CLI, including a
|
|
38
|
+
* deploy's pre-start self-heal run of it) MUST fall back to THIS value when no
|
|
39
|
+
* explicit prefix is set, otherwise they compute divergent sentinel keys and
|
|
40
|
+
* crash-loop the orchestrator boot on a spurious "Cluster identity mismatch".
|
|
41
41
|
*/
|
|
42
42
|
export declare const DEFAULT_CACHE_STORAGE_S3_PREFIX = "";
|
|
43
43
|
/**
|
|
@@ -79,8 +79,8 @@ export declare class JoinTokenManager {
|
|
|
79
79
|
/**
|
|
80
80
|
* Build a JoinTokenManager backed by its own connection pool to the given
|
|
81
81
|
* orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl`;
|
|
82
|
-
*
|
|
83
|
-
*
|
|
82
|
+
* used by tests that need to exercise token validation/reuse against a real
|
|
83
|
+
* cluster DB.
|
|
84
84
|
*/
|
|
85
85
|
export declare function createJoinTokenManagerFromUrl(databaseUrl: string, opts?: {
|
|
86
86
|
maxConnections?: number;
|
|
@@ -23,7 +23,8 @@ export interface LeaderGatedSchedulerOptions {
|
|
|
23
23
|
/**
|
|
24
24
|
* Raft-leader-only periodic scheduler. Encapsulates the become -> (optional async setup) ->
|
|
25
25
|
* start-interval / lose -> stop lifecycle that the orchestrator's leader-gated schedulers share.
|
|
26
|
-
* The interval runs only while this node is the leader.
|
|
26
|
+
* The interval runs only while this node is the leader. `stop()` is final: no
|
|
27
|
+
* later leadership change starts another tenure.
|
|
27
28
|
*/
|
|
28
29
|
export declare class LeaderGatedScheduler {
|
|
29
30
|
private readonly opts;
|
|
@@ -39,6 +40,11 @@ export declare class LeaderGatedScheduler {
|
|
|
39
40
|
* would leak a timer running tick() while no longer leader.
|
|
40
41
|
*/
|
|
41
42
|
private generation;
|
|
43
|
+
/**
|
|
44
|
+
* Set by {@link stop}: the owning process is shutting down, so no later
|
|
45
|
+
* leadership change starts another tenure.
|
|
46
|
+
*/
|
|
47
|
+
private stopped;
|
|
42
48
|
constructor(opts: LeaderGatedSchedulerOptions);
|
|
43
49
|
get isLeader(): boolean;
|
|
44
50
|
onBecomeLeader(): Promise<void>;
|
|
@@ -107,7 +107,7 @@ export declare function readCredentialFile(filePath: string): Promise<Credential
|
|
|
107
107
|
* store plus a disposer that closes the underlying pool. Lets tests
|
|
108
108
|
* exercise the real Kysely-backed store without importing pg/kysely.
|
|
109
109
|
*
|
|
110
|
-
* Intended for
|
|
110
|
+
* Intended for tests of the store's own CRUD methods — production
|
|
111
111
|
* code constructs the store with an already-allocated Kysely instance
|
|
112
112
|
* shared with the orchestrator.
|
|
113
113
|
*/
|
|
@@ -28,6 +28,12 @@ export declare class RaftStateStore {
|
|
|
28
28
|
* Upsert full Raft state into the DB.
|
|
29
29
|
*/
|
|
30
30
|
save(state: RaftPersistentState): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Upsert full Raft state unless the stored term is newer. A coordinator that
|
|
33
|
+
* leaves the cluster saves through this: the row is shared, and the remaining
|
|
34
|
+
* coordinators may already have elected a leader in a newer term.
|
|
35
|
+
*/
|
|
36
|
+
saveUnlessNewerTerm(state: RaftPersistentState): Promise<void>;
|
|
31
37
|
/**
|
|
32
38
|
* Update just the leader and term fields (lightweight update for heartbeat acceptance).
|
|
33
39
|
*/
|
package/dist/cluster/raft.d.ts
CHANGED
|
@@ -4,12 +4,16 @@
|
|
|
4
4
|
* Implements only leader election (no log replication) for cluster coordination.
|
|
5
5
|
* State machine: follower -> candidate -> leader. State is persisted to Postgres
|
|
6
6
|
* via RaftStateStore for crash recovery. Election timeout is 5-10s with randomized
|
|
7
|
-
* jitter (WAN-appropriate
|
|
7
|
+
* jitter (WAN-appropriate). Leader sends heartbeats every 2s.
|
|
8
8
|
*
|
|
9
9
|
* Dormant mode: When peerRegistry has 0 connected peers, the node self-elects
|
|
10
10
|
* immediately (single-orchestrator deployment). If peers connect later, normal
|
|
11
11
|
* election resumes.
|
|
12
12
|
*
|
|
13
|
+
* `stop()` takes the node out of every election until the next `start()`: it
|
|
14
|
+
* ends the node's leadership and ignores every election input afterwards, so a
|
|
15
|
+
* process that is shutting down cannot be elected again.
|
|
16
|
+
*
|
|
13
17
|
* Raft heartbeat (raft.append.entries every 2s) is separate from peer inventory
|
|
14
18
|
* heartbeat (peer.heartbeat every 30s). Raft heartbeat is purely for leader
|
|
15
19
|
* liveness detection.
|
|
@@ -47,6 +51,13 @@ export declare class RaftNode {
|
|
|
47
51
|
private votedFor;
|
|
48
52
|
private leaderId;
|
|
49
53
|
private role;
|
|
54
|
+
/**
|
|
55
|
+
* Set by {@link stop}. A stopped node takes no further part in any election:
|
|
56
|
+
* it belongs to a process that is shutting down, and an election would restart
|
|
57
|
+
* the leader-only services the shutdown is stopping and write Raft state to a
|
|
58
|
+
* database that is being closed.
|
|
59
|
+
*/
|
|
60
|
+
private stopped;
|
|
50
61
|
private electionTimer;
|
|
51
62
|
private leaderHeartbeatTimer;
|
|
52
63
|
/** Votes received during the current election as candidate. */
|
|
@@ -65,7 +76,10 @@ export declare class RaftNode {
|
|
|
65
76
|
*/
|
|
66
77
|
start(): Promise<void>;
|
|
67
78
|
/**
|
|
68
|
-
* Stop the Raft node. Clears
|
|
79
|
+
* Stop the Raft node. Clears its timers, ends its leadership (so the
|
|
80
|
+
* leader-only services stop with it), and persists its state. Every election
|
|
81
|
+
* input after this is ignored until the next {@link start}. A second call does
|
|
82
|
+
* nothing.
|
|
69
83
|
*/
|
|
70
84
|
stop(): Promise<void>;
|
|
71
85
|
/** Whether this node is the current leader. */
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Read-through helper for `access_log` (Orchestrator)
|
|
2
|
+
* Read-through helper for `access_log` (Orchestrator).
|
|
3
3
|
*
|
|
4
4
|
* Unlike `audit_log` and `secret_audit_log`, the `access_log` callers
|
|
5
5
|
* (dashboard "Data access" tab, admin HTTP route, `kici-admin
|
|
6
6
|
* access-log list` CLI) all use the same cursor-based pagination
|
|
7
|
-
* already implemented by `AccessLogWriter.query()`.
|
|
8
|
-
*
|
|
7
|
+
* already implemented by `AccessLogWriter.query()`. access_log is
|
|
8
|
+
* "paginated/transparent" — there
|
|
9
9
|
* is NO `--include-archived` flag. The helper merges cold rows
|
|
10
10
|
* automatically when pagination crosses the warm cutoff or the caller
|
|
11
11
|
* supplies a `from` timestamp older than warm.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Read-through helper for `event_log` (Orchestrator)
|
|
2
|
+
* Read-through helper for `event_log` (Orchestrator).
|
|
3
3
|
*
|
|
4
4
|
* Mirrors the Platform-side `load-event-log-range.ts`. Tenant column is
|
|
5
5
|
* `routing_key` (NOT NULL on this side) so cold-store partitioning
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Read-through helper for `secret_audit_log` (Orchestrator)
|
|
2
|
+
* Read-through helper for `secret_audit_log` (Orchestrator).
|
|
3
3
|
*
|
|
4
|
-
* Mirrors the
|
|
4
|
+
* Mirrors the execution-table reader pattern. Wraps `AuditLogger.query()`'s
|
|
5
5
|
* existing offset/limit shape; cold-store rows are fetched only when
|
|
6
6
|
* `includeArchived === true`.
|
|
7
7
|
*
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Orchestrator-side cold-store.
|
|
3
3
|
*
|
|
4
|
-
* Extends BaseColdStore with orchestrator-specific table adapters
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* and access_log.
|
|
4
|
+
* Extends BaseColdStore with orchestrator-specific table adapters:
|
|
5
|
+
* execution_runs / execution_jobs / execution_steps, secret_audit_log,
|
|
6
|
+
* access_log and event_log.
|
|
8
7
|
*/
|
|
9
8
|
import { type Kysely } from 'kysely';
|
|
10
9
|
import { BaseColdStore, type BaseColdStoreDeps, type ColdStoreConfig, type PurgeableChunk } from '@kici-dev/shared';
|
|
@@ -17,7 +16,7 @@ export declare class OrchestratorColdStore extends BaseColdStore {
|
|
|
17
16
|
private readonly kdb;
|
|
18
17
|
constructor(deps: OrchestratorColdStoreDeps);
|
|
19
18
|
/**
|
|
20
|
-
*
|
|
19
|
+
* Query the orchestrator's `cold_store_chunks` table for
|
|
21
20
|
* chunks past their per-row cold-retention horizon. See the matching
|
|
22
21
|
* Platform-side method for the full rationale.
|
|
23
22
|
*/
|
|
@@ -27,7 +26,7 @@ export declare class OrchestratorColdStore extends BaseColdStore {
|
|
|
27
26
|
limit: number;
|
|
28
27
|
}): Promise<PurgeableChunk[]>;
|
|
29
28
|
/**
|
|
30
|
-
*
|
|
29
|
+
* Per-chunk advisory lock keyed on
|
|
31
30
|
* `hashtext('cold-store-purge|orchestrator|<table>|<chunkId>')`.
|
|
32
31
|
*/
|
|
33
32
|
protected withPurgeLock<T>(args: {
|
|
@@ -37,10 +36,9 @@ export declare class OrchestratorColdStore extends BaseColdStore {
|
|
|
37
36
|
}
|
|
38
37
|
/**
|
|
39
38
|
* Build the orchestrator cold-store config from env vars. Reads
|
|
40
|
-
* `KICI_COLD_STORE_*` vars mirrored with the Platform side.
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* Phase A ships the env-var path only.
|
|
39
|
+
* `KICI_COLD_STORE_*` vars mirrored with the Platform side. The
|
|
40
|
+
* orchestrator config schema has no `coldStore` section, so env vars are
|
|
41
|
+
* the only input.
|
|
44
42
|
*/
|
|
45
43
|
export declare function readOrchestratorColdStoreConfig(env?: NodeJS.ProcessEnv): ColdStoreConfig;
|
|
46
44
|
/**
|
|
@@ -50,7 +48,7 @@ export declare function readOrchestratorColdStoreConfig(env?: NodeJS.ProcessEnv)
|
|
|
50
48
|
*/
|
|
51
49
|
export declare function createColdStoreArchiveHandler(instanceId: string, kdb: Kysely<Database>): () => Promise<void>;
|
|
52
50
|
/**
|
|
53
|
-
*
|
|
51
|
+
* Build the cold-store-purge scheduled-job handler. Same
|
|
54
52
|
* env-var re-read pattern as the archive handler so SIGHUP picks up
|
|
55
53
|
* config changes mid-run.
|
|
56
54
|
*/
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Contract:
|
|
6
6
|
* - tenant column: `org_id` (NULL → synthetic `__orchestrator__`)
|
|
7
7
|
* - partition column: `created_at`
|
|
8
|
-
* - warm TTL: 30 days
|
|
8
|
+
* - warm TTL: 30 days
|
|
9
9
|
*
|
|
10
10
|
* Replaces the previous 90-day `expires_at`-based hard delete (removed
|
|
11
11
|
* by migration 007). Rows older than 30 days now live in S3
|
|
@@ -57,7 +57,7 @@ export declare class AccessLogAdapter implements TableAdapter<AccessLogColdStore
|
|
|
57
57
|
rowId(row: AccessLogColdStoreRow): string | number;
|
|
58
58
|
rowTimestamp(row: AccessLogColdStoreRow): Date | string;
|
|
59
59
|
/**
|
|
60
|
-
*
|
|
60
|
+
* Per-row cold-retention TTL. Mirrors the warm-side override
|
|
61
61
|
* layering: outcome=denied/error → 730d (forensic), platform_operator →
|
|
62
62
|
* `'forever'` (compliance), otherwise per-action map. The framework
|
|
63
63
|
* groups rows by `coldDaysToBucket(coldTtlDays(row))` at archive time.
|
|
@@ -1,25 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `event_log` cold-store adapter (Orchestrator side)
|
|
3
|
-
*
|
|
4
|
-
* §2 row 6
|
|
5
|
-
* and §10.
|
|
2
|
+
* `event_log` cold-store adapter (Orchestrator side).
|
|
6
3
|
*
|
|
7
4
|
* Contract:
|
|
8
5
|
* - tenant column: `routing_key` (NOT NULL on this side — no
|
|
9
6
|
* synthetic-tenant fallback like access_log / secret_audit_log)
|
|
10
7
|
* - partition column: `received_at`
|
|
11
|
-
* - warm TTL: 30 days (
|
|
12
|
-
* `packages/orchestrator/src/webhook/event-log.ts:cleanup` +
|
|
13
|
-
* `packages/orchestrator/src/queue/cleanup.ts:runCleanup` step 4)
|
|
8
|
+
* - warm TTL: 30 days (rows are archived, not hard-deleted)
|
|
14
9
|
*
|
|
15
10
|
* Per-row payload retention: rows on this table carry a `payload_key`
|
|
16
11
|
* pointing at a gzipped webhook body in object storage (LogStorage).
|
|
17
12
|
* Cold-store packages the row metadata — including `payload_key` —
|
|
18
13
|
* but does NOT touch the body the key points to. The body remains in
|
|
19
14
|
* S3 indefinitely so the dashboard delivery-detail page resolves
|
|
20
|
-
* payload reads identically for hot and cold rows.
|
|
21
|
-
*
|
|
22
|
-
* lock-step is retired.
|
|
15
|
+
* payload reads identically for hot and cold rows. Nothing deletes the
|
|
16
|
+
* row and the payload blob in lock-step.
|
|
23
17
|
*
|
|
24
18
|
* Recursive write: archiving event_log rows writes one access_log
|
|
25
19
|
* audit row per chunk (the orchestrator's audit surface). The new
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Contract:
|
|
6
6
|
* - tenant column: `routing_key` (denormalized in migration 006)
|
|
7
7
|
* - partition column: `created_at`
|
|
8
|
-
* - warm TTL: 30 days
|
|
8
|
+
* - warm TTL: 30 days
|
|
9
9
|
* - eligibility: terminal job status AND no live steps remain for this
|
|
10
10
|
* (run, job)
|
|
11
11
|
*
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Contract:
|
|
6
6
|
* - tenant column: `routing_key`
|
|
7
7
|
* - partition column: `created_at`
|
|
8
|
-
* - warm TTL: 30 days
|
|
8
|
+
* - warm TTL: 30 days
|
|
9
9
|
* - eligibility: terminal run status AND no live jobs reference this run
|
|
10
10
|
*
|
|
11
11
|
* Order: registered LAST in the orchestrator cold-store. Within one
|
|
@@ -51,17 +51,17 @@ export declare class ExecutionRunsAdapter implements TableAdapter<ExecutionRunRo
|
|
|
51
51
|
rowId(row: ExecutionRunRow): string | number;
|
|
52
52
|
rowTimestamp(row: ExecutionRunRow): Date | string;
|
|
53
53
|
/**
|
|
54
|
-
*
|
|
54
|
+
* Natural-key extractor used by `replayRow` so callers
|
|
55
55
|
* (the rerun pipeline) can locate the chunk holding a given UUID
|
|
56
56
|
* `run_id` without knowing the internal SERIAL `id` returned by
|
|
57
57
|
* `rowId()`.
|
|
58
58
|
*/
|
|
59
59
|
replayLookupKey(row: ExecutionRunRow): string;
|
|
60
60
|
/**
|
|
61
|
-
*
|
|
61
|
+
* Replay a chunk's `execution_runs` rows back into
|
|
62
62
|
* Orchestrator PG. Mirrors the Platform-side replayInsert; the
|
|
63
63
|
* audit row goes to `access_log` (not `audit_log`) because the
|
|
64
|
-
* orchestrator's audit surface is access_log
|
|
64
|
+
* orchestrator's audit surface is access_log.
|
|
65
65
|
*
|
|
66
66
|
* Idempotent on duplicate replays via `ON CONFLICT (run_id) DO
|
|
67
67
|
* NOTHING`. Decrements `cold_store_chunk_counts` (floored at 0).
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Contract:
|
|
6
6
|
* - tenant column: `routing_key` (denormalized in migration 006)
|
|
7
7
|
* - partition column: `created_at`
|
|
8
|
-
* - warm TTL: 30 days
|
|
8
|
+
* - warm TTL: 30 days
|
|
9
9
|
* - eligibility: created_at < cutoff (steps are append-only and have no
|
|
10
10
|
* terminal-status predicate — there is no UPDATE statement against
|
|
11
11
|
* `execution_steps` in the orchestrator codebase, so age alone is
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* Contract:
|
|
6
6
|
* - tenant column: `routing_key` (NULL → synthetic `__orchestrator__`)
|
|
7
7
|
* - partition column: `timestamp`
|
|
8
|
-
* - warm TTL: 90 days (longer window than other audit tables —
|
|
9
|
-
*
|
|
8
|
+
* - warm TTL: 90 days (longer window than other audit tables — volume is
|
|
9
|
+
* low, forensic value is high)
|
|
10
10
|
*
|
|
11
11
|
* Synthetic tenant: rows without `routing_key` (e.g. orchestrator-level
|
|
12
12
|
* key rotations done before any source is bound) collapse to a single
|
|
@@ -51,7 +51,7 @@ export declare class SecretAuditLogAdapter implements TableAdapter<SecretAuditLo
|
|
|
51
51
|
rowId(row: SecretAuditLogRow): string | number;
|
|
52
52
|
rowTimestamp(row: SecretAuditLogRow): Date | string;
|
|
53
53
|
/**
|
|
54
|
-
*
|
|
54
|
+
* Per-row cold-retention TTL. Sampled `resolve` /
|
|
55
55
|
* `resolve_named` rows get 180d cold; mutations get `'forever'`. The
|
|
56
56
|
* denied-outcome override promotes to 730d (forensic).
|
|
57
57
|
*
|
package/dist/config.d.ts
CHANGED
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
* and PgSecretStore, not through config. See SourceStore and SourceManager.
|
|
16
16
|
*/
|
|
17
17
|
import { z } from 'zod';
|
|
18
|
+
/** How long the database pool waits for a connection when `KICI_DB_POOL_ACQUIRE_TIMEOUT_MS` is unset. */
|
|
19
|
+
export declare const DB_POOL_ACQUIRE_TIMEOUT_DEFAULT_MS = 5000;
|
|
18
20
|
/**
|
|
19
21
|
* Default for `sealed_secrets_retry_backoff_ms`: how long a coordinator stops
|
|
20
22
|
* offering a queued job it put back because it could not open its seal. The
|
|
@@ -484,7 +486,7 @@ export type AppConfig = z.infer<typeof configSchema> & {
|
|
|
484
486
|
};
|
|
485
487
|
/**
|
|
486
488
|
* Env-var definition for the orchestrator. Exported so the docs generator and
|
|
487
|
-
*
|
|
489
|
+
* config pre-validation can re-parse without going through process.env.
|
|
488
490
|
*
|
|
489
491
|
* Note: passes the inner `baseSchema` (a ZodObject) so describe() can walk
|
|
490
492
|
* `.shape`, but uses the outer `configSchema` (with .superRefine) as the parser
|
|
@@ -321,8 +321,8 @@ export declare class HeldRunStore {
|
|
|
321
321
|
/**
|
|
322
322
|
* Build a HeldRunStore backed by its own connection pool to the given
|
|
323
323
|
* orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl` /
|
|
324
|
-
* `createJoinTokenManagerFromUrl`;
|
|
325
|
-
*
|
|
324
|
+
* `createJoinTokenManagerFromUrl`; used by tests that exercise the PR-scoped
|
|
325
|
+
* hold selection against a real orchestrator DB.
|
|
326
326
|
*/
|
|
327
327
|
export declare function createHeldRunStoreFromUrl(databaseUrl: string, opts?: {
|
|
328
328
|
maxConnections?: number;
|
|
@@ -5,7 +5,7 @@ import type { Database } from '../db/types.js';
|
|
|
5
5
|
* The org-wide attestations base query: joins `attestations` to `execution_jobs`
|
|
6
6
|
* (for the job name) and `execution_runs` (for repository / workflow context).
|
|
7
7
|
*
|
|
8
|
-
* `attestations.run_id` / `job_id` are TEXT
|
|
8
|
+
* `attestations.run_id` / `job_id` are TEXT while the
|
|
9
9
|
* `execution_*` keys are `uuid`; Postgres won't compare `uuid = text` implicitly,
|
|
10
10
|
* so the join casts the uuid side to text — mirroring `resolveAttestationsForRun`.
|
|
11
11
|
* The `execution_runs` join is a LEFT join so a row with no matching run still
|
|
@@ -23,7 +23,7 @@ interface DashboardHandlerDeps {
|
|
|
23
23
|
* so the dashboard verifies it client-side without a second fetch. Optional —
|
|
24
24
|
* when absent (e.g. orchestrators without provenance configured) the handler
|
|
25
25
|
* replies with an `error` and an empty list. Reuses the cache storage backend
|
|
26
|
-
* (the same one
|
|
26
|
+
* (the same one attestation bundles are written to).
|
|
27
27
|
*/
|
|
28
28
|
provenanceStorage?: CacheStorage | null;
|
|
29
29
|
/**
|
|
@@ -58,14 +58,14 @@ interface DashboardHandlerDeps {
|
|
|
58
58
|
* Long-lived cold-store handle for read fallback. When set, run /
|
|
59
59
|
* job / step lookups fall through to S3 if the row has aged out of
|
|
60
60
|
* PG. `null` means the orchestrator is running without cold-store
|
|
61
|
-
* configured — handler then
|
|
61
|
+
* configured — handler then serves PG rows only.
|
|
62
62
|
*/
|
|
63
63
|
coldStore?: ColdStore | null;
|
|
64
64
|
/**
|
|
65
65
|
* Callback for handling re-run requests.
|
|
66
66
|
* Returns { newRunId } on success or throws on failure.
|
|
67
67
|
*
|
|
68
|
-
*
|
|
68
|
+
* `routingKey` is forwarded from the Platform (read from
|
|
69
69
|
* Platform's denormalized `execution_runs.routing_key`) so the
|
|
70
70
|
* orchestrator can probe its cold-store under the right tenant
|
|
71
71
|
* prefix when the run row is missing from PG. Optional for back-
|
|
@@ -114,7 +114,7 @@ interface DashboardHandlerDeps {
|
|
|
114
114
|
/**
|
|
115
115
|
* **Test-only fault injection**, supplied only by the build-time test double:
|
|
116
116
|
* `handleRerunRequest` awaits this hook before invoking `onRerun`, so an HA
|
|
117
|
-
*
|
|
117
|
+
* test can force a slow first coordinator and trigger a relay failover.
|
|
118
118
|
* Undefined (the shipped default) means no delay.
|
|
119
119
|
*/
|
|
120
120
|
beforeRerun?: () => Promise<void>;
|
|
@@ -167,7 +167,7 @@ export declare class DashboardHandler {
|
|
|
167
167
|
* Resolve the org owning a routing key by consulting both source tables
|
|
168
168
|
* (GitHub-app `sources` first, then `generic_webhook_sources`). Used
|
|
169
169
|
* directly for `handleRerunRequest` (which gets a `routingKey` hint from
|
|
170
|
-
* Platform
|
|
170
|
+
* Platform's denormalized copy) and indirectly via
|
|
171
171
|
* `resolveOrgForRun`.
|
|
172
172
|
*/
|
|
173
173
|
private resolveOrgForRoutingKey;
|
|
@@ -206,7 +206,7 @@ export declare class DashboardHandler {
|
|
|
206
206
|
handleRunDetail(msg: DashboardRunDetailRequest): Promise<void>;
|
|
207
207
|
/**
|
|
208
208
|
* Handle a dashboard.run.structured request — the user-plane equivalent of
|
|
209
|
-
* the orchestrator-admin `/runs/:id/structured` endpoint. Reuses the
|
|
209
|
+
* the orchestrator-admin `/runs/:id/structured` endpoint. Reuses the
|
|
210
210
|
* aggregator + provenance mapper so the result is byte-identical to the admin
|
|
211
211
|
* surface; emits an `access_log` `run.structured.read` row (carrying the
|
|
212
212
|
* agent label when the actor came through an agent PAT).
|
|
@@ -396,7 +396,7 @@ export declare class DashboardHandler {
|
|
|
396
396
|
* Handle a run.rerun.request.
|
|
397
397
|
* Delegates to the onRerun callback which invokes handleRerun from the rerun module.
|
|
398
398
|
*
|
|
399
|
-
* Resolution: prefer `msg.routingKey` (
|
|
399
|
+
* Resolution: prefer `msg.routingKey` (denormalization hint from
|
|
400
400
|
* Platform — `execution_runs.routing_key` mirror). The hint is the ONLY
|
|
401
401
|
* org context we have when the run row has been cold-archived out of PG,
|
|
402
402
|
* so consult sources/generic_webhook_sources directly. If the hint is
|
|
@@ -512,7 +512,7 @@ export declare class DashboardHandler {
|
|
|
512
512
|
*/
|
|
513
513
|
handleManualScheduleRequest(msg: ManualScheduleRequest): Promise<void>;
|
|
514
514
|
/**
|
|
515
|
-
*
|
|
515
|
+
* Cold-store fallback: fetch the run + its jobs + its steps
|
|
516
516
|
* from S3 when the PG copy has been archived. Bounded scan: stops at
|
|
517
517
|
* the warm cutoff (older entries don't exist in cold-store yet by
|
|
518
518
|
* construction). Each row is filtered by `run_id` because chunks
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
* server is already serving. `IF NOT EXISTS` makes every build idempotent, and
|
|
16
16
|
* a failed build retries on the next boot.
|
|
17
17
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
18
|
+
* A repository check enforces that new indexes on those tables land here
|
|
19
|
+
* rather than in a migration.
|
|
20
20
|
*/
|
|
21
21
|
import type pg from 'pg';
|
|
22
22
|
import { type Logger } from '@kici-dev/shared';
|
|
@@ -7,12 +7,10 @@ import { type Kysely } from 'kysely';
|
|
|
7
7
|
* `kici-admin cold-store list-chunks` CLI can report totals without
|
|
8
8
|
* issuing S3 LIST calls (expensive + rate-limited).
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* The table starts empty; the cold-store archiver
|
|
11
|
+
* updates it transactionally with every successful chunk
|
|
12
12
|
* write via `INSERT ... ON CONFLICT (db, table_name, tenant_id) DO
|
|
13
13
|
* UPDATE SET chunk_count = chunk_count + EXCLUDED.chunk_count, ...`.
|
|
14
|
-
*
|
|
15
|
-
* sections 5 and 8.
|
|
16
14
|
*/
|
|
17
15
|
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
18
16
|
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
@@ -3,12 +3,12 @@ import { type Kysely } from 'kysely';
|
|
|
3
3
|
* `execution_runs` / `execution_jobs` / `execution_steps` cold-store
|
|
4
4
|
* schema additions.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* Part of the cold-storage archival system.
|
|
7
7
|
*
|
|
8
8
|
* Each table gets `archived_at TIMESTAMPTZ NULL` and
|
|
9
9
|
* `archive_object_key TEXT NULL` (set inside the archive tx before the
|
|
10
|
-
* DELETE; survivors carry NULL — same pattern
|
|
11
|
-
*
|
|
10
|
+
* DELETE; survivors carry NULL — the same pattern `run_events` carries
|
|
11
|
+
* in the Platform's schema).
|
|
12
12
|
*
|
|
13
13
|
* Tenant column denormalization: the orchestrator side partitions by
|
|
14
14
|
* `routing_key`, but only `execution_runs` carries the column today.
|
|
@@ -17,7 +17,7 @@ import { type Kysely } from 'kysely';
|
|
|
17
17
|
* denormalize `routing_key` onto both tables and backfill from the
|
|
18
18
|
* existing run rows. Backfill is one-shot inside this migration. New
|
|
19
19
|
* inserts must populate the column going forward (enforced at the
|
|
20
|
-
* application layer
|
|
20
|
+
* application layer).
|
|
21
21
|
*
|
|
22
22
|
* The denormalized column is left NULLable for safety: if an insert
|
|
23
23
|
* site is missed in the rollout, the row still lands in PG; the
|
|
@@ -3,13 +3,12 @@ import { type Kysely } from 'kysely';
|
|
|
3
3
|
* `secret_audit_log` and `access_log` cold-store schema additions, plus
|
|
4
4
|
* the removal of `access_log`'s `expires_at`-based hard delete.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* Part of the cold-storage archival system.
|
|
7
7
|
*
|
|
8
8
|
* Both tables get the same `archived_at TIMESTAMPTZ NULL` /
|
|
9
|
-
* `archive_object_key TEXT NULL` pair that
|
|
10
|
-
* (`
|
|
11
|
-
*
|
|
12
|
-
* `execution_steps` (`006_runs_jobs_steps_archived_at.ts`). Set inside
|
|
9
|
+
* `archive_object_key TEXT NULL` pair that `run_events` carries
|
|
10
|
+
* (in the Platform's schema) and that `execution_runs` / `execution_jobs` /
|
|
11
|
+
* `execution_steps` got in `006_runs_jobs_steps_archived_at.ts`. Set inside
|
|
13
12
|
* the archive transaction before the DELETE; survivors carry NULL.
|
|
14
13
|
*
|
|
15
14
|
* `secret_audit_log` gets a composite
|
|
@@ -23,7 +22,7 @@ import { type Kysely } from 'kysely';
|
|
|
23
22
|
* (range scans use either direction).
|
|
24
23
|
*
|
|
25
24
|
* Drops `access_log.expires_at` column and `access_log_expires_idx`.
|
|
26
|
-
*
|
|
25
|
+
* Before this migration, `expires_at` powered a 90-day hard-delete sweep in
|
|
27
26
|
* `packages/orchestrator/src/audit/access-log.ts:cleanup` (called from
|
|
28
27
|
* `packages/orchestrator/src/queue/cleanup.ts:runCleanup` step 5). With
|
|
29
28
|
* cold-store, `access_log` rows older than 30 days are archived rather
|
|
@@ -31,11 +30,9 @@ import { type Kysely } from 'kysely';
|
|
|
31
30
|
* column would mean either (a) double-evicting rows or (b) silently
|
|
32
31
|
* mismatching the new contract. Drop it.
|
|
33
32
|
*
|
|
34
|
-
* Per the project's "no backward compatibility (pre-release)" rule,
|
|
35
33
|
* `down()` recreates `expires_at` with a default of `now() + 90 days`
|
|
36
34
|
* — best effort; rows inserted between `up()` and a hypothetical
|
|
37
|
-
* `down()` would not have meaningful retention bounds.
|
|
38
|
-
* staging.
|
|
35
|
+
* `down()` would not have meaningful retention bounds.
|
|
39
36
|
*/
|
|
40
37
|
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
41
38
|
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
@@ -3,13 +3,11 @@ import { type Kysely } from 'kysely';
|
|
|
3
3
|
* `event_log` cold-store schema additions plus removal of the
|
|
4
4
|
* `expires_at`-based 30-day hard delete.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
* §10
|
|
8
|
-
* Phase E.
|
|
6
|
+
* Part of the cold-storage archival system.
|
|
9
7
|
*
|
|
10
8
|
* Adds the same `archived_at TIMESTAMPTZ NULL` /
|
|
11
|
-
* `archive_object_key TEXT NULL` pair
|
|
12
|
-
*
|
|
9
|
+
* `archive_object_key TEXT NULL` pair
|
|
10
|
+
* `007_audit_logs_archived_at.ts` added on this side. Set inside the archive
|
|
13
11
|
* transaction before the DELETE; survivors carry NULL.
|
|
14
12
|
*
|
|
15
13
|
* Adds `idx_event_log_routing_received` on `(routing_key, received_at)`
|
|
@@ -20,7 +18,7 @@ import { type Kysely } from 'kysely';
|
|
|
20
18
|
* archiver scan.
|
|
21
19
|
*
|
|
22
20
|
* Drops `event_log.expires_at` and the matching
|
|
23
|
-
* `event_log_expires_at_idx` index.
|
|
21
|
+
* `event_log_expires_at_idx` index. Before this migration, `expires_at`
|
|
24
22
|
* powered the 30-day hard-delete sweep in
|
|
25
23
|
* `packages/orchestrator/src/webhook/event-log.ts:cleanup`, called
|
|
26
24
|
* from `packages/orchestrator/src/queue/cleanup.ts:runCleanup` step 4
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Kysely } from 'kysely';
|
|
2
2
|
/**
|
|
3
|
-
* Cold-store chunk index
|
|
3
|
+
* Cold-store chunk index (cold-store purge).
|
|
4
4
|
*
|
|
5
5
|
* One row per archived chunk that the GC sweep can later purge from S3.
|
|
6
6
|
* Inserted inside the same transaction as `markArchivedAndDelete` so a
|
|
@@ -25,7 +25,7 @@ import { type Kysely } from 'kysely';
|
|
|
25
25
|
* uses this to issue `DeleteObject` directly without recomputing the
|
|
26
26
|
* key from the prefix.
|
|
27
27
|
*
|
|
28
|
-
*
|
|
28
|
+
* V1-manifest chunks are NOT in this table — they live
|
|
29
29
|
* forever. Adapters that don't opt into per-bucket archival via
|
|
30
30
|
* `coldTtlDays` don't insert here either.
|
|
31
31
|
*/
|
|
@@ -12,13 +12,11 @@ import { type Kysely } from 'kysely';
|
|
|
12
12
|
* nothing and both `INSERT`s succeeded, leaving 2+ ACTIVE rows for the same
|
|
13
13
|
* `instance_id`. The peer's local credential file then ended up holding a
|
|
14
14
|
* hash that a later concurrent `save()` had revoked, causing an endless
|
|
15
|
-
* "Peer HMAC proof invalid" loop.
|
|
16
|
-
* for the
|
|
17
|
-
* full incident write-up (DB rows + log evidence from real staging deploys).
|
|
15
|
+
* "Peer HMAC proof invalid" loop.
|
|
18
16
|
*
|
|
19
17
|
* The `up()` first dedupes any pre-existing duplicates (revoke all but the
|
|
20
18
|
* newest active row per `instance_id`) — required because earlier migrations
|
|
21
|
-
* predate the constraint and existing
|
|
19
|
+
* predate the constraint and existing databases may already hold the dupes.
|
|
22
20
|
*
|
|
23
21
|
* `down()` only drops the index; it does NOT undo the dedupe (there's no
|
|
24
22
|
* safe way to recreate revoked rows, and the dedupe is monotonic).
|
|
@@ -5,8 +5,8 @@ import { type Kysely } from 'kysely';
|
|
|
5
5
|
* Operators flip this on a per-org basis to permit `http://` (non-HTTPS)
|
|
6
6
|
* registry URLs in workflow `registries:` declarations. By default only
|
|
7
7
|
* `https://` registries are accepted, with an automatic exemption for
|
|
8
|
-
* loopback / `*.local` / private link-local hosts so
|
|
9
|
-
* (
|
|
8
|
+
* loopback / `*.local` / private link-local hosts so a local registry
|
|
9
|
+
* (for example `http://registry.local:4873`) keeps working.
|
|
10
10
|
*
|
|
11
11
|
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
12
12
|
*/
|
|
@@ -9,7 +9,7 @@ import { type Kysely } from 'kysely';
|
|
|
9
9
|
* the separate queryable host-vars dimension.
|
|
10
10
|
*
|
|
11
11
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`): re-running on a DB that already has
|
|
12
|
-
* the column is a no-op.
|
|
12
|
+
* the column is a no-op. Existing data is preserved (additive column).
|
|
13
13
|
*/
|
|
14
14
|
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
15
15
|
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
@@ -10,7 +10,7 @@ import { type Kysely } from 'kysely';
|
|
|
10
10
|
* join token already in its env — no operator action, no cluster redeploy.
|
|
11
11
|
*
|
|
12
12
|
* Idempotent (`ADD COLUMN IF NOT EXISTS`): re-running on a DB that already has
|
|
13
|
-
* the column is a no-op.
|
|
13
|
+
* the column is a no-op. Existing data is preserved (additive nullable column).
|
|
14
14
|
*/
|
|
15
15
|
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
16
16
|
export declare function down(db: Kysely<unknown>): Promise<void>;
|