@kici-dev/orchestrator 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +10 -1
- package/dist/agent/host-roster.d.ts +8 -3
- package/dist/agent/registry.d.ts +60 -5
- package/dist/app.d.ts +10 -5
- package/dist/cache/dep-cache.d.ts +14 -6
- package/dist/cache/global-eval-round-cache.d.ts +1 -1
- package/dist/cache/index.d.ts +1 -1
- package/dist/cache/source-cache.d.ts +92 -15
- package/dist/cancel/cancel-run.d.ts +20 -2
- package/dist/cancel/sweep-stuck-cancelling.d.ts +30 -0
- package/dist/cli/api-client.d.ts +10 -0
- package/dist/cli/commands/agent-service/logs.d.ts +1 -1
- package/dist/cli/commands/agent-service/restart.d.ts +1 -1
- package/dist/cli/commands/agent-service/start.d.ts +1 -1
- package/dist/cli/commands/agent-service/status.d.ts +1 -1
- package/dist/cli/commands/agent-service/stop.d.ts +1 -1
- package/dist/cli/commands/agent-service/uninstall.d.ts +1 -1
- package/dist/cli/commands/agent.d.ts +25 -0
- package/dist/cli/commands/config.d.ts +16 -0
- package/dist/cli/commands/db-backup.d.ts +46 -0
- package/dist/cli/commands/held-run.d.ts +9 -0
- package/dist/cli/commands/orchestrator-service/drain.d.ts +2 -19
- package/dist/cli/commands/orchestrator-service/index.d.ts +9 -1
- package/dist/cli/commands/orchestrator-service/install-env.d.ts +43 -2
- package/dist/cli/commands/orchestrator-service/logs.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/restart.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/start.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/status.d.ts +17 -1
- package/dist/cli/commands/orchestrator-service/stop.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/uninstall.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +2 -1
- package/dist/cli/commands/rotate.d.ts +1 -1
- package/dist/cli/commands/shared/env-file-mode.d.ts +16 -0
- package/dist/cli/commands/shared/upgrade-hooks.d.ts +32 -0
- package/dist/cli/commands/shared/upgrade-safety.d.ts +81 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +67 -12
- package/dist/cli/commands/source.d.ts +1 -0
- package/dist/cli/commands/token.d.ts +4 -0
- package/dist/cli/commands/trust-policy.d.ts +57 -1
- package/dist/cli/join.d.ts +1 -1
- package/dist/cli/service/backup-timer.d.ts +108 -0
- package/dist/cli/service/compose-path.d.ts +11 -0
- package/dist/cli/service/compose.d.ts +22 -0
- package/dist/cli/service/deploy-env.d.ts +10 -11
- package/dist/cli/service/env-assignment.d.ts +24 -0
- package/dist/cli/service/index.d.ts +2 -2
- package/dist/cli/service/instance/resolve.d.ts +71 -7
- package/dist/cli/service/instance/types.d.ts +15 -0
- package/dist/cli/service/launchd.d.ts +7 -0
- package/dist/cli/service/systemd.d.ts +1 -0
- package/dist/cli/service/types.d.ts +36 -0
- package/dist/cli/service/windows.d.ts +16 -0
- package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -3
- package/dist/cli/wizard/prompts.d.ts +7 -0
- package/dist/cli.js +12525 -9298
- package/dist/cluster/cluster-settings-reader.d.ts +2 -2
- package/dist/cluster/coordinator.d.ts +11 -0
- package/dist/cluster/instance-heartbeat.d.ts +96 -0
- package/dist/cluster/join-client.d.ts +55 -5
- package/dist/cluster/join-handler.d.ts +1 -0
- package/dist/cluster/orphan-recovery.d.ts +16 -0
- package/dist/cluster/peer-registry.d.ts +10 -0
- package/dist/concurrency/dispatch-next-queued.d.ts +1 -13
- package/dist/concurrency/index.d.ts +1 -2
- package/dist/concurrency/queue-manager.d.ts +63 -18
- package/dist/config/env-overlay.d.ts +3 -2
- package/dist/config.d.ts +24 -0
- package/dist/contexts/release-queued-holds.d.ts +23 -1
- package/dist/dashboard/attestation-filters.d.ts +2 -0
- package/dist/dashboard/handler.d.ts +22 -2
- package/dist/db/deferred-indexes.d.ts +72 -0
- package/dist/db/migration-test-harness.d.ts +22 -0
- package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +30 -0
- package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +23 -0
- package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +23 -0
- package/dist/db/migrations/134_dispatch_queue_ownership.d.ts +4 -0
- package/dist/db/migrations/135_cluster_settings_retention_days.d.ts +4 -0
- package/dist/db/migrations/136_master_key_wrapped_tables_key_version.d.ts +4 -0
- package/dist/db/migrations/137_execution_runs_pr_head_context.d.ts +4 -0
- package/dist/db/migrations/138_execution_runs_subject_trigger_event.d.ts +4 -0
- package/dist/db/migrations/139_admin_token_subject.d.ts +4 -0
- package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +54 -0
- package/dist/db/migrations/141_event_log_shed.d.ts +15 -0
- package/dist/db/migrator.d.ts +47 -0
- package/dist/db/notify-listener.d.ts +60 -0
- package/dist/db/synthetic-job-ids.d.ts +24 -0
- package/dist/db/types.d.ts +169 -4
- package/dist/deployment/deployment-identity.d.ts +31 -1
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/checks/retention.d.ts +24 -0
- package/dist/diagnostics/types.d.ts +7 -0
- package/dist/events/event-router.d.ts +1 -1
- package/dist/fault-injection-types.d.ts +53 -0
- package/dist/firecracker/host-network.d.ts +63 -8
- package/dist/git/credential-broker.d.ts +35 -2
- package/dist/git/inherited-secret.d.ts +7 -2
- package/dist/git/job-context.d.ts +7 -4
- package/dist/helpers/secure-write.d.ts +35 -0
- package/dist/index.js +196 -67
- package/dist/metrics/prometheus.d.ts +116 -1
- package/dist/oidc/db-signer.d.ts +27 -5
- package/dist/oidc/id-token-claims.d.ts +33 -2
- package/dist/oidc/oidc-mint-registration.d.ts +6 -0
- package/dist/oidc/orchestrator-mint.d.ts +8 -0
- package/dist/oidc/reconcile-signing-key.d.ts +37 -0
- package/dist/orchestrator-core.d.ts +9 -3
- package/dist/pipeline/decorating-secret-resolver.d.ts +1 -1
- package/dist/pipeline/dispatch-matched-workflow.d.ts +65 -14
- package/dist/pipeline/manual-schedule.d.ts +1 -0
- package/dist/pipeline/process-webhook.d.ts +20 -2
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +7 -0
- package/dist/pipeline/run-event-context.d.ts +39 -0
- package/dist/policy/dashboard-write-policy-listener.d.ts +1 -1
- package/dist/policy/dashboard-write-policy.d.ts +47 -1
- package/dist/provenance/dispatch-context.d.ts +46 -0
- package/dist/provenance/verify-deferred-capture.d.ts +39 -0
- package/dist/providers/github/auth.d.ts +9 -0
- package/dist/providers/github/deliveries.d.ts +118 -0
- package/dist/providers/universal-git/config.d.ts +1 -1
- package/dist/queue/cleanup.d.ts +13 -0
- package/dist/queue/job-queue.d.ts +162 -14
- package/dist/queue/retention.d.ts +105 -0
- package/dist/reporting/execution-tracker.d.ts +87 -36
- package/dist/routes/admin-db.d.ts +5 -0
- package/dist/routes/admin-events.d.ts +13 -0
- package/dist/routes/admin-held-runs.d.ts +11 -1
- package/dist/routes/admin-org-settings.d.ts +9 -0
- package/dist/routes/admin-runs.d.ts +7 -4
- package/dist/routes/admin-sources.d.ts +15 -0
- package/dist/routes/admin-trust-policy.d.ts +11 -0
- package/dist/routes/admin.d.ts +25 -0
- package/dist/scaler/backend-factory.d.ts +16 -0
- package/dist/scaler/bare-metal-backend.d.ts +22 -4
- package/dist/scaler/config.d.ts +6 -3
- package/dist/scaler/container-backend.d.ts +102 -3
- package/dist/scaler/container-hostconfig.d.ts +60 -0
- package/dist/scaler/file-tail.d.ts +25 -2
- package/dist/scaler/firecracker-api.d.ts +9 -1
- package/dist/scaler/firecracker-backend.d.ts +135 -2
- package/dist/scaler/host-access.d.ts +59 -0
- package/dist/scaler/index.d.ts +1 -0
- package/dist/scaler/manager.d.ts +33 -0
- package/dist/scaler/scaler-state-store.d.ts +6 -0
- package/dist/scaler/types.d.ts +5 -7
- package/dist/secrets/config.d.ts +48 -0
- package/dist/secrets/dashboard-encryption-key.d.ts +19 -0
- package/dist/secrets/ephemeral-keys.d.ts +9 -3
- package/dist/secrets/index.d.ts +3 -1
- package/dist/secrets/job-secret-gate.d.ts +76 -0
- package/dist/secrets/master-key-rotation.d.ts +65 -0
- package/dist/secrets/secret-output-crypto.d.ts +17 -0
- package/dist/secrets/secret-resolver.d.ts +23 -6
- package/dist/secrets/source-credentials.d.ts +1 -1
- package/dist/secrets/token-manager.d.ts +4 -1
- package/dist/security/reduced-privilege-note.d.ts +5 -0
- package/dist/security/trust-policy-gate.d.ts +70 -6
- package/dist/security/trust-tier.d.ts +31 -0
- package/dist/server.d.ts +1 -1
- package/dist/server.js +10674 -5283
- package/dist/sources/source-manager.d.ts +1 -1
- package/dist/stale-detector/stale-dispatch-candidates.d.ts +43 -0
- package/dist/standalone.js +9258 -4122
- package/dist/storage/loopback-guard.d.ts +53 -6
- package/dist/webhook/event-log.d.ts +14 -1
- package/dist/webhook/generic-sources-listener.d.ts +1 -1
- package/dist/webhook/ingest-admission.d.ts +24 -0
- package/dist/webhook/ingest-overflow-replayer.d.ts +78 -14
- package/dist/webhook/ingest-overflow-types.d.ts +13 -1
- package/dist/webhook/register-source-bundle.d.ts +23 -0
- package/dist/webhook/relay-reinject.d.ts +11 -12
- package/dist/webhook/shed-breadcrumb.d.ts +15 -0
- package/dist/ws/git-credential-relay.d.ts +41 -12
- package/dist/ws/platform-client.d.ts +28 -5
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +53 -53
- package/dist/concurrency/group-tracker.d.ts +0 -51
- package/dist/scaler/nftables.d.ts +0 -111
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub App webhook-delivery replay.
|
|
3
|
+
*
|
|
4
|
+
* GitHub keeps every App webhook delivery for a retention window and can be
|
|
5
|
+
* asked to send one again. That is the only way to recover a window of events
|
|
6
|
+
* an orchestrator never received — GitHub itself does not retry a delivery its
|
|
7
|
+
* destination failed to accept, so a window lost while the destination was
|
|
8
|
+
* unreachable is gone unless somebody replays it.
|
|
9
|
+
*
|
|
10
|
+
* Both endpoints are App-level and accept only a JWT signed with the App
|
|
11
|
+
* private key, which the orchestrator holds and nothing upstream does:
|
|
12
|
+
*
|
|
13
|
+
* GET /app/hook/deliveries (cursor-paginated)
|
|
14
|
+
* POST /app/hook/deliveries/{delivery_id}/attempts
|
|
15
|
+
*
|
|
16
|
+
* Verified against docs.github.com/en/rest/apps/webhooks on 2026-09-03.
|
|
17
|
+
*/
|
|
18
|
+
import { z } from 'zod';
|
|
19
|
+
import type { Octokit } from '@octokit/rest';
|
|
20
|
+
import { type GitHubAppConfig } from './auth.js';
|
|
21
|
+
/**
|
|
22
|
+
* Safety ceiling on pages walked in one replay. GitHub returns deliveries
|
|
23
|
+
* newest-first, so the walk normally stops as soon as it passes `since`; this
|
|
24
|
+
* bounds a pathological case (a very old `since` on a very busy App) instead of
|
|
25
|
+
* paginating an App's whole retention window.
|
|
26
|
+
*/
|
|
27
|
+
export declare const MAX_DELIVERY_PAGES = 100;
|
|
28
|
+
/** What happened to one delivery in a replay. */
|
|
29
|
+
export declare const RedeliverOutcome: z.ZodEnum<{
|
|
30
|
+
failed: "failed";
|
|
31
|
+
redelivered: "redelivered";
|
|
32
|
+
"would-redeliver": "would-redeliver";
|
|
33
|
+
}>;
|
|
34
|
+
export type RedeliverOutcome = z.infer<typeof RedeliverOutcome>;
|
|
35
|
+
/** One delivery as `GET /app/hook/deliveries` returns it (the fields used here). */
|
|
36
|
+
export interface GithubWebhookDelivery {
|
|
37
|
+
id: number;
|
|
38
|
+
guid: string;
|
|
39
|
+
/** ISO-8601 timestamp GitHub attempted the delivery. */
|
|
40
|
+
delivered_at: string;
|
|
41
|
+
redelivery: boolean;
|
|
42
|
+
event: string;
|
|
43
|
+
action: string | null;
|
|
44
|
+
status: string;
|
|
45
|
+
status_code: number;
|
|
46
|
+
installation_id: number | null;
|
|
47
|
+
repository_id: number | null;
|
|
48
|
+
}
|
|
49
|
+
/** Per-delivery result of a replay. */
|
|
50
|
+
export interface RedeliverResult {
|
|
51
|
+
deliveryId: number;
|
|
52
|
+
guid: string;
|
|
53
|
+
deliveredAt: string;
|
|
54
|
+
event: string;
|
|
55
|
+
action: string | null;
|
|
56
|
+
/** The original delivery's HTTP status as GitHub recorded it. */
|
|
57
|
+
originalStatusCode: number;
|
|
58
|
+
outcome: RedeliverOutcome;
|
|
59
|
+
/** Present when `outcome` is `failed`. */
|
|
60
|
+
error?: string;
|
|
61
|
+
}
|
|
62
|
+
/** The whole replay, as the admin route and the CLI report it. */
|
|
63
|
+
export interface RedeliverWindowResult {
|
|
64
|
+
since: string;
|
|
65
|
+
until: string;
|
|
66
|
+
dryRun: boolean;
|
|
67
|
+
/** Deliveries GitHub holds whose `delivered_at` falls in the window. */
|
|
68
|
+
matched: number;
|
|
69
|
+
/**
|
|
70
|
+
* True when the page walk hit its ceiling before reaching a delivery older
|
|
71
|
+
* than `since`, so `matched` counts only what was gathered and the window
|
|
72
|
+
* holds more. Callers must say so rather than reporting a partial replay as
|
|
73
|
+
* a complete one.
|
|
74
|
+
*/
|
|
75
|
+
truncated: boolean;
|
|
76
|
+
redelivered: number;
|
|
77
|
+
failed: number;
|
|
78
|
+
results: RedeliverResult[];
|
|
79
|
+
}
|
|
80
|
+
/** The one Octokit method this module needs, so a test can hand in a stub. */
|
|
81
|
+
export type GithubRequest = Pick<Octokit, 'request'>;
|
|
82
|
+
export interface RedeliverWindowOptions {
|
|
83
|
+
/** Inclusive lower bound on `delivered_at`. */
|
|
84
|
+
since: Date;
|
|
85
|
+
/** Exclusive upper bound on `delivered_at`. */
|
|
86
|
+
until: Date;
|
|
87
|
+
/** List and report the matching deliveries without asking GitHub to resend. */
|
|
88
|
+
dryRun?: boolean;
|
|
89
|
+
/** Injected for tests; defaults to a JWT-authenticated App client. */
|
|
90
|
+
octokit?: GithubRequest;
|
|
91
|
+
}
|
|
92
|
+
/** Read the `cursor` query value out of the `Link: <...>; rel="next"` header. */
|
|
93
|
+
export declare function nextCursorFromLink(link: string | undefined): string | null;
|
|
94
|
+
/**
|
|
95
|
+
* Walk `GET /app/hook/deliveries` newest-first and return every delivery whose
|
|
96
|
+
* `delivered_at` falls in `[since, until)`.
|
|
97
|
+
*
|
|
98
|
+
* The walk stops at the first page whose oldest delivery predates `since`:
|
|
99
|
+
* GitHub orders the feed newest-first, so nothing older can match afterwards.
|
|
100
|
+
*/
|
|
101
|
+
export declare function listDeliveriesInWindow(octokit: GithubRequest, opts: {
|
|
102
|
+
since: Date;
|
|
103
|
+
until: Date;
|
|
104
|
+
}): Promise<{
|
|
105
|
+
deliveries: GithubWebhookDelivery[];
|
|
106
|
+
truncated: boolean;
|
|
107
|
+
}>;
|
|
108
|
+
/** Ask GitHub to send one delivery again. Resolves once GitHub accepts (202). */
|
|
109
|
+
export declare function redeliverDelivery(octokit: GithubRequest, deliveryId: number): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Replay every delivery GitHub holds for this App in `[since, until)`.
|
|
112
|
+
*
|
|
113
|
+
* A per-delivery failure is recorded and the replay continues: an operator
|
|
114
|
+
* recovering an outage window wants the deliveries that can be replayed, plus
|
|
115
|
+
* an honest list of the ones that could not.
|
|
116
|
+
*/
|
|
117
|
+
export declare function redeliverWindow(app: GitHubAppConfig, opts: RedeliverWindowOptions): Promise<RedeliverWindowResult>;
|
|
118
|
+
//# sourceMappingURL=deliveries.d.ts.map
|
|
@@ -51,7 +51,7 @@ export declare const SshHostKeyPolicy: z.ZodEnum<{
|
|
|
51
51
|
export type SshHostKeyPolicy = z.infer<typeof SshHostKeyPolicy>;
|
|
52
52
|
/**
|
|
53
53
|
* Reference to a secret in the orchestrator's secret store. Resolved at
|
|
54
|
-
* dispatch time via `SecretResolver.
|
|
54
|
+
* dispatch time via `SecretResolver.resolveNamedInternal()`. The secret itself lives
|
|
55
55
|
* at scope `__source__/<sourceId>`, key = `credentialRef.key`, optionally
|
|
56
56
|
* in a specific backend via `credentialRef.store` (defaults to `pg`).
|
|
57
57
|
*/
|
package/dist/queue/cleanup.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.j
|
|
|
7
7
|
import type { LogStorage } from '../reporting/log-storage.js';
|
|
8
8
|
import { JobQueue } from './job-queue.js';
|
|
9
9
|
import { type CanRouteLabels, type TerminalizeDeps } from './terminalize-unroutable.js';
|
|
10
|
+
import { type RetentionWindows } from './retention.js';
|
|
10
11
|
/**
|
|
11
12
|
* Optional cleanup dependencies + retention knobs. Present in platform/hybrid
|
|
12
13
|
* mode (the only wiring site, `orchestrator-core`, always supplies them); the
|
|
@@ -33,6 +34,17 @@ export interface CleanupExtras {
|
|
|
33
34
|
stepLogTtlDays?: number;
|
|
34
35
|
/** Store for `check_run_tracking`; the retention sweep runs only when wired. */
|
|
35
36
|
checkRunTrackingStore?: CheckRunTrackingStore;
|
|
37
|
+
/**
|
|
38
|
+
* Database-side retention windows, in days. Absent leaves the history sweep
|
|
39
|
+
* off entirely, which is what a caller that has not wired it gets.
|
|
40
|
+
*/
|
|
41
|
+
retentionWindows?: RetentionWindows;
|
|
42
|
+
/**
|
|
43
|
+
* True when the cold store is configured. Its archivers then own the tables
|
|
44
|
+
* they register and the history sweep leaves those alone — one owner per
|
|
45
|
+
* table.
|
|
46
|
+
*/
|
|
47
|
+
coldStoreEnabled?: boolean;
|
|
36
48
|
/** Live cluster-settings reader; absent ⇒ the config default is used as-is. */
|
|
37
49
|
clusterSettings?: ClusterSettingsReader;
|
|
38
50
|
/** `check_run_tracking` row retention (days); 0 disables. Default 7. */
|
|
@@ -96,6 +108,7 @@ export declare function runCleanup(dedup: {
|
|
|
96
108
|
dispatchRowsPruned: number;
|
|
97
109
|
logObjectsPruned: number;
|
|
98
110
|
checkRunRowsPruned: number;
|
|
111
|
+
historyRowsPruned: number;
|
|
99
112
|
}>;
|
|
100
113
|
/**
|
|
101
114
|
* Build a per-tick handler for the queue-cleanup scheduled job.
|
|
@@ -1,7 +1,50 @@
|
|
|
1
|
-
import { type Kysely } from 'kysely';
|
|
2
|
-
import { type LabelMatcher, type ResourceRequest } from '@kici-dev/engine';
|
|
1
|
+
import { type Kysely, type SqlBool } from 'kysely';
|
|
2
|
+
import { type CanonicalLabel, type CanonicalMatcher, type LabelMatcher, type ResourceRequest } from '@kici-dev/engine';
|
|
3
3
|
import type { Database } from '../db/types.js';
|
|
4
4
|
import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
|
|
5
|
+
/**
|
|
6
|
+
* The ownership predicate, in its two forms. Both come from here so they cannot
|
|
7
|
+
* drift apart, because they answer the same question from opposite directions.
|
|
8
|
+
*
|
|
9
|
+
* **Strict selection** (`recoverableByOwner`) picks the rows a cluster-wide
|
|
10
|
+
* sweep may act on: mine, or a dead instance's. A NULL owner is EXCLUDED — it
|
|
11
|
+
* reads as "unknown owner", never as "not mine", which is the convention
|
|
12
|
+
* migration 119 established for the scaler plane. Sparing it is what makes a
|
|
13
|
+
* rolling upgrade safe: every row a pre-upgrade coordinator dispatched carries
|
|
14
|
+
* a NULL owner, and recovering those would fail every job the rest of the
|
|
15
|
+
* cluster is running, on every coordinator boot — precisely the window in which
|
|
16
|
+
* coordinators boot. A spared orphan is not lost; the stale detector reaps it on
|
|
17
|
+
* the dispatch-time clock.
|
|
18
|
+
*
|
|
19
|
+
* **Permissive write guard** (`writableByOwner`) gates a write on a row this
|
|
20
|
+
* coordinator has already selected some other way: mine, unknown, or a dead
|
|
21
|
+
* instance's. It INCLUDES a NULL owner, because a legitimate local disconnect
|
|
22
|
+
* must still be able to move a pre-upgrade row. Its job is to stop a path that
|
|
23
|
+
* forgot the selection filter from flipping a row a LIVE sibling owns.
|
|
24
|
+
*/
|
|
25
|
+
export interface OwnershipPredicateOptions {
|
|
26
|
+
/** This coordinator's instance id. Undefined ⇒ no instance identity. */
|
|
27
|
+
instanceId?: string;
|
|
28
|
+
/** How stale a `cluster_instances` heartbeat may be and still read as live. */
|
|
29
|
+
graceMs: number;
|
|
30
|
+
/** Fully qualified owner column, defaulting to the dispatch queue's. */
|
|
31
|
+
ownerColumn?: string;
|
|
32
|
+
/**
|
|
33
|
+
* Instance ids currently connected as peers. OR-ed in as live, so a peer whose
|
|
34
|
+
* heartbeat row is momentarily stale is not mistaken for dead.
|
|
35
|
+
*/
|
|
36
|
+
livePeerIds?: readonly string[];
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Strict selection: the rows a cluster-wide sweep may recover. Excludes a NULL
|
|
40
|
+
* owner and excludes any row whose owner is still live.
|
|
41
|
+
*/
|
|
42
|
+
export declare function recoverableByOwner(opts: OwnershipPredicateOptions): import("kysely").RawBuilder<SqlBool>;
|
|
43
|
+
/**
|
|
44
|
+
* Permissive write guard: allow the write unless a LIVE sibling owns the row.
|
|
45
|
+
* A NULL owner passes.
|
|
46
|
+
*/
|
|
47
|
+
export declare function writableByOwner(opts: OwnershipPredicateOptions): import("kysely").RawBuilder<SqlBool>;
|
|
5
48
|
/** Info about an expired dispatch_queue entry, returned by markExpired(). */
|
|
6
49
|
export interface ExpiredJobInfo {
|
|
7
50
|
/** dispatch_queue row ID */
|
|
@@ -15,11 +58,15 @@ export interface ExpiredJobInfo {
|
|
|
15
58
|
* any agent could ever have run it. A job that expires with NO matching agent
|
|
16
59
|
* is `unroutable` (a fleet/label problem); one whose agent existed but never
|
|
17
60
|
* freed up is `timed_out_stale` (a capacity problem).
|
|
61
|
+
*
|
|
62
|
+
* Canonical: the probe compares them against a live agent's canonical labels,
|
|
63
|
+
* so a case-sensitive read here would report a routable job `unroutable` and
|
|
64
|
+
* send the operator hunting a fleet problem that does not exist.
|
|
18
65
|
*/
|
|
19
|
-
runsOnLabels:
|
|
20
|
-
runsOnPatterns:
|
|
21
|
-
excludeLabels:
|
|
22
|
-
excludePatterns:
|
|
66
|
+
runsOnLabels: CanonicalLabel[];
|
|
67
|
+
runsOnPatterns: CanonicalMatcher[];
|
|
68
|
+
excludeLabels: CanonicalLabel[];
|
|
69
|
+
excludePatterns: CanonicalMatcher[];
|
|
23
70
|
}
|
|
24
71
|
/**
|
|
25
72
|
* A pending job as the unroutable probe sees it: the same routing facts the
|
|
@@ -103,6 +150,8 @@ export interface QueuedJobInput {
|
|
|
103
150
|
sourceTarUrl?: string;
|
|
104
151
|
/** Content hash of the pre-compiled bundle for verification. */
|
|
105
152
|
sourceTarHash?: string;
|
|
153
|
+
/** SHA-256 of the source tarball's own bytes (what `sourceTarHash` never was). */
|
|
154
|
+
sourceTarDigest?: string;
|
|
106
155
|
/** URL to pre-built dependency tarball (from dep cache). Passed through to job.dispatch. */
|
|
107
156
|
depsUrl?: string;
|
|
108
157
|
/** SHA-256 hash of the dependency tarball for integrity verification. */
|
|
@@ -140,7 +189,8 @@ export interface QueuedJob {
|
|
|
140
189
|
runId: string;
|
|
141
190
|
workflowName: string;
|
|
142
191
|
jobName: string;
|
|
143
|
-
|
|
192
|
+
/** Canonical: folded by `rowToQueuedJob`, whatever case the row holds. */
|
|
193
|
+
runsOnLabels: CanonicalLabel[];
|
|
144
194
|
jobConfig: Record<string, unknown>;
|
|
145
195
|
repoUrl: string;
|
|
146
196
|
ref: string;
|
|
@@ -160,18 +210,24 @@ export interface QueuedJob {
|
|
|
160
210
|
sourceTarUrl?: string;
|
|
161
211
|
/** Content hash of the pre-compiled bundle for verification. */
|
|
162
212
|
sourceTarHash?: string;
|
|
213
|
+
/** SHA-256 of the source tarball's own bytes (what `sourceTarHash` never was). */
|
|
214
|
+
sourceTarDigest?: string;
|
|
163
215
|
/** URL to pre-built dependency tarball (from dep cache). Passed through to job.dispatch. */
|
|
164
216
|
depsUrl?: string;
|
|
165
217
|
/** SHA-256 hash of the dependency tarball for integrity verification. */
|
|
166
218
|
depsHash?: string;
|
|
167
219
|
/** Request trace ID for cross-tier correlation. Passed through to job.dispatch. */
|
|
168
220
|
requestId?: string;
|
|
169
|
-
/** Labels that the dispatched agent must NOT have. */
|
|
170
|
-
excludeLabels:
|
|
171
|
-
/**
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
221
|
+
/** Labels that the dispatched agent must NOT have. Canonical. */
|
|
222
|
+
excludeLabels: CanonicalLabel[];
|
|
223
|
+
/**
|
|
224
|
+
* Regex matchers the agent's labels must satisfy (JS post-filter on the exact
|
|
225
|
+
* `@>` prefilter). Canonical, so the stored pattern carries the `i` flag
|
|
226
|
+
* whether or not the lock file that produced it did.
|
|
227
|
+
*/
|
|
228
|
+
runsOnPatterns: CanonicalMatcher[];
|
|
229
|
+
/** Regex matchers that disqualify an agent (JS post-filter). Canonical. */
|
|
230
|
+
excludePatterns: CanonicalMatcher[];
|
|
175
231
|
/**
|
|
176
232
|
* Per-job resource request and limit (K8s-style). Materialized from `jobConfig.resources`
|
|
177
233
|
* by `rowToQueuedJob` so callers can read it without re-parsing the JSON column.
|
|
@@ -207,6 +263,8 @@ export declare class JobQueue {
|
|
|
207
263
|
private readonly defaultMaxDepth;
|
|
208
264
|
private readonly defaultTimeoutMs;
|
|
209
265
|
private readonly clusterSettings?;
|
|
266
|
+
private readonly instanceId?;
|
|
267
|
+
private readonly ownershipGraceMs;
|
|
210
268
|
/** Per-job (per-org) queue-timeout resolver; falls back to defaultTimeoutMs. */
|
|
211
269
|
private readonly getQueueTimeoutMs?;
|
|
212
270
|
/** 1-second TTL cache for pending depth count to avoid extra SELECT COUNT per enqueue. */
|
|
@@ -225,7 +283,23 @@ export declare class JobQueue {
|
|
|
225
283
|
getQueueTimeoutMs?: (job: {
|
|
226
284
|
jobConfig?: Record<string, unknown>;
|
|
227
285
|
}) => Promise<number>;
|
|
286
|
+
/**
|
|
287
|
+
* This coordinator's instance id, stamped onto every row it dispatches so
|
|
288
|
+
* a cluster-wide sweep can tell its own rows from a sibling's. Optional
|
|
289
|
+
* for the same reason `AgentRegistryDeps.instanceId` is: a worker with no
|
|
290
|
+
* cluster identity (and every unit test) leaves it undefined and the
|
|
291
|
+
* column stays NULL, which reads as "unknown owner".
|
|
292
|
+
*/
|
|
293
|
+
instanceId?: string;
|
|
294
|
+
/**
|
|
295
|
+
* How stale a `cluster_instances` heartbeat may be and still read as
|
|
296
|
+
* live. Defaults to {@link instanceLivenessGraceMs} at the default
|
|
297
|
+
* recovery grace period.
|
|
298
|
+
*/
|
|
299
|
+
ownershipGraceMs?: number;
|
|
228
300
|
});
|
|
301
|
+
/** Ownership predicate inputs, in the shape both predicate builders take. */
|
|
302
|
+
private ownershipOptions;
|
|
229
303
|
/**
|
|
230
304
|
* Enqueue a job. Checks depth first, rejects with 'queue full' if >= the
|
|
231
305
|
* fleet-wide `queue_max_depth` (cluster_settings, falling back to the config
|
|
@@ -530,6 +604,19 @@ export declare class JobQueue {
|
|
|
530
604
|
* refreshing the cache periodically via {@link JobQueue.getDepthBreakdown}.
|
|
531
605
|
*/
|
|
532
606
|
readCachedDepthBreakdown(): DispatchQueueDepthBreakdown | null;
|
|
607
|
+
/**
|
|
608
|
+
* The run's dispatched-or-recovering rows with the durable facts a cancel
|
|
609
|
+
* needs to route: which agent holds the job, and which coordinator dispatched
|
|
610
|
+
* it. Both come from the row rather than from a process map, because the
|
|
611
|
+
* coordinator handling a cancel is the owner only by chance — the Platform
|
|
612
|
+
* routes `run.cancel.request` by routing key and picks any connected pool
|
|
613
|
+
* member.
|
|
614
|
+
*/
|
|
615
|
+
getDispatchedJobOwnersByRunId(runId: string): Promise<Array<{
|
|
616
|
+
jobId: string;
|
|
617
|
+
agentId: string | null;
|
|
618
|
+
ownerInstanceId: string | null;
|
|
619
|
+
}>>;
|
|
533
620
|
/**
|
|
534
621
|
* Get job IDs for a run that are currently dispatched or recovering.
|
|
535
622
|
* Used by the cancel-run API to send job.cancel to the right agents.
|
|
@@ -544,8 +631,47 @@ export declare class JobQueue {
|
|
|
544
631
|
* replacement coord on Raft leader switch can recreate the recovery
|
|
545
632
|
* timer (via `getRecoveringJobs()` on boot) or expire the row in
|
|
546
633
|
* the leader-gated sweep (`sweepExpiredRecoveries()`).
|
|
634
|
+
*
|
|
635
|
+
* Carries the permissive ownership write guard: a row owned by a LIVE sibling
|
|
636
|
+
* is left alone. Callers are expected to have selected owned rows already
|
|
637
|
+
* ({@link JobQueue.getOrphanedDispatchedJobs}); this is the backstop for a
|
|
638
|
+
* path that forgets to, since flipping a live sibling's row to `recovering`
|
|
639
|
+
* starts a 120-second clock that ends by failing a job nobody disconnected
|
|
640
|
+
* from. A NULL owner still passes — a local disconnect must be able to move a
|
|
641
|
+
* pre-upgrade row.
|
|
642
|
+
*
|
|
643
|
+
* @returns whether a row was flipped.
|
|
644
|
+
*/
|
|
645
|
+
markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<boolean>;
|
|
646
|
+
/**
|
|
647
|
+
* The `dispatched` rows this coordinator may start recovery timers for on
|
|
648
|
+
* boot: its own, and those whose owning coordinator is no longer alive.
|
|
649
|
+
*
|
|
650
|
+
* Replaces an unfiltered `getJobsByStatus(Dispatched)` over the shared table.
|
|
651
|
+
* That scan made every coordinator boot flip every sibling's in-flight row to
|
|
652
|
+
* `recovering`; the agents running those jobs were never disconnected, so
|
|
653
|
+
* nothing reclaimed them, and 120 seconds later the booting coordinator
|
|
654
|
+
* failed every one of them while they ran to completion. The documented
|
|
655
|
+
* rolling-restart procedure triggered it on every restart.
|
|
656
|
+
*
|
|
657
|
+
* A NULL owner is spared — see {@link recoverableByOwner} for why, and for
|
|
658
|
+
* what reaps a genuinely orphaned one instead.
|
|
659
|
+
*/
|
|
660
|
+
getOrphanedDispatchedJobs(): Promise<Array<{
|
|
661
|
+
id: string;
|
|
662
|
+
runId: string;
|
|
663
|
+
agentId: string | null;
|
|
664
|
+
}>>;
|
|
665
|
+
/**
|
|
666
|
+
* How many `dispatched` rows this coordinator deliberately left alone on
|
|
667
|
+
* boot: owned by a live sibling, or of unknown ownership. Read only to log
|
|
668
|
+
* and count the decision, so "recovery spared N rows" is visible rather than
|
|
669
|
+
* being the silent absence of a log line.
|
|
547
670
|
*/
|
|
548
|
-
|
|
671
|
+
countSparedDispatchedJobs(): Promise<{
|
|
672
|
+
ownedElsewhere: number;
|
|
673
|
+
unknownOwner: number;
|
|
674
|
+
}>;
|
|
549
675
|
/**
|
|
550
676
|
* Stamp the dispatch-acknowledgment deadline for a dispatched job.
|
|
551
677
|
* Only touches rows still in 'dispatched' for safety.
|
|
@@ -621,6 +747,28 @@ export declare class JobQueue {
|
|
|
621
747
|
* was not in 'dispatched' state (nothing requeued).
|
|
622
748
|
*/
|
|
623
749
|
requeue(jobId: string): Promise<number | null>;
|
|
750
|
+
/**
|
|
751
|
+
* Requeue a dispatched job **only** when its ack genuinely never landed.
|
|
752
|
+
*
|
|
753
|
+
* One atomic guarded UPDATE that both proves the ack is still outstanding
|
|
754
|
+
* and requeues. It exists because {@link requeue}'s only guard is
|
|
755
|
+
* `status = 'dispatched'`, and an acked, executing job is still
|
|
756
|
+
* `dispatched` — so an ack timer that fires late (a coordinator restart
|
|
757
|
+
* re-armed it, or the ack raced the deadline) requeues a running job and a
|
|
758
|
+
* second agent claims it. Both agents then execute the same job.
|
|
759
|
+
*
|
|
760
|
+
* The proof has to happen in the same statement as the requeue: the ack path
|
|
761
|
+
* clears `ack_deadline`/`ack_agent_id`, so a separate read-then-requeue is a
|
|
762
|
+
* race, and a guard bolted onto a caller that clears the deadline first can
|
|
763
|
+
* never see the evidence it needs.
|
|
764
|
+
*
|
|
765
|
+
* {@link requeue} stays for `job.reject` and the scaler-disconnect path,
|
|
766
|
+
* whose rows legitimately carry no deadline.
|
|
767
|
+
*
|
|
768
|
+
* @returns the post-increment dispatch_attempts, or null when the ack landed
|
|
769
|
+
* (or the row moved on) and nothing was requeued.
|
|
770
|
+
*/
|
|
771
|
+
requeueIfAwaitingAck(jobId: string, expectedAgentId: string | null): Promise<number | null>;
|
|
624
772
|
/**
|
|
625
773
|
* Get the full QueuedJob row by ID regardless of status. Used by the
|
|
626
774
|
* dispatcher's redispatch path, which needs runsOnLabels / excludeLabels /
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Database-side retention for the tables that scale with run volume.
|
|
3
|
+
*
|
|
4
|
+
* The cold store is the other half of this: its archivers do
|
|
5
|
+
* archive-then-delete inside one transaction and own the four tables they
|
|
6
|
+
* register. But the cold store needs `KICI_COLD_STORE_ENABLED=true` and a
|
|
7
|
+
* bucket, and with neither — the default, and the quickstart path — nothing
|
|
8
|
+
* aged at all, so `execution_steps`, `event_log` and `access_log` grew without
|
|
9
|
+
* bound on the operator's own disk.
|
|
10
|
+
*
|
|
11
|
+
* Two rules keep the tiers from fighting:
|
|
12
|
+
*
|
|
13
|
+
* - When the cold store is on, this tier skips every table an archiver
|
|
14
|
+
* registers. Two owners deleting the same rows would race the archiver's
|
|
15
|
+
* FK guard and could delete a parent whose archive chunk was never
|
|
16
|
+
* written. One owner per table.
|
|
17
|
+
* - Nothing is deleted on the first pass. The first pass with a non-zero
|
|
18
|
+
* window stamps `cluster_settings.retention_announced_at` and reports what
|
|
19
|
+
* it would remove; deletion begins a week later. An upgrade that silently
|
|
20
|
+
* deleted an operator's history on its first night would be a behaviour
|
|
21
|
+
* change nobody was warned about.
|
|
22
|
+
*
|
|
23
|
+
* Every delete is batched — select ids under a `LIMIT`, then delete by id,
|
|
24
|
+
* looping until a pass comes back short. One unbounded `DELETE` over a table
|
|
25
|
+
* holding millions of rows holds locks for as long as it runs.
|
|
26
|
+
*/
|
|
27
|
+
import { type Kysely } from 'kysely';
|
|
28
|
+
import type { Database } from '../db/types.js';
|
|
29
|
+
/** Rows selected, and deleted, per statement. */
|
|
30
|
+
export declare const RETENTION_BATCH_SIZE = 5000;
|
|
31
|
+
/** Days between the announce stamp and the first deletion. */
|
|
32
|
+
export declare const RETENTION_ANNOUNCE_DAYS = 7;
|
|
33
|
+
/** Terminal run statuses. A run still queued, running, held or cancelling is
|
|
34
|
+
* live work whatever its age. */
|
|
35
|
+
export declare const TERMINAL_RUN_STATUS_LIST: readonly string[];
|
|
36
|
+
/** Terminal held-run statuses. A pending hold is never pruned. */
|
|
37
|
+
export declare const TERMINAL_HELD_RUN_STATUSES: readonly string[];
|
|
38
|
+
/**
|
|
39
|
+
* Tables the cold-store archivers register. When the cold store is on these
|
|
40
|
+
* are theirs alone — including the `execution_jobs` / `execution_steps`
|
|
41
|
+
* children, which the run archiver removes with their parent.
|
|
42
|
+
*/
|
|
43
|
+
export declare const COLD_STORE_OWNED_TABLES: readonly string[];
|
|
44
|
+
/** The `cluster_settings` columns this module reads. */
|
|
45
|
+
export type ClusterRetentionColumn = 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days';
|
|
46
|
+
export interface RetentionWindows {
|
|
47
|
+
/** Terminal runs, and the jobs and steps beneath them. 0 disables. */
|
|
48
|
+
runRetentionDays: number;
|
|
49
|
+
/** access_log, secret_audit_log, event_log. 0 disables. */
|
|
50
|
+
auditRetentionDays: number;
|
|
51
|
+
/** attestations, pending_attestations. 0 disables. */
|
|
52
|
+
provenanceRetentionDays: number;
|
|
53
|
+
/** Terminal held_runs. 0 disables. */
|
|
54
|
+
heldRunRetentionDays: number;
|
|
55
|
+
}
|
|
56
|
+
export interface PruneExpiredHistoryDeps {
|
|
57
|
+
db: Kysely<Database>;
|
|
58
|
+
windows: RetentionWindows;
|
|
59
|
+
/** True when the cold store is configured and owns its tables. */
|
|
60
|
+
coldStoreEnabled: boolean;
|
|
61
|
+
now?: () => Date;
|
|
62
|
+
batchSize?: number;
|
|
63
|
+
}
|
|
64
|
+
export interface RetentionSummary {
|
|
65
|
+
/** Rows deleted, per table. */
|
|
66
|
+
deleted: Record<string, number>;
|
|
67
|
+
/** Rows that WOULD have been deleted, while the announce window runs. */
|
|
68
|
+
pending: Record<string, number>;
|
|
69
|
+
/** True while the announce window has not elapsed, so nothing was deleted. */
|
|
70
|
+
announcing: boolean;
|
|
71
|
+
}
|
|
72
|
+
/** True when every window is 0, i.e. the operator disabled the whole tier. */
|
|
73
|
+
export declare function allWindowsDisabled(windows: RetentionWindows): boolean;
|
|
74
|
+
/**
|
|
75
|
+
* Delete rows in bounded batches, returning the total removed.
|
|
76
|
+
*
|
|
77
|
+
* The loop stops as soon as a pass comes back short, which is what bounds it
|
|
78
|
+
* on a table that is still being written to.
|
|
79
|
+
*/
|
|
80
|
+
export declare function deleteInBatches(table: string, selectIds: (limit: number) => Promise<string[]>, deleteIds: (ids: string[]) => Promise<void>, batchSize: number): Promise<number>;
|
|
81
|
+
/**
|
|
82
|
+
* Read the announce stamp, setting it on the first pass that has work to do.
|
|
83
|
+
*
|
|
84
|
+
* Returns true once the window has elapsed and deletion may begin.
|
|
85
|
+
*/
|
|
86
|
+
export declare function announceGatePassed(db: Kysely<Database>, now: Date, announceDays?: number): Promise<boolean>;
|
|
87
|
+
/**
|
|
88
|
+
* Prune every table this tier owns, honouring the windows, the cold-store
|
|
89
|
+
* ownership split, and the announce gate.
|
|
90
|
+
*/
|
|
91
|
+
export declare function pruneExpiredHistory(deps: PruneExpiredHistoryDeps): Promise<RetentionSummary>;
|
|
92
|
+
/**
|
|
93
|
+
* Resolve the effective windows: a live `cluster_settings` value wins over the
|
|
94
|
+
* configured default.
|
|
95
|
+
*
|
|
96
|
+
* `getNumber` returns a genuine 0 when the operator set the knob to 0, which is
|
|
97
|
+
* the documented way to disable a window — so this stays a fallback-on-null
|
|
98
|
+
* read, never `||`.
|
|
99
|
+
*/
|
|
100
|
+
export declare function resolveRetentionWindows(defaults: RetentionWindows, clusterSettings?: {
|
|
101
|
+
getNumber(column: ClusterRetentionColumn, fallback: number): Promise<number>;
|
|
102
|
+
}): Promise<RetentionWindows>;
|
|
103
|
+
/** Wrap {@link pruneExpiredHistory} so a failure never aborts the sweep. */
|
|
104
|
+
export declare function pruneExpiredHistorySafely(deps: PruneExpiredHistoryDeps): Promise<RetentionSummary>;
|
|
105
|
+
//# sourceMappingURL=retention.d.ts.map
|