@kici-dev/orchestrator 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/dist/agent/dispatcher.d.ts +10 -1
  2. package/dist/agent/host-roster.d.ts +8 -3
  3. package/dist/agent/registry.d.ts +60 -5
  4. package/dist/app.d.ts +10 -5
  5. package/dist/cache/dep-cache.d.ts +14 -6
  6. package/dist/cache/global-eval-round-cache.d.ts +1 -1
  7. package/dist/cache/index.d.ts +1 -1
  8. package/dist/cache/source-cache.d.ts +92 -15
  9. package/dist/cancel/cancel-run.d.ts +20 -2
  10. package/dist/cancel/sweep-stuck-cancelling.d.ts +30 -0
  11. package/dist/cli/api-client.d.ts +10 -0
  12. package/dist/cli/commands/agent-service/logs.d.ts +1 -1
  13. package/dist/cli/commands/agent-service/restart.d.ts +1 -1
  14. package/dist/cli/commands/agent-service/start.d.ts +1 -1
  15. package/dist/cli/commands/agent-service/status.d.ts +1 -1
  16. package/dist/cli/commands/agent-service/stop.d.ts +1 -1
  17. package/dist/cli/commands/agent-service/uninstall.d.ts +1 -1
  18. package/dist/cli/commands/agent.d.ts +25 -0
  19. package/dist/cli/commands/config.d.ts +16 -0
  20. package/dist/cli/commands/db-backup.d.ts +46 -0
  21. package/dist/cli/commands/held-run.d.ts +9 -0
  22. package/dist/cli/commands/orchestrator-service/drain.d.ts +2 -19
  23. package/dist/cli/commands/orchestrator-service/index.d.ts +9 -1
  24. package/dist/cli/commands/orchestrator-service/install-env.d.ts +43 -2
  25. package/dist/cli/commands/orchestrator-service/logs.d.ts +1 -1
  26. package/dist/cli/commands/orchestrator-service/restart.d.ts +1 -1
  27. package/dist/cli/commands/orchestrator-service/start.d.ts +1 -1
  28. package/dist/cli/commands/orchestrator-service/status.d.ts +17 -1
  29. package/dist/cli/commands/orchestrator-service/stop.d.ts +1 -1
  30. package/dist/cli/commands/orchestrator-service/uninstall.d.ts +1 -1
  31. package/dist/cli/commands/orchestrator-service/upgrade.d.ts +2 -1
  32. package/dist/cli/commands/rotate.d.ts +1 -1
  33. package/dist/cli/commands/shared/env-file-mode.d.ts +16 -0
  34. package/dist/cli/commands/shared/upgrade-hooks.d.ts +32 -0
  35. package/dist/cli/commands/shared/upgrade-safety.d.ts +81 -0
  36. package/dist/cli/commands/shared/versioned-upgrade.d.ts +67 -12
  37. package/dist/cli/commands/source.d.ts +1 -0
  38. package/dist/cli/commands/token.d.ts +4 -0
  39. package/dist/cli/commands/trust-policy.d.ts +57 -1
  40. package/dist/cli/join.d.ts +1 -1
  41. package/dist/cli/service/backup-timer.d.ts +108 -0
  42. package/dist/cli/service/compose-path.d.ts +11 -0
  43. package/dist/cli/service/compose.d.ts +22 -0
  44. package/dist/cli/service/deploy-env.d.ts +10 -11
  45. package/dist/cli/service/env-assignment.d.ts +24 -0
  46. package/dist/cli/service/index.d.ts +2 -2
  47. package/dist/cli/service/instance/resolve.d.ts +71 -7
  48. package/dist/cli/service/instance/types.d.ts +15 -0
  49. package/dist/cli/service/launchd.d.ts +7 -0
  50. package/dist/cli/service/systemd.d.ts +1 -0
  51. package/dist/cli/service/types.d.ts +36 -0
  52. package/dist/cli/service/windows.d.ts +16 -0
  53. package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -3
  54. package/dist/cli/wizard/prompts.d.ts +7 -0
  55. package/dist/cli.js +12544 -9299
  56. package/dist/cluster/cluster-settings-reader.d.ts +2 -2
  57. package/dist/cluster/coordinator.d.ts +11 -0
  58. package/dist/cluster/instance-heartbeat.d.ts +96 -0
  59. package/dist/cluster/join-client.d.ts +55 -5
  60. package/dist/cluster/join-handler.d.ts +1 -0
  61. package/dist/cluster/orphan-recovery.d.ts +16 -0
  62. package/dist/cluster/peer-registry.d.ts +10 -0
  63. package/dist/concurrency/dispatch-next-queued.d.ts +1 -13
  64. package/dist/concurrency/index.d.ts +1 -2
  65. package/dist/concurrency/queue-manager.d.ts +63 -18
  66. package/dist/config/env-overlay.d.ts +3 -2
  67. package/dist/config.d.ts +24 -0
  68. package/dist/contexts/release-queued-holds.d.ts +23 -1
  69. package/dist/dashboard/attestation-filters.d.ts +2 -0
  70. package/dist/dashboard/handler.d.ts +22 -2
  71. package/dist/db/deferred-indexes.d.ts +72 -0
  72. package/dist/db/migration-test-harness.d.ts +22 -0
  73. package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +30 -0
  74. package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +23 -0
  75. package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +23 -0
  76. package/dist/db/migrations/134_dispatch_queue_ownership.d.ts +4 -0
  77. package/dist/db/migrations/135_cluster_settings_retention_days.d.ts +4 -0
  78. package/dist/db/migrations/136_master_key_wrapped_tables_key_version.d.ts +4 -0
  79. package/dist/db/migrations/137_execution_runs_pr_head_context.d.ts +4 -0
  80. package/dist/db/migrations/138_execution_runs_subject_trigger_event.d.ts +4 -0
  81. package/dist/db/migrations/139_admin_token_subject.d.ts +4 -0
  82. package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +54 -0
  83. package/dist/db/migrations/141_event_log_shed.d.ts +15 -0
  84. package/dist/db/migrator.d.ts +47 -0
  85. package/dist/db/notify-listener.d.ts +60 -0
  86. package/dist/db/synthetic-job-ids.d.ts +24 -0
  87. package/dist/db/types.d.ts +169 -4
  88. package/dist/deployment/deployment-identity.d.ts +31 -1
  89. package/dist/diagnostics/checks/index.d.ts +2 -1
  90. package/dist/diagnostics/checks/retention.d.ts +24 -0
  91. package/dist/diagnostics/types.d.ts +7 -0
  92. package/dist/events/event-router.d.ts +1 -1
  93. package/dist/fault-injection-types.d.ts +53 -0
  94. package/dist/firecracker/host-network.d.ts +63 -8
  95. package/dist/git/credential-broker.d.ts +35 -2
  96. package/dist/git/inherited-secret.d.ts +7 -2
  97. package/dist/git/job-context.d.ts +7 -4
  98. package/dist/helpers/secure-write.d.ts +35 -0
  99. package/dist/index.js +196 -67
  100. package/dist/metrics/prometheus.d.ts +116 -1
  101. package/dist/oidc/db-signer.d.ts +27 -5
  102. package/dist/oidc/id-token-claims.d.ts +33 -2
  103. package/dist/oidc/oidc-mint-registration.d.ts +6 -0
  104. package/dist/oidc/orchestrator-mint.d.ts +8 -0
  105. package/dist/oidc/reconcile-signing-key.d.ts +37 -0
  106. package/dist/orchestrator-core.d.ts +9 -3
  107. package/dist/pipeline/decorating-secret-resolver.d.ts +1 -1
  108. package/dist/pipeline/dispatch-matched-workflow.d.ts +65 -14
  109. package/dist/pipeline/manual-schedule.d.ts +1 -0
  110. package/dist/pipeline/process-webhook.d.ts +20 -2
  111. package/dist/pipeline/route-or-dispatch-jobs.d.ts +7 -0
  112. package/dist/pipeline/run-event-context.d.ts +39 -0
  113. package/dist/policy/dashboard-write-policy-listener.d.ts +1 -1
  114. package/dist/policy/dashboard-write-policy.d.ts +47 -1
  115. package/dist/provenance/dispatch-context.d.ts +46 -0
  116. package/dist/provenance/verify-deferred-capture.d.ts +39 -0
  117. package/dist/providers/github/auth.d.ts +9 -0
  118. package/dist/providers/github/deliveries.d.ts +118 -0
  119. package/dist/providers/local/lock-file-fetcher.d.ts +8 -0
  120. package/dist/providers/universal-git/config.d.ts +1 -1
  121. package/dist/queue/cleanup.d.ts +13 -0
  122. package/dist/queue/job-queue.d.ts +162 -14
  123. package/dist/queue/retention.d.ts +105 -0
  124. package/dist/reporting/execution-tracker.d.ts +87 -36
  125. package/dist/routes/admin-db.d.ts +5 -0
  126. package/dist/routes/admin-events.d.ts +13 -0
  127. package/dist/routes/admin-held-runs.d.ts +11 -1
  128. package/dist/routes/admin-org-settings.d.ts +9 -0
  129. package/dist/routes/admin-runs.d.ts +7 -4
  130. package/dist/routes/admin-sources.d.ts +15 -0
  131. package/dist/routes/admin-trust-policy.d.ts +11 -0
  132. package/dist/routes/admin.d.ts +25 -0
  133. package/dist/scaler/backend-factory.d.ts +16 -0
  134. package/dist/scaler/bare-metal-backend.d.ts +22 -4
  135. package/dist/scaler/claim-store.d.ts +0 -2
  136. package/dist/scaler/config.d.ts +6 -3
  137. package/dist/scaler/container-backend.d.ts +102 -3
  138. package/dist/scaler/container-hostconfig.d.ts +60 -0
  139. package/dist/scaler/event-backend.d.ts +7 -6
  140. package/dist/scaler/file-tail.d.ts +25 -2
  141. package/dist/scaler/firecracker-api.d.ts +9 -1
  142. package/dist/scaler/firecracker-backend.d.ts +135 -2
  143. package/dist/scaler/host-access.d.ts +59 -0
  144. package/dist/scaler/index.d.ts +1 -0
  145. package/dist/scaler/manager.d.ts +51 -2
  146. package/dist/scaler/scaler-state-store.d.ts +13 -0
  147. package/dist/scaler/types.d.ts +15 -7
  148. package/dist/scaler/warm-pool.d.ts +6 -4
  149. package/dist/secrets/config.d.ts +48 -0
  150. package/dist/secrets/dashboard-encryption-key.d.ts +19 -0
  151. package/dist/secrets/ephemeral-keys.d.ts +9 -3
  152. package/dist/secrets/index.d.ts +3 -1
  153. package/dist/secrets/job-secret-gate.d.ts +76 -0
  154. package/dist/secrets/master-key-rotation.d.ts +65 -0
  155. package/dist/secrets/secret-output-crypto.d.ts +17 -0
  156. package/dist/secrets/secret-resolver.d.ts +23 -6
  157. package/dist/secrets/source-credentials.d.ts +1 -1
  158. package/dist/secrets/token-manager.d.ts +4 -1
  159. package/dist/security/identity-link.d.ts +1 -1
  160. package/dist/security/reduced-privilege-note.d.ts +5 -0
  161. package/dist/security/trust-directory-store.d.ts +1 -1
  162. package/dist/security/trust-policy-gate.d.ts +70 -6
  163. package/dist/security/trust-tier.d.ts +31 -0
  164. package/dist/server.d.ts +1 -1
  165. package/dist/server.js +10793 -5323
  166. package/dist/sources/source-manager.d.ts +1 -1
  167. package/dist/stale-detector/stale-dispatch-candidates.d.ts +43 -0
  168. package/dist/standalone.js +9385 -4160
  169. package/dist/storage/loopback-guard.d.ts +53 -6
  170. package/dist/webhook/event-log.d.ts +14 -1
  171. package/dist/webhook/generic-sources-listener.d.ts +1 -1
  172. package/dist/webhook/ingest-admission.d.ts +24 -0
  173. package/dist/webhook/ingest-overflow-replayer.d.ts +78 -14
  174. package/dist/webhook/ingest-overflow-types.d.ts +13 -1
  175. package/dist/webhook/register-source-bundle.d.ts +23 -0
  176. package/dist/webhook/relay-reinject.d.ts +11 -12
  177. package/dist/webhook/shed-breadcrumb.d.ts +15 -0
  178. package/dist/worker/in-memory-job-queue.d.ts +9 -0
  179. package/dist/ws/fleet-agent-collector.d.ts +7 -0
  180. package/dist/ws/git-credential-relay.d.ts +41 -12
  181. package/dist/ws/platform-client.d.ts +28 -5
  182. package/dist/ws/test-relay-handlers.d.ts +2 -0
  183. package/installer-image-digests.json +3 -3
  184. package/package.json +4 -4
  185. package/sbom.spdx.json +53 -53
  186. package/dist/concurrency/group-tracker.d.ts +0 -51
  187. package/dist/scaler/nftables.d.ts +0 -111
@@ -0,0 +1,118 @@
1
+ /**
2
+ * GitHub App webhook-delivery replay.
3
+ *
4
+ * GitHub keeps every App webhook delivery for a retention window and can be
5
+ * asked to send one again. That is the only way to recover a window of events
6
+ * an orchestrator never received — GitHub itself does not retry a delivery its
7
+ * destination failed to accept, so a window lost while the destination was
8
+ * unreachable is gone unless somebody replays it.
9
+ *
10
+ * Both endpoints are App-level and accept only a JWT signed with the App
11
+ * private key, which the orchestrator holds and nothing upstream does:
12
+ *
13
+ * GET /app/hook/deliveries (cursor-paginated)
14
+ * POST /app/hook/deliveries/{delivery_id}/attempts
15
+ *
16
+ * Verified against docs.github.com/en/rest/apps/webhooks on 2026-09-03.
17
+ */
18
+ import { z } from 'zod';
19
+ import type { Octokit } from '@octokit/rest';
20
+ import { type GitHubAppConfig } from './auth.js';
21
+ /**
22
+ * Safety ceiling on pages walked in one replay. GitHub returns deliveries
23
+ * newest-first, so the walk normally stops as soon as it passes `since`; this
24
+ * bounds a pathological case (a very old `since` on a very busy App) instead of
25
+ * paginating an App's whole retention window.
26
+ */
27
+ export declare const MAX_DELIVERY_PAGES = 100;
28
+ /** What happened to one delivery in a replay. */
29
+ export declare const RedeliverOutcome: z.ZodEnum<{
30
+ failed: "failed";
31
+ redelivered: "redelivered";
32
+ "would-redeliver": "would-redeliver";
33
+ }>;
34
+ export type RedeliverOutcome = z.infer<typeof RedeliverOutcome>;
35
+ /** One delivery as `GET /app/hook/deliveries` returns it (the fields used here). */
36
+ export interface GithubWebhookDelivery {
37
+ id: number;
38
+ guid: string;
39
+ /** ISO-8601 timestamp GitHub attempted the delivery. */
40
+ delivered_at: string;
41
+ redelivery: boolean;
42
+ event: string;
43
+ action: string | null;
44
+ status: string;
45
+ status_code: number;
46
+ installation_id: number | null;
47
+ repository_id: number | null;
48
+ }
49
+ /** Per-delivery result of a replay. */
50
+ export interface RedeliverResult {
51
+ deliveryId: number;
52
+ guid: string;
53
+ deliveredAt: string;
54
+ event: string;
55
+ action: string | null;
56
+ /** The original delivery's HTTP status as GitHub recorded it. */
57
+ originalStatusCode: number;
58
+ outcome: RedeliverOutcome;
59
+ /** Present when `outcome` is `failed`. */
60
+ error?: string;
61
+ }
62
+ /** The whole replay, as the admin route and the CLI report it. */
63
+ export interface RedeliverWindowResult {
64
+ since: string;
65
+ until: string;
66
+ dryRun: boolean;
67
+ /** Deliveries GitHub holds whose `delivered_at` falls in the window. */
68
+ matched: number;
69
+ /**
70
+ * True when the page walk hit its ceiling before reaching a delivery older
71
+ * than `since`, so `matched` counts only what was gathered and the window
72
+ * holds more. Callers must say so rather than reporting a partial replay as
73
+ * a complete one.
74
+ */
75
+ truncated: boolean;
76
+ redelivered: number;
77
+ failed: number;
78
+ results: RedeliverResult[];
79
+ }
80
+ /** The one Octokit method this module needs, so a test can hand in a stub. */
81
+ export type GithubRequest = Pick<Octokit, 'request'>;
82
+ export interface RedeliverWindowOptions {
83
+ /** Inclusive lower bound on `delivered_at`. */
84
+ since: Date;
85
+ /** Exclusive upper bound on `delivered_at`. */
86
+ until: Date;
87
+ /** List and report the matching deliveries without asking GitHub to resend. */
88
+ dryRun?: boolean;
89
+ /** Injected for tests; defaults to a JWT-authenticated App client. */
90
+ octokit?: GithubRequest;
91
+ }
92
+ /** Read the `cursor` query value out of the `Link: <...>; rel="next"` header. */
93
+ export declare function nextCursorFromLink(link: string | undefined): string | null;
94
+ /**
95
+ * Walk `GET /app/hook/deliveries` newest-first and return every delivery whose
96
+ * `delivered_at` falls in `[since, until)`.
97
+ *
98
+ * The walk stops at the first page whose oldest delivery predates `since`:
99
+ * GitHub orders the feed newest-first, so nothing older can match afterwards.
100
+ */
101
+ export declare function listDeliveriesInWindow(octokit: GithubRequest, opts: {
102
+ since: Date;
103
+ until: Date;
104
+ }): Promise<{
105
+ deliveries: GithubWebhookDelivery[];
106
+ truncated: boolean;
107
+ }>;
108
+ /** Ask GitHub to send one delivery again. Resolves once GitHub accepts (202). */
109
+ export declare function redeliverDelivery(octokit: GithubRequest, deliveryId: number): Promise<void>;
110
+ /**
111
+ * Replay every delivery GitHub holds for this App in `[since, until)`.
112
+ *
113
+ * A per-delivery failure is recorded and the replay continues: an operator
114
+ * recovering an outage window wants the deliveries that can be replayed, plus
115
+ * an honest list of the ones that could not.
116
+ */
117
+ export declare function redeliverWindow(app: GitHubAppConfig, opts: RedeliverWindowOptions): Promise<RedeliverWindowResult>;
118
+ //# sourceMappingURL=deliveries.d.ts.map
@@ -15,6 +15,14 @@ import type { LockFileFetcher, LockFile } from '@kici-dev/engine';
15
15
  export declare class LocalLockFileFetcher implements LockFileFetcher {
16
16
  private readonly repoBasePath;
17
17
  readonly provider: 'local';
18
+ /**
19
+ * Never cached: `ref` is ignored below, so a cache keyed on it would serve the
20
+ * first lock file read to every later trigger for the lifetime of the entry —
21
+ * a `kici run --local` dispatch always triggers at `HEAD`, and an in-place run
22
+ * carries its uncommitted edits under an unchanged sha. Reading the file from
23
+ * disk is what makes those edits reach the run.
24
+ */
25
+ readonly cacheable: false;
18
26
  /**
19
27
  * @param repoBasePath - Base directory for the repo(s). When repoIdentifier
20
28
  * starts with 'file://', it is stripped and used as-is. Otherwise
@@ -51,7 +51,7 @@ export declare const SshHostKeyPolicy: z.ZodEnum<{
51
51
  export type SshHostKeyPolicy = z.infer<typeof SshHostKeyPolicy>;
52
52
  /**
53
53
  * Reference to a secret in the orchestrator's secret store. Resolved at
54
- * dispatch time via `SecretResolver.resolveNamed()`. The secret itself lives
54
+ * dispatch time via `SecretResolver.resolveNamedInternal()`. The secret itself lives
55
55
  * at scope `__source__/<sourceId>`, key = `credentialRef.key`, optionally
56
56
  * in a specific backend via `credentialRef.store` (defaults to `pg`).
57
57
  */
@@ -7,6 +7,7 @@ import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.j
7
7
  import type { LogStorage } from '../reporting/log-storage.js';
8
8
  import { JobQueue } from './job-queue.js';
9
9
  import { type CanRouteLabels, type TerminalizeDeps } from './terminalize-unroutable.js';
10
+ import { type RetentionWindows } from './retention.js';
10
11
  /**
11
12
  * Optional cleanup dependencies + retention knobs. Present in platform/hybrid
12
13
  * mode (the only wiring site, `orchestrator-core`, always supplies them); the
@@ -33,6 +34,17 @@ export interface CleanupExtras {
33
34
  stepLogTtlDays?: number;
34
35
  /** Store for `check_run_tracking`; the retention sweep runs only when wired. */
35
36
  checkRunTrackingStore?: CheckRunTrackingStore;
37
+ /**
38
+ * Database-side retention windows, in days. Absent leaves the history sweep
39
+ * off entirely, which is what a caller that has not wired it gets.
40
+ */
41
+ retentionWindows?: RetentionWindows;
42
+ /**
43
+ * True when the cold store is configured. Its archivers then own the tables
44
+ * they register and the history sweep leaves those alone — one owner per
45
+ * table.
46
+ */
47
+ coldStoreEnabled?: boolean;
36
48
  /** Live cluster-settings reader; absent ⇒ the config default is used as-is. */
37
49
  clusterSettings?: ClusterSettingsReader;
38
50
  /** `check_run_tracking` row retention (days); 0 disables. Default 7. */
@@ -96,6 +108,7 @@ export declare function runCleanup(dedup: {
96
108
  dispatchRowsPruned: number;
97
109
  logObjectsPruned: number;
98
110
  checkRunRowsPruned: number;
111
+ historyRowsPruned: number;
99
112
  }>;
100
113
  /**
101
114
  * Build a per-tick handler for the queue-cleanup scheduled job.
@@ -1,7 +1,50 @@
1
- import { type Kysely } from 'kysely';
2
- import { type LabelMatcher, type ResourceRequest } from '@kici-dev/engine';
1
+ import { type Kysely, type SqlBool } from 'kysely';
2
+ import { type CanonicalLabel, type CanonicalMatcher, type LabelMatcher, type ResourceRequest } from '@kici-dev/engine';
3
3
  import type { Database } from '../db/types.js';
4
4
  import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
5
+ /**
6
+ * The ownership predicate, in its two forms. Both come from here so they cannot
7
+ * drift apart, because they answer the same question from opposite directions.
8
+ *
9
+ * **Strict selection** (`recoverableByOwner`) picks the rows a cluster-wide
10
+ * sweep may act on: mine, or a dead instance's. A NULL owner is EXCLUDED — it
11
+ * reads as "unknown owner", never as "not mine", which is the convention
12
+ * migration 119 established for the scaler plane. Sparing it is what makes a
13
+ * rolling upgrade safe: every row a pre-upgrade coordinator dispatched carries
14
+ * a NULL owner, and recovering those would fail every job the rest of the
15
+ * cluster is running, on every coordinator boot — precisely the window in which
16
+ * coordinators boot. A spared orphan is not lost; the stale detector reaps it on
17
+ * the dispatch-time clock.
18
+ *
19
+ * **Permissive write guard** (`writableByOwner`) gates a write on a row this
20
+ * coordinator has already selected some other way: mine, unknown, or a dead
21
+ * instance's. It INCLUDES a NULL owner, because a legitimate local disconnect
22
+ * must still be able to move a pre-upgrade row. Its job is to stop a path that
23
+ * forgot the selection filter from flipping a row a LIVE sibling owns.
24
+ */
25
+ export interface OwnershipPredicateOptions {
26
+ /** This coordinator's instance id. Undefined ⇒ no instance identity. */
27
+ instanceId?: string;
28
+ /** How stale a `cluster_instances` heartbeat may be and still read as live. */
29
+ graceMs: number;
30
+ /** Fully qualified owner column, defaulting to the dispatch queue's. */
31
+ ownerColumn?: string;
32
+ /**
33
+ * Instance ids currently connected as peers. OR-ed in as live, so a peer whose
34
+ * heartbeat row is momentarily stale is not mistaken for dead.
35
+ */
36
+ livePeerIds?: readonly string[];
37
+ }
38
+ /**
39
+ * Strict selection: the rows a cluster-wide sweep may recover. Excludes a NULL
40
+ * owner and excludes any row whose owner is still live.
41
+ */
42
+ export declare function recoverableByOwner(opts: OwnershipPredicateOptions): import("kysely").RawBuilder<SqlBool>;
43
+ /**
44
+ * Permissive write guard: allow the write unless a LIVE sibling owns the row.
45
+ * A NULL owner passes.
46
+ */
47
+ export declare function writableByOwner(opts: OwnershipPredicateOptions): import("kysely").RawBuilder<SqlBool>;
5
48
  /** Info about an expired dispatch_queue entry, returned by markExpired(). */
6
49
  export interface ExpiredJobInfo {
7
50
  /** dispatch_queue row ID */
@@ -15,11 +58,15 @@ export interface ExpiredJobInfo {
15
58
  * any agent could ever have run it. A job that expires with NO matching agent
16
59
  * is `unroutable` (a fleet/label problem); one whose agent existed but never
17
60
  * freed up is `timed_out_stale` (a capacity problem).
61
+ *
62
+ * Canonical: the probe compares them against a live agent's canonical labels,
63
+ * so a case-sensitive read here would report a routable job `unroutable` and
64
+ * send the operator hunting a fleet problem that does not exist.
18
65
  */
19
- runsOnLabels: string[];
20
- runsOnPatterns: LabelMatcher[];
21
- excludeLabels: string[];
22
- excludePatterns: LabelMatcher[];
66
+ runsOnLabels: CanonicalLabel[];
67
+ runsOnPatterns: CanonicalMatcher[];
68
+ excludeLabels: CanonicalLabel[];
69
+ excludePatterns: CanonicalMatcher[];
23
70
  }
24
71
  /**
25
72
  * A pending job as the unroutable probe sees it: the same routing facts the
@@ -103,6 +150,8 @@ export interface QueuedJobInput {
103
150
  sourceTarUrl?: string;
104
151
  /** Content hash of the pre-compiled bundle for verification. */
105
152
  sourceTarHash?: string;
153
+ /** SHA-256 of the source tarball's own bytes (what `sourceTarHash` never was). */
154
+ sourceTarDigest?: string;
106
155
  /** URL to pre-built dependency tarball (from dep cache). Passed through to job.dispatch. */
107
156
  depsUrl?: string;
108
157
  /** SHA-256 hash of the dependency tarball for integrity verification. */
@@ -140,7 +189,8 @@ export interface QueuedJob {
140
189
  runId: string;
141
190
  workflowName: string;
142
191
  jobName: string;
143
- runsOnLabels: string[];
192
+ /** Canonical: folded by `rowToQueuedJob`, whatever case the row holds. */
193
+ runsOnLabels: CanonicalLabel[];
144
194
  jobConfig: Record<string, unknown>;
145
195
  repoUrl: string;
146
196
  ref: string;
@@ -160,18 +210,24 @@ export interface QueuedJob {
160
210
  sourceTarUrl?: string;
161
211
  /** Content hash of the pre-compiled bundle for verification. */
162
212
  sourceTarHash?: string;
213
+ /** SHA-256 of the source tarball's own bytes (what `sourceTarHash` never was). */
214
+ sourceTarDigest?: string;
163
215
  /** URL to pre-built dependency tarball (from dep cache). Passed through to job.dispatch. */
164
216
  depsUrl?: string;
165
217
  /** SHA-256 hash of the dependency tarball for integrity verification. */
166
218
  depsHash?: string;
167
219
  /** Request trace ID for cross-tier correlation. Passed through to job.dispatch. */
168
220
  requestId?: string;
169
- /** Labels that the dispatched agent must NOT have. */
170
- excludeLabels: string[];
171
- /** Regex matchers the agent's labels must satisfy (JS post-filter on the exact @> prefilter). */
172
- runsOnPatterns: LabelMatcher[];
173
- /** Regex matchers that disqualify an agent (JS post-filter). */
174
- excludePatterns: LabelMatcher[];
221
+ /** Labels that the dispatched agent must NOT have. Canonical. */
222
+ excludeLabels: CanonicalLabel[];
223
+ /**
224
+ * Regex matchers the agent's labels must satisfy (JS post-filter on the exact
225
+ * `@>` prefilter). Canonical, so the stored pattern carries the `i` flag
226
+ * whether or not the lock file that produced it did.
227
+ */
228
+ runsOnPatterns: CanonicalMatcher[];
229
+ /** Regex matchers that disqualify an agent (JS post-filter). Canonical. */
230
+ excludePatterns: CanonicalMatcher[];
175
231
  /**
176
232
  * Per-job resource request and limit (K8s-style). Materialized from `jobConfig.resources`
177
233
  * by `rowToQueuedJob` so callers can read it without re-parsing the JSON column.
@@ -207,6 +263,8 @@ export declare class JobQueue {
207
263
  private readonly defaultMaxDepth;
208
264
  private readonly defaultTimeoutMs;
209
265
  private readonly clusterSettings?;
266
+ private readonly instanceId?;
267
+ private readonly ownershipGraceMs;
210
268
  /** Per-job (per-org) queue-timeout resolver; falls back to defaultTimeoutMs. */
211
269
  private readonly getQueueTimeoutMs?;
212
270
  /** 1-second TTL cache for pending depth count to avoid extra SELECT COUNT per enqueue. */
@@ -225,7 +283,23 @@ export declare class JobQueue {
225
283
  getQueueTimeoutMs?: (job: {
226
284
  jobConfig?: Record<string, unknown>;
227
285
  }) => Promise<number>;
286
+ /**
287
+ * This coordinator's instance id, stamped onto every row it dispatches so
288
+ * a cluster-wide sweep can tell its own rows from a sibling's. Optional
289
+ * for the same reason `AgentRegistryDeps.instanceId` is: a worker with no
290
+ * cluster identity (and every unit test) leaves it undefined and the
291
+ * column stays NULL, which reads as "unknown owner".
292
+ */
293
+ instanceId?: string;
294
+ /**
295
+ * How stale a `cluster_instances` heartbeat may be and still read as
296
+ * live. Defaults to {@link instanceLivenessGraceMs} at the default
297
+ * recovery grace period.
298
+ */
299
+ ownershipGraceMs?: number;
228
300
  });
301
+ /** Ownership predicate inputs, in the shape both predicate builders take. */
302
+ private ownershipOptions;
229
303
  /**
230
304
  * Enqueue a job. Checks depth first, rejects with 'queue full' if >= the
231
305
  * fleet-wide `queue_max_depth` (cluster_settings, falling back to the config
@@ -530,6 +604,19 @@ export declare class JobQueue {
530
604
  * refreshing the cache periodically via {@link JobQueue.getDepthBreakdown}.
531
605
  */
532
606
  readCachedDepthBreakdown(): DispatchQueueDepthBreakdown | null;
607
+ /**
608
+ * The run's dispatched-or-recovering rows with the durable facts a cancel
609
+ * needs to route: which agent holds the job, and which coordinator dispatched
610
+ * it. Both come from the row rather than from a process map, because the
611
+ * coordinator handling a cancel is the owner only by chance — the Platform
612
+ * routes `run.cancel.request` by routing key and picks any connected pool
613
+ * member.
614
+ */
615
+ getDispatchedJobOwnersByRunId(runId: string): Promise<Array<{
616
+ jobId: string;
617
+ agentId: string | null;
618
+ ownerInstanceId: string | null;
619
+ }>>;
533
620
  /**
534
621
  * Get job IDs for a run that are currently dispatched or recovering.
535
622
  * Used by the cancel-run API to send job.cancel to the right agents.
@@ -544,8 +631,47 @@ export declare class JobQueue {
544
631
  * replacement coord on Raft leader switch can recreate the recovery
545
632
  * timer (via `getRecoveringJobs()` on boot) or expire the row in
546
633
  * the leader-gated sweep (`sweepExpiredRecoveries()`).
634
+ *
635
+ * Carries the permissive ownership write guard: a row owned by a LIVE sibling
636
+ * is left alone. Callers are expected to have selected owned rows already
637
+ * ({@link JobQueue.getOrphanedDispatchedJobs}); this is the backstop for a
638
+ * path that forgets to, since flipping a live sibling's row to `recovering`
639
+ * starts a 120-second clock that ends by failing a job nobody disconnected
640
+ * from. A NULL owner still passes — a local disconnect must be able to move a
641
+ * pre-upgrade row.
642
+ *
643
+ * @returns whether a row was flipped.
644
+ */
645
+ markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<boolean>;
646
+ /**
647
+ * The `dispatched` rows this coordinator may start recovery timers for on
648
+ * boot: its own, and those whose owning coordinator is no longer alive.
649
+ *
650
+ * Replaces an unfiltered `getJobsByStatus(Dispatched)` over the shared table.
651
+ * That scan made every coordinator boot flip every sibling's in-flight row to
652
+ * `recovering`; the agents running those jobs were never disconnected, so
653
+ * nothing reclaimed them, and 120 seconds later the booting coordinator
654
+ * failed every one of them while they ran to completion. The documented
655
+ * rolling-restart procedure triggered it on every restart.
656
+ *
657
+ * A NULL owner is spared — see {@link recoverableByOwner} for why, and for
658
+ * what reaps a genuinely orphaned one instead.
659
+ */
660
+ getOrphanedDispatchedJobs(): Promise<Array<{
661
+ id: string;
662
+ runId: string;
663
+ agentId: string | null;
664
+ }>>;
665
+ /**
666
+ * How many `dispatched` rows this coordinator deliberately left alone on
667
+ * boot: owned by a live sibling, or of unknown ownership. Read only to log
668
+ * and count the decision, so "recovery spared N rows" is visible rather than
669
+ * being the silent absence of a log line.
547
670
  */
548
- markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<void>;
671
+ countSparedDispatchedJobs(): Promise<{
672
+ ownedElsewhere: number;
673
+ unknownOwner: number;
674
+ }>;
549
675
  /**
550
676
  * Stamp the dispatch-acknowledgment deadline for a dispatched job.
551
677
  * Only touches rows still in 'dispatched' for safety.
@@ -621,6 +747,28 @@ export declare class JobQueue {
621
747
  * was not in 'dispatched' state (nothing requeued).
622
748
  */
623
749
  requeue(jobId: string): Promise<number | null>;
750
+ /**
751
+ * Requeue a dispatched job **only** when its ack genuinely never landed.
752
+ *
753
+ * One atomic guarded UPDATE that both proves the ack is still outstanding
754
+ * and requeues. It exists because {@link requeue}'s only guard is
755
+ * `status = 'dispatched'`, and an acked, executing job is still
756
+ * `dispatched` — so an ack timer that fires late (a coordinator restart
757
+ * re-armed it, or the ack raced the deadline) requeues a running job and a
758
+ * second agent claims it. Both agents then execute the same job.
759
+ *
760
+ * The proof has to happen in the same statement as the requeue: the ack path
761
+ * clears `ack_deadline`/`ack_agent_id`, so a separate read-then-requeue is a
762
+ * race, and a guard bolted onto a caller that clears the deadline first can
763
+ * never see the evidence it needs.
764
+ *
765
+ * {@link requeue} stays for `job.reject` and the scaler-disconnect path,
766
+ * whose rows legitimately carry no deadline.
767
+ *
768
+ * @returns the post-increment dispatch_attempts, or null when the ack landed
769
+ * (or the row moved on) and nothing was requeued.
770
+ */
771
+ requeueIfAwaitingAck(jobId: string, expectedAgentId: string | null): Promise<number | null>;
624
772
  /**
625
773
  * Get the full QueuedJob row by ID regardless of status. Used by the
626
774
  * dispatcher's redispatch path, which needs runsOnLabels / excludeLabels /
@@ -0,0 +1,105 @@
1
+ /**
2
+ * Database-side retention for the tables that scale with run volume.
3
+ *
4
+ * The cold store is the other half of this: its archivers do
5
+ * archive-then-delete inside one transaction and own the four tables they
6
+ * register. But the cold store needs `KICI_COLD_STORE_ENABLED=true` and a
7
+ * bucket, and with neither — the default, and the quickstart path — nothing
8
+ * aged at all, so `execution_steps`, `event_log` and `access_log` grew without
9
+ * bound on the operator's own disk.
10
+ *
11
+ * Two rules keep the tiers from fighting:
12
+ *
13
+ * - When the cold store is on, this tier skips every table an archiver
14
+ * registers. Two owners deleting the same rows would race the archiver's
15
+ * FK guard and could delete a parent whose archive chunk was never
16
+ * written. One owner per table.
17
+ * - Nothing is deleted on the first pass. The first pass with a non-zero
18
+ * window stamps `cluster_settings.retention_announced_at` and reports what
19
+ * it would remove; deletion begins a week later. An upgrade that silently
20
+ * deleted an operator's history on its first night would be a behaviour
21
+ * change nobody was warned about.
22
+ *
23
+ * Every delete is batched — select ids under a `LIMIT`, then delete by id,
24
+ * looping until a pass comes back short. One unbounded `DELETE` over a table
25
+ * holding millions of rows holds locks for as long as it runs.
26
+ */
27
+ import { type Kysely } from 'kysely';
28
+ import type { Database } from '../db/types.js';
29
+ /** Rows selected, and deleted, per statement. */
30
+ export declare const RETENTION_BATCH_SIZE = 5000;
31
+ /** Days between the announce stamp and the first deletion. */
32
+ export declare const RETENTION_ANNOUNCE_DAYS = 7;
33
+ /** Terminal run statuses. A run still queued, running, held or cancelling is
34
+ * live work whatever its age. */
35
+ export declare const TERMINAL_RUN_STATUS_LIST: readonly string[];
36
+ /** Terminal held-run statuses. A pending hold is never pruned. */
37
+ export declare const TERMINAL_HELD_RUN_STATUSES: readonly string[];
38
+ /**
39
+ * Tables the cold-store archivers register. When the cold store is on these
40
+ * are theirs alone — including the `execution_jobs` / `execution_steps`
41
+ * children, which the run archiver removes with their parent.
42
+ */
43
+ export declare const COLD_STORE_OWNED_TABLES: readonly string[];
44
+ /** The `cluster_settings` columns this module reads. */
45
+ export type ClusterRetentionColumn = 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days';
46
+ export interface RetentionWindows {
47
+ /** Terminal runs, and the jobs and steps beneath them. 0 disables. */
48
+ runRetentionDays: number;
49
+ /** access_log, secret_audit_log, event_log. 0 disables. */
50
+ auditRetentionDays: number;
51
+ /** attestations, pending_attestations. 0 disables. */
52
+ provenanceRetentionDays: number;
53
+ /** Terminal held_runs. 0 disables. */
54
+ heldRunRetentionDays: number;
55
+ }
56
+ export interface PruneExpiredHistoryDeps {
57
+ db: Kysely<Database>;
58
+ windows: RetentionWindows;
59
+ /** True when the cold store is configured and owns its tables. */
60
+ coldStoreEnabled: boolean;
61
+ now?: () => Date;
62
+ batchSize?: number;
63
+ }
64
+ export interface RetentionSummary {
65
+ /** Rows deleted, per table. */
66
+ deleted: Record<string, number>;
67
+ /** Rows that WOULD have been deleted, while the announce window runs. */
68
+ pending: Record<string, number>;
69
+ /** True while the announce window has not elapsed, so nothing was deleted. */
70
+ announcing: boolean;
71
+ }
72
+ /** True when every window is 0, i.e. the operator disabled the whole tier. */
73
+ export declare function allWindowsDisabled(windows: RetentionWindows): boolean;
74
+ /**
75
+ * Delete rows in bounded batches, returning the total removed.
76
+ *
77
+ * The loop stops as soon as a pass comes back short, which is what bounds it
78
+ * on a table that is still being written to.
79
+ */
80
+ export declare function deleteInBatches(table: string, selectIds: (limit: number) => Promise<string[]>, deleteIds: (ids: string[]) => Promise<void>, batchSize: number): Promise<number>;
81
+ /**
82
+ * Read the announce stamp, setting it on the first pass that has work to do.
83
+ *
84
+ * Returns true once the window has elapsed and deletion may begin.
85
+ */
86
+ export declare function announceGatePassed(db: Kysely<Database>, now: Date, announceDays?: number): Promise<boolean>;
87
+ /**
88
+ * Prune every table this tier owns, honouring the windows, the cold-store
89
+ * ownership split, and the announce gate.
90
+ */
91
+ export declare function pruneExpiredHistory(deps: PruneExpiredHistoryDeps): Promise<RetentionSummary>;
92
+ /**
93
+ * Resolve the effective windows: a live `cluster_settings` value wins over the
94
+ * configured default.
95
+ *
96
+ * `getNumber` returns a genuine 0 when the operator set the knob to 0, which is
97
+ * the documented way to disable a window — so this stays a fallback-on-null
98
+ * read, never `||`.
99
+ */
100
+ export declare function resolveRetentionWindows(defaults: RetentionWindows, clusterSettings?: {
101
+ getNumber(column: ClusterRetentionColumn, fallback: number): Promise<number>;
102
+ }): Promise<RetentionWindows>;
103
+ /** Wrap {@link pruneExpiredHistory} so a failure never aborts the sweep. */
104
+ export declare function pruneExpiredHistorySafely(deps: PruneExpiredHistoryDeps): Promise<RetentionSummary>;
105
+ //# sourceMappingURL=retention.d.ts.map