@kici-dev/orchestrator 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/dist/__test-helpers__/fake-scaler-state-store.d.ts +24 -0
  2. package/dist/__test-helpers__/mock-db-query.d.ts +130 -0
  3. package/dist/__test-helpers__/mock-db.d.ts +23 -0
  4. package/dist/agent/dispatcher.d.ts +81 -13
  5. package/dist/agent/host-roster.d.ts +37 -5
  6. package/dist/agent/registry.d.ts +2 -0
  7. package/dist/app.d.ts +33 -6
  8. package/dist/approvals/apply-decision.d.ts +67 -19
  9. package/dist/approvals/held-run-release-wiring.d.ts +45 -0
  10. package/dist/approvals/independent-wiring.d.ts +55 -0
  11. package/dist/approvals/resume-router.d.ts +9 -0
  12. package/dist/approvals/triggerer-subject.d.ts +37 -0
  13. package/dist/cache/dep-cache.d.ts +54 -10
  14. package/dist/cache/global-eval-round-cache.d.ts +7 -0
  15. package/dist/cli/api-client.d.ts +18 -0
  16. package/dist/cli/commands/cluster-settings.d.ts +11 -0
  17. package/dist/cli/commands/diagnose.d.ts +6 -0
  18. package/dist/cli/commands/held-run.d.ts +58 -0
  19. package/dist/cli/commands/trust-policy.d.ts +117 -12
  20. package/dist/cli/kici-admin.d.ts +10 -0
  21. package/dist/cli.js +2272 -1025
  22. package/dist/cluster/cluster-identity.d.ts +3 -3
  23. package/dist/cluster/cluster-settings-reader.d.ts +7 -1
  24. package/dist/cluster/coordinator.d.ts +44 -0
  25. package/dist/cluster/index.d.ts +4 -0
  26. package/dist/cluster/peer-handler.d.ts +19 -0
  27. package/dist/cluster/peer-registry.d.ts +17 -0
  28. package/dist/cluster/plan-headroom-store.d.ts +26 -0
  29. package/dist/cluster/worker-eviction.d.ts +69 -0
  30. package/dist/cold-store/load-event-log-range.d.ts +10 -3
  31. package/dist/cold-store/load-secret-audit-log-range.d.ts +4 -3
  32. package/dist/cold-store/tables/secret-audit-log.d.ts +4 -3
  33. package/dist/config.d.ts +21 -21
  34. package/dist/contexts/held-runs.d.ts +123 -16
  35. package/dist/contexts/protection/aggregate.d.ts +3 -2
  36. package/dist/contexts/protection/branch-gate.d.ts +15 -0
  37. package/dist/contexts/protection/concurrency-gate.d.ts +8 -2
  38. package/dist/contexts/protection/pipeline.d.ts +18 -0
  39. package/dist/contexts/release-queued-holds.d.ts +35 -0
  40. package/dist/dashboard/attestation-filters.d.ts +8 -0
  41. package/dist/dashboard/handler.d.ts +6 -24
  42. package/dist/db/migrations/116_org_plan_headroom.d.ts +4 -0
  43. package/dist/db/migrations/117_invoke_gate_proxy_jobs.d.ts +4 -0
  44. package/dist/db/migrations/118_invoke_gate_scheduling.d.ts +4 -0
  45. package/dist/db/migrations/119_scaler_ha_ownership.d.ts +4 -0
  46. package/dist/db/migrations/120_cluster_settings_scaler_reaper.d.ts +4 -0
  47. package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +19 -0
  48. package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +25 -0
  49. package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +24 -0
  50. package/dist/db/migrations/124_cluster_settings_provision_backoff.d.ts +4 -0
  51. package/dist/db/migrations/125_org_trust_directory.d.ts +30 -0
  52. package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +28 -0
  53. package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +26 -0
  54. package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +23 -0
  55. package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +24 -0
  56. package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +32 -0
  57. package/dist/db/types.d.ts +368 -10
  58. package/dist/entry-helpers.d.ts +28 -0
  59. package/dist/events/event-emitter.d.ts +22 -0
  60. package/dist/events/event-router.d.ts +87 -1
  61. package/dist/events/types.d.ts +2 -23
  62. package/dist/git/credential-broker.d.ts +94 -0
  63. package/dist/git/inherited-secret.d.ts +24 -0
  64. package/dist/git/installation-token.d.ts +38 -0
  65. package/dist/git/job-context.d.ts +13 -0
  66. package/dist/index.js +324 -84
  67. package/dist/metrics/agent-metrics-aggregator.d.ts +2 -2
  68. package/dist/metrics/prometheus.d.ts +111 -3
  69. package/dist/oidc/oidc-mint-registration.d.ts +7 -3
  70. package/dist/orchestrator-core.d.ts +79 -23
  71. package/dist/pipeline/direct-ingress-deps.d.ts +100 -0
  72. package/dist/pipeline/dispatch-matched-workflow.d.ts +302 -31
  73. package/dist/pipeline/global-eval-round.d.ts +14 -0
  74. package/dist/pipeline/internal-event-pipeline.d.ts +140 -0
  75. package/dist/pipeline/invoke-gate.d.ts +164 -0
  76. package/dist/pipeline/needs-scheduler.d.ts +13 -0
  77. package/dist/pipeline/process-webhook.d.ts +119 -31
  78. package/dist/pipeline/processor.d.ts +226 -21
  79. package/dist/pipeline/rerun.d.ts +21 -0
  80. package/dist/pipeline/resume-workflow.d.ts +40 -7
  81. package/dist/pipeline/route-or-dispatch-jobs.d.ts +9 -0
  82. package/dist/pipeline/security-hold-check.d.ts +217 -0
  83. package/dist/pipeline/undispatched-hold-checks.d.ts +23 -0
  84. package/dist/policy/dashboard-write-policy-listener.d.ts +62 -0
  85. package/dist/policy/dashboard-write-policy.d.ts +14 -0
  86. package/dist/provider-registry.d.ts +10 -2
  87. package/dist/providers/github/check-status-poster.d.ts +11 -0
  88. package/dist/providers/github/index.d.ts +0 -2
  89. package/dist/providers/github/normalizer.d.ts +3 -2
  90. package/dist/providers/local/normalizer.d.ts +3 -5
  91. package/dist/providers/universal-git/index.d.ts +11 -2
  92. package/dist/queue/job-queue.d.ts +43 -1
  93. package/dist/registration/extractor.d.ts +3 -3
  94. package/dist/registration/registration-index.d.ts +7 -0
  95. package/dist/registration/registration-store.d.ts +22 -0
  96. package/dist/reporting/check-run-reporter.d.ts +94 -1
  97. package/dist/reporting/check-run-summary.d.ts +25 -1
  98. package/dist/reporting/execution-tracker.d.ts +207 -2
  99. package/dist/reporting/run-aggregator.d.ts +4 -14
  100. package/dist/reporting/step-display-order.d.ts +43 -0
  101. package/dist/routes/admin-held-runs.d.ts +142 -0
  102. package/dist/routes/admin-trust-policy.d.ts +34 -6
  103. package/dist/routes/admin.d.ts +9 -0
  104. package/dist/scaler/backend-factory.d.ts +55 -0
  105. package/dist/scaler/bare-metal-backend.d.ts +43 -5
  106. package/dist/scaler/claim-store.d.ts +119 -0
  107. package/dist/scaler/config.d.ts +4 -0
  108. package/dist/scaler/container-backend.d.ts +18 -4
  109. package/dist/scaler/container-routing.d.ts +23 -0
  110. package/dist/scaler/container-spawn.d.ts +28 -0
  111. package/dist/scaler/event-backend.d.ts +124 -0
  112. package/dist/scaler/event-provision-reaper.d.ts +350 -0
  113. package/dist/scaler/failure-tracker.d.ts +1 -1
  114. package/dist/scaler/firecracker-backend.d.ts +33 -6
  115. package/dist/scaler/index.d.ts +11 -1
  116. package/dist/scaler/label-matcher.d.ts +4 -3
  117. package/dist/scaler/manager.d.ts +899 -72
  118. package/dist/scaler/resolve-container-auth.d.ts +45 -0
  119. package/dist/scaler/scaler-events.d.ts +11 -0
  120. package/dist/scaler/scaler-state-store.d.ts +294 -10
  121. package/dist/scaler/types.d.ts +131 -13
  122. package/dist/scaler/warm-pool.d.ts +132 -29
  123. package/dist/security/comment-handler.d.ts +50 -11
  124. package/dist/security/identity-link.d.ts +60 -0
  125. package/dist/security/lock-source.d.ts +11 -13
  126. package/dist/security/reduced-privilege-note.d.ts +59 -0
  127. package/dist/security/trust-directory-store.d.ts +233 -0
  128. package/dist/security/trust-policy-gate.d.ts +94 -59
  129. package/dist/security/trust-policy-store.d.ts +3 -0
  130. package/dist/security/trust-resolver.d.ts +14 -98
  131. package/dist/server.d.ts +11 -1
  132. package/dist/server.js +17671 -7350
  133. package/dist/stale-detector/gate-deadline-detector.d.ts +52 -0
  134. package/dist/stale-detector/stale-run-detector.d.ts +61 -3
  135. package/dist/standalone.js +24238 -13822
  136. package/dist/worker/in-memory-job-queue.d.ts +1 -1
  137. package/dist/ws/agent-handler.d.ts +28 -4
  138. package/dist/ws/dashboard-context-handler.d.ts +28 -4
  139. package/dist/ws/dashboard-global-workflows-handler.d.ts +6 -0
  140. package/dist/ws/failure-messages.d.ts +2 -0
  141. package/dist/ws/git-credential-relay.d.ts +39 -0
  142. package/dist/ws/oidc-token-relay.d.ts +6 -11
  143. package/dist/ws/platform-client.d.ts +54 -9
  144. package/installer-image-digests.json +3 -3
  145. package/package.json +24 -19
  146. package/sbom.spdx.json +864 -822
  147. package/dist/approvals/team-membership-lookup.d.ts +0 -13
  148. package/dist/providers/github/contributor-resolver.d.ts +0 -30
  149. package/dist/security/contributor-cache.d.ts +0 -83
@@ -30,7 +30,7 @@ import type { ProviderRegistry } from '../provider-registry.js';
30
30
  import type { StepLogBuffer } from './step-log-buffer.js';
31
31
  import { type SourceLocationData } from './check-run-summary.js';
32
32
  import type { CheckRunTrackingStore } from './check-run-tracking-store.js';
33
- import { type TerminalJobStatus } from '@kici-dev/engine';
33
+ import { CheckRunConclusion, type TerminalJobStatus } from '@kici-dev/engine';
34
34
  /**
35
35
  * Dependencies for the CheckRunReporter.
36
36
  */
@@ -172,6 +172,15 @@ interface UpdateJobStatusOptions {
172
172
  runIdForLogs?: string;
173
173
  /** Job ID for StepLogBuffer lookup. */
174
174
  jobId?: string;
175
+ /**
176
+ * The run's resolved trust tier and lock-file branch. When the tier is
177
+ * anything other than `trusted`, the completion summary leads with the
178
+ * reduced-privilege note so a contributor reading a failed job on a fork pull
179
+ * request can see which parts of the build environment the run did not have.
180
+ * Both absent for a run whose trust never resolved.
181
+ */
182
+ trustTier?: string;
183
+ lockFileSource?: string;
175
184
  }
176
185
  /**
177
186
  * Options for updateWorkflowStatus.
@@ -193,6 +202,43 @@ interface UpdateWorkflowStatusOptions {
193
202
  requestId?: string;
194
203
  /** Explicit runId for trace context (falls back to AsyncLocalStorage context). */
195
204
  runId?: string;
205
+ /**
206
+ * The run's resolved trust tier and lock-file branch, same contract as
207
+ * {@link UpdateJobStatusOptions}. The roll-up check carries the note too: a
208
+ * contributor who reads only `kici/<workflow>` — the one branch protection
209
+ * usually requires — would otherwise see a run fail with no explanation.
210
+ */
211
+ trustTier?: string;
212
+ lockFileSource?: string;
213
+ }
214
+ /**
215
+ * Options for completeUndispatchedCheckRuns.
216
+ */
217
+ export interface CompleteUndispatchedCheckRunsOptions {
218
+ provider: string;
219
+ owner: string;
220
+ repo: string;
221
+ sha: string;
222
+ workflowName: string;
223
+ /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
224
+ workflowRepoIdentifier?: string;
225
+ /**
226
+ * The job names `setPending` / `setPendingAwait` created a check run for.
227
+ * Pass the same list that call used, or the names will not match the check
228
+ * runs on the commit.
229
+ */
230
+ jobNames: string[];
231
+ installationId?: number;
232
+ /** Routing key for per-app credential lookup (e.g., "github:12345"). */
233
+ routingKey?: string;
234
+ /** Explicit requestId for trace context (falls back to AsyncLocalStorage context). */
235
+ requestId?: string;
236
+ /** Explicit runId for trace context (falls back to AsyncLocalStorage context). */
237
+ runId?: string;
238
+ /** The conclusion to complete each check run with. */
239
+ conclusion: CheckRunConclusion;
240
+ /** The check-run output body. The title stays the standard `KiCI: <label>` form. */
241
+ summary: string;
196
242
  }
197
243
  /**
198
244
  * Options for updateStepProgress.
@@ -276,7 +322,26 @@ export declare class CheckRunReporter {
276
322
  * still find every key for a runId at cleanup time.
277
323
  */
278
324
  private readonly runIdToKeys;
325
+ /**
326
+ * Per-key serialization of GitHub check-run PATCHes. Two updates for one
327
+ * check run must never be in flight at GitHub simultaneously: the terminal
328
+ * `completed` write and an earlier `in_progress` write race last-write-wins,
329
+ * and if the `in_progress` PATCH lands second it reopens the check run to the
330
+ * permanently-unresolved `{ status: in_progress, conclusion: <terminal> }`
331
+ * state. The `terminalSent` re-check inside `updateCheckRun` cannot help once
332
+ * an `in_progress` PATCH has passed it and is awaiting the network — it is a
333
+ * check-then-await. Chaining every PATCH for a key through this map makes the
334
+ * re-check and the PATCH atomic relative to the completion write.
335
+ */
336
+ private readonly updateLocks;
279
337
  constructor(deps: CheckRunReporterDeps);
338
+ /**
339
+ * Run `fn` after every previously-queued check-run PATCH for `key` has
340
+ * settled, so all updates to one check run execute strictly in order. The
341
+ * chain swallows prior errors (each is surfaced to its own caller) so one
342
+ * failed PATCH does not wedge the key.
343
+ */
344
+ private runUpdateExclusive;
280
345
  /**
281
346
  * Update the provider registry used for per-routing-key credential lookup.
282
347
  * Called after config reload when the provider registry is rebuilt.
@@ -320,6 +385,33 @@ export declare class CheckRunReporter {
320
385
  * Fire-and-forget: errors are logged but don't block the pipeline.
321
386
  */
322
387
  updateWorkflowStatus(opts: UpdateWorkflowStatusOptions): void;
388
+ /**
389
+ * Complete the queued check runs of a workflow that never dispatched a job.
390
+ *
391
+ * `setPendingAwait` creates `kici/<workflow>` and one
392
+ * `kici/<workflow>/job/<name>` per static job before the pipeline knows
393
+ * whether the workflow will run. When the workflow then ends without
394
+ * dispatching one — a hold rejected or expired, or any pre-dispatch init
395
+ * failure — none of these names ever reaches a job or run record, so
396
+ * `updateJobStatus` and `updateWorkflowStatus` are never called for them, and
397
+ * `doCleanupStaleCheckRuns` skips them because that sweep only updates check
398
+ * runs whose status is `in_progress`. They stay `queued` on the commit, which
399
+ * on a pull request reads as a check that never finishes and blocks branch
400
+ * protection.
401
+ *
402
+ * Completes only the check runs this reporter can resolve an id for (L1 cache
403
+ * or the `check_run_tracking` row), and skips a key already latched terminal.
404
+ *
405
+ * The build check `kici/<workflow>/setup` is deliberately not in the set,
406
+ * because that latch does not reliably cover it. `setBuildComplete` stamps
407
+ * `terminal_sent_at` on the row but adds nothing to the in-process
408
+ * `terminalSent` set, and `resolveCheckRunId` returns on an L1 hit without
409
+ * reading the row — so a build check this process completed a moment ago
410
+ * resolves with no latch, and completing it again would overwrite a real
411
+ * build conclusion. An unreadable row is the harmless direction: it yields
412
+ * no id and the name is skipped.
413
+ */
414
+ completeUndispatchedCheckRuns(opts: CompleteUndispatchedCheckRunsOptions): Promise<void>;
323
415
  /**
324
416
  * Clean up stale check runs left by a dead orchestrator.
325
417
  *
@@ -519,6 +611,7 @@ export declare class CheckRunReporter {
519
611
  * Supports both 'completed' and 'in_progress' statuses.
520
612
  */
521
613
  private updateCheckRun;
614
+ private updateCheckRunLocked;
522
615
  }
523
616
  /**
524
617
  * Build a meaningful failure description from agent job status data.
@@ -58,6 +58,17 @@ interface BuildCheckRunSummaryOptions {
58
58
  };
59
59
  /** Total job duration in milliseconds. */
60
60
  jobDurationMs?: number;
61
+ /**
62
+ * Bytes a caller will add around the returned summary — the reduced-privilege
63
+ * note it prepends, the annotation-count line it appends — subtracted from
64
+ * the byte budget below.
65
+ *
66
+ * Without it the budget is spent in full here and the caller's own bytes push
67
+ * the final string over the API's cap, which rejects the entire update and
68
+ * leaves the check run unresolved. Reserving instead drops one more tier of
69
+ * log lines, which costs context rather than the whole report.
70
+ */
71
+ reservedBytes?: number;
61
72
  }
62
73
  interface BuildAnnotationsOptions {
63
74
  stepResults: StepResultData[];
@@ -71,12 +82,25 @@ interface BuildProgressTextOptions {
71
82
  runId: string;
72
83
  };
73
84
  }
85
+ /** GitHub Checks API output.summary byte limit. */
86
+ export declare const SUMMARY_BYTE_LIMIT = 65535;
87
+ /**
88
+ * Cut `summary` so it fits the API's cap, whatever a caller wrapped around it.
89
+ *
90
+ * The last line of defence, not the primary mechanism: `reservedBytes` is what
91
+ * keeps a summary readable by dropping log tiers, and this only runs when the
92
+ * final string is still over. Truncating at a UTF-8 boundary matters — a cut
93
+ * through a multi-byte sequence produces a replacement character and can leave
94
+ * the string one byte over the cap it was cut to fit.
95
+ */
96
+ export declare function clampSummaryToLimit(summary: string, limit?: number): string;
74
97
  export { formatDuration } from '@kici-dev/shared';
75
98
  /**
76
99
  * Build a rich markdown summary for a GitHub Check run output.summary field.
77
100
  *
78
101
  * Includes: headline, step table, failure details with log context, trace footer.
79
- * Respects the 65535 byte limit with progressive log truncation.
102
+ * Respects the 65535 byte limit with progressive log truncation, minus whatever
103
+ * `reservedBytes` the caller will wrap around the result.
80
104
  */
81
105
  export declare function buildCheckRunSummary(opts: BuildCheckRunSummaryOptions): string;
82
106
  /**
@@ -13,6 +13,7 @@
13
13
  */
14
14
  import { type Kysely } from 'kysely';
15
15
  import type { Database } from '../db/types.js';
16
+ import { JobKind } from '../db/types.js';
16
17
  import { ExecutionRunStatus, type InitFailure, RunFailureClass, ScalerEventType, OrchLogPhase } from '@kici-dev/engine';
17
18
  import type { ObserverRegistry } from '../ws/observer-registry.js';
18
19
  import type { LogStorage } from './log-storage.js';
@@ -36,6 +37,12 @@ export interface ExecutionContext {
36
37
  * `execution.status`.
37
38
  */
38
39
  workflowRepoIdentifier?: string;
40
+ /**
41
+ * True when this run records a global evaluation round rather than a
42
+ * workflow. Forwarded to the Platform so its own re-run refusal can admit the
43
+ * round's re-evaluation.
44
+ */
45
+ isGlobalEvalRound?: boolean;
39
46
  /** Git branch or tag (e.g. "main", "feature/foo"). */
40
47
  ref?: string;
41
48
  /** Trigger event type (e.g. "push", "pr:open"). */
@@ -64,6 +71,15 @@ export interface ExecutionContext {
64
71
  * subscriptions can match on it. Null/undefined for success or non-terminal.
65
72
  */
66
73
  failureClass?: RunFailureClass | null;
74
+ /**
75
+ * Resolved trust tier of the run's ref, and which branch's lock file it was
76
+ * evaluated against. Mirror the `execution_runs.trust_tier` /
77
+ * `lock_file_source` columns; both absent for a run whose trust never
78
+ * resolved. Read by the check-run reporter to name the reduced-privilege
79
+ * posture on a job's completion summary.
80
+ */
81
+ trustTier?: string;
82
+ lockFileSource?: string;
67
83
  /** Workflow-level concurrency config from the lock file. */
68
84
  concurrency?: {
69
85
  cancelInProgress?: boolean;
@@ -79,6 +95,7 @@ interface WorkflowCompleteCallbackData {
79
95
  jobResults: Array<{
80
96
  name: string;
81
97
  status: string;
98
+ outputs?: Record<string, unknown>;
82
99
  }>;
83
100
  routingKey?: string;
84
101
  repo: string;
@@ -106,6 +123,24 @@ export interface ExecutionTrackerDeps {
106
123
  observerRegistry?: ObserverRegistry;
107
124
  /** Optional callback when execution reaches terminal state. */
108
125
  onExecutionComplete?: (runId: string, status: Extract<ExecutionRunStatus, 'success' | 'failed' | 'cancelled'>, context: ExecutionContext, description?: string) => void;
126
+ /**
127
+ * Drop the in-process state a run leaves behind — its pending job contexts
128
+ * and its still-open eval gates.
129
+ *
130
+ * Separate from {@link ExecutionTrackerDeps.onExecutionComplete} because that
131
+ * callback is NOT the terminalization chokepoint it reads as. It fires on the
132
+ * three paths that finish a run whose jobs ran (`completeRun`, and both
133
+ * stale-detector arms) and on none of the paths that terminalize a run whose
134
+ * jobs never did — `failRun` (the expired-approval sweep and every
135
+ * pre-dispatch abort) and `cancelHeldRun` (an install gate rejected). Those
136
+ * are exactly the runs most likely to be holding a pending job context, since
137
+ * a context is only stored for a job that was gated rather than dispatched.
138
+ *
139
+ * Synchronous by contract: the in-memory half of the cleanup must not be
140
+ * deferred behind an await the caller does not hold, and the DB half is
141
+ * fire-and-forget inside the callback.
142
+ */
143
+ onRunTerminalCleanup?: (runId: string) => void;
109
144
  /** Optional callback to forward step status to Platform. */
110
145
  onStepStatusForward?: (runId: string, jobId: string, jobName: string, stepIndex: number, stepName: string, state: string, timestamp: number, data?: Record<string, unknown>, requestId?: string) => void;
111
146
  /** Optional callback when a completed run is pruned from memory. */
@@ -202,6 +237,7 @@ export declare class ExecutionTracker {
202
237
  private readonly db;
203
238
  private readonly observerRegistry?;
204
239
  private readonly onExecutionComplete?;
240
+ private readonly onRunTerminalCleanup?;
205
241
  private readonly onStepStatusForward?;
206
242
  private readonly onRunPruned?;
207
243
  private readonly onWorkflowComplete?;
@@ -350,6 +386,34 @@ export declare class ExecutionTracker {
350
386
  * after a peer ACKs a reroute.
351
387
  */
352
388
  markJobReroutedToPeer(runId: string, jobId: string, peerId: string): Promise<void>;
389
+ /**
390
+ * Whether a rerouted job has visibly started, read from the shared
391
+ * `execution_jobs` row rather than from a relayed progress frame.
392
+ *
393
+ * The coordinator's spawn-window backstop learns about progress from
394
+ * `job.progress`, which only a WORKER peer sends — a worker has no database,
395
+ * so relaying is its only channel. A peer COORDINATOR writes the job's status
396
+ * straight into this table instead, so silence on the wire says nothing about
397
+ * whether the job is running, and the backstop would cancel a healthy job.
398
+ *
399
+ * False when no row exists yet (the peer has not started the job) and when the
400
+ * row is still `pending`; the `started_at` half covers a status that has
401
+ * already moved on to a terminal value.
402
+ */
403
+ hasJobStarted(runId: string, jobId: string): Promise<boolean>;
404
+ /**
405
+ * Whether a job has reached a terminal state in the shared `execution_jobs`
406
+ * row.
407
+ *
408
+ * The companion read to {@link hasJobStarted}: a peer COORDINATOR runs a
409
+ * rerouted job against this same table and relays no terminal `job.progress`,
410
+ * so the row is the only signal the routing coordinator gets that the job is
411
+ * over and its reroute tracking can be released.
412
+ *
413
+ * False when no row exists yet — an absent row is a job that has not started,
414
+ * never a finished one.
415
+ */
416
+ isJobTerminal(runId: string, jobId: string): Promise<boolean>;
353
417
  /**
354
418
  * Run `fn` while holding a per-run lock, serializing the run-mutating methods
355
419
  * (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
@@ -377,6 +441,9 @@ export declare class ExecutionTracker {
377
441
  contexts?: string[];
378
442
  skippedContexts?: string[];
379
443
  envWarning?: string;
444
+ jobKind?: JobKind;
445
+ summonedRunId?: string;
446
+ timeoutMs?: number;
380
447
  }>, dispatchedContexts?: string[],
381
448
  /** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
382
449
  replaceSyntheticId?: string): Promise<void>;
@@ -470,6 +537,13 @@ export declare class ExecutionTracker {
470
537
  * - `noop`: nothing — a later terminal will free the next slot.
471
538
  */
472
539
  private runWaveSchedulerHook;
540
+ /**
541
+ * When a proxy job of an invoke gate reaches terminal, aggregate the gate once
542
+ * every sibling proxy is terminal: the gate is set `failed` if any proxy
543
+ * failed, else `success`. Setting the gate terminal drives its own downstream
544
+ * `needs` release via the scheduler hook. A no-op for any non-proxy job.
545
+ */
546
+ private runInvokeGateAggregationHook;
473
547
  /**
474
548
  * Phase 9: stuck-jobs invariant check ( Layer 3).
475
549
  * Before declaring a run complete, verify no stuck jobs exist. If any are
@@ -484,6 +558,51 @@ export declare class ExecutionTracker {
484
558
  * observer callbacks, and schedules in-memory pruning.
485
559
  */
486
560
  private finalizeRunCompletion;
561
+ /**
562
+ * Map a terminal run status to the terminal job status a proxy should carry.
563
+ * A failure class collapses to `failed`; a cancel to `cancelled`; everything
564
+ * else (success) to `success`.
565
+ */
566
+ private mapRunStatusToProxyStatus;
567
+ /**
568
+ * Read a summoned run's non-secret declared outputs — the merged `outputs`
569
+ * JSONB of its jobs. Secret-masked outputs live in `run_secret_outputs` and
570
+ * are deliberately NOT read here, so a repo's secret output never crosses into
571
+ * the summoning global run.
572
+ */
573
+ private readNonSecretRunOutputs;
574
+ /**
575
+ * Normalize an `outputs` JSONB cell to a plain object. The Postgres driver may
576
+ * return it already parsed (object) or as a JSON string; both are accepted. A
577
+ * null / empty / malformed cell yields `null`.
578
+ */
579
+ private parseOutputsCell;
580
+ /**
581
+ * When `spawnedRunId` was summoned by an invoke gate, set the gate's proxy job
582
+ * to the run's mapped terminal status and attach its non-secret outputs. The
583
+ * proxy's terminal transition drives the summoning run's scheduler (release the
584
+ * next held proxy, and aggregate the gate once every proxy is terminal).
585
+ */
586
+ private mirrorSummonedRunOntoProxy;
587
+ /**
588
+ * Set a proxy job's timeline span to the summoned run's own started_at /
589
+ * completed_at so its dashboard bar reflects the run it mirrors. A run rejected
590
+ * before executing any step has no start; fall back to its completion (and, for
591
+ * a run with no completion recorded, to the mirror timestamp) so the proxy
592
+ * still renders a point-width bar rather than a status-only placeholder.
593
+ */
594
+ private mirrorSummonedRunSpanOntoProxy;
595
+ /**
596
+ * Reconcile a just-tagged summoned run against its proxy: if the run is already
597
+ * terminal, drive the mirror now. Closes the tag race — a run that finalizes
598
+ * before `summoned_by_run_id` lands (a synchronous all-jobs-rejected completion,
599
+ * or an HA peer finalizing it first) reads a null tag in `mirrorSummonedRunOntoProxy`
600
+ * and skips, so nothing else terminalizes its proxy and the gate hangs until
601
+ * its timeout. The invoke gate calls this after tagging each spawned run; the
602
+ * `onJobStatus` idempotency guard makes a double-fire with the normal finalize
603
+ * a no-op.
604
+ */
605
+ reconcileSummonedRunIfTerminal(spawnedRunId: string): Promise<void>;
487
606
  /**
488
607
  * Mark a run as failed when its build fails (timeout or error).
489
608
  *
@@ -498,8 +617,25 @@ export declare class ExecutionTracker {
498
617
  *
499
618
  * Inserts a minimal execution_runs row with status='failed' directly so the E2E
500
619
  * test (and dashboard) can observe the failure instead of a missing run.
501
- */
502
- onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure): Promise<void>;
620
+ *
621
+ * The fifth pre-dispatch recording site, and the third that writes the row and
622
+ * returns. Its row is terminal with no resume path, so no invoke gate ever
623
+ * reads its `chain_depth` — the stamp is here for the same reason as on
624
+ * {@link ExecutionTracker.recordInitFailureRun}: this row is the run's ONLY
625
+ * record, and one that says `0` claims to have started the chain it actually
626
+ * died inside.
627
+ */
628
+ onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure,
629
+ /**
630
+ * Internal-trigger provenance, as one object rather than two more
631
+ * positional args on an already 13-wide signature. Shape matches
632
+ * `preDispatchRunProvenance` in `dispatch-matched-workflow.ts`, which is
633
+ * the only caller.
634
+ */
635
+ provenance?: {
636
+ chainDepth?: number;
637
+ dispatchedByFailureLifecycle?: boolean;
638
+ }): Promise<void>;
503
639
  /**
504
640
  * Write a `failed` execution_runs row directly for an init failure, and drop
505
641
  * any in-memory run so the recorded failure is the run's final word. Also
@@ -543,6 +679,29 @@ export declare class ExecutionTracker {
543
679
  * and records nothing.
544
680
  */
545
681
  workflowRepoIdentifier: string;
682
+ /**
683
+ * Inherited invoke-chain depth for a run summoned by an invoke gate.
684
+ *
685
+ * An init failure is terminal, so unlike the hold on
686
+ * {@link ExecutionTracker.recordRunHeld} this run never fires a gate of its
687
+ * own and nothing reads the column back to bound recursion. It is recorded
688
+ * because this row is the run's ONLY record: one that says `0` claims to
689
+ * have started the chain it actually died inside.
690
+ */
691
+ chainDepth?: number;
692
+ /**
693
+ * True when a failure-lifecycle trigger dispatched this run, recorded as
694
+ * the run's `trigger_decision` marker
695
+ * (`EventRouter.isFailureLifecycleRun`).
696
+ *
697
+ * Consistency rather than a live reader: this path emits no
698
+ * `__workflow_complete`, so no accumulator reads the marker back today. It
699
+ * is recorded so the row does not disagree with the one
700
+ * {@link ExecutionTracker.recordRunHeld} writes for the same dispatch, and
701
+ * so adding an emit here later cannot silently re-open the self-retrigger
702
+ * loop.
703
+ */
704
+ dispatchedByFailureLifecycle?: boolean;
546
705
  }): Promise<void>;
547
706
  /**
548
707
  * Write one `failed` execution_runs row for a global eval round that never
@@ -573,6 +732,16 @@ export declare class ExecutionTracker {
573
732
  deliveryId: string | null;
574
733
  providerContext: Record<string, unknown>;
575
734
  routingKey: string;
735
+ /**
736
+ * The source `providerContext` was taken from, when a cross-provider
737
+ * lock-file fallback made it a different source from `routingKey`.
738
+ *
739
+ * Recorded because the two are a PAIR: a re-run of this round re-drives the
740
+ * organization-wide pass and must hand it the same bundle those credentials
741
+ * belong to. Pairing the stored context with the inbound routing key instead
742
+ * gives one source's credentials to another source's API client.
743
+ */
744
+ dispatchRoutingKey?: string;
576
745
  failureReason: string;
577
746
  triggerEvent?: string;
578
747
  /**
@@ -623,6 +792,20 @@ export declare class ExecutionTracker {
623
792
  * whole time it sits in the queue and for the run it resumes into.
624
793
  */
625
794
  workflowRepoIdentifier: string;
795
+ /**
796
+ * Inherited invoke-chain depth for a run summoned by an invoke gate.
797
+ * REQUIRED to be threaded by any caller that has one: a hold is resumable,
798
+ * so the resumed run can fire its own invoke gate, and the chain-depth
799
+ * circuit breaker reads this column back. Absent ⇒ the column's `0`
800
+ * default, which means "this run starts a chain".
801
+ */
802
+ chainDepth?: number;
803
+ /**
804
+ * True when a failure-lifecycle trigger dispatched this run, recorded as
805
+ * the run's `trigger_decision` marker so its eventual completion is
806
+ * excluded from batch accumulation (`EventRouter.isFailureLifecycleRun`).
807
+ */
808
+ dispatchedByFailureLifecycle?: boolean;
626
809
  }): Promise<void>;
627
810
  /**
628
811
  * Flip a `held` run back to `pending` so the resumed dispatch can proceed
@@ -769,6 +952,28 @@ export declare class ExecutionTracker {
769
952
  * Used by commit status reporting to access provider/repo/sha info.
770
953
  */
771
954
  getExecutionContext(runId: string): ExecutionContext | undefined;
955
+ /**
956
+ * Stamp the run's resolved trust tier and lock-file branch onto the in-memory
957
+ * state, after `onExecutionStarted` recorded the row.
958
+ *
959
+ * Separate from the start call because trust resolves on its own path and is
960
+ * written to `execution_runs` by its own update. The dispatch site that
961
+ * writes those two columns calls this with the same values, but the two
962
+ * writes are independent and either can be lost: the DB update is
963
+ * fire-and-forget with its own `.catch()`, and this call is wrapped in a
964
+ * `try`/`catch` there. Both are deliberate — the note is worth less than the
965
+ * run — so the in-memory copy and the row CAN disagree, and neither is
966
+ * authoritative for the other. `recoverRunFromDb` reconciles by reading the
967
+ * row, so the durable value is the one that survives.
968
+ *
969
+ * A run this tracker does not know is a no-op: a missing run means the status
970
+ * updates that would read the fields are not coming either.
971
+ *
972
+ * Keep this synchronous. Its one caller wraps it in `try`/`catch`, which
973
+ * catches a throw but not a rejected promise, so an async body would put the
974
+ * failure back outside the guard.
975
+ */
976
+ setRunTrustContext(runId: string, trustTier: string, lockFileSource?: string): void;
772
977
  /**
773
978
  * Update the heartbeat timestamp for a running job.
774
979
  *
@@ -1,17 +1,3 @@
1
- /**
2
- * Shared run-detail aggregator.
3
- *
4
- * One source of truth for "DB rows → canonical nested run detail (jobs with
5
- * nested steps + needs + outputs)". The dashboard handler and the agent-facing
6
- * structured run-result route both build on the same `buildRunDetailJobs` /
7
- * `mapRunDetailStep` mapping, so the row→jobs shape never diverges between the
8
- * two read surfaces.
9
- *
10
- * `aggregateRunDetail` is the warm-path convenience used by the agent route: it
11
- * runs the run-header + per-run batch queries and returns the canonical detail.
12
- * (Cold-store fallback stays a dashboard-handler concern — the agent read path
13
- * is warm-only in v1.)
14
- */
15
1
  import type { Kysely } from 'kysely';
16
2
  import { ExecutionJobStatus, type InitFailure } from '@kici-dev/engine';
17
3
  import type { Database } from '../db/types.js';
@@ -58,6 +44,8 @@ export interface RunDetailJobRow {
58
44
  base_job_name: string | null;
59
45
  variant_kind: string | null;
60
46
  variant_label: string | null;
47
+ job_kind: string | null;
48
+ summoned_run_id: string | null;
61
49
  started_at: Date | null;
62
50
  completed_at: Date | null;
63
51
  duration_ms: number | null;
@@ -104,6 +92,8 @@ export declare function buildRunDetailJobs(jobs: RunDetailJobRow[], lookups: Run
104
92
  baseJobName: string | null;
105
93
  variantKind: string | null;
106
94
  variantLabel: string | null;
95
+ jobKind: string | null;
96
+ summonedRunId: string | null;
107
97
  startedAt: number | null;
108
98
  completedAt: number | null;
109
99
  durationMs: number | null;
@@ -0,0 +1,43 @@
1
+ /**
2
+ * The order a run's steps are shown in.
3
+ *
4
+ * Every reader of `execution_steps` sorted by `step_index` alone, which puts a
5
+ * pseudo-step wherever its index landed rather than where it ran. That is fine
6
+ * for the cache pseudo-steps, whose indices sit above the real steps and which
7
+ * mostly bracket the work — but a job's image build runs strictly FIRST, and
8
+ * rendering it last reads as though it happened last.
9
+ *
10
+ * The index cannot express this: `step.status` rejects a negative index (an
11
+ * agent sending one to an older orchestrator is disconnected mid-job), and two
12
+ * consumers treat a negative index as absent — the check-run reporter writes
13
+ * `steps[stepIndex]`, and the dashboard gates log fetching on `stepIndex >= 0`.
14
+ * So the ordering is expressed by TYPE, here, once, rather than by each reader
15
+ * inventing its own rule.
16
+ */
17
+ import { type Expression } from 'kysely';
18
+ /**
19
+ * Sort key placing setup pseudo-steps (the ones that ran before the job's real
20
+ * steps) ahead of everything else. Apply BEFORE the `step_index` sort:
21
+ *
22
+ * ```ts
23
+ * qb.orderBy(setupStepsFirst()).orderBy('step_index', 'asc')
24
+ * ```
25
+ *
26
+ * Steps with no type, and every real step, share the same key — so within each
27
+ * group `step_index` remains the only thing that orders them, exactly as before.
28
+ */
29
+ export declare function setupStepsFirst(): Expression<number>;
30
+ /**
31
+ * In-memory equivalent, for a caller that already holds the rows.
32
+ *
33
+ * Kept beside the SQL so the two cannot drift into disagreeing about what
34
+ * "setup" means.
35
+ */
36
+ export declare function compareStepsForDisplay(a: {
37
+ step_type?: string | null;
38
+ step_index: number;
39
+ }, b: {
40
+ step_type?: string | null;
41
+ step_index: number;
42
+ }): number;
43
+ //# sourceMappingURL=step-display-order.d.ts.map