@kici-dev/orchestrator 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__test-helpers__/fake-scaler-state-store.d.ts +24 -0
- package/dist/__test-helpers__/mock-db-query.d.ts +130 -0
- package/dist/__test-helpers__/mock-db.d.ts +23 -0
- package/dist/agent/dispatcher.d.ts +81 -13
- package/dist/agent/host-roster.d.ts +37 -5
- package/dist/agent/registry.d.ts +2 -0
- package/dist/app.d.ts +33 -6
- package/dist/approvals/apply-decision.d.ts +67 -19
- package/dist/approvals/held-run-release-wiring.d.ts +45 -0
- package/dist/approvals/independent-wiring.d.ts +55 -0
- package/dist/approvals/resume-router.d.ts +9 -0
- package/dist/approvals/triggerer-subject.d.ts +37 -0
- package/dist/cache/dep-cache.d.ts +54 -10
- package/dist/cache/global-eval-round-cache.d.ts +7 -0
- package/dist/cli/api-client.d.ts +18 -0
- package/dist/cli/commands/cluster-settings.d.ts +11 -0
- package/dist/cli/commands/diagnose.d.ts +6 -0
- package/dist/cli/commands/held-run.d.ts +58 -0
- package/dist/cli/commands/trust-policy.d.ts +117 -12
- package/dist/cli/kici-admin.d.ts +10 -0
- package/dist/cli.js +2272 -1025
- package/dist/cluster/cluster-identity.d.ts +3 -3
- package/dist/cluster/cluster-settings-reader.d.ts +7 -1
- package/dist/cluster/coordinator.d.ts +44 -0
- package/dist/cluster/index.d.ts +4 -0
- package/dist/cluster/peer-handler.d.ts +19 -0
- package/dist/cluster/peer-registry.d.ts +17 -0
- package/dist/cluster/plan-headroom-store.d.ts +26 -0
- package/dist/cluster/worker-eviction.d.ts +69 -0
- package/dist/cold-store/load-event-log-range.d.ts +10 -3
- package/dist/cold-store/load-secret-audit-log-range.d.ts +4 -3
- package/dist/cold-store/tables/secret-audit-log.d.ts +4 -3
- package/dist/config.d.ts +21 -21
- package/dist/contexts/held-runs.d.ts +123 -16
- package/dist/contexts/protection/aggregate.d.ts +3 -2
- package/dist/contexts/protection/branch-gate.d.ts +15 -0
- package/dist/contexts/protection/concurrency-gate.d.ts +8 -2
- package/dist/contexts/protection/pipeline.d.ts +18 -0
- package/dist/contexts/release-queued-holds.d.ts +35 -0
- package/dist/dashboard/attestation-filters.d.ts +8 -0
- package/dist/dashboard/handler.d.ts +6 -24
- package/dist/db/migrations/116_org_plan_headroom.d.ts +4 -0
- package/dist/db/migrations/117_invoke_gate_proxy_jobs.d.ts +4 -0
- package/dist/db/migrations/118_invoke_gate_scheduling.d.ts +4 -0
- package/dist/db/migrations/119_scaler_ha_ownership.d.ts +4 -0
- package/dist/db/migrations/120_cluster_settings_scaler_reaper.d.ts +4 -0
- package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +19 -0
- package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +25 -0
- package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +24 -0
- package/dist/db/migrations/124_cluster_settings_provision_backoff.d.ts +4 -0
- package/dist/db/migrations/125_org_trust_directory.d.ts +30 -0
- package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +28 -0
- package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +26 -0
- package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +23 -0
- package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +24 -0
- package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +32 -0
- package/dist/db/types.d.ts +368 -10
- package/dist/entry-helpers.d.ts +28 -0
- package/dist/events/event-emitter.d.ts +22 -0
- package/dist/events/event-router.d.ts +87 -1
- package/dist/events/types.d.ts +2 -23
- package/dist/git/credential-broker.d.ts +94 -0
- package/dist/git/inherited-secret.d.ts +24 -0
- package/dist/git/installation-token.d.ts +38 -0
- package/dist/git/job-context.d.ts +13 -0
- package/dist/index.js +324 -84
- package/dist/metrics/agent-metrics-aggregator.d.ts +2 -2
- package/dist/metrics/prometheus.d.ts +111 -3
- package/dist/oidc/oidc-mint-registration.d.ts +7 -3
- package/dist/orchestrator-core.d.ts +79 -23
- package/dist/pipeline/direct-ingress-deps.d.ts +100 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +302 -31
- package/dist/pipeline/global-eval-round.d.ts +14 -0
- package/dist/pipeline/internal-event-pipeline.d.ts +140 -0
- package/dist/pipeline/invoke-gate.d.ts +164 -0
- package/dist/pipeline/needs-scheduler.d.ts +13 -0
- package/dist/pipeline/process-webhook.d.ts +119 -31
- package/dist/pipeline/processor.d.ts +226 -21
- package/dist/pipeline/rerun.d.ts +21 -0
- package/dist/pipeline/resume-workflow.d.ts +40 -7
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +9 -0
- package/dist/pipeline/security-hold-check.d.ts +217 -0
- package/dist/pipeline/undispatched-hold-checks.d.ts +23 -0
- package/dist/policy/dashboard-write-policy-listener.d.ts +62 -0
- package/dist/policy/dashboard-write-policy.d.ts +14 -0
- package/dist/provider-registry.d.ts +10 -2
- package/dist/providers/github/check-status-poster.d.ts +11 -0
- package/dist/providers/github/index.d.ts +0 -2
- package/dist/providers/github/normalizer.d.ts +3 -2
- package/dist/providers/local/normalizer.d.ts +3 -5
- package/dist/providers/universal-git/index.d.ts +11 -2
- package/dist/queue/job-queue.d.ts +43 -1
- package/dist/registration/extractor.d.ts +3 -3
- package/dist/registration/registration-index.d.ts +7 -0
- package/dist/registration/registration-store.d.ts +22 -0
- package/dist/reporting/check-run-reporter.d.ts +94 -1
- package/dist/reporting/check-run-summary.d.ts +25 -1
- package/dist/reporting/execution-tracker.d.ts +207 -2
- package/dist/reporting/run-aggregator.d.ts +4 -14
- package/dist/reporting/step-display-order.d.ts +43 -0
- package/dist/routes/admin-held-runs.d.ts +142 -0
- package/dist/routes/admin-trust-policy.d.ts +34 -6
- package/dist/routes/admin.d.ts +9 -0
- package/dist/scaler/backend-factory.d.ts +55 -0
- package/dist/scaler/bare-metal-backend.d.ts +43 -5
- package/dist/scaler/claim-store.d.ts +119 -0
- package/dist/scaler/config.d.ts +4 -0
- package/dist/scaler/container-backend.d.ts +18 -4
- package/dist/scaler/container-routing.d.ts +23 -0
- package/dist/scaler/container-spawn.d.ts +28 -0
- package/dist/scaler/event-backend.d.ts +124 -0
- package/dist/scaler/event-provision-reaper.d.ts +350 -0
- package/dist/scaler/failure-tracker.d.ts +1 -1
- package/dist/scaler/firecracker-backend.d.ts +33 -6
- package/dist/scaler/index.d.ts +11 -1
- package/dist/scaler/label-matcher.d.ts +4 -3
- package/dist/scaler/manager.d.ts +899 -72
- package/dist/scaler/resolve-container-auth.d.ts +45 -0
- package/dist/scaler/scaler-events.d.ts +11 -0
- package/dist/scaler/scaler-state-store.d.ts +294 -10
- package/dist/scaler/types.d.ts +131 -13
- package/dist/scaler/warm-pool.d.ts +132 -29
- package/dist/security/comment-handler.d.ts +50 -11
- package/dist/security/identity-link.d.ts +60 -0
- package/dist/security/lock-source.d.ts +11 -13
- package/dist/security/reduced-privilege-note.d.ts +59 -0
- package/dist/security/trust-directory-store.d.ts +233 -0
- package/dist/security/trust-policy-gate.d.ts +94 -59
- package/dist/security/trust-policy-store.d.ts +3 -0
- package/dist/security/trust-resolver.d.ts +14 -98
- package/dist/server.d.ts +11 -1
- package/dist/server.js +17671 -7350
- package/dist/stale-detector/gate-deadline-detector.d.ts +52 -0
- package/dist/stale-detector/stale-run-detector.d.ts +61 -3
- package/dist/standalone.js +24238 -13822
- package/dist/worker/in-memory-job-queue.d.ts +1 -1
- package/dist/ws/agent-handler.d.ts +28 -4
- package/dist/ws/dashboard-context-handler.d.ts +28 -4
- package/dist/ws/dashboard-global-workflows-handler.d.ts +6 -0
- package/dist/ws/failure-messages.d.ts +2 -0
- package/dist/ws/git-credential-relay.d.ts +39 -0
- package/dist/ws/oidc-token-relay.d.ts +6 -11
- package/dist/ws/platform-client.d.ts +54 -9
- package/installer-image-digests.json +3 -3
- package/package.json +24 -19
- package/sbom.spdx.json +864 -822
- package/dist/approvals/team-membership-lookup.d.ts +0 -13
- package/dist/providers/github/contributor-resolver.d.ts +0 -30
- package/dist/security/contributor-cache.d.ts +0 -83
|
@@ -30,7 +30,7 @@ import type { ProviderRegistry } from '../provider-registry.js';
|
|
|
30
30
|
import type { StepLogBuffer } from './step-log-buffer.js';
|
|
31
31
|
import { type SourceLocationData } from './check-run-summary.js';
|
|
32
32
|
import type { CheckRunTrackingStore } from './check-run-tracking-store.js';
|
|
33
|
-
import { type TerminalJobStatus } from '@kici-dev/engine';
|
|
33
|
+
import { CheckRunConclusion, type TerminalJobStatus } from '@kici-dev/engine';
|
|
34
34
|
/**
|
|
35
35
|
* Dependencies for the CheckRunReporter.
|
|
36
36
|
*/
|
|
@@ -172,6 +172,15 @@ interface UpdateJobStatusOptions {
|
|
|
172
172
|
runIdForLogs?: string;
|
|
173
173
|
/** Job ID for StepLogBuffer lookup. */
|
|
174
174
|
jobId?: string;
|
|
175
|
+
/**
|
|
176
|
+
* The run's resolved trust tier and lock-file branch. When the tier is
|
|
177
|
+
* anything other than `trusted`, the completion summary leads with the
|
|
178
|
+
* reduced-privilege note so a contributor reading a failed job on a fork pull
|
|
179
|
+
* request can see which parts of the build environment the run did not have.
|
|
180
|
+
* Both absent for a run whose trust never resolved.
|
|
181
|
+
*/
|
|
182
|
+
trustTier?: string;
|
|
183
|
+
lockFileSource?: string;
|
|
175
184
|
}
|
|
176
185
|
/**
|
|
177
186
|
* Options for updateWorkflowStatus.
|
|
@@ -193,6 +202,43 @@ interface UpdateWorkflowStatusOptions {
|
|
|
193
202
|
requestId?: string;
|
|
194
203
|
/** Explicit runId for trace context (falls back to AsyncLocalStorage context). */
|
|
195
204
|
runId?: string;
|
|
205
|
+
/**
|
|
206
|
+
* The run's resolved trust tier and lock-file branch, same contract as
|
|
207
|
+
* {@link UpdateJobStatusOptions}. The roll-up check carries the note too: a
|
|
208
|
+
* contributor who reads only `kici/<workflow>` — the one branch protection
|
|
209
|
+
* usually requires — would otherwise see a run fail with no explanation.
|
|
210
|
+
*/
|
|
211
|
+
trustTier?: string;
|
|
212
|
+
lockFileSource?: string;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Options for completeUndispatchedCheckRuns.
|
|
216
|
+
*/
|
|
217
|
+
export interface CompleteUndispatchedCheckRunsOptions {
|
|
218
|
+
provider: string;
|
|
219
|
+
owner: string;
|
|
220
|
+
repo: string;
|
|
221
|
+
sha: string;
|
|
222
|
+
workflowName: string;
|
|
223
|
+
/** See {@link SetPendingOptions.workflowRepoIdentifier}. */
|
|
224
|
+
workflowRepoIdentifier?: string;
|
|
225
|
+
/**
|
|
226
|
+
* The job names `setPending` / `setPendingAwait` created a check run for.
|
|
227
|
+
* Pass the same list that call used, or the names will not match the check
|
|
228
|
+
* runs on the commit.
|
|
229
|
+
*/
|
|
230
|
+
jobNames: string[];
|
|
231
|
+
installationId?: number;
|
|
232
|
+
/** Routing key for per-app credential lookup (e.g., "github:12345"). */
|
|
233
|
+
routingKey?: string;
|
|
234
|
+
/** Explicit requestId for trace context (falls back to AsyncLocalStorage context). */
|
|
235
|
+
requestId?: string;
|
|
236
|
+
/** Explicit runId for trace context (falls back to AsyncLocalStorage context). */
|
|
237
|
+
runId?: string;
|
|
238
|
+
/** The conclusion to complete each check run with. */
|
|
239
|
+
conclusion: CheckRunConclusion;
|
|
240
|
+
/** The check-run output body. The title stays the standard `KiCI: <label>` form. */
|
|
241
|
+
summary: string;
|
|
196
242
|
}
|
|
197
243
|
/**
|
|
198
244
|
* Options for updateStepProgress.
|
|
@@ -276,7 +322,26 @@ export declare class CheckRunReporter {
|
|
|
276
322
|
* still find every key for a runId at cleanup time.
|
|
277
323
|
*/
|
|
278
324
|
private readonly runIdToKeys;
|
|
325
|
+
/**
|
|
326
|
+
* Per-key serialization of GitHub check-run PATCHes. Two updates for one
|
|
327
|
+
* check run must never be in flight at GitHub simultaneously: the terminal
|
|
328
|
+
* `completed` write and an earlier `in_progress` write race last-write-wins,
|
|
329
|
+
* and if the `in_progress` PATCH lands second it reopens the check run to the
|
|
330
|
+
* permanently-unresolved `{ status: in_progress, conclusion: <terminal> }`
|
|
331
|
+
* state. The `terminalSent` re-check inside `updateCheckRun` cannot help once
|
|
332
|
+
* an `in_progress` PATCH has passed it and is awaiting the network — it is a
|
|
333
|
+
* check-then-await. Chaining every PATCH for a key through this map makes the
|
|
334
|
+
* re-check and the PATCH atomic relative to the completion write.
|
|
335
|
+
*/
|
|
336
|
+
private readonly updateLocks;
|
|
279
337
|
constructor(deps: CheckRunReporterDeps);
|
|
338
|
+
/**
|
|
339
|
+
* Run `fn` after every previously-queued check-run PATCH for `key` has
|
|
340
|
+
* settled, so all updates to one check run execute strictly in order. The
|
|
341
|
+
* chain swallows prior errors (each is surfaced to its own caller) so one
|
|
342
|
+
* failed PATCH does not wedge the key.
|
|
343
|
+
*/
|
|
344
|
+
private runUpdateExclusive;
|
|
280
345
|
/**
|
|
281
346
|
* Update the provider registry used for per-routing-key credential lookup.
|
|
282
347
|
* Called after config reload when the provider registry is rebuilt.
|
|
@@ -320,6 +385,33 @@ export declare class CheckRunReporter {
|
|
|
320
385
|
* Fire-and-forget: errors are logged but don't block the pipeline.
|
|
321
386
|
*/
|
|
322
387
|
updateWorkflowStatus(opts: UpdateWorkflowStatusOptions): void;
|
|
388
|
+
/**
|
|
389
|
+
* Complete the queued check runs of a workflow that never dispatched a job.
|
|
390
|
+
*
|
|
391
|
+
* `setPendingAwait` creates `kici/<workflow>` and one
|
|
392
|
+
* `kici/<workflow>/job/<name>` per static job before the pipeline knows
|
|
393
|
+
* whether the workflow will run. When the workflow then ends without
|
|
394
|
+
* dispatching one — a hold rejected or expired, or any pre-dispatch init
|
|
395
|
+
* failure — none of these names ever reaches a job or run record, so
|
|
396
|
+
* `updateJobStatus` and `updateWorkflowStatus` are never called for them, and
|
|
397
|
+
* `doCleanupStaleCheckRuns` skips them because that sweep only updates check
|
|
398
|
+
* runs whose status is `in_progress`. They stay `queued` on the commit, which
|
|
399
|
+
* on a pull request reads as a check that never finishes and blocks branch
|
|
400
|
+
* protection.
|
|
401
|
+
*
|
|
402
|
+
* Completes only the check runs this reporter can resolve an id for (L1 cache
|
|
403
|
+
* or the `check_run_tracking` row), and skips a key already latched terminal.
|
|
404
|
+
*
|
|
405
|
+
* The build check `kici/<workflow>/setup` is deliberately not in the set,
|
|
406
|
+
* because that latch does not reliably cover it. `setBuildComplete` stamps
|
|
407
|
+
* `terminal_sent_at` on the row but adds nothing to the in-process
|
|
408
|
+
* `terminalSent` set, and `resolveCheckRunId` returns on an L1 hit without
|
|
409
|
+
* reading the row — so a build check this process completed a moment ago
|
|
410
|
+
* resolves with no latch, and completing it again would overwrite a real
|
|
411
|
+
* build conclusion. An unreadable row is the harmless direction: it yields
|
|
412
|
+
* no id and the name is skipped.
|
|
413
|
+
*/
|
|
414
|
+
completeUndispatchedCheckRuns(opts: CompleteUndispatchedCheckRunsOptions): Promise<void>;
|
|
323
415
|
/**
|
|
324
416
|
* Clean up stale check runs left by a dead orchestrator.
|
|
325
417
|
*
|
|
@@ -519,6 +611,7 @@ export declare class CheckRunReporter {
|
|
|
519
611
|
* Supports both 'completed' and 'in_progress' statuses.
|
|
520
612
|
*/
|
|
521
613
|
private updateCheckRun;
|
|
614
|
+
private updateCheckRunLocked;
|
|
522
615
|
}
|
|
523
616
|
/**
|
|
524
617
|
* Build a meaningful failure description from agent job status data.
|
|
@@ -58,6 +58,17 @@ interface BuildCheckRunSummaryOptions {
|
|
|
58
58
|
};
|
|
59
59
|
/** Total job duration in milliseconds. */
|
|
60
60
|
jobDurationMs?: number;
|
|
61
|
+
/**
|
|
62
|
+
* Bytes a caller will add around the returned summary — the reduced-privilege
|
|
63
|
+
* note it prepends, the annotation-count line it appends — subtracted from
|
|
64
|
+
* the byte budget below.
|
|
65
|
+
*
|
|
66
|
+
* Without it the budget is spent in full here and the caller's own bytes push
|
|
67
|
+
* the final string over the API's cap, which rejects the entire update and
|
|
68
|
+
* leaves the check run unresolved. Reserving instead drops one more tier of
|
|
69
|
+
* log lines, which costs context rather than the whole report.
|
|
70
|
+
*/
|
|
71
|
+
reservedBytes?: number;
|
|
61
72
|
}
|
|
62
73
|
interface BuildAnnotationsOptions {
|
|
63
74
|
stepResults: StepResultData[];
|
|
@@ -71,12 +82,25 @@ interface BuildProgressTextOptions {
|
|
|
71
82
|
runId: string;
|
|
72
83
|
};
|
|
73
84
|
}
|
|
85
|
+
/** GitHub Checks API output.summary byte limit. */
|
|
86
|
+
export declare const SUMMARY_BYTE_LIMIT = 65535;
|
|
87
|
+
/**
|
|
88
|
+
* Cut `summary` so it fits the API's cap, whatever a caller wrapped around it.
|
|
89
|
+
*
|
|
90
|
+
* The last line of defence, not the primary mechanism: `reservedBytes` is what
|
|
91
|
+
* keeps a summary readable by dropping log tiers, and this only runs when the
|
|
92
|
+
* final string is still over. Truncating at a UTF-8 boundary matters — a cut
|
|
93
|
+
* through a multi-byte sequence produces a replacement character and can leave
|
|
94
|
+
* the string one byte over the cap it was cut to fit.
|
|
95
|
+
*/
|
|
96
|
+
export declare function clampSummaryToLimit(summary: string, limit?: number): string;
|
|
74
97
|
export { formatDuration } from '@kici-dev/shared';
|
|
75
98
|
/**
|
|
76
99
|
* Build a rich markdown summary for a GitHub Check run output.summary field.
|
|
77
100
|
*
|
|
78
101
|
* Includes: headline, step table, failure details with log context, trace footer.
|
|
79
|
-
* Respects the 65535 byte limit with progressive log truncation
|
|
102
|
+
* Respects the 65535 byte limit with progressive log truncation, minus whatever
|
|
103
|
+
* `reservedBytes` the caller will wrap around the result.
|
|
80
104
|
*/
|
|
81
105
|
export declare function buildCheckRunSummary(opts: BuildCheckRunSummaryOptions): string;
|
|
82
106
|
/**
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { type Kysely } from 'kysely';
|
|
15
15
|
import type { Database } from '../db/types.js';
|
|
16
|
+
import { JobKind } from '../db/types.js';
|
|
16
17
|
import { ExecutionRunStatus, type InitFailure, RunFailureClass, ScalerEventType, OrchLogPhase } from '@kici-dev/engine';
|
|
17
18
|
import type { ObserverRegistry } from '../ws/observer-registry.js';
|
|
18
19
|
import type { LogStorage } from './log-storage.js';
|
|
@@ -36,6 +37,12 @@ export interface ExecutionContext {
|
|
|
36
37
|
* `execution.status`.
|
|
37
38
|
*/
|
|
38
39
|
workflowRepoIdentifier?: string;
|
|
40
|
+
/**
|
|
41
|
+
* True when this run records a global evaluation round rather than a
|
|
42
|
+
* workflow. Forwarded to the Platform so its own re-run refusal can admit the
|
|
43
|
+
* round's re-evaluation.
|
|
44
|
+
*/
|
|
45
|
+
isGlobalEvalRound?: boolean;
|
|
39
46
|
/** Git branch or tag (e.g. "main", "feature/foo"). */
|
|
40
47
|
ref?: string;
|
|
41
48
|
/** Trigger event type (e.g. "push", "pr:open"). */
|
|
@@ -64,6 +71,15 @@ export interface ExecutionContext {
|
|
|
64
71
|
* subscriptions can match on it. Null/undefined for success or non-terminal.
|
|
65
72
|
*/
|
|
66
73
|
failureClass?: RunFailureClass | null;
|
|
74
|
+
/**
|
|
75
|
+
* Resolved trust tier of the run's ref, and which branch's lock file it was
|
|
76
|
+
* evaluated against. Mirror the `execution_runs.trust_tier` /
|
|
77
|
+
* `lock_file_source` columns; both absent for a run whose trust never
|
|
78
|
+
* resolved. Read by the check-run reporter to name the reduced-privilege
|
|
79
|
+
* posture on a job's completion summary.
|
|
80
|
+
*/
|
|
81
|
+
trustTier?: string;
|
|
82
|
+
lockFileSource?: string;
|
|
67
83
|
/** Workflow-level concurrency config from the lock file. */
|
|
68
84
|
concurrency?: {
|
|
69
85
|
cancelInProgress?: boolean;
|
|
@@ -79,6 +95,7 @@ interface WorkflowCompleteCallbackData {
|
|
|
79
95
|
jobResults: Array<{
|
|
80
96
|
name: string;
|
|
81
97
|
status: string;
|
|
98
|
+
outputs?: Record<string, unknown>;
|
|
82
99
|
}>;
|
|
83
100
|
routingKey?: string;
|
|
84
101
|
repo: string;
|
|
@@ -106,6 +123,24 @@ export interface ExecutionTrackerDeps {
|
|
|
106
123
|
observerRegistry?: ObserverRegistry;
|
|
107
124
|
/** Optional callback when execution reaches terminal state. */
|
|
108
125
|
onExecutionComplete?: (runId: string, status: Extract<ExecutionRunStatus, 'success' | 'failed' | 'cancelled'>, context: ExecutionContext, description?: string) => void;
|
|
126
|
+
/**
|
|
127
|
+
* Drop the in-process state a run leaves behind — its pending job contexts
|
|
128
|
+
* and its still-open eval gates.
|
|
129
|
+
*
|
|
130
|
+
* Separate from {@link ExecutionTrackerDeps.onExecutionComplete} because that
|
|
131
|
+
* callback is NOT the terminalization chokepoint it reads as. It fires on the
|
|
132
|
+
* three paths that finish a run whose jobs ran (`completeRun`, and both
|
|
133
|
+
* stale-detector arms) and on none of the paths that terminalize a run whose
|
|
134
|
+
* jobs never did — `failRun` (the expired-approval sweep and every
|
|
135
|
+
* pre-dispatch abort) and `cancelHeldRun` (an install gate rejected). Those
|
|
136
|
+
* are exactly the runs most likely to be holding a pending job context, since
|
|
137
|
+
* a context is only stored for a job that was gated rather than dispatched.
|
|
138
|
+
*
|
|
139
|
+
* Synchronous by contract: the in-memory half of the cleanup must not be
|
|
140
|
+
* deferred behind an await the caller does not hold, and the DB half is
|
|
141
|
+
* fire-and-forget inside the callback.
|
|
142
|
+
*/
|
|
143
|
+
onRunTerminalCleanup?: (runId: string) => void;
|
|
109
144
|
/** Optional callback to forward step status to Platform. */
|
|
110
145
|
onStepStatusForward?: (runId: string, jobId: string, jobName: string, stepIndex: number, stepName: string, state: string, timestamp: number, data?: Record<string, unknown>, requestId?: string) => void;
|
|
111
146
|
/** Optional callback when a completed run is pruned from memory. */
|
|
@@ -202,6 +237,7 @@ export declare class ExecutionTracker {
|
|
|
202
237
|
private readonly db;
|
|
203
238
|
private readonly observerRegistry?;
|
|
204
239
|
private readonly onExecutionComplete?;
|
|
240
|
+
private readonly onRunTerminalCleanup?;
|
|
205
241
|
private readonly onStepStatusForward?;
|
|
206
242
|
private readonly onRunPruned?;
|
|
207
243
|
private readonly onWorkflowComplete?;
|
|
@@ -350,6 +386,34 @@ export declare class ExecutionTracker {
|
|
|
350
386
|
* after a peer ACKs a reroute.
|
|
351
387
|
*/
|
|
352
388
|
markJobReroutedToPeer(runId: string, jobId: string, peerId: string): Promise<void>;
|
|
389
|
+
/**
|
|
390
|
+
* Whether a rerouted job has visibly started, read from the shared
|
|
391
|
+
* `execution_jobs` row rather than from a relayed progress frame.
|
|
392
|
+
*
|
|
393
|
+
* The coordinator's spawn-window backstop learns about progress from
|
|
394
|
+
* `job.progress`, which only a WORKER peer sends — a worker has no database,
|
|
395
|
+
* so relaying is its only channel. A peer COORDINATOR writes the job's status
|
|
396
|
+
* straight into this table instead, so silence on the wire says nothing about
|
|
397
|
+
* whether the job is running, and the backstop would cancel a healthy job.
|
|
398
|
+
*
|
|
399
|
+
* False when no row exists yet (the peer has not started the job) and when the
|
|
400
|
+
* row is still `pending`; the `started_at` half covers a status that has
|
|
401
|
+
* already moved on to a terminal value.
|
|
402
|
+
*/
|
|
403
|
+
hasJobStarted(runId: string, jobId: string): Promise<boolean>;
|
|
404
|
+
/**
|
|
405
|
+
* Whether a job has reached a terminal state in the shared `execution_jobs`
|
|
406
|
+
* row.
|
|
407
|
+
*
|
|
408
|
+
* The companion read to {@link hasJobStarted}: a peer COORDINATOR runs a
|
|
409
|
+
* rerouted job against this same table and relays no terminal `job.progress`,
|
|
410
|
+
* so the row is the only signal the routing coordinator gets that the job is
|
|
411
|
+
* over and its reroute tracking can be released.
|
|
412
|
+
*
|
|
413
|
+
* False when no row exists yet — an absent row is a job that has not started,
|
|
414
|
+
* never a finished one.
|
|
415
|
+
*/
|
|
416
|
+
isJobTerminal(runId: string, jobId: string): Promise<boolean>;
|
|
353
417
|
/**
|
|
354
418
|
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
355
419
|
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
@@ -377,6 +441,9 @@ export declare class ExecutionTracker {
|
|
|
377
441
|
contexts?: string[];
|
|
378
442
|
skippedContexts?: string[];
|
|
379
443
|
envWarning?: string;
|
|
444
|
+
jobKind?: JobKind;
|
|
445
|
+
summonedRunId?: string;
|
|
446
|
+
timeoutMs?: number;
|
|
380
447
|
}>, dispatchedContexts?: string[],
|
|
381
448
|
/** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
|
|
382
449
|
replaceSyntheticId?: string): Promise<void>;
|
|
@@ -470,6 +537,13 @@ export declare class ExecutionTracker {
|
|
|
470
537
|
* - `noop`: nothing — a later terminal will free the next slot.
|
|
471
538
|
*/
|
|
472
539
|
private runWaveSchedulerHook;
|
|
540
|
+
/**
|
|
541
|
+
* When a proxy job of an invoke gate reaches terminal, aggregate the gate once
|
|
542
|
+
* every sibling proxy is terminal: the gate is set `failed` if any proxy
|
|
543
|
+
* failed, else `success`. Setting the gate terminal drives its own downstream
|
|
544
|
+
* `needs` release via the scheduler hook. A no-op for any non-proxy job.
|
|
545
|
+
*/
|
|
546
|
+
private runInvokeGateAggregationHook;
|
|
473
547
|
/**
|
|
474
548
|
* Phase 9: stuck-jobs invariant check ( Layer 3).
|
|
475
549
|
* Before declaring a run complete, verify no stuck jobs exist. If any are
|
|
@@ -484,6 +558,51 @@ export declare class ExecutionTracker {
|
|
|
484
558
|
* observer callbacks, and schedules in-memory pruning.
|
|
485
559
|
*/
|
|
486
560
|
private finalizeRunCompletion;
|
|
561
|
+
/**
|
|
562
|
+
* Map a terminal run status to the terminal job status a proxy should carry.
|
|
563
|
+
* A failure class collapses to `failed`; a cancel to `cancelled`; everything
|
|
564
|
+
* else (success) to `success`.
|
|
565
|
+
*/
|
|
566
|
+
private mapRunStatusToProxyStatus;
|
|
567
|
+
/**
|
|
568
|
+
* Read a summoned run's non-secret declared outputs — the merged `outputs`
|
|
569
|
+
* JSONB of its jobs. Secret-masked outputs live in `run_secret_outputs` and
|
|
570
|
+
* are deliberately NOT read here, so a repo's secret output never crosses into
|
|
571
|
+
* the summoning global run.
|
|
572
|
+
*/
|
|
573
|
+
private readNonSecretRunOutputs;
|
|
574
|
+
/**
|
|
575
|
+
* Normalize an `outputs` JSONB cell to a plain object. The Postgres driver may
|
|
576
|
+
* return it already parsed (object) or as a JSON string; both are accepted. A
|
|
577
|
+
* null / empty / malformed cell yields `null`.
|
|
578
|
+
*/
|
|
579
|
+
private parseOutputsCell;
|
|
580
|
+
/**
|
|
581
|
+
* When `spawnedRunId` was summoned by an invoke gate, set the gate's proxy job
|
|
582
|
+
* to the run's mapped terminal status and attach its non-secret outputs. The
|
|
583
|
+
* proxy's terminal transition drives the summoning run's scheduler (release the
|
|
584
|
+
* next held proxy, and aggregate the gate once every proxy is terminal).
|
|
585
|
+
*/
|
|
586
|
+
private mirrorSummonedRunOntoProxy;
|
|
587
|
+
/**
|
|
588
|
+
* Set a proxy job's timeline span to the summoned run's own started_at /
|
|
589
|
+
* completed_at so its dashboard bar reflects the run it mirrors. A run rejected
|
|
590
|
+
* before executing any step has no start; fall back to its completion (and, for
|
|
591
|
+
* a run with no completion recorded, to the mirror timestamp) so the proxy
|
|
592
|
+
* still renders a point-width bar rather than a status-only placeholder.
|
|
593
|
+
*/
|
|
594
|
+
private mirrorSummonedRunSpanOntoProxy;
|
|
595
|
+
/**
|
|
596
|
+
* Reconcile a just-tagged summoned run against its proxy: if the run is already
|
|
597
|
+
* terminal, drive the mirror now. Closes the tag race — a run that finalizes
|
|
598
|
+
* before `summoned_by_run_id` lands (a synchronous all-jobs-rejected completion,
|
|
599
|
+
* or an HA peer finalizing it first) reads a null tag in `mirrorSummonedRunOntoProxy`
|
|
600
|
+
* and skips, so nothing else terminalizes its proxy and the gate hangs until
|
|
601
|
+
* its timeout. The invoke gate calls this after tagging each spawned run; the
|
|
602
|
+
* `onJobStatus` idempotency guard makes a double-fire with the normal finalize
|
|
603
|
+
* a no-op.
|
|
604
|
+
*/
|
|
605
|
+
reconcileSummonedRunIfTerminal(spawnedRunId: string): Promise<void>;
|
|
487
606
|
/**
|
|
488
607
|
* Mark a run as failed when its build fails (timeout or error).
|
|
489
608
|
*
|
|
@@ -498,8 +617,25 @@ export declare class ExecutionTracker {
|
|
|
498
617
|
*
|
|
499
618
|
* Inserts a minimal execution_runs row with status='failed' directly so the E2E
|
|
500
619
|
* test (and dashboard) can observe the failure instead of a missing run.
|
|
501
|
-
|
|
502
|
-
|
|
620
|
+
*
|
|
621
|
+
* The fifth pre-dispatch recording site, and the third that writes the row and
|
|
622
|
+
* returns. Its row is terminal with no resume path, so no invoke gate ever
|
|
623
|
+
* reads its `chain_depth` — the stamp is here for the same reason as on
|
|
624
|
+
* {@link ExecutionTracker.recordInitFailureRun}: this row is the run's ONLY
|
|
625
|
+
* record, and one that says `0` claims to have started the chain it actually
|
|
626
|
+
* died inside.
|
|
627
|
+
*/
|
|
628
|
+
onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure,
|
|
629
|
+
/**
|
|
630
|
+
* Internal-trigger provenance, as one object rather than two more
|
|
631
|
+
* positional args on an already 13-wide signature. Shape matches
|
|
632
|
+
* `preDispatchRunProvenance` in `dispatch-matched-workflow.ts`, which is
|
|
633
|
+
* the only caller.
|
|
634
|
+
*/
|
|
635
|
+
provenance?: {
|
|
636
|
+
chainDepth?: number;
|
|
637
|
+
dispatchedByFailureLifecycle?: boolean;
|
|
638
|
+
}): Promise<void>;
|
|
503
639
|
/**
|
|
504
640
|
* Write a `failed` execution_runs row directly for an init failure, and drop
|
|
505
641
|
* any in-memory run so the recorded failure is the run's final word. Also
|
|
@@ -543,6 +679,29 @@ export declare class ExecutionTracker {
|
|
|
543
679
|
* and records nothing.
|
|
544
680
|
*/
|
|
545
681
|
workflowRepoIdentifier: string;
|
|
682
|
+
/**
|
|
683
|
+
* Inherited invoke-chain depth for a run summoned by an invoke gate.
|
|
684
|
+
*
|
|
685
|
+
* An init failure is terminal, so unlike the hold on
|
|
686
|
+
* {@link ExecutionTracker.recordRunHeld} this run never fires a gate of its
|
|
687
|
+
* own and nothing reads the column back to bound recursion. It is recorded
|
|
688
|
+
* because this row is the run's ONLY record: one that says `0` claims to
|
|
689
|
+
* have started the chain it actually died inside.
|
|
690
|
+
*/
|
|
691
|
+
chainDepth?: number;
|
|
692
|
+
/**
|
|
693
|
+
* True when a failure-lifecycle trigger dispatched this run, recorded as
|
|
694
|
+
* the run's `trigger_decision` marker
|
|
695
|
+
* (`EventRouter.isFailureLifecycleRun`).
|
|
696
|
+
*
|
|
697
|
+
* Consistency rather than a live reader: this path emits no
|
|
698
|
+
* `__workflow_complete`, so no accumulator reads the marker back today. It
|
|
699
|
+
* is recorded so the row does not disagree with the one
|
|
700
|
+
* {@link ExecutionTracker.recordRunHeld} writes for the same dispatch, and
|
|
701
|
+
* so adding an emit here later cannot silently re-open the self-retrigger
|
|
702
|
+
* loop.
|
|
703
|
+
*/
|
|
704
|
+
dispatchedByFailureLifecycle?: boolean;
|
|
546
705
|
}): Promise<void>;
|
|
547
706
|
/**
|
|
548
707
|
* Write one `failed` execution_runs row for a global eval round that never
|
|
@@ -573,6 +732,16 @@ export declare class ExecutionTracker {
|
|
|
573
732
|
deliveryId: string | null;
|
|
574
733
|
providerContext: Record<string, unknown>;
|
|
575
734
|
routingKey: string;
|
|
735
|
+
/**
|
|
736
|
+
* The source `providerContext` was taken from, when a cross-provider
|
|
737
|
+
* lock-file fallback made it a different source from `routingKey`.
|
|
738
|
+
*
|
|
739
|
+
* Recorded because the two are a PAIR: a re-run of this round re-drives the
|
|
740
|
+
* organization-wide pass and must hand it the same bundle those credentials
|
|
741
|
+
* belong to. Pairing the stored context with the inbound routing key instead
|
|
742
|
+
* gives one source's credentials to another source's API client.
|
|
743
|
+
*/
|
|
744
|
+
dispatchRoutingKey?: string;
|
|
576
745
|
failureReason: string;
|
|
577
746
|
triggerEvent?: string;
|
|
578
747
|
/**
|
|
@@ -623,6 +792,20 @@ export declare class ExecutionTracker {
|
|
|
623
792
|
* whole time it sits in the queue and for the run it resumes into.
|
|
624
793
|
*/
|
|
625
794
|
workflowRepoIdentifier: string;
|
|
795
|
+
/**
|
|
796
|
+
* Inherited invoke-chain depth for a run summoned by an invoke gate.
|
|
797
|
+
* REQUIRED to be threaded by any caller that has one: a hold is resumable,
|
|
798
|
+
* so the resumed run can fire its own invoke gate, and the chain-depth
|
|
799
|
+
* circuit breaker reads this column back. Absent ⇒ the column's `0`
|
|
800
|
+
* default, which means "this run starts a chain".
|
|
801
|
+
*/
|
|
802
|
+
chainDepth?: number;
|
|
803
|
+
/**
|
|
804
|
+
* True when a failure-lifecycle trigger dispatched this run, recorded as
|
|
805
|
+
* the run's `trigger_decision` marker so its eventual completion is
|
|
806
|
+
* excluded from batch accumulation (`EventRouter.isFailureLifecycleRun`).
|
|
807
|
+
*/
|
|
808
|
+
dispatchedByFailureLifecycle?: boolean;
|
|
626
809
|
}): Promise<void>;
|
|
627
810
|
/**
|
|
628
811
|
* Flip a `held` run back to `pending` so the resumed dispatch can proceed
|
|
@@ -769,6 +952,28 @@ export declare class ExecutionTracker {
|
|
|
769
952
|
* Used by commit status reporting to access provider/repo/sha info.
|
|
770
953
|
*/
|
|
771
954
|
getExecutionContext(runId: string): ExecutionContext | undefined;
|
|
955
|
+
/**
|
|
956
|
+
* Stamp the run's resolved trust tier and lock-file branch onto the in-memory
|
|
957
|
+
* state, after `onExecutionStarted` recorded the row.
|
|
958
|
+
*
|
|
959
|
+
* Separate from the start call because trust resolves on its own path and is
|
|
960
|
+
* written to `execution_runs` by its own update. The dispatch site that
|
|
961
|
+
* writes those two columns calls this with the same values, but the two
|
|
962
|
+
* writes are independent and either can be lost: the DB update is
|
|
963
|
+
* fire-and-forget with its own `.catch()`, and this call is wrapped in a
|
|
964
|
+
* `try`/`catch` there. Both are deliberate — the note is worth less than the
|
|
965
|
+
* run — so the in-memory copy and the row CAN disagree, and neither is
|
|
966
|
+
* authoritative for the other. `recoverRunFromDb` reconciles by reading the
|
|
967
|
+
* row, so the durable value is the one that survives.
|
|
968
|
+
*
|
|
969
|
+
* A run this tracker does not know is a no-op: a missing run means the status
|
|
970
|
+
* updates that would read the fields are not coming either.
|
|
971
|
+
*
|
|
972
|
+
* Keep this synchronous. Its one caller wraps it in `try`/`catch`, which
|
|
973
|
+
* catches a throw but not a rejected promise, so an async body would put the
|
|
974
|
+
* failure back outside the guard.
|
|
975
|
+
*/
|
|
976
|
+
setRunTrustContext(runId: string, trustTier: string, lockFileSource?: string): void;
|
|
772
977
|
/**
|
|
773
978
|
* Update the heartbeat timestamp for a running job.
|
|
774
979
|
*
|
|
@@ -1,17 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Shared run-detail aggregator.
|
|
3
|
-
*
|
|
4
|
-
* One source of truth for "DB rows → canonical nested run detail (jobs with
|
|
5
|
-
* nested steps + needs + outputs)". The dashboard handler and the agent-facing
|
|
6
|
-
* structured run-result route both build on the same `buildRunDetailJobs` /
|
|
7
|
-
* `mapRunDetailStep` mapping, so the row→jobs shape never diverges between the
|
|
8
|
-
* two read surfaces.
|
|
9
|
-
*
|
|
10
|
-
* `aggregateRunDetail` is the warm-path convenience used by the agent route: it
|
|
11
|
-
* runs the run-header + per-run batch queries and returns the canonical detail.
|
|
12
|
-
* (Cold-store fallback stays a dashboard-handler concern — the agent read path
|
|
13
|
-
* is warm-only in v1.)
|
|
14
|
-
*/
|
|
15
1
|
import type { Kysely } from 'kysely';
|
|
16
2
|
import { ExecutionJobStatus, type InitFailure } from '@kici-dev/engine';
|
|
17
3
|
import type { Database } from '../db/types.js';
|
|
@@ -58,6 +44,8 @@ export interface RunDetailJobRow {
|
|
|
58
44
|
base_job_name: string | null;
|
|
59
45
|
variant_kind: string | null;
|
|
60
46
|
variant_label: string | null;
|
|
47
|
+
job_kind: string | null;
|
|
48
|
+
summoned_run_id: string | null;
|
|
61
49
|
started_at: Date | null;
|
|
62
50
|
completed_at: Date | null;
|
|
63
51
|
duration_ms: number | null;
|
|
@@ -104,6 +92,8 @@ export declare function buildRunDetailJobs(jobs: RunDetailJobRow[], lookups: Run
|
|
|
104
92
|
baseJobName: string | null;
|
|
105
93
|
variantKind: string | null;
|
|
106
94
|
variantLabel: string | null;
|
|
95
|
+
jobKind: string | null;
|
|
96
|
+
summonedRunId: string | null;
|
|
107
97
|
startedAt: number | null;
|
|
108
98
|
completedAt: number | null;
|
|
109
99
|
durationMs: number | null;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The order a run's steps are shown in.
|
|
3
|
+
*
|
|
4
|
+
* Every reader of `execution_steps` sorted by `step_index` alone, which puts a
|
|
5
|
+
* pseudo-step wherever its index landed rather than where it ran. That is fine
|
|
6
|
+
* for the cache pseudo-steps, whose indices sit above the real steps and which
|
|
7
|
+
* mostly bracket the work — but a job's image build runs strictly FIRST, and
|
|
8
|
+
* rendering it last reads as though it happened last.
|
|
9
|
+
*
|
|
10
|
+
* The index cannot express this: `step.status` rejects a negative index (an
|
|
11
|
+
* agent sending one to an older orchestrator is disconnected mid-job), and two
|
|
12
|
+
* consumers treat a negative index as absent — the check-run reporter writes
|
|
13
|
+
* `steps[stepIndex]`, and the dashboard gates log fetching on `stepIndex >= 0`.
|
|
14
|
+
* So the ordering is expressed by TYPE, here, once, rather than by each reader
|
|
15
|
+
* inventing its own rule.
|
|
16
|
+
*/
|
|
17
|
+
import { type Expression } from 'kysely';
|
|
18
|
+
/**
|
|
19
|
+
* Sort key placing setup pseudo-steps (the ones that ran before the job's real
|
|
20
|
+
* steps) ahead of everything else. Apply BEFORE the `step_index` sort:
|
|
21
|
+
*
|
|
22
|
+
* ```ts
|
|
23
|
+
* qb.orderBy(setupStepsFirst()).orderBy('step_index', 'asc')
|
|
24
|
+
* ```
|
|
25
|
+
*
|
|
26
|
+
* Steps with no type, and every real step, share the same key — so within each
|
|
27
|
+
* group `step_index` remains the only thing that orders them, exactly as before.
|
|
28
|
+
*/
|
|
29
|
+
export declare function setupStepsFirst(): Expression<number>;
|
|
30
|
+
/**
|
|
31
|
+
* In-memory equivalent, for a caller that already holds the rows.
|
|
32
|
+
*
|
|
33
|
+
* Kept beside the SQL so the two cannot drift into disagreeing about what
|
|
34
|
+
* "setup" means.
|
|
35
|
+
*/
|
|
36
|
+
export declare function compareStepsForDisplay(a: {
|
|
37
|
+
step_type?: string | null;
|
|
38
|
+
step_index: number;
|
|
39
|
+
}, b: {
|
|
40
|
+
step_type?: string | null;
|
|
41
|
+
step_index: number;
|
|
42
|
+
}): number;
|
|
43
|
+
//# sourceMappingURL=step-display-order.d.ts.map
|