@kici-dev/orchestrator 0.1.20 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +44 -0
- package/dist/agent/host-roster-reaper.d.ts +2 -1
- package/dist/agent/host-roster.d.ts +54 -1
- package/dist/agent/registry.d.ts +22 -0
- package/dist/app.d.ts +5 -0
- package/dist/approvals/step-approval-bridge.d.ts +5 -0
- package/dist/cli/commands/source-manifest.d.ts +42 -0
- package/dist/cli/loopback-callback.d.ts +24 -0
- package/dist/cli/open-browser.d.ts +12 -0
- package/dist/cli/service/compose.d.ts +13 -0
- package/dist/cli/service/deploy-env.d.ts +31 -0
- package/dist/cli.js +1204 -256
- package/dist/cluster/coordinator.d.ts +5 -3
- package/dist/cluster/index.d.ts +2 -0
- package/dist/cluster/join-token.d.ts +27 -5
- package/dist/cluster/peer-auth-coordinator.d.ts +36 -0
- package/dist/cluster/peer-client.d.ts +34 -14
- package/dist/cluster/peer-credentials.d.ts +14 -2
- package/dist/cluster/peer-handler.d.ts +2 -2
- package/dist/cluster/rerouted-job-guard.d.ts +39 -0
- package/dist/config.d.ts +6 -0
- package/dist/dashboard/needs-edges.d.ts +4 -3
- package/dist/db/migrations/043_rerouted_to_peer.d.ts +4 -0
- package/dist/db/migrations/044_check_mode.d.ts +4 -0
- package/dist/db/migrations/045_host_properties.d.ts +16 -0
- package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +17 -0
- package/dist/db/migrations/047_needs_run_on.d.ts +4 -0
- package/dist/db/migrations/048_host_reboot_pending.d.ts +19 -0
- package/dist/db/migrations/049_held_runs_payload.d.ts +14 -0
- package/dist/db/migrations/050_sources_slug.d.ts +17 -0
- package/dist/db/types.d.ts +60 -5
- package/dist/deployment/deployment-identity.d.ts +9 -0
- package/dist/entry-helpers.d.ts +7 -0
- package/dist/environments/held-runs.d.ts +7 -1
- package/dist/github-app-name-refresher/github-app-name-refresher.d.ts +77 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.js +1187 -33
- package/dist/lockfile-validate.d.ts +24 -0
- package/dist/orchestrator-core.d.ts +13 -1
- package/dist/pipeline/decorating-secret-resolver.d.ts +32 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +127 -3
- package/dist/pipeline/install-secrets-resolver.d.ts +2 -2
- package/dist/pipeline/needs-scheduler.d.ts +22 -13
- package/dist/pipeline/processor.d.ts +2 -2
- package/dist/pipeline/test-pipeline.d.ts +37 -58
- package/dist/providers/github/manifest-form.d.ts +15 -0
- package/dist/providers/github/manifest.d.ts +103 -0
- package/dist/reporting/execution-tracker.d.ts +10 -1
- package/dist/routes/admin-sources.d.ts +18 -0
- package/dist/routes/admin.d.ts +8 -0
- package/dist/secrets/pg-secret-store.d.ts +9 -0
- package/dist/secrets/secret-resolver.d.ts +16 -1
- package/dist/server.js +4373 -2322
- package/dist/sources/source-store.d.ts +4 -0
- package/dist/sources/source-validator.d.ts +2 -0
- package/dist/stale-detector/reboot-deadline-sweep.d.ts +29 -0
- package/dist/stale-detector/stale-run-detector.d.ts +8 -0
- package/dist/standalone.js +3582 -1932
- package/dist/worker/in-memory-job-queue.d.ts +17 -0
- package/dist/worker/peer-outbox.d.ts +36 -0
- package/dist/worker/worker-outbox-relay.d.ts +13 -0
- package/dist/ws/agent-handler.d.ts +11 -6
- package/dist/ws/dashboard-fleet-handler.d.ts +33 -0
- package/dist/ws/dashboard-fleet-write-handler.d.ts +60 -0
- package/dist/ws/fleet-runs-on-all.d.ts +16 -0
- package/dist/ws/inventory-api.d.ts +17 -0
- package/dist/ws/platform-client.d.ts +13 -1
- package/dist/ws/test-relay-handlers.d.ts +4 -2
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +77 -128
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lock-load shape + compatibility validation.
|
|
3
|
+
*
|
|
4
|
+
* A fetched lock must be a valid JSON document whose schemaVersion matches this
|
|
5
|
+
* orchestrator's engine SCHEMA_VERSION exactly, and whose routing matchers are
|
|
6
|
+
* well-formed LabelMatcher objects. A lock compiled by an older engine stored
|
|
7
|
+
* `runsOn` as a plain string array; without this gate those strings would parse
|
|
8
|
+
* past the dispatch path as an empty label set and mis-route jobs to an
|
|
9
|
+
* arbitrary scaler. These checks run at the cache choke point and surface as a
|
|
10
|
+
* `LockFileParseError` (the established corrupt-lock signal).
|
|
11
|
+
*/
|
|
12
|
+
import { type LockFile } from '@kici-dev/engine';
|
|
13
|
+
/** Parse the raw lock JSON and assert the minimal document shape. */
|
|
14
|
+
export declare function parseLockDocument(raw: string, repoIdentifier: string, ref: string): LockFile;
|
|
15
|
+
/** Reject a lock whose schemaVersion does not exactly match this orchestrator. */
|
|
16
|
+
export declare function assertLockFileSchemaCompatible(lockFile: LockFile): void;
|
|
17
|
+
/**
|
|
18
|
+
* Walk every static job's runsOn / excludeLabels / runsOnAll matchers and throw
|
|
19
|
+
* if any element is not a valid LabelMatcher. Dynamic job generators carry no
|
|
20
|
+
* static routing matchers, so only static jobs are checked (mirrors
|
|
21
|
+
* assertLockFileRegexesSafe).
|
|
22
|
+
*/
|
|
23
|
+
export declare function assertLockFileMatchersValid(lockFile: LockFile): void;
|
|
24
|
+
//# sourceMappingURL=lockfile-validate.d.ts.map
|
|
@@ -23,7 +23,7 @@ import { DedupCache } from './webhook/dedup.js';
|
|
|
23
23
|
import { ObserverRegistry } from './ws/observer-registry.js';
|
|
24
24
|
import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
|
|
25
25
|
import { SourceLocationStore } from './app.js';
|
|
26
|
-
import { type LabelMatcher, type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
|
|
26
|
+
import { ExecutionJobStatus, type LabelMatcher, type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
|
|
27
27
|
import { ScalerManager } from './scaler/index.js';
|
|
28
28
|
import type { ScalerConfig } from './scaler/index.js';
|
|
29
29
|
import type { CacheStorage } from './storage/index.js';
|
|
@@ -253,6 +253,17 @@ export declare function buildUpstreamOutputsByBase(baseNames: string[], rows: Ar
|
|
|
253
253
|
variant_label?: string | null;
|
|
254
254
|
status?: string | null;
|
|
255
255
|
}>): Record<string, Record<string, unknown>> | undefined;
|
|
256
|
+
/**
|
|
257
|
+
* Build the downstream `upstreamJobStatuses` map keyed by each upstream job
|
|
258
|
+
* row's name. A single non-fanned upstream is keyed by its base name; a
|
|
259
|
+
* fanned-out upstream contributes one entry per expanded child name (`base
|
|
260
|
+
* (child)`). The agent uses this to stamp `ctx.needs.<job>.status` (single) and
|
|
261
|
+
* the per-child status of group / matrix / host-fanout entries.
|
|
262
|
+
*/
|
|
263
|
+
export declare function buildUpstreamStatusesByBase(rows: Array<{
|
|
264
|
+
job_name: string;
|
|
265
|
+
status?: string | null;
|
|
266
|
+
}>): Record<string, ExecutionJobStatus> | undefined;
|
|
256
267
|
/**
|
|
257
268
|
* Fold a `runsOnAll` upstream's host children into the `byHost` envelope
|
|
258
269
|
* `{ byHost: { '<host>': outputs }, summary: { succeededHosts, failedHosts, outputs } }`.
|
|
@@ -302,6 +313,7 @@ export declare function buildMatrixOutputsEnvelope(baseName: string, children: A
|
|
|
302
313
|
export declare function mergeUpstreamOutputs(db: Kysely<Database>, runId: string, jobName: string, needs: unknown, dispatchSecrets: Record<string, string> | undefined, secretKey: string): Promise<{
|
|
303
314
|
mergedSecrets: Record<string, string> | undefined;
|
|
304
315
|
upstreamJobOutputs: Record<string, Record<string, unknown>> | undefined;
|
|
316
|
+
upstreamJobStatuses: Record<string, ExecutionJobStatus> | undefined;
|
|
305
317
|
}>;
|
|
306
318
|
export declare function bootstrapOrchestrator(config: AppConfig, hooks: OrchestratorHooks, options?: {
|
|
307
319
|
otelSdk?: {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI-secret overlay for `kici run` test dispatch.
|
|
3
|
+
*
|
|
4
|
+
* Wraps the orchestrator's environment `SecretResolver` and overlays the
|
|
5
|
+
* developer's CLI-uploaded local secrets on top of the env-resolved secrets,
|
|
6
|
+
* with CLI winning on collision. Passed to the shared dispatch core via
|
|
7
|
+
* `ProcessingDeps.secretResolver`, so the core's secret-resolution path is
|
|
8
|
+
* unchanged and oblivious to "test secrets".
|
|
9
|
+
*
|
|
10
|
+
* Precedence: env-resolved secrets → CLI context for the requested environment
|
|
11
|
+
* → CLI flat. The CLI flat overlay is applied last so a CLI flat key always
|
|
12
|
+
* wins.
|
|
13
|
+
*/
|
|
14
|
+
import type { SecretResolverApi, ResolvedSecretMeta } from '../secrets/secret-resolver.js';
|
|
15
|
+
/** Decrypted CLI-uploaded local secrets: flat keys + per-context namespaces. */
|
|
16
|
+
export interface CliSecrets {
|
|
17
|
+
flat: Record<string, string>;
|
|
18
|
+
contexts: Record<string, Record<string, string>>;
|
|
19
|
+
}
|
|
20
|
+
export declare class DecoratingSecretResolver implements SecretResolverApi {
|
|
21
|
+
private readonly base;
|
|
22
|
+
private readonly cli;
|
|
23
|
+
constructor(base: SecretResolverApi, cli: CliSecrets);
|
|
24
|
+
resolveForJob(orgId: string, environmentName: string): Promise<Record<string, string>>;
|
|
25
|
+
resolveNamed(orgId: string, scope: string, key: string, opts?: {
|
|
26
|
+
store?: string;
|
|
27
|
+
runId?: string;
|
|
28
|
+
jobId?: string;
|
|
29
|
+
}): Promise<string | null>;
|
|
30
|
+
resolveForJobWithMeta(orgId: string, environmentName: string): Promise<Record<string, ResolvedSecretMeta>>;
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=decorating-secret-resolver.d.ts.map
|
|
@@ -13,8 +13,9 @@
|
|
|
13
13
|
* exported function is a narrative orchestrator that threads the typed
|
|
14
14
|
* results through the pipeline.
|
|
15
15
|
*/
|
|
16
|
-
import { CacheRefScope } from '@kici-dev/engine';
|
|
17
|
-
import type { LabelMatcher, LockWorkflow, SimulatedEvent, WorkflowDecision, MaterializedJob } from '@kici-dev/engine';
|
|
16
|
+
import { ExecutionJobStatus, InitFailureCategory, CacheRefScope } from '@kici-dev/engine';
|
|
17
|
+
import type { LabelMatcher, LockWorkflow, LockJob, HostTargetSelector, SimulatedEvent, WorkflowDecision, MaterializedJob, ResolvedHostAgent } from '@kici-dev/engine';
|
|
18
|
+
import { type HostRosterStore } from '../agent/host-roster.js';
|
|
18
19
|
import type { WebhookInfo } from '../webhook/handler.js';
|
|
19
20
|
import type { ProviderBundle } from '../provider-registry.js';
|
|
20
21
|
import type { TrustResolution } from '../security/trust-resolver.js';
|
|
@@ -40,7 +41,15 @@ export declare function deriveCacheRefScope(trust: TrustResolution | undefined):
|
|
|
40
41
|
export interface WorkflowDispatchContext {
|
|
41
42
|
info: WebhookInfo;
|
|
42
43
|
deps: ProcessingDeps;
|
|
43
|
-
|
|
44
|
+
/**
|
|
45
|
+
* Provider bundle for the matched source. Undefined for local-repo test runs
|
|
46
|
+
* (`kici run` against an inline lock file with no remote provider): in that
|
|
47
|
+
* mode there is no clone-url builder / check-status poster / clone-token
|
|
48
|
+
* provider, and `repoUrl` falls back to `''` (the agent treats a missing url
|
|
49
|
+
* as a local/`fullRepo` clone). The webhook adapter always passes a defined
|
|
50
|
+
* bundle, so its dispatch behavior is unchanged.
|
|
51
|
+
*/
|
|
52
|
+
bundle?: ProviderBundle;
|
|
44
53
|
payload: unknown;
|
|
45
54
|
repoIdentifier: string;
|
|
46
55
|
credentials: Record<string, unknown>;
|
|
@@ -87,10 +96,36 @@ export interface WorkflowDispatchContext {
|
|
|
87
96
|
* correct clone + logging.
|
|
88
97
|
*/
|
|
89
98
|
extraJobConfig?: Record<string, unknown>;
|
|
99
|
+
/**
|
|
100
|
+
* Test-run provenance. Present only for `kici run` / test-trigger dispatches.
|
|
101
|
+
* When set, `recordRunStart` stamps `is_test_run = true` and
|
|
102
|
+
* `fixture_id = testRun.fixtureId` on the `execution_runs` row. Undefined for
|
|
103
|
+
* webhook runs (the stamp block is skipped).
|
|
104
|
+
*/
|
|
105
|
+
testRun?: {
|
|
106
|
+
fixtureId: string;
|
|
107
|
+
};
|
|
108
|
+
/**
|
|
109
|
+
* Run-wide flat secrets layered onto EVERY dispatched job's `jobConfig.secrets`
|
|
110
|
+
* (env-declaring or not). Used by the test path to deliver `kici run --secret`
|
|
111
|
+
* / `--env` CLI flat secrets, which must reach a job regardless of whether it
|
|
112
|
+
* declares an `environment:`. Merged UNDER the per-job env-resolved secrets so
|
|
113
|
+
* the CLI value wins on a key collision (matching the prior B1-env -> A-CLI
|
|
114
|
+
* precedence). Undefined for webhook runs.
|
|
115
|
+
*/
|
|
116
|
+
runWideFlatSecrets?: Record<string, string>;
|
|
117
|
+
/**
|
|
118
|
+
* Runtime host narrowing from `kici run --target` (Ansible `--limit`). Applied
|
|
119
|
+
* as a post-filter over each runsOnAll job's matched roster: effective hosts =
|
|
120
|
+
* runsOnAll ∩ target. Narrow-only. Undefined for webhook runs (no narrowing).
|
|
121
|
+
*/
|
|
122
|
+
target?: HostTargetSelector;
|
|
90
123
|
}
|
|
91
124
|
export interface DispatchMatchedWorkflowResult {
|
|
92
125
|
/** Number of jobs successfully dispatched (non-rejected). */
|
|
93
126
|
dispatchedJobCount: number;
|
|
127
|
+
/** Execution job ids of every dispatched/tracked job (root, gated, synthetic). */
|
|
128
|
+
dispatchedJobIds: string[];
|
|
94
129
|
/** True when the workflow install gate paused the dispatch (held run). */
|
|
95
130
|
held?: boolean;
|
|
96
131
|
}
|
|
@@ -109,6 +144,39 @@ export interface DispatchMatchedWorkflowOptions {
|
|
|
109
144
|
*/
|
|
110
145
|
reuseRunId?: string;
|
|
111
146
|
}
|
|
147
|
+
interface RejectedJob {
|
|
148
|
+
jobId: string;
|
|
149
|
+
jobName: string;
|
|
150
|
+
reason: string;
|
|
151
|
+
/** Explicit init-failure category override; inferred from reason when absent. */
|
|
152
|
+
category?: InitFailureCategory;
|
|
153
|
+
/**
|
|
154
|
+
* Terminal status to record for this job. Defaults to `failed`. A zeroed
|
|
155
|
+
* `runsOnAll` that intentionally narrowed to no hosts is recorded as `skipped`
|
|
156
|
+
* (no init-failure) so its downstreams' `when` sets govern propagation.
|
|
157
|
+
*/
|
|
158
|
+
terminalStatus?: ExecutionJobStatus;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Phase B orchestrator: probe caches, dispatch the build job (if needed),
|
|
162
|
+
* and surface enough state for downstream phases to skip / continue
|
|
163
|
+
* appropriately.
|
|
164
|
+
*/
|
|
165
|
+
/**
|
|
166
|
+
* Materialize each static job's matrix into dispatchable children. A job whose
|
|
167
|
+
* matrix is invalid (zero combinations / over the cap) is dropped from the
|
|
168
|
+
* dispatch list and recorded as a `matrix_expansion` matrix failure so the run's
|
|
169
|
+
* other jobs still proceed. Dynamic-matrix jobs pass through with a
|
|
170
|
+
* `pendingDynamicMatrix` marker for the eval flow.
|
|
171
|
+
*/
|
|
172
|
+
/**
|
|
173
|
+
* Resolve a `runsOnAll` lock job against the declared host roster and partition
|
|
174
|
+
* the matched hosts into the target set per the `onUnreachable` policy (R2):
|
|
175
|
+
* `ready` hosts always run; unreachable durable (`static`) hosts hold / fail /
|
|
176
|
+
* skip; stale ephemeral hosts are always skipped. Throws {@link FanoutError}
|
|
177
|
+
* when the run can't proceed (fail policy with an absent host, or zero targets).
|
|
178
|
+
*/
|
|
179
|
+
export declare function resolveHostFanoutTargets(lockJob: LockJob, deps: ProcessingDeps, target?: HostTargetSelector): Promise<ResolvedHostAgent[]>;
|
|
112
180
|
/** Exact labels + regex patterns partitioned from a lock job's selectors. */
|
|
113
181
|
interface JobRoutingSelectors {
|
|
114
182
|
runsOnLabels: string[];
|
|
@@ -126,6 +194,25 @@ export declare function runsOnSelectorsForLockJob(lockJob: {
|
|
|
126
194
|
runsOn?: readonly LabelMatcher[];
|
|
127
195
|
excludeLabels?: readonly LabelMatcher[];
|
|
128
196
|
}): JobRoutingSelectors;
|
|
197
|
+
/**
|
|
198
|
+
* Resolve a generated job's single bare-`agentId` `runsOn` into a host pin.
|
|
199
|
+
*
|
|
200
|
+
* The documented inventory fan-out pattern is `runsOn: [h.agentId]`. A bare
|
|
201
|
+
* agentId is not a label any agent advertises, so the normal label path leaves
|
|
202
|
+
* the job `queued-no-backend`. When the single exact label names a known roster
|
|
203
|
+
* host, resolve it to a `pinnedAgentId` dispatch (+ the host's coordinator for
|
|
204
|
+
* cross-cluster reroute) — exact parity with how `runsOnAll` children pin. Any
|
|
205
|
+
* other shape (multi-label, a regex pattern, a non-roster label, or no roster
|
|
206
|
+
* store) returns null and the caller keeps normal label routing.
|
|
207
|
+
*/
|
|
208
|
+
export declare function resolveRosterAgentPin(args: {
|
|
209
|
+
runsOnLabels: string[];
|
|
210
|
+
runsOnPatterns: LabelMatcher[];
|
|
211
|
+
hostRosterStore: HostRosterStore | undefined;
|
|
212
|
+
}): Promise<{
|
|
213
|
+
pinnedAgentId: string;
|
|
214
|
+
connectedInstanceId: string | null;
|
|
215
|
+
} | null>;
|
|
129
216
|
/** Per-child rolling-wave plan: which children are held + the base's wave policy. */
|
|
130
217
|
export interface WavePlan {
|
|
131
218
|
/** `expandedName`s held behind the wave gate (beyond the maxParallel window). */
|
|
@@ -149,6 +236,43 @@ export interface WavePlan {
|
|
|
149
236
|
* job (single child) or one without `maxParallel` contributes nothing.
|
|
150
237
|
*/
|
|
151
238
|
export declare function computeWavePlan(materializedJobs: readonly MaterializedJob[]): WavePlan;
|
|
239
|
+
export declare function materializeStaticJobsSafe(staticJobs: readonly LockJob[], deps: ProcessingDeps, target?: HostTargetSelector): Promise<{
|
|
240
|
+
materializedJobs: MaterializedJob[];
|
|
241
|
+
expansionMap: Map<string, readonly string[]>;
|
|
242
|
+
matrixFailures: RejectedJob[];
|
|
243
|
+
}>;
|
|
244
|
+
export interface GeneratedJobConfig {
|
|
245
|
+
/**
|
|
246
|
+
* The generated lock job with its `name` and `needs` rewritten to expanded
|
|
247
|
+
* matrix-child names (identical to the base job for non-matrix generated
|
|
248
|
+
* jobs). Downstream dispatch / needs-edge / tracking code keys on this name.
|
|
249
|
+
*/
|
|
250
|
+
genJob: LockJob;
|
|
251
|
+
genJobConfig: Record<string, unknown>;
|
|
252
|
+
runsOnLabels: string[];
|
|
253
|
+
/** Regex matchers the agent's labels must satisfy (JS post-filter). */
|
|
254
|
+
runsOnPatterns: LabelMatcher[];
|
|
255
|
+
/** Exact labels the dispatched agent must NOT have. */
|
|
256
|
+
excludeLabels: string[];
|
|
257
|
+
/** Regex matchers that disqualify an agent (JS post-filter). */
|
|
258
|
+
excludePatterns: LabelMatcher[];
|
|
259
|
+
/** Host-fanout pin: when runsOn resolved to a roster host, route only to it. */
|
|
260
|
+
pinnedAgentId?: string;
|
|
261
|
+
/** The host's current coordinator (cross-cluster reroute hint); null = not connected. */
|
|
262
|
+
connectedInstanceId?: string | null;
|
|
263
|
+
/** The matrix combination for this child; absent for non-matrix generated jobs. */
|
|
264
|
+
matrixValues?: Record<string, unknown>;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Split generated configs into pinned (host-pin dispatch) and unpinned (normal
|
|
268
|
+
* label routing). A pinned config always rides the dispatcher pin path because
|
|
269
|
+
* the coordinator's `JobToRoute` shape carries no pin field — routing it via the
|
|
270
|
+
* coordinator would silently drop the pin.
|
|
271
|
+
*/
|
|
272
|
+
export declare function partitionGeneratedConfigsByPin(configs: readonly GeneratedJobConfig[]): {
|
|
273
|
+
pinnedConfigs: GeneratedJobConfig[];
|
|
274
|
+
unpinnedConfigs: GeneratedJobConfig[];
|
|
275
|
+
};
|
|
152
276
|
/**
|
|
153
277
|
* Dispatch a single matched workflow.
|
|
154
278
|
*
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
*/
|
|
32
32
|
import type { ApproverClause, LockRegistry } from '@kici-dev/engine';
|
|
33
33
|
import type { TrustResolution } from '../security/trust-resolver.js';
|
|
34
|
-
import type {
|
|
34
|
+
import type { SecretResolverApi } from '../secrets/secret-resolver.js';
|
|
35
35
|
import type { EnvironmentStore } from '../environments/environment-store.js';
|
|
36
36
|
import { type JobDispatchContext } from '../environments/protection/pipeline.js';
|
|
37
37
|
/** Registry spec carried on the dispatch message (token already resolved). */
|
|
@@ -48,7 +48,7 @@ export interface ResolveInstallSecretsArgs {
|
|
|
48
48
|
resolvedOrgId: string;
|
|
49
49
|
trustResolution: TrustResolution | undefined;
|
|
50
50
|
environmentStore: EnvironmentStore | undefined;
|
|
51
|
-
secretResolver:
|
|
51
|
+
secretResolver: SecretResolverApi | undefined;
|
|
52
52
|
protectionContext: JobDispatchContext;
|
|
53
53
|
/**
|
|
54
54
|
* Resume path: skip the protection-rule gate (already satisfied) and resolve
|
|
@@ -1,24 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* DB-backed needs-aware dispatch scheduler
|
|
2
|
+
* DB-backed needs-aware dispatch scheduler.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* needs edges (static-to-static, static-to-dyn-group, dyn-to-static, dyn-to-dyn).
|
|
4
|
+
* Gates ALL needs edges (static-to-static, static-to-dyn-group, dyn-to-static,
|
|
5
|
+
* dyn-to-dyn) with event-driven scheduling instead of concurrent dispatch.
|
|
7
6
|
*
|
|
8
7
|
* The scheduler is pure DB — no in-memory state. Every scheduling decision is
|
|
9
8
|
* a fresh DB query against execution_jobs + execution_job_needs. This means
|
|
10
9
|
* zero recovery code on orchestrator restart.
|
|
11
10
|
*
|
|
11
|
+
* Each edge carries a `run_on` status-set (the upstream terminal statuses that
|
|
12
|
+
* satisfy the edge). A downstream edge is dispatch-satisfied when the upstream's
|
|
13
|
+
* terminal status is a member of the edge's run_on set; otherwise the downstream
|
|
14
|
+
* is skipped. A downstream dispatches only when every edge is satisfied.
|
|
15
|
+
*
|
|
12
16
|
* Entry points:
|
|
13
17
|
* - insertEdgesForRun: called at run start for static-to-static edges
|
|
14
18
|
* - resolveGroupEdges: called on dynamic-eval completion
|
|
15
19
|
* - evaluateDownstreams: called from onJobStatus(terminal)
|
|
16
20
|
* - recomputeNeedsSatisfied: batch recompute after group resolution
|
|
17
|
-
* - checkSchedulerInvariant:
|
|
21
|
+
* - checkSchedulerInvariant: defensive stuck-job check
|
|
18
22
|
* - getFailurePropagationTargets: cascade for transitive skip
|
|
19
23
|
*/
|
|
20
24
|
import type { Kysely } from 'kysely';
|
|
21
25
|
import type { Database } from '../db/types.js';
|
|
26
|
+
import { ExecutionJobStatus } from '@kici-dev/engine';
|
|
22
27
|
import type { MaterializedJob } from '@kici-dev/engine';
|
|
23
28
|
/** Result of evaluating downstream jobs after an upstream completes. */
|
|
24
29
|
export interface SchedulerResult {
|
|
@@ -48,21 +53,22 @@ export declare function insertEdgesForRun(db: Kysely<Database>, runId: string, j
|
|
|
48
53
|
* edge row. Empty groups (0 members) trigger immediate needs_satisfied=true
|
|
49
54
|
* for dependents.
|
|
50
55
|
*
|
|
51
|
-
* CRITICAL: dependentStaticJobs carries per-job
|
|
56
|
+
* CRITICAL: dependentStaticJobs carries the per-job run_on status-set from the
|
|
52
57
|
* NeedsGroupEntry in the lock file. Without this, all group edges would
|
|
53
|
-
* silently default to
|
|
58
|
+
* silently default to success-only.
|
|
54
59
|
*/
|
|
55
60
|
export declare function resolveGroupEdges(db: Kysely<Database>, runId: string, groupName: string, memberJobNames: string[], dependentStaticJobs: Array<{
|
|
56
61
|
jobName: string;
|
|
57
|
-
|
|
62
|
+
runOn: ExecutionJobStatus[];
|
|
58
63
|
}>): Promise<void>;
|
|
59
64
|
/**
|
|
60
65
|
* Evaluate downstream jobs after an upstream reaches terminal state.
|
|
61
66
|
*
|
|
62
67
|
* This is the core scheduler hook. For each downstream of the completed job:
|
|
63
|
-
* 1. If
|
|
64
|
-
*
|
|
65
|
-
*
|
|
68
|
+
* 1. If the completed status is not in this edge's run_on set, mark the
|
|
69
|
+
* downstream as 'skip' immediately.
|
|
70
|
+
* 2. Otherwise, check if ALL upstreams are terminal and satisfied.
|
|
71
|
+
* 3. If all satisfied, mark needs_satisfied=true and return for dispatch.
|
|
66
72
|
*/
|
|
67
73
|
export declare function evaluateDownstreams(db: Kysely<Database>, runId: string, completedJobName: string, completedStatus: string): Promise<SchedulerResult[]>;
|
|
68
74
|
/**
|
|
@@ -80,8 +86,11 @@ export declare function recomputeNeedsSatisfied(db: Kysely<Database>, runId: str
|
|
|
80
86
|
*/
|
|
81
87
|
export declare function checkSchedulerInvariant(db: Kysely<Database>, runId: string): Promise<string[]>;
|
|
82
88
|
/**
|
|
83
|
-
* cascade: find all transitive downstreams that should be
|
|
84
|
-
*
|
|
89
|
+
* Failure-propagation cascade: find all transitive downstreams that should be
|
|
90
|
+
* skipped because a terminal upstream's status is not in their edge's run_on
|
|
91
|
+
* set. At each hop, the propagating job's actual terminal status decides which
|
|
92
|
+
* downstream edges propagate (status not in run_on → the downstream skips and
|
|
93
|
+
* propagates further).
|
|
85
94
|
*/
|
|
86
95
|
export declare function getFailurePropagationTargets(db: Kysely<Database>, runId: string, failedJobName: string): Promise<string[]>;
|
|
87
96
|
//# sourceMappingURL=needs-scheduler.d.ts.map
|
|
@@ -33,7 +33,7 @@ import type { HostRosterStore } from '../agent/host-roster.js';
|
|
|
33
33
|
import type { RunCoordinator } from '../cluster/coordinator.js';
|
|
34
34
|
import type { TeamMembershipLookup } from '../approvals/team-membership-lookup.js';
|
|
35
35
|
import type { LogStorage } from '../reporting/log-storage.js';
|
|
36
|
-
import type {
|
|
36
|
+
import type { SecretResolverApi } from '../secrets/secret-resolver.js';
|
|
37
37
|
import type { ContributorCache } from '../security/contributor-cache.js';
|
|
38
38
|
import type { AccessLogWriter } from '../audit/access-log.js';
|
|
39
39
|
import type { LockFile as FullLockFile, LockWorkflow, SimulatedEvent, WebhookNormalizer } from '@kici-dev/engine';
|
|
@@ -281,7 +281,7 @@ export interface ProcessingDeps {
|
|
|
281
281
|
/** Run coordinator for multi-orchestrator job routing. Optional -- if not set, all jobs dispatch locally (single-orchestrator mode). */
|
|
282
282
|
coordinator?: RunCoordinator;
|
|
283
283
|
/** Secret resolver for dispatch-time secret resolution. Optional -- if not set, secrets are not resolved. */
|
|
284
|
-
secretResolver?:
|
|
284
|
+
secretResolver?: SecretResolverApi;
|
|
285
285
|
/** Optional callback when source locations are extracted from a lock file workflow. */
|
|
286
286
|
onSourceLocationsExtracted?: (workflowName: string, jobName: string, sourceLocations: Array<{
|
|
287
287
|
file: string;
|
|
@@ -1,35 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Test
|
|
2
|
+
* Test-trigger adapter for CLI-initiated runs (`kici run`).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Resolves the lock file, matched decisions, and fixture-specific concerns
|
|
5
|
+
* (inline-vs-provider lock, decision selection, fixture-payload storage, the
|
|
6
|
+
* `allow_local_execution` environment gate, CLI-secret overlay, in-memory
|
|
7
|
+
* test-run marking), then dispatches each matched workflow through the SAME
|
|
8
|
+
* shared core as webhooks (`dispatchMatchedWorkflow`). The test path is a thin
|
|
9
|
+
* adapter — needs-DAG scheduling, `expansionMap` fan-out edges, `runsOnAll`
|
|
10
|
+
* host fan-out, and deferred init/dynamic dispatch all come from the core.
|
|
8
11
|
*
|
|
9
|
-
*
|
|
10
|
-
* -
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
* - deliveryId
|
|
14
|
-
* -
|
|
15
|
-
* -
|
|
12
|
+
* Differences from a webhook run:
|
|
13
|
+
* - the synthetic event is injected directly (no provider normalization/dedup);
|
|
14
|
+
* - the lock file may be inline (local repos have no remote provider, so
|
|
15
|
+
* `bundle` is undefined);
|
|
16
|
+
* - `deliveryId` carries a `test:` prefix;
|
|
17
|
+
* - the run is stamped `is_test_run = true` + `fixture_id` (via the core's
|
|
18
|
+
* `testRun` meta) and marked in-memory for live-log broadcast to the CLI;
|
|
19
|
+
* - CLI-uploaded local secrets win over orchestrator env secrets (the
|
|
20
|
+
* decorating secret resolver), and the fixture's `secrets` context mapping
|
|
21
|
+
* resolves into namespaced secrets;
|
|
22
|
+
* - direct workflow execution (bypass trigger matching) is supported.
|
|
16
23
|
*/
|
|
17
|
-
import type
|
|
18
|
-
import type {
|
|
19
|
-
import type { CheckRunReporter } from '../reporting/check-run-reporter.js';
|
|
20
|
-
import type { ExecutionTracker } from '../reporting/execution-tracker.js';
|
|
21
|
-
import type { AgentRegistry } from '../agent/registry.js';
|
|
22
|
-
import type { ProviderRegistry } from '../provider-registry.js';
|
|
23
|
-
import type { SourceCache } from '../cache/index.js';
|
|
24
|
-
import type { BuildCoordinator } from '../cache/index.js';
|
|
25
|
-
import type { DepCache } from '../cache/index.js';
|
|
26
|
-
import type { PendingBuildTracker } from '../cache/index.js';
|
|
27
|
-
import type { SecretResolver } from '../secrets/secret-resolver.js';
|
|
28
|
-
import type { EnvironmentStore } from '../environments/environment-store.js';
|
|
29
|
-
import type { VariableStore } from '../environments/variable-store.js';
|
|
30
|
-
import type { LogStorage } from '../reporting/log-storage.js';
|
|
31
|
-
import type { Kysely } from 'kysely';
|
|
32
|
-
import type { Database } from '../db/types.js';
|
|
24
|
+
import { type ProcessingDeps } from './processor.js';
|
|
25
|
+
import type { CheckMode, HostTargetSelector } from '@kici-dev/engine';
|
|
33
26
|
/**
|
|
34
27
|
* Input for a test trigger request.
|
|
35
28
|
*/
|
|
@@ -69,6 +62,17 @@ export interface TestTriggerInput {
|
|
|
69
62
|
inlineLockFile?: string;
|
|
70
63
|
/** When true, repo has no remote -- skip provider lookup, skip clone. */
|
|
71
64
|
fullRepo?: boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Run mode for idempotent steps (`apply` | `check` | `check-fail-on-drift`).
|
|
67
|
+
* Threaded onto each dispatched job's config and persisted on the run.
|
|
68
|
+
* Omitted means `apply`.
|
|
69
|
+
*/
|
|
70
|
+
checkMode?: CheckMode;
|
|
71
|
+
/**
|
|
72
|
+
* Runtime host narrowing from `kici run --target`. Threaded onto the dispatch
|
|
73
|
+
* context, where it post-filters each runsOnAll job's matched roster.
|
|
74
|
+
*/
|
|
75
|
+
target?: HostTargetSelector;
|
|
72
76
|
}
|
|
73
77
|
/**
|
|
74
78
|
* Result of processing a test trigger.
|
|
@@ -84,37 +88,12 @@ interface TestTriggerResult {
|
|
|
84
88
|
jobIds: string[];
|
|
85
89
|
}
|
|
86
90
|
/**
|
|
87
|
-
*
|
|
88
|
-
* All injected for testability.
|
|
89
|
-
*/
|
|
90
|
-
export interface TestPipelineDeps {
|
|
91
|
-
lockFileCache: LockFileCache;
|
|
92
|
-
dispatcher: Dispatcher;
|
|
93
|
-
executionTracker?: ExecutionTracker;
|
|
94
|
-
checkRunReporter?: CheckRunReporter;
|
|
95
|
-
sourceCache?: SourceCache;
|
|
96
|
-
buildCoordinator?: BuildCoordinator;
|
|
97
|
-
depCache?: DepCache;
|
|
98
|
-
pendingBuilds?: PendingBuildTracker;
|
|
99
|
-
secretResolver?: SecretResolver;
|
|
100
|
-
agentRegistry: AgentRegistry;
|
|
101
|
-
providerRegistry: ProviderRegistry;
|
|
102
|
-
/** Log storage for persisting test fixture payloads. Optional -- if not set, payload storage is skipped. */
|
|
103
|
-
logStorage?: LogStorage;
|
|
104
|
-
/** Database connection for environment protection checks. Optional. */
|
|
105
|
-
db?: Kysely<Database>;
|
|
106
|
-
/** Environment store for resolving environment ids in test dispatch parity. Optional. */
|
|
107
|
-
environmentStore?: EnvironmentStore;
|
|
108
|
-
/** Variable store for resolving environment variables in test dispatch parity. Optional. */
|
|
109
|
-
variableStore?: VariableStore;
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* Process a test trigger through the existing pipeline.
|
|
91
|
+
* Process a test trigger through the shared dispatch core.
|
|
113
92
|
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
93
|
+
* Resolves the lock file + matched decisions + fixture concerns, then builds a
|
|
94
|
+
* `WorkflowDispatchContext` per matched workflow and calls
|
|
95
|
+
* `dispatchMatchedWorkflow` — the same core the webhook path uses.
|
|
117
96
|
*/
|
|
118
|
-
export declare function processTestTrigger(input: TestTriggerInput, deps:
|
|
97
|
+
export declare function processTestTrigger(input: TestTriggerInput, deps: ProcessingDeps): Promise<TestTriggerResult>;
|
|
119
98
|
export {};
|
|
120
99
|
//# sourceMappingURL=test-pipeline.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub's create-from-manifest flow is an HTML form POST (the `manifest`
|
|
3
|
+
* JSON cannot ride in a query string). These helpers render the auto-submitting
|
|
4
|
+
* form and the headless code-display page. Pure string builders — no IO — so
|
|
5
|
+
* both the CLI loopback server and the static marketing-site page can reuse the
|
|
6
|
+
* same shapes.
|
|
7
|
+
*/
|
|
8
|
+
export declare function manifestCreateUrl(githubOrg?: string): string;
|
|
9
|
+
export declare function renderManifestFormHtml(opts: {
|
|
10
|
+
createUrl: string;
|
|
11
|
+
state: string;
|
|
12
|
+
manifestJson: string;
|
|
13
|
+
}): string;
|
|
14
|
+
export declare function renderCodeDisplayHtml(code: string): string;
|
|
15
|
+
//# sourceMappingURL=manifest-form.d.ts.map
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub App Manifest flow helpers. The manifest encodes KiCI's exact
|
|
3
|
+
* permissions + events + webhook config so the operator never picks them by
|
|
4
|
+
* hand — GitHub creates a correctly-configured App from this object in one
|
|
5
|
+
* click. See docs.github.com "Registering a GitHub App from a manifest".
|
|
6
|
+
*
|
|
7
|
+
* The webhook secret is NOT part of the manifest: GitHub generates it during
|
|
8
|
+
* registration and returns it on the conversion response, so both GitHub and
|
|
9
|
+
* the Platform end up sharing the same secret with zero operator effort.
|
|
10
|
+
*/
|
|
11
|
+
import { Octokit } from '@octokit/rest';
|
|
12
|
+
export interface GithubManifestInput {
|
|
13
|
+
/** App name shown on GitHub. */
|
|
14
|
+
name: string;
|
|
15
|
+
/** Full https webhook URL (org-scoped) GitHub will POST events to. */
|
|
16
|
+
webhookUrl: string;
|
|
17
|
+
/** Loopback or static-page callback GitHub redirects to with the setup code. */
|
|
18
|
+
redirectUrl: string;
|
|
19
|
+
/** Optional post-install redirect. */
|
|
20
|
+
setupUrl?: string;
|
|
21
|
+
}
|
|
22
|
+
/** The JSON object GitHub's create-from-manifest endpoint expects. */
|
|
23
|
+
export interface GithubAppManifest {
|
|
24
|
+
name: string;
|
|
25
|
+
url: string;
|
|
26
|
+
hook_attributes: {
|
|
27
|
+
url: string;
|
|
28
|
+
active: boolean;
|
|
29
|
+
};
|
|
30
|
+
redirect_url: string;
|
|
31
|
+
setup_url?: string;
|
|
32
|
+
public: boolean;
|
|
33
|
+
default_permissions: Record<string, string>;
|
|
34
|
+
default_events: string[];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Validate a self-hosted webhook URL supplied via `source add github
|
|
38
|
+
* --webhook-url`. Must be a well-formed absolute `https://` URL. Returns the URL
|
|
39
|
+
* verbatim on success; throws a clear error otherwise. The validated URL is
|
|
40
|
+
* baked into `manifest.hook_attributes.url` as-is — KiCI adds no ingress and
|
|
41
|
+
* does not receive events at it; the operator owns delivery.
|
|
42
|
+
*/
|
|
43
|
+
export declare function validateWebhookUrl(value: string): string;
|
|
44
|
+
export declare function buildGithubAppManifest(input: GithubManifestInput): GithubAppManifest;
|
|
45
|
+
/** Credentials returned by GitHub's manifest-conversion endpoint. */
|
|
46
|
+
export interface GithubAppCredentials {
|
|
47
|
+
appId: string;
|
|
48
|
+
slug: string;
|
|
49
|
+
/** GitHub's display name for the App (the authoritative stored name). */
|
|
50
|
+
name: string;
|
|
51
|
+
privateKey: string;
|
|
52
|
+
webhookSecret: string;
|
|
53
|
+
clientId?: string;
|
|
54
|
+
clientSecret?: string;
|
|
55
|
+
htmlUrl?: string;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Exchange the short-lived manifest `code` for the App's id, private key, and
|
|
59
|
+
* webhook secret. Runs server-to-server directly against GitHub — the private
|
|
60
|
+
* key never transits the Platform.
|
|
61
|
+
*/
|
|
62
|
+
export declare function convertManifestCode(code: string, deps?: {
|
|
63
|
+
octokit?: Pick<Octokit, 'request'>;
|
|
64
|
+
}): Promise<GithubAppCredentials>;
|
|
65
|
+
/**
|
|
66
|
+
* Ask GitHub who this App is (`GET /app`, authenticated as the App via its
|
|
67
|
+
* JWT) and return its authoritative display `name` + `slug`. This is the single
|
|
68
|
+
* "fetch the App's identity from GitHub" helper, reused at source creation, by
|
|
69
|
+
* the daily refresher, and by `kici-admin source refresh`.
|
|
70
|
+
*
|
|
71
|
+
* GitHub is the source of truth: a rename in the GitHub UI changes the value
|
|
72
|
+
* `GET /app` returns, which is what keeps the dashboard name fresh.
|
|
73
|
+
*/
|
|
74
|
+
export declare function fetchGithubAppIdentity(creds: Pick<GithubAppCredentials, 'appId' | 'privateKey'>, deps?: {
|
|
75
|
+
appOctokit?: Pick<Octokit, 'request'>;
|
|
76
|
+
}): Promise<{
|
|
77
|
+
name: string;
|
|
78
|
+
slug: string;
|
|
79
|
+
}>;
|
|
80
|
+
/**
|
|
81
|
+
* Poll GitHub (as the App, via a JWT) until at least one installation exists,
|
|
82
|
+
* returning the first installation's id + account login. Throws on timeout.
|
|
83
|
+
*/
|
|
84
|
+
export declare function waitForInstallation(creds: Pick<GithubAppCredentials, 'appId' | 'privateKey'>, opts: {
|
|
85
|
+
timeoutMs: number;
|
|
86
|
+
pollMs: number;
|
|
87
|
+
now?: () => number;
|
|
88
|
+
appOctokit?: Pick<Octokit, 'request'>;
|
|
89
|
+
}): Promise<{
|
|
90
|
+
installationId: number;
|
|
91
|
+
accountLogin: string;
|
|
92
|
+
}>;
|
|
93
|
+
/**
|
|
94
|
+
* Mint an installation token from the captured private key and confirm the App
|
|
95
|
+
* can reach repos — proves the key works end-to-end (the same path the agent's
|
|
96
|
+
* clone uses at runtime).
|
|
97
|
+
*/
|
|
98
|
+
export declare function verifyRepoAccess(creds: Pick<GithubAppCredentials, 'appId' | 'privateKey'>, installationId: number, deps?: {
|
|
99
|
+
octokit?: Pick<Octokit, 'request'>;
|
|
100
|
+
}): Promise<{
|
|
101
|
+
repoCount: number;
|
|
102
|
+
}>;
|
|
103
|
+
//# sourceMappingURL=manifest.d.ts.map
|
|
@@ -227,7 +227,9 @@ export declare class ExecutionTracker {
|
|
|
227
227
|
max?: number;
|
|
228
228
|
},
|
|
229
229
|
/** Workflow-level wall-clock timeout in ms from the lock file. Sets the run deadline. */
|
|
230
|
-
workflowTimeoutMs?: number
|
|
230
|
+
workflowTimeoutMs?: number,
|
|
231
|
+
/** Run mode for idempotent steps; non-apply labels the run a check-mode preview. */
|
|
232
|
+
checkMode?: string): Promise<void>;
|
|
231
233
|
/**
|
|
232
234
|
* Add additional jobs to an already-started execution run.
|
|
233
235
|
* Used when build jobs are tracked early and regular jobs are dispatched later.
|
|
@@ -250,6 +252,13 @@ export declare class ExecutionTracker {
|
|
|
250
252
|
* uses, so dispatchEvalJob can swap it for the real eval job id.
|
|
251
253
|
*/
|
|
252
254
|
findDynamicEvalSyntheticId(runId: string, evalJobName: string): Promise<string | undefined>;
|
|
255
|
+
/**
|
|
256
|
+
* Durably mark the projected `execution_jobs` row so run-recovery sweepers
|
|
257
|
+
* know this job lives on a remote worker peer and must not be force-failed
|
|
258
|
+
* while that worker is connected. Called by the owning coordinator right
|
|
259
|
+
* after a peer ACKs a reroute.
|
|
260
|
+
*/
|
|
261
|
+
markJobReroutedToPeer(runId: string, jobId: string, peerId: string): Promise<void>;
|
|
253
262
|
/**
|
|
254
263
|
* Run `fn` while holding a per-run lock, serializing the run-mutating methods
|
|
255
264
|
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|