@kici-dev/orchestrator 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__test-helpers__/mock-db.d.ts +4 -0
- package/dist/agent/agent-version.d.ts +34 -0
- package/dist/agent/dispatcher.d.ts +70 -0
- package/dist/app.d.ts +23 -1
- package/dist/cache/global-eval-round-cache.d.ts +88 -0
- package/dist/cache/index.d.ts +3 -0
- package/dist/cache/pending-global-evals.d.ts +42 -0
- package/dist/cache/pending-inits.d.ts +10 -0
- package/dist/cli/commands/cluster-settings.d.ts +41 -3
- package/dist/cli/commands/runs.d.ts +1 -0
- package/dist/cli.js +1252 -623
- package/dist/cluster/cluster-settings-reader.d.ts +53 -1
- package/dist/config.d.ts +27 -0
- package/dist/content-requirements-cache.d.ts +55 -0
- package/dist/db/migrations/109_cluster_settings_cache_knobs.d.ts +4 -0
- package/dist/db/migrations/110_cluster_settings_global_eval_knobs.d.ts +4 -0
- package/dist/db/migrations/111_cluster_settings_global_eval_wait.d.ts +4 -0
- package/dist/db/migrations/112_execution_runs_workflow_repo.d.ts +4 -0
- package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +30 -0
- package/dist/db/migrations/114_ingest_queue_claim.d.ts +4 -0
- package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +9 -0
- package/dist/db/types.d.ts +60 -2
- package/dist/metrics/agent-metrics-aggregator.d.ts +2 -2
- package/dist/metrics/prometheus.d.ts +59 -0
- package/dist/orchestrator-core.d.ts +12 -1
- package/dist/pipeline/content-filter.d.ts +71 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +247 -8
- package/dist/pipeline/global-eval-round.d.ts +293 -0
- package/dist/pipeline/job-contexts.d.ts +16 -17
- package/dist/pipeline/process-webhook.d.ts +7 -0
- package/dist/pipeline/processor.d.ts +56 -2
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +6 -0
- package/dist/pipeline/test-pipeline.d.ts +12 -0
- package/dist/pipeline/webhook-payload-store.d.ts +20 -0
- package/dist/provenance/backfill-run.d.ts +10 -1
- package/dist/provider-registry.d.ts +38 -3
- package/dist/providers/github/check-status-poster.d.ts +22 -3
- package/dist/providers/github/commit-message.d.ts +20 -0
- package/dist/providers/github/file-contents.d.ts +40 -0
- package/dist/providers/github/index.d.ts +2 -0
- package/dist/providers/universal-git/config.d.ts +2 -0
- package/dist/providers/universal-git/normalizer.d.ts +10 -0
- package/dist/queue/cleanup.d.ts +7 -1
- package/dist/queue/job-queue.d.ts +69 -6
- package/dist/queue/terminalize-unroutable.d.ts +13 -0
- package/dist/registration/registration-run-match.d.ts +47 -0
- package/dist/reporting/check-run-reporter.d.ts +52 -1
- package/dist/reporting/execution-tracker.d.ts +117 -7
- package/dist/reporting/log-chunk-sink.d.ts +8 -5
- package/dist/routes/admin-org-settings.d.ts +5 -0
- package/dist/routes/admin.d.ts +6 -0
- package/dist/scaler/manager.d.ts +10 -8
- package/dist/secrets/pg-secret-store.d.ts +20 -2
- package/dist/security/global-workflow-policy.d.ts +52 -12
- package/dist/server.js +27579 -23783
- package/dist/standalone.js +5865 -2211
- package/dist/webhook/ingest-accept.d.ts +70 -0
- package/dist/webhook/ingest-overflow-buffer.d.ts +35 -4
- package/dist/webhook/ingest-overflow-replayer.d.ts +50 -6
- package/dist/ws/agent-handler.d.ts +3 -0
- package/dist/ws/dashboard-global-workflows-handler.d.ts +30 -9
- package/dist/ws/execution-status-frame.d.ts +32 -0
- package/dist/ws/platform-client.d.ts +14 -0
- package/dist/ws/test-relay-handlers.d.ts +35 -10
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +57 -52
- package/dist/pipeline/inline-eval.d.ts +0 -44
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* exported function is a narrative orchestrator that threads the typed
|
|
14
14
|
* results through the pipeline.
|
|
15
15
|
*/
|
|
16
|
-
import { ExecutionJobStatus, InitFailureCategory, CacheRefScope } from '@kici-dev/engine';
|
|
17
|
-
import type { LabelMatcher, LockWorkflow, LockJob, HostTargetSelector, SimulatedEvent, WorkflowDecision, MaterializedJob, ResolvedHostAgent, HostFacts } from '@kici-dev/engine';
|
|
16
|
+
import { ExecutionJobStatus, InitFailureCategory, CacheRefScope, HoldScope, TriggerSource } from '@kici-dev/engine';
|
|
17
|
+
import type { LabelMatcher, LockWorkflow, LockJob, ApprovalRequirement, HostTargetSelector, SimulatedEvent, WorkflowDecision, MaterializedJob, ResolvedHostAgent, HostFacts } from '@kici-dev/engine';
|
|
18
18
|
import { type HostRosterStore } from '../agent/host-roster.js';
|
|
19
19
|
import type { WebhookInfo } from '../webhook/handler.js';
|
|
20
20
|
import type { ProviderBundle } from '../provider-registry.js';
|
|
@@ -55,6 +55,24 @@ export interface WorkflowDispatchContext {
|
|
|
55
55
|
bundle?: ProviderBundle;
|
|
56
56
|
payload: unknown;
|
|
57
57
|
repoIdentifier: string;
|
|
58
|
+
/**
|
|
59
|
+
* The repository that DEFINES the workflow being dispatched. `repoIdentifier`
|
|
60
|
+
* is the repository the run acts on; for an organization-wide workflow the
|
|
61
|
+
* two are different repositories, and every run row this dispatch writes has
|
|
62
|
+
* to say which one defined it.
|
|
63
|
+
*
|
|
64
|
+
* REQUIRED, for the same reason `securityDecision` is: a dispatch path that
|
|
65
|
+
* does not state it must not compile. Left optional, a new caller omits it
|
|
66
|
+
* silently and every row it records claims the workflow lives in the
|
|
67
|
+
* repository the run acted on — a null marker is read as that fact, not as
|
|
68
|
+
* "unknown" (`registration/registration-run-match.ts`).
|
|
69
|
+
*
|
|
70
|
+
* Every caller today states `repoIdentifier` or a value equal to it, because
|
|
71
|
+
* no cross-repository global dispatch enters this function — the global path
|
|
72
|
+
* builds its job inputs directly and dispatches them itself. The recording
|
|
73
|
+
* sites narrow, so stating the acted-on repository records nothing.
|
|
74
|
+
*/
|
|
75
|
+
workflowRepoIdentifier: string;
|
|
58
76
|
credentials: Record<string, unknown>;
|
|
59
77
|
event: SimulatedEvent;
|
|
60
78
|
eventWithFiles: SimulatedEvent;
|
|
@@ -74,6 +92,18 @@ export interface WorkflowDispatchContext {
|
|
|
74
92
|
lockFileSource: string | undefined;
|
|
75
93
|
/** True when this run executes an uploaded local working tree (CLI remote run). */
|
|
76
94
|
localWorkingTree: boolean;
|
|
95
|
+
/**
|
|
96
|
+
* Identity that initiated this run, for `execution_runs.triggered_by`.
|
|
97
|
+
*
|
|
98
|
+
* Set by the CLI remote-run path, which knows its caller: the Platform relays
|
|
99
|
+
* the developer's actor on `test.relay.trigger` and the test pipeline renders
|
|
100
|
+
* it here. Undefined on the webhook path, where the initiator is a provider
|
|
101
|
+
* account rather than a KiCI principal — that attribution is carried by
|
|
102
|
+
* `triggerActorUsername` / `triggerActorUserId` instead.
|
|
103
|
+
*/
|
|
104
|
+
triggeredBy?: string | null;
|
|
105
|
+
/** Agent provenance label when the run was initiated through an agent credential. */
|
|
106
|
+
triggeredByAgentLabel?: string | null;
|
|
77
107
|
/** True only when invoked from the cross-source dispatch shell. */
|
|
78
108
|
crossSource: boolean;
|
|
79
109
|
/**
|
|
@@ -91,13 +121,22 @@ export interface WorkflowDispatchContext {
|
|
|
91
121
|
*/
|
|
92
122
|
securityDecision: TrustPolicyOutcome;
|
|
93
123
|
/**
|
|
94
|
-
* Set when this dispatch call took a pending-jobs token
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
124
|
+
* Set when this dispatch call took a pending-jobs token covering the window
|
|
125
|
+
* between registering the run and registering its jobs — the source-pack
|
|
126
|
+
* build window, or the plain dispatch window when there is no build. Tokens
|
|
127
|
+
* are fungible, so the `finally` must release only one it actually took — an
|
|
128
|
+
* unpaired release would consume a token held by a deferred init / dynamic
|
|
129
|
+
* task and un-hold the run while its jobs are still being registered.
|
|
99
130
|
*/
|
|
100
|
-
|
|
131
|
+
dispatchWindowTokenHeld?: boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Set when this dispatch call inserted the `execution_runs` row before
|
|
134
|
+
* handing the first job to an agent. A row that exists with zero jobs can
|
|
135
|
+
* never complete (`isRunComplete` ends `run.jobs.size > 0`) and no sweeper
|
|
136
|
+
* reaps it, so a throw inside that window has to terminalize the run
|
|
137
|
+
* explicitly rather than leave it `pending` forever.
|
|
138
|
+
*/
|
|
139
|
+
runRegisteredBeforeDispatch?: boolean;
|
|
101
140
|
/** Composite dedup key `${info.deliveryId}:${reg.id}` (cross-source only). */
|
|
102
141
|
crossSourceDeliveryId?: string;
|
|
103
142
|
/**
|
|
@@ -166,6 +205,14 @@ export interface DispatchMatchedWorkflowResult {
|
|
|
166
205
|
dispatchedJobCount: number;
|
|
167
206
|
/** Execution job ids of every dispatched/tracked job (root, gated, synthetic). */
|
|
168
207
|
dispatchedJobIds: string[];
|
|
208
|
+
/**
|
|
209
|
+
* Jobs whose dispatch is deferred to the agent init round (a dynamic context
|
|
210
|
+
* or a deferred-init job) and therefore not yet in `dispatchedJobIds`. These
|
|
211
|
+
* still run — they are dispatched asynchronously by `startDeferredPhases` —
|
|
212
|
+
* so a caller must not treat a run with pending deferred work as "nothing
|
|
213
|
+
* dispatched". Absent/0 on the early-return paths.
|
|
214
|
+
*/
|
|
215
|
+
deferredJobCount?: number;
|
|
169
216
|
/** True when the workflow install gate paused the dispatch (held run). */
|
|
170
217
|
held?: boolean;
|
|
171
218
|
/**
|
|
@@ -214,6 +261,116 @@ interface DispatchSetup {
|
|
|
214
261
|
*/
|
|
215
262
|
checkMode: string | undefined;
|
|
216
263
|
}
|
|
264
|
+
interface BuildPrepResult {
|
|
265
|
+
sourceTarUrl: string | undefined;
|
|
266
|
+
sourceTarHash: string | undefined;
|
|
267
|
+
depsUrl: string | undefined;
|
|
268
|
+
depsHash: string | undefined;
|
|
269
|
+
contentHash: string | undefined;
|
|
270
|
+
lockfileHash: string | undefined;
|
|
271
|
+
hasDynamicEntries: boolean;
|
|
272
|
+
dynamicEntries: ReadonlyArray<Extract<LockWorkflow['jobs'][number], {
|
|
273
|
+
_type: 'dynamic';
|
|
274
|
+
}>>;
|
|
275
|
+
staticJobs: readonly LockJob[];
|
|
276
|
+
/**
|
|
277
|
+
* Static jobs expanded into dispatchable children (matrix fan-out). Non-matrix
|
|
278
|
+
* jobs pass through 1:1. Every dispatch phase iterates this list, keying by
|
|
279
|
+
* `expandedName`. Dynamic-matrix jobs are flagged `pendingDynamicMatrix`.
|
|
280
|
+
*/
|
|
281
|
+
materializedJobs: readonly MaterializedJob[];
|
|
282
|
+
/** baseName -> expanded child names; drives needs-edge expansion. */
|
|
283
|
+
expansionMap: ReadonlyMap<string, readonly string[]>;
|
|
284
|
+
/** Jobs whose matrix could not be materialized (cap / zero-combination). */
|
|
285
|
+
matrixFailures: readonly RejectedJob[];
|
|
286
|
+
targetPlatform: string;
|
|
287
|
+
targetArch: string;
|
|
288
|
+
buildJobId: string | undefined;
|
|
289
|
+
buildJobName: string | undefined;
|
|
290
|
+
buildJobLabels: string[] | undefined;
|
|
291
|
+
buildJobTrackedEarly: boolean;
|
|
292
|
+
/** True when the build failed but dynamic entries can still proceed. */
|
|
293
|
+
buildFailed: boolean;
|
|
294
|
+
/**
|
|
295
|
+
* True when the helper has fully short-circuited the dispatch (build failed
|
|
296
|
+
* and no dynamic entries to fall back on, or build job rejected). Caller
|
|
297
|
+
* MUST early-return with `dispatchedJobCount: 0`.
|
|
298
|
+
*/
|
|
299
|
+
abort: boolean;
|
|
300
|
+
}
|
|
301
|
+
interface JobEnvData {
|
|
302
|
+
contextName?: string;
|
|
303
|
+
/** Configured env id matched for the first declared context name. */
|
|
304
|
+
contextId?: string;
|
|
305
|
+
/**
|
|
306
|
+
* Ordered bound-context names persisted on the job row (`(dynamic)`
|
|
307
|
+
* placeholder for elements unresolved at dispatch; overwritten with the
|
|
308
|
+
* agent-resolved list for dynamic contexts). Empty/undefined = no binding.
|
|
309
|
+
*/
|
|
310
|
+
contextNames?: string[];
|
|
311
|
+
contextVars?: Record<string, string>;
|
|
312
|
+
jobEnv?: Record<string, string>;
|
|
313
|
+
jobSecrets?: Record<string, string>;
|
|
314
|
+
jobNamespacedSecrets?: Record<string, Record<string, string>>;
|
|
315
|
+
held?: boolean;
|
|
316
|
+
/**
|
|
317
|
+
* Pending approval hold for this job, set when a context policy or
|
|
318
|
+
* explicit lock `approval` requires human sign-off. The dispatch loop turns
|
|
319
|
+
* this into a `held_runs` row + a stored pending job context so `release()`
|
|
320
|
+
* can re-dispatch after approval.
|
|
321
|
+
*/
|
|
322
|
+
approvalHold?: PendingApprovalHold;
|
|
323
|
+
rejected?: boolean;
|
|
324
|
+
rejectReason?: string;
|
|
325
|
+
pendingInit?: boolean;
|
|
326
|
+
/** Bound contexts skipped on a test/local run because they disallow local execution. */
|
|
327
|
+
skippedEnvs?: string[];
|
|
328
|
+
/**
|
|
329
|
+
* User-visible warning set whenever any bound context was unavailable for
|
|
330
|
+
* a test run (non-test or unconfigured) and skipped — surfaced on the CLI run
|
|
331
|
+
* output and the dashboard run view.
|
|
332
|
+
*/
|
|
333
|
+
envWarning?: string;
|
|
334
|
+
}
|
|
335
|
+
/** A resolved approval requirement awaiting hold creation in the dispatch loop. */
|
|
336
|
+
interface PendingApprovalHold {
|
|
337
|
+
scope: HoldScope;
|
|
338
|
+
triggerSource: TriggerSource;
|
|
339
|
+
requirement: ApprovalRequirement;
|
|
340
|
+
contextId: string | null;
|
|
341
|
+
queueType: 'context' | 'security';
|
|
342
|
+
}
|
|
343
|
+
interface DeferredInitJob {
|
|
344
|
+
mat: MaterializedJob;
|
|
345
|
+
initJobInput: QueuedJobInput;
|
|
346
|
+
}
|
|
347
|
+
interface JobEnvEvalResult {
|
|
348
|
+
jobContextData: Map<string, JobEnvData>;
|
|
349
|
+
deferredInitJobs: DeferredInitJob[];
|
|
350
|
+
runContextName: string | undefined;
|
|
351
|
+
runContextId: string | undefined;
|
|
352
|
+
}
|
|
353
|
+
interface DispatchedJob {
|
|
354
|
+
jobId: string;
|
|
355
|
+
jobName: string;
|
|
356
|
+
runsOnLabels?: string[];
|
|
357
|
+
matrixValues?: Record<string, unknown>;
|
|
358
|
+
baseJobName?: string;
|
|
359
|
+
variantKind?: string;
|
|
360
|
+
variantLabel?: string;
|
|
361
|
+
/** Held by the rolling-wave gate (a fan-out child beyond maxParallel). */
|
|
362
|
+
waveGated?: boolean;
|
|
363
|
+
/** The base's wave width, stamped on every child of a bounded wave. */
|
|
364
|
+
waveMaxParallel?: number;
|
|
365
|
+
/** The base's failFast policy, stamped on every child of a bounded wave. */
|
|
366
|
+
waveFailFast?: boolean;
|
|
367
|
+
/** Ordered bound-context names persisted on the job row (multi-env jobs). */
|
|
368
|
+
contexts?: string[];
|
|
369
|
+
/** Bound contexts skipped on a test run (non-test / unconfigured). */
|
|
370
|
+
skippedContexts?: string[];
|
|
371
|
+
/** User-visible warning naming the skipped test-run contexts. */
|
|
372
|
+
envWarning?: string;
|
|
373
|
+
}
|
|
217
374
|
interface RejectedJob {
|
|
218
375
|
jobId: string;
|
|
219
376
|
jobName: string;
|
|
@@ -345,6 +502,77 @@ export declare function findInvalidApprovalTimeout(workflow: LockWorkflow): {
|
|
|
345
502
|
* no host facts to scope by.
|
|
346
503
|
*/
|
|
347
504
|
export declare function hostCtxFromMat(mat: MaterializedJob): HostFacts | undefined;
|
|
505
|
+
/**
|
|
506
|
+
* Phase D — evaluate static jobs' context data, queue deferred-init jobs
|
|
507
|
+
* for jobs with dynamic fields, and pick the first `runContextName` for
|
|
508
|
+
* the run.
|
|
509
|
+
*/
|
|
510
|
+
export declare function evaluateJobContexts(args: {
|
|
511
|
+
ctx: WorkflowDispatchContext;
|
|
512
|
+
setup: DispatchSetup;
|
|
513
|
+
buildPrep: BuildPrepResult;
|
|
514
|
+
}): Promise<JobEnvEvalResult>;
|
|
515
|
+
/**
|
|
516
|
+
* Open the needs gate for any job whose upstreams already reached terminal
|
|
517
|
+
* before this run's edges existed.
|
|
518
|
+
*
|
|
519
|
+
* **The race this closes.** Root jobs are dispatched to agents inside the
|
|
520
|
+
* dispatch loop, but `execution_job_needs` is only written afterwards, here. An
|
|
521
|
+
* agent that reports a root job terminal in that window drives
|
|
522
|
+
* `evaluateDownstreams`, which reads zero edges, returns an empty result, and
|
|
523
|
+
* the gate never fires again — the downstream stays `pending` forever and the
|
|
524
|
+
* run hangs with no error anywhere. Nothing re-evaluates on its own: the
|
|
525
|
+
* scheduler is purely event-driven off job completion, and that event has
|
|
526
|
+
* already been consumed.
|
|
527
|
+
*
|
|
528
|
+
* The window is small (single-digit milliseconds) but entirely reachable: a job
|
|
529
|
+
* that fails immediately on arrival — a rejected dispatch, an init failure, a
|
|
530
|
+
* capability mismatch — reports terminal in about the time one DB write takes.
|
|
531
|
+
* It was invisible for as long as the webhook route answered only after the
|
|
532
|
+
* whole pipeline had run, because the caller could not learn a job had been
|
|
533
|
+
* dispatched until every edge was already committed.
|
|
534
|
+
*
|
|
535
|
+
* Recomputing here is the same guard the deferred result-aware eval registration
|
|
536
|
+
* already applies for its own edges, and it is safe to run unconditionally: the
|
|
537
|
+
* claim inside `recomputeNeedsSatisfied` is a conditional UPDATE, so a job the
|
|
538
|
+
* normal completion path already claimed is skipped rather than dispatched
|
|
539
|
+
* twice.
|
|
540
|
+
*
|
|
541
|
+
* Wave-held jobs share the synthetic-id prefix but are gated by the rolling-wave
|
|
542
|
+
* scheduler, not by needs, so they are excluded — opening their gate here would
|
|
543
|
+
* bypass the `maxParallel` window.
|
|
544
|
+
*/
|
|
545
|
+
export declare function catchUpNeedsGatedJobs(args: {
|
|
546
|
+
ctx: WorkflowDispatchContext;
|
|
547
|
+
dispatchedJobs: readonly DispatchedJob[];
|
|
548
|
+
}): Promise<void>;
|
|
549
|
+
/** Why an init result must not lead to a dispatch. */
|
|
550
|
+
export declare enum InitDispatchSuppression {
|
|
551
|
+
/** The workflow's own `filter` decided the workflow does not apply. */
|
|
552
|
+
Filter = "filter",
|
|
553
|
+
/** The job is already rejected by a context rule, or held for approval. */
|
|
554
|
+
Gated = "gated"
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* Decide whether an arrived init result may dispatch its job.
|
|
558
|
+
*
|
|
559
|
+
* `Filter` requires the workflow to actually declare a filter as well as the
|
|
560
|
+
* agent to have reported `false`: a buggy or rogue agent must not be able to
|
|
561
|
+
* suppress a filter-less workflow by inventing the field, and an agent that
|
|
562
|
+
* predates the filter reports no verdict at all — reading that absence as
|
|
563
|
+
* "suppress" would silently stop every dispatch it handles.
|
|
564
|
+
*
|
|
565
|
+
* `Gated` is belt-and-braces. `evaluateJobContexts` gives a rejected or held job
|
|
566
|
+
* no init job in the first place, so this is unreachable today; if it ever
|
|
567
|
+
* becomes reachable, dispatching would mean going straight past a protection
|
|
568
|
+
* rule or an approval hold, which is the one outcome worth a redundant check.
|
|
569
|
+
*
|
|
570
|
+
* Exported for its own test: inline, the second branch could not be exercised at
|
|
571
|
+
* all, and an untestable security check is one nobody can prove still works.
|
|
572
|
+
*/
|
|
573
|
+
export declare function initDispatchSuppression(workflow: Pick<LockWorkflow, 'hasFilter'>, initResult: {
|
|
574
|
+
filterPassed?: boolean;
|
|
575
|
+
}, jobEnvData: Pick<JobEnvData, 'rejected' | 'held'>): InitDispatchSuppression | null;
|
|
348
576
|
export interface GeneratedJobConfig {
|
|
349
577
|
/**
|
|
350
578
|
* The generated lock job with its `name` and `needs` rewritten to expanded
|
|
@@ -377,6 +605,17 @@ export declare function partitionGeneratedConfigsByPin(configs: readonly Generat
|
|
|
377
605
|
pinnedConfigs: GeneratedJobConfig[];
|
|
378
606
|
unpinnedConfigs: GeneratedJobConfig[];
|
|
379
607
|
};
|
|
608
|
+
/**
|
|
609
|
+
* Recompute the needs gate for `jobNames` and act on whatever became ready:
|
|
610
|
+
* open the gate for a satisfied job, terminalize a job an upstream's status
|
|
611
|
+
* excluded.
|
|
612
|
+
*
|
|
613
|
+
* `recomputeNeedsSatisfied` claims each job with a conditional
|
|
614
|
+
* `needs_satisfied = false → true` UPDATE, so a job a concurrent
|
|
615
|
+
* `evaluateDownstreams` already claimed is not returned here and cannot be
|
|
616
|
+
* dispatched twice.
|
|
617
|
+
*/
|
|
618
|
+
export declare function recomputeAndApplyReady(ctx: WorkflowDispatchContext, jobNames: readonly string[]): Promise<void>;
|
|
380
619
|
/**
|
|
381
620
|
* Dispatch a single matched workflow.
|
|
382
621
|
*
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tier-2 global eval round — orchestrator side.
|
|
3
|
+
*
|
|
4
|
+
* A global workflow can declare a `filter` predicate and `DynamicJobFn`
|
|
5
|
+
* generators, neither of which the orchestrator may run: author code never
|
|
6
|
+
* returns to this process. So the candidates that need either one are grouped by
|
|
7
|
+
* the workflow repo they live in and handed to ONE pre-run job per
|
|
8
|
+
* (event × workflow repo × registered SHA). That job checks out the workflow
|
|
9
|
+
* repo and the source repo once, runs every candidate's filter and then its
|
|
10
|
+
* generators, and reports a verdict per candidate.
|
|
11
|
+
*
|
|
12
|
+
* The round precedes any run row by design: its whole purpose is to decide which
|
|
13
|
+
* global workflows produce a run at all, so creating one up-front would defeat
|
|
14
|
+
* it.
|
|
15
|
+
*
|
|
16
|
+
* This module owns the partition, the grouping, and the dispatch-and-await. The
|
|
17
|
+
* caller decides what to do with the verdicts.
|
|
18
|
+
*/
|
|
19
|
+
import type { GlobalEvalCandidateResult, LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
|
|
20
|
+
import { type GlobalEvalRoundCache } from '../cache/global-eval-round-cache.js';
|
|
21
|
+
import type { PendingGlobalEvalTracker } from '../cache/pending-global-evals.js';
|
|
22
|
+
import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
|
|
23
|
+
import type { ProviderBundle } from '../provider-registry.js';
|
|
24
|
+
import type { QueuedJobInput } from '../queue/job-queue.js';
|
|
25
|
+
import type { RegisteredWorkflow } from '../registration/registration-index.js';
|
|
26
|
+
import type { WebhookInfo } from '../webhook/handler.js';
|
|
27
|
+
/** Name prefix identifying a round job in the queue, logs, and the dashboard. */
|
|
28
|
+
export declare const ROUND_JOB_PREFIX = "__globaleval__";
|
|
29
|
+
/** Metric label values for a per-candidate round verdict. */
|
|
30
|
+
export declare const GlobalEvalVerdictOutcome: {
|
|
31
|
+
/** The candidate's `filter` admitted it and its jobs are dispatching. */
|
|
32
|
+
readonly Run: 'run';
|
|
33
|
+
/** The candidate's `filter` returned false — a decided exclusion. */
|
|
34
|
+
readonly Filtered: 'filtered';
|
|
35
|
+
/** The round could not decide: it failed, breached a budget, or never reported. */
|
|
36
|
+
readonly Indeterminate: 'indeterminate';
|
|
37
|
+
};
|
|
38
|
+
export type GlobalEvalVerdictOutcome = (typeof GlobalEvalVerdictOutcome)[keyof typeof GlobalEvalVerdictOutcome];
|
|
39
|
+
/** Metric label values for a dispatched round's outcome. */
|
|
40
|
+
export declare const GlobalEvalRoundResultLabel: {
|
|
41
|
+
readonly Success: 'success';
|
|
42
|
+
readonly Error: 'error';
|
|
43
|
+
};
|
|
44
|
+
export type GlobalEvalRoundResultLabel = (typeof GlobalEvalRoundResultLabel)[keyof typeof GlobalEvalRoundResultLabel];
|
|
45
|
+
/** Metric label values for one round-cache lookup. */
|
|
46
|
+
export declare const GlobalEvalCacheLookupResult: {
|
|
47
|
+
readonly Hit: 'hit';
|
|
48
|
+
readonly Miss: 'miss';
|
|
49
|
+
/** The round input could not be serialized, so no key exists to look up. */
|
|
50
|
+
readonly Unkeyable: 'unkeyable';
|
|
51
|
+
};
|
|
52
|
+
export type GlobalEvalCacheLookupResult = (typeof GlobalEvalCacheLookupResult)[keyof typeof GlobalEvalCacheLookupResult];
|
|
53
|
+
/** One matched global workflow the round may have to decide on. */
|
|
54
|
+
export interface GlobalEvalCandidate {
|
|
55
|
+
reg: RegisteredWorkflow;
|
|
56
|
+
lockEntry: LockWorkflow;
|
|
57
|
+
/**
|
|
58
|
+
* The trigger-match decision this candidate came from, carried so the round's
|
|
59
|
+
* verdict can be appended to its trace. Optional because the round itself
|
|
60
|
+
* never reads it — only the caller that explains an exclusion does.
|
|
61
|
+
*/
|
|
62
|
+
decision?: WorkflowDecision;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Stable identity for one candidate across a round's results.
|
|
66
|
+
*
|
|
67
|
+
* The registration id alone is unique, but the workflow name is appended so a
|
|
68
|
+
* lookup reads honestly at the call site and a malformed registration with a
|
|
69
|
+
* blank id still separates two workflows.
|
|
70
|
+
*/
|
|
71
|
+
export declare function candidateKey(candidate: GlobalEvalCandidate): string;
|
|
72
|
+
/**
|
|
73
|
+
* Split matched global candidates into those that can dispatch straight away and
|
|
74
|
+
* those that must go through an eval round first.
|
|
75
|
+
*
|
|
76
|
+
* A candidate needs the round when it declares a `filter` (only the agent may
|
|
77
|
+
* run the predicate) or carries a `DynamicJobFn` (only the agent may run the
|
|
78
|
+
* generator). Everything else is fully described by the lock file, so routing it
|
|
79
|
+
* through a round would add a job dispatch and an agent round trip for nothing.
|
|
80
|
+
*/
|
|
81
|
+
export declare function partitionCandidates(candidates: readonly GlobalEvalCandidate[]): {
|
|
82
|
+
immediate: GlobalEvalCandidate[];
|
|
83
|
+
needsRound: GlobalEvalCandidate[];
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* Group by workflow repo, routing key, and registered SHA.
|
|
87
|
+
*
|
|
88
|
+
* All three parts are load-bearing. The repo identifier and the SHA together pin
|
|
89
|
+
* the exact tree the round checks out, so two registrations of one repo at
|
|
90
|
+
* different commits can never share a checkout; the routing key pins which
|
|
91
|
+
* provider bundle mints the clone credentials, so two providers owning the same
|
|
92
|
+
* repo path stay separate.
|
|
93
|
+
*/
|
|
94
|
+
export declare function groupCandidates(candidates: readonly GlobalEvalCandidate[]): Map<string, GlobalEvalCandidate[]>;
|
|
95
|
+
/** Minimal dispatcher surface the round needs (the real one is the job dispatcher). */
|
|
96
|
+
export interface GlobalEvalDispatcher {
|
|
97
|
+
dispatch(input: QueuedJobInput): Promise<{
|
|
98
|
+
status: string;
|
|
99
|
+
jobId: string;
|
|
100
|
+
}>;
|
|
101
|
+
/**
|
|
102
|
+
* Take a still-queued round job out of the queue when nobody is waiting for
|
|
103
|
+
* it any more. Optional so a test double can omit it.
|
|
104
|
+
*/
|
|
105
|
+
cancelQueuedJob?(jobId: string, reason: string): Promise<void>;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Minimal provider-registry surface: resolving the bundle that owns a routing
|
|
109
|
+
* key. The round needs it because the workflow repo and the event's source repo
|
|
110
|
+
* can live behind different providers.
|
|
111
|
+
*/
|
|
112
|
+
export interface GlobalEvalProviderRegistry {
|
|
113
|
+
getByRoutingKey(routingKey: string): ProviderBundle | undefined;
|
|
114
|
+
}
|
|
115
|
+
/** Minimal agent-registry surface used to pick the round job's platform labels. */
|
|
116
|
+
export interface GlobalEvalAgentRegistry {
|
|
117
|
+
findAvailable(labels: string[]): Array<{
|
|
118
|
+
platform: string;
|
|
119
|
+
arch: string;
|
|
120
|
+
version?: string | null;
|
|
121
|
+
}>;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* First agent release whose job runner understands a `globalEvalRound`
|
|
125
|
+
* dispatch.
|
|
126
|
+
*
|
|
127
|
+
* Below it the agent has no round branch at all: the job falls through to the
|
|
128
|
+
* standard executor, whose `jobConfig` carries no `source.file`, so it either
|
|
129
|
+
* fails outright or reports success without the `globalEvalComplete` the
|
|
130
|
+
* orchestrator is waiting for. Either way the round never settles.
|
|
131
|
+
*
|
|
132
|
+
* Customers upgrade their orchestrator and their agents on their own schedule
|
|
133
|
+
* (`.claude/rules/compatibility.md`), so an orchestrator ahead of its fleet is
|
|
134
|
+
* a supported state and not an error — but it is one this module has to
|
|
135
|
+
* recognise, because the damage is not confined to the new feature. A global
|
|
136
|
+
* workflow that merely *contains* a generator now routes through the round, so
|
|
137
|
+
* an unrecognised round means static jobs that ran yesterday stop running.
|
|
138
|
+
*/
|
|
139
|
+
export declare const MIN_GLOBAL_EVAL_AGENT_VERSION = "0.5.0";
|
|
140
|
+
export interface GlobalEvalRoundDeps {
|
|
141
|
+
dispatcher: GlobalEvalDispatcher;
|
|
142
|
+
pendingGlobalEvals: PendingGlobalEvalTracker;
|
|
143
|
+
/**
|
|
144
|
+
* Required, unlike the three optional deps below: it is what resolves the
|
|
145
|
+
* WORKFLOW repo's own provider bundle. Omitting it would leave the round with
|
|
146
|
+
* no clone URL for the workflow repo at all, so it is not a dep that can
|
|
147
|
+
* degrade quietly.
|
|
148
|
+
*/
|
|
149
|
+
providerRegistry: GlobalEvalProviderRegistry;
|
|
150
|
+
clusterSettings?: ClusterSettingsReader;
|
|
151
|
+
globalEvalCache?: GlobalEvalRoundCache;
|
|
152
|
+
agentRegistry?: GlobalEvalAgentRegistry;
|
|
153
|
+
}
|
|
154
|
+
export interface GlobalEvalRoundArgs {
|
|
155
|
+
deps: GlobalEvalRoundDeps;
|
|
156
|
+
info: WebhookInfo;
|
|
157
|
+
event: SimulatedEvent;
|
|
158
|
+
candidates: readonly GlobalEvalCandidate[];
|
|
159
|
+
/** Source repo that triggered the event. */
|
|
160
|
+
repoIdentifier: string;
|
|
161
|
+
/** Source repo SHA the event lands on. */
|
|
162
|
+
ref: string;
|
|
163
|
+
dispatchBundle: ProviderBundle;
|
|
164
|
+
dispatchCredentials: Record<string, unknown>;
|
|
165
|
+
/** Cluster defaults for the round budgets and the wait ceiling (`config.ts`). */
|
|
166
|
+
config: {
|
|
167
|
+
globalEvalRoundTimeoutMs: number;
|
|
168
|
+
globalEvalCandidateTimeoutMs: number;
|
|
169
|
+
globalEvalWaitTimeoutMs: number;
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Why a fleet cannot decide a round, or `null` when it might.
|
|
174
|
+
*
|
|
175
|
+
* Only ONE state is refused: every registered init-runner reports a version, and
|
|
176
|
+
* every one of them is below the minimum. Everything else returns `null`.
|
|
177
|
+
*
|
|
178
|
+
* An empty fleet is not refused — nothing is registered yet, an agent may
|
|
179
|
+
* register a second from now, and the queue's own unroutable handling already
|
|
180
|
+
* covers a round nobody picks up.
|
|
181
|
+
*
|
|
182
|
+
* A fleet containing even ONE agent whose version cannot be read is not refused
|
|
183
|
+
* either, and the bar is deliberately that strict rather than "all unknown".
|
|
184
|
+
* Refusing suppresses every global workflow for the delivery — the exact damage
|
|
185
|
+
* this function exists to prevent — so it must rest on proof, not on ignorance
|
|
186
|
+
* about part of the fleet. Getting it wrong in the other direction merely costs
|
|
187
|
+
* a slow failure with a less useful message.
|
|
188
|
+
*
|
|
189
|
+
* Refusing here rather than at dispatch is deliberate. A too-old fleet does not
|
|
190
|
+
* clear within a retry, so dispatching would spend two full wait ceilings of
|
|
191
|
+
* inline webhook latency to reach the same verdict — and reach it with a
|
|
192
|
+
* timeout message that says nothing about agent versions.
|
|
193
|
+
*/
|
|
194
|
+
export declare function unsupportedFleetReason(agentRegistry?: GlobalEvalAgentRegistry): string | null;
|
|
195
|
+
/**
|
|
196
|
+
* Cap on the joined reason string a decided-nothing round reports.
|
|
197
|
+
*
|
|
198
|
+
* The reasons are authored by an agent running author code, so their combined
|
|
199
|
+
* length is unbounded — and this string travels verbatim into the commit
|
|
200
|
+
* check's `output.summary`, which GitHub caps at 65535 characters. An overflow
|
|
201
|
+
* is rejected with a 422 that the best-effort post swallows, so the check
|
|
202
|
+
* disappears in exactly the case it exists for. Realistic reasons are one short
|
|
203
|
+
* sentence each; this bound makes that structural instead of lucky.
|
|
204
|
+
*/
|
|
205
|
+
export declare const MAX_ROUND_REASON_CHARS = 4000;
|
|
206
|
+
/**
|
|
207
|
+
* Truncate `text` to at most `max` UTF-16 code units, marking that it was cut.
|
|
208
|
+
*
|
|
209
|
+
* Two edges the naive slice gets wrong, both of which matter because the result
|
|
210
|
+
* is posted to an API with a hard character limit:
|
|
211
|
+
*
|
|
212
|
+
* - A `max` smaller than the marker would make `max - marker.length` negative,
|
|
213
|
+
* and appending the marker to an empty slice then returns a string LONGER
|
|
214
|
+
* than `max`. Such a `max` cannot carry both content and a marker, so the
|
|
215
|
+
* marker itself is truncated instead.
|
|
216
|
+
* - Slicing at an arbitrary index can cut a surrogate pair in half, leaving a
|
|
217
|
+
* lone surrogate that is not valid text. The cut backs off by one unit when
|
|
218
|
+
* it lands between the halves of a pair.
|
|
219
|
+
*/
|
|
220
|
+
export declare function truncateReasonText(text: string, max: number): string;
|
|
221
|
+
/**
|
|
222
|
+
* One round that produced no verdicts after every attempt, described well enough
|
|
223
|
+
* for the caller to record it.
|
|
224
|
+
*
|
|
225
|
+
* There is exactly one of these per failed round — never one per candidate. The
|
|
226
|
+
* round exists to collapse N candidate workflows into a single pre-run job, so
|
|
227
|
+
* fanning its failure back out into N run rows and N checks would undo the fan-out
|
|
228
|
+
* reduction the whole design is for. `workflowNames` is what makes the single
|
|
229
|
+
* record honest: it names every workflow the failure suppressed.
|
|
230
|
+
*/
|
|
231
|
+
export interface GlobalEvalRoundFailure {
|
|
232
|
+
/** The last attempt's round-job run id — what an errored run row is written under. */
|
|
233
|
+
runId: string;
|
|
234
|
+
/** Repo the suppressed workflows are authored in. */
|
|
235
|
+
workflowRepoIdentifier: string;
|
|
236
|
+
/** Every candidate workflow this round suppressed. */
|
|
237
|
+
workflowNames: string[];
|
|
238
|
+
/** The last attempt's error. */
|
|
239
|
+
error: string;
|
|
240
|
+
/** How many attempts were made before giving up. `0` ⇒ never dispatched. */
|
|
241
|
+
attempts: number;
|
|
242
|
+
/**
|
|
243
|
+
* True when the round itself completed and decided some candidates, leaving
|
|
244
|
+
* only {@link workflowNames} undecided.
|
|
245
|
+
*
|
|
246
|
+
* Recorded as its own record rather than folded into a total failure because
|
|
247
|
+
* the two read differently to an author: a total failure means nothing from
|
|
248
|
+
* this repo ran, a partial one means their workflow specifically could not be
|
|
249
|
+
* decided while its neighbours ran fine. Without it, whether a broken filter
|
|
250
|
+
* is visible at all depends on how many unrelated global workflows happen to
|
|
251
|
+
* share a workflow repo — a per-candidate budget breach in a group of one
|
|
252
|
+
* produces a check, and the identical fault in a group of two produces
|
|
253
|
+
* nothing.
|
|
254
|
+
*/
|
|
255
|
+
partial?: boolean;
|
|
256
|
+
}
|
|
257
|
+
/** Every candidate's verdict, plus the rounds that produced none. */
|
|
258
|
+
export interface GlobalEvalRoundsOutcome {
|
|
259
|
+
/** One verdict per candidate, keyed by {@link candidateKey}. */
|
|
260
|
+
verdicts: Map<string, GlobalEvalCandidateResult>;
|
|
261
|
+
/** One entry per round that failed outright — never one per candidate. */
|
|
262
|
+
failures: GlobalEvalRoundFailure[];
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Run every eval round the candidate set needs and return one verdict per
|
|
266
|
+
* candidate, plus one failure record per round that produced none.
|
|
267
|
+
*
|
|
268
|
+
* Rounds run one group at a time: a group is one dual checkout on one agent, and
|
|
269
|
+
* a push touching several workflow repos is the uncommon case. A group whose
|
|
270
|
+
* round fails marks only its own candidates indeterminate — one unreachable
|
|
271
|
+
* workflow repo must not suppress the workflows living in another.
|
|
272
|
+
*
|
|
273
|
+
* The failures are returned rather than recorded here: this module owns the
|
|
274
|
+
* partition, the grouping, and the dispatch-and-await, and has neither a
|
|
275
|
+
* database handle nor the inbound provider bundle a commit check must be posted
|
|
276
|
+
* through.
|
|
277
|
+
*/
|
|
278
|
+
export declare function runGlobalEvalRounds(args: GlobalEvalRoundArgs): Promise<GlobalEvalRoundsOutcome>;
|
|
279
|
+
/**
|
|
280
|
+
* Meter candidates a round never got to decide on.
|
|
281
|
+
*
|
|
282
|
+
* The caller's fail-closed paths — no pending-eval tracker, so a round could
|
|
283
|
+
* never settle — return before {@link runGlobalEvalRounds} is reached, and they
|
|
284
|
+
* suppress EVERY global workflow for that delivery. That is this subsystem's
|
|
285
|
+
* most severe outcome, and without this call it appeared in none of its
|
|
286
|
+
* metrics: one `logger.warn` was the whole record.
|
|
287
|
+
*
|
|
288
|
+
* Counted as `indeterminate` because that is exactly what happened — nothing
|
|
289
|
+
* evaluated the workflows — which also keeps `candidates` equal to the sum of
|
|
290
|
+
* the verdicts on every path.
|
|
291
|
+
*/
|
|
292
|
+
export declare function recordUnrunCandidates(count: number): void;
|
|
293
|
+
//# sourceMappingURL=global-eval-round.d.ts.map
|
|
@@ -3,43 +3,42 @@
|
|
|
3
3
|
*
|
|
4
4
|
* A job binds an ordered list of contexts (`LockJob.contexts`). This
|
|
5
5
|
* module resolves that list into concrete context names (static values
|
|
6
|
-
* verbatim
|
|
7
|
-
* the per-context secrets/variables last-wins. It keeps the heavy fold logic
|
|
6
|
+
* verbatim; every dynamic element is resolved by the agent's init job) and
|
|
7
|
+
* folds the per-context secrets/variables last-wins. It keeps the heavy fold logic
|
|
8
8
|
* out of `dispatchMatchedWorkflow`, which must stay under the function-length cap.
|
|
9
9
|
*/
|
|
10
10
|
import { type Context, type HostFacts, type LockJob } from '@kici-dev/engine';
|
|
11
11
|
import type { SecretResolverApi } from '../secrets/secret-resolver.js';
|
|
12
12
|
import type { VariableStore } from '../contexts/variable-store.js';
|
|
13
13
|
/**
|
|
14
|
-
* Placeholder written into the persisted bound-env list for
|
|
15
|
-
*
|
|
16
|
-
*
|
|
14
|
+
* Placeholder written into the persisted bound-env list for a dynamic element
|
|
15
|
+
* the orchestrator does not resolve at dispatch. The agent's init eval later
|
|
16
|
+
* overwrites the list with the resolved name.
|
|
17
17
|
*/
|
|
18
18
|
export declare const DYNAMIC_ENV_PLACEHOLDER = "(dynamic)";
|
|
19
19
|
/** Ordered resolved context names plus whether any element still needs agent init. */
|
|
20
20
|
export interface ResolvedJobContexts {
|
|
21
|
-
/** Resolved static
|
|
21
|
+
/** Resolved static names, in order. */
|
|
22
22
|
names: string[];
|
|
23
|
-
/** True when
|
|
23
|
+
/** True when a dynamic element must be resolved by an agent init job. */
|
|
24
24
|
needsInit: boolean;
|
|
25
25
|
}
|
|
26
26
|
/**
|
|
27
27
|
* Resolve the ordered bound-context names from a lock job. Static elements
|
|
28
|
-
* use their value verbatim;
|
|
29
|
-
*
|
|
30
|
-
* be resolved here and flags `needsInit`.
|
|
28
|
+
* use their value verbatim; any dynamic element (inline or impure) is resolved
|
|
29
|
+
* by the agent's init job and flags `needsInit`.
|
|
31
30
|
*/
|
|
32
|
-
export declare function resolveJobContextNames(lockJob: LockJob
|
|
31
|
+
export declare function resolveJobContextNames(lockJob: LockJob): ResolvedJobContexts;
|
|
33
32
|
/**
|
|
34
33
|
* Build the ordered bound-context display list for persistence at dispatch.
|
|
35
34
|
* Unlike {@link resolveJobContextNames}, this never drops an unresolved
|
|
36
|
-
* element: a static element uses its value,
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
35
|
+
* element: a static element uses its value, and any dynamic element (which the
|
|
36
|
+
* orchestrator no longer resolves at dispatch) becomes the `(dynamic)`
|
|
37
|
+
* placeholder — so the persisted column reflects every declared slot in order.
|
|
38
|
+
* The deferred-init flow-back overwrites the placeholder once the agent
|
|
39
|
+
* resolves the name. Returns an empty array when the job binds no context.
|
|
41
40
|
*/
|
|
42
|
-
export declare function buildJobContextDisplayNames(lockJob: LockJob
|
|
41
|
+
export declare function buildJobContextDisplayNames(lockJob: LockJob): string[];
|
|
43
42
|
/** Merged secrets/variables across an ordered list of resolved contexts. */
|
|
44
43
|
export interface MultiEnvMergedData {
|
|
45
44
|
contextVars?: Record<string, string>;
|