@kici-dev/orchestrator 0.1.17 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/agent/dispatcher.d.ts +9 -0
  2. package/dist/agent/host-roster-reaper.d.ts +42 -0
  3. package/dist/agent/host-roster.d.ts +110 -0
  4. package/dist/agent/registry.d.ts +65 -2
  5. package/dist/app.d.ts +3 -0
  6. package/dist/cli/commands/host.d.ts +13 -0
  7. package/dist/cli/commands/shared/versioned-upgrade.d.ts +23 -0
  8. package/dist/cli/service/compose.d.ts +2 -1
  9. package/dist/cli/service/index.d.ts +2 -2
  10. package/dist/cli/service/instance/manifest.d.ts +12 -0
  11. package/dist/cli/service/instance/resolve.d.ts +12 -2
  12. package/dist/cli/service/launchd.d.ts +4 -1
  13. package/dist/cli/service/platform-detect.d.ts +5 -0
  14. package/dist/cli/service/systemd.d.ts +2 -1
  15. package/dist/cli/service/types.d.ts +37 -0
  16. package/dist/cli/service/windows.d.ts +2 -1
  17. package/dist/cli.js +797 -202
  18. package/dist/cluster/coordinator.d.ts +5 -1
  19. package/dist/config/schema.d.ts +4 -0
  20. package/dist/config/types.d.ts +9 -0
  21. package/dist/config.d.ts +6 -0
  22. package/dist/db/migrations/039_host_roster.d.ts +19 -0
  23. package/dist/db/migrations/040_runsonall_pin.d.ts +4 -0
  24. package/dist/db/migrations/041_wave_gated.d.ts +4 -0
  25. package/dist/db/migrations/042_dispatch_queue_patterns.d.ts +4 -0
  26. package/dist/db/types.d.ts +61 -0
  27. package/dist/diagnostics/fleet-collector.d.ts +1 -1
  28. package/dist/environments/held-runs.d.ts +9 -0
  29. package/dist/lockfile-redos-guard.d.ts +19 -0
  30. package/dist/metrics/prometheus.d.ts +13 -0
  31. package/dist/metrics/scheduled-jobs.d.ts +2 -2
  32. package/dist/orchestrator-core.d.ts +46 -1
  33. package/dist/pipeline/dispatch-matched-workflow.d.ts +42 -1
  34. package/dist/pipeline/processor.d.ts +25 -0
  35. package/dist/pipeline/wave-scheduler.d.ts +60 -0
  36. package/dist/queue/job-queue.d.ts +61 -2
  37. package/dist/reporting/execution-tracker.d.ts +28 -0
  38. package/dist/server.js +52760 -51263
  39. package/dist/standalone.js +1953 -500
  40. package/installer-image-digests.json +3 -3
  41. package/package.json +22 -22
  42. package/sbom.spdx.json +3227 -6682
@@ -10,7 +10,7 @@
10
10
  * - "Peers report step-by-step progress back to coordinator"
11
11
  * - "Cancel mode: graceful -- finish current step, cancel remaining"
12
12
  */
13
- import type { JobReroute, JobProgress, PeerScalerEvent, PeerToPeerMessage, ResourceRequest } from '@kici-dev/engine';
13
+ import type { JobReroute, JobProgress, PeerScalerEvent, PeerToPeerMessage, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
14
14
  import type { PeerRegistry } from './peer-registry.js';
15
15
  import type { PeerClient } from './peer-client.js';
16
16
  import type { Dispatcher } from '../agent/dispatcher.js';
@@ -37,6 +37,10 @@ export interface RunContext {
37
37
  export interface JobToRoute {
38
38
  jobName: string;
39
39
  runsOnLabels: string[][];
40
+ /** Regex matchers the agent's labels must satisfy (JS post-filter). */
41
+ runsOnPatterns?: LabelMatcher[];
42
+ /** Regex matchers that disqualify an agent (JS post-filter). */
43
+ excludePatterns?: LabelMatcher[];
40
44
  jobConfig: Record<string, unknown>;
41
45
  repoUrl: string;
42
46
  ref: string;
@@ -69,6 +69,8 @@ export declare const sharedConfigSchema: z.ZodObject<{
69
69
  none: "none";
70
70
  }>>;
71
71
  agentTokenTtlMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
72
+ rosterGraceMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
73
+ rosterTtlMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
72
74
  queue: z.ZodOptional<z.ZodObject<{
73
75
  maxDepth: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
74
76
  timeoutMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
@@ -156,6 +158,8 @@ export declare const appConfigSchema: z.ZodObject<{
156
158
  none: "none";
157
159
  }>>;
158
160
  agentTokenTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
161
+ rosterGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
162
+ rosterTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
159
163
  queueMaxDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
160
164
  queueTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
161
165
  queueBackpressureThreshold: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -50,6 +50,9 @@ export interface SharedConfig {
50
50
  };
51
51
  agentAuth?: 'token' | 'none';
52
52
  agentTokenTtlMs?: number;
53
+ rosterGraceMs?: number;
54
+ rosterTtlMs?: number;
55
+ maxFanoutHosts?: number;
53
56
  queue?: {
54
57
  maxDepth?: number;
55
58
  timeoutMs?: number;
@@ -135,6 +138,12 @@ export interface AppConfig {
135
138
  agentAuth: 'token' | 'none';
136
139
  /** Agent token TTL in milliseconds */
137
140
  agentTokenTtlMs: number;
141
+ /** Host roster: static grace before a disconnected static host reads unreachable (ms) */
142
+ rosterGraceMs: number;
143
+ /** Host roster: ephemeral GC ttl — past this a disconnected ephemeral host is reaped (ms) */
144
+ rosterTtlMs: number;
145
+ /** Cap on per-host children produced by a runsOnAll fan-out */
146
+ maxFanoutHosts: number;
138
147
  /** Queue settings */
139
148
  queueMaxDepth: number;
140
149
  queueTimeoutMs: number;
package/dist/config.d.ts CHANGED
@@ -82,6 +82,9 @@ declare const configSchema: z.ZodObject<{
82
82
  none: "none";
83
83
  }>>;
84
84
  agentTokenTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
85
+ rosterGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
86
+ rosterTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
87
+ maxFanoutHosts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
85
88
  eventRouterMaxChainDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
86
89
  eventRouterRateLimitPerWorkflowPerMinute: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
87
90
  eventRouterEventTtlSeconds: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -197,6 +200,9 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
197
200
  pgCustomerSecrets: boolean;
198
201
  agentAuth: "token" | "none";
199
202
  agentTokenTtlMs: number;
203
+ rosterGraceMs: number;
204
+ rosterTtlMs: number;
205
+ maxFanoutHosts: number;
200
206
  eventRouterMaxChainDepth: number;
201
207
  eventRouterRateLimitPerWorkflowPerMinute: number;
202
208
  eventRouterEventTtlSeconds: number;
@@ -0,0 +1,19 @@
1
+ import { type Kysely } from 'kysely';
2
+ /**
3
+ * `host_roster` is KiCI's declared inventory: one durable row per agent the
4
+ * cluster has ever enrolled, reconciled from the in-memory AgentRegistry on
5
+ * every register/unregister. `lifecycle_class` (snapshot of the auth token's
6
+ * agent_type) drives reaping — `ephemeral` rows are GC'd past their TTL,
7
+ * `static` rows persist and read as `unreachable` when their heartbeat goes
8
+ * stale. `connected_instance_id` records which orchestrator holds the live WS
9
+ * (cluster liveness + the host-fanout reroute target); NULL = disconnected.
10
+ *
11
+ * The roster lives in the shared cluster DB (one table, all instances). Status
12
+ * is derived at read from the shared `last_seen` + `connected_instance_id`, so
13
+ * every instance agrees regardless of which one holds the agent's live WS.
14
+ *
15
+ * Idempotent: a re-run on a DB that already has the table is a no-op.
16
+ */
17
+ export declare function up(db: Kysely<unknown>): Promise<void>;
18
+ export declare function down(db: Kysely<unknown>): Promise<void>;
19
+ //# sourceMappingURL=039_host_roster.d.ts.map
@@ -0,0 +1,4 @@
1
+ import { type Kysely } from 'kysely';
2
+ export declare function up(db: Kysely<unknown>): Promise<void>;
3
+ export declare function down(db: Kysely<unknown>): Promise<void>;
4
+ //# sourceMappingURL=040_runsonall_pin.d.ts.map
@@ -0,0 +1,4 @@
1
+ import { type Kysely } from 'kysely';
2
+ export declare function up(db: Kysely<unknown>): Promise<void>;
3
+ export declare function down(db: Kysely<unknown>): Promise<void>;
4
+ //# sourceMappingURL=041_wave_gated.d.ts.map
@@ -0,0 +1,4 @@
1
+ import { type Kysely } from 'kysely';
2
+ export declare function up(db: Kysely<unknown>): Promise<void>;
3
+ export declare function down(db: Kysely<unknown>): Promise<void>;
4
+ //# sourceMappingURL=042_dispatch_queue_patterns.d.ts.map
@@ -50,6 +50,7 @@ export interface Database {
50
50
  scaler_reservations: ScalerReservationsTable;
51
51
  attestations: AttestationsTable;
52
52
  remote_sources: RemoteSourcesTable;
53
+ host_roster: HostRosterTable;
53
54
  }
54
55
  /**
55
56
  * Cluster metadata table
@@ -138,6 +139,10 @@ export interface DispatchQueueTable {
138
139
  request_id: string | null;
139
140
  /** JSONB array of exclusion labels. Default '[]'. */
140
141
  exclude_labels: Generated<string>;
142
+ /** Regex matchers (LabelMatcher[]) the job requires; JS post-filter on top of runs_on_labels. Default '[]'. */
143
+ runs_on_patterns: Generated<string>;
144
+ /** Regex matchers (LabelMatcher[]) that disqualify an agent; JS post-filter. Default '[]'. */
145
+ exclude_patterns: Generated<string>;
141
146
  /** Routing key (e.g. "github:12345") so dispatch can pick the right
142
147
  * per-app provider bundle in multi-app setups. Required (NOT NULL). */
143
148
  routing_key: string;
@@ -173,6 +178,12 @@ export interface DispatchQueueTable {
173
178
  ack_deadline: ColumnType<Date | null, Date | null | undefined, Date | null>;
174
179
  /** Agent the dispatch was sent to (for ack-timeout disconnect + logging). */
175
180
  ack_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
181
+ /**
182
+ * For a runsOnAll host-fanout child: the agent this job is pinned to. The
183
+ * dispatcher routes it only to that agent and the queue drain never hands it
184
+ * to another. NULL for normal label-routed jobs.
185
+ */
186
+ pinned_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
176
187
  }
177
188
  /**
178
189
  * Deduplication cache table
@@ -354,6 +365,23 @@ export interface ExecutionJobTable {
354
365
  ready_at: Date | null;
355
366
  /** Dynamic group membership tag (NULL for static jobs). */
356
367
  group_name: string | null;
368
+ /** Base (logical) job name for a fan-out child. NULL for non-fanned jobs. */
369
+ base_job_name: string | null;
370
+ /** Fan-out kind for a child: 'matrix' | 'host'. NULL for non-fanned jobs. */
371
+ variant_kind: string | null;
372
+ /** Fan-out label for a child: matrix suffix or hostname. NULL for non-fanned jobs. */
373
+ variant_label: string | null;
374
+ /**
375
+ * Wave gate: a fan-out child beyond the job's `maxParallel` window is held
376
+ * (`true`) instead of dispatched. Cleared by the wave-scheduler when a sibling
377
+ * reaches terminal and an in-flight slot frees up. NULL/false for any job not
378
+ * held by a rolling wave.
379
+ */
380
+ wave_gated: Generated<boolean>;
381
+ /** The fan-out base's `maxParallel` wave width, stamped on every child. NULL = no bounded wave. */
382
+ wave_max_parallel: number | null;
383
+ /** The fan-out base's `failFast` policy, stamped on every child. NULL = no bounded wave. */
384
+ wave_fail_fast: boolean | null;
357
385
  /** When this record was created */
358
386
  created_at: Generated<Date>;
359
387
  /**
@@ -1529,6 +1557,39 @@ export interface RemoteSourcesTable {
1529
1557
  }
1530
1558
  export type RemoteSourceRow = Selectable<RemoteSourcesTable>;
1531
1559
  export type NewRemoteSourceRow = Insertable<RemoteSourcesTable>;
1560
+ /**
1561
+ * Host roster table (host_roster).
1562
+ *
1563
+ * KiCI's declared inventory: one durable row per agent the cluster has ever
1564
+ * enrolled, reconciled from the in-memory AgentRegistry on register/unregister.
1565
+ * `lifecycle_class` (snapshot of the auth token's `agent_type`) drives reaping;
1566
+ * `connected_instance_id` records which orchestrator holds the live WS (cluster
1567
+ * liveness + the host-fanout reroute target), null when disconnected. Status is
1568
+ * derived at read from the shared `last_seen` + `connected_instance_id`.
1569
+ */
1570
+ export interface HostRosterTable {
1571
+ /** UUID primary key */
1572
+ id: Generated<string>;
1573
+ /** The agent identity the pin targets; unique. */
1574
+ agent_id: string;
1575
+ /** FK to agent_tokens.id (provenance), or null when auth mode is none. */
1576
+ token_id: string | null;
1577
+ /** Snapshot of the token's agent_type: 'static' | 'ephemeral'. */
1578
+ lifecycle_class: string;
1579
+ /** JSON-encoded string[] of the post-Gate-1 validated labels. */
1580
+ labels: string;
1581
+ hostname: string | null;
1582
+ platform: string | null;
1583
+ arch: string | null;
1584
+ /** Which orchestrator instance holds the live WS; null = disconnected. */
1585
+ connected_instance_id: string | null;
1586
+ last_seen: ColumnType<Date, Date | string | undefined, Date | string>;
1587
+ created_at: Generated<Date>;
1588
+ updated_at: ColumnType<Date, Date | string | undefined, Date | string>;
1589
+ }
1590
+ export type HostRosterRow = Selectable<HostRosterTable>;
1591
+ export type NewHostRosterRow = Insertable<HostRosterTable>;
1592
+ export type HostRosterUpdate = Updateable<HostRosterTable>;
1532
1593
  /**
1533
1594
  * Scaler spawning-agents table (scaler_spawning_agents).
1534
1595
  *
@@ -12,8 +12,8 @@ import { z } from 'zod';
12
12
  /** Per-node collection outcome recorded in the fleet manifest. */
13
13
  export declare const FleetNodeStatus: z.ZodEnum<{
14
14
  error: "error";
15
- ok: "ok";
16
15
  unreachable: "unreachable";
16
+ ok: "ok";
17
17
  timeout: "timeout";
18
18
  }>;
19
19
  export type FleetNodeStatus = z.infer<typeof FleetNodeStatus>;
@@ -71,6 +71,15 @@ export interface ReleaseSignal {
71
71
  scope: HoldScope;
72
72
  /** Set only for step-scoped holds. */
73
73
  stepIndex: number | null;
74
+ /**
75
+ * What kind of gate created the hold. `explicit` (SDK `requireApproval`) holds
76
+ * a real root job and resumes by re-dispatching it; `environment` covers the
77
+ * workflow install-gate (wait-timer / concurrency / env approval) which resumes
78
+ * by rebuilding the workflow dispatch context. The resume router keys off this
79
+ * so a workflow-scoped explicit hold goes through the job re-dispatch path
80
+ * rather than the install-gate path (which has no pending workflow context).
81
+ */
82
+ triggerSource: TriggerSource;
74
83
  }
75
84
  /** Options for listing held runs. */
76
85
  export interface ListHeldRunsOptions {
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Lock-load ReDoS revalidation.
3
+ *
4
+ * Re-validates every regex matcher in a fetched lock file before it is cached
5
+ * or dispatched. Defense-in-depth against a hand-edited or non-compiled lock
6
+ * that smuggled a ReDoS-prone pattern past the compile-time gate: the compiler
7
+ * runs the same `assertMatchersSafe` check when it emits the lock, but the
8
+ * orchestrator does not trust that the lock it fetched was produced by our
9
+ * compiler.
10
+ */
11
+ import type { LockFile } from '@kici-dev/engine';
12
+ /**
13
+ * Walk every static job's `runsOn` / `excludeLabels` / `runsOnAll` matchers and
14
+ * throw if any regex matcher is ReDoS-prone. Dynamic job generators carry no
15
+ * static routing matchers (they materialize jobs at eval time, which re-runs the
16
+ * compile-time gate), so only static jobs are checked.
17
+ */
18
+ export declare function assertLockFileRegexesSafe(lockFile: LockFile): void;
19
+ //# sourceMappingURL=lockfile-redos-guard.d.ts.map
@@ -2,6 +2,8 @@
2
2
  export declare function setAgentsActive(value: number): void;
3
3
  /** Set the current config version number. */
4
4
  export declare function setConfigVersion(value: number): void;
5
+ /** Set the current number of declared (static) roster hosts that are unreachable. */
6
+ export declare function setDeclaredHostsUnreachable(value: number): void;
5
7
  /** Set the current number of stale runs detected. */
6
8
  export declare function setStaleRunsCurrent(value: number): void;
7
9
  interface ScalerUsageRow {
@@ -322,5 +324,16 @@ export declare const installSecretsContributorStrippedTotal: import("@openteleme
322
324
  * - environment: the environment name referenced in the qualified `<environment>:<secret>` ref (per-org count is typically <10)
323
325
  */
324
326
  export declare const installSecretsTokenResolutionDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
327
+ /**
328
+ * Register every queued orchestrator observable gauge on the real meter.
329
+ *
330
+ * Must run AFTER `initTelemetry()` has wired the global MeterProvider —
331
+ * `createApp()` calls it once during bootstrap. Registering the gauges at
332
+ * module-eval time instead would bind them to the no-op provider (the
333
+ * bundler hoists some module init above the entry's `initTelemetry()` call),
334
+ * leaving every `kici_orch_*` gauge absent from the /metrics scrape and the
335
+ * Platform push. Idempotent: repeat calls are no-ops.
336
+ */
337
+ export declare function registerOrchestratorMetrics(): void;
325
338
  export {};
326
339
  //# sourceMappingURL=prometheus.d.ts.map
@@ -4,13 +4,13 @@
4
4
  * - job: scheduled-job name (one of `OrchestratorScheduledJobName`)
5
5
  * - result: success | failure
6
6
  */
7
- export declare const jobRunsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
7
+ export declare const jobRunsTotal: Pick<import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>, "add">;
8
8
  /**
9
9
  * Histogram of per-tick duration, seconds.
10
10
  * Labels:
11
11
  * - job: scheduled-job name (one of `OrchestratorScheduledJobName`)
12
12
  */
13
- export declare const jobDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
13
+ export declare const jobDurationSeconds: Pick<import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>, "record">;
14
14
  /**
15
15
  * Unix timestamp (seconds) of the most recent successful tick.
16
16
  * Labels:
@@ -13,6 +13,7 @@ import { type ColdStore } from '@kici-dev/shared';
13
13
  import type { AppConfig } from './config.js';
14
14
  import { ConfigReloader } from './config/reload.js';
15
15
  import { AgentRegistry } from './agent/registry.js';
16
+ import { HostRosterStore } from './agent/host-roster.js';
16
17
  import { JobQueue } from './queue/job-queue.js';
17
18
  import { EventLogWriter } from './webhook/event-log.js';
18
19
  import { AccessLogWriter } from './audit/access-log.js';
@@ -22,7 +23,7 @@ import { DedupCache } from './webhook/dedup.js';
22
23
  import { ObserverRegistry } from './ws/observer-registry.js';
23
24
  import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
24
25
  import { SourceLocationStore } from './app.js';
25
- import { type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
26
+ import { type LabelMatcher, type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
26
27
  import { ScalerManager } from './scaler/index.js';
27
28
  import type { ScalerConfig } from './scaler/index.js';
28
29
  import type { CacheStorage } from './storage/index.js';
@@ -63,6 +64,7 @@ export interface OrchestratorSubsystems {
63
64
  pool: pg.Pool;
64
65
  providerRegistry: ProviderRegistry;
65
66
  agentRegistry: AgentRegistry;
67
+ hostRosterStore: HostRosterStore;
66
68
  dispatcher: Dispatcher;
67
69
  queue: JobQueue;
68
70
  scalerManager: ScalerManager | null;
@@ -216,6 +218,27 @@ export interface OrchestratorHooks {
216
218
  * are skipped here — group fan-in is resolved separately by the scheduler.
217
219
  */
218
220
  export declare function upstreamBaseNamesFromNeeds(needs: unknown): string[];
221
+ /**
222
+ * Partition a lock job's `runsOn` / `excludeLabels` matchers into exact label
223
+ * strings and regex patterns for internal-event (cron / `ctx.emit`) dispatch.
224
+ * Lock jobs carry `runsOn` as `LabelMatcher[]`; the coordinator routing and the
225
+ * direct dispatcher both need exact labels for the indexed/SQL fast path and
226
+ * regex patterns as a separate JS post-filter — never the raw matcher objects.
227
+ */
228
+ export declare function internalJobRunsOnSelectors(job: {
229
+ runsOn?: readonly LabelMatcher[];
230
+ excludeLabels?: readonly LabelMatcher[];
231
+ }): {
232
+ runsOnLabels: string[];
233
+ runsOnPatterns: LabelMatcher[];
234
+ excludeLabels: string[];
235
+ excludePatterns: LabelMatcher[];
236
+ };
237
+ /**
238
+ * Parse an `execution_jobs.outputs` cell (string JSON or object) to a plain
239
+ * object, or null when empty / unparseable.
240
+ */
241
+ export declare function parseOutputsCell(outputs: unknown): Record<string, unknown> | null;
219
242
  /**
220
243
  * Build the downstream `upstreamJobOutputs` map keyed by BASE name. A base name
221
244
  * that fanned into matrix children (rows with `matrix_values`) gets the
@@ -226,7 +249,29 @@ export declare function buildUpstreamOutputsByBase(baseNames: string[], rows: Ar
226
249
  job_name: string;
227
250
  outputs: unknown;
228
251
  matrix_values: unknown;
252
+ variant_kind?: string | null;
253
+ variant_label?: string | null;
254
+ status?: string | null;
229
255
  }>): Record<string, Record<string, unknown>> | undefined;
256
+ /**
257
+ * Fold a `runsOnAll` upstream's host children into the `byHost` envelope
258
+ * `{ byHost: { '<host>': outputs }, summary: { succeededHosts, failedHosts, outputs } }`.
259
+ * Unlike the matrix envelope, `summary.outputs[key]` is an array view across hosts
260
+ * (host order), never a last-write-wins scalar; `succeededHosts`/`failedHosts`
261
+ * record each host's terminal outcome.
262
+ */
263
+ export declare function buildHostOutputsEnvelope(children: Array<{
264
+ host: string;
265
+ status: string | null;
266
+ parsed: Record<string, unknown>;
267
+ }>): {
268
+ byHost: Record<string, Record<string, unknown>>;
269
+ summary: {
270
+ succeededHosts: string[];
271
+ failedHosts: string[];
272
+ outputs: Record<string, unknown[]>;
273
+ };
274
+ };
230
275
  /**
231
276
  * Group an upstream's child rows into the matrix outputs envelope
232
277
  * `{ byMatrix: { '<suffix>': outputs }, merged: <last-write-wins> }`. The suffix
@@ -14,7 +14,7 @@
14
14
  * results through the pipeline.
15
15
  */
16
16
  import { CacheRefScope } from '@kici-dev/engine';
17
- import type { LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
17
+ import type { LabelMatcher, LockWorkflow, SimulatedEvent, WorkflowDecision, MaterializedJob } from '@kici-dev/engine';
18
18
  import type { WebhookInfo } from '../webhook/handler.js';
19
19
  import type { ProviderBundle } from '../provider-registry.js';
20
20
  import type { TrustResolution } from '../security/trust-resolver.js';
@@ -109,6 +109,46 @@ export interface DispatchMatchedWorkflowOptions {
109
109
  */
110
110
  reuseRunId?: string;
111
111
  }
112
+ /** Exact labels + regex patterns partitioned from a lock job's selectors. */
113
+ interface JobRoutingSelectors {
114
+ runsOnLabels: string[];
115
+ runsOnPatterns: LabelMatcher[];
116
+ excludeLabels: string[];
117
+ excludePatterns: LabelMatcher[];
118
+ }
119
+ /**
120
+ * Partition a lock job's runsOn / excludeLabels matchers into exact labels (SQL
121
+ * `@>` prefilter + registry index) and regex patterns (JS post-filter). A
122
+ * `runsOnAll` host-fanout job has no `runsOn`; its pinned children carry no
123
+ * routing (the pin targets the resolved agent directly).
124
+ */
125
+ export declare function runsOnSelectorsForLockJob(lockJob: {
126
+ runsOn?: readonly LabelMatcher[];
127
+ excludeLabels?: readonly LabelMatcher[];
128
+ }): JobRoutingSelectors;
129
+ /** Per-child rolling-wave plan: which children are held + the base's wave policy. */
130
+ export interface WavePlan {
131
+ /** `expandedName`s held behind the wave gate (beyond the maxParallel window). */
132
+ held: Set<string>;
133
+ /** `expandedName` → the base's `{maxParallel, failFast}`, stamped on every child of a bounded wave. */
134
+ policy: Map<string, {
135
+ maxParallel: number;
136
+ failFast: boolean;
137
+ }>;
138
+ }
139
+ /**
140
+ * Compute the rolling-wave plan for a materialized job set.
141
+ *
142
+ * For each base job declaring `maxParallel` whose fan-out produced more than one
143
+ * child, children are ordered deterministically by `variant_label` (the matrix
144
+ * suffix / hostname, via `expandedName`) and every child at index `>=
145
+ * maxParallel` is held (`wave_gated=true`). The first `maxParallel` dispatch
146
+ * immediately; held children release one-per-terminal via the wave-scheduler.
147
+ * Every child of a bounded-wave base — held or not — gets a `policy` entry so
148
+ * the wave-scheduler can read the width/failFast at terminal time. A non-fan-out
149
+ * job (single child) or one without `maxParallel` contributes nothing.
150
+ */
151
+ export declare function computeWavePlan(materializedJobs: readonly MaterializedJob[]): WavePlan;
112
152
  /**
113
153
  * Dispatch a single matched workflow.
114
154
  *
@@ -125,4 +165,5 @@ export interface DispatchMatchedWorkflowOptions {
125
165
  * J. deferred dynamic dispatch (fire-and-forget per dynamic entry)
126
166
  */
127
167
  export declare function dispatchMatchedWorkflow(ctx: WorkflowDispatchContext, opts?: DispatchMatchedWorkflowOptions): Promise<DispatchMatchedWorkflowResult>;
168
+ export {};
128
169
  //# sourceMappingURL=dispatch-matched-workflow.d.ts.map
@@ -29,6 +29,7 @@ import type { PendingDynamicTracker } from '../cache/pending-dynamics.js';
29
29
  import type { CheckRunReporter } from '../reporting/check-run-reporter.js';
30
30
  import type { ExecutionTracker } from '../reporting/execution-tracker.js';
31
31
  import type { AgentRegistry } from '../agent/registry.js';
32
+ import type { HostRosterStore } from '../agent/host-roster.js';
32
33
  import type { RunCoordinator } from '../cluster/coordinator.js';
33
34
  import type { TeamMembershipLookup } from '../approvals/team-membership-lookup.js';
34
35
  import type { LogStorage } from '../reporting/log-storage.js';
@@ -62,6 +63,21 @@ interface PendingJobContext {
62
63
  jobInput: QueuedJobInput;
63
64
  runsOnLabels: string[];
64
65
  }
66
+ /**
67
+ * Register an eval gate and return a promise that resolves when the scheduler
68
+ * opens it (the eval job's upstream needs are all satisfied).
69
+ */
70
+ export declare function trackEvalGate(runId: string, evalJobName: string): Promise<void>;
71
+ /**
72
+ * Open a registered eval gate, unblocking the deferred dispatch task. Returns
73
+ * true if a gate was registered for this eval job (so the scheduler knows it
74
+ * handled the ready signal itself and must not run the normal dispatch path).
75
+ */
76
+ export declare function openEvalGate(runId: string, evalJobName: string): boolean;
77
+ /** True when a job name is a result-aware dynamic eval job awaiting its gate. */
78
+ export declare function isEvalGatePending(runId: string, evalJobName: string): boolean;
79
+ /** Clear all eval gates for a run (called on run completion / cleanup). */
80
+ export declare function clearEvalGatesForRun(runId: string): void;
65
81
  /**
66
82
  * Store a pending dispatch context for a job that will be dispatched later
67
83
  * by the needs scheduler. The key is `${runId}:${jobName}`.
@@ -320,6 +336,15 @@ export interface ProcessingDeps {
320
336
  /** Access-log writer for the orchestrator audit stream. Optional -- if not
321
337
  * set, hold-creation audit rows (`held_run.request`) are skipped. */
322
338
  accessLogWriter?: AccessLogWriter;
339
+ /** Host roster store for runsOnAll fan-out resolution. Optional -- if not set,
340
+ * runsOnAll jobs cannot be resolved and fail at materialize. */
341
+ hostRosterStore?: HostRosterStore;
342
+ /** This orchestrator instance id (for the cross-cluster host-fanout pin reroute). */
343
+ instanceId?: string;
344
+ /** Static-host grace before a disconnected static host reads unreachable (ms). */
345
+ rosterGraceMs?: number;
346
+ /** Cap on runsOnAll per-host children (default 1024). */
347
+ maxFanoutHosts?: number;
323
348
  }
324
349
  /**
325
350
  * Check if any trigger in the lock file workflows uses path filters.
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Rolling-wave scheduler for bounded fan-out (`maxParallel` / `failFast`).
3
+ *
4
+ * Sibling of the needs-scheduler (`needs-scheduler.ts`): both fire on a job
5
+ * reaching terminal state and decide what to do with held downstream/sibling
6
+ * jobs. The needs-scheduler releases jobs whose `needs` edges are now satisfied;
7
+ * the wave-scheduler releases the next `wave_gated` sibling of a fan-out base
8
+ * whenever an in-flight slot frees up (or, under `failFast`, skips the held
9
+ * remainder on the first child failure).
10
+ *
11
+ * Pure DB — no in-memory state. Every decision is a fresh query against
12
+ * `execution_jobs` keyed by `(run_id, base_job_name)`, so the scheduler needs
13
+ * zero recovery code on orchestrator restart. The caller performs the DB write
14
+ * (clear `wave_gated` / mark skipped) and the dispatch (`onJobReady`), keeping
15
+ * this module a pure decision function.
16
+ */
17
+ import type { Kysely } from 'kysely';
18
+ import type { Database } from '../db/types.js';
19
+ /** Inputs identifying a completed fan-out child. The wave policy is read from the base group. */
20
+ export interface WaveEvaluation {
21
+ runId: string;
22
+ /** The base (logical) job name shared by every fan-out child. */
23
+ baseJobName: string;
24
+ /** Terminal status of the child that just completed. */
25
+ completedStatus: string;
26
+ }
27
+ /** The wave-scheduler's decision for one completed child. */
28
+ export type WaveResult = {
29
+ action: 'release';
30
+ jobName: string;
31
+ baseJobName: string;
32
+ maxParallel: number;
33
+ failFast: boolean;
34
+ } | {
35
+ action: 'skip-remaining';
36
+ jobNames: string[];
37
+ } | {
38
+ action: 'noop';
39
+ };
40
+ /**
41
+ * Decide what happens after a fan-out child of `baseJobName` reaches terminal.
42
+ *
43
+ * The wave policy (`maxParallel` / `failFast`) is read from the base group's
44
+ * own rows — every child of a bounded wave carries the same stamped
45
+ * `wave_max_parallel` / `wave_fail_fast`, so the just-completed child's slot
46
+ * being re-inserted without the policy on release does not break the chain
47
+ * (the still-held siblings carry it). If no sibling carries a policy, this is
48
+ * not a bounded wave → `noop`.
49
+ *
50
+ * - `failFast` + a child failure → `skip-remaining` every still-held sibling.
51
+ * - in-flight count `< maxParallel` AND a held sibling exists → `release` the
52
+ * next held sibling (lowest `variant_label`).
53
+ * - otherwise → `noop`.
54
+ *
55
+ * "In-flight" = a non-terminal, non-`wave_gated` child (it has been dispatched
56
+ * and not yet completed). The just-completed child is terminal, so it does not
57
+ * count against the window — its slot is the one we are filling.
58
+ */
59
+ export declare function evaluateWave(db: Kysely<Database>, evaluation: WaveEvaluation): Promise<WaveResult>;
60
+ //# sourceMappingURL=wave-scheduler.d.ts.map