@kici-dev/orchestrator 0.1.17 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +9 -0
- package/dist/agent/host-roster-reaper.d.ts +42 -0
- package/dist/agent/host-roster.d.ts +110 -0
- package/dist/agent/registry.d.ts +65 -2
- package/dist/app.d.ts +3 -0
- package/dist/cli/commands/host.d.ts +13 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +23 -0
- package/dist/cli/service/compose.d.ts +2 -1
- package/dist/cli/service/index.d.ts +2 -2
- package/dist/cli/service/instance/manifest.d.ts +12 -0
- package/dist/cli/service/instance/resolve.d.ts +12 -2
- package/dist/cli/service/launchd.d.ts +4 -1
- package/dist/cli/service/platform-detect.d.ts +5 -0
- package/dist/cli/service/systemd.d.ts +2 -1
- package/dist/cli/service/types.d.ts +37 -0
- package/dist/cli/service/windows.d.ts +2 -1
- package/dist/cli.js +794 -201
- package/dist/cluster/coordinator.d.ts +5 -1
- package/dist/config/schema.d.ts +4 -0
- package/dist/config/types.d.ts +9 -0
- package/dist/config.d.ts +6 -0
- package/dist/db/migrations/039_host_roster.d.ts +19 -0
- package/dist/db/migrations/040_runsonall_pin.d.ts +4 -0
- package/dist/db/migrations/041_wave_gated.d.ts +4 -0
- package/dist/db/migrations/042_dispatch_queue_patterns.d.ts +4 -0
- package/dist/db/types.d.ts +61 -0
- package/dist/diagnostics/fleet-collector.d.ts +1 -1
- package/dist/environments/held-runs.d.ts +9 -0
- package/dist/lockfile-redos-guard.d.ts +19 -0
- package/dist/metrics/prometheus.d.ts +13 -0
- package/dist/metrics/scheduled-jobs.d.ts +2 -2
- package/dist/orchestrator-core.d.ts +46 -1
- package/dist/pipeline/dispatch-matched-workflow.d.ts +42 -1
- package/dist/pipeline/processor.d.ts +25 -0
- package/dist/pipeline/wave-scheduler.d.ts +60 -0
- package/dist/queue/job-queue.d.ts +61 -2
- package/dist/reporting/execution-tracker.d.ts +28 -0
- package/dist/server.js +52720 -51280
- package/dist/standalone.js +1908 -488
- package/installer-image-digests.json +3 -3
- package/package.json +22 -22
- package/sbom.spdx.json +3227 -6682
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* - "Peers report step-by-step progress back to coordinator"
|
|
11
11
|
* - "Cancel mode: graceful -- finish current step, cancel remaining"
|
|
12
12
|
*/
|
|
13
|
-
import type { JobReroute, JobProgress, PeerScalerEvent, PeerToPeerMessage, ResourceRequest } from '@kici-dev/engine';
|
|
13
|
+
import type { JobReroute, JobProgress, PeerScalerEvent, PeerToPeerMessage, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
|
|
14
14
|
import type { PeerRegistry } from './peer-registry.js';
|
|
15
15
|
import type { PeerClient } from './peer-client.js';
|
|
16
16
|
import type { Dispatcher } from '../agent/dispatcher.js';
|
|
@@ -37,6 +37,10 @@ export interface RunContext {
|
|
|
37
37
|
export interface JobToRoute {
|
|
38
38
|
jobName: string;
|
|
39
39
|
runsOnLabels: string[][];
|
|
40
|
+
/** Regex matchers the agent's labels must satisfy (JS post-filter). */
|
|
41
|
+
runsOnPatterns?: LabelMatcher[];
|
|
42
|
+
/** Regex matchers that disqualify an agent (JS post-filter). */
|
|
43
|
+
excludePatterns?: LabelMatcher[];
|
|
40
44
|
jobConfig: Record<string, unknown>;
|
|
41
45
|
repoUrl: string;
|
|
42
46
|
ref: string;
|
package/dist/config/schema.d.ts
CHANGED
|
@@ -69,6 +69,8 @@ export declare const sharedConfigSchema: z.ZodObject<{
|
|
|
69
69
|
none: "none";
|
|
70
70
|
}>>;
|
|
71
71
|
agentTokenTtlMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
72
|
+
rosterGraceMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
73
|
+
rosterTtlMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
72
74
|
queue: z.ZodOptional<z.ZodObject<{
|
|
73
75
|
maxDepth: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
74
76
|
timeoutMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
|
|
@@ -156,6 +158,8 @@ export declare const appConfigSchema: z.ZodObject<{
|
|
|
156
158
|
none: "none";
|
|
157
159
|
}>>;
|
|
158
160
|
agentTokenTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
161
|
+
rosterGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
162
|
+
rosterTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
159
163
|
queueMaxDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
160
164
|
queueTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
161
165
|
queueBackpressureThreshold: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
package/dist/config/types.d.ts
CHANGED
|
@@ -50,6 +50,9 @@ export interface SharedConfig {
|
|
|
50
50
|
};
|
|
51
51
|
agentAuth?: 'token' | 'none';
|
|
52
52
|
agentTokenTtlMs?: number;
|
|
53
|
+
rosterGraceMs?: number;
|
|
54
|
+
rosterTtlMs?: number;
|
|
55
|
+
maxFanoutHosts?: number;
|
|
53
56
|
queue?: {
|
|
54
57
|
maxDepth?: number;
|
|
55
58
|
timeoutMs?: number;
|
|
@@ -135,6 +138,12 @@ export interface AppConfig {
|
|
|
135
138
|
agentAuth: 'token' | 'none';
|
|
136
139
|
/** Agent token TTL in milliseconds */
|
|
137
140
|
agentTokenTtlMs: number;
|
|
141
|
+
/** Host roster: static grace before a disconnected static host reads unreachable (ms) */
|
|
142
|
+
rosterGraceMs: number;
|
|
143
|
+
/** Host roster: ephemeral GC ttl — past this a disconnected ephemeral host is reaped (ms) */
|
|
144
|
+
rosterTtlMs: number;
|
|
145
|
+
/** Cap on per-host children produced by a runsOnAll fan-out */
|
|
146
|
+
maxFanoutHosts: number;
|
|
138
147
|
/** Queue settings */
|
|
139
148
|
queueMaxDepth: number;
|
|
140
149
|
queueTimeoutMs: number;
|
package/dist/config.d.ts
CHANGED
|
@@ -82,6 +82,9 @@ declare const configSchema: z.ZodObject<{
|
|
|
82
82
|
none: "none";
|
|
83
83
|
}>>;
|
|
84
84
|
agentTokenTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
85
|
+
rosterGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
86
|
+
rosterTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
87
|
+
maxFanoutHosts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
85
88
|
eventRouterMaxChainDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
86
89
|
eventRouterRateLimitPerWorkflowPerMinute: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
87
90
|
eventRouterEventTtlSeconds: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -197,6 +200,9 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
|
|
|
197
200
|
pgCustomerSecrets: boolean;
|
|
198
201
|
agentAuth: "token" | "none";
|
|
199
202
|
agentTokenTtlMs: number;
|
|
203
|
+
rosterGraceMs: number;
|
|
204
|
+
rosterTtlMs: number;
|
|
205
|
+
maxFanoutHosts: number;
|
|
200
206
|
eventRouterMaxChainDepth: number;
|
|
201
207
|
eventRouterRateLimitPerWorkflowPerMinute: number;
|
|
202
208
|
eventRouterEventTtlSeconds: number;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { type Kysely } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* `host_roster` is KiCI's declared inventory: one durable row per agent the
|
|
4
|
+
* cluster has ever enrolled, reconciled from the in-memory AgentRegistry on
|
|
5
|
+
* every register/unregister. `lifecycle_class` (snapshot of the auth token's
|
|
6
|
+
* agent_type) drives reaping — `ephemeral` rows are GC'd past their TTL,
|
|
7
|
+
* `static` rows persist and read as `unreachable` when their heartbeat goes
|
|
8
|
+
* stale. `connected_instance_id` records which orchestrator holds the live WS
|
|
9
|
+
* (cluster liveness + the host-fanout reroute target); NULL = disconnected.
|
|
10
|
+
*
|
|
11
|
+
* The roster lives in the shared cluster DB (one table, all instances). Status
|
|
12
|
+
* is derived at read from the shared `last_seen` + `connected_instance_id`, so
|
|
13
|
+
* every instance agrees regardless of which one holds the agent's live WS.
|
|
14
|
+
*
|
|
15
|
+
* Idempotent: a re-run on a DB that already has the table is a no-op.
|
|
16
|
+
*/
|
|
17
|
+
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
18
|
+
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
19
|
+
//# sourceMappingURL=039_host_roster.d.ts.map
|
package/dist/db/types.d.ts
CHANGED
|
@@ -50,6 +50,7 @@ export interface Database {
|
|
|
50
50
|
scaler_reservations: ScalerReservationsTable;
|
|
51
51
|
attestations: AttestationsTable;
|
|
52
52
|
remote_sources: RemoteSourcesTable;
|
|
53
|
+
host_roster: HostRosterTable;
|
|
53
54
|
}
|
|
54
55
|
/**
|
|
55
56
|
* Cluster metadata table
|
|
@@ -138,6 +139,10 @@ export interface DispatchQueueTable {
|
|
|
138
139
|
request_id: string | null;
|
|
139
140
|
/** JSONB array of exclusion labels. Default '[]'. */
|
|
140
141
|
exclude_labels: Generated<string>;
|
|
142
|
+
/** Regex matchers (LabelMatcher[]) the job requires; JS post-filter on top of runs_on_labels. Default '[]'. */
|
|
143
|
+
runs_on_patterns: Generated<string>;
|
|
144
|
+
/** Regex matchers (LabelMatcher[]) that disqualify an agent; JS post-filter. Default '[]'. */
|
|
145
|
+
exclude_patterns: Generated<string>;
|
|
141
146
|
/** Routing key (e.g. "github:12345") so dispatch can pick the right
|
|
142
147
|
* per-app provider bundle in multi-app setups. Required (NOT NULL). */
|
|
143
148
|
routing_key: string;
|
|
@@ -173,6 +178,12 @@ export interface DispatchQueueTable {
|
|
|
173
178
|
ack_deadline: ColumnType<Date | null, Date | null | undefined, Date | null>;
|
|
174
179
|
/** Agent the dispatch was sent to (for ack-timeout disconnect + logging). */
|
|
175
180
|
ack_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
|
|
181
|
+
/**
|
|
182
|
+
* For a runsOnAll host-fanout child: the agent this job is pinned to. The
|
|
183
|
+
* dispatcher routes it only to that agent and the queue drain never hands it
|
|
184
|
+
* to another. NULL for normal label-routed jobs.
|
|
185
|
+
*/
|
|
186
|
+
pinned_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
|
|
176
187
|
}
|
|
177
188
|
/**
|
|
178
189
|
* Deduplication cache table
|
|
@@ -354,6 +365,23 @@ export interface ExecutionJobTable {
|
|
|
354
365
|
ready_at: Date | null;
|
|
355
366
|
/** Dynamic group membership tag (NULL for static jobs). */
|
|
356
367
|
group_name: string | null;
|
|
368
|
+
/** Base (logical) job name for a fan-out child. NULL for non-fanned jobs. */
|
|
369
|
+
base_job_name: string | null;
|
|
370
|
+
/** Fan-out kind for a child: 'matrix' | 'host'. NULL for non-fanned jobs. */
|
|
371
|
+
variant_kind: string | null;
|
|
372
|
+
/** Fan-out label for a child: matrix suffix or hostname. NULL for non-fanned jobs. */
|
|
373
|
+
variant_label: string | null;
|
|
374
|
+
/**
|
|
375
|
+
* Wave gate: a fan-out child beyond the job's `maxParallel` window is held
|
|
376
|
+
* (`true`) instead of dispatched. Cleared by the wave-scheduler when a sibling
|
|
377
|
+
* reaches terminal and an in-flight slot frees up. NULL/false for any job not
|
|
378
|
+
* held by a rolling wave.
|
|
379
|
+
*/
|
|
380
|
+
wave_gated: Generated<boolean>;
|
|
381
|
+
/** The fan-out base's `maxParallel` wave width, stamped on every child. NULL = no bounded wave. */
|
|
382
|
+
wave_max_parallel: number | null;
|
|
383
|
+
/** The fan-out base's `failFast` policy, stamped on every child. NULL = no bounded wave. */
|
|
384
|
+
wave_fail_fast: boolean | null;
|
|
357
385
|
/** When this record was created */
|
|
358
386
|
created_at: Generated<Date>;
|
|
359
387
|
/**
|
|
@@ -1529,6 +1557,39 @@ export interface RemoteSourcesTable {
|
|
|
1529
1557
|
}
|
|
1530
1558
|
export type RemoteSourceRow = Selectable<RemoteSourcesTable>;
|
|
1531
1559
|
export type NewRemoteSourceRow = Insertable<RemoteSourcesTable>;
|
|
1560
|
+
/**
|
|
1561
|
+
* Host roster table (host_roster).
|
|
1562
|
+
*
|
|
1563
|
+
* KiCI's declared inventory: one durable row per agent the cluster has ever
|
|
1564
|
+
* enrolled, reconciled from the in-memory AgentRegistry on register/unregister.
|
|
1565
|
+
* `lifecycle_class` (snapshot of the auth token's `agent_type`) drives reaping;
|
|
1566
|
+
* `connected_instance_id` records which orchestrator holds the live WS (cluster
|
|
1567
|
+
* liveness + the host-fanout reroute target), null when disconnected. Status is
|
|
1568
|
+
* derived at read from the shared `last_seen` + `connected_instance_id`.
|
|
1569
|
+
*/
|
|
1570
|
+
export interface HostRosterTable {
|
|
1571
|
+
/** UUID primary key */
|
|
1572
|
+
id: Generated<string>;
|
|
1573
|
+
/** The agent identity the pin targets; unique. */
|
|
1574
|
+
agent_id: string;
|
|
1575
|
+
/** FK to agent_tokens.id (provenance), or null when auth mode is none. */
|
|
1576
|
+
token_id: string | null;
|
|
1577
|
+
/** Snapshot of the token's agent_type: 'static' | 'ephemeral'. */
|
|
1578
|
+
lifecycle_class: string;
|
|
1579
|
+
/** JSON-encoded string[] of the post-Gate-1 validated labels. */
|
|
1580
|
+
labels: string;
|
|
1581
|
+
hostname: string | null;
|
|
1582
|
+
platform: string | null;
|
|
1583
|
+
arch: string | null;
|
|
1584
|
+
/** Which orchestrator instance holds the live WS; null = disconnected. */
|
|
1585
|
+
connected_instance_id: string | null;
|
|
1586
|
+
last_seen: ColumnType<Date, Date | string | undefined, Date | string>;
|
|
1587
|
+
created_at: Generated<Date>;
|
|
1588
|
+
updated_at: ColumnType<Date, Date | string | undefined, Date | string>;
|
|
1589
|
+
}
|
|
1590
|
+
export type HostRosterRow = Selectable<HostRosterTable>;
|
|
1591
|
+
export type NewHostRosterRow = Insertable<HostRosterTable>;
|
|
1592
|
+
export type HostRosterUpdate = Updateable<HostRosterTable>;
|
|
1532
1593
|
/**
|
|
1533
1594
|
* Scaler spawning-agents table (scaler_spawning_agents).
|
|
1534
1595
|
*
|
|
@@ -12,8 +12,8 @@ import { z } from 'zod';
|
|
|
12
12
|
/** Per-node collection outcome recorded in the fleet manifest. */
|
|
13
13
|
export declare const FleetNodeStatus: z.ZodEnum<{
|
|
14
14
|
error: "error";
|
|
15
|
-
ok: "ok";
|
|
16
15
|
unreachable: "unreachable";
|
|
16
|
+
ok: "ok";
|
|
17
17
|
timeout: "timeout";
|
|
18
18
|
}>;
|
|
19
19
|
export type FleetNodeStatus = z.infer<typeof FleetNodeStatus>;
|
|
@@ -71,6 +71,15 @@ export interface ReleaseSignal {
|
|
|
71
71
|
scope: HoldScope;
|
|
72
72
|
/** Set only for step-scoped holds. */
|
|
73
73
|
stepIndex: number | null;
|
|
74
|
+
/**
|
|
75
|
+
* What kind of gate created the hold. `explicit` (SDK `requireApproval`) holds
|
|
76
|
+
* a real root job and resumes by re-dispatching it; `environment` covers the
|
|
77
|
+
* workflow install-gate (wait-timer / concurrency / env approval) which resumes
|
|
78
|
+
* by rebuilding the workflow dispatch context. The resume router keys off this
|
|
79
|
+
* so a workflow-scoped explicit hold goes through the job re-dispatch path
|
|
80
|
+
* rather than the install-gate path (which has no pending workflow context).
|
|
81
|
+
*/
|
|
82
|
+
triggerSource: TriggerSource;
|
|
74
83
|
}
|
|
75
84
|
/** Options for listing held runs. */
|
|
76
85
|
export interface ListHeldRunsOptions {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lock-load ReDoS revalidation.
|
|
3
|
+
*
|
|
4
|
+
* Re-validates every regex matcher in a fetched lock file before it is cached
|
|
5
|
+
* or dispatched. Defense-in-depth against a hand-edited or non-compiled lock
|
|
6
|
+
* that smuggled a ReDoS-prone pattern past the compile-time gate: the compiler
|
|
7
|
+
* runs the same `assertMatchersSafe` check when it emits the lock, but the
|
|
8
|
+
* orchestrator does not trust that the lock it fetched was produced by our
|
|
9
|
+
* compiler.
|
|
10
|
+
*/
|
|
11
|
+
import type { LockFile } from '@kici-dev/engine';
|
|
12
|
+
/**
|
|
13
|
+
* Walk every static job's `runsOn` / `excludeLabels` / `runsOnAll` matchers and
|
|
14
|
+
* throw if any regex matcher is ReDoS-prone. Dynamic job generators carry no
|
|
15
|
+
* static routing matchers (they materialize jobs at eval time, which re-runs the
|
|
16
|
+
* compile-time gate), so only static jobs are checked.
|
|
17
|
+
*/
|
|
18
|
+
export declare function assertLockFileRegexesSafe(lockFile: LockFile): void;
|
|
19
|
+
//# sourceMappingURL=lockfile-redos-guard.d.ts.map
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
export declare function setAgentsActive(value: number): void;
|
|
3
3
|
/** Set the current config version number. */
|
|
4
4
|
export declare function setConfigVersion(value: number): void;
|
|
5
|
+
/** Set the current number of declared (static) roster hosts that are unreachable. */
|
|
6
|
+
export declare function setDeclaredHostsUnreachable(value: number): void;
|
|
5
7
|
/** Set the current number of stale runs detected. */
|
|
6
8
|
export declare function setStaleRunsCurrent(value: number): void;
|
|
7
9
|
interface ScalerUsageRow {
|
|
@@ -322,5 +324,16 @@ export declare const installSecretsContributorStrippedTotal: import("@openteleme
|
|
|
322
324
|
* - environment: the environment name referenced in the qualified `<environment>:<secret>` ref (per-org count is typically <10)
|
|
323
325
|
*/
|
|
324
326
|
export declare const installSecretsTokenResolutionDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
|
|
327
|
+
/**
|
|
328
|
+
* Register every queued orchestrator observable gauge on the real meter.
|
|
329
|
+
*
|
|
330
|
+
* Must run AFTER `initTelemetry()` has wired the global MeterProvider —
|
|
331
|
+
* `createApp()` calls it once during bootstrap. Registering the gauges at
|
|
332
|
+
* module-eval time instead would bind them to the no-op provider (the
|
|
333
|
+
* bundler hoists some module init above the entry's `initTelemetry()` call),
|
|
334
|
+
* leaving every `kici_orch_*` gauge absent from the /metrics scrape and the
|
|
335
|
+
* Platform push. Idempotent: repeat calls are no-ops.
|
|
336
|
+
*/
|
|
337
|
+
export declare function registerOrchestratorMetrics(): void;
|
|
325
338
|
export {};
|
|
326
339
|
//# sourceMappingURL=prometheus.d.ts.map
|
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
* - job: scheduled-job name (one of `OrchestratorScheduledJobName`)
|
|
5
5
|
* - result: success | failure
|
|
6
6
|
*/
|
|
7
|
-
export declare const jobRunsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
|
|
7
|
+
export declare const jobRunsTotal: Pick<import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>, "add">;
|
|
8
8
|
/**
|
|
9
9
|
* Histogram of per-tick duration, seconds.
|
|
10
10
|
* Labels:
|
|
11
11
|
* - job: scheduled-job name (one of `OrchestratorScheduledJobName`)
|
|
12
12
|
*/
|
|
13
|
-
export declare const jobDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
|
|
13
|
+
export declare const jobDurationSeconds: Pick<import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>, "record">;
|
|
14
14
|
/**
|
|
15
15
|
* Unix timestamp (seconds) of the most recent successful tick.
|
|
16
16
|
* Labels:
|
|
@@ -13,6 +13,7 @@ import { type ColdStore } from '@kici-dev/shared';
|
|
|
13
13
|
import type { AppConfig } from './config.js';
|
|
14
14
|
import { ConfigReloader } from './config/reload.js';
|
|
15
15
|
import { AgentRegistry } from './agent/registry.js';
|
|
16
|
+
import { HostRosterStore } from './agent/host-roster.js';
|
|
16
17
|
import { JobQueue } from './queue/job-queue.js';
|
|
17
18
|
import { EventLogWriter } from './webhook/event-log.js';
|
|
18
19
|
import { AccessLogWriter } from './audit/access-log.js';
|
|
@@ -22,7 +23,7 @@ import { DedupCache } from './webhook/dedup.js';
|
|
|
22
23
|
import { ObserverRegistry } from './ws/observer-registry.js';
|
|
23
24
|
import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
|
|
24
25
|
import { SourceLocationStore } from './app.js';
|
|
25
|
-
import { type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
|
|
26
|
+
import { type LabelMatcher, type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
|
|
26
27
|
import { ScalerManager } from './scaler/index.js';
|
|
27
28
|
import type { ScalerConfig } from './scaler/index.js';
|
|
28
29
|
import type { CacheStorage } from './storage/index.js';
|
|
@@ -63,6 +64,7 @@ export interface OrchestratorSubsystems {
|
|
|
63
64
|
pool: pg.Pool;
|
|
64
65
|
providerRegistry: ProviderRegistry;
|
|
65
66
|
agentRegistry: AgentRegistry;
|
|
67
|
+
hostRosterStore: HostRosterStore;
|
|
66
68
|
dispatcher: Dispatcher;
|
|
67
69
|
queue: JobQueue;
|
|
68
70
|
scalerManager: ScalerManager | null;
|
|
@@ -216,6 +218,27 @@ export interface OrchestratorHooks {
|
|
|
216
218
|
* are skipped here — group fan-in is resolved separately by the scheduler.
|
|
217
219
|
*/
|
|
218
220
|
export declare function upstreamBaseNamesFromNeeds(needs: unknown): string[];
|
|
221
|
+
/**
|
|
222
|
+
* Partition a lock job's `runsOn` / `excludeLabels` matchers into exact label
|
|
223
|
+
* strings and regex patterns for internal-event (cron / `ctx.emit`) dispatch.
|
|
224
|
+
* Lock jobs carry `runsOn` as `LabelMatcher[]`; the coordinator routing and the
|
|
225
|
+
* direct dispatcher both need exact labels for the indexed/SQL fast path and
|
|
226
|
+
* regex patterns as a separate JS post-filter — never the raw matcher objects.
|
|
227
|
+
*/
|
|
228
|
+
export declare function internalJobRunsOnSelectors(job: {
|
|
229
|
+
runsOn?: readonly LabelMatcher[];
|
|
230
|
+
excludeLabels?: readonly LabelMatcher[];
|
|
231
|
+
}): {
|
|
232
|
+
runsOnLabels: string[];
|
|
233
|
+
runsOnPatterns: LabelMatcher[];
|
|
234
|
+
excludeLabels: string[];
|
|
235
|
+
excludePatterns: LabelMatcher[];
|
|
236
|
+
};
|
|
237
|
+
/**
|
|
238
|
+
* Parse an `execution_jobs.outputs` cell (string JSON or object) to a plain
|
|
239
|
+
* object, or null when empty / unparseable.
|
|
240
|
+
*/
|
|
241
|
+
export declare function parseOutputsCell(outputs: unknown): Record<string, unknown> | null;
|
|
219
242
|
/**
|
|
220
243
|
* Build the downstream `upstreamJobOutputs` map keyed by BASE name. A base name
|
|
221
244
|
* that fanned into matrix children (rows with `matrix_values`) gets the
|
|
@@ -226,7 +249,29 @@ export declare function buildUpstreamOutputsByBase(baseNames: string[], rows: Ar
|
|
|
226
249
|
job_name: string;
|
|
227
250
|
outputs: unknown;
|
|
228
251
|
matrix_values: unknown;
|
|
252
|
+
variant_kind?: string | null;
|
|
253
|
+
variant_label?: string | null;
|
|
254
|
+
status?: string | null;
|
|
229
255
|
}>): Record<string, Record<string, unknown>> | undefined;
|
|
256
|
+
/**
|
|
257
|
+
* Fold a `runsOnAll` upstream's host children into the `byHost` envelope
|
|
258
|
+
* `{ byHost: { '<host>': outputs }, summary: { succeededHosts, failedHosts, outputs } }`.
|
|
259
|
+
* Unlike the matrix envelope, `summary.outputs[key]` is an array view across hosts
|
|
260
|
+
* (host order), never a last-write-wins scalar; `succeededHosts`/`failedHosts`
|
|
261
|
+
* record each host's terminal outcome.
|
|
262
|
+
*/
|
|
263
|
+
export declare function buildHostOutputsEnvelope(children: Array<{
|
|
264
|
+
host: string;
|
|
265
|
+
status: string | null;
|
|
266
|
+
parsed: Record<string, unknown>;
|
|
267
|
+
}>): {
|
|
268
|
+
byHost: Record<string, Record<string, unknown>>;
|
|
269
|
+
summary: {
|
|
270
|
+
succeededHosts: string[];
|
|
271
|
+
failedHosts: string[];
|
|
272
|
+
outputs: Record<string, unknown[]>;
|
|
273
|
+
};
|
|
274
|
+
};
|
|
230
275
|
/**
|
|
231
276
|
* Group an upstream's child rows into the matrix outputs envelope
|
|
232
277
|
* `{ byMatrix: { '<suffix>': outputs }, merged: <last-write-wins> }`. The suffix
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* results through the pipeline.
|
|
15
15
|
*/
|
|
16
16
|
import { CacheRefScope } from '@kici-dev/engine';
|
|
17
|
-
import type { LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
|
|
17
|
+
import type { LabelMatcher, LockWorkflow, SimulatedEvent, WorkflowDecision, MaterializedJob } from '@kici-dev/engine';
|
|
18
18
|
import type { WebhookInfo } from '../webhook/handler.js';
|
|
19
19
|
import type { ProviderBundle } from '../provider-registry.js';
|
|
20
20
|
import type { TrustResolution } from '../security/trust-resolver.js';
|
|
@@ -109,6 +109,46 @@ export interface DispatchMatchedWorkflowOptions {
|
|
|
109
109
|
*/
|
|
110
110
|
reuseRunId?: string;
|
|
111
111
|
}
|
|
112
|
+
/** Exact labels + regex patterns partitioned from a lock job's selectors. */
|
|
113
|
+
interface JobRoutingSelectors {
|
|
114
|
+
runsOnLabels: string[];
|
|
115
|
+
runsOnPatterns: LabelMatcher[];
|
|
116
|
+
excludeLabels: string[];
|
|
117
|
+
excludePatterns: LabelMatcher[];
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Partition a lock job's runsOn / excludeLabels matchers into exact labels (SQL
|
|
121
|
+
* `@>` prefilter + registry index) and regex patterns (JS post-filter). A
|
|
122
|
+
* `runsOnAll` host-fanout job has no `runsOn`; its pinned children carry no
|
|
123
|
+
* routing (the pin targets the resolved agent directly).
|
|
124
|
+
*/
|
|
125
|
+
export declare function runsOnSelectorsForLockJob(lockJob: {
|
|
126
|
+
runsOn?: readonly LabelMatcher[];
|
|
127
|
+
excludeLabels?: readonly LabelMatcher[];
|
|
128
|
+
}): JobRoutingSelectors;
|
|
129
|
+
/** Per-child rolling-wave plan: which children are held + the base's wave policy. */
|
|
130
|
+
export interface WavePlan {
|
|
131
|
+
/** `expandedName`s held behind the wave gate (beyond the maxParallel window). */
|
|
132
|
+
held: Set<string>;
|
|
133
|
+
/** `expandedName` → the base's `{maxParallel, failFast}`, stamped on every child of a bounded wave. */
|
|
134
|
+
policy: Map<string, {
|
|
135
|
+
maxParallel: number;
|
|
136
|
+
failFast: boolean;
|
|
137
|
+
}>;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Compute the rolling-wave plan for a materialized job set.
|
|
141
|
+
*
|
|
142
|
+
* For each base job declaring `maxParallel` whose fan-out produced more than one
|
|
143
|
+
* child, children are ordered deterministically by `variant_label` (the matrix
|
|
144
|
+
* suffix / hostname, via `expandedName`) and every child at index `>=
|
|
145
|
+
* maxParallel` is held (`wave_gated=true`). The first `maxParallel` dispatch
|
|
146
|
+
* immediately; held children release one-per-terminal via the wave-scheduler.
|
|
147
|
+
* Every child of a bounded-wave base — held or not — gets a `policy` entry so
|
|
148
|
+
* the wave-scheduler can read the width/failFast at terminal time. A non-fan-out
|
|
149
|
+
* job (single child) or one without `maxParallel` contributes nothing.
|
|
150
|
+
*/
|
|
151
|
+
export declare function computeWavePlan(materializedJobs: readonly MaterializedJob[]): WavePlan;
|
|
112
152
|
/**
|
|
113
153
|
* Dispatch a single matched workflow.
|
|
114
154
|
*
|
|
@@ -125,4 +165,5 @@ export interface DispatchMatchedWorkflowOptions {
|
|
|
125
165
|
* J. deferred dynamic dispatch (fire-and-forget per dynamic entry)
|
|
126
166
|
*/
|
|
127
167
|
export declare function dispatchMatchedWorkflow(ctx: WorkflowDispatchContext, opts?: DispatchMatchedWorkflowOptions): Promise<DispatchMatchedWorkflowResult>;
|
|
168
|
+
export {};
|
|
128
169
|
//# sourceMappingURL=dispatch-matched-workflow.d.ts.map
|
|
@@ -29,6 +29,7 @@ import type { PendingDynamicTracker } from '../cache/pending-dynamics.js';
|
|
|
29
29
|
import type { CheckRunReporter } from '../reporting/check-run-reporter.js';
|
|
30
30
|
import type { ExecutionTracker } from '../reporting/execution-tracker.js';
|
|
31
31
|
import type { AgentRegistry } from '../agent/registry.js';
|
|
32
|
+
import type { HostRosterStore } from '../agent/host-roster.js';
|
|
32
33
|
import type { RunCoordinator } from '../cluster/coordinator.js';
|
|
33
34
|
import type { TeamMembershipLookup } from '../approvals/team-membership-lookup.js';
|
|
34
35
|
import type { LogStorage } from '../reporting/log-storage.js';
|
|
@@ -62,6 +63,21 @@ interface PendingJobContext {
|
|
|
62
63
|
jobInput: QueuedJobInput;
|
|
63
64
|
runsOnLabels: string[];
|
|
64
65
|
}
|
|
66
|
+
/**
|
|
67
|
+
* Register an eval gate and return a promise that resolves when the scheduler
|
|
68
|
+
* opens it (the eval job's upstream needs are all satisfied).
|
|
69
|
+
*/
|
|
70
|
+
export declare function trackEvalGate(runId: string, evalJobName: string): Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* Open a registered eval gate, unblocking the deferred dispatch task. Returns
|
|
73
|
+
* true if a gate was registered for this eval job (so the scheduler knows it
|
|
74
|
+
* handled the ready signal itself and must not run the normal dispatch path).
|
|
75
|
+
*/
|
|
76
|
+
export declare function openEvalGate(runId: string, evalJobName: string): boolean;
|
|
77
|
+
/** True when a job name is a result-aware dynamic eval job awaiting its gate. */
|
|
78
|
+
export declare function isEvalGatePending(runId: string, evalJobName: string): boolean;
|
|
79
|
+
/** Clear all eval gates for a run (called on run completion / cleanup). */
|
|
80
|
+
export declare function clearEvalGatesForRun(runId: string): void;
|
|
65
81
|
/**
|
|
66
82
|
* Store a pending dispatch context for a job that will be dispatched later
|
|
67
83
|
* by the needs scheduler. The key is `${runId}:${jobName}`.
|
|
@@ -320,6 +336,15 @@ export interface ProcessingDeps {
|
|
|
320
336
|
/** Access-log writer for the orchestrator audit stream. Optional -- if not
|
|
321
337
|
* set, hold-creation audit rows (`held_run.request`) are skipped. */
|
|
322
338
|
accessLogWriter?: AccessLogWriter;
|
|
339
|
+
/** Host roster store for runsOnAll fan-out resolution. Optional -- if not set,
|
|
340
|
+
* runsOnAll jobs cannot be resolved and fail at materialize. */
|
|
341
|
+
hostRosterStore?: HostRosterStore;
|
|
342
|
+
/** This orchestrator instance id (for the cross-cluster host-fanout pin reroute). */
|
|
343
|
+
instanceId?: string;
|
|
344
|
+
/** Static-host grace before a disconnected static host reads unreachable (ms). */
|
|
345
|
+
rosterGraceMs?: number;
|
|
346
|
+
/** Cap on runsOnAll per-host children (default 1024). */
|
|
347
|
+
maxFanoutHosts?: number;
|
|
323
348
|
}
|
|
324
349
|
/**
|
|
325
350
|
* Check if any trigger in the lock file workflows uses path filters.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rolling-wave scheduler for bounded fan-out (`maxParallel` / `failFast`).
|
|
3
|
+
*
|
|
4
|
+
* Sibling of the needs-scheduler (`needs-scheduler.ts`): both fire on a job
|
|
5
|
+
* reaching terminal state and decide what to do with held downstream/sibling
|
|
6
|
+
* jobs. The needs-scheduler releases jobs whose `needs` edges are now satisfied;
|
|
7
|
+
* the wave-scheduler releases the next `wave_gated` sibling of a fan-out base
|
|
8
|
+
* whenever an in-flight slot frees up (or, under `failFast`, skips the held
|
|
9
|
+
* remainder on the first child failure).
|
|
10
|
+
*
|
|
11
|
+
* Pure DB — no in-memory state. Every decision is a fresh query against
|
|
12
|
+
* `execution_jobs` keyed by `(run_id, base_job_name)`, so the scheduler needs
|
|
13
|
+
* zero recovery code on orchestrator restart. The caller performs the DB write
|
|
14
|
+
* (clear `wave_gated` / mark skipped) and the dispatch (`onJobReady`), keeping
|
|
15
|
+
* this module a pure decision function.
|
|
16
|
+
*/
|
|
17
|
+
import type { Kysely } from 'kysely';
|
|
18
|
+
import type { Database } from '../db/types.js';
|
|
19
|
+
/** Inputs identifying a completed fan-out child. The wave policy is read from the base group. */
|
|
20
|
+
export interface WaveEvaluation {
|
|
21
|
+
runId: string;
|
|
22
|
+
/** The base (logical) job name shared by every fan-out child. */
|
|
23
|
+
baseJobName: string;
|
|
24
|
+
/** Terminal status of the child that just completed. */
|
|
25
|
+
completedStatus: string;
|
|
26
|
+
}
|
|
27
|
+
/** The wave-scheduler's decision for one completed child. */
|
|
28
|
+
export type WaveResult = {
|
|
29
|
+
action: 'release';
|
|
30
|
+
jobName: string;
|
|
31
|
+
baseJobName: string;
|
|
32
|
+
maxParallel: number;
|
|
33
|
+
failFast: boolean;
|
|
34
|
+
} | {
|
|
35
|
+
action: 'skip-remaining';
|
|
36
|
+
jobNames: string[];
|
|
37
|
+
} | {
|
|
38
|
+
action: 'noop';
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Decide what happens after a fan-out child of `baseJobName` reaches terminal.
|
|
42
|
+
*
|
|
43
|
+
* The wave policy (`maxParallel` / `failFast`) is read from the base group's
|
|
44
|
+
* own rows — every child of a bounded wave carries the same stamped
|
|
45
|
+
* `wave_max_parallel` / `wave_fail_fast`, so the just-completed child's slot
|
|
46
|
+
* being re-inserted without the policy on release does not break the chain
|
|
47
|
+
* (the still-held siblings carry it). If no sibling carries a policy, this is
|
|
48
|
+
* not a bounded wave → `noop`.
|
|
49
|
+
*
|
|
50
|
+
* - `failFast` + a child failure → `skip-remaining` every still-held sibling.
|
|
51
|
+
* - in-flight count `< maxParallel` AND a held sibling exists → `release` the
|
|
52
|
+
* next held sibling (lowest `variant_label`).
|
|
53
|
+
* - otherwise → `noop`.
|
|
54
|
+
*
|
|
55
|
+
* "In-flight" = a non-terminal, non-`wave_gated` child (it has been dispatched
|
|
56
|
+
* and not yet completed). The just-completed child is terminal, so it does not
|
|
57
|
+
* count against the window — its slot is the one we are filling.
|
|
58
|
+
*/
|
|
59
|
+
export declare function evaluateWave(db: Kysely<Database>, evaluation: WaveEvaluation): Promise<WaveResult>;
|
|
60
|
+
//# sourceMappingURL=wave-scheduler.d.ts.map
|