@kici-dev/orchestrator 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +35 -17
- package/dist/providers/local/lock-file-fetcher.d.ts +8 -0
- package/dist/scaler/claim-store.d.ts +0 -2
- package/dist/scaler/event-backend.d.ts +7 -6
- package/dist/scaler/manager.d.ts +18 -2
- package/dist/scaler/scaler-state-store.d.ts +7 -0
- package/dist/scaler/types.d.ts +10 -0
- package/dist/scaler/warm-pool.d.ts +6 -4
- package/dist/security/identity-link.d.ts +1 -1
- package/dist/security/trust-directory-store.d.ts +1 -1
- package/dist/server.js +167 -76
- package/dist/standalone.js +161 -72
- package/dist/webhook/ingest-overflow-replayer.d.ts +1 -1
- package/dist/webhook/relay-reinject.d.ts +1 -1
- package/dist/worker/in-memory-job-queue.d.ts +9 -0
- package/dist/ws/fleet-agent-collector.d.ts +7 -0
- package/dist/ws/test-relay-handlers.d.ts +2 -0
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +53 -53
package/dist/standalone.js
CHANGED
|
@@ -1382,7 +1382,7 @@ var logger$134, SOFTWARE_VERSION$1, PeerClient$1;
|
|
|
1382
1382
|
var init_peer_client = __esmMin((() => {
|
|
1383
1383
|
init_peer_crypto();
|
|
1384
1384
|
logger$134 = createLogger({ prefix: "peer-client" });
|
|
1385
|
-
SOFTWARE_VERSION$1 = "0.
|
|
1385
|
+
SOFTWARE_VERSION$1 = "0.8.0";
|
|
1386
1386
|
PeerClient$1 = class {
|
|
1387
1387
|
ws = null;
|
|
1388
1388
|
_state = "disconnected";
|
|
@@ -3396,7 +3396,7 @@ var init_peer_handler = __esmMin((() => {
|
|
|
3396
3396
|
init_peer_crypto();
|
|
3397
3397
|
init_join_token();
|
|
3398
3398
|
logger$132 = createLogger({ prefix: "peer-handler" });
|
|
3399
|
-
SOFTWARE_VERSION = "0.
|
|
3399
|
+
SOFTWARE_VERSION = "0.8.0";
|
|
3400
3400
|
RATE_LIMIT_MAX = 5;
|
|
3401
3401
|
RATE_LIMIT_WINDOW_MS = 6e4;
|
|
3402
3402
|
}));
|
|
@@ -24348,7 +24348,10 @@ async function clusterRouteRootJobs(args) {
|
|
|
24348
24348
|
runsOnLabels: [sel.runsOnLabels],
|
|
24349
24349
|
runsOnPatterns: sel.runsOnPatterns,
|
|
24350
24350
|
excludePatterns: sel.excludePatterns,
|
|
24351
|
-
jobConfig:
|
|
24351
|
+
jobConfig: {
|
|
24352
|
+
...buildJobConfig(mj),
|
|
24353
|
+
...ctx.extraJobConfig
|
|
24354
|
+
},
|
|
24352
24355
|
repoUrl: bundle?.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
24353
24356
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
24354
24357
|
sha: ref,
|
|
@@ -25328,7 +25331,10 @@ async function dispatchExecutionAfterInit(args) {
|
|
|
25328
25331
|
runsOnLabels: [runsOnLabels],
|
|
25329
25332
|
runsOnPatterns: selectors.runsOnPatterns,
|
|
25330
25333
|
excludePatterns: selectors.excludePatterns,
|
|
25331
|
-
jobConfig:
|
|
25334
|
+
jobConfig: {
|
|
25335
|
+
...buildJobConfig(mat),
|
|
25336
|
+
...ctx.extraJobConfig
|
|
25337
|
+
},
|
|
25332
25338
|
repoUrl: bundle?.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
25333
25339
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
25334
25340
|
sha: ref,
|
|
@@ -29323,6 +29329,7 @@ var init_lockfile_cache = __esmMin((() => {
|
|
|
29323
29329
|
*/
|
|
29324
29330
|
async get(fetcher, repoIdentifier, ref, credentials) {
|
|
29325
29331
|
const cacheKey = `${fetcher.provider}:${repoIdentifier}:${ref}`;
|
|
29332
|
+
if (fetcher.cacheable === false) return this.fetchAndValidate(fetcher, repoIdentifier, ref, credentials, void 0);
|
|
29326
29333
|
const cached = this.cache.get(cacheKey);
|
|
29327
29334
|
if (cached !== void 0) {
|
|
29328
29335
|
this.hits++;
|
|
@@ -29378,7 +29385,7 @@ var init_lockfile_cache = __esmMin((() => {
|
|
|
29378
29385
|
if (error instanceof LockFileParseError) throw error;
|
|
29379
29386
|
throw new LockFileParseError(repoIdentifier, ref, message);
|
|
29380
29387
|
}
|
|
29381
|
-
this.cache.set(cacheKey, lockFile);
|
|
29388
|
+
if (cacheKey !== void 0) this.cache.set(cacheKey, lockFile);
|
|
29382
29389
|
return lockFile;
|
|
29383
29390
|
}
|
|
29384
29391
|
/**
|
|
@@ -33859,10 +33866,12 @@ var init_warm_pool = __esmMin((() => {
|
|
|
33859
33866
|
* Start the periodic tick: reap surplus agents past their idle timeout, then
|
|
33860
33867
|
* top the pools back up.
|
|
33861
33868
|
*
|
|
33862
|
-
* The first pass runs immediately rather than a tick later.
|
|
33863
|
-
*
|
|
33864
|
-
*
|
|
33865
|
-
*
|
|
33869
|
+
* The first pass runs immediately rather than a tick later. The one caller
|
|
33870
|
+
* is `ScalerManager.startWarmPools()`, which each host reaches only after
|
|
33871
|
+
* `ensureHostsReady()` and — on the coordinator — after the event emitter
|
|
33872
|
+
* is assigned, so spawning here is safe. Waiting a tick instead would leave
|
|
33873
|
+
* the pool empty for 30 seconds after every restart, which is precisely the
|
|
33874
|
+
* cold start it exists to remove.
|
|
33866
33875
|
*/
|
|
33867
33876
|
start() {
|
|
33868
33877
|
if (this.idleCheckInterval) return;
|
|
@@ -34775,10 +34784,10 @@ var init_event_backend = __esmMin((() => {
|
|
|
34775
34784
|
async spawn(labelSet, agentId, orchestratorUrl, _onEvent, effectiveLimits, spawnContext, signal) {
|
|
34776
34785
|
if (signal?.aborted) throw new Error("event scaler spawn aborted before start");
|
|
34777
34786
|
const fullLabels = scalerAgentLabels(labelSet, this.type, this.entry.name, this.entry.roles, spawnContext?.platformTaints);
|
|
34787
|
+
const mandatoryLabels = [...spawnContext?.mandatoryLabels ?? this.entry.mandatoryLabels ?? []];
|
|
34778
34788
|
const claimCode = await this.claimStore.register({
|
|
34779
34789
|
agentId,
|
|
34780
34790
|
labels: fullLabels,
|
|
34781
|
-
mandatoryLabels: this.entry.mandatoryLabels ?? [],
|
|
34782
34791
|
agentTokenTtlSeconds: this.entry.agentTokenTtlSeconds ?? DEFAULT_AGENT_TOKEN_TTL_SECONDS,
|
|
34783
34792
|
orchestratorUrl
|
|
34784
34793
|
});
|
|
@@ -34797,7 +34806,7 @@ var init_event_backend = __esmMin((() => {
|
|
|
34797
34806
|
scalerName: this.entry.name,
|
|
34798
34807
|
agentId,
|
|
34799
34808
|
labels: fullLabels,
|
|
34800
|
-
mandatoryLabels
|
|
34809
|
+
mandatoryLabels,
|
|
34801
34810
|
resources,
|
|
34802
34811
|
orchestratorUrl,
|
|
34803
34812
|
claimCode,
|
|
@@ -34911,12 +34920,13 @@ var init_event_backend = __esmMin((() => {
|
|
|
34911
34920
|
if (this.agents.size > 0) logger$81.info(`Leaving ${this.agents.size} event provision(s) to the cluster on shutdown; teardown is the holding coordinator's or the reaper's`);
|
|
34912
34921
|
}
|
|
34913
34922
|
/**
|
|
34914
|
-
* Apply the new config. The backend reads `roles`, `
|
|
34915
|
-
*
|
|
34916
|
-
*
|
|
34917
|
-
*
|
|
34918
|
-
*
|
|
34919
|
-
*
|
|
34923
|
+
* Apply the new config. The backend reads `roles`, `agentTokenTtlSeconds`
|
|
34924
|
+
* and `provisioningTargets` off its entry at every spawn (and
|
|
34925
|
+
* `mandatoryLabels` only when a spawn reaches it without a context), so the
|
|
34926
|
+
* entry is replaced here — otherwise a reload that retargets the
|
|
34927
|
+
* provisioning workflow, or changes the scaler's roles, would keep emitting
|
|
34928
|
+
* scale-up events to the old workflow refs and minting agent labels from the
|
|
34929
|
+
* old roles, while the manager's own routing gate used the new ones.
|
|
34920
34930
|
*/
|
|
34921
34931
|
reload(labelSets, opts) {
|
|
34922
34932
|
this.labelSets = labelSets;
|
|
@@ -35943,7 +35953,10 @@ var init_manager = __esmMin((() => {
|
|
|
35943
35953
|
await this.spawnSemaphoreFor(backendName).run(() => {
|
|
35944
35954
|
const entry = this.spawningAgents.get(agentId);
|
|
35945
35955
|
if (entry) entry.spawnStartedAt = Date.now();
|
|
35946
|
-
return this.runSpawnWithTimeout(void 0, (signal) => backend.spawn(labelSet, agentId, this.getOrchestratorUrl(backendName), onEvent, spawnLimitsFor(effective.limits), {
|
|
35956
|
+
return this.runSpawnWithTimeout(void 0, (signal) => backend.spawn(labelSet, agentId, this.getOrchestratorUrl(backendName), onEvent, spawnLimitsFor(effective.limits), {
|
|
35957
|
+
platformTaints: this.platformTaintsFor(backendName, backend.type),
|
|
35958
|
+
mandatoryLabels: this.labelSetMandatoryLabels(backendName, backend, labelSet)
|
|
35959
|
+
}, signal));
|
|
35947
35960
|
});
|
|
35948
35961
|
this.startLogForwarding(backend, agentId);
|
|
35949
35962
|
} catch (err) {
|
|
@@ -36459,6 +36472,7 @@ var init_manager = __esmMin((() => {
|
|
|
36459
36472
|
boundJobId: jobId,
|
|
36460
36473
|
runId,
|
|
36461
36474
|
platformTaints: this.platformTaintsFor(backendName, backend.type),
|
|
36475
|
+
mandatoryLabels: this.labelSetMandatoryLabels(backendName, backend, spawnLabelSet),
|
|
36462
36476
|
...containerSpawn ? { container: containerSpawn } : {}
|
|
36463
36477
|
};
|
|
36464
36478
|
this.spawnSemaphoreFor(backendName).run(() => {
|
|
@@ -37820,12 +37834,16 @@ var init_manager = __esmMin((() => {
|
|
|
37820
37834
|
return backend.getScalerContext?.(agentId);
|
|
37821
37835
|
}
|
|
37822
37836
|
/**
|
|
37823
|
-
* Start the
|
|
37824
|
-
*
|
|
37837
|
+
* Start the machine-pool ledger reaper and the retirement sweep that tears
|
|
37838
|
+
* down drained retiring backends, and publish the warm-pool gauges so each
|
|
37839
|
+
* pool's target is visible before its first fill.
|
|
37840
|
+
*
|
|
37841
|
+
* The warm pools themselves are NOT started here — see
|
|
37842
|
+
* {@link startWarmPools}, which the host calls once every dependency a fill
|
|
37843
|
+
* can reach is wired.
|
|
37825
37844
|
*/
|
|
37826
37845
|
start() {
|
|
37827
37846
|
this.started = true;
|
|
37828
|
-
this.warmPool.start();
|
|
37829
37847
|
this.publishWarmPoolGauges();
|
|
37830
37848
|
if (this.machineLedger) this.machineLedger.start();
|
|
37831
37849
|
this.retirementSweep ??= setInterval(() => {
|
|
@@ -37834,6 +37852,18 @@ var init_manager = __esmMin((() => {
|
|
|
37834
37852
|
this.retirementSweep.unref?.();
|
|
37835
37853
|
}
|
|
37836
37854
|
/**
|
|
37855
|
+
* Run the warm pools' first deficit pass and arm their periodic tick.
|
|
37856
|
+
*
|
|
37857
|
+
* Separate from {@link start} because the first pass spawns immediately,
|
|
37858
|
+
* and on an event backend a spawn emits through the event emitter — which
|
|
37859
|
+
* the coordinator assigns after the scaler is built. The coordinator calls
|
|
37860
|
+
* this once the emitter exists; the worker, which hosts no event backend,
|
|
37861
|
+
* calls it right after `start()`.
|
|
37862
|
+
*/
|
|
37863
|
+
startWarmPools() {
|
|
37864
|
+
this.warmPool.start();
|
|
37865
|
+
}
|
|
37866
|
+
/**
|
|
37837
37867
|
* Provision/heal every backend's host prerequisites before spawning starts.
|
|
37838
37868
|
* Awaits each backend's optional ensureHostReady, catching per-backend so one
|
|
37839
37869
|
* scaler's host-prep failure degrades only that scaler (its spawns will fail
|
|
@@ -38085,6 +38115,7 @@ var init_manager = __esmMin((() => {
|
|
|
38085
38115
|
* coord boot or Raft leader switch. Reconstructs:
|
|
38086
38116
|
*
|
|
38087
38117
|
* - `spawningAgents` (with `boundJobId` preserved for eager-dispatch on register)
|
|
38118
|
+
* — event-backend rows only; a local-backend row is dropped, see below
|
|
38088
38119
|
* - `agentJobCorrelation` (so scaler-lifecycle events route correctly)
|
|
38089
38120
|
* - `reservations` + `perScalerUsage` (so the cap-check critical
|
|
38090
38121
|
* section reflects the cluster-wide truth, not the local empty
|
|
@@ -38106,6 +38137,7 @@ var init_manager = __esmMin((() => {
|
|
|
38106
38137
|
async recoverState() {
|
|
38107
38138
|
const recovery = {
|
|
38108
38139
|
spawningAgentsRehydrated: 0,
|
|
38140
|
+
spawningDropped: 0,
|
|
38109
38141
|
agentJobsRehydrated: 0,
|
|
38110
38142
|
reservationsRehydrated: 0,
|
|
38111
38143
|
reservationsDropped: 0,
|
|
@@ -38114,17 +38146,25 @@ var init_manager = __esmMin((() => {
|
|
|
38114
38146
|
if (!this.stateStore) return recovery;
|
|
38115
38147
|
try {
|
|
38116
38148
|
const spawning = await this.stateStore.listSpawningAgentsForOwner(this.instanceId);
|
|
38117
|
-
for (const entry of spawning)
|
|
38118
|
-
|
|
38119
|
-
|
|
38120
|
-
|
|
38121
|
-
|
|
38122
|
-
|
|
38123
|
-
|
|
38124
|
-
|
|
38125
|
-
|
|
38126
|
-
|
|
38127
|
-
|
|
38149
|
+
for (const entry of spawning) {
|
|
38150
|
+
const backendType = entry.backendType ?? this.backends.get(entry.scalerName)?.type;
|
|
38151
|
+
if (backendType !== void 0 && backendType !== ScalerBackendType.enum.event) {
|
|
38152
|
+
this.deleteUnadoptedSpawningAgentFromStore(entry.agentId);
|
|
38153
|
+
recovery.spawningDropped += 1;
|
|
38154
|
+
continue;
|
|
38155
|
+
}
|
|
38156
|
+
this.spawningAgents.set(entry.agentId, {
|
|
38157
|
+
labelSet: entry.labelSet,
|
|
38158
|
+
backendName: entry.scalerName,
|
|
38159
|
+
provisioningTargets: entry.provisioningTargets ?? [],
|
|
38160
|
+
spawnedAt: entry.spawnedAt.getTime(),
|
|
38161
|
+
spawnStartedAt: entry.spawnedAt.getTime(),
|
|
38162
|
+
...entry.backendType !== void 0 && { backendType: entry.backendType },
|
|
38163
|
+
...entry.boundJobId !== void 0 && { boundJobId: entry.boundJobId },
|
|
38164
|
+
...entry.runId !== void 0 && { runId: entry.runId }
|
|
38165
|
+
});
|
|
38166
|
+
}
|
|
38167
|
+
recovery.spawningAgentsRehydrated = spawning.length - recovery.spawningDropped;
|
|
38128
38168
|
const correlations = await this.stateStore.listAgentJobs();
|
|
38129
38169
|
for (const c of correlations) this.agentJobCorrelation.set(c.agentId, {
|
|
38130
38170
|
runId: c.runId,
|
|
@@ -39979,7 +40019,8 @@ function createAgentWsHandler(deps) {
|
|
|
39979
40019
|
if (rateLimiter) {
|
|
39980
40020
|
const messageSize = typeof evt.data === "string" ? evt.data.length : 0;
|
|
39981
40021
|
const isHeartbeat = raw !== null && typeof raw === "object" && "type" in raw && raw.type === "heartbeat";
|
|
39982
|
-
const
|
|
40022
|
+
const isSolicitedFleetChunk = raw !== null && typeof raw === "object" && raw.type === "fleet.bundle.chunk" && typeof raw.requestId === "string" && deps.fleetAgentCollector !== void 0 && wsToAgentId.has(ws) && deps.fleetAgentCollector.isPendingFor(raw.requestId, wsToAgentId.get(ws));
|
|
40023
|
+
const rlResult = rateLimiter.check(messageSize, isHeartbeat || isSolicitedFleetChunk);
|
|
39983
40024
|
if (!rlResult.allowed) {
|
|
39984
40025
|
if (rlResult.action === "disconnect") {
|
|
39985
40026
|
logger$75.warn("Rate limit disconnect", { reason: rlResult.reason });
|
|
@@ -57591,6 +57632,14 @@ var init_lock_file_fetcher = __esmMin((() => {
|
|
|
57591
57632
|
repoBasePath;
|
|
57592
57633
|
provider = "local";
|
|
57593
57634
|
/**
|
|
57635
|
+
* Never cached: `ref` is ignored below, so a cache keyed on it would serve the
|
|
57636
|
+
* first lock file read to every later trigger for the lifetime of the entry —
|
|
57637
|
+
* a `kici run --local` dispatch always triggers at `HEAD`, and an in-place run
|
|
57638
|
+
* carries its uncommitted edits under an unchanged sha. Reading the file from
|
|
57639
|
+
* disk is what makes those edits reach the run.
|
|
57640
|
+
*/
|
|
57641
|
+
cacheable = false;
|
|
57642
|
+
/**
|
|
57594
57643
|
* @param repoBasePath - Base directory for the repo(s). When repoIdentifier
|
|
57595
57644
|
* starts with 'file://', it is stripped and used as-is. Otherwise
|
|
57596
57645
|
* repoBasePath is used as the root.
|
|
@@ -61486,15 +61535,15 @@ var init_admin_config = __esmMin((() => {
|
|
|
61486
61535
|
function createHealthRoutes$1(deps = {}) {
|
|
61487
61536
|
return createHealthRoutes({
|
|
61488
61537
|
livenessInfo: () => ({
|
|
61489
|
-
version: "0.
|
|
61490
|
-
buildDate: "2026-09-
|
|
61491
|
-
buildCommit: "
|
|
61492
|
-
sdkVersion: "0.
|
|
61538
|
+
version: "0.8.0",
|
|
61539
|
+
buildDate: "2026-09-13T11:42:43.287Z",
|
|
61540
|
+
buildCommit: "57712173f",
|
|
61541
|
+
sdkVersion: "0.8.0",
|
|
61493
61542
|
sdkBundleHash: "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5",
|
|
61494
|
-
sharedVersion: "0.
|
|
61495
|
-
sharedBundleHash: "
|
|
61496
|
-
engineVersion: "0.
|
|
61497
|
-
engineBundleHash: "
|
|
61543
|
+
sharedVersion: "0.8.0",
|
|
61544
|
+
sharedBundleHash: "c1c70e41a1ec14fd7cf3a6047251e9aa11258a21e3694ea89a50e8bf14b99f9d",
|
|
61545
|
+
engineVersion: "0.8.0",
|
|
61546
|
+
engineBundleHash: "b26b2f23059c2ceb06a65feb2dcfe2d8c8be45ccf66622823ffedf742d2d7599"
|
|
61498
61547
|
}),
|
|
61499
61548
|
readinessCheck: deps.db ? async () => {
|
|
61500
61549
|
const checks = {};
|
|
@@ -61529,7 +61578,7 @@ function createCapabilitiesRoutes() {
|
|
|
61529
61578
|
const app = new Hono();
|
|
61530
61579
|
app.get("/api/v1/capabilities", (c) => {
|
|
61531
61580
|
const manifest = {
|
|
61532
|
-
orchestratorVersion: "0.
|
|
61581
|
+
orchestratorVersion: "0.8.0",
|
|
61533
61582
|
protocolVersion: PROTOCOL_VERSION,
|
|
61534
61583
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
61535
61584
|
};
|
|
@@ -64497,7 +64546,7 @@ var init_app = __esmMin((() => {
|
|
|
64497
64546
|
init_log_chunk_sink();
|
|
64498
64547
|
init_agent_metrics_aggregator();
|
|
64499
64548
|
logger$31 = createLogger({ prefix: "app" });
|
|
64500
|
-
ORCHESTRATOR_VERSION$2 = "0.
|
|
64549
|
+
ORCHESTRATOR_VERSION$2 = "0.8.0";
|
|
64501
64550
|
SourceLocationStore = class {
|
|
64502
64551
|
cache = /* @__PURE__ */ new Map();
|
|
64503
64552
|
key(workflowName, jobName) {
|
|
@@ -64552,6 +64601,15 @@ var init_fleet_agent_collector = __esmMin((() => {
|
|
|
64552
64601
|
onChunk(requestId, seq, dataB64, isLast) {
|
|
64553
64602
|
this.waiter.onChunk(requestId, seq, dataB64, isLast);
|
|
64554
64603
|
}
|
|
64604
|
+
/**
|
|
64605
|
+
* Whether `requestId` is a collection this orchestrator asked `agentId` for
|
|
64606
|
+
* and is still waiting on. The agent handler consults this before rate
|
|
64607
|
+
* limiting a `fleet.bundle.chunk`: a solicited transfer is exempt, an
|
|
64608
|
+
* unsolicited one is not.
|
|
64609
|
+
*/
|
|
64610
|
+
isPendingFor(requestId, agentId) {
|
|
64611
|
+
return this.requestAgent.get(requestId) === agentId;
|
|
64612
|
+
}
|
|
64555
64613
|
onError(requestId, message) {
|
|
64556
64614
|
this.waiter.onError(requestId, message);
|
|
64557
64615
|
}
|
|
@@ -65700,6 +65758,9 @@ var init_container_backend = __esmMin((() => {
|
|
|
65700
65758
|
state: "spawning"
|
|
65701
65759
|
};
|
|
65702
65760
|
this.agents.set(managed.id, managed);
|
|
65761
|
+
const assertStillTracked = () => {
|
|
65762
|
+
if (this.agents.get(managed.id) !== managed) throw new Error(`Agent ${agentId} was torn down while its container was being provisioned`);
|
|
65763
|
+
};
|
|
65703
65764
|
try {
|
|
65704
65765
|
const agentEnvForwarded = [];
|
|
65705
65766
|
for (const [key, value] of Object.entries(process.env)) if (key.startsWith(KICI_AGENT_ENV_PREFIX) && value !== void 0) {
|
|
@@ -65773,17 +65834,24 @@ var init_container_backend = __esmMin((() => {
|
|
|
65773
65834
|
...this.networkIsolation && { NetworkingConfig: { EndpointsConfig: { ["kici-agent-net"]: {} } } }
|
|
65774
65835
|
});
|
|
65775
65836
|
createdContainer = container;
|
|
65837
|
+
managed.backendRef = container.id;
|
|
65838
|
+
this.containerToManaged.set(container.id, managed.id);
|
|
65839
|
+
assertStillTracked();
|
|
65776
65840
|
if (this.networkIsolation) emit(ScalerEventType$1.enum["scaler.network"], "configuring network isolation");
|
|
65777
65841
|
await container.start({ abortSignal: signal });
|
|
65842
|
+
assertStillTracked();
|
|
65778
65843
|
emit(ScalerEventType$1.enum["scaler.ready"], "container started");
|
|
65779
|
-
if (this.networkIsolation)
|
|
65780
|
-
|
|
65781
|
-
|
|
65782
|
-
|
|
65783
|
-
|
|
65784
|
-
|
|
65785
|
-
|
|
65786
|
-
|
|
65844
|
+
if (this.networkIsolation) {
|
|
65845
|
+
await this.applyIsolationRules({
|
|
65846
|
+
agentId,
|
|
65847
|
+
managedId: managed.id,
|
|
65848
|
+
containerId: container.id,
|
|
65849
|
+
networkPolicy: matchedLabelSet.networkPolicy,
|
|
65850
|
+
orchestratorUrl,
|
|
65851
|
+
...signal ? { signal } : {}
|
|
65852
|
+
});
|
|
65853
|
+
assertStillTracked();
|
|
65854
|
+
}
|
|
65787
65855
|
try {
|
|
65788
65856
|
const capture = await createContainerLogCapture(this.docker, container.id);
|
|
65789
65857
|
this.logCaptures.set(managed.id, capture);
|
|
@@ -65793,13 +65861,17 @@ var init_container_backend = __esmMin((() => {
|
|
|
65793
65861
|
error: toErrorMessage(err)
|
|
65794
65862
|
});
|
|
65795
65863
|
}
|
|
65864
|
+
assertStillTracked();
|
|
65796
65865
|
managed.state = "running";
|
|
65797
|
-
managed.backendRef = container.id;
|
|
65798
|
-
this.containerToManaged.set(container.id, managed.id);
|
|
65799
65866
|
emit(ScalerEventType$1.enum["agent.connecting"], "waiting for agent WS registration");
|
|
65800
65867
|
return managed;
|
|
65801
65868
|
} catch (err) {
|
|
65802
|
-
const
|
|
65869
|
+
const capture = this.logCaptures.get(managed.id);
|
|
65870
|
+
const t = capture?.tail() ?? "";
|
|
65871
|
+
if (capture) {
|
|
65872
|
+
capture.close();
|
|
65873
|
+
this.logCaptures.delete(managed.id);
|
|
65874
|
+
}
|
|
65803
65875
|
const base = toErrorMessage(err);
|
|
65804
65876
|
emit(ScalerEventType$1.enum["scaler.failed"], t ? `${base}\n--- captured output ---\n${t}` : base);
|
|
65805
65877
|
const failedIp = this.containerIps.get(managed.id);
|
|
@@ -65809,10 +65881,13 @@ var init_container_backend = __esmMin((() => {
|
|
|
65809
65881
|
} catch {}
|
|
65810
65882
|
this.containerIps.delete(managed.id);
|
|
65811
65883
|
}
|
|
65812
|
-
if (createdContainer)
|
|
65813
|
-
|
|
65814
|
-
|
|
65815
|
-
|
|
65884
|
+
if (createdContainer) {
|
|
65885
|
+
this.containerToManaged.delete(createdContainer.id);
|
|
65886
|
+
try {
|
|
65887
|
+
await createdContainer.remove({ force: true });
|
|
65888
|
+
} catch {}
|
|
65889
|
+
}
|
|
65890
|
+
if (this.agents.get(managed.id) === managed) this.agents.delete(managed.id);
|
|
65816
65891
|
throw err;
|
|
65817
65892
|
}
|
|
65818
65893
|
}
|
|
@@ -65849,6 +65924,7 @@ var init_container_backend = __esmMin((() => {
|
|
|
65849
65924
|
capture.close();
|
|
65850
65925
|
this.logCaptures.delete(managedId);
|
|
65851
65926
|
}
|
|
65927
|
+
if (!managed.backendRef) return;
|
|
65852
65928
|
try {
|
|
65853
65929
|
const container = this.docker.getContainer(managed.backendRef);
|
|
65854
65930
|
try {
|
|
@@ -81376,6 +81452,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
81376
81452
|
nodeId: config.instanceId
|
|
81377
81453
|
});
|
|
81378
81454
|
eventEmitter = new EventEmitter$1(eventRouter);
|
|
81455
|
+
scalerManager?.startWarmPools();
|
|
81379
81456
|
const invokeGateDeps = {
|
|
81380
81457
|
db,
|
|
81381
81458
|
executionTracker,
|
|
@@ -82909,6 +82986,17 @@ var init_in_memory_job_queue = __esmMin((() => {
|
|
|
82909
82986
|
async getPendingJobs() {
|
|
82910
82987
|
return [...this.jobs.values()].filter((j) => j.status === "pending");
|
|
82911
82988
|
}
|
|
82989
|
+
/**
|
|
82990
|
+
* Oldest-first listing of pending jobs, capped at `limit` — the worker-side
|
|
82991
|
+
* twin of `JobQueue.listPending`. The dispatcher's capacity-freed re-drive
|
|
82992
|
+
* (`retryPendingScaleRequests`) reads it on every worker whose scaler frees
|
|
82993
|
+
* a slot; `worker-core` wires that hook exactly as the coordinator does.
|
|
82994
|
+
* Insertion order is the enqueue order, which is what `created_at ASC` gives
|
|
82995
|
+
* the DB queue.
|
|
82996
|
+
*/
|
|
82997
|
+
async listPending(limit) {
|
|
82998
|
+
return (await this.getPendingJobs()).slice(0, Math.max(0, limit));
|
|
82999
|
+
}
|
|
82912
83000
|
/** Always returns empty array. */
|
|
82913
83001
|
async getDispatchedJobIdsByRunId(_runId) {
|
|
82914
83002
|
return [];
|
|
@@ -83417,6 +83505,7 @@ async function initializeWorkerScaler(config, tokenStore, onScalerEvent, tokenTt
|
|
|
83417
83505
|
}
|
|
83418
83506
|
await scalerManager.ensureHostsReady();
|
|
83419
83507
|
scalerManager.start();
|
|
83508
|
+
scalerManager.startWarmPools();
|
|
83420
83509
|
logger$2.info("Worker scaler initialized", {
|
|
83421
83510
|
backends: backends.map((b) => b.name),
|
|
83422
83511
|
globalMaxAgents: scalerConfig.globalMaxAgents
|
|
@@ -84109,14 +84198,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
84109
84198
|
init_peer_outbox();
|
|
84110
84199
|
init_worker_outbox_relay();
|
|
84111
84200
|
init_app_on_error();
|
|
84112
|
-
ORCHESTRATOR_VERSION$1 = "0.
|
|
84113
|
-
WORKER_BUILD_COMMIT = "
|
|
84114
|
-
WORKER_SDK_VERSION = "0.
|
|
84201
|
+
ORCHESTRATOR_VERSION$1 = "0.8.0";
|
|
84202
|
+
WORKER_BUILD_COMMIT = "57712173f";
|
|
84203
|
+
WORKER_SDK_VERSION = "0.8.0";
|
|
84115
84204
|
WORKER_SDK_BUNDLE_HASH = "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5";
|
|
84116
|
-
WORKER_SHARED_VERSION = "0.
|
|
84117
|
-
WORKER_SHARED_BUNDLE_HASH = "
|
|
84118
|
-
WORKER_ENGINE_VERSION = "0.
|
|
84119
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
84205
|
+
WORKER_SHARED_VERSION = "0.8.0";
|
|
84206
|
+
WORKER_SHARED_BUNDLE_HASH = "c1c70e41a1ec14fd7cf3a6047251e9aa11258a21e3694ea89a50e8bf14b99f9d";
|
|
84207
|
+
WORKER_ENGINE_VERSION = "0.8.0";
|
|
84208
|
+
WORKER_ENGINE_BUNDLE_HASH = "b26b2f23059c2ceb06a65feb2dcfe2d8c8be45ccf66622823ffedf742d2d7599";
|
|
84120
84209
|
logger$2 = createLogger({ prefix: "worker" });
|
|
84121
84210
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
84122
84211
|
}));
|
|
@@ -84140,14 +84229,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
84140
84229
|
* Graceful shutdown:
|
|
84141
84230
|
* agent WS -> heartbeat -> HTTP -> DB
|
|
84142
84231
|
*/
|
|
84143
|
-
const ORCHESTRATOR_VERSION = "0.
|
|
84144
|
-
const BUILD_COMMIT = "
|
|
84145
|
-
const SDK_VERSION = "0.
|
|
84232
|
+
const ORCHESTRATOR_VERSION = "0.8.0";
|
|
84233
|
+
const BUILD_COMMIT = "57712173f";
|
|
84234
|
+
const SDK_VERSION = "0.8.0";
|
|
84146
84235
|
const SDK_BUNDLE_HASH = "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5";
|
|
84147
|
-
const SHARED_VERSION = "0.
|
|
84148
|
-
const SHARED_BUNDLE_HASH = "
|
|
84149
|
-
const ENGINE_VERSION = "0.
|
|
84150
|
-
const ENGINE_BUNDLE_HASH = "
|
|
84236
|
+
const SHARED_VERSION = "0.8.0";
|
|
84237
|
+
const SHARED_BUNDLE_HASH = "c1c70e41a1ec14fd7cf3a6047251e9aa11258a21e3694ea89a50e8bf14b99f9d";
|
|
84238
|
+
const ENGINE_VERSION = "0.8.0";
|
|
84239
|
+
const ENGINE_BUNDLE_HASH = "b26b2f23059c2ceb06a65feb2dcfe2d8c8be45ccf66622823ffedf742d2d7599";
|
|
84151
84240
|
const otelSdk = initTelemetry({
|
|
84152
84241
|
serviceName: "kici-orchestrator",
|
|
84153
84242
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
@@ -80,7 +80,7 @@ export interface IngestOverflowReplayerDeps {
|
|
|
80
80
|
* Refusal is per fairness key: a saturated org's rows are skipped for the rest
|
|
81
81
|
* of the pass while other orgs' rows keep draining, mirroring the fair skip the
|
|
82
82
|
* controller's own queue does. Reclaiming is not gated on load at all: a
|
|
83
|
-
* stranded claim is stranded regardless, and the row it frees
|
|
83
|
+
* stranded claim is stranded regardless, and the row it frees waits in
|
|
84
84
|
* `buffered` until a reservation succeeds.
|
|
85
85
|
*/
|
|
86
86
|
export declare class IngestOverflowReplayer {
|
|
@@ -36,7 +36,7 @@ export declare function parseRelayPayload(body: Buffer, headers: Record<string,
|
|
|
36
36
|
* would be a second, redundant gate — and a refusal from it would be charged to
|
|
37
37
|
* the delivery as a failed replay attempt, which is the defect this ordering
|
|
38
38
|
* exists to remove. The slot is genuinely held: nothing here bypasses the
|
|
39
|
-
* controller, the decision was
|
|
39
|
+
* controller, the decision was made one step earlier.
|
|
40
40
|
*/
|
|
41
41
|
export declare function buildRelayReinject(seams: RelayReinjectSeams): ReinjectFn;
|
|
42
42
|
//# sourceMappingURL=relay-reinject.d.ts.map
|
|
@@ -104,6 +104,15 @@ export declare class InMemoryJobQueue {
|
|
|
104
104
|
getFullJobById(jobId: string): Promise<QueuedJob | null>;
|
|
105
105
|
/** Return all pending jobs. */
|
|
106
106
|
getPendingJobs(): Promise<QueuedJob[]>;
|
|
107
|
+
/**
|
|
108
|
+
* Oldest-first listing of pending jobs, capped at `limit` — the worker-side
|
|
109
|
+
* twin of `JobQueue.listPending`. The dispatcher's capacity-freed re-drive
|
|
110
|
+
* (`retryPendingScaleRequests`) reads it on every worker whose scaler frees
|
|
111
|
+
* a slot; `worker-core` wires that hook exactly as the coordinator does.
|
|
112
|
+
* Insertion order is the enqueue order, which is what `created_at ASC` gives
|
|
113
|
+
* the DB queue.
|
|
114
|
+
*/
|
|
115
|
+
listPending(limit: number): Promise<QueuedJob[]>;
|
|
107
116
|
/** Always returns empty array. */
|
|
108
117
|
getDispatchedJobIdsByRunId(_runId: string): Promise<string[]>;
|
|
109
118
|
/** Lookup across pending and dispatched jobs (id / runId / status). */
|
|
@@ -14,6 +14,13 @@ export declare class FleetAgentCollector {
|
|
|
14
14
|
*/
|
|
15
15
|
request(requestId: string, agentId: string, send: () => void): Promise<Buffer>;
|
|
16
16
|
onChunk(requestId: string, seq: number, dataB64: string, isLast: boolean): void;
|
|
17
|
+
/**
|
|
18
|
+
* Whether `requestId` is a collection this orchestrator asked `agentId` for
|
|
19
|
+
* and is still waiting on. The agent handler consults this before rate
|
|
20
|
+
* limiting a `fleet.bundle.chunk`: a solicited transfer is exempt, an
|
|
21
|
+
* unsolicited one is not.
|
|
22
|
+
*/
|
|
23
|
+
isPendingFor(requestId: string, agentId: string): boolean;
|
|
17
24
|
onError(requestId: string, message: string): void;
|
|
18
25
|
/** Reject every pending request owned by `agentId` (called on its disconnect). */
|
|
19
26
|
rejectAgent(agentId: string, reason: string): void;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "0.
|
|
2
|
+
"version": "0.8.0",
|
|
3
3
|
"images": {
|
|
4
|
-
"kici-agent": "sha256:
|
|
5
|
-
"kici-orchestrator": "sha256:
|
|
4
|
+
"kici-agent": "sha256:81dbd07842282809140f620eceec60978793e1aea6972297b78933c2f66f34b1",
|
|
5
|
+
"kici-orchestrator": "sha256:0cc0c914b0ce08cb1f3532073d1c70f18532024f69dceeb3a719f1160a62554f"
|
|
6
6
|
}
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kici-dev/orchestrator",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Customer-deployable orchestrator for the KiCI CI/CD stack. Receives webhook events (direct or via Platform relay), matches triggers against the lock file, and dispatches jobs to connected agents.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ci",
|
|
@@ -89,8 +89,8 @@
|
|
|
89
89
|
"ws": "^8.21.3",
|
|
90
90
|
"yaml": "^2.9.0",
|
|
91
91
|
"zod": "^4.4.3",
|
|
92
|
-
"@kici-dev/
|
|
93
|
-
"@kici-dev/
|
|
92
|
+
"@kici-dev/engine": "0.8.0",
|
|
93
|
+
"@kici-dev/shared": "0.8.0"
|
|
94
94
|
},
|
|
95
95
|
"kici": {
|
|
96
96
|
"metrics": {
|
|
@@ -105,7 +105,7 @@
|
|
|
105
105
|
"@types/dockerode": "^4.0.1",
|
|
106
106
|
"@types/ws": "^8.18.1",
|
|
107
107
|
"kysely-ctl": "^0.21.0",
|
|
108
|
-
"@kici-dev/agent": "0.
|
|
108
|
+
"@kici-dev/agent": "0.8.0"
|
|
109
109
|
},
|
|
110
110
|
"scripts": {
|
|
111
111
|
"build": "node ../../scripts/build-service.mjs && tsgo --emitDeclarationOnly",
|