@kici-dev/orchestrator 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +35 -17
- package/dist/providers/local/lock-file-fetcher.d.ts +8 -0
- package/dist/scaler/claim-store.d.ts +0 -2
- package/dist/scaler/event-backend.d.ts +7 -6
- package/dist/scaler/manager.d.ts +18 -2
- package/dist/scaler/scaler-state-store.d.ts +7 -0
- package/dist/scaler/types.d.ts +10 -0
- package/dist/scaler/warm-pool.d.ts +6 -4
- package/dist/security/identity-link.d.ts +1 -1
- package/dist/security/trust-directory-store.d.ts +1 -1
- package/dist/server.js +167 -76
- package/dist/standalone.js +161 -72
- package/dist/webhook/ingest-overflow-replayer.d.ts +1 -1
- package/dist/webhook/relay-reinject.d.ts +1 -1
- package/dist/worker/in-memory-job-queue.d.ts +9 -0
- package/dist/ws/fleet-agent-collector.d.ts +7 -0
- package/dist/ws/test-relay-handlers.d.ts +2 -0
- package/installer-image-digests.json +3 -3
- package/package.json +4 -4
- package/sbom.spdx.json +53 -53
package/dist/server.js
CHANGED
|
@@ -1469,7 +1469,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
1469
1469
|
* the answer is unambiguous; when they disagree there is no way to tell here
|
|
1470
1470
|
* which provider serves the commit, so this answers undefined rather than
|
|
1471
1471
|
* naming the wrong account. Undefined is a safe answer at every call site: the
|
|
1472
|
-
* attribution is
|
|
1472
|
+
* attribution is omitted, never replaced by the raw subject id.
|
|
1473
1473
|
*/
|
|
1474
1474
|
function resolveLinkedUsername(identityLinks, userId) {
|
|
1475
1475
|
const names = new Set(identityLinks.filter((link) => link.userId === userId).map((link) => link.providerUsername));
|
|
@@ -13658,6 +13658,7 @@ var init_lockfile_cache = __esmMin((() => {
|
|
|
13658
13658
|
*/
|
|
13659
13659
|
async get(fetcher, repoIdentifier, ref, credentials) {
|
|
13660
13660
|
const cacheKey = `${fetcher.provider}:${repoIdentifier}:${ref}`;
|
|
13661
|
+
if (fetcher.cacheable === false) return this.fetchAndValidate(fetcher, repoIdentifier, ref, credentials, void 0);
|
|
13661
13662
|
const cached = this.cache.get(cacheKey);
|
|
13662
13663
|
if (cached !== void 0) {
|
|
13663
13664
|
this.hits++;
|
|
@@ -13713,7 +13714,7 @@ var init_lockfile_cache = __esmMin((() => {
|
|
|
13713
13714
|
if (error instanceof LockFileParseError) throw error;
|
|
13714
13715
|
throw new LockFileParseError(repoIdentifier, ref, message);
|
|
13715
13716
|
}
|
|
13716
|
-
this.cache.set(cacheKey, lockFile);
|
|
13717
|
+
if (cacheKey !== void 0) this.cache.set(cacheKey, lockFile);
|
|
13717
13718
|
return lockFile;
|
|
13718
13719
|
}
|
|
13719
13720
|
/**
|
|
@@ -18635,10 +18636,12 @@ var init_warm_pool = __esmMin((() => {
|
|
|
18635
18636
|
* Start the periodic tick: reap surplus agents past their idle timeout, then
|
|
18636
18637
|
* top the pools back up.
|
|
18637
18638
|
*
|
|
18638
|
-
* The first pass runs immediately rather than a tick later.
|
|
18639
|
-
*
|
|
18640
|
-
*
|
|
18641
|
-
*
|
|
18639
|
+
* The first pass runs immediately rather than a tick later. The one caller
|
|
18640
|
+
* is `ScalerManager.startWarmPools()`, which each host reaches only after
|
|
18641
|
+
* `ensureHostsReady()` and — on the coordinator — after the event emitter
|
|
18642
|
+
* is assigned, so spawning here is safe. Waiting a tick instead would leave
|
|
18643
|
+
* the pool empty for 30 seconds after every restart, which is precisely the
|
|
18644
|
+
* cold start it exists to remove.
|
|
18642
18645
|
*/
|
|
18643
18646
|
start() {
|
|
18644
18647
|
if (this.idleCheckInterval) return;
|
|
@@ -19551,10 +19554,10 @@ var init_event_backend = __esmMin((() => {
|
|
|
19551
19554
|
async spawn(labelSet, agentId, orchestratorUrl, _onEvent, effectiveLimits, spawnContext, signal) {
|
|
19552
19555
|
if (signal?.aborted) throw new Error("event scaler spawn aborted before start");
|
|
19553
19556
|
const fullLabels = scalerAgentLabels(labelSet, this.type, this.entry.name, this.entry.roles, spawnContext?.platformTaints);
|
|
19557
|
+
const mandatoryLabels = [...spawnContext?.mandatoryLabels ?? this.entry.mandatoryLabels ?? []];
|
|
19554
19558
|
const claimCode = await this.claimStore.register({
|
|
19555
19559
|
agentId,
|
|
19556
19560
|
labels: fullLabels,
|
|
19557
|
-
mandatoryLabels: this.entry.mandatoryLabels ?? [],
|
|
19558
19561
|
agentTokenTtlSeconds: this.entry.agentTokenTtlSeconds ?? DEFAULT_AGENT_TOKEN_TTL_SECONDS,
|
|
19559
19562
|
orchestratorUrl
|
|
19560
19563
|
});
|
|
@@ -19573,7 +19576,7 @@ var init_event_backend = __esmMin((() => {
|
|
|
19573
19576
|
scalerName: this.entry.name,
|
|
19574
19577
|
agentId,
|
|
19575
19578
|
labels: fullLabels,
|
|
19576
|
-
mandatoryLabels
|
|
19579
|
+
mandatoryLabels,
|
|
19577
19580
|
resources,
|
|
19578
19581
|
orchestratorUrl,
|
|
19579
19582
|
claimCode,
|
|
@@ -19687,12 +19690,13 @@ var init_event_backend = __esmMin((() => {
|
|
|
19687
19690
|
if (this.agents.size > 0) logger$113.info(`Leaving ${this.agents.size} event provision(s) to the cluster on shutdown; teardown is the holding coordinator's or the reaper's`);
|
|
19688
19691
|
}
|
|
19689
19692
|
/**
|
|
19690
|
-
* Apply the new config. The backend reads `roles`, `
|
|
19691
|
-
*
|
|
19692
|
-
*
|
|
19693
|
-
*
|
|
19694
|
-
*
|
|
19695
|
-
*
|
|
19693
|
+
* Apply the new config. The backend reads `roles`, `agentTokenTtlSeconds`
|
|
19694
|
+
* and `provisioningTargets` off its entry at every spawn (and
|
|
19695
|
+
* `mandatoryLabels` only when a spawn reaches it without a context), so the
|
|
19696
|
+
* entry is replaced here — otherwise a reload that retargets the
|
|
19697
|
+
* provisioning workflow, or changes the scaler's roles, would keep emitting
|
|
19698
|
+
* scale-up events to the old workflow refs and minting agent labels from the
|
|
19699
|
+
* old roles, while the manager's own routing gate used the new ones.
|
|
19696
19700
|
*/
|
|
19697
19701
|
reload(labelSets, opts) {
|
|
19698
19702
|
this.labelSets = labelSets;
|
|
@@ -20719,7 +20723,10 @@ var init_manager = __esmMin((() => {
|
|
|
20719
20723
|
await this.spawnSemaphoreFor(backendName).run(() => {
|
|
20720
20724
|
const entry = this.spawningAgents.get(agentId);
|
|
20721
20725
|
if (entry) entry.spawnStartedAt = Date.now();
|
|
20722
|
-
return this.runSpawnWithTimeout(void 0, (signal) => backend.spawn(labelSet, agentId, this.getOrchestratorUrl(backendName), onEvent, spawnLimitsFor(effective.limits), {
|
|
20726
|
+
return this.runSpawnWithTimeout(void 0, (signal) => backend.spawn(labelSet, agentId, this.getOrchestratorUrl(backendName), onEvent, spawnLimitsFor(effective.limits), {
|
|
20727
|
+
platformTaints: this.platformTaintsFor(backendName, backend.type),
|
|
20728
|
+
mandatoryLabels: this.labelSetMandatoryLabels(backendName, backend, labelSet)
|
|
20729
|
+
}, signal));
|
|
20723
20730
|
});
|
|
20724
20731
|
this.startLogForwarding(backend, agentId);
|
|
20725
20732
|
} catch (err) {
|
|
@@ -21235,6 +21242,7 @@ var init_manager = __esmMin((() => {
|
|
|
21235
21242
|
boundJobId: jobId,
|
|
21236
21243
|
runId,
|
|
21237
21244
|
platformTaints: this.platformTaintsFor(backendName, backend.type),
|
|
21245
|
+
mandatoryLabels: this.labelSetMandatoryLabels(backendName, backend, spawnLabelSet),
|
|
21238
21246
|
...containerSpawn ? { container: containerSpawn } : {}
|
|
21239
21247
|
};
|
|
21240
21248
|
this.spawnSemaphoreFor(backendName).run(() => {
|
|
@@ -22596,12 +22604,16 @@ var init_manager = __esmMin((() => {
|
|
|
22596
22604
|
return backend.getScalerContext?.(agentId);
|
|
22597
22605
|
}
|
|
22598
22606
|
/**
|
|
22599
|
-
* Start the
|
|
22600
|
-
*
|
|
22607
|
+
* Start the machine-pool ledger reaper and the retirement sweep that tears
|
|
22608
|
+
* down drained retiring backends, and publish the warm-pool gauges so each
|
|
22609
|
+
* pool's target is visible before its first fill.
|
|
22610
|
+
*
|
|
22611
|
+
* The warm pools themselves are NOT started here — see
|
|
22612
|
+
* {@link startWarmPools}, which the host calls once every dependency a fill
|
|
22613
|
+
* can reach is wired.
|
|
22601
22614
|
*/
|
|
22602
22615
|
start() {
|
|
22603
22616
|
this.started = true;
|
|
22604
|
-
this.warmPool.start();
|
|
22605
22617
|
this.publishWarmPoolGauges();
|
|
22606
22618
|
if (this.machineLedger) this.machineLedger.start();
|
|
22607
22619
|
this.retirementSweep ??= setInterval(() => {
|
|
@@ -22610,6 +22622,18 @@ var init_manager = __esmMin((() => {
|
|
|
22610
22622
|
this.retirementSweep.unref?.();
|
|
22611
22623
|
}
|
|
22612
22624
|
/**
|
|
22625
|
+
* Run the warm pools' first deficit pass and arm their periodic tick.
|
|
22626
|
+
*
|
|
22627
|
+
* Separate from {@link start} because the first pass spawns immediately,
|
|
22628
|
+
* and on an event backend a spawn emits through the event emitter — which
|
|
22629
|
+
* the coordinator assigns after the scaler is built. The coordinator calls
|
|
22630
|
+
* this once the emitter exists; the worker, which hosts no event backend,
|
|
22631
|
+
* calls it right after `start()`.
|
|
22632
|
+
*/
|
|
22633
|
+
startWarmPools() {
|
|
22634
|
+
this.warmPool.start();
|
|
22635
|
+
}
|
|
22636
|
+
/**
|
|
22613
22637
|
* Provision/heal every backend's host prerequisites before spawning starts.
|
|
22614
22638
|
* Awaits each backend's optional ensureHostReady, catching per-backend so one
|
|
22615
22639
|
* scaler's host-prep failure degrades only that scaler (its spawns will fail
|
|
@@ -22861,6 +22885,7 @@ var init_manager = __esmMin((() => {
|
|
|
22861
22885
|
* coord boot or Raft leader switch. Reconstructs:
|
|
22862
22886
|
*
|
|
22863
22887
|
* - `spawningAgents` (with `boundJobId` preserved for eager-dispatch on register)
|
|
22888
|
+
* — event-backend rows only; a local-backend row is dropped, see below
|
|
22864
22889
|
* - `agentJobCorrelation` (so scaler-lifecycle events route correctly)
|
|
22865
22890
|
* - `reservations` + `perScalerUsage` (so the cap-check critical
|
|
22866
22891
|
* section reflects the cluster-wide truth, not the local empty
|
|
@@ -22882,6 +22907,7 @@ var init_manager = __esmMin((() => {
|
|
|
22882
22907
|
async recoverState() {
|
|
22883
22908
|
const recovery = {
|
|
22884
22909
|
spawningAgentsRehydrated: 0,
|
|
22910
|
+
spawningDropped: 0,
|
|
22885
22911
|
agentJobsRehydrated: 0,
|
|
22886
22912
|
reservationsRehydrated: 0,
|
|
22887
22913
|
reservationsDropped: 0,
|
|
@@ -22890,17 +22916,25 @@ var init_manager = __esmMin((() => {
|
|
|
22890
22916
|
if (!this.stateStore) return recovery;
|
|
22891
22917
|
try {
|
|
22892
22918
|
const spawning = await this.stateStore.listSpawningAgentsForOwner(this.instanceId);
|
|
22893
|
-
for (const entry of spawning)
|
|
22894
|
-
|
|
22895
|
-
|
|
22896
|
-
|
|
22897
|
-
|
|
22898
|
-
|
|
22899
|
-
|
|
22900
|
-
|
|
22901
|
-
|
|
22902
|
-
|
|
22903
|
-
|
|
22919
|
+
for (const entry of spawning) {
|
|
22920
|
+
const backendType = entry.backendType ?? this.backends.get(entry.scalerName)?.type;
|
|
22921
|
+
if (backendType !== void 0 && backendType !== ScalerBackendType.enum.event) {
|
|
22922
|
+
this.deleteUnadoptedSpawningAgentFromStore(entry.agentId);
|
|
22923
|
+
recovery.spawningDropped += 1;
|
|
22924
|
+
continue;
|
|
22925
|
+
}
|
|
22926
|
+
this.spawningAgents.set(entry.agentId, {
|
|
22927
|
+
labelSet: entry.labelSet,
|
|
22928
|
+
backendName: entry.scalerName,
|
|
22929
|
+
provisioningTargets: entry.provisioningTargets ?? [],
|
|
22930
|
+
spawnedAt: entry.spawnedAt.getTime(),
|
|
22931
|
+
spawnStartedAt: entry.spawnedAt.getTime(),
|
|
22932
|
+
...entry.backendType !== void 0 && { backendType: entry.backendType },
|
|
22933
|
+
...entry.boundJobId !== void 0 && { boundJobId: entry.boundJobId },
|
|
22934
|
+
...entry.runId !== void 0 && { runId: entry.runId }
|
|
22935
|
+
});
|
|
22936
|
+
}
|
|
22937
|
+
recovery.spawningAgentsRehydrated = spawning.length - recovery.spawningDropped;
|
|
22904
22938
|
const correlations = await this.stateStore.listAgentJobs();
|
|
22905
22939
|
for (const c of correlations) this.agentJobCorrelation.set(c.agentId, {
|
|
22906
22940
|
runId: c.runId,
|
|
@@ -24570,7 +24604,8 @@ function createAgentWsHandler(deps) {
|
|
|
24570
24604
|
if (rateLimiter) {
|
|
24571
24605
|
const messageSize = typeof evt.data === "string" ? evt.data.length : 0;
|
|
24572
24606
|
const isHeartbeat = raw !== null && typeof raw === "object" && "type" in raw && raw.type === "heartbeat";
|
|
24573
|
-
const
|
|
24607
|
+
const isSolicitedFleetChunk = raw !== null && typeof raw === "object" && raw.type === "fleet.bundle.chunk" && typeof raw.requestId === "string" && deps.fleetAgentCollector !== void 0 && wsToAgentId.has(ws) && deps.fleetAgentCollector.isPendingFor(raw.requestId, wsToAgentId.get(ws));
|
|
24608
|
+
const rlResult = rateLimiter.check(messageSize, isHeartbeat || isSolicitedFleetChunk);
|
|
24574
24609
|
if (!rlResult.allowed) {
|
|
24575
24610
|
if (rlResult.action === "disconnect") {
|
|
24576
24611
|
logger$108.warn("Rate limit disconnect", { reason: rlResult.reason });
|
|
@@ -42433,6 +42468,14 @@ var init_lock_file_fetcher = __esmMin((() => {
|
|
|
42433
42468
|
repoBasePath;
|
|
42434
42469
|
provider = "local";
|
|
42435
42470
|
/**
|
|
42471
|
+
* Never cached: `ref` is ignored below, so a cache keyed on it would serve the
|
|
42472
|
+
* first lock file read to every later trigger for the lifetime of the entry —
|
|
42473
|
+
* a `kici run --local` dispatch always triggers at `HEAD`, and an in-place run
|
|
42474
|
+
* carries its uncommitted edits under an unchanged sha. Reading the file from
|
|
42475
|
+
* disk is what makes those edits reach the run.
|
|
42476
|
+
*/
|
|
42477
|
+
cacheable = false;
|
|
42478
|
+
/**
|
|
42436
42479
|
* @param repoBasePath - Base directory for the repo(s). When repoIdentifier
|
|
42437
42480
|
* starts with 'file://', it is stripped and used as-is. Otherwise
|
|
42438
42481
|
* repoBasePath is used as the root.
|
|
@@ -46328,15 +46371,15 @@ var init_admin_config = __esmMin((() => {
|
|
|
46328
46371
|
function createHealthRoutes$1(deps = {}) {
|
|
46329
46372
|
return createHealthRoutes({
|
|
46330
46373
|
livenessInfo: () => ({
|
|
46331
|
-
version: "0.
|
|
46332
|
-
buildDate: "2026-09-
|
|
46333
|
-
buildCommit: "
|
|
46334
|
-
sdkVersion: "0.
|
|
46374
|
+
version: "0.8.0",
|
|
46375
|
+
buildDate: "2026-09-13T11:42:43.287Z",
|
|
46376
|
+
buildCommit: "57712173f",
|
|
46377
|
+
sdkVersion: "0.8.0",
|
|
46335
46378
|
sdkBundleHash: "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5",
|
|
46336
|
-
sharedVersion: "0.
|
|
46337
|
-
sharedBundleHash: "
|
|
46338
|
-
engineVersion: "0.
|
|
46339
|
-
engineBundleHash: "
|
|
46379
|
+
sharedVersion: "0.8.0",
|
|
46380
|
+
sharedBundleHash: "c1c70e41a1ec14fd7cf3a6047251e9aa11258a21e3694ea89a50e8bf14b99f9d",
|
|
46381
|
+
engineVersion: "0.8.0",
|
|
46382
|
+
engineBundleHash: "b26b2f23059c2ceb06a65feb2dcfe2d8c8be45ccf66622823ffedf742d2d7599"
|
|
46340
46383
|
}),
|
|
46341
46384
|
readinessCheck: deps.db ? async () => {
|
|
46342
46385
|
const checks = {};
|
|
@@ -46371,7 +46414,7 @@ function createCapabilitiesRoutes() {
|
|
|
46371
46414
|
const app = new Hono();
|
|
46372
46415
|
app.get("/api/v1/capabilities", (c) => {
|
|
46373
46416
|
const manifest = {
|
|
46374
|
-
orchestratorVersion: "0.
|
|
46417
|
+
orchestratorVersion: "0.8.0",
|
|
46375
46418
|
protocolVersion: PROTOCOL_VERSION,
|
|
46376
46419
|
minProtocolVersion: MIN_PROTOCOL_VERSION
|
|
46377
46420
|
};
|
|
@@ -49690,7 +49733,7 @@ var init_app = __esmMin((() => {
|
|
|
49690
49733
|
init_log_chunk_sink();
|
|
49691
49734
|
init_agent_metrics_aggregator();
|
|
49692
49735
|
logger$62 = createLogger({ prefix: "app" });
|
|
49693
|
-
ORCHESTRATOR_VERSION$2 = "0.
|
|
49736
|
+
ORCHESTRATOR_VERSION$2 = "0.8.0";
|
|
49694
49737
|
SourceLocationStore = class {
|
|
49695
49738
|
cache = /* @__PURE__ */ new Map();
|
|
49696
49739
|
key(workflowName, jobName) {
|
|
@@ -49745,6 +49788,15 @@ var init_fleet_agent_collector = __esmMin((() => {
|
|
|
49745
49788
|
onChunk(requestId, seq, dataB64, isLast) {
|
|
49746
49789
|
this.waiter.onChunk(requestId, seq, dataB64, isLast);
|
|
49747
49790
|
}
|
|
49791
|
+
/**
|
|
49792
|
+
* Whether `requestId` is a collection this orchestrator asked `agentId` for
|
|
49793
|
+
* and is still waiting on. The agent handler consults this before rate
|
|
49794
|
+
* limiting a `fleet.bundle.chunk`: a solicited transfer is exempt, an
|
|
49795
|
+
* unsolicited one is not.
|
|
49796
|
+
*/
|
|
49797
|
+
isPendingFor(requestId, agentId) {
|
|
49798
|
+
return this.requestAgent.get(requestId) === agentId;
|
|
49799
|
+
}
|
|
49748
49800
|
onError(requestId, message) {
|
|
49749
49801
|
this.waiter.onError(requestId, message);
|
|
49750
49802
|
}
|
|
@@ -50893,6 +50945,9 @@ var init_container_backend = __esmMin((() => {
|
|
|
50893
50945
|
state: "spawning"
|
|
50894
50946
|
};
|
|
50895
50947
|
this.agents.set(managed.id, managed);
|
|
50948
|
+
const assertStillTracked = () => {
|
|
50949
|
+
if (this.agents.get(managed.id) !== managed) throw new Error(`Agent ${agentId} was torn down while its container was being provisioned`);
|
|
50950
|
+
};
|
|
50896
50951
|
try {
|
|
50897
50952
|
const agentEnvForwarded = [];
|
|
50898
50953
|
for (const [key, value] of Object.entries(process.env)) if (key.startsWith(KICI_AGENT_ENV_PREFIX) && value !== void 0) {
|
|
@@ -50966,17 +51021,24 @@ var init_container_backend = __esmMin((() => {
|
|
|
50966
51021
|
...this.networkIsolation && { NetworkingConfig: { EndpointsConfig: { ["kici-agent-net"]: {} } } }
|
|
50967
51022
|
});
|
|
50968
51023
|
createdContainer = container;
|
|
51024
|
+
managed.backendRef = container.id;
|
|
51025
|
+
this.containerToManaged.set(container.id, managed.id);
|
|
51026
|
+
assertStillTracked();
|
|
50969
51027
|
if (this.networkIsolation) emit(ScalerEventType$1.enum["scaler.network"], "configuring network isolation");
|
|
50970
51028
|
await container.start({ abortSignal: signal });
|
|
51029
|
+
assertStillTracked();
|
|
50971
51030
|
emit(ScalerEventType$1.enum["scaler.ready"], "container started");
|
|
50972
|
-
if (this.networkIsolation)
|
|
50973
|
-
|
|
50974
|
-
|
|
50975
|
-
|
|
50976
|
-
|
|
50977
|
-
|
|
50978
|
-
|
|
50979
|
-
|
|
51031
|
+
if (this.networkIsolation) {
|
|
51032
|
+
await this.applyIsolationRules({
|
|
51033
|
+
agentId,
|
|
51034
|
+
managedId: managed.id,
|
|
51035
|
+
containerId: container.id,
|
|
51036
|
+
networkPolicy: matchedLabelSet.networkPolicy,
|
|
51037
|
+
orchestratorUrl,
|
|
51038
|
+
...signal ? { signal } : {}
|
|
51039
|
+
});
|
|
51040
|
+
assertStillTracked();
|
|
51041
|
+
}
|
|
50980
51042
|
try {
|
|
50981
51043
|
const capture = await createContainerLogCapture(this.docker, container.id);
|
|
50982
51044
|
this.logCaptures.set(managed.id, capture);
|
|
@@ -50986,13 +51048,17 @@ var init_container_backend = __esmMin((() => {
|
|
|
50986
51048
|
error: toErrorMessage(err)
|
|
50987
51049
|
});
|
|
50988
51050
|
}
|
|
51051
|
+
assertStillTracked();
|
|
50989
51052
|
managed.state = "running";
|
|
50990
|
-
managed.backendRef = container.id;
|
|
50991
|
-
this.containerToManaged.set(container.id, managed.id);
|
|
50992
51053
|
emit(ScalerEventType$1.enum["agent.connecting"], "waiting for agent WS registration");
|
|
50993
51054
|
return managed;
|
|
50994
51055
|
} catch (err) {
|
|
50995
|
-
const
|
|
51056
|
+
const capture = this.logCaptures.get(managed.id);
|
|
51057
|
+
const t = capture?.tail() ?? "";
|
|
51058
|
+
if (capture) {
|
|
51059
|
+
capture.close();
|
|
51060
|
+
this.logCaptures.delete(managed.id);
|
|
51061
|
+
}
|
|
50996
51062
|
const base = toErrorMessage(err);
|
|
50997
51063
|
emit(ScalerEventType$1.enum["scaler.failed"], t ? `${base}\n--- captured output ---\n${t}` : base);
|
|
50998
51064
|
const failedIp = this.containerIps.get(managed.id);
|
|
@@ -51002,10 +51068,13 @@ var init_container_backend = __esmMin((() => {
|
|
|
51002
51068
|
} catch {}
|
|
51003
51069
|
this.containerIps.delete(managed.id);
|
|
51004
51070
|
}
|
|
51005
|
-
if (createdContainer)
|
|
51006
|
-
|
|
51007
|
-
|
|
51008
|
-
|
|
51071
|
+
if (createdContainer) {
|
|
51072
|
+
this.containerToManaged.delete(createdContainer.id);
|
|
51073
|
+
try {
|
|
51074
|
+
await createdContainer.remove({ force: true });
|
|
51075
|
+
} catch {}
|
|
51076
|
+
}
|
|
51077
|
+
if (this.agents.get(managed.id) === managed) this.agents.delete(managed.id);
|
|
51009
51078
|
throw err;
|
|
51010
51079
|
}
|
|
51011
51080
|
}
|
|
@@ -51042,6 +51111,7 @@ var init_container_backend = __esmMin((() => {
|
|
|
51042
51111
|
capture.close();
|
|
51043
51112
|
this.logCaptures.delete(managedId);
|
|
51044
51113
|
}
|
|
51114
|
+
if (!managed.backendRef) return;
|
|
51045
51115
|
try {
|
|
51046
51116
|
const container = this.docker.getContainer(managed.backendRef);
|
|
51047
51117
|
try {
|
|
@@ -63127,7 +63197,7 @@ var logger$45, SOFTWARE_VERSION$1, PeerClient$1;
|
|
|
63127
63197
|
var init_peer_client = __esmMin((() => {
|
|
63128
63198
|
init_peer_crypto();
|
|
63129
63199
|
logger$45 = createLogger({ prefix: "peer-client" });
|
|
63130
|
-
SOFTWARE_VERSION$1 = "0.
|
|
63200
|
+
SOFTWARE_VERSION$1 = "0.8.0";
|
|
63131
63201
|
PeerClient$1 = class {
|
|
63132
63202
|
ws = null;
|
|
63133
63203
|
_state = "disconnected";
|
|
@@ -65141,7 +65211,7 @@ var init_peer_handler = __esmMin((() => {
|
|
|
65141
65211
|
init_peer_crypto();
|
|
65142
65212
|
init_join_token();
|
|
65143
65213
|
logger$43 = createLogger({ prefix: "peer-handler" });
|
|
65144
|
-
SOFTWARE_VERSION = "0.
|
|
65214
|
+
SOFTWARE_VERSION = "0.8.0";
|
|
65145
65215
|
RATE_LIMIT_MAX = 5;
|
|
65146
65216
|
RATE_LIMIT_WINDOW_MS = 6e4;
|
|
65147
65217
|
}));
|
|
@@ -73868,6 +73938,7 @@ async function bootstrapOrchestrator$1(config, hooks, options) {
|
|
|
73868
73938
|
nodeId: config.instanceId
|
|
73869
73939
|
});
|
|
73870
73940
|
eventEmitter = new EventEmitter$1(eventRouter);
|
|
73941
|
+
scalerManager?.startWarmPools();
|
|
73871
73942
|
const invokeGateDeps = {
|
|
73872
73943
|
db,
|
|
73873
73944
|
executionTracker,
|
|
@@ -77753,7 +77824,10 @@ async function clusterRouteRootJobs(args) {
|
|
|
77753
77824
|
runsOnLabels: [sel.runsOnLabels],
|
|
77754
77825
|
runsOnPatterns: sel.runsOnPatterns,
|
|
77755
77826
|
excludePatterns: sel.excludePatterns,
|
|
77756
|
-
jobConfig:
|
|
77827
|
+
jobConfig: {
|
|
77828
|
+
...buildJobConfig(mj),
|
|
77829
|
+
...ctx.extraJobConfig
|
|
77830
|
+
},
|
|
77757
77831
|
repoUrl: bundle?.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
77758
77832
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
77759
77833
|
sha: ref,
|
|
@@ -78733,7 +78807,10 @@ async function dispatchExecutionAfterInit(args) {
|
|
|
78733
78807
|
runsOnLabels: [runsOnLabels],
|
|
78734
78808
|
runsOnPatterns: selectors.runsOnPatterns,
|
|
78735
78809
|
excludePatterns: selectors.excludePatterns,
|
|
78736
|
-
jobConfig:
|
|
78810
|
+
jobConfig: {
|
|
78811
|
+
...buildJobConfig(mat),
|
|
78812
|
+
...ctx.extraJobConfig
|
|
78813
|
+
},
|
|
78737
78814
|
repoUrl: bundle?.repoUrlBuilder?.buildCloneUrl(repoIdentifier) ?? "",
|
|
78738
78815
|
ref: event.sourceBranch ?? event.targetBranch,
|
|
78739
78816
|
sha: ref,
|
|
@@ -85310,7 +85387,8 @@ async function handleTestRunStatus(msg, deps) {
|
|
|
85310
85387
|
"job_id",
|
|
85311
85388
|
"job_name",
|
|
85312
85389
|
"status",
|
|
85313
|
-
"error_message"
|
|
85390
|
+
"error_message",
|
|
85391
|
+
"duration_ms"
|
|
85314
85392
|
]).where("run_id", "=", msg.runId).execute();
|
|
85315
85393
|
return {
|
|
85316
85394
|
runId: run.run_id,
|
|
@@ -85319,7 +85397,8 @@ async function handleTestRunStatus(msg, deps) {
|
|
|
85319
85397
|
jobId: j.job_id,
|
|
85320
85398
|
jobName: j.job_name,
|
|
85321
85399
|
status: j.status,
|
|
85322
|
-
errorMessage: j.error_message ?? null
|
|
85400
|
+
errorMessage: j.error_message ?? null,
|
|
85401
|
+
durationMs: j.duration_ms ?? null
|
|
85323
85402
|
})),
|
|
85324
85403
|
done: TERMINAL_RUN_STATES.has(run.status)
|
|
85325
85404
|
};
|
|
@@ -85682,7 +85761,7 @@ function parseRelayPayload(body, headers) {
|
|
|
85682
85761
|
* would be a second, redundant gate — and a refusal from it would be charged to
|
|
85683
85762
|
* the delivery as a failed replay attempt, which is the defect this ordering
|
|
85684
85763
|
* exists to remove. The slot is genuinely held: nothing here bypasses the
|
|
85685
|
-
* controller, the decision was
|
|
85764
|
+
* controller, the decision was made one step earlier.
|
|
85686
85765
|
*/
|
|
85687
85766
|
function buildRelayReinject(seams) {
|
|
85688
85767
|
return async (d) => {
|
|
@@ -93842,6 +93921,17 @@ var init_in_memory_job_queue = __esmMin((() => {
|
|
|
93842
93921
|
async getPendingJobs() {
|
|
93843
93922
|
return [...this.jobs.values()].filter((j) => j.status === "pending");
|
|
93844
93923
|
}
|
|
93924
|
+
/**
|
|
93925
|
+
* Oldest-first listing of pending jobs, capped at `limit` — the worker-side
|
|
93926
|
+
* twin of `JobQueue.listPending`. The dispatcher's capacity-freed re-drive
|
|
93927
|
+
* (`retryPendingScaleRequests`) reads it on every worker whose scaler frees
|
|
93928
|
+
* a slot; `worker-core` wires that hook exactly as the coordinator does.
|
|
93929
|
+
* Insertion order is the enqueue order, which is what `created_at ASC` gives
|
|
93930
|
+
* the DB queue.
|
|
93931
|
+
*/
|
|
93932
|
+
async listPending(limit) {
|
|
93933
|
+
return (await this.getPendingJobs()).slice(0, Math.max(0, limit));
|
|
93934
|
+
}
|
|
93845
93935
|
/** Always returns empty array. */
|
|
93846
93936
|
async getDispatchedJobIdsByRunId(_runId) {
|
|
93847
93937
|
return [];
|
|
@@ -94350,6 +94440,7 @@ async function initializeWorkerScaler(config, tokenStore, onScalerEvent, tokenTt
|
|
|
94350
94440
|
}
|
|
94351
94441
|
await scalerManager.ensureHostsReady();
|
|
94352
94442
|
scalerManager.start();
|
|
94443
|
+
scalerManager.startWarmPools();
|
|
94353
94444
|
logger$2.info("Worker scaler initialized", {
|
|
94354
94445
|
backends: backends.map((b) => b.name),
|
|
94355
94446
|
globalMaxAgents: scalerConfig.globalMaxAgents
|
|
@@ -95042,14 +95133,14 @@ var init_worker_core = __esmMin((() => {
|
|
|
95042
95133
|
init_peer_outbox();
|
|
95043
95134
|
init_worker_outbox_relay();
|
|
95044
95135
|
init_app_on_error();
|
|
95045
|
-
ORCHESTRATOR_VERSION$1 = "0.
|
|
95046
|
-
WORKER_BUILD_COMMIT = "
|
|
95047
|
-
WORKER_SDK_VERSION = "0.
|
|
95136
|
+
ORCHESTRATOR_VERSION$1 = "0.8.0";
|
|
95137
|
+
WORKER_BUILD_COMMIT = "57712173f";
|
|
95138
|
+
WORKER_SDK_VERSION = "0.8.0";
|
|
95048
95139
|
WORKER_SDK_BUNDLE_HASH = "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5";
|
|
95049
|
-
WORKER_SHARED_VERSION = "0.
|
|
95050
|
-
WORKER_SHARED_BUNDLE_HASH = "
|
|
95051
|
-
WORKER_ENGINE_VERSION = "0.
|
|
95052
|
-
WORKER_ENGINE_BUNDLE_HASH = "
|
|
95140
|
+
WORKER_SHARED_VERSION = "0.8.0";
|
|
95141
|
+
WORKER_SHARED_BUNDLE_HASH = "c1c70e41a1ec14fd7cf3a6047251e9aa11258a21e3694ea89a50e8bf14b99f9d";
|
|
95142
|
+
WORKER_ENGINE_VERSION = "0.8.0";
|
|
95143
|
+
WORKER_ENGINE_BUNDLE_HASH = "b26b2f23059c2ceb06a65feb2dcfe2d8c8be45ccf66622823ffedf742d2d7599";
|
|
95053
95144
|
logger$2 = createLogger({ prefix: "worker" });
|
|
95054
95145
|
DRAIN_TIMEOUT_MS = 3e5;
|
|
95055
95146
|
}));
|
|
@@ -95076,14 +95167,14 @@ init_processor();
|
|
|
95076
95167
|
init_verify_inbound();
|
|
95077
95168
|
init_ingest_overflow_types();
|
|
95078
95169
|
init_scope_routing();
|
|
95079
|
-
const ORCHESTRATOR_VERSION = "0.
|
|
95080
|
-
const BUILD_COMMIT = "
|
|
95081
|
-
const SDK_VERSION = "0.
|
|
95170
|
+
const ORCHESTRATOR_VERSION = "0.8.0";
|
|
95171
|
+
const BUILD_COMMIT = "57712173f";
|
|
95172
|
+
const SDK_VERSION = "0.8.0";
|
|
95082
95173
|
const SDK_BUNDLE_HASH = "065963c7765dc8d87e04d45f57d7e15be1613da705e4ff3ec3742fd1408b7bf5";
|
|
95083
|
-
const SHARED_VERSION = "0.
|
|
95084
|
-
const SHARED_BUNDLE_HASH = "
|
|
95085
|
-
const ENGINE_VERSION = "0.
|
|
95086
|
-
const ENGINE_BUNDLE_HASH = "
|
|
95174
|
+
const SHARED_VERSION = "0.8.0";
|
|
95175
|
+
const SHARED_BUNDLE_HASH = "c1c70e41a1ec14fd7cf3a6047251e9aa11258a21e3694ea89a50e8bf14b99f9d";
|
|
95176
|
+
const ENGINE_VERSION = "0.8.0";
|
|
95177
|
+
const ENGINE_BUNDLE_HASH = "b26b2f23059c2ceb06a65feb2dcfe2d8c8be45ccf66622823ffedf742d2d7599";
|
|
95087
95178
|
const otelSdk = initTelemetry({
|
|
95088
95179
|
serviceName: "kici-orchestrator",
|
|
95089
95180
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|