@kici-dev/agent 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/diagnostics/mini-bundle.d.ts +9 -0
- package/dist/execution/cache/cache-engine.d.ts +62 -0
- package/dist/execution/cache/cache-phase.d.ts +29 -0
- package/dist/execution/cache/index.d.ts +9 -0
- package/dist/execution/dynamic-job-serializer.d.ts +12 -1
- package/dist/execution/env-init/init-phase.d.ts +64 -0
- package/dist/execution/init-runner.d.ts +1 -1
- package/dist/execution/job-runner.d.ts +30 -0
- package/dist/execution/sandbox/env-delta.d.ts +44 -0
- package/dist/execution/sandbox/env-file.d.ts +39 -0
- package/dist/execution/sandbox/index.d.ts +1 -1
- package/dist/execution/sandbox/ipc-protocol.d.ts +111 -3
- package/dist/execution/sandbox/job-deadline.d.ts +13 -0
- package/dist/execution/sandbox/step-loop.d.ts +53 -1
- package/dist/execution/sandbox/types.d.ts +20 -1
- package/dist/execution/sandbox/workflow-runner.d.ts +9 -1
- package/dist/execution/tmp-gc.d.ts +17 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +342 -12
- package/dist/server.js +623 -168
- package/dist/workflow-runner.js +2001 -950
- package/dist/ws/orchestrator-client.d.ts +55 -3
- package/package.json +16 -10
- package/sbom.spdx.json +1923 -148
package/dist/server.js
CHANGED
|
@@ -3,21 +3,24 @@ import { dirname as __cjs_dirname } from "node:path";
|
|
|
3
3
|
__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
|
|
4
4
|
import { register } from "node:module";
|
|
5
5
|
import crypto$1, { createCipheriv, createHash, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
|
6
|
+
import * as os$1 from "node:os";
|
|
6
7
|
import os, { hostname, tmpdir } from "node:os";
|
|
7
8
|
import { PassThrough, Readable, Transform, Writable } from "node:stream";
|
|
8
9
|
import { serve } from "@hono/node-server";
|
|
9
10
|
import { Hono } from "hono";
|
|
10
11
|
import winston from "winston";
|
|
11
|
-
import { RingBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, normalizeLineEndings, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
12
|
+
import { RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
12
13
|
import { z } from "zod";
|
|
13
14
|
import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
|
|
14
|
-
import { ALLOWED_SYSTEM_VARS, ExecutionJobStatus, ExecutionStepStatus, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, deriveOsArchLabels, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, resolveRoleLabels, validateNoReservedLabels } from "@kici-dev/engine";
|
|
15
|
+
import { ALLOWED_SYSTEM_VARS, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, deriveOsArchLabels, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, resolveRoleLabels, validateNoReservedLabels } from "@kici-dev/engine";
|
|
15
16
|
import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
|
|
16
17
|
import WebSocket from "ws";
|
|
18
|
+
import archiver from "archiver";
|
|
17
19
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
18
20
|
import { format, promisify } from "node:util";
|
|
19
21
|
import { existsSync } from "node:fs";
|
|
20
22
|
import path, { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
23
|
+
import { gcStaleTmpDirs } from "@kici-dev/core/tmp-gc";
|
|
21
24
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
22
25
|
import fs, { access, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, unlink, writeFile } from "node:fs/promises";
|
|
23
26
|
import Docker from "dockerode";
|
|
@@ -176,6 +179,46 @@ function agentClientConnectionOptions(config) {
|
|
|
176
179
|
};
|
|
177
180
|
}
|
|
178
181
|
//#endregion
|
|
182
|
+
//#region src/diagnostics/mini-bundle.ts
|
|
183
|
+
/**
|
|
184
|
+
* Agent fleet mini-bundle assembler.
|
|
185
|
+
*
|
|
186
|
+
* Builds an in-memory ZIP of the agent's recent logs, system info, redacted
|
|
187
|
+
* config, and current Prometheus metrics text, streamed to the orchestrator on
|
|
188
|
+
* a fleet.logs.request. No diagnostics runner exists agent-side, so this is a
|
|
189
|
+
* lean subset of the orchestrator's createDebugBundle.
|
|
190
|
+
*/
|
|
191
|
+
async function buildAgentMiniBundle(opts) {
|
|
192
|
+
const archive = archiver("zip", { zlib: { level: 6 } });
|
|
193
|
+
const chunks = [];
|
|
194
|
+
archive.on("data", (d) => chunks.push(d));
|
|
195
|
+
const done = new Promise((resolve, reject) => {
|
|
196
|
+
archive.on("end", resolve);
|
|
197
|
+
archive.on("error", reject);
|
|
198
|
+
});
|
|
199
|
+
archive.append(JSON.stringify({
|
|
200
|
+
kind: "agent",
|
|
201
|
+
agentId: opts.agentId,
|
|
202
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
203
|
+
}, null, 2), { name: "manifest.json" });
|
|
204
|
+
archive.append(JSON.stringify(redactConfig(opts.config), null, 2), { name: "config/config.json" });
|
|
205
|
+
archive.append(JSON.stringify({
|
|
206
|
+
hostname: os$1.hostname(),
|
|
207
|
+
platform: process.platform,
|
|
208
|
+
arch: process.arch,
|
|
209
|
+
nodeVersion: process.version,
|
|
210
|
+
cpus: os$1.cpus().length,
|
|
211
|
+
totalmem: os$1.totalmem(),
|
|
212
|
+
freemem: os$1.freemem(),
|
|
213
|
+
uptime: os$1.uptime()
|
|
214
|
+
}, null, 2), { name: "system/info.json" });
|
|
215
|
+
if (opts.metricsText) archive.append(opts.metricsText, { name: "system/metrics.txt" });
|
|
216
|
+
if (opts.logDir) await addLogsToArchive(archive, opts.logDir, opts.logWindowHours);
|
|
217
|
+
await archive.finalize();
|
|
218
|
+
await done;
|
|
219
|
+
return Buffer.concat(chunks);
|
|
220
|
+
}
|
|
221
|
+
//#endregion
|
|
179
222
|
//#region src/ws/event-buffer.ts
|
|
180
223
|
/**
|
|
181
224
|
* In-memory buffer for agent-to-orchestrator messages during disconnection.
|
|
@@ -207,7 +250,7 @@ var LogBuffer = class extends RingBuffer {
|
|
|
207
250
|
};
|
|
208
251
|
//#endregion
|
|
209
252
|
//#region src/ws/orchestrator-client.ts
|
|
210
|
-
const logger$
|
|
253
|
+
const logger$11 = createLogger({ prefix: "orchestrator-client" });
|
|
211
254
|
/**
|
|
212
255
|
* WebSocket client that connects the agent to the customer orchestrator.
|
|
213
256
|
*
|
|
@@ -240,6 +283,15 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
240
283
|
pendingEventEmitRequests = /* @__PURE__ */ new Map();
|
|
241
284
|
/** Pending agent.api.request calls awaiting orchestrator response. */
|
|
242
285
|
pendingApiRequests = /* @__PURE__ */ new Map();
|
|
286
|
+
/** Pending user-cache restore/save requests awaiting orchestrator response. */
|
|
287
|
+
pendingUserCacheRequests = /* @__PURE__ */ new Map();
|
|
288
|
+
/**
|
|
289
|
+
* Pending step-approval requests awaiting the orchestrator's resolution.
|
|
290
|
+
* No client-side timeout: the orchestrator owns the (org-/SDK-configured)
|
|
291
|
+
* expiry and sends `step.approval-resolved: expired` when it lapses. The
|
|
292
|
+
* workflow-runner carries an outer safety-net timeout.
|
|
293
|
+
*/
|
|
294
|
+
pendingStepApprovals = /* @__PURE__ */ new Map();
|
|
243
295
|
/** Pending concurrency report requests awaiting orchestrator ack. */
|
|
244
296
|
pendingConcurrencyRequests = /* @__PURE__ */ new Map();
|
|
245
297
|
url;
|
|
@@ -253,6 +305,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
253
305
|
getInFlightJobs;
|
|
254
306
|
roles;
|
|
255
307
|
scalerManaged;
|
|
308
|
+
getFleetBundleInputs;
|
|
256
309
|
/** Timestamp when the connection was lost, used for gap marker outage duration. */
|
|
257
310
|
disconnectedAt = null;
|
|
258
311
|
/** Set to true when auth.failure is received. Prevents retrying with a bad token. */
|
|
@@ -279,6 +332,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
279
332
|
this.getInFlightJobs = options.getInFlightJobs;
|
|
280
333
|
this.roles = options.roles;
|
|
281
334
|
this.scalerManaged = options.scalerManaged ?? false;
|
|
335
|
+
this.getFleetBundleInputs = options.getFleetBundleInputs;
|
|
282
336
|
this.eventBuffer = new EventBuffer({ maxSize: options.maxBufferSize ?? 5e3 });
|
|
283
337
|
this.logBuffer = new LogBuffer({ maxLines: options.maxLogBufferLines ?? 1e4 });
|
|
284
338
|
}
|
|
@@ -311,7 +365,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
311
365
|
*/
|
|
312
366
|
connect() {
|
|
313
367
|
if (this._state !== "disconnected") {
|
|
314
|
-
logger$
|
|
368
|
+
logger$11.warn("connect() called while not disconnected", { state: this._state });
|
|
315
369
|
return;
|
|
316
370
|
}
|
|
317
371
|
this.intentionalDisconnect = false;
|
|
@@ -450,6 +504,36 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
450
504
|
});
|
|
451
505
|
}
|
|
452
506
|
/**
|
|
507
|
+
* Build this agent's fleet mini-bundle and stream it back to the orchestrator
|
|
508
|
+
* as ordered fleet.bundle.chunk frames (the WS frame cap forbids one frame).
|
|
509
|
+
* On failure, sends a single fleet.bundle.error. Public for unit testing.
|
|
510
|
+
*/
|
|
511
|
+
async streamFleetBundle(req) {
|
|
512
|
+
try {
|
|
513
|
+
const inputs = await this.getFleetBundleInputs?.() ?? { config: {} };
|
|
514
|
+
const buf = await buildAgentMiniBundle({
|
|
515
|
+
agentId: this.agentId,
|
|
516
|
+
logDir: inputs.logDir,
|
|
517
|
+
logWindowHours: req.logWindowHours,
|
|
518
|
+
config: inputs.config,
|
|
519
|
+
metricsText: inputs.metricsText
|
|
520
|
+
});
|
|
521
|
+
for (const f of chunkBuffer(buf)) this.sendDirect({
|
|
522
|
+
type: "fleet.bundle.chunk",
|
|
523
|
+
requestId: req.requestId,
|
|
524
|
+
seq: f.seq,
|
|
525
|
+
isLast: f.isLast,
|
|
526
|
+
dataB64: f.dataB64
|
|
527
|
+
});
|
|
528
|
+
} catch (err) {
|
|
529
|
+
this.sendDirect({
|
|
530
|
+
type: "fleet.bundle.error",
|
|
531
|
+
requestId: req.requestId,
|
|
532
|
+
message: toErrorMessage(err)
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
453
537
|
* Send a typed API request to the orchestrator and await the response.
|
|
454
538
|
*
|
|
455
539
|
* This is the transport layer for the agent private API. The SDK's typed
|
|
@@ -481,6 +565,100 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
481
565
|
});
|
|
482
566
|
}
|
|
483
567
|
/**
|
|
568
|
+
* Relay a user-facing cache request from the sandbox to the orchestrator.
|
|
569
|
+
*
|
|
570
|
+
* Translates the sandbox `cache.request` IPC into the matching `cache.user.*`
|
|
571
|
+
* WS message and resolves with the orchestrator's response mapped onto the
|
|
572
|
+
* IPC response shape:
|
|
573
|
+
*
|
|
574
|
+
* - `restore` -> `cache.user.restore.request`, awaits `cache.user.restore.response`.
|
|
575
|
+
* - `beginSave` -> `cache.user.save.request`, awaits `cache.user.save.response`.
|
|
576
|
+
* - `completeSave` -> `cache.user.save.complete` (fire-and-forget; the
|
|
577
|
+
* orchestrator commits temp -> final without replying), resolves immediately.
|
|
578
|
+
*
|
|
579
|
+
* Times out after 30 seconds for the round-trip ops.
|
|
580
|
+
*/
|
|
581
|
+
async requestUserCache(jobId, request) {
|
|
582
|
+
if (request.op === "completeSave") {
|
|
583
|
+
this.sendDirect({
|
|
584
|
+
type: "cache.user.save.complete",
|
|
585
|
+
messageId: randomUUID(),
|
|
586
|
+
jobId,
|
|
587
|
+
key: request.key,
|
|
588
|
+
tarHash: request.tarHash,
|
|
589
|
+
sizeBytes: request.sizeBytes
|
|
590
|
+
});
|
|
591
|
+
return {
|
|
592
|
+
type: "cache.response",
|
|
593
|
+
requestId: request.requestId
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
const messageId = randomUUID();
|
|
597
|
+
return new Promise((resolve, reject) => {
|
|
598
|
+
const timer = setTimeout(() => {
|
|
599
|
+
this.pendingUserCacheRequests.delete(messageId);
|
|
600
|
+
reject(/* @__PURE__ */ new Error("User-cache request timed out (30s)"));
|
|
601
|
+
}, 3e4);
|
|
602
|
+
this.pendingUserCacheRequests.set(messageId, {
|
|
603
|
+
resolve: (response) => {
|
|
604
|
+
clearTimeout(timer);
|
|
605
|
+
resolve({
|
|
606
|
+
...response,
|
|
607
|
+
requestId: request.requestId
|
|
608
|
+
});
|
|
609
|
+
},
|
|
610
|
+
reject: (err) => {
|
|
611
|
+
clearTimeout(timer);
|
|
612
|
+
reject(err);
|
|
613
|
+
}
|
|
614
|
+
});
|
|
615
|
+
if (request.op === "restore") this.sendDirect({
|
|
616
|
+
type: "cache.user.restore.request",
|
|
617
|
+
messageId,
|
|
618
|
+
jobId,
|
|
619
|
+
key: request.key,
|
|
620
|
+
...request.restoreKeys && { restoreKeys: request.restoreKeys }
|
|
621
|
+
});
|
|
622
|
+
else this.sendDirect({
|
|
623
|
+
type: "cache.user.save.request",
|
|
624
|
+
messageId,
|
|
625
|
+
jobId,
|
|
626
|
+
key: request.key
|
|
627
|
+
});
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Relay a step-level approval request to the orchestrator. Sends a
|
|
632
|
+
* `step.approval-request` WS message and resolves with the orchestrator's
|
|
633
|
+
* `step.approval-resolved` mapped onto the IPC response shape. No client-side
|
|
634
|
+
* timeout — the orchestrator owns the approval expiry and replies with an
|
|
635
|
+
* `expired` outcome when it lapses. Rejects only on disconnect (the relay
|
|
636
|
+
* caller treats a rejection as a fail-closed reject).
|
|
637
|
+
*/
|
|
638
|
+
async sendStepApproval(runId, jobId, request) {
|
|
639
|
+
const messageId = randomUUID();
|
|
640
|
+
return new Promise((resolve, reject) => {
|
|
641
|
+
this.pendingStepApprovals.set(messageId, {
|
|
642
|
+
resolve: (response) => resolve({
|
|
643
|
+
...response,
|
|
644
|
+
requestId: request.requestId
|
|
645
|
+
}),
|
|
646
|
+
reject
|
|
647
|
+
});
|
|
648
|
+
this.sendDirect({
|
|
649
|
+
type: "step.approval-request",
|
|
650
|
+
messageId,
|
|
651
|
+
runId,
|
|
652
|
+
jobId,
|
|
653
|
+
stepIndex: request.stepIndex,
|
|
654
|
+
stepName: request.stepName,
|
|
655
|
+
clauses: request.clauses,
|
|
656
|
+
reason: request.reason,
|
|
657
|
+
...request.timeoutSeconds !== void 0 && { timeoutSeconds: request.timeoutSeconds }
|
|
658
|
+
});
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
/**
|
|
484
662
|
* Send a job.context message to the orchestrator.
|
|
485
663
|
*
|
|
486
664
|
* Conveys execution environment details (runtime, sandbox type, env vars)
|
|
@@ -562,7 +740,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
562
740
|
}
|
|
563
741
|
});
|
|
564
742
|
} catch (err) {
|
|
565
|
-
logger$
|
|
743
|
+
logger$11.error("Failed to create WebSocket", { error: toErrorMessage(err) });
|
|
566
744
|
this._state = "disconnected";
|
|
567
745
|
this.scheduleReconnect();
|
|
568
746
|
return;
|
|
@@ -570,7 +748,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
570
748
|
this.ws.on("open", () => {
|
|
571
749
|
if (this.token) {
|
|
572
750
|
this._state = "authenticating";
|
|
573
|
-
logger$
|
|
751
|
+
logger$11.info("Connected to orchestrator, sending auth.request", {
|
|
574
752
|
url: this.url,
|
|
575
753
|
agentId: this.agentId
|
|
576
754
|
});
|
|
@@ -581,7 +759,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
581
759
|
}));
|
|
582
760
|
} else {
|
|
583
761
|
this._state = "registering";
|
|
584
|
-
logger$
|
|
762
|
+
logger$11.info("Connected to orchestrator, sending agent.register (no token)", {
|
|
585
763
|
url: this.url,
|
|
586
764
|
agentId: this.agentId
|
|
587
765
|
});
|
|
@@ -592,12 +770,12 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
592
770
|
this.handleMessage(data);
|
|
593
771
|
});
|
|
594
772
|
this.ws.on("close", (code, reason) => {
|
|
595
|
-
logger$
|
|
773
|
+
logger$11.info("Orchestrator connection closed", {
|
|
596
774
|
code,
|
|
597
775
|
reason: reason.toString()
|
|
598
776
|
});
|
|
599
777
|
if (code === WS_CLOSE_AGENT_AUTH_FAILED) {
|
|
600
|
-
logger$
|
|
778
|
+
logger$11.error("Orchestrator closed with auth-failed code -- token is invalid or revoked. NOT retrying.", {
|
|
601
779
|
code,
|
|
602
780
|
reason: reason.toString()
|
|
603
781
|
});
|
|
@@ -614,12 +792,16 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
614
792
|
this.pendingEventEmitRequests.clear();
|
|
615
793
|
for (const [_id, pending] of this.pendingApiRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
616
794
|
this.pendingApiRequests.clear();
|
|
795
|
+
for (const [_id, pending] of this.pendingUserCacheRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
796
|
+
this.pendingUserCacheRequests.clear();
|
|
617
797
|
for (const [_id, pending] of this.pendingConcurrencyRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
618
798
|
this.pendingConcurrencyRequests.clear();
|
|
799
|
+
for (const [_id, pending] of this.pendingStepApprovals) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
800
|
+
this.pendingStepApprovals.clear();
|
|
619
801
|
if (!this.intentionalDisconnect) this.scheduleReconnect();
|
|
620
802
|
});
|
|
621
803
|
this.ws.on("error", (err) => {
|
|
622
|
-
logger$
|
|
804
|
+
logger$11.error(`Orchestrator WebSocket error: ${err.message}`);
|
|
623
805
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
624
806
|
});
|
|
625
807
|
}
|
|
@@ -628,7 +810,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
628
810
|
try {
|
|
629
811
|
raw = JSON.parse(data.toString());
|
|
630
812
|
} catch {
|
|
631
|
-
logger$
|
|
813
|
+
logger$11.warn("Malformed JSON received from orchestrator");
|
|
632
814
|
return;
|
|
633
815
|
}
|
|
634
816
|
const rawMsg = raw;
|
|
@@ -662,19 +844,37 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
662
844
|
}
|
|
663
845
|
return;
|
|
664
846
|
}
|
|
847
|
+
if (rawMsg.type === "cache.user.restore.response" || rawMsg.type === "cache.user.save.response") {
|
|
848
|
+
const cacheMsg = raw;
|
|
849
|
+
const pending = this.pendingUserCacheRequests.get(cacheMsg.requestId);
|
|
850
|
+
if (pending) {
|
|
851
|
+
this.pendingUserCacheRequests.delete(cacheMsg.requestId);
|
|
852
|
+
pending.resolve({
|
|
853
|
+
type: "cache.response",
|
|
854
|
+
requestId: cacheMsg.requestId,
|
|
855
|
+
...cacheMsg.hit !== void 0 && { hit: cacheMsg.hit },
|
|
856
|
+
...cacheMsg.matchedKey && { matchedKey: cacheMsg.matchedKey },
|
|
857
|
+
...cacheMsg.downloadUrl && { downloadUrl: cacheMsg.downloadUrl },
|
|
858
|
+
...cacheMsg.tarHash && { tarHash: cacheMsg.tarHash },
|
|
859
|
+
...cacheMsg.skip !== void 0 && { skip: cacheMsg.skip },
|
|
860
|
+
...cacheMsg.uploadUrl && { uploadUrl: cacheMsg.uploadUrl }
|
|
861
|
+
});
|
|
862
|
+
}
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
665
865
|
const parsed = orchestratorToAgentMessageSchema.safeParse(raw);
|
|
666
866
|
if (parsed.success) {
|
|
667
867
|
const msg = parsed.data;
|
|
668
868
|
switch (msg.type) {
|
|
669
869
|
case "auth.success":
|
|
670
870
|
if (this._state === "authenticating") {
|
|
671
|
-
logger$
|
|
871
|
+
logger$11.info("Authentication successful, sending agent.register", { connectionId: msg.connectionId });
|
|
672
872
|
this._state = "registering";
|
|
673
873
|
this.sendAgentRegister();
|
|
674
874
|
}
|
|
675
875
|
break;
|
|
676
876
|
case "auth.failure":
|
|
677
|
-
logger$
|
|
877
|
+
logger$11.error("Authentication FAILED -- token is invalid or expired. NOT retrying.", { reason: msg.reason });
|
|
678
878
|
this.authFailed = true;
|
|
679
879
|
this.intentionalDisconnect = true;
|
|
680
880
|
if (this.ws) {
|
|
@@ -684,7 +884,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
684
884
|
this._state = "disconnected";
|
|
685
885
|
break;
|
|
686
886
|
case "register.ack":
|
|
687
|
-
logger$
|
|
887
|
+
logger$11.info("Registration acknowledged by orchestrator", {
|
|
688
888
|
agentId: msg.agentId,
|
|
689
889
|
labels: msg.labels,
|
|
690
890
|
scalerManaged: msg.scalerManaged,
|
|
@@ -699,14 +899,14 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
699
899
|
this.sendConfigAck(msg.agentId);
|
|
700
900
|
break;
|
|
701
901
|
case "job.dispatch":
|
|
702
|
-
logger$
|
|
902
|
+
logger$11.info("Job dispatch received", {
|
|
703
903
|
runId: msg.runId,
|
|
704
904
|
jobId: msg.jobId
|
|
705
905
|
});
|
|
706
906
|
this.onJobDispatch(msg);
|
|
707
907
|
break;
|
|
708
908
|
case "job.cancel":
|
|
709
|
-
logger$
|
|
909
|
+
logger$11.info("Job cancel received", {
|
|
710
910
|
runId: msg.runId,
|
|
711
911
|
jobId: msg.jobId,
|
|
712
912
|
reason: msg.reason
|
|
@@ -714,7 +914,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
714
914
|
this.onJobCancel(msg);
|
|
715
915
|
break;
|
|
716
916
|
case "job.concurrency.ack": {
|
|
717
|
-
logger$
|
|
917
|
+
logger$11.info("Concurrency ack received", {
|
|
718
918
|
requestId: msg.requestId,
|
|
719
919
|
action: msg.action
|
|
720
920
|
});
|
|
@@ -728,11 +928,38 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
728
928
|
}
|
|
729
929
|
break;
|
|
730
930
|
}
|
|
931
|
+
case "step.approval-resolved": {
|
|
932
|
+
logger$11.info("Step approval resolved", {
|
|
933
|
+
requestId: msg.requestId,
|
|
934
|
+
runId: msg.runId,
|
|
935
|
+
jobId: msg.jobId,
|
|
936
|
+
stepIndex: msg.stepIndex,
|
|
937
|
+
outcome: msg.outcome
|
|
938
|
+
});
|
|
939
|
+
const pending = this.pendingStepApprovals.get(msg.requestId);
|
|
940
|
+
if (pending) {
|
|
941
|
+
this.pendingStepApprovals.delete(msg.requestId);
|
|
942
|
+
pending.resolve({
|
|
943
|
+
type: "approval.resolved",
|
|
944
|
+
requestId: msg.requestId,
|
|
945
|
+
outcome: msg.outcome,
|
|
946
|
+
...msg.reason !== void 0 && { reason: msg.reason }
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
break;
|
|
950
|
+
}
|
|
951
|
+
case "fleet.logs.request":
|
|
952
|
+
logger$11.info("Fleet log collection requested", {
|
|
953
|
+
requestId: msg.requestId,
|
|
954
|
+
logWindowHours: msg.logWindowHours
|
|
955
|
+
});
|
|
956
|
+
this.streamFleetBundle(msg);
|
|
957
|
+
break;
|
|
731
958
|
}
|
|
732
959
|
return;
|
|
733
960
|
}
|
|
734
961
|
if (heartbeatSchema.safeParse(raw).success) return;
|
|
735
|
-
logger$
|
|
962
|
+
logger$11.warn("Invalid message from orchestrator", { errors: parsed.error.issues });
|
|
736
963
|
}
|
|
737
964
|
flushBuffer() {
|
|
738
965
|
const events = this.eventBuffer.flush();
|
|
@@ -746,11 +973,11 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
746
973
|
}
|
|
747
974
|
this.disconnectedAt = null;
|
|
748
975
|
if (events.length > 0) {
|
|
749
|
-
logger$
|
|
976
|
+
logger$11.info("Flushing event buffer", { count: events.length });
|
|
750
977
|
for (const msg of events) this.sendDirect(msg);
|
|
751
978
|
}
|
|
752
979
|
if (logLines.length > 0) {
|
|
753
|
-
logger$
|
|
980
|
+
logger$11.info("Flushing log buffer", { count: logLines.length });
|
|
754
981
|
for (let i = 0; i < logLines.length; i += OrchestratorClient.LOG_BATCH_SIZE) {
|
|
755
982
|
const batch = logLines.slice(i, i + OrchestratorClient.LOG_BATCH_SIZE);
|
|
756
983
|
this.sendAgentLogMessage(batch);
|
|
@@ -803,14 +1030,14 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
803
1030
|
*/
|
|
804
1031
|
blockMmdsAccess() {
|
|
805
1032
|
if (process.getuid?.() !== 0) {
|
|
806
|
-
logger$
|
|
1033
|
+
logger$11.info("MMDS iptables block skipped (non-root) — network isolation handled by orchestrator");
|
|
807
1034
|
return;
|
|
808
1035
|
}
|
|
809
1036
|
try {
|
|
810
1037
|
execSync("iptables -A OUTPUT -d 169.254.169.254 -j DROP", { timeout: 5e3 });
|
|
811
|
-
logger$
|
|
1038
|
+
logger$11.info("MMDS access blocked via iptables");
|
|
812
1039
|
} catch (err) {
|
|
813
|
-
logger$
|
|
1040
|
+
logger$11.warn("Failed to block MMDS access via iptables", { error: toErrorMessage(err) });
|
|
814
1041
|
}
|
|
815
1042
|
}
|
|
816
1043
|
/**
|
|
@@ -824,7 +1051,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
824
1051
|
messageId: `config-ack-${agentId}-${Date.now()}`,
|
|
825
1052
|
agentId
|
|
826
1053
|
}));
|
|
827
|
-
logger$
|
|
1054
|
+
logger$11.info("Config ACK sent to orchestrator", { agentId });
|
|
828
1055
|
}
|
|
829
1056
|
}
|
|
830
1057
|
startHeartbeat() {
|
|
@@ -885,12 +1112,12 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
885
1112
|
scheduleReconnect() {
|
|
886
1113
|
this.cancelReconnect();
|
|
887
1114
|
if (this.authFailed) {
|
|
888
|
-
logger$
|
|
1115
|
+
logger$11.error("Not reconnecting: authentication permanently failed");
|
|
889
1116
|
return;
|
|
890
1117
|
}
|
|
891
1118
|
const delay = this.getReconnectDelay();
|
|
892
1119
|
this.reconnectAttempts++;
|
|
893
|
-
logger$
|
|
1120
|
+
logger$11.info("Scheduling reconnect", {
|
|
894
1121
|
attempt: this.reconnectAttempts,
|
|
895
1122
|
delayMs: Math.round(delay)
|
|
896
1123
|
});
|
|
@@ -975,14 +1202,14 @@ var init_console_capture = __esmMin((() => {
|
|
|
975
1202
|
init_console_capture();
|
|
976
1203
|
function safe(name, fallback = "unknown") {
|
|
977
1204
|
switch (name) {
|
|
978
|
-
case "version": return "0.1.
|
|
979
|
-
case "buildCommit": return "
|
|
980
|
-
case "sdkVersion": return "0.1.
|
|
981
|
-
case "sdkBundleHash": return "
|
|
982
|
-
case "sharedVersion": return "0.1.
|
|
983
|
-
case "sharedBundleHash": return "
|
|
984
|
-
case "engineVersion": return "0.1.
|
|
985
|
-
case "engineBundleHash": return "
|
|
1205
|
+
case "version": return "0.1.16";
|
|
1206
|
+
case "buildCommit": return "7d97bb32c";
|
|
1207
|
+
case "sdkVersion": return "0.1.16";
|
|
1208
|
+
case "sdkBundleHash": return "cd4b8e0d91efe578c96e7c746f89b44a3df8fcaebaec2185399168ef8eb4ebca";
|
|
1209
|
+
case "sharedVersion": return "0.1.16";
|
|
1210
|
+
case "sharedBundleHash": return "c58b1596e92c8423ef83958e86894cb150315d8b91578a0080f1c645000e93e8";
|
|
1211
|
+
case "engineVersion": return "0.1.16";
|
|
1212
|
+
case "engineBundleHash": return "a611c0017d08faa9c5aa4fd97c3dd6259f53f3cca248bd2b369ad5d30c394847";
|
|
986
1213
|
default: return fallback;
|
|
987
1214
|
}
|
|
988
1215
|
}
|
|
@@ -1203,6 +1430,42 @@ function verifyNpmAvailable() {
|
|
|
1203
1430
|
}
|
|
1204
1431
|
var init_npm_resolver = __esmMin((() => {}));
|
|
1205
1432
|
//#endregion
|
|
1433
|
+
//#region src/execution/tmp-gc.ts
|
|
1434
|
+
init_npm_resolver();
|
|
1435
|
+
/**
|
|
1436
|
+
* Startup garbage collection for this agent's own temp-directory families.
|
|
1437
|
+
*
|
|
1438
|
+
* Job workdirs (`kici-<6 random chars>`, see job-runner.ts) and isolated
|
|
1439
|
+
* pnpm stores (`kici-pnpm-store-*`, see dep-installer.ts) clean themselves
|
|
1440
|
+
* up in `finally` blocks — but a hard process death (SIGKILL, OOM kill)
|
|
1441
|
+
* skips those, and on a long-lived bare-metal agent the leftovers then
|
|
1442
|
+
* accumulate forever. Collecting anything older than a day at startup is
|
|
1443
|
+
* safe on shared hosts: no job lives remotely that long (job timeouts are
|
|
1444
|
+
* minutes), so a concurrent agent's in-flight dirs are never eligible.
|
|
1445
|
+
*/
|
|
1446
|
+
const AGENT_TMP_GC_MAX_AGE_MS = 1440 * 60 * 1e3;
|
|
1447
|
+
/** mkdtemp's 6-char suffix on the bare `kici-` prefix — job workdirs only. */
|
|
1448
|
+
const AGENT_WORKDIR_PATTERN = /^kici-[A-Za-z0-9]{6}$/;
|
|
1449
|
+
const PNPM_STORE_PATTERN = /^kici-pnpm-store-/;
|
|
1450
|
+
/**
|
|
1451
|
+
* Collect this agent's stale temp dirs. `base` is overridable for tests;
|
|
1452
|
+
* production callers use the default temp root. Never throws.
|
|
1453
|
+
*/
|
|
1454
|
+
async function gcStaleAgentTmpDirs(base = tmpdir()) {
|
|
1455
|
+
const log = (m) => logger.info(m);
|
|
1456
|
+
return [...await gcStaleTmpDirs({
|
|
1457
|
+
base,
|
|
1458
|
+
pattern: AGENT_WORKDIR_PATTERN,
|
|
1459
|
+
maxAgeMs: AGENT_TMP_GC_MAX_AGE_MS,
|
|
1460
|
+
log
|
|
1461
|
+
}), ...await gcStaleTmpDirs({
|
|
1462
|
+
base,
|
|
1463
|
+
pattern: PNPM_STORE_PATTERN,
|
|
1464
|
+
maxAgeMs: AGENT_TMP_GC_MAX_AGE_MS,
|
|
1465
|
+
log
|
|
1466
|
+
})];
|
|
1467
|
+
}
|
|
1468
|
+
//#endregion
|
|
1206
1469
|
//#region src/metrics/prometheus.ts
|
|
1207
1470
|
var prometheus_exports = /* @__PURE__ */ __exportAll({
|
|
1208
1471
|
cloneDurationSeconds: () => cloneDurationSeconds,
|
|
@@ -1630,8 +1893,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
|
|
|
1630
1893
|
}
|
|
1631
1894
|
var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
|
|
1632
1895
|
var init_workflow_loader = __esmMin((() => {
|
|
1633
|
-
AGENT_SDK_VERSION = "0.1.
|
|
1634
|
-
AGENT_SDK_BUNDLE_HASH = "
|
|
1896
|
+
AGENT_SDK_VERSION = "0.1.16";
|
|
1897
|
+
AGENT_SDK_BUNDLE_HASH = "cd4b8e0d91efe578c96e7c746f89b44a3df8fcaebaec2185399168ef8eb4ebca";
|
|
1635
1898
|
hookRegistered = false;
|
|
1636
1899
|
}));
|
|
1637
1900
|
//#endregion
|
|
@@ -1652,7 +1915,7 @@ var init_workflow_loader = __esmMin((() => {
|
|
|
1652
1915
|
async function packKiciSource(workDir) {
|
|
1653
1916
|
const kiciDir = join(workDir, ".kici");
|
|
1654
1917
|
if (!existsSync(kiciDir)) throw new Error(`.kici/ not found at ${kiciDir}`);
|
|
1655
|
-
logger$
|
|
1918
|
+
logger$10.info("Packing .kici/ source tarball", { dir: workDir });
|
|
1656
1919
|
const startTime = Date.now();
|
|
1657
1920
|
const stream = c({
|
|
1658
1921
|
gzip: true,
|
|
@@ -1666,7 +1929,7 @@ async function packKiciSource(workDir) {
|
|
|
1666
1929
|
const hash = sha256(tarball);
|
|
1667
1930
|
const sizeKB = (tarball.length / 1024).toFixed(2);
|
|
1668
1931
|
const durationMs = Date.now() - startTime;
|
|
1669
|
-
logger$
|
|
1932
|
+
logger$10.info(".kici/ source packed", {
|
|
1670
1933
|
sizeKB,
|
|
1671
1934
|
hash: hash.slice(0, 12),
|
|
1672
1935
|
durationMs
|
|
@@ -1676,9 +1939,9 @@ async function packKiciSource(workDir) {
|
|
|
1676
1939
|
hash
|
|
1677
1940
|
};
|
|
1678
1941
|
}
|
|
1679
|
-
var logger$
|
|
1942
|
+
var logger$10;
|
|
1680
1943
|
var init_source_packer = __esmMin((() => {
|
|
1681
|
-
logger$
|
|
1944
|
+
logger$10 = createLogger({ prefix: "source-packer" });
|
|
1682
1945
|
}));
|
|
1683
1946
|
//#endregion
|
|
1684
1947
|
//#region src/execution/dep-restore.ts
|
|
@@ -1819,7 +2082,7 @@ async function cleanupScratch(scratchDir) {
|
|
|
1819
2082
|
force: true
|
|
1820
2083
|
});
|
|
1821
2084
|
} catch (cleanupErr) {
|
|
1822
|
-
logger$
|
|
2085
|
+
logger$9.warn("Scratch dir cleanup failed (orphan left behind)", {
|
|
1823
2086
|
scratchDir,
|
|
1824
2087
|
error: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
|
|
1825
2088
|
});
|
|
@@ -1844,7 +2107,7 @@ async function cleanupScratch(scratchDir) {
|
|
|
1844
2107
|
*/
|
|
1845
2108
|
async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
1846
2109
|
depsUrl = resolveOrchestratorUrl(depsUrl);
|
|
1847
|
-
logger$
|
|
2110
|
+
logger$9.info("Downloading dependency tarball", { url: depsUrl });
|
|
1848
2111
|
const kiciDir = join(workDir, ".kici");
|
|
1849
2112
|
if (depsUrl.startsWith("file://")) {
|
|
1850
2113
|
const localPath = fileURLToPath(depsUrl);
|
|
@@ -1858,7 +2121,7 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
1858
2121
|
await moveScratchIntoRepo(scratchDir, workDir);
|
|
1859
2122
|
await cleanupScratch(scratchDir);
|
|
1860
2123
|
const sizeMB = (data.length / (1024 * 1024)).toFixed(2);
|
|
1861
|
-
logger$
|
|
2124
|
+
logger$9.info("Dependencies restored from cache (file)", {
|
|
1862
2125
|
sizeMB,
|
|
1863
2126
|
targetDir: workDir
|
|
1864
2127
|
});
|
|
@@ -1867,7 +2130,7 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
1867
2130
|
if (!depsUrl.startsWith("http://") && !depsUrl.startsWith("https://")) throw new Error(`Unsupported deps URL scheme: ${depsUrl}`);
|
|
1868
2131
|
let lastError;
|
|
1869
2132
|
for (let attempt = 0; attempt <= 2; attempt++) {
|
|
1870
|
-
if (attempt > 0) logger$
|
|
2133
|
+
if (attempt > 0) logger$9.warn("Retrying dep tarball download", {
|
|
1871
2134
|
attempt,
|
|
1872
2135
|
url: depsUrl
|
|
1873
2136
|
});
|
|
@@ -1876,11 +2139,11 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
1876
2139
|
if (depsHash && hash !== depsHash) throw new Error(`Dep tarball hash mismatch: expected ${depsHash}, got ${hash}`);
|
|
1877
2140
|
await moveScratchIntoRepo(scratchDir, workDir);
|
|
1878
2141
|
await cleanupScratch(scratchDir);
|
|
1879
|
-
logger$
|
|
2142
|
+
logger$9.info("Dependencies restored from cache (stream)", { targetDir: workDir });
|
|
1880
2143
|
return;
|
|
1881
2144
|
} catch (err) {
|
|
1882
2145
|
lastError = err instanceof Error ? err : new Error(String(err));
|
|
1883
|
-
logger$
|
|
2146
|
+
logger$9.warn("Dep tarball download failed", {
|
|
1884
2147
|
attempt,
|
|
1885
2148
|
error: lastError.message
|
|
1886
2149
|
});
|
|
@@ -1888,9 +2151,9 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
1888
2151
|
}
|
|
1889
2152
|
throw new Error(`Dep tarball download failed after 3 attempts: ${lastError?.message}`);
|
|
1890
2153
|
}
|
|
1891
|
-
var logger$
|
|
2154
|
+
var logger$9, DOWNLOAD_TIMEOUT_MS$1, SCRATCH_DIR_BASENAME_PREFIX;
|
|
1892
2155
|
var init_dep_restore = __esmMin((() => {
|
|
1893
|
-
logger$
|
|
2156
|
+
logger$9 = createLogger({ prefix: "dep-restore" });
|
|
1894
2157
|
DOWNLOAD_TIMEOUT_MS$1 = 300 * 1e3;
|
|
1895
2158
|
SCRATCH_DIR_BASENAME_PREFIX = ".dep-restore-scratch-";
|
|
1896
2159
|
`${SCRATCH_DIR_BASENAME_PREFIX}`;
|
|
@@ -2002,7 +2265,7 @@ async function extractSourceTarball(data, targetDir) {
|
|
|
2002
2265
|
}
|
|
2003
2266
|
async function restoreSource(workDir, sourceTarUrl) {
|
|
2004
2267
|
sourceTarUrl = resolveOrchestratorUrl(sourceTarUrl);
|
|
2005
|
-
logger$
|
|
2268
|
+
logger$8.info("Restoring .kici/ source from tarball", { sourceTarUrl });
|
|
2006
2269
|
const startTime = Date.now();
|
|
2007
2270
|
let data;
|
|
2008
2271
|
if (sourceTarUrl.startsWith("file://")) {
|
|
@@ -2012,16 +2275,16 @@ async function restoreSource(workDir, sourceTarUrl) {
|
|
|
2012
2275
|
else throw new Error(`Unsupported source tarball URL scheme: ${sourceTarUrl}`);
|
|
2013
2276
|
await extractSourceTarball(data, workDir);
|
|
2014
2277
|
const durationMs = Date.now() - startTime;
|
|
2015
|
-
logger$
|
|
2278
|
+
logger$8.info(".kici/ source restored", {
|
|
2016
2279
|
sizeKB: (data.length / 1024).toFixed(2),
|
|
2017
2280
|
durationMs
|
|
2018
2281
|
});
|
|
2019
2282
|
}
|
|
2020
|
-
var logger$
|
|
2283
|
+
var logger$8;
|
|
2021
2284
|
var init_source_restore = __esmMin((() => {
|
|
2022
2285
|
init_download();
|
|
2023
2286
|
init_dep_restore();
|
|
2024
|
-
logger$
|
|
2287
|
+
logger$8 = createLogger({ prefix: "source-restore" });
|
|
2025
2288
|
}));
|
|
2026
2289
|
//#endregion
|
|
2027
2290
|
//#region src/execution/timeout-util.ts
|
|
@@ -2069,7 +2332,7 @@ function findJobByName(workflow, jobName) {
|
|
|
2069
2332
|
*
|
|
2070
2333
|
* @param workflow - The extracted Workflow object
|
|
2071
2334
|
* @param jobName - Name of the job whose dynamic fields to evaluate
|
|
2072
|
-
* @param event - Normalized
|
|
2335
|
+
* @param event - Normalized event envelope — same shape every dynamic-function call site receives.
|
|
2073
2336
|
* @param flags - Which fields are dynamic and need evaluation
|
|
2074
2337
|
* @param timeoutMs - Timeout per dynamic function call (default 60_000ms)
|
|
2075
2338
|
*/
|
|
@@ -2258,7 +2521,12 @@ async function serializeMatrix(matrix, jobName, runsOn, ctx) {
|
|
|
2258
2521
|
log: ctx.log,
|
|
2259
2522
|
env: ctx.env
|
|
2260
2523
|
};
|
|
2261
|
-
|
|
2524
|
+
let values;
|
|
2525
|
+
try {
|
|
2526
|
+
values = await withTimeout(() => matrix(matrixCtx), DYNAMIC_FIELD_TIMEOUT_MS, `dynamic matrix for generated job '${jobName}'`);
|
|
2527
|
+
} catch (err) {
|
|
2528
|
+
throw new MatrixExpansionError(jobName, `Matrix expansion failed for job '${jobName}': ${err.message}`);
|
|
2529
|
+
}
|
|
2262
2530
|
if (Array.isArray(values)) return {
|
|
2263
2531
|
_type: "static",
|
|
2264
2532
|
values
|
|
@@ -2267,11 +2535,19 @@ async function serializeMatrix(matrix, jobName, runsOn, ctx) {
|
|
|
2267
2535
|
_type: "static",
|
|
2268
2536
|
values
|
|
2269
2537
|
};
|
|
2270
|
-
throw new
|
|
2538
|
+
throw new MatrixExpansionError(jobName, `Job '${jobName}': dynamic matrix function returned an unsupported value (expected array or object, got ${typeof values})`);
|
|
2271
2539
|
}
|
|
2272
|
-
var DYNAMIC_FIELD_TIMEOUT_MS;
|
|
2540
|
+
var MatrixExpansionError, DYNAMIC_FIELD_TIMEOUT_MS;
|
|
2273
2541
|
var init_dynamic_job_serializer = __esmMin((() => {
|
|
2274
2542
|
init_timeout_util();
|
|
2543
|
+
MatrixExpansionError = class MatrixExpansionError extends Error {
|
|
2544
|
+
name = "MatrixExpansionError";
|
|
2545
|
+
constructor(jobName, message) {
|
|
2546
|
+
super(message);
|
|
2547
|
+
this.jobName = jobName;
|
|
2548
|
+
Object.setPrototypeOf(this, MatrixExpansionError.prototype);
|
|
2549
|
+
}
|
|
2550
|
+
};
|
|
2275
2551
|
DYNAMIC_FIELD_TIMEOUT_MS = 6e4;
|
|
2276
2552
|
})), DEFAULT_MAX_LOG_SIZE_BYTES, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_FLUSH_LINE_THRESHOLD, PAUSE_SAFETY_TIMEOUT_MS, LogStreamer;
|
|
2277
2553
|
var init_log_streamer = __esmMin((() => {
|
|
@@ -2540,7 +2816,7 @@ async function applyOverlay(config) {
|
|
|
2540
2816
|
const { tarballUrl, cliPublicKey, orchestratorPrivateKey, repoDir } = config;
|
|
2541
2817
|
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "kici-overlay-"));
|
|
2542
2818
|
try {
|
|
2543
|
-
logger$
|
|
2819
|
+
logger$7.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
|
|
2544
2820
|
let encryptedData;
|
|
2545
2821
|
try {
|
|
2546
2822
|
encryptedData = await downloadUrl(tarballUrl);
|
|
@@ -2550,7 +2826,7 @@ async function applyOverlay(config) {
|
|
|
2550
2826
|
const cliPubKeyBuf = Buffer.from(cliPublicKey, "base64");
|
|
2551
2827
|
const aesKey = deriveSharedSecret(Buffer.from(orchestratorPrivateKey, "base64"), cliPubKeyBuf);
|
|
2552
2828
|
const decryptedData = decryptBuffer(encryptedData, aesKey);
|
|
2553
|
-
logger$
|
|
2829
|
+
logger$7.info("Extracting overlay tarball", { size: decryptedData.length });
|
|
2554
2830
|
const extractDir = path.join(tmpDir, "extracted");
|
|
2555
2831
|
await fs.mkdir(extractDir, { recursive: true });
|
|
2556
2832
|
try {
|
|
@@ -2599,10 +2875,10 @@ async function applyOverlay(config) {
|
|
|
2599
2875
|
await fs.unlink(targetPath);
|
|
2600
2876
|
filesDeleted++;
|
|
2601
2877
|
} catch {
|
|
2602
|
-
logger$
|
|
2878
|
+
logger$7.debug("Deletion target not found, skipping", { file });
|
|
2603
2879
|
}
|
|
2604
2880
|
}
|
|
2605
|
-
logger$
|
|
2881
|
+
logger$7.info("Overlay applied successfully", {
|
|
2606
2882
|
filesApplied,
|
|
2607
2883
|
filesDeleted
|
|
2608
2884
|
});
|
|
@@ -2618,10 +2894,10 @@ async function applyOverlay(config) {
|
|
|
2618
2894
|
}).catch(() => {});
|
|
2619
2895
|
}
|
|
2620
2896
|
}
|
|
2621
|
-
var logger$
|
|
2897
|
+
var logger$7, IV_LENGTH$1, AUTH_TAG_LENGTH;
|
|
2622
2898
|
var init_overlay_applier = __esmMin((() => {
|
|
2623
2899
|
init_download();
|
|
2624
|
-
logger$
|
|
2900
|
+
logger$7 = createLogger({ prefix: "overlay-applier" });
|
|
2625
2901
|
IV_LENGTH$1 = 12;
|
|
2626
2902
|
AUTH_TAG_LENGTH = 16;
|
|
2627
2903
|
}));
|
|
@@ -2935,7 +3211,7 @@ async function detectKiciPackageManager(repoRoot, kiciDir) {
|
|
|
2935
3211
|
async function installDeps(kiciDir, opts = {}) {
|
|
2936
3212
|
const repoRoot = opts.repoRoot ?? dirname(kiciDir);
|
|
2937
3213
|
const packageManager = await detectKiciPackageManager(repoRoot, kiciDir);
|
|
2938
|
-
logger$
|
|
3214
|
+
logger$6.info("Installing deps inline", {
|
|
2939
3215
|
packageManager,
|
|
2940
3216
|
dir: kiciDir
|
|
2941
3217
|
});
|
|
@@ -2976,7 +3252,7 @@ async function installDeps(kiciDir, opts = {}) {
|
|
|
2976
3252
|
if (packageManager === PackageManager.Pnpm && await kiciHasLocalProtocolDeps(kiciDir)) await buildWorkspaceClosure(repoRoot);
|
|
2977
3253
|
const durationMs = Date.now() - startTime;
|
|
2978
3254
|
process.stderr.write(`[dep-installer:trace] install complete: ${durationMs}ms\n`);
|
|
2979
|
-
logger$
|
|
3255
|
+
logger$6.info("Deps installed inline", {
|
|
2980
3256
|
packageManager,
|
|
2981
3257
|
durationMs
|
|
2982
3258
|
});
|
|
@@ -3114,12 +3390,11 @@ function logSubprocessStreams(e, tokens) {
|
|
|
3114
3390
|
if (e && typeof e === "object" && "stdout" in e) process.stderr.write(`[dep-installer:trace] stdout: ${redactNpmOutput(String(e.stdout), tokens).slice(0, 500)}\n`);
|
|
3115
3391
|
if (e && typeof e === "object" && "stderr" in e) process.stderr.write(`[dep-installer:trace] stderr: ${redactNpmOutput(String(e.stderr), tokens).slice(0, 500)}\n`);
|
|
3116
3392
|
}
|
|
3117
|
-
var logger$
|
|
3393
|
+
var logger$6, execFileAsync, INSTALL_TIMEOUT_MS, INSTALL_MAX_BUFFER;
|
|
3118
3394
|
var init_dep_installer = __esmMin((() => {
|
|
3119
|
-
init_npm_resolver();
|
|
3120
3395
|
init_npm_registry_config();
|
|
3121
3396
|
init_validate_kici_deps();
|
|
3122
|
-
logger$
|
|
3397
|
+
logger$6 = createLogger({ prefix: "dep-installer" });
|
|
3123
3398
|
execFileAsync = promisify(execFile);
|
|
3124
3399
|
INSTALL_TIMEOUT_MS = 6e5;
|
|
3125
3400
|
INSTALL_MAX_BUFFER = 128 * 1024 * 1024;
|
|
@@ -3158,7 +3433,7 @@ async function packNodeModules(kiciDir) {
|
|
|
3158
3433
|
const workDir = dirname(kiciDir);
|
|
3159
3434
|
const packageManager = await detectPackageManagerFromManifests(workDir) ?? await detectPackageManagerFromManifests(kiciDir) ?? PackageManager.Npm;
|
|
3160
3435
|
const entries = await closureEntries(workDir, kiciDir, packageManager);
|
|
3161
|
-
logger$
|
|
3436
|
+
logger$5.info("Packing dependency closure into tarball", {
|
|
3162
3437
|
dir: workDir,
|
|
3163
3438
|
packageManager,
|
|
3164
3439
|
entries
|
|
@@ -3174,7 +3449,7 @@ async function packNodeModules(kiciDir) {
|
|
|
3174
3449
|
const tarball = Buffer.concat(chunks);
|
|
3175
3450
|
const hash = sha256(tarball);
|
|
3176
3451
|
const sizeMB = (tarball.length / (1024 * 1024)).toFixed(2);
|
|
3177
|
-
logger$
|
|
3452
|
+
logger$5.info("Dependency closure packed", {
|
|
3178
3453
|
sizeMB,
|
|
3179
3454
|
hash: hash.slice(0, 12),
|
|
3180
3455
|
durationMs: Date.now() - startTime
|
|
@@ -3262,9 +3537,9 @@ function isInside(root, target) {
|
|
|
3262
3537
|
function isAbsoluteRel(rel) {
|
|
3263
3538
|
return rel.length > 1 && rel[1] === ":";
|
|
3264
3539
|
}
|
|
3265
|
-
var logger$
|
|
3540
|
+
var logger$5;
|
|
3266
3541
|
var init_dep_packer = __esmMin((() => {
|
|
3267
|
-
logger$
|
|
3542
|
+
logger$5 = createLogger({ prefix: "dep-packer" });
|
|
3268
3543
|
}));
|
|
3269
3544
|
//#endregion
|
|
3270
3545
|
//#region src/execution/sandbox/env-sanitizer.ts
|
|
@@ -3427,6 +3702,7 @@ function buildRequest(dispatch, workDir) {
|
|
|
3427
3702
|
contentHash: jobConfig.contentHash,
|
|
3428
3703
|
resolvedHashFiles: jobConfig.resolvedHashFiles,
|
|
3429
3704
|
maxLogSizeBytes: dispatch.maxLogSizeBytes,
|
|
3705
|
+
jobTimeoutMs: jobConfig.timeout,
|
|
3430
3706
|
container: jobConfig.container,
|
|
3431
3707
|
event: jobConfig.event,
|
|
3432
3708
|
provider: jobConfig.provider,
|
|
@@ -3668,6 +3944,42 @@ function relayAgentApiRequest(msg, ctx) {
|
|
|
3668
3944
|
error: toErrorMessage(err)
|
|
3669
3945
|
}));
|
|
3670
3946
|
}
|
|
3947
|
+
/** Relay `cache.request` and pipe the orchestrator response (or an error
|
|
3948
|
+
* response, or a "not configured" response when the agent didn't provide
|
|
3949
|
+
* the callback) back into the sandbox runner. */
|
|
3950
|
+
function relayCacheRequest$1(msg, ctx) {
|
|
3951
|
+
if (!ctx.execOptions.onCacheRequest) {
|
|
3952
|
+
safeSendToChild(ctx.child, {
|
|
3953
|
+
type: "cache.response",
|
|
3954
|
+
requestId: msg.requestId,
|
|
3955
|
+
error: "Cache not available in this agent configuration"
|
|
3956
|
+
});
|
|
3957
|
+
return;
|
|
3958
|
+
}
|
|
3959
|
+
ctx.execOptions.onCacheRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
|
|
3960
|
+
type: "cache.response",
|
|
3961
|
+
requestId: msg.requestId,
|
|
3962
|
+
error: toErrorMessage(err)
|
|
3963
|
+
}));
|
|
3964
|
+
}
|
|
3965
|
+
/** Relay `approval.request` and pipe the orchestrator's resolution (or a
|
|
3966
|
+
* fail-closed reject when the callback isn't wired or the relay throws) back
|
|
3967
|
+
* into the sandbox runner. */
|
|
3968
|
+
function relayApprovalRequest$1(msg, ctx) {
|
|
3969
|
+
if (!ctx.execOptions.onApprovalRequest) {
|
|
3970
|
+
safeSendToChild(ctx.child, {
|
|
3971
|
+
type: "approval.resolved",
|
|
3972
|
+
requestId: msg.requestId,
|
|
3973
|
+
error: "Approvals not available in this agent configuration"
|
|
3974
|
+
});
|
|
3975
|
+
return;
|
|
3976
|
+
}
|
|
3977
|
+
ctx.execOptions.onApprovalRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
|
|
3978
|
+
type: "approval.resolved",
|
|
3979
|
+
requestId: msg.requestId,
|
|
3980
|
+
error: toErrorMessage(err)
|
|
3981
|
+
}));
|
|
3982
|
+
}
|
|
3671
3983
|
/** Resolve the result promise for `job.complete` IPC messages. Encrypts
|
|
3672
3984
|
* secret outputs (when a runPublicKey is available) and overrides status to
|
|
3673
3985
|
* `cancelled` if a cancel was already in flight. */
|
|
@@ -3714,7 +4026,9 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
3714
4026
|
ctx.execOptions.onStepStatus(msg.stepIndex, ctx.stepNames.get(msg.stepIndex) ?? "", msg.status, {
|
|
3715
4027
|
durationMs: msg.durationMs,
|
|
3716
4028
|
...msg.error && { error: msg.error },
|
|
3717
|
-
...msg.secretsAccessed && { secretsAccessed: msg.secretsAccessed }
|
|
4029
|
+
...msg.secretsAccessed && { secretsAccessed: msg.secretsAccessed },
|
|
4030
|
+
...msg.step_type && { step_type: msg.step_type },
|
|
4031
|
+
...msg.data && msg.data
|
|
3718
4032
|
});
|
|
3719
4033
|
return;
|
|
3720
4034
|
case "step.secret_mount":
|
|
@@ -3735,6 +4049,12 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
3735
4049
|
case "agent.api.request":
|
|
3736
4050
|
relayAgentApiRequest(msg, ctx);
|
|
3737
4051
|
return;
|
|
4052
|
+
case "cache.request":
|
|
4053
|
+
relayCacheRequest$1(msg, ctx);
|
|
4054
|
+
return;
|
|
4055
|
+
case "approval.request":
|
|
4056
|
+
relayApprovalRequest$1(msg, ctx);
|
|
4057
|
+
return;
|
|
3738
4058
|
case "job.complete":
|
|
3739
4059
|
handleJobComplete(msg, dispatch, ctx);
|
|
3740
4060
|
return;
|
|
@@ -3905,10 +4225,10 @@ var init_fork_runner = __esmMin((() => {
|
|
|
3905
4225
|
* network access. This mode provides credential isolation only and should
|
|
3906
4226
|
* be used in trusted environments.
|
|
3907
4227
|
*/
|
|
3908
|
-
var logger$
|
|
4228
|
+
var logger$4, BareMetalSandbox;
|
|
3909
4229
|
var init_bare_metal_sandbox = __esmMin((() => {
|
|
3910
4230
|
init_fork_runner();
|
|
3911
|
-
logger$
|
|
4231
|
+
logger$4 = createLogger({ prefix: "bare-metal-sandbox" });
|
|
3912
4232
|
BareMetalSandbox = class {
|
|
3913
4233
|
runnerPath;
|
|
3914
4234
|
useBwrap;
|
|
@@ -3935,12 +4255,12 @@ var init_bare_metal_sandbox = __esmMin((() => {
|
|
|
3935
4255
|
if (this.useBwrap) try {
|
|
3936
4256
|
const { execSync } = await import("node:child_process");
|
|
3937
4257
|
execSync("which bwrap", { stdio: "ignore" });
|
|
3938
|
-
if (this.sandboxNetwork === "isolated") logger$
|
|
3939
|
-
else logger$
|
|
4258
|
+
if (this.sandboxNetwork === "isolated") logger$4.info("Bubblewrap (bwrap) sandbox enabled with network isolation (--unshare-net)");
|
|
4259
|
+
else logger$4.info("Bubblewrap (bwrap) sandbox enabled with host network (KICI_SANDBOX_NETWORK=host)");
|
|
3940
4260
|
} catch {
|
|
3941
4261
|
throw new Error("Bubblewrap (bwrap) not found. Install bubblewrap or set sandbox=false. On Debian/Ubuntu: apt install bubblewrap");
|
|
3942
4262
|
}
|
|
3943
|
-
else logger$
|
|
4263
|
+
else logger$4.warn("Bare-metal without sandbox provides limited isolation. Only environment sanitization is active. Enable sandbox=true with bubblewrap for PID/IPC/filesystem namespace isolation.");
|
|
3944
4264
|
}
|
|
3945
4265
|
/**
|
|
3946
4266
|
* Execute a job by forking the workflow runner with sanitized environment.
|
|
@@ -4130,6 +4450,58 @@ function relayApiRequest(stream, options, apiMsg) {
|
|
|
4130
4450
|
} catch {}
|
|
4131
4451
|
}
|
|
4132
4452
|
/**
|
|
4453
|
+
* Relay cache.request from the container runner to the orchestrator via
|
|
4454
|
+
* options.onCacheRequest, then write the response back through `stream`. If
|
|
4455
|
+
* the agent doesn't expose a cache relay, write a structured error response
|
|
4456
|
+
* so the runner doesn't hang.
|
|
4457
|
+
*/
|
|
4458
|
+
function relayCacheRequest(stream, options, cacheMsg) {
|
|
4459
|
+
const writeResponse = (response) => {
|
|
4460
|
+
try {
|
|
4461
|
+
stream.write(JSON.stringify(response) + "\n");
|
|
4462
|
+
} catch {}
|
|
4463
|
+
};
|
|
4464
|
+
if (!options.onCacheRequest) {
|
|
4465
|
+
writeResponse({
|
|
4466
|
+
type: "cache.response",
|
|
4467
|
+
requestId: cacheMsg.requestId,
|
|
4468
|
+
error: "Cache not available in this agent configuration"
|
|
4469
|
+
});
|
|
4470
|
+
return;
|
|
4471
|
+
}
|
|
4472
|
+
options.onCacheRequest(cacheMsg).then((response) => writeResponse(response), (err) => writeResponse({
|
|
4473
|
+
type: "cache.response",
|
|
4474
|
+
requestId: cacheMsg.requestId,
|
|
4475
|
+
error: toErrorMessage(err)
|
|
4476
|
+
}));
|
|
4477
|
+
}
|
|
4478
|
+
/**
|
|
4479
|
+
* Relay approval.request from the container runner to the orchestrator via
|
|
4480
|
+
* options.onApprovalRequest, then write the resolution back through `stream`.
|
|
4481
|
+
* If the agent doesn't expose an approval relay (or it throws), write a
|
|
4482
|
+
* fail-closed reject so the runner doesn't hang.
|
|
4483
|
+
*/
|
|
4484
|
+
function relayApprovalRequest(stream, options, approvalMsg) {
|
|
4485
|
+
const writeResponse = (response) => {
|
|
4486
|
+
try {
|
|
4487
|
+
stream.write(JSON.stringify(response) + "\n");
|
|
4488
|
+
} catch {}
|
|
4489
|
+
};
|
|
4490
|
+
if (!options.onApprovalRequest) {
|
|
4491
|
+
writeResponse({
|
|
4492
|
+
type: "approval.resolved",
|
|
4493
|
+
requestId: approvalMsg.requestId,
|
|
4494
|
+
error: "Approvals not available in this agent configuration"
|
|
4495
|
+
});
|
|
4496
|
+
return;
|
|
4497
|
+
}
|
|
4498
|
+
options.onApprovalRequest(approvalMsg).then((response) => writeResponse(response), (err) => writeResponse({
|
|
4499
|
+
type: "approval.resolved",
|
|
4500
|
+
requestId: approvalMsg.requestId,
|
|
4501
|
+
error: toErrorMessage(err)
|
|
4502
|
+
}));
|
|
4503
|
+
}
|
|
4504
|
+
/**
|
|
4133
4505
|
* Apply a job.complete message to the mutable runner state: capture status,
|
|
4134
4506
|
* merge any bulk-reported step results, propagate plain outputs, and encrypt
|
|
4135
4507
|
* secret outputs if a run public key is available.
|
|
@@ -4144,14 +4516,14 @@ function applyJobComplete(msg, stepResults, state, options) {
|
|
|
4144
4516
|
if (msg.secretOutputs && options.dispatch.runPublicKey) try {
|
|
4145
4517
|
state.encryptedSecretOutputs = encryptSecretOutputs(msg.secretOutputs, options.dispatch.runPublicKey);
|
|
4146
4518
|
} catch (err) {
|
|
4147
|
-
logger$
|
|
4519
|
+
logger$3.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
|
|
4148
4520
|
}
|
|
4149
4521
|
}
|
|
4150
|
-
var logger$
|
|
4522
|
+
var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, ContainerSandbox;
|
|
4151
4523
|
var init_container_sandbox = __esmMin((() => {
|
|
4152
4524
|
init_fork_runner();
|
|
4153
4525
|
init_secret_encryption();
|
|
4154
|
-
logger$
|
|
4526
|
+
logger$3 = createLogger({ prefix: "container-sandbox" });
|
|
4155
4527
|
MAX_STDERR_LINES = 20;
|
|
4156
4528
|
ABORT_GRACE_MS = 1e4;
|
|
4157
4529
|
CONTAINER_STOP_TIMEOUT = 10;
|
|
@@ -4183,7 +4555,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4183
4555
|
async setup(options) {
|
|
4184
4556
|
this.containerName = `kici-sandbox-${this.jobId}-${Date.now()}`;
|
|
4185
4557
|
const envArray = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
|
|
4186
|
-
logger$
|
|
4558
|
+
logger$3.info("Creating sandbox container", {
|
|
4187
4559
|
name: this.containerName,
|
|
4188
4560
|
image: this.image,
|
|
4189
4561
|
workDir: options.workDir
|
|
@@ -4201,7 +4573,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4201
4573
|
HostConfig: { Binds: [`${options.workDir}:/workspace`, `${this.runnerPath}:${this.runnerMountPath}:ro`] }
|
|
4202
4574
|
});
|
|
4203
4575
|
await this.container.start();
|
|
4204
|
-
logger$
|
|
4576
|
+
logger$3.info("Sandbox container started", {
|
|
4205
4577
|
name: this.containerName,
|
|
4206
4578
|
containerId: this.container.id.slice(0, 12)
|
|
4207
4579
|
});
|
|
@@ -4214,7 +4586,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4214
4586
|
try {
|
|
4215
4587
|
outcome = await this.awaitJobCompletion(streamCtx, options);
|
|
4216
4588
|
} catch (err) {
|
|
4217
|
-
logger$
|
|
4589
|
+
logger$3.error("Job execution error", {
|
|
4218
4590
|
error: toErrorMessage(err),
|
|
4219
4591
|
stderrTail: streamCtx.stderrLines.slice(-5).join("\n")
|
|
4220
4592
|
});
|
|
@@ -4264,7 +4636,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4264
4636
|
});
|
|
4265
4637
|
const abortHandler = () => {
|
|
4266
4638
|
this.handleAbort().catch((err) => {
|
|
4267
|
-
logger$
|
|
4639
|
+
logger$3.warn("Error during abort", { error: toErrorMessage(err) });
|
|
4268
4640
|
});
|
|
4269
4641
|
};
|
|
4270
4642
|
options.signal.addEventListener("abort", abortHandler, { once: true });
|
|
@@ -4301,7 +4673,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4301
4673
|
try {
|
|
4302
4674
|
msg = JSON.parse(line);
|
|
4303
4675
|
} catch {
|
|
4304
|
-
logger$
|
|
4676
|
+
logger$3.warn("Non-JSON output from runner", { line: line.slice(0, 200) });
|
|
4305
4677
|
return;
|
|
4306
4678
|
}
|
|
4307
4679
|
if (this.dispatchRunnerMessage(msg, stream, options, stepNames, stepResults, state)) resolve({
|
|
@@ -4349,7 +4721,9 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4349
4721
|
options.onStepStatus(msg.stepIndex, name, msg.status, {
|
|
4350
4722
|
durationMs: msg.durationMs,
|
|
4351
4723
|
...msg.error && { error: msg.error },
|
|
4352
|
-
...msg.secretsAccessed && { secretsAccessed: msg.secretsAccessed }
|
|
4724
|
+
...msg.secretsAccessed && { secretsAccessed: msg.secretsAccessed },
|
|
4725
|
+
...msg.step_type && { step_type: msg.step_type },
|
|
4726
|
+
...msg.data && msg.data
|
|
4353
4727
|
});
|
|
4354
4728
|
stepResults.push({
|
|
4355
4729
|
name,
|
|
@@ -4381,11 +4755,17 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4381
4755
|
case "agent.api.request":
|
|
4382
4756
|
relayApiRequest(stream, options, msg);
|
|
4383
4757
|
return false;
|
|
4758
|
+
case "cache.request":
|
|
4759
|
+
relayCacheRequest(stream, options, msg);
|
|
4760
|
+
return false;
|
|
4761
|
+
case "approval.request":
|
|
4762
|
+
relayApprovalRequest(stream, options, msg);
|
|
4763
|
+
return false;
|
|
4384
4764
|
case "job.complete":
|
|
4385
4765
|
applyJobComplete(msg, stepResults, state, options);
|
|
4386
4766
|
return true;
|
|
4387
4767
|
default:
|
|
4388
|
-
logger$
|
|
4768
|
+
logger$3.warn("Unrecognized IPC message from container runner", { type: msg.type });
|
|
4389
4769
|
return false;
|
|
4390
4770
|
}
|
|
4391
4771
|
}
|
|
@@ -4412,14 +4792,14 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4412
4792
|
async teardown() {
|
|
4413
4793
|
if (!this.container) return;
|
|
4414
4794
|
if (this.keepFailed && this.jobFailed) {
|
|
4415
|
-
logger$
|
|
4795
|
+
logger$3.info("Keeping failed container for debugging", {
|
|
4416
4796
|
name: this.containerName,
|
|
4417
4797
|
containerId: this.container.id.slice(0, 12)
|
|
4418
4798
|
});
|
|
4419
4799
|
this.container = null;
|
|
4420
4800
|
return;
|
|
4421
4801
|
}
|
|
4422
|
-
logger$
|
|
4802
|
+
logger$3.info("Tearing down sandbox container", { name: this.containerName });
|
|
4423
4803
|
try {
|
|
4424
4804
|
await this.container.stop({ t: CONTAINER_STOP_TIMEOUT });
|
|
4425
4805
|
} catch {}
|
|
@@ -4450,7 +4830,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
4450
4830
|
*/
|
|
4451
4831
|
async handleAbort() {
|
|
4452
4832
|
if (!this.execStream && !this.container) return;
|
|
4453
|
-
logger$
|
|
4833
|
+
logger$3.info("Aborting sandbox execution", { name: this.containerName });
|
|
4454
4834
|
if (this.execStream) try {
|
|
4455
4835
|
this.execStream.write(JSON.stringify({ type: "abort" }) + "\n");
|
|
4456
4836
|
} catch {}
|
|
@@ -4519,7 +4899,7 @@ function determineExecutionMode(jobConfig, agentConfig) {
|
|
|
4519
4899
|
if (agentConfig.scalerManaged) return "firecracker";
|
|
4520
4900
|
return "bare-metal";
|
|
4521
4901
|
}
|
|
4522
|
-
var logger$
|
|
4902
|
+
var logger$2, JobRunner$1;
|
|
4523
4903
|
var init_job_runner = __esmMin((() => {
|
|
4524
4904
|
init_git_clone();
|
|
4525
4905
|
init_workflow_loader();
|
|
@@ -4536,7 +4916,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4536
4916
|
init_download();
|
|
4537
4917
|
init_sandbox();
|
|
4538
4918
|
init_prometheus();
|
|
4539
|
-
logger$
|
|
4919
|
+
logger$2 = createLogger({ prefix: "job-runner" });
|
|
4540
4920
|
JobRunner$1 = class {
|
|
4541
4921
|
send;
|
|
4542
4922
|
sendDirect;
|
|
@@ -4550,6 +4930,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
4550
4930
|
_sendRunEvent;
|
|
4551
4931
|
_sendConcurrencyReport;
|
|
4552
4932
|
_sendApiRequest;
|
|
4933
|
+
_requestUserCache;
|
|
4934
|
+
_sendStepApproval;
|
|
4553
4935
|
/** Tracks running jobs for concurrency and cancellation */
|
|
4554
4936
|
activeJobs = /* @__PURE__ */ new Map();
|
|
4555
4937
|
/** Active sandbox for the current job (used for abort). */
|
|
@@ -4567,6 +4949,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
4567
4949
|
this._sendRunEvent = deps.sendRunEvent;
|
|
4568
4950
|
this._sendConcurrencyReport = deps.sendConcurrencyReport;
|
|
4569
4951
|
this._sendApiRequest = deps.sendApiRequest;
|
|
4952
|
+
this._requestUserCache = deps.requestUserCache;
|
|
4953
|
+
this._sendStepApproval = deps.sendStepApproval;
|
|
4570
4954
|
}
|
|
4571
4955
|
/**
|
|
4572
4956
|
* Execute a dispatched job through its full lifecycle.
|
|
@@ -4634,7 +5018,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4634
5018
|
}
|
|
4635
5019
|
if (jobConfig.buildOnly === true) {
|
|
4636
5020
|
if (jobConfig.fullRepo) {
|
|
4637
|
-
logger$
|
|
5021
|
+
logger$2.warn("Build job received for fullRepo run -- skipping (should not happen)", {
|
|
4638
5022
|
jobId,
|
|
4639
5023
|
runId
|
|
4640
5024
|
});
|
|
@@ -4656,7 +5040,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4656
5040
|
async executeStandardJob(dispatch, workDir, abortController) {
|
|
4657
5041
|
const { runId, jobId } = dispatch;
|
|
4658
5042
|
const ctx = getRequestContext();
|
|
4659
|
-
logger$
|
|
5043
|
+
logger$2.info(`Run: ${ctx.runId ?? runId} | Trace: ${ctx.requestId ?? "N/A"}`);
|
|
4660
5044
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
|
|
4661
5045
|
const heartbeatTimer = setInterval(() => {
|
|
4662
5046
|
this.send({
|
|
@@ -4681,7 +5065,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4681
5065
|
if (sandbox) {
|
|
4682
5066
|
this.emitRunEvent(runId, "agent.teardown", { jobId });
|
|
4683
5067
|
await sandbox.teardown().catch((err) => {
|
|
4684
|
-
logger$
|
|
5068
|
+
logger$2.warn("Sandbox teardown error", { error: toErrorMessage(err) });
|
|
4685
5069
|
});
|
|
4686
5070
|
}
|
|
4687
5071
|
}
|
|
@@ -4709,7 +5093,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4709
5093
|
environmentVars: typedConfig.environmentVars ?? void 0,
|
|
4710
5094
|
jobEnv: typedConfig.jobEnv ?? void 0
|
|
4711
5095
|
});
|
|
4712
|
-
logger$
|
|
5096
|
+
logger$2.info("Creating execution sandbox", {
|
|
4713
5097
|
executionMode,
|
|
4714
5098
|
jobId,
|
|
4715
5099
|
runnerPath
|
|
@@ -4780,6 +5164,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4780
5164
|
onStepStatus: (stepIndex, stepName, state, data) => {
|
|
4781
5165
|
let logBytesStreamed;
|
|
4782
5166
|
if (state === ExecutionStepStatus.enum.success || state === ExecutionStepStatus.enum.failed || state === ExecutionStepStatus.enum.skipped) logBytesStreamed = logStreamers.get(stepIndex)?.getTotalBytes() ?? 0;
|
|
5167
|
+
this.maybeEmitCacheRunEvent(runId, jobId, stepIndex, state, data);
|
|
4783
5168
|
this.sendStepStatus(dispatch, stepIndex, stepName, state, data, logBytesStreamed);
|
|
4784
5169
|
},
|
|
4785
5170
|
onLogLine: (stepIndex, line) => {
|
|
@@ -4804,6 +5189,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
4804
5189
|
};
|
|
4805
5190
|
},
|
|
4806
5191
|
onApiRequest: this._sendApiRequest ? async (method, params) => this._sendApiRequest(method, params) : void 0,
|
|
5192
|
+
onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
|
|
5193
|
+
onApprovalRequest: this._sendStepApproval ? async (request) => this._sendStepApproval(dispatch.runId, dispatch.jobId, request) : void 0,
|
|
4807
5194
|
onSecretMount: (event) => {
|
|
4808
5195
|
this.emitRunEvent(runId, "step.secret_mount", {
|
|
4809
5196
|
jobId,
|
|
@@ -4837,7 +5224,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4837
5224
|
stepsTotal.add(1, { status: stepResult.status });
|
|
4838
5225
|
if (stepResult.durationMs > 0) stepDurationSeconds.record(stepResult.durationMs / 1e3);
|
|
4839
5226
|
}
|
|
4840
|
-
if (result.status === ExecutionJobStatus.enum.failed) logger$
|
|
5227
|
+
if (result.status === ExecutionJobStatus.enum.failed) logger$2.error("Sandbox returned failed result", {
|
|
4841
5228
|
durationMs: result.durationMs,
|
|
4842
5229
|
stepCount: result.stepResults.length,
|
|
4843
5230
|
steps: result.stepResults.map((r) => `${r.name}:${r.status}`).join(","),
|
|
@@ -4866,7 +5253,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
4866
5253
|
async handleBuildJob(dispatch, workDir, abortController) {
|
|
4867
5254
|
const { runId, jobId, jobConfig } = dispatch;
|
|
4868
5255
|
const buildCtx = getRequestContext();
|
|
4869
|
-
logger$
|
|
5256
|
+
logger$2.info(`Run: ${buildCtx.runId ?? runId} | Trace: ${buildCtx.requestId ?? "N/A"}`);
|
|
4870
5257
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
|
|
4871
5258
|
const buildStreamer = this.createStepStreamer(dispatch, 0);
|
|
4872
5259
|
const buildLog = (msg) => buildStreamer.addLine(msg);
|
|
@@ -4955,14 +5342,14 @@ var init_job_runner = __esmMin((() => {
|
|
|
4955
5342
|
const cliPublicKey = jobConfig.cliPublicKey;
|
|
4956
5343
|
const orchestratorPrivateKey = jobConfig.orchestratorPrivateKey;
|
|
4957
5344
|
if (tarballUrl && cliPublicKey && orchestratorPrivateKey) {
|
|
4958
|
-
logger$
|
|
5345
|
+
logger$2.info("Applying overlay tarball for test run", { jobId });
|
|
4959
5346
|
const overlayResult = await applyOverlay({
|
|
4960
5347
|
tarballUrl,
|
|
4961
5348
|
cliPublicKey,
|
|
4962
5349
|
orchestratorPrivateKey,
|
|
4963
5350
|
repoDir: workDir
|
|
4964
5351
|
});
|
|
4965
|
-
logger$
|
|
5352
|
+
logger$2.info("Overlay applied", {
|
|
4966
5353
|
filesApplied: overlayResult.filesApplied,
|
|
4967
5354
|
filesDeleted: overlayResult.filesDeleted
|
|
4968
5355
|
});
|
|
@@ -4990,9 +5377,9 @@ var init_job_runner = __esmMin((() => {
|
|
|
4990
5377
|
platform: os.platform(),
|
|
4991
5378
|
arch: os.arch()
|
|
4992
5379
|
};
|
|
4993
|
-
logger$
|
|
5380
|
+
logger$2.info("Requesting dep upload URL from orchestrator", { lockfileHash: buildConfig.lockfileHash });
|
|
4994
5381
|
const depUploadUrl = await this.requestUploadUrl(dispatch.jobId, "deps", depKey);
|
|
4995
|
-
logger$
|
|
5382
|
+
logger$2.info("Uploading dep tarball to S3", {
|
|
4996
5383
|
size: tarball.length,
|
|
4997
5384
|
hash: hash.slice(0, 12)
|
|
4998
5385
|
});
|
|
@@ -5001,7 +5388,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5001
5388
|
...depKey,
|
|
5002
5389
|
depsHash: hash
|
|
5003
5390
|
});
|
|
5004
|
-
logger$
|
|
5391
|
+
logger$2.info("Dep tarball upload complete", { lockfileHash: buildConfig.lockfileHash });
|
|
5005
5392
|
buildLog(`Deps tarball uploaded (${tarball.length} bytes)`);
|
|
5006
5393
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running, {
|
|
5007
5394
|
buildEvent: "deps_packed",
|
|
@@ -5027,15 +5414,15 @@ var init_job_runner = __esmMin((() => {
|
|
|
5027
5414
|
platform: os.platform(),
|
|
5028
5415
|
arch: os.arch()
|
|
5029
5416
|
};
|
|
5030
|
-
logger$
|
|
5417
|
+
logger$2.info("Requesting source tarball upload URL from orchestrator", { contentHash: buildConfig.contentHash });
|
|
5031
5418
|
const sourceUploadUrl = await this.requestUploadUrl(dispatch.jobId, "source", sourceKey);
|
|
5032
|
-
logger$
|
|
5419
|
+
logger$2.info("Uploading source tarball to S3", {
|
|
5033
5420
|
size: tarball.length,
|
|
5034
5421
|
contentHash: buildConfig.contentHash
|
|
5035
5422
|
});
|
|
5036
5423
|
await uploadToPresignedUrl(sourceUploadUrl, tarball);
|
|
5037
5424
|
this.sendUploadComplete(dispatch.jobId, "source", sourceKey);
|
|
5038
|
-
logger$
|
|
5425
|
+
logger$2.info("Source tarball upload complete", { contentHash: buildConfig.contentHash });
|
|
5039
5426
|
buildLog(`Source tarball packed and uploaded (${tarball.length} bytes, hash: ${buildConfig.contentHash.slice(0, 12)})`);
|
|
5040
5427
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running, {
|
|
5041
5428
|
buildEvent: "source_packed",
|
|
@@ -5058,7 +5445,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5058
5445
|
async handleInitJob(dispatch, workDir, abortController) {
|
|
5059
5446
|
const { runId, jobId, jobConfig } = dispatch;
|
|
5060
5447
|
const config = jobConfig;
|
|
5061
|
-
logger$
|
|
5448
|
+
logger$2.info("Starting init job", {
|
|
5062
5449
|
jobId,
|
|
5063
5450
|
targetJobName: config.targetJobName,
|
|
5064
5451
|
workflowName: config.workflowName
|
|
@@ -5106,7 +5493,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5106
5493
|
}
|
|
5107
5494
|
const kiciDir = join(workDir, ".kici");
|
|
5108
5495
|
const hasPackage = await fileExists(join(kiciDir, "package.json"));
|
|
5109
|
-
logger$
|
|
5496
|
+
logger$2.info("Init job: checking deps", {
|
|
5110
5497
|
kiciDir,
|
|
5111
5498
|
hasPackageJson: hasPackage,
|
|
5112
5499
|
source: config.source
|
|
@@ -5129,7 +5516,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5129
5516
|
dynamicConcurrencyGroup: config.dynamicConcurrencyGroup
|
|
5130
5517
|
}, config.timeoutMs);
|
|
5131
5518
|
});
|
|
5132
|
-
logger$
|
|
5519
|
+
logger$2.info("Init job completed successfully", {
|
|
5133
5520
|
jobId,
|
|
5134
5521
|
hasEnvironment: initResult.environmentName !== void 0,
|
|
5135
5522
|
hasEnv: initResult.env !== void 0,
|
|
@@ -5145,7 +5532,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5145
5532
|
});
|
|
5146
5533
|
} catch (err) {
|
|
5147
5534
|
const errorMsg = toErrorMessage(err);
|
|
5148
|
-
logger$
|
|
5535
|
+
logger$2.error("Init job failed", {
|
|
5149
5536
|
jobId,
|
|
5150
5537
|
error: errorMsg
|
|
5151
5538
|
});
|
|
@@ -5172,7 +5559,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5172
5559
|
const { runId, jobId, jobConfig } = dispatch;
|
|
5173
5560
|
const config = jobConfig;
|
|
5174
5561
|
const timeoutMs = config.timeoutMs ?? 12e4;
|
|
5175
|
-
logger$
|
|
5562
|
+
logger$2.info("Starting DynamicJobFn evaluation", {
|
|
5176
5563
|
jobId,
|
|
5177
5564
|
workflowName: config.workflowName,
|
|
5178
5565
|
sourceIndex: config.source.index
|
|
@@ -5273,7 +5660,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5273
5660
|
workflowName: config.workflowName
|
|
5274
5661
|
});
|
|
5275
5662
|
});
|
|
5276
|
-
logger$
|
|
5663
|
+
logger$2.info("DynamicJobFn evaluation completed", {
|
|
5277
5664
|
jobId,
|
|
5278
5665
|
generatedJobCount: lockJobs.length,
|
|
5279
5666
|
jobNames: lockJobs.map((j) => j.name)
|
|
@@ -5288,7 +5675,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5288
5675
|
});
|
|
5289
5676
|
} catch (err) {
|
|
5290
5677
|
const errorMsg = toErrorMessage(err);
|
|
5291
|
-
logger$
|
|
5678
|
+
logger$2.error("DynamicJobFn evaluation failed", {
|
|
5292
5679
|
jobId,
|
|
5293
5680
|
error: errorMsg
|
|
5294
5681
|
});
|
|
@@ -5296,10 +5683,17 @@ var init_job_runner = __esmMin((() => {
|
|
|
5296
5683
|
await evalStreamer.flush();
|
|
5297
5684
|
evalStreamer.destroy();
|
|
5298
5685
|
this.sendStepStatus(dispatch, 0, "evaluate", ExecutionStepStatus.enum.failed, { error: errorMsg }, evalStreamer.getTotalBytes());
|
|
5299
|
-
|
|
5686
|
+
const dynamicData = {
|
|
5300
5687
|
error: errorMsg,
|
|
5301
5688
|
dynamicFailed: true
|
|
5302
|
-
}
|
|
5689
|
+
};
|
|
5690
|
+
if (err instanceof MatrixExpansionError) dynamicData.initFailure = {
|
|
5691
|
+
scope: "job",
|
|
5692
|
+
category: InitFailureCategory.enum.matrix_expansion,
|
|
5693
|
+
message: errorMsg,
|
|
5694
|
+
jobName: err.jobName
|
|
5695
|
+
};
|
|
5696
|
+
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, dynamicData);
|
|
5303
5697
|
} finally {
|
|
5304
5698
|
clearInterval(heartbeatTimer);
|
|
5305
5699
|
}
|
|
@@ -5361,6 +5755,31 @@ var init_job_runner = __esmMin((() => {
|
|
|
5361
5755
|
this._sendRunEvent(runId, eventType, opts);
|
|
5362
5756
|
}
|
|
5363
5757
|
/**
|
|
5758
|
+
* Emit a `cache.restore` / `cache.save` run event for a cache pseudo-step.
|
|
5759
|
+
*
|
|
5760
|
+
* The cache phase tags its `step.complete` IPC with a {@link CacheStepType}
|
|
5761
|
+
* `step_type` and a `data.cacheOutcome` ({@link CacheOutcome}); when one of
|
|
5762
|
+
* those terminal pseudo-step statuses arrives here, mirror it onto the run
|
|
5763
|
+
* timeline as a `run.event` so hit/miss/saved/skipped/error is recorded for
|
|
5764
|
+
* the dashboard. A no-op for regular steps and hooks.
|
|
5765
|
+
*/
|
|
5766
|
+
maybeEmitCacheRunEvent(runId, jobId, stepIndex, state, data) {
|
|
5767
|
+
if (state === ExecutionStepStatus.enum.running) return;
|
|
5768
|
+
const stepType = data?.step_type;
|
|
5769
|
+
const eventType = stepType === CacheStepType.enum["cache:restore"] ? CacheRunEventType.enum["cache.restore"] : stepType === CacheStepType.enum["cache:save"] ? CacheRunEventType.enum["cache.save"] : void 0;
|
|
5770
|
+
if (!eventType) return;
|
|
5771
|
+
this.emitRunEvent(runId, eventType, {
|
|
5772
|
+
jobId,
|
|
5773
|
+
metadata: {
|
|
5774
|
+
stepIndex,
|
|
5775
|
+
...data?.cacheOutcome !== void 0 && { outcome: data.cacheOutcome },
|
|
5776
|
+
...data?.key !== void 0 && { key: data.key },
|
|
5777
|
+
...data?.matchedKey !== void 0 && { matchedKey: data.matchedKey },
|
|
5778
|
+
...data?.bytes !== void 0 && { bytes: data.bytes }
|
|
5779
|
+
}
|
|
5780
|
+
});
|
|
5781
|
+
}
|
|
5782
|
+
/**
|
|
5364
5783
|
* Create a LogStreamer for a synthetic step (build, evaluate, etc.).
|
|
5365
5784
|
*/
|
|
5366
5785
|
createStepStreamer(dispatch, stepIndex) {
|
|
@@ -5444,14 +5863,14 @@ var init_job_runner = __esmMin((() => {
|
|
|
5444
5863
|
*/
|
|
5445
5864
|
init_console_capture();
|
|
5446
5865
|
init_npm_resolver();
|
|
5447
|
-
const AGENT_VERSION = "0.1.
|
|
5448
|
-
const BUILD_COMMIT = "
|
|
5449
|
-
const SDK_VERSION = "0.1.
|
|
5450
|
-
const SDK_BUNDLE_HASH = "
|
|
5451
|
-
const SHARED_VERSION = "0.1.
|
|
5452
|
-
const SHARED_BUNDLE_HASH = "
|
|
5453
|
-
const ENGINE_VERSION = "0.1.
|
|
5454
|
-
const ENGINE_BUNDLE_HASH = "
|
|
5866
|
+
const AGENT_VERSION = "0.1.16";
|
|
5867
|
+
const BUILD_COMMIT = "7d97bb32c";
|
|
5868
|
+
const SDK_VERSION = "0.1.16";
|
|
5869
|
+
const SDK_BUNDLE_HASH = "cd4b8e0d91efe578c96e7c746f89b44a3df8fcaebaec2185399168ef8eb4ebca";
|
|
5870
|
+
const SHARED_VERSION = "0.1.16";
|
|
5871
|
+
const SHARED_BUNDLE_HASH = "c58b1596e92c8423ef83958e86894cb150315d8b91578a0080f1c645000e93e8";
|
|
5872
|
+
const ENGINE_VERSION = "0.1.16";
|
|
5873
|
+
const ENGINE_BUNDLE_HASH = "a611c0017d08faa9c5aa4fd97c3dd6259f53f3cca248bd2b369ad5d30c394847";
|
|
5455
5874
|
initTelemetry({
|
|
5456
5875
|
serviceName: "kici-agent",
|
|
5457
5876
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
@@ -5459,18 +5878,39 @@ initTelemetry({
|
|
|
5459
5878
|
const { connectionStatus, jobsActive, jobsTotal } = await Promise.resolve().then(() => (init_prometheus(), prometheus_exports));
|
|
5460
5879
|
const { JobRunner } = await Promise.resolve().then(() => (init_job_runner(), job_runner_exports));
|
|
5461
5880
|
setServiceName("agent");
|
|
5462
|
-
const logger = createLogger({ prefix: "agent" });
|
|
5881
|
+
const logger$1 = createLogger({ prefix: "agent" });
|
|
5882
|
+
/**
|
|
5883
|
+
* Serialize the agent's current Prometheus metrics to text. The OTel
|
|
5884
|
+
* PrometheusExporter exposes no direct serialize method, so the metrics are
|
|
5885
|
+
* piped through its request handler with a mock ServerResponse. Returns an
|
|
5886
|
+
* empty string when no exporter is configured. Shared by the /metrics health
|
|
5887
|
+
* route and the fleet mini-bundle.
|
|
5888
|
+
*/
|
|
5889
|
+
async function serializeAgentMetrics() {
|
|
5890
|
+
const exporter = getPrometheusExporter();
|
|
5891
|
+
if (!exporter) return "";
|
|
5892
|
+
return new Promise((resolve) => {
|
|
5893
|
+
exporter.getMetricsRequestHandler({}, {
|
|
5894
|
+
statusCode: 200,
|
|
5895
|
+
setHeader: () => {},
|
|
5896
|
+
end: (data) => {
|
|
5897
|
+
resolve(typeof data === "string" ? data : data ? data.toString() : "");
|
|
5898
|
+
}
|
|
5899
|
+
});
|
|
5900
|
+
});
|
|
5901
|
+
}
|
|
5463
5902
|
installConsoleCapture();
|
|
5464
|
-
await guardStartup(logger, async () => {
|
|
5903
|
+
await guardStartup(logger$1, async () => {
|
|
5465
5904
|
const config = loadConfig();
|
|
5466
|
-
logger.info("Agent starting", {
|
|
5905
|
+
logger$1.info("Agent starting", {
|
|
5467
5906
|
agentId: config.agentId,
|
|
5468
5907
|
orchestratorUrl: config.orchestratorUrl,
|
|
5469
5908
|
labels: config.labels,
|
|
5470
5909
|
roles: config.roles,
|
|
5471
5910
|
port: config.port
|
|
5472
5911
|
});
|
|
5473
|
-
|
|
5912
|
+
gcStaleAgentTmpDirs();
|
|
5913
|
+
logger$1.info("agent.build.info", {
|
|
5474
5914
|
agentVersion: AGENT_VERSION,
|
|
5475
5915
|
buildCommit: BUILD_COMMIT,
|
|
5476
5916
|
sdkVersion: SDK_VERSION,
|
|
@@ -5492,7 +5932,7 @@ await guardStartup(logger, async () => {
|
|
|
5492
5932
|
if (toolErrors.length > 0) throw new Error("Agent required-tools validation failed:\n" + toolErrors.map((e) => ` - ${e}`).join("\n"));
|
|
5493
5933
|
if (config.roles === void 0 || config.roles.includes("builder")) {
|
|
5494
5934
|
const npmVersion = verifyNpmAvailable();
|
|
5495
|
-
logger.info("Builder role: npm verified", { npmVersion });
|
|
5935
|
+
logger$1.info("Builder role: npm verified", { npmVersion });
|
|
5496
5936
|
}
|
|
5497
5937
|
let isDraining = false;
|
|
5498
5938
|
let idleShutdownTimer;
|
|
@@ -5509,7 +5949,9 @@ await guardStartup(logger, async () => {
|
|
|
5509
5949
|
sendJobContext: (runId, jobId, context) => client.sendJobContext(runId, jobId, context),
|
|
5510
5950
|
sendRunEvent: (runId, eventType, opts) => client.sendRunEvent(runId, eventType, opts),
|
|
5511
5951
|
sendConcurrencyReport: (runId, jobId, group) => client.sendConcurrencyReport(runId, jobId, group),
|
|
5512
|
-
sendApiRequest: (method, params) => client.sendApiRequest(method, params ?? {})
|
|
5952
|
+
sendApiRequest: (method, params) => client.sendApiRequest(method, params ?? {}),
|
|
5953
|
+
requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
|
|
5954
|
+
sendStepApproval: (runId, jobId, request) => client.sendStepApproval(runId, jobId, request)
|
|
5513
5955
|
});
|
|
5514
5956
|
/** Build and send an agent.status message with dynamic OS metadata. */
|
|
5515
5957
|
function sendAgentStatus() {
|
|
@@ -5525,6 +5967,11 @@ await guardStartup(logger, async () => {
|
|
|
5525
5967
|
}
|
|
5526
5968
|
client = new OrchestratorClient({
|
|
5527
5969
|
...agentClientConnectionOptions(config),
|
|
5970
|
+
getFleetBundleInputs: async () => ({
|
|
5971
|
+
config,
|
|
5972
|
+
logDir: process.env.KICI_LOG_DIR,
|
|
5973
|
+
metricsText: await serializeAgentMetrics()
|
|
5974
|
+
}),
|
|
5528
5975
|
onJobDispatch: (dispatch) => {
|
|
5529
5976
|
const reqId = dispatch.requestId ?? randomUUID();
|
|
5530
5977
|
requestContext.run({
|
|
@@ -5533,15 +5980,31 @@ await guardStartup(logger, async () => {
|
|
|
5533
5980
|
jobId: dispatch.jobId
|
|
5534
5981
|
}, () => {
|
|
5535
5982
|
if (isDraining) {
|
|
5536
|
-
logger.info("Draining: rejecting job dispatch", { jobId: dispatch.jobId });
|
|
5983
|
+
logger$1.info("Draining: rejecting job dispatch", { jobId: dispatch.jobId });
|
|
5984
|
+
client.sendDirect({
|
|
5985
|
+
type: "job.reject",
|
|
5986
|
+
messageId: randomUUID(),
|
|
5987
|
+
runId: dispatch.runId,
|
|
5988
|
+
jobId: dispatch.jobId,
|
|
5989
|
+
reason: "draining",
|
|
5990
|
+
timestamp: Date.now()
|
|
5991
|
+
});
|
|
5537
5992
|
sendAgentStatus();
|
|
5538
5993
|
return;
|
|
5539
5994
|
}
|
|
5540
5995
|
if (jobRunner.activeJobs.size > 0) {
|
|
5541
|
-
logger.warn("Already running a job,
|
|
5996
|
+
logger$1.warn("Already running a job, rejecting dispatch", {
|
|
5542
5997
|
jobId: dispatch.jobId,
|
|
5543
5998
|
activeJobs: jobRunner.activeJobs.size
|
|
5544
5999
|
});
|
|
6000
|
+
client.sendDirect({
|
|
6001
|
+
type: "job.reject",
|
|
6002
|
+
messageId: randomUUID(),
|
|
6003
|
+
runId: dispatch.runId,
|
|
6004
|
+
jobId: dispatch.jobId,
|
|
6005
|
+
reason: "busy",
|
|
6006
|
+
timestamp: Date.now()
|
|
6007
|
+
});
|
|
5545
6008
|
sendAgentStatus();
|
|
5546
6009
|
return;
|
|
5547
6010
|
}
|
|
@@ -5549,7 +6012,14 @@ await guardStartup(logger, async () => {
|
|
|
5549
6012
|
clearTimeout(idleShutdownTimer);
|
|
5550
6013
|
idleShutdownTimer = void 0;
|
|
5551
6014
|
}
|
|
5552
|
-
|
|
6015
|
+
client.sendDirect({
|
|
6016
|
+
type: "job.ack",
|
|
6017
|
+
messageId: randomUUID(),
|
|
6018
|
+
runId: dispatch.runId,
|
|
6019
|
+
jobId: dispatch.jobId,
|
|
6020
|
+
timestamp: Date.now()
|
|
6021
|
+
});
|
|
6022
|
+
logger$1.info("Accepting job dispatch", {
|
|
5553
6023
|
jobId: dispatch.jobId,
|
|
5554
6024
|
runId: dispatch.runId,
|
|
5555
6025
|
activeJobs: jobRunner.activeJobs.size + 1
|
|
@@ -5558,7 +6028,7 @@ await guardStartup(logger, async () => {
|
|
|
5558
6028
|
jobRunner.execute(dispatch).then(() => {
|
|
5559
6029
|
jobsTotal.add(1, { status: "success" });
|
|
5560
6030
|
}).catch((err) => {
|
|
5561
|
-
logger.error("Job execution error", {
|
|
6031
|
+
logger$1.error("Job execution error", {
|
|
5562
6032
|
jobId: dispatch.jobId,
|
|
5563
6033
|
error: toErrorMessage(err)
|
|
5564
6034
|
});
|
|
@@ -5569,7 +6039,7 @@ await guardStartup(logger, async () => {
|
|
|
5569
6039
|
sendAgentStatus();
|
|
5570
6040
|
if (config.scalerManaged && jobRunner.activeJobs.size === 0) {
|
|
5571
6041
|
if (client.state !== "registered") {
|
|
5572
|
-
logger.info("Scaler-managed agent idle but disconnected, deferring shutdown until reconnected");
|
|
6042
|
+
logger$1.info("Scaler-managed agent idle but disconnected, deferring shutdown until reconnected");
|
|
5573
6043
|
return;
|
|
5574
6044
|
}
|
|
5575
6045
|
startIdleShutdownTimer();
|
|
@@ -5578,7 +6048,7 @@ await guardStartup(logger, async () => {
|
|
|
5578
6048
|
});
|
|
5579
6049
|
},
|
|
5580
6050
|
onJobCancel: (cancel) => {
|
|
5581
|
-
logger.info("Job cancel received", {
|
|
6051
|
+
logger$1.info("Job cancel received", {
|
|
5582
6052
|
jobId: cancel.jobId,
|
|
5583
6053
|
reason: cancel.reason
|
|
5584
6054
|
});
|
|
@@ -5596,13 +6066,13 @@ await guardStartup(logger, async () => {
|
|
|
5596
6066
|
const idleMs = config.scalerIdleTimeoutMs;
|
|
5597
6067
|
if (idleShutdownTimer) clearTimeout(idleShutdownTimer);
|
|
5598
6068
|
if (idleMs <= 0) {
|
|
5599
|
-
logger.info("Scaler-managed agent idle after job completion, shutting down");
|
|
6069
|
+
logger$1.info("Scaler-managed agent idle after job completion, shutting down");
|
|
5600
6070
|
gracefulShutdown("scaler-idle");
|
|
5601
6071
|
} else {
|
|
5602
|
-
logger.info(`Scaler-managed agent idle, waiting ${idleMs}ms for follow-up jobs`);
|
|
6072
|
+
logger$1.info(`Scaler-managed agent idle, waiting ${idleMs}ms for follow-up jobs`);
|
|
5603
6073
|
idleShutdownTimer = setTimeout(() => {
|
|
5604
6074
|
if (jobRunner.activeJobs.size === 0) {
|
|
5605
|
-
logger.info("Scaler-managed agent still idle after timeout, shutting down");
|
|
6075
|
+
logger$1.info("Scaler-managed agent still idle after timeout, shutting down");
|
|
5606
6076
|
gracefulShutdown("scaler-idle");
|
|
5607
6077
|
}
|
|
5608
6078
|
}, idleMs);
|
|
@@ -5612,17 +6082,17 @@ await guardStartup(logger, async () => {
|
|
|
5612
6082
|
if (!config.scalerManaged || jobRunner.activeJobs.size > 0) return;
|
|
5613
6083
|
if (pendingDispatch) {
|
|
5614
6084
|
const safetyMs = config.scalerPendingDispatchTimeoutMs;
|
|
5615
|
-
logger.info(`Scaler-managed agent registered with pending bound dispatch, deferring idle shutdown for ${safetyMs}ms`);
|
|
6085
|
+
logger$1.info(`Scaler-managed agent registered with pending bound dispatch, deferring idle shutdown for ${safetyMs}ms`);
|
|
5616
6086
|
if (idleShutdownTimer) clearTimeout(idleShutdownTimer);
|
|
5617
6087
|
idleShutdownTimer = setTimeout(() => {
|
|
5618
6088
|
if (jobRunner.activeJobs.size === 0) {
|
|
5619
|
-
logger.warn("Scaler-managed agent pending-dispatch safety timeout exceeded, shutting down");
|
|
6089
|
+
logger$1.warn("Scaler-managed agent pending-dispatch safety timeout exceeded, shutting down");
|
|
5620
6090
|
gracefulShutdown("scaler-pending-dispatch-timeout");
|
|
5621
6091
|
}
|
|
5622
6092
|
}, safetyMs);
|
|
5623
6093
|
return;
|
|
5624
6094
|
}
|
|
5625
|
-
logger.info("Scaler-managed agent reconnected and idle, starting idle shutdown timer");
|
|
6095
|
+
logger$1.info("Scaler-managed agent reconnected and idle, starting idle shutdown timer");
|
|
5626
6096
|
startIdleShutdownTimer();
|
|
5627
6097
|
};
|
|
5628
6098
|
const wsTransport = new winston.transports.Stream({
|
|
@@ -5632,14 +6102,14 @@ await guardStartup(logger, async () => {
|
|
|
5632
6102
|
} }),
|
|
5633
6103
|
format: winston.format.combine(winston.format.timestamp(), winston.format.json())
|
|
5634
6104
|
});
|
|
5635
|
-
logger.add(wsTransport);
|
|
6105
|
+
logger$1.add(wsTransport);
|
|
5636
6106
|
const envProbes = {};
|
|
5637
6107
|
for (const [k, v] of Object.entries(process.env)) {
|
|
5638
6108
|
if (v === void 0) continue;
|
|
5639
6109
|
if (!/_ENV_PROBE$|_ENV_PROBE_/.test(k)) continue;
|
|
5640
6110
|
envProbes[k] = v.length <= 64 ? v : `${v.slice(0, 61)}...`;
|
|
5641
6111
|
}
|
|
5642
|
-
if (Object.keys(envProbes).length > 0) logger.info("Agent startup env probes (diagnostic)", envProbes);
|
|
6112
|
+
if (Object.keys(envProbes).length > 0) logger$1.info("Agent startup env probes (diagnostic)", envProbes);
|
|
5643
6113
|
client.connect();
|
|
5644
6114
|
connectionStatus.add(1);
|
|
5645
6115
|
const metricsReporter = new MetricsReporter({
|
|
@@ -5649,25 +6119,10 @@ await guardStartup(logger, async () => {
|
|
|
5649
6119
|
metricsReporter.start();
|
|
5650
6120
|
const app = new Hono();
|
|
5651
6121
|
const healthRoutes = createHealthRoutes$1({
|
|
5652
|
-
getMetrics: async () => {
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
body: ""
|
|
5657
|
-
};
|
|
5658
|
-
return new Promise((resolve) => {
|
|
5659
|
-
exporter.getMetricsRequestHandler({}, {
|
|
5660
|
-
statusCode: 200,
|
|
5661
|
-
setHeader: () => {},
|
|
5662
|
-
end: (data) => {
|
|
5663
|
-
resolve({
|
|
5664
|
-
contentType: "text/plain",
|
|
5665
|
-
body: typeof data === "string" ? data : data ? data.toString() : ""
|
|
5666
|
-
});
|
|
5667
|
-
}
|
|
5668
|
-
});
|
|
5669
|
-
});
|
|
5670
|
-
},
|
|
6122
|
+
getMetrics: async () => ({
|
|
6123
|
+
contentType: "text/plain",
|
|
6124
|
+
body: await serializeAgentMetrics()
|
|
6125
|
+
}),
|
|
5671
6126
|
getStatus: () => ({
|
|
5672
6127
|
agentId: config.agentId,
|
|
5673
6128
|
connected: client.state === "registered",
|
|
@@ -5679,13 +6134,13 @@ await guardStartup(logger, async () => {
|
|
|
5679
6134
|
fetch: app.fetch,
|
|
5680
6135
|
port: config.port
|
|
5681
6136
|
}, (info) => {
|
|
5682
|
-
logger.info(`Agent started on port ${info.port}`, {
|
|
6137
|
+
logger$1.info(`Agent started on port ${info.port}`, {
|
|
5683
6138
|
port: info.port,
|
|
5684
6139
|
agentId: config.agentId
|
|
5685
6140
|
});
|
|
5686
6141
|
});
|
|
5687
6142
|
const { shutdown: gracefulShutdown } = setupGracefulShutdown({
|
|
5688
|
-
logger,
|
|
6143
|
+
logger: logger$1,
|
|
5689
6144
|
timeoutMs: 1e4,
|
|
5690
6145
|
onForceExit: () => {
|
|
5691
6146
|
for (const job of jobRunner.activeJobs.values()) job.abortController.abort();
|
|
@@ -5703,7 +6158,7 @@ await guardStartup(logger, async () => {
|
|
|
5703
6158
|
name: "Waiting for active jobs to complete",
|
|
5704
6159
|
fn: async () => {
|
|
5705
6160
|
if (jobRunner.activeJobs.size > 0) {
|
|
5706
|
-
logger.info("Active jobs remaining", { activeJobs: jobRunner.activeJobs.size });
|
|
6161
|
+
logger$1.info("Active jobs remaining", { activeJobs: jobRunner.activeJobs.size });
|
|
5707
6162
|
await Promise.allSettled([...jobRunner.activeJobs.values()].map((j) => j.completionPromise));
|
|
5708
6163
|
}
|
|
5709
6164
|
}
|
|
@@ -5737,17 +6192,17 @@ await guardStartup(logger, async () => {
|
|
|
5737
6192
|
]
|
|
5738
6193
|
});
|
|
5739
6194
|
process.on("SIGUSR1", () => {
|
|
5740
|
-
logger.info("Received SIGUSR1, entering drain mode");
|
|
6195
|
+
logger$1.info("Received SIGUSR1, entering drain mode");
|
|
5741
6196
|
isDraining = true;
|
|
5742
6197
|
if (jobRunner.activeJobs.size === 0) {
|
|
5743
|
-
logger.info("No active jobs, shutting down immediately");
|
|
6198
|
+
logger$1.info("No active jobs, shutting down immediately");
|
|
5744
6199
|
gracefulShutdown("SIGUSR1-drain");
|
|
5745
6200
|
return;
|
|
5746
6201
|
}
|
|
5747
6202
|
const checkDrained = setInterval(() => {
|
|
5748
6203
|
if (jobRunner.activeJobs.size === 0) {
|
|
5749
6204
|
clearInterval(checkDrained);
|
|
5750
|
-
logger.info("All jobs drained, shutting down");
|
|
6205
|
+
logger$1.info("All jobs drained, shutting down");
|
|
5751
6206
|
gracefulShutdown("SIGUSR1-drain");
|
|
5752
6207
|
}
|
|
5753
6208
|
}, 1e3);
|