@kici-dev/agent 0.1.26 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/ensure-init-runner.d.ts +23 -22
- package/dist/bootstrap/payload-source.d.ts +32 -0
- package/dist/bootstrap/probe-platform.d.ts +21 -0
- package/dist/bootstrap/restage-agent.d.ts +43 -0
- package/dist/bootstrap/run-restage.d.ts +12 -0
- package/dist/bootstrap/s3-payload-source.d.ts +35 -0
- package/dist/bootstrap/ssh-exec.d.ts +14 -0
- package/dist/bootstrap/stage-agent-payload.d.ts +41 -0
- package/dist/checkout/changed-files.d.ts +34 -0
- package/dist/config.d.ts +42 -14
- package/dist/container-ts-loader-hook.js +147710 -0
- package/dist/execution/artifacts/artifact-engine.d.ts +51 -0
- package/dist/execution/dep-installer.d.ts +3 -3
- package/dist/execution/init-runner.d.ts +4 -4
- package/dist/execution/job-runner.d.ts +41 -6
- package/dist/execution/log-streamer.d.ts +17 -2
- package/dist/execution/rule-evaluator.d.ts +1 -12
- package/dist/execution/sandbox/container-hardening.d.ts +80 -0
- package/dist/execution/sandbox/container-sandbox.d.ts +54 -0
- package/dist/execution/sandbox/container-ts-loader-hook.d.ts +26 -0
- package/dist/execution/sandbox/env-sanitizer.d.ts +12 -3
- package/dist/execution/sandbox/fork-runner.d.ts +14 -0
- package/dist/execution/sandbox/index.d.ts +2 -1
- package/dist/execution/sandbox/ipc-protocol.d.ts +82 -6
- package/dist/execution/sandbox/step-loop.d.ts +4 -0
- package/dist/execution/sandbox/types.d.ts +25 -4
- package/dist/execution/sandbox/workflow-runner.d.ts +111 -5
- package/dist/execution/streaming-zx-log.d.ts +38 -0
- package/dist/execution/tmp-gc.d.ts +22 -7
- package/dist/execution/workflow-loader.d.ts +32 -1
- package/dist/index.js +50 -45
- package/dist/provenance/statement-builder.d.ts +3 -2
- package/dist/server.d.ts +1 -1
- package/dist/server.js +1360 -193
- package/dist/workflow-runner-bundle.js +215393 -0
- package/dist/workflow-runner.js +937 -256
- package/dist/ws/orchestrator-client.d.ts +105 -1
- package/package.json +14 -12
- package/sbom.spdx.json +1092 -1723
package/dist/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
|
|
2
2
|
import { dirname as __cjs_dirname } from "node:path";
|
|
3
3
|
__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
|
|
4
|
-
import { register } from "node:module";
|
|
4
|
+
import { createRequire, register } from "node:module";
|
|
5
5
|
import crypto$1, { createCipheriv, createHash, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
|
6
6
|
import * as os$1 from "node:os";
|
|
7
7
|
import os, { hostname, tmpdir } from "node:os";
|
|
@@ -9,23 +9,25 @@ import { PassThrough, Readable, Transform, Writable } from "node:stream";
|
|
|
9
9
|
import { serve } from "@hono/node-server";
|
|
10
10
|
import { Hono } from "hono";
|
|
11
11
|
import winston from "winston";
|
|
12
|
-
import { RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
12
|
+
import { AgentDeliveryMode, AgentPlatform, RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, kiciTmpBase, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
13
13
|
import { z } from "zod";
|
|
14
14
|
import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
|
|
15
|
-
import { ALLOWED_SYSTEM_VARS, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, deriveOsArchLabels, expandMatrix, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
|
|
15
|
+
import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
|
|
16
16
|
import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
|
|
17
17
|
import WebSocket from "ws";
|
|
18
18
|
import * as fs$1 from "node:fs";
|
|
19
|
-
import fs, { existsSync } from "node:fs";
|
|
19
|
+
import fs, { createReadStream, createWriteStream, existsSync } from "node:fs";
|
|
20
20
|
import { ZipArchive } from "archiver";
|
|
21
21
|
import path, { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
22
22
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
23
23
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
24
24
|
import { format, promisify } from "node:util";
|
|
25
|
+
import { OTEL_DATA_POINT_TYPE, mapDataPointTypeToWireKind } from "@kici-dev/engine/metrics/metric-kind-compat";
|
|
25
26
|
import { gcStaleTmpDirs } from "@kici-dev/core/tmp-gc";
|
|
26
|
-
import fsPromises, { access, lstat, mkdir,
|
|
27
|
+
import fsPromises, { access, lstat, mkdir, readFile, readdir, realpath, unlink, writeFile } from "node:fs/promises";
|
|
28
|
+
import { makeTempDir } from "@kici-dev/core/tmp";
|
|
27
29
|
import Docker from "dockerode";
|
|
28
|
-
import { buildKiciApi, buildNeedsContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject } from "@kici-dev/sdk";
|
|
30
|
+
import { buildKiciApi, buildNeedsContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject, setJobOutputsMap, setStepOutputsMap, setStepRefMap } from "@kici-dev/sdk";
|
|
29
31
|
import { c, x } from "tar";
|
|
30
32
|
import https from "node:https";
|
|
31
33
|
import http from "node:http";
|
|
@@ -101,8 +103,18 @@ const envDef = defineEnv({
|
|
|
101
103
|
dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
|
|
102
104
|
jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
|
|
103
105
|
backpressureMode: z.enum(["pause", "drop"]).default("pause"),
|
|
106
|
+
agentPayloadDir: z.string().optional(),
|
|
107
|
+
agentCommand: z.string().optional(),
|
|
104
108
|
sandbox: z.string().default("false").transform((s) => s === "true"),
|
|
109
|
+
trustedEnv: z.string().default("false").transform((s) => s === "true"),
|
|
110
|
+
inPlace: z.string().default("false").transform((s) => s === "true"),
|
|
105
111
|
sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
|
|
112
|
+
sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
|
|
113
|
+
sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
|
|
114
|
+
sandboxUser: z.string().optional(),
|
|
115
|
+
sandboxPidsLimit: z.coerce.number().int().positive().default(512),
|
|
116
|
+
sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
|
|
117
|
+
sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
|
|
106
118
|
scalerManaged: z.string().optional().transform((s) => s === "1"),
|
|
107
119
|
scalerIdleTimeoutMs: z.coerce.number().default(5e3),
|
|
108
120
|
scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
|
|
@@ -127,8 +139,18 @@ const envDef = defineEnv({
|
|
|
127
139
|
dockerKeepFailed: "KICI_DOCKER_KEEP_FAILED",
|
|
128
140
|
jobHeartbeatIntervalMs: "KICI_JOB_HEARTBEAT_INTERVAL_MS",
|
|
129
141
|
backpressureMode: "KICI_BACKPRESSURE_MODE",
|
|
142
|
+
agentPayloadDir: "KICI_AGENT_PAYLOAD_DIR",
|
|
143
|
+
agentCommand: "KICI_AGENT_COMMAND",
|
|
130
144
|
sandbox: "KICI_SANDBOX",
|
|
145
|
+
trustedEnv: "KICI_TRUSTED_ENV",
|
|
146
|
+
inPlace: "KICI_IN_PLACE",
|
|
131
147
|
sandboxNetwork: "KICI_SANDBOX_NETWORK",
|
|
148
|
+
sandboxHardened: "KICI_SANDBOX_HARDENED",
|
|
149
|
+
sandboxReadonlyRootfs: "KICI_SANDBOX_READONLY_ROOTFS",
|
|
150
|
+
sandboxUser: "KICI_SANDBOX_USER",
|
|
151
|
+
sandboxPidsLimit: "KICI_SANDBOX_PIDS_LIMIT",
|
|
152
|
+
sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
|
|
153
|
+
sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
|
|
132
154
|
scalerManaged: "KICI_SCALER_MANAGED",
|
|
133
155
|
scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
|
|
134
156
|
scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
|
|
@@ -156,7 +178,15 @@ const envDef = defineEnv({
|
|
|
156
178
|
* - KICI_JOB_HEARTBEAT_INTERVAL_MS (default: 60000)
|
|
157
179
|
* - KICI_BACKPRESSURE_MODE (default: pause, options: pause | drop)
|
|
158
180
|
* - KICI_SANDBOX (default: false) — enable bubblewrap (bwrap) namespace isolation for bare-metal execution
|
|
159
|
-
* -
|
|
181
|
+
* - KICI_TRUSTED_ENV (default: false) — trusted fleet-agent profile: pass the ambient host env (minus the agent's own KiCI identity secrets) through to steps
|
|
182
|
+
* - KICI_IN_PLACE (default: false) — in-place no-clone profile: for a file:// source, use the real repo path as workDir and skip the clone (the routed deploy:stg profile)
|
|
183
|
+
* - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — sandbox network posture for BOTH backends: the bwrap network namespace (when sandbox=true) and the container job network. `host` shares the host network (container backend also binds host /etc/hosts read-only for name resolution); applies to the container backend under the default hardened posture (KICI_SANDBOX_HARDENED=true)
|
|
184
|
+
* - KICI_SANDBOX_HARDENED (default: true) — hardened-by-default job containers (CapDrop ALL, no-new-privileges, cgroup caps, tmpfs /tmp); set false to roll back to the legacy unhardened posture
|
|
185
|
+
* - KICI_SANDBOX_READONLY_ROOTFS (default: false) — opt-in read-only container rootfs (/tmp stays a writable tmpfs)
|
|
186
|
+
* - KICI_SANDBOX_USER (optional) — container user override (uid, uid:gid, or name); honors the image user when unset
|
|
187
|
+
* - KICI_SANDBOX_PIDS_LIMIT (default: 512) — max PIDs in the job container cgroup
|
|
188
|
+
* - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
|
|
189
|
+
* - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
|
|
160
190
|
* - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
|
|
161
191
|
* - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
|
|
162
192
|
* - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
|
|
@@ -165,7 +195,7 @@ const envDef = defineEnv({
|
|
|
165
195
|
*/
|
|
166
196
|
function loadConfig() {
|
|
167
197
|
const data = envDef.parse();
|
|
168
|
-
if (!data.scalerManaged) validateNoReservedLabels(data.labels, "KICI_LABELS");
|
|
198
|
+
if (!data.scalerManaged && !data.agentToken) validateNoReservedLabels(data.labels, "KICI_LABELS");
|
|
169
199
|
validateUnknownKiciVars([...envDef.listKnownEnvVars(), ...LOGGER_ENV_VARS]);
|
|
170
200
|
return {
|
|
171
201
|
...data,
|
|
@@ -296,6 +326,22 @@ var LogBuffer = class extends RingBuffer {
|
|
|
296
326
|
//#region src/ws/orchestrator-client.ts
|
|
297
327
|
const logger$12 = createLogger({ prefix: "orchestrator-client" });
|
|
298
328
|
/**
|
|
329
|
+
* How long to wait for `artifacts.upload.complete.ack` before failing the step.
|
|
330
|
+
*
|
|
331
|
+
* Deliberately shorter than the sandbox's own 30s artifact-request timeout
|
|
332
|
+
* (`CACHE_RESPONSE_TIMEOUT_MS` in the workflow runner) so a lost ack surfaces
|
|
333
|
+
* this specific reason rather than the sandbox's generic "artifact request
|
|
334
|
+
* timed out". The orchestrator's bounded commit retry finishes in well under a
|
|
335
|
+
* second, so the margin costs nothing.
|
|
336
|
+
*/
|
|
337
|
+
const ARTIFACT_COMPLETE_ACK_TIMEOUT_MS = 25e3;
|
|
338
|
+
/**
|
|
339
|
+
* Tail shared by both give-up messages for an unacknowledged commit. The
|
|
340
|
+
* operator must not be sent hunting for a lost artifact that in fact landed:
|
|
341
|
+
* the orchestrator may well have committed it and only the ack was lost.
|
|
342
|
+
*/
|
|
343
|
+
const COMMIT_MAY_HAVE_LANDED = "the artifact may have been committed";
|
|
344
|
+
/**
|
|
299
345
|
* WebSocket client that connects the agent to the customer orchestrator.
|
|
300
346
|
*
|
|
301
347
|
* Handles:
|
|
@@ -317,6 +363,13 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
317
363
|
reconnectTimer = null;
|
|
318
364
|
reconnectAttempts = 0;
|
|
319
365
|
intentionalDisconnect = false;
|
|
366
|
+
/**
|
|
367
|
+
* Agent-facing capabilities advertised by the orchestrator on `register.ack`.
|
|
368
|
+
* Undefined until the first ack arrives, and on a pre-capability orchestrator
|
|
369
|
+
* that never advertises — both cases mean "assume unsupported" for every
|
|
370
|
+
* optional feature gated on it.
|
|
371
|
+
*/
|
|
372
|
+
orchCapabilities;
|
|
320
373
|
pendingLogBatch = [];
|
|
321
374
|
logFlushTimer = null;
|
|
322
375
|
static LOG_BATCH_SIZE = 50;
|
|
@@ -329,6 +382,26 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
329
382
|
pendingApiRequests = /* @__PURE__ */ new Map();
|
|
330
383
|
/** Pending user-cache restore/save requests awaiting orchestrator response. */
|
|
331
384
|
pendingUserCacheRequests = /* @__PURE__ */ new Map();
|
|
385
|
+
/** Pending user-artifact upload/download requests awaiting orchestrator response. */
|
|
386
|
+
pendingUserArtifactRequests = /* @__PURE__ */ new Map();
|
|
387
|
+
/**
|
|
388
|
+
* In-flight `artifacts.upload.complete` frames that are safe to re-send after a
|
|
389
|
+
* reconnect, keyed by their current `messageId`.
|
|
390
|
+
*
|
|
391
|
+
* The commit is idempotent server-side (the storage key is derived server-side,
|
|
392
|
+
* `initMeta` is a re-writable sidecar, and the DB insert is
|
|
393
|
+
* `onConflict.doNothing()`), so a dropped ack means "we did not hear the
|
|
394
|
+
* answer", not "it did not happen". `beginUpload` / `download` are deliberately
|
|
395
|
+
* absent: `beginUpload` mints a presigned PUT and is not idempotent the same
|
|
396
|
+
* way, and `download` is read-only and cheap to fail.
|
|
397
|
+
*/
|
|
398
|
+
resendableCompletes = /* @__PURE__ */ new Map();
|
|
399
|
+
/**
|
|
400
|
+
* Completes taken off `pendingUserArtifactRequests` by the disconnect sweep and
|
|
401
|
+
* parked until the next `register.ack` re-sends them. Their ack timers keep
|
|
402
|
+
* running — parking does not extend the deadline.
|
|
403
|
+
*/
|
|
404
|
+
heldCompletes = /* @__PURE__ */ new Set();
|
|
332
405
|
/**
|
|
333
406
|
* Pending step-approval requests awaiting the orchestrator's resolution.
|
|
334
407
|
* No client-side timeout: the orchestrator owns the (org-/SDK-configured)
|
|
@@ -424,6 +497,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
424
497
|
this.intentionalDisconnect = true;
|
|
425
498
|
this.stopHeartbeat();
|
|
426
499
|
this.cancelReconnect();
|
|
500
|
+
this.rejectHeldCompletes("the agent disconnected");
|
|
427
501
|
this.drainPendingLogBatch();
|
|
428
502
|
if (this.ws) {
|
|
429
503
|
this.ws.close(1e3, "Agent disconnect");
|
|
@@ -715,6 +789,105 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
715
789
|
});
|
|
716
790
|
}
|
|
717
791
|
/**
|
|
792
|
+
* Relay a user-facing artifact request from the sandbox to the orchestrator.
|
|
793
|
+
*
|
|
794
|
+
* Translates the sandbox `artifacts.request` IPC into the matching
|
|
795
|
+
* `artifacts.*` WS message and resolves with the orchestrator's response
|
|
796
|
+
* mapped onto the IPC response shape:
|
|
797
|
+
*
|
|
798
|
+
* - `beginUpload` -> `artifacts.upload.request`, awaits `artifacts.upload.response`.
|
|
799
|
+
* - `completeUpload` -> `artifacts.upload.complete`, awaits
|
|
800
|
+
* `artifacts.upload.complete.ack` when the orchestrator advertises the
|
|
801
|
+
* `artifactCompleteAck` capability, and rejects when the commit failed so
|
|
802
|
+
* the workflow step fails instead of losing the artifact behind a green run.
|
|
803
|
+
* Against an orchestrator that never advertises it, the message stays
|
|
804
|
+
* fire-and-forget and resolves immediately.
|
|
805
|
+
* - `download` -> `artifacts.download.request`, awaits `artifacts.download.response`.
|
|
806
|
+
*
|
|
807
|
+
* Times out after 30 seconds for the round-trip ops.
|
|
808
|
+
*/
|
|
809
|
+
async requestUserArtifact(jobId, request) {
|
|
810
|
+
if (request.op === "completeUpload") {
|
|
811
|
+
const completeId = randomUUID();
|
|
812
|
+
const complete = {
|
|
813
|
+
type: "artifacts.upload.complete",
|
|
814
|
+
messageId: completeId,
|
|
815
|
+
jobId,
|
|
816
|
+
name: request.name,
|
|
817
|
+
sizeBytes: request.sizeBytes,
|
|
818
|
+
sha256: request.sha256,
|
|
819
|
+
storageKey: request.storageKey
|
|
820
|
+
};
|
|
821
|
+
if (!hasOrchAgentCapability(this.orchCapabilities, "artifactCompleteAck")) {
|
|
822
|
+
this.sendDirect(complete);
|
|
823
|
+
return {
|
|
824
|
+
type: "artifacts.response",
|
|
825
|
+
requestId: request.requestId
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
return new Promise((resolve, reject) => {
|
|
829
|
+
const record = {
|
|
830
|
+
messageId: completeId,
|
|
831
|
+
frame: complete,
|
|
832
|
+
attempts: 0,
|
|
833
|
+
pending: {
|
|
834
|
+
resolve: () => {
|
|
835
|
+
clearTimeout(timer);
|
|
836
|
+
resolve({
|
|
837
|
+
type: "artifacts.response",
|
|
838
|
+
requestId: request.requestId
|
|
839
|
+
});
|
|
840
|
+
},
|
|
841
|
+
reject: (err) => {
|
|
842
|
+
clearTimeout(timer);
|
|
843
|
+
reject(err);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
const timer = setTimeout(() => {
|
|
848
|
+
this.forgetComplete(record);
|
|
849
|
+
reject(/* @__PURE__ */ new Error(`Artifact upload-complete ack timed out (${ARTIFACT_COMPLETE_ACK_TIMEOUT_MS}ms)`));
|
|
850
|
+
}, ARTIFACT_COMPLETE_ACK_TIMEOUT_MS);
|
|
851
|
+
this.pendingUserArtifactRequests.set(completeId, record.pending);
|
|
852
|
+
this.resendableCompletes.set(completeId, record);
|
|
853
|
+
this.sendDirect(complete);
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
const messageId = randomUUID();
|
|
857
|
+
return new Promise((resolve, reject) => {
|
|
858
|
+
const timer = setTimeout(() => {
|
|
859
|
+
this.pendingUserArtifactRequests.delete(messageId);
|
|
860
|
+
reject(/* @__PURE__ */ new Error("User-artifact request timed out (30s)"));
|
|
861
|
+
}, 3e4);
|
|
862
|
+
this.pendingUserArtifactRequests.set(messageId, {
|
|
863
|
+
resolve: (response) => {
|
|
864
|
+
clearTimeout(timer);
|
|
865
|
+
resolve({
|
|
866
|
+
...response,
|
|
867
|
+
requestId: request.requestId
|
|
868
|
+
});
|
|
869
|
+
},
|
|
870
|
+
reject: (err) => {
|
|
871
|
+
clearTimeout(timer);
|
|
872
|
+
reject(err);
|
|
873
|
+
}
|
|
874
|
+
});
|
|
875
|
+
if (request.op === "beginUpload") this.sendDirect({
|
|
876
|
+
type: "artifacts.upload.request",
|
|
877
|
+
messageId,
|
|
878
|
+
jobId,
|
|
879
|
+
name: request.name,
|
|
880
|
+
declaredSizeBytes: request.declaredSizeBytes
|
|
881
|
+
});
|
|
882
|
+
else this.sendDirect({
|
|
883
|
+
type: "artifacts.download.request",
|
|
884
|
+
messageId,
|
|
885
|
+
jobId,
|
|
886
|
+
name: request.name
|
|
887
|
+
});
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
/**
|
|
718
891
|
* Relay a provenance bundle upload operation to the orchestrator. Maps the
|
|
719
892
|
* IPC `provenance.request` onto `requestProvenanceUploadUrl` (returns the
|
|
720
893
|
* presigned URL) or `sendProvenanceUploadComplete` (fire-and-forget) and
|
|
@@ -929,6 +1102,17 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
929
1102
|
this.pendingApiRequests.clear();
|
|
930
1103
|
for (const [_id, pending] of this.pendingUserCacheRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
931
1104
|
this.pendingUserCacheRequests.clear();
|
|
1105
|
+
for (const [id, pending] of this.pendingUserArtifactRequests) {
|
|
1106
|
+
const record = this.resendableCompletes.get(id);
|
|
1107
|
+
if (record && !this.intentionalDisconnect) {
|
|
1108
|
+
this.heldCompletes.add(record);
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
1111
|
+
pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
1112
|
+
}
|
|
1113
|
+
this.pendingUserArtifactRequests.clear();
|
|
1114
|
+
this.resendableCompletes.clear();
|
|
1115
|
+
if (this.intentionalDisconnect) this.rejectHeldCompletes("the agent disconnected");
|
|
932
1116
|
for (const [_id, pending] of this.pendingConcurrencyRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
933
1117
|
this.pendingConcurrencyRequests.clear();
|
|
934
1118
|
for (const [_id, pending] of this.pendingStepApprovals) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
@@ -940,6 +1124,132 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
940
1124
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
941
1125
|
});
|
|
942
1126
|
}
|
|
1127
|
+
/**
|
|
1128
|
+
* Resolve a pending user-artifact request from an `artifacts.upload.response`
|
|
1129
|
+
* / `artifacts.download.response` WS message, mapping the wire fields onto the
|
|
1130
|
+
* IPC response shape. No-op when no request is pending for the requestId.
|
|
1131
|
+
*/
|
|
1132
|
+
resolveArtifactResponse(type, raw) {
|
|
1133
|
+
const artMsg = raw;
|
|
1134
|
+
const pending = this.pendingUserArtifactRequests.get(artMsg.requestId);
|
|
1135
|
+
if (!pending) return;
|
|
1136
|
+
this.pendingUserArtifactRequests.delete(artMsg.requestId);
|
|
1137
|
+
const isUpload = type === "artifacts.upload.response";
|
|
1138
|
+
pending.resolve({
|
|
1139
|
+
type: "artifacts.response",
|
|
1140
|
+
requestId: artMsg.requestId,
|
|
1141
|
+
...isUpload && artMsg.outcome !== void 0 && { uploadOutcome: artMsg.outcome },
|
|
1142
|
+
...!isUpload && artMsg.outcome !== void 0 && { downloadOutcome: artMsg.outcome },
|
|
1143
|
+
...artMsg.uploadUrl && { uploadUrl: artMsg.uploadUrl },
|
|
1144
|
+
...artMsg.storageKey && { storageKey: artMsg.storageKey },
|
|
1145
|
+
...artMsg.reason && { reason: artMsg.reason },
|
|
1146
|
+
...artMsg.error && { rejectionDetail: artMsg.error },
|
|
1147
|
+
...artMsg.downloadUrl && { downloadUrl: artMsg.downloadUrl },
|
|
1148
|
+
...artMsg.sizeBytes !== void 0 && { sizeBytes: artMsg.sizeBytes },
|
|
1149
|
+
...artMsg.sha256 && { sha256: artMsg.sha256 }
|
|
1150
|
+
});
|
|
1151
|
+
}
|
|
1152
|
+
/**
|
|
1153
|
+
* Resolve or reject a pending completeUpload from an
|
|
1154
|
+
* `artifacts.upload.complete.ack`. `committed` resolves the relayed IPC
|
|
1155
|
+
* request; `failed` rejects it (carrying the orchestrator's reason), which
|
|
1156
|
+
* propagates through the relay's `response.error` and fails the workflow step
|
|
1157
|
+
* rather than losing the artifact silently. No-op when nothing is pending for
|
|
1158
|
+
* the requestId (an ack for an already-timed-out complete).
|
|
1159
|
+
*/
|
|
1160
|
+
resolveArtifactCompleteAck(raw) {
|
|
1161
|
+
const ack = raw;
|
|
1162
|
+
const pending = this.pendingUserArtifactRequests.get(ack.requestId);
|
|
1163
|
+
if (!pending) return;
|
|
1164
|
+
this.pendingUserArtifactRequests.delete(ack.requestId);
|
|
1165
|
+
this.resendableCompletes.delete(ack.requestId);
|
|
1166
|
+
if (ack.outcome === ArtifactCompleteAckOutcome.enum.committed) pending.resolve({
|
|
1167
|
+
type: "artifacts.response",
|
|
1168
|
+
requestId: ack.requestId
|
|
1169
|
+
});
|
|
1170
|
+
else pending.reject(new Error(ack.reason ?? "artifact upload-complete failed"));
|
|
1171
|
+
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Drop every trace of an in-flight upload-complete, whichever key it currently
|
|
1174
|
+
* lives under and whether or not it is parked awaiting a resend.
|
|
1175
|
+
*
|
|
1176
|
+
* Called from the ack timer, which fires on the original schedule regardless of
|
|
1177
|
+
* how many times the complete has been re-keyed since.
|
|
1178
|
+
*/
|
|
1179
|
+
forgetComplete(record) {
|
|
1180
|
+
this.pendingUserArtifactRequests.delete(record.messageId);
|
|
1181
|
+
this.resendableCompletes.delete(record.messageId);
|
|
1182
|
+
this.heldCompletes.delete(record);
|
|
1183
|
+
}
|
|
1184
|
+
/**
|
|
1185
|
+
* Fail every parked upload-complete closed, for the paths where the reconnect
|
|
1186
|
+
* the park was betting on will never arrive (a deliberate shutdown, a
|
|
1187
|
+
* permanently rejected token). Naming the ambiguity keeps the operator from
|
|
1188
|
+
* hunting for an artifact that in fact landed.
|
|
1189
|
+
*/
|
|
1190
|
+
rejectHeldCompletes(cause) {
|
|
1191
|
+
if (this.heldCompletes.size === 0) return;
|
|
1192
|
+
const held = [...this.heldCompletes];
|
|
1193
|
+
this.heldCompletes.clear();
|
|
1194
|
+
for (const record of held) record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete could not be acknowledged before ${cause} — the artifact may have been committed`));
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Re-send every upload-complete parked by the disconnect sweep.
|
|
1198
|
+
*
|
|
1199
|
+
* Called from `register.ack`, once the new connection's capabilities are known.
|
|
1200
|
+
* Each frame is re-keyed with a fresh `messageId` before it goes out: the id is
|
|
1201
|
+
* the correlation key, so reusing it would let a late ack for the pre-disconnect
|
|
1202
|
+
* send resolve the new pending.
|
|
1203
|
+
*
|
|
1204
|
+
* The ack timer created when the complete was first sent is deliberately left
|
|
1205
|
+
* alone, so the hold plus every resend stays inside the original
|
|
1206
|
+
* ARTIFACT_COMPLETE_ACK_TIMEOUT_MS budget.
|
|
1207
|
+
*/
|
|
1208
|
+
resendHeldCompletes() {
|
|
1209
|
+
if (this.heldCompletes.size === 0) return;
|
|
1210
|
+
const held = [...this.heldCompletes];
|
|
1211
|
+
this.heldCompletes.clear();
|
|
1212
|
+
const canAck = hasOrchAgentCapability(this.orchCapabilities, "artifactCompleteAck");
|
|
1213
|
+
for (const record of held) {
|
|
1214
|
+
if (!canAck) {
|
|
1215
|
+
record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete cannot be acknowledged by the reconnected orchestrator — ${COMMIT_MAY_HAVE_LANDED}`));
|
|
1216
|
+
continue;
|
|
1217
|
+
}
|
|
1218
|
+
if (record.attempts >= 2) {
|
|
1219
|
+
record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete could not be acknowledged after 2 reconnects — ${COMMIT_MAY_HAVE_LANDED}`));
|
|
1220
|
+
continue;
|
|
1221
|
+
}
|
|
1222
|
+
record.messageId = randomUUID();
|
|
1223
|
+
record.attempts += 1;
|
|
1224
|
+
record.frame = {
|
|
1225
|
+
...record.frame,
|
|
1226
|
+
messageId: record.messageId
|
|
1227
|
+
};
|
|
1228
|
+
this.pendingUserArtifactRequests.set(record.messageId, record.pending);
|
|
1229
|
+
this.resendableCompletes.set(record.messageId, record);
|
|
1230
|
+
this.sendDirect(record.frame);
|
|
1231
|
+
logger$12.info("Re-sent artifact upload-complete after reconnect", {
|
|
1232
|
+
messageId: record.messageId,
|
|
1233
|
+
attempt: record.attempts
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Route the orchestrator's replies to a relayed user-artifact request, which
|
|
1239
|
+
* are dispatched off the raw frame rather than the parsed protocol union.
|
|
1240
|
+
* Returns true when the message was consumed.
|
|
1241
|
+
*/
|
|
1242
|
+
handleArtifactReply(type, raw) {
|
|
1243
|
+
if (type === "artifacts.upload.response" || type === "artifacts.download.response") {
|
|
1244
|
+
this.resolveArtifactResponse(type, raw);
|
|
1245
|
+
return true;
|
|
1246
|
+
}
|
|
1247
|
+
if (type === "artifacts.upload.complete.ack") {
|
|
1248
|
+
this.resolveArtifactCompleteAck(raw);
|
|
1249
|
+
return true;
|
|
1250
|
+
}
|
|
1251
|
+
return false;
|
|
1252
|
+
}
|
|
943
1253
|
handleMessage(data) {
|
|
944
1254
|
let raw;
|
|
945
1255
|
try {
|
|
@@ -997,6 +1307,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
997
1307
|
}
|
|
998
1308
|
return;
|
|
999
1309
|
}
|
|
1310
|
+
if (this.handleArtifactReply(rawMsg.type, raw)) return;
|
|
1000
1311
|
const parsed = orchestratorToAgentMessageSchema.safeParse(raw);
|
|
1001
1312
|
if (parsed.success) {
|
|
1002
1313
|
const msg = parsed.data;
|
|
@@ -1025,10 +1336,12 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1025
1336
|
scalerManaged: msg.scalerManaged,
|
|
1026
1337
|
pendingDispatch: msg.pendingDispatch ?? false
|
|
1027
1338
|
});
|
|
1339
|
+
this.orchCapabilities = msg.capabilities;
|
|
1028
1340
|
this._state = "registered";
|
|
1029
1341
|
this.reconnectAttempts = 0;
|
|
1030
1342
|
this.startHeartbeat();
|
|
1031
1343
|
this.flushBuffer();
|
|
1344
|
+
this.resendHeldCompletes();
|
|
1032
1345
|
this.onRegistered?.({ pendingDispatch: msg.pendingDispatch ?? false });
|
|
1033
1346
|
if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
|
|
1034
1347
|
this.sendConfigAck(msg.agentId);
|
|
@@ -1342,14 +1655,14 @@ var init_console_capture = __esmMin((() => {
|
|
|
1342
1655
|
init_console_capture();
|
|
1343
1656
|
function safe(name, fallback = "unknown") {
|
|
1344
1657
|
switch (name) {
|
|
1345
|
-
case "version": return "0.
|
|
1346
|
-
case "buildCommit": return "
|
|
1347
|
-
case "sdkVersion": return "0.
|
|
1348
|
-
case "sdkBundleHash": return "
|
|
1349
|
-
case "sharedVersion": return "0.
|
|
1350
|
-
case "sharedBundleHash": return "
|
|
1351
|
-
case "engineVersion": return "0.
|
|
1352
|
-
case "engineBundleHash": return "
|
|
1658
|
+
case "version": return "0.2.0";
|
|
1659
|
+
case "buildCommit": return "15d5e4447";
|
|
1660
|
+
case "sdkVersion": return "0.2.0";
|
|
1661
|
+
case "sdkBundleHash": return "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
|
|
1662
|
+
case "sharedVersion": return "0.2.0";
|
|
1663
|
+
case "sharedBundleHash": return "f16fa528a8a9ef1bed8df369f0e88efafca8a9954a749005299fdbcdb93fa188";
|
|
1664
|
+
case "engineVersion": return "0.2.0";
|
|
1665
|
+
case "engineBundleHash": return "c579e54f7d0b0d5b16587a0528a3e66d32f63e2434bb471a93a01366296efd58";
|
|
1353
1666
|
default: return fallback;
|
|
1354
1667
|
}
|
|
1355
1668
|
}
|
|
@@ -1400,15 +1713,6 @@ function createHealthRoutes$1(deps) {
|
|
|
1400
1713
|
* The orchestrator aggregates these metrics from all agents and exposes
|
|
1401
1714
|
* them on its `/metrics` endpoint for Prometheus scraping.
|
|
1402
1715
|
*/
|
|
1403
|
-
/**
|
|
1404
|
-
* OTel DataPointType enum values (from @opentelemetry/sdk-metrics).
|
|
1405
|
-
* We duplicate them here to avoid a direct runtime dependency on sdk-metrics.
|
|
1406
|
-
*/
|
|
1407
|
-
const DataPointType = {
|
|
1408
|
-
HISTOGRAM: 0,
|
|
1409
|
-
GAUGE: 2,
|
|
1410
|
-
SUM: 3
|
|
1411
|
-
};
|
|
1412
1716
|
var MetricsReporter = class {
|
|
1413
1717
|
timer;
|
|
1414
1718
|
agentId;
|
|
@@ -1456,7 +1760,7 @@ var MetricsReporter = class {
|
|
|
1456
1760
|
const wireType = this.mapDataPointType(metricData.dataPointType, metricData.isMonotonic);
|
|
1457
1761
|
for (const dp of metricData.dataPoints) {
|
|
1458
1762
|
const labels = this.extractLabels(dp.attributes);
|
|
1459
|
-
if (metricData.dataPointType ===
|
|
1763
|
+
if (metricData.dataPointType === OTEL_DATA_POINT_TYPE.HISTOGRAM) {
|
|
1460
1764
|
const histValue = dp.value;
|
|
1461
1765
|
const boundaries = histValue.buckets.boundaries;
|
|
1462
1766
|
const counts = histValue.buckets.counts;
|
|
@@ -1489,12 +1793,7 @@ var MetricsReporter = class {
|
|
|
1489
1793
|
}
|
|
1490
1794
|
/** Map OTel DataPointType to wire format type string. */
|
|
1491
1795
|
mapDataPointType(dataPointType, isMonotonic) {
|
|
1492
|
-
|
|
1493
|
-
case DataPointType.HISTOGRAM: return "histogram";
|
|
1494
|
-
case DataPointType.GAUGE: return "gauge";
|
|
1495
|
-
case DataPointType.SUM: return isMonotonic === false ? "upDownCounter" : "counter";
|
|
1496
|
-
default: return "gauge";
|
|
1497
|
-
}
|
|
1796
|
+
return mapDataPointTypeToWireKind(dataPointType, isMonotonic);
|
|
1498
1797
|
}
|
|
1499
1798
|
/** Extract string labels from OTel Attributes. */
|
|
1500
1799
|
extractLabels(attributes) {
|
|
@@ -1575,28 +1874,57 @@ init_npm_resolver();
|
|
|
1575
1874
|
/**
|
|
1576
1875
|
* Startup garbage collection for this agent's own temp-directory families.
|
|
1577
1876
|
*
|
|
1578
|
-
*
|
|
1579
|
-
*
|
|
1580
|
-
*
|
|
1581
|
-
*
|
|
1582
|
-
*
|
|
1583
|
-
*
|
|
1584
|
-
*
|
|
1877
|
+
* Bare job workdirs (`kici-<6 random chars>`, see job-runner.ts), labeled
|
|
1878
|
+
* allocator dirs (`kici-<label>-<6 random chars>`, minted by the global temp
|
|
1879
|
+
* allocator `@kici-dev/core/tmp`, which guarantees every allocation carries
|
|
1880
|
+
* the `kici-` prefix), and isolated pnpm stores (`kici-pnpm-store-*`, see
|
|
1881
|
+
* dep-installer.ts) clean themselves up in `finally` blocks — but a hard
|
|
1882
|
+
* process death (SIGKILL, OOM kill) skips those, and on a long-lived
|
|
1883
|
+
* bare-metal agent the leftovers then accumulate forever. Collecting anything
|
|
1884
|
+
* older than a day at startup is safe on shared hosts: no job lives remotely
|
|
1885
|
+
* that long (job timeouts are minutes), so a concurrent agent's in-flight
|
|
1886
|
+
* dirs are never eligible.
|
|
1887
|
+
*
|
|
1888
|
+
* The deterministic persistent caches (`kici-agent-payloads`, `kici-data`,
|
|
1889
|
+
* `kici-scaler-ledger`) also live directly under the temp root and can be far
|
|
1890
|
+
* older than a day, so they are explicitly excluded — `kici-scaler-ledger`
|
|
1891
|
+
* even structurally matches the allocator pattern (`ledger` is a 6-char label
|
|
1892
|
+
* suffix), which a regex alone cannot distinguish.
|
|
1585
1893
|
*/
|
|
1586
1894
|
const AGENT_TMP_GC_MAX_AGE_MS = 1440 * 60 * 1e3;
|
|
1587
|
-
/**
|
|
1588
|
-
|
|
1895
|
+
/**
|
|
1896
|
+
* Bare `kici-<6 chars>` workdirs and labeled `kici-<label>-<6 chars>`
|
|
1897
|
+
* allocator dirs. The optional label group makes both families eligible.
|
|
1898
|
+
*/
|
|
1899
|
+
const AGENT_WORKDIR_PATTERN = /^kici-([a-z0-9-]+-)?[A-Za-z0-9]{6}$/;
|
|
1589
1900
|
const PNPM_STORE_PATTERN = /^kici-pnpm-store-/;
|
|
1590
1901
|
/**
|
|
1902
|
+
* Deterministic persistent caches that share the `kici-` prefix but must
|
|
1903
|
+
* never be collected — `kici-scaler-ledger` even matches the allocator
|
|
1904
|
+
* pattern, so a basename exclude (not a regex) is the only correct guard.
|
|
1905
|
+
*/
|
|
1906
|
+
const PERSISTENT_CACHES = /* @__PURE__ */ new Set([
|
|
1907
|
+
"kici-agent-payloads",
|
|
1908
|
+
"kici-data",
|
|
1909
|
+
"kici-scaler-ledger"
|
|
1910
|
+
]);
|
|
1911
|
+
/**
|
|
1591
1912
|
* Collect this agent's stale temp dirs. `base` is overridable for tests;
|
|
1592
1913
|
* production callers use the default temp root. Never throws.
|
|
1914
|
+
*
|
|
1915
|
+
* The default base is `kiciTmpBase()`, not the OS temp root, so it scans the
|
|
1916
|
+
* exact directory the agent now writes payloads/clones/pnpm stores into: the
|
|
1917
|
+
* global temp allocator and the payload cache both honor `KICI_TMPDIR` via the
|
|
1918
|
+
* same helper. If the GC scanned a different root, stale-temp reaping would
|
|
1919
|
+
* silently stop working whenever `KICI_TMPDIR` is set.
|
|
1593
1920
|
*/
|
|
1594
|
-
async function gcStaleAgentTmpDirs(base =
|
|
1921
|
+
async function gcStaleAgentTmpDirs(base = kiciTmpBase()) {
|
|
1595
1922
|
const log = (m) => logger.info(m);
|
|
1596
1923
|
return [...await gcStaleTmpDirs({
|
|
1597
1924
|
base,
|
|
1598
1925
|
pattern: AGENT_WORKDIR_PATTERN,
|
|
1599
1926
|
maxAgeMs: AGENT_TMP_GC_MAX_AGE_MS,
|
|
1927
|
+
exclude: PERSISTENT_CACHES,
|
|
1600
1928
|
log
|
|
1601
1929
|
}), ...await gcStaleTmpDirs({
|
|
1602
1930
|
base,
|
|
@@ -1752,7 +2080,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
1752
2080
|
*/
|
|
1753
2081
|
async function setupSshAuth(opts) {
|
|
1754
2082
|
if (opts.hostKeyPolicy === "pinned" && !opts.knownHosts) throw new Error("pinned hostKeyPolicy requires knownHosts content");
|
|
1755
|
-
const tempDir = await
|
|
2083
|
+
const { path: tempDir, cleanup } = await makeTempDir("ssh");
|
|
1756
2084
|
const keyPath = join(tempDir, "id");
|
|
1757
2085
|
await writeFile(keyPath, opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`, { mode: 384 });
|
|
1758
2086
|
const knownHostsPath = join(tempDir, "known_hosts");
|
|
@@ -1773,12 +2101,7 @@ async function setupSshAuth(opts) {
|
|
|
1773
2101
|
return {
|
|
1774
2102
|
gitSshCommand: parts.join(" "),
|
|
1775
2103
|
tempDir,
|
|
1776
|
-
|
|
1777
|
-
await rm(tempDir, {
|
|
1778
|
-
recursive: true,
|
|
1779
|
-
force: true
|
|
1780
|
-
});
|
|
1781
|
-
}
|
|
2104
|
+
cleanup
|
|
1782
2105
|
};
|
|
1783
2106
|
}
|
|
1784
2107
|
/**
|
|
@@ -1830,19 +2153,16 @@ async function gitClone(options) {
|
|
|
1830
2153
|
let needsCustomEnv = false;
|
|
1831
2154
|
let safeDirCleanup;
|
|
1832
2155
|
if (repoUrl.startsWith("file://")) {
|
|
1833
|
-
const {
|
|
1834
|
-
const { tmpdir } = await import("node:os");
|
|
2156
|
+
const { writeFile } = await import("node:fs/promises");
|
|
1835
2157
|
const path = await import("node:path");
|
|
1836
|
-
const
|
|
2158
|
+
const { makeTempDir } = await import("@kici-dev/core/tmp");
|
|
2159
|
+
const { path: dir, cleanup } = await makeTempDir("gitcfg");
|
|
1837
2160
|
const cfgPath = path.join(dir, "config");
|
|
1838
2161
|
await writeFile(cfgPath, "[safe]\n directory = *\n", { mode: 384 });
|
|
1839
2162
|
envEntries.GIT_CONFIG_GLOBAL = cfgPath;
|
|
1840
2163
|
needsCustomEnv = true;
|
|
1841
2164
|
safeDirCleanup = async () => {
|
|
1842
|
-
await
|
|
1843
|
-
recursive: true,
|
|
1844
|
-
force: true
|
|
1845
|
-
}).catch(() => {});
|
|
2165
|
+
await cleanup().catch(() => {});
|
|
1846
2166
|
};
|
|
1847
2167
|
}
|
|
1848
2168
|
let sshSetup;
|
|
@@ -1955,11 +2275,46 @@ var workflow_loader_exports = /* @__PURE__ */ __exportAll({
|
|
|
1955
2275
|
extractSteps: () => extractSteps,
|
|
1956
2276
|
extractStepsFromDynamicJob: () => extractStepsFromDynamicJob,
|
|
1957
2277
|
extractWorkflow: () => extractWorkflow,
|
|
1958
|
-
loadWorkflowSource: () => loadWorkflowSource
|
|
2278
|
+
loadWorkflowSource: () => loadWorkflowSource,
|
|
2279
|
+
resolveWorkflowSdkSetters: () => resolveWorkflowSdkSetters
|
|
1959
2280
|
});
|
|
2281
|
+
/**
|
|
2282
|
+
* Resolve the `@kici-dev/sdk` instance the workflow module itself imports.
|
|
2283
|
+
*
|
|
2284
|
+
* The workflow's `.result` proxies read the module-global step-outputs map of
|
|
2285
|
+
* whichever SDK copy the workflow file resolves — which is generally a
|
|
2286
|
+
* different physical module than the agent's bundled SDK (the workflow is
|
|
2287
|
+
* imported from the cloned source tree and resolves its deps against that
|
|
2288
|
+
* tree's `node_modules`). Resolving via `createRequire(workflowFilePath)` walks
|
|
2289
|
+
* `node_modules` from the workflow file exactly the way the workflow's own
|
|
2290
|
+
* `import '@kici-dev/sdk'` does — including any hoisted copy — so the returned
|
|
2291
|
+
* setters mutate the SAME module-global map object the proxies read. Node caches
|
|
2292
|
+
* ESM modules by resolved URL, so importing that path yields the workflow's live
|
|
2293
|
+
* singleton, not a fresh copy.
|
|
2294
|
+
*
|
|
2295
|
+
* Falls back to the agent's bundled setters when resolution fails (mirrors
|
|
2296
|
+
* `resolveSdkSetters` in the compiler's test runner).
|
|
2297
|
+
*/
|
|
2298
|
+
async function resolveWorkflowSdkSetters(workflowFilePath) {
|
|
2299
|
+
try {
|
|
2300
|
+
const sdk = await import(pathToFileURL(createRequire(workflowFilePath).resolve("@kici-dev/sdk")).href);
|
|
2301
|
+
if (typeof sdk.setStepOutputsMap === "function" && typeof sdk.setStepRefMap === "function" && typeof sdk.setJobOutputsMap === "function") return {
|
|
2302
|
+
setStepOutputsMap: sdk.setStepOutputsMap,
|
|
2303
|
+
setStepRefMap: sdk.setStepRefMap,
|
|
2304
|
+
setJobOutputsMap: sdk.setJobOutputsMap
|
|
2305
|
+
};
|
|
2306
|
+
} catch {}
|
|
2307
|
+
return {
|
|
2308
|
+
setStepOutputsMap,
|
|
2309
|
+
setStepRefMap,
|
|
2310
|
+
setJobOutputsMap
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
1960
2313
|
function ensureLoaderHookRegistered() {
|
|
1961
2314
|
if (hookRegistered) return;
|
|
1962
|
-
|
|
2315
|
+
const hookPath = process.env.KICI_TS_LOADER_HOOK_PATH;
|
|
2316
|
+
if (hookPath) register(pathToFileURL(hookPath).href, import.meta.url);
|
|
2317
|
+
else register("@kici-dev/core/ts-loader-hook", import.meta.url);
|
|
1963
2318
|
hookRegistered = true;
|
|
1964
2319
|
}
|
|
1965
2320
|
/**
|
|
@@ -2014,7 +2369,10 @@ async function loadWorkflowSource(workDir, sourceFile, expectedContentHash, reso
|
|
|
2014
2369
|
const actualHash = computeContentHash(rawSource, assetDigest);
|
|
2015
2370
|
if (actualHash !== expectedContentHash) throw new Error(`Lock file is out of date: workflow source changed without regenerating kici.lock.json (expected contentHash ${expectedContentHash}, got ${actualHash}, agent baked @kici-dev/sdk@${AGENT_SDK_VERSION} bundleHash=${AGENT_SDK_BUNDLE_HASH}). Run 'kici compile' and commit the updated lock file.`);
|
|
2016
2371
|
}
|
|
2017
|
-
return {
|
|
2372
|
+
return {
|
|
2373
|
+
module: await import(pathToFileURL(filePath).href + `?t=${Date.now()}`),
|
|
2374
|
+
sdkSetters: await resolveWorkflowSdkSetters(filePath)
|
|
2375
|
+
};
|
|
2018
2376
|
}
|
|
2019
2377
|
/**
|
|
2020
2378
|
* Type guard for Workflow shape (discriminant: `_tag === 'Workflow'`).
|
|
@@ -2112,8 +2470,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
|
|
|
2112
2470
|
}
|
|
2113
2471
|
var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
|
|
2114
2472
|
var init_workflow_loader = __esmMin((() => {
|
|
2115
|
-
AGENT_SDK_VERSION = "0.
|
|
2116
|
-
AGENT_SDK_BUNDLE_HASH = "
|
|
2473
|
+
AGENT_SDK_VERSION = "0.2.0";
|
|
2474
|
+
AGENT_SDK_BUNDLE_HASH = "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
|
|
2117
2475
|
hookRegistered = false;
|
|
2118
2476
|
}));
|
|
2119
2477
|
//#endregion
|
|
@@ -2539,7 +2897,7 @@ function findJobByName(workflow, jobName) {
|
|
|
2539
2897
|
throw new Error(`Job '${jobName}' not found in workflow '${workflow.name}'`);
|
|
2540
2898
|
}
|
|
2541
2899
|
/**
|
|
2542
|
-
* Evaluate dynamic fields (
|
|
2900
|
+
* Evaluate dynamic fields (context, env, concurrencyGroup) on a job.
|
|
2543
2901
|
*
|
|
2544
2902
|
* Only fields with their corresponding flag set to true AND whose property
|
|
2545
2903
|
* on the job is a function will be evaluated. All evaluations happen in a
|
|
@@ -2576,19 +2934,28 @@ async function evaluateDynamicFields(workflow, jobName, event, flags, timeoutMs
|
|
|
2576
2934
|
},
|
|
2577
2935
|
env: { ...process.env }
|
|
2578
2936
|
};
|
|
2579
|
-
|
|
2937
|
+
const resolved = await withTimeout(() => job.matrix(matrixContext), timeoutMs, `dynamicMatrix for job '${jobName}'`);
|
|
2938
|
+
let combos;
|
|
2939
|
+
try {
|
|
2940
|
+
const rawCount = matrixCombinationCount(resolved);
|
|
2941
|
+
if (rawCount > MAX_MATRIX_MATERIALIZATION) throw new MatrixShapeError(`matrix is too large to expand: ${rawCount} raw combinations (max ${MAX_MATRIX_MATERIALIZATION})`);
|
|
2942
|
+
combos = expandMatrix(resolved);
|
|
2943
|
+
} catch (err) {
|
|
2944
|
+
if (err instanceof MatrixShapeError) throw new MatrixShapeError(`dynamicMatrix for job '${jobName}': ${err.message}`);
|
|
2945
|
+
throw err;
|
|
2946
|
+
}
|
|
2580
2947
|
if (job.include || job.exclude) combos = applyIncludeExclude(combos, job.include, job.exclude);
|
|
2581
2948
|
result.matrixValues = combos;
|
|
2582
2949
|
}
|
|
2583
|
-
if (flags.
|
|
2584
|
-
const envRefs = job.
|
|
2950
|
+
if (flags.dynamicContext) {
|
|
2951
|
+
const envRefs = job.contexts ?? (job.context !== void 0 ? [job.context] : void 0);
|
|
2585
2952
|
if (envRefs && envRefs.length > 0) {
|
|
2586
2953
|
const names = [];
|
|
2587
2954
|
for (const ref of envRefs) if (typeof ref === "function") {
|
|
2588
|
-
const value = await withTimeout(() => ref(event), timeoutMs, `
|
|
2955
|
+
const value = await withTimeout(() => ref(event), timeoutMs, `dynamicContext for job '${jobName}'`);
|
|
2589
2956
|
if (value !== void 0 && value !== null) names.push(value);
|
|
2590
2957
|
} else if (typeof ref === "string") names.push(ref);
|
|
2591
|
-
if (names.length > 0) result.
|
|
2958
|
+
if (names.length > 0) result.contextNames = names;
|
|
2592
2959
|
}
|
|
2593
2960
|
}
|
|
2594
2961
|
if (flags.dynamicEnv && typeof job.env === "function") {
|
|
@@ -2659,13 +3026,14 @@ async function sshExec(reach, privateKey, command, opts = {}, deps = {}) {
|
|
|
2659
3026
|
const hostKeyMode = opts.hostKeyMode ?? "accept-new";
|
|
2660
3027
|
const { dest, port } = resolveTarget(reach, opts.port);
|
|
2661
3028
|
return withEphemeralAgent(privateKey, spawnFn, async (env) => {
|
|
2662
|
-
|
|
3029
|
+
const args = [
|
|
2663
3030
|
...baseSshOptions(hostKeyMode),
|
|
2664
3031
|
"-p",
|
|
2665
3032
|
String(port),
|
|
2666
3033
|
dest,
|
|
2667
3034
|
command
|
|
2668
|
-
]
|
|
3035
|
+
];
|
|
3036
|
+
return spawnFn("ssh", args, {
|
|
2669
3037
|
env,
|
|
2670
3038
|
stdin: opts.stdin
|
|
2671
3039
|
});
|
|
@@ -2681,13 +3049,14 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
|
|
|
2681
3049
|
const hostKeyMode = opts.hostKeyMode ?? "accept-new";
|
|
2682
3050
|
const { dest, port } = resolveTarget(reach, opts.port);
|
|
2683
3051
|
const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
|
|
2684
|
-
|
|
3052
|
+
const args = [
|
|
2685
3053
|
...baseSshOptions(hostKeyMode),
|
|
2686
3054
|
"-p",
|
|
2687
3055
|
String(port),
|
|
2688
3056
|
dest,
|
|
2689
3057
|
`cat > '${remotePath.replace(/'/g, `'\\''`)}'`
|
|
2690
|
-
]
|
|
3058
|
+
];
|
|
3059
|
+
return spawnFn("ssh", args, {
|
|
2691
3060
|
env,
|
|
2692
3061
|
stdin: localBytes
|
|
2693
3062
|
});
|
|
@@ -2695,6 +3064,32 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
|
|
|
2695
3064
|
if (result.exitCode !== 0) throw new Error(`sshPush(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
|
|
2696
3065
|
}
|
|
2697
3066
|
/**
|
|
3067
|
+
* Ship a local FILE to a remote path over `scp` (a binary channel), using the
|
|
3068
|
+
* same ephemeral-key discipline as `sshExec`/`sshPush`.
|
|
3069
|
+
*
|
|
3070
|
+
* Unlike `sshPush` — which pipes a JS string through `ssh 'cat > path'` and so
|
|
3071
|
+
* corrupts binary payloads (a ~50 MB tarball) — `scp` streams the file's raw
|
|
3072
|
+
* bytes untouched. This is the ONLY sanctioned path for a binary payload
|
|
3073
|
+
* transfer; the string `cat >` path is for text (launcher scripts) only. Throws
|
|
3074
|
+
* on a non-zero exit (a payload push must succeed end-to-end).
|
|
3075
|
+
*/
|
|
3076
|
+
async function sshPushFile(reach, privateKey, localFilePath, remotePath, opts = {}, deps = {}) {
|
|
3077
|
+
const spawnFn = deps.spawnFn ?? defaultSpawn;
|
|
3078
|
+
const hostKeyMode = opts.hostKeyMode ?? "accept-new";
|
|
3079
|
+
const { dest, port } = resolveTarget(reach, opts.port);
|
|
3080
|
+
const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
|
|
3081
|
+
const args = [
|
|
3082
|
+
...baseSshOptions(hostKeyMode),
|
|
3083
|
+
"-P",
|
|
3084
|
+
String(port),
|
|
3085
|
+
localFilePath,
|
|
3086
|
+
`${dest}:${remotePath}`
|
|
3087
|
+
];
|
|
3088
|
+
return spawnFn("scp", args, { env });
|
|
3089
|
+
});
|
|
3090
|
+
if (result.exitCode !== 0) throw new Error(`sshPushFile(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
|
|
3091
|
+
}
|
|
3092
|
+
/**
|
|
2698
3093
|
* Start a per-call ephemeral ssh-agent, load the key via stdin (never a file),
|
|
2699
3094
|
* run `body` with `SSH_AUTH_SOCK` in env, and kill the agent in `finally`.
|
|
2700
3095
|
*
|
|
@@ -2710,7 +3105,7 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
|
|
|
2710
3105
|
*/
|
|
2711
3106
|
async function withEphemeralAgent(privateKey, spawnFn, body) {
|
|
2712
3107
|
const baseEnv = { ...process.env };
|
|
2713
|
-
const agentDir = await
|
|
3108
|
+
const { path: agentDir, cleanup } = await makeTempDir("bootstrap-ssh");
|
|
2714
3109
|
const sock = join(agentDir, "agent.sock");
|
|
2715
3110
|
try {
|
|
2716
3111
|
const start = await spawnFn("ssh-agent", [
|
|
@@ -2723,10 +3118,11 @@ async function withEphemeralAgent(privateKey, spawnFn, body) {
|
|
|
2723
3118
|
const agentEnv = {
|
|
2724
3119
|
...baseEnv,
|
|
2725
3120
|
SSH_AUTH_SOCK: sock,
|
|
2726
|
-
...pid ? { SSH_AGENT_PID: pid } : {},
|
|
2727
3121
|
SSH_ASKPASS: "/bin/false",
|
|
2728
3122
|
DISPLAY: ""
|
|
2729
3123
|
};
|
|
3124
|
+
if (pid) agentEnv.SSH_AGENT_PID = pid;
|
|
3125
|
+
else delete agentEnv.SSH_AGENT_PID;
|
|
2730
3126
|
try {
|
|
2731
3127
|
const add = await spawnFn("ssh-add", ["-"], {
|
|
2732
3128
|
env: agentEnv,
|
|
@@ -2738,10 +3134,7 @@ async function withEphemeralAgent(privateKey, spawnFn, body) {
|
|
|
2738
3134
|
await spawnFn("ssh-agent", ["-k"], { env: agentEnv }).catch(() => {});
|
|
2739
3135
|
}
|
|
2740
3136
|
} finally {
|
|
2741
|
-
await
|
|
2742
|
-
recursive: true,
|
|
2743
|
-
force: true
|
|
2744
|
-
}).catch(() => {});
|
|
3137
|
+
await cleanup().catch(() => {});
|
|
2745
3138
|
}
|
|
2746
3139
|
}
|
|
2747
3140
|
/** Extract `SSH_AGENT_PID=<n>;` from `ssh-agent -s` output (best-effort). */
|
|
@@ -2773,7 +3166,10 @@ var init_ssh_exec = __esmMin((() => {
|
|
|
2773
3166
|
stdout,
|
|
2774
3167
|
stderr
|
|
2775
3168
|
}));
|
|
2776
|
-
if (opts.stdin !== void 0)
|
|
3169
|
+
if (opts.stdin !== void 0) {
|
|
3170
|
+
child.stdin?.on("error", () => {});
|
|
3171
|
+
child.stdin?.end(opts.stdin);
|
|
3172
|
+
}
|
|
2777
3173
|
});
|
|
2778
3174
|
SSH_USER_DEFAULT = "root";
|
|
2779
3175
|
SSH_PORT_DEFAULT = 22;
|
|
@@ -2783,8 +3179,166 @@ var init_ssh_exec = __esmMin((() => {
|
|
|
2783
3179
|
};
|
|
2784
3180
|
}));
|
|
2785
3181
|
//#endregion
|
|
3182
|
+
//#region src/bootstrap/probe-platform.ts
|
|
3183
|
+
/**
|
|
3184
|
+
* Probe a bring-up target's platform over SSH.
|
|
3185
|
+
*
|
|
3186
|
+
* A stock rescue box has nothing but sshd — no `kici-agent`, no Node — so we
|
|
3187
|
+
* cannot ask the box what to run; we must ask what it IS, then stage the
|
|
3188
|
+
* matching self-contained payload. One `uname -s -m` (+ a libc check) maps to
|
|
3189
|
+
* an `AgentPlatform`; an unsupported target (musl-only, non-Linux, unknown
|
|
3190
|
+
* arch) throws early, naming what was detected, rather than staging bytes that
|
|
3191
|
+
* won't boot.
|
|
3192
|
+
*/
|
|
3193
|
+
/**
|
|
3194
|
+
* Map `uname -s -m` (+ libc line) output to an `AgentPlatform`. Pure + directly
|
|
3195
|
+
* unit-tested. Throws a self-describing error for any unsupported target.
|
|
3196
|
+
*/
|
|
3197
|
+
function parseUname(stdout) {
|
|
3198
|
+
const lines = stdout.split("\n");
|
|
3199
|
+
const unameLine = (lines[0] ?? "").trim();
|
|
3200
|
+
const libc = lines.slice(1).join("\n").toLowerCase();
|
|
3201
|
+
const [os, arch] = unameLine.split(/\s+/);
|
|
3202
|
+
if (os !== "Linux") throw new Error(`unsupported bring-up target OS "${os}" (uname: "${unameLine}") — Linux only`);
|
|
3203
|
+
if (libc.includes("musl") || libc.includes("ld-musl")) throw new Error(`unsupported bring-up target: musl libc detected (uname: "${unameLine}") — glibc Linux only`);
|
|
3204
|
+
if (arch === "x86_64") return AgentPlatform.enum["linux-x64"];
|
|
3205
|
+
if (arch === "aarch64") return AgentPlatform.enum["linux-arm64"];
|
|
3206
|
+
throw new Error(`unsupported bring-up target arch "${arch}" (uname: "${unameLine}") — x86_64 or aarch64 only`);
|
|
3207
|
+
}
|
|
3208
|
+
/** SSH into `reach` and detect its `AgentPlatform` (one probe, fail-fast on unsupported). */
|
|
3209
|
+
async function probeTargetPlatform(reach, privateKey, deps = {}) {
|
|
3210
|
+
const res = await sshExec(reach, privateKey, PROBE_COMMAND, {}, deps);
|
|
3211
|
+
if (res.exitCode !== 0) throw new Error(`platform probe on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3212
|
+
return parseUname(res.stdout);
|
|
3213
|
+
}
|
|
3214
|
+
var PROBE_COMMAND;
|
|
3215
|
+
var init_probe_platform = __esmMin((() => {
|
|
3216
|
+
init_ssh_exec();
|
|
3217
|
+
PROBE_COMMAND = "uname -s -m; (ldd --version 2>&1 | head -1) || true";
|
|
3218
|
+
}));
|
|
3219
|
+
//#endregion
|
|
3220
|
+
//#region src/bootstrap/stage-agent-payload.ts
|
|
3221
|
+
/**
|
|
3222
|
+
* Stage a self-contained agent payload onto a bring-up target.
|
|
3223
|
+
*
|
|
3224
|
+
* The reusable core mechanism behind fresh-box bring-up: resolve a version-keyed
|
|
3225
|
+
* payload from an `AgentPayloadSource`, verify it, deliver it to the box, verify
|
|
3226
|
+
* it again ON the box before extracting, and return the launcher path. The
|
|
3227
|
+
* launcher boots the agent on its VENDORED Node, so a stock rescue box (no Node)
|
|
3228
|
+
* becomes a connected agent.
|
|
3229
|
+
*
|
|
3230
|
+
* Two delivery modes: `ssh-push` (the ops agent pulls the payload from its
|
|
3231
|
+
* source, then streams it to the box over a binary-safe scp) and `s3-direct`
|
|
3232
|
+
* (the box pulls a presigned URL itself, so no 50 MB transits the ops agent).
|
|
3233
|
+
* Every path is fail-closed: a hash it cannot verify is never extracted.
|
|
3234
|
+
*/
|
|
3235
|
+
/** Streamed sha256 of a local file (the default ops-agent-side verify). */
|
|
3236
|
+
function hashFileSha256(filePath) {
|
|
3237
|
+
return new Promise((resolve, reject) => {
|
|
3238
|
+
const hash = createHash("sha256");
|
|
3239
|
+
createReadStream(filePath).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve(hash.digest("hex")));
|
|
3240
|
+
});
|
|
3241
|
+
}
|
|
3242
|
+
/** Single-quote a value for safe embedding in a remote shell command. */
|
|
3243
|
+
function shQuote$2(v) {
|
|
3244
|
+
return `'${v.replace(/'/g, `'\\''`)}'`;
|
|
3245
|
+
}
|
|
3246
|
+
/**
|
|
3247
|
+
* Verify the resolved payload on the ops agent BEFORE any transfer. A payload
|
|
3248
|
+
* with no sidecar hash is unverifiable and refused (fail-closed); a hash
|
|
3249
|
+
* mismatch throws so corrupt/tampered bytes never leave this process.
|
|
3250
|
+
*/
|
|
3251
|
+
async function verifyLocalPayload(tarballPath, expectedSha256, deps) {
|
|
3252
|
+
if (!expectedSha256) throw new Error(`refusing to stage ${tarballPath}: no sha256 sidecar — cannot verify the payload (run \`kici-admin agent package\` to (re)generate it with a hash)`);
|
|
3253
|
+
const actual = await (deps.hashLocalFile ?? hashFileSha256)(tarballPath);
|
|
3254
|
+
if (actual !== expectedSha256) throw new Error(`payload hash mismatch for ${tarballPath}: expected ${expectedSha256}, got ${actual}`);
|
|
3255
|
+
return expectedSha256;
|
|
3256
|
+
}
|
|
3257
|
+
/**
|
|
3258
|
+
* On the box: verify the pushed tarball's hash, then extract it into
|
|
3259
|
+
* `extractDir`. `sha256sum -c` fails the whole `&&` chain if the hash differs,
|
|
3260
|
+
* so extraction only ever runs on verified bytes.
|
|
3261
|
+
*/
|
|
3262
|
+
async function verifyAndExtractRemote(reach, privateKey, sha256, extractDir, deps) {
|
|
3263
|
+
const remoteBase = path.posix.basename(REMOTE_TARBALL_PATH);
|
|
3264
|
+
const res = await sshExec(reach, privateKey, [
|
|
3265
|
+
`cd ${shQuote$2(path.posix.dirname(REMOTE_TARBALL_PATH))}`,
|
|
3266
|
+
`echo ${shQuote$2(`${sha256} ${remoteBase}`)} | sha256sum -c -`,
|
|
3267
|
+
`mkdir -p ${shQuote$2(extractDir)}`,
|
|
3268
|
+
`tar xzf ${shQuote$2(remoteBase)} -C ${shQuote$2(extractDir)}`
|
|
3269
|
+
].join(" && "), {}, deps);
|
|
3270
|
+
if (res.exitCode !== 0) throw new Error(`on-box verify/extract on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3271
|
+
}
|
|
3272
|
+
/** Resolve → verify locally → push → verify+extract on box. */
|
|
3273
|
+
async function stageSshPush(reach, privateKey, opts, extractDir, deps) {
|
|
3274
|
+
if (!deps.payloadSource) throw new Error("ssh-push delivery requires a payload source (KICI_AGENT_BINARY_SOURCE / KICI_AGENT_PAYLOAD_DIR)");
|
|
3275
|
+
const staged = await deps.payloadSource.resolve(opts.platform, opts.version);
|
|
3276
|
+
const sha256 = await verifyLocalPayload(staged.tarballPath, staged.sha256, deps);
|
|
3277
|
+
await sshPushFile(reach, privateKey, staged.tarballPath, REMOTE_TARBALL_PATH, {}, deps);
|
|
3278
|
+
await verifyAndExtractRemote(reach, privateKey, sha256, extractDir, deps);
|
|
3279
|
+
}
|
|
3280
|
+
/**
|
|
3281
|
+
* `s3-direct`: the box pulls the presigned URL itself, verifies the sha256, and
|
|
3282
|
+
* extracts — all in one `sshExec`, so the 50 MB payload never transits the ops
|
|
3283
|
+
* agent. Fail-closed: `sha256sum -c` breaks the `&&` chain on mismatch, so the
|
|
3284
|
+
* box only ever extracts verified bytes, and `curl -f` fails the chain on a
|
|
3285
|
+
* non-2xx (expired/absent presign).
|
|
3286
|
+
*/
|
|
3287
|
+
async function stageS3Direct(reach, privateKey, presignedUrl, sha256, extractDir, deps) {
|
|
3288
|
+
const remoteBase = path.posix.basename(REMOTE_TARBALL_PATH);
|
|
3289
|
+
const res = await sshExec(reach, privateKey, [
|
|
3290
|
+
`cd ${shQuote$2(path.posix.dirname(REMOTE_TARBALL_PATH))}`,
|
|
3291
|
+
`curl -fsSL ${shQuote$2(presignedUrl)} -o ${shQuote$2(remoteBase)}`,
|
|
3292
|
+
`echo ${shQuote$2(`${sha256} ${remoteBase}`)} | sha256sum -c -`,
|
|
3293
|
+
`mkdir -p ${shQuote$2(extractDir)}`,
|
|
3294
|
+
`tar xzf ${shQuote$2(remoteBase)} -C ${shQuote$2(extractDir)}`
|
|
3295
|
+
].join(" && "), {}, deps);
|
|
3296
|
+
if (res.exitCode !== 0) throw new Error(`s3-direct pull/verify/extract on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3297
|
+
}
|
|
3298
|
+
/**
|
|
3299
|
+
* Stage a payload onto `reach` and return the launcher path the caller runs as
|
|
3300
|
+
* the init-runner's `agentCommand`.
|
|
3301
|
+
*/
|
|
3302
|
+
async function stageAgentPayload(reach, privateKey, opts, deps) {
|
|
3303
|
+
const extractDir = deps.extractDir ?? DEFAULT_EXTRACT_DIR;
|
|
3304
|
+
switch (opts.delivery.mode) {
|
|
3305
|
+
case "ssh-push":
|
|
3306
|
+
await stageSshPush(reach, privateKey, opts, extractDir, deps);
|
|
3307
|
+
break;
|
|
3308
|
+
case "s3-direct":
|
|
3309
|
+
await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
|
|
3310
|
+
break;
|
|
3311
|
+
}
|
|
3312
|
+
return { launcherPath: path.posix.join(extractDir, "kici-agent") };
|
|
3313
|
+
}
|
|
3314
|
+
var DEFAULT_EXTRACT_DIR, REMOTE_TARBALL_PATH;
|
|
3315
|
+
var init_stage_agent_payload = __esmMin((() => {
|
|
3316
|
+
init_ssh_exec();
|
|
3317
|
+
DEFAULT_EXTRACT_DIR = "/opt/kici-init";
|
|
3318
|
+
REMOTE_TARBALL_PATH = "/tmp/kici-agent-payload.tar.gz";
|
|
3319
|
+
}));
|
|
3320
|
+
//#endregion
|
|
2786
3321
|
//#region src/bootstrap/ensure-init-runner.ts
|
|
2787
3322
|
/**
|
|
3323
|
+
* Agent-side init-runner bring-up.
|
|
3324
|
+
*
|
|
3325
|
+
* Runs in the AGENT process (never the workflow sandbox), so the bring-up SSH
|
|
3326
|
+
* key and bootstrap token the orchestrator hands back never reach user
|
|
3327
|
+
* workflow code. The flow:
|
|
3328
|
+
*
|
|
3329
|
+
* 1. Call the orchestrator's privileged `kici.ensureInitRunner` handler — it
|
|
3330
|
+
* gates on this agent's `kici:capability:ssh-transport` capability,
|
|
3331
|
+
* resolves the target's reach + SSH key, mints a single-use bootstrap
|
|
3332
|
+
* token, audits, and returns the material (or `{ broughtUp: false }` when
|
|
3333
|
+
* the target already has a live agent).
|
|
3334
|
+
* 2. Over SSH (ephemeral key, never on disk): drop a launcher onto the target
|
|
3335
|
+
* that starts `kici-agent` with the bootstrap env (token + agent id +
|
|
3336
|
+
* orchestrator URL + labels), and start it detached.
|
|
3337
|
+
*
|
|
3338
|
+
* The init-runner then connects → `auth.request` (bootstrap token) →
|
|
3339
|
+
* `agent.register` auto-enroll as a temporary `kici:init` agent.
|
|
3340
|
+
*/
|
|
3341
|
+
/**
|
|
2788
3342
|
* Build the launcher script that starts the init-runner on the target with its
|
|
2789
3343
|
* bootstrap env. Detached (`setsid … &`) so the SSH session can return while
|
|
2790
3344
|
* the agent keeps running and dials the orchestrator.
|
|
@@ -2794,10 +3348,11 @@ function buildLauncher(material, agentCommand) {
|
|
|
2794
3348
|
"#!/usr/bin/env bash",
|
|
2795
3349
|
"set -euo pipefail",
|
|
2796
3350
|
`setsid env ${[
|
|
2797
|
-
`KICI_AGENT_TOKEN=${shQuote(material.bootstrapToken)}`,
|
|
2798
|
-
`KICI_AGENT_ID=${shQuote(material.targetAgentId)}`,
|
|
2799
|
-
`KICI_ORCHESTRATOR_URL=${shQuote(material.orchestratorUrl)}`,
|
|
2800
|
-
`KICI_LABELS=${shQuote(material.labels.join(","))}`,
|
|
3351
|
+
`KICI_AGENT_TOKEN=${shQuote$1(material.bootstrapToken)}`,
|
|
3352
|
+
`KICI_AGENT_ID=${shQuote$1(material.targetAgentId)}`,
|
|
3353
|
+
`KICI_ORCHESTRATOR_URL=${shQuote$1(material.orchestratorUrl)}`,
|
|
3354
|
+
`KICI_LABELS=${shQuote$1(material.labels.join(","))}`,
|
|
3355
|
+
"KICI_ROLES=",
|
|
2801
3356
|
"KICI_EXECUTION_MODE=bare-metal",
|
|
2802
3357
|
"KICI_PORT=0"
|
|
2803
3358
|
].join(" \\\n ")} \\`,
|
|
@@ -2806,10 +3361,60 @@ function buildLauncher(material, agentCommand) {
|
|
|
2806
3361
|
].join("\n");
|
|
2807
3362
|
}
|
|
2808
3363
|
/** Single-quote a value for safe embedding in the launcher's env assignment. */
|
|
2809
|
-
function shQuote(v) {
|
|
3364
|
+
function shQuote$1(v) {
|
|
2810
3365
|
return `'${v.replace(/'/g, `'\\''`)}'`;
|
|
2811
3366
|
}
|
|
2812
3367
|
/**
|
|
3368
|
+
* Resolve the command the launcher invokes to start the init-runner.
|
|
3369
|
+
*
|
|
3370
|
+
* - A `deps.agentCommand` override short-circuits staging (golden image).
|
|
3371
|
+
* - Otherwise a payload source is required: probe the target's platform, stage
|
|
3372
|
+
* the version-keyed self-contained payload over SSH, and return the staged
|
|
3373
|
+
* launcher path (which boots the agent on its vendored Node — no system Node).
|
|
3374
|
+
* - No override AND no source is a misconfiguration, not a silent Node
|
|
3375
|
+
* assumption — throw a clear error.
|
|
3376
|
+
*/
|
|
3377
|
+
async function resolveAgentCommand(transport, reach, privateKey, material, deps) {
|
|
3378
|
+
if (deps.agentCommand) return deps.agentCommand;
|
|
3379
|
+
if (!material.version) throw new Error(`orchestrator returned no version for ${material.targetAgentId} — cannot stage a payload`);
|
|
3380
|
+
if (!(deps.delivery?.mode === AgentDeliveryMode.enum["s3-direct"] || material.deliveryMode === AgentDeliveryMode.enum["s3-direct"]) && !deps.payloadSource) throw new Error(`no agent payload source configured for bringing up ${material.targetAgentId}: set KICI_AGENT_BINARY_SOURCE (object storage), KICI_AGENT_PAYLOAD_DIR (air-gap), or KICI_AGENT_COMMAND (golden image)`);
|
|
3381
|
+
const platform = await probeTargetPlatform(reach, privateKey, deps);
|
|
3382
|
+
const delivery = await resolveDelivery$1(transport, material, platform, deps);
|
|
3383
|
+
const { launcherPath } = await stageAgentPayload(reach, privateKey, {
|
|
3384
|
+
platform,
|
|
3385
|
+
version: material.version,
|
|
3386
|
+
delivery
|
|
3387
|
+
}, {
|
|
3388
|
+
spawnFn: deps.spawnFn,
|
|
3389
|
+
payloadSource: deps.payloadSource,
|
|
3390
|
+
extractDir: deps.extractDir,
|
|
3391
|
+
hashLocalFile: deps.hashLocalFile
|
|
3392
|
+
});
|
|
3393
|
+
return launcherPath;
|
|
3394
|
+
}
|
|
3395
|
+
/**
|
|
3396
|
+
* Decide how the payload reaches the box. An explicit `deps.delivery` override
|
|
3397
|
+
* wins (test/escape hatch); otherwise honor the orchestrator's per-host choice.
|
|
3398
|
+
* For `s3-direct` we ask the orchestrator (which knows the probed platform now)
|
|
3399
|
+
* to mint a box-routable presigned URL via `kici.presignAgentPackage`; a payload
|
|
3400
|
+
* with no sha256 is refused (fail-closed — never extract unverifiable bytes).
|
|
3401
|
+
*/
|
|
3402
|
+
async function resolveDelivery$1(transport, material, platform, deps) {
|
|
3403
|
+
if (deps.delivery) return deps.delivery;
|
|
3404
|
+
if (material.deliveryMode !== AgentDeliveryMode.enum["s3-direct"]) return { mode: "ssh-push" };
|
|
3405
|
+
const presigned = await transport("kici.presignAgentPackage", {
|
|
3406
|
+
targetAgentId: material.targetAgentId,
|
|
3407
|
+
platform
|
|
3408
|
+
});
|
|
3409
|
+
if (!presigned?.url) throw new Error(`orchestrator returned no presigned URL for ${material.targetAgentId}`);
|
|
3410
|
+
if (!presigned.sha256) throw new Error(`refusing s3-direct delivery for ${material.targetAgentId}: no sha256 for the payload (cannot verify on the box)`);
|
|
3411
|
+
return {
|
|
3412
|
+
mode: "s3-direct",
|
|
3413
|
+
presignedUrl: presigned.url,
|
|
3414
|
+
sha256: presigned.sha256
|
|
3415
|
+
};
|
|
3416
|
+
}
|
|
3417
|
+
/**
|
|
2813
3418
|
* Bring up a temporary init-runner on `targetAgentId`. Returns `{ broughtUp }`:
|
|
2814
3419
|
* false when the target already had a live agent (the orchestrator no-op'd),
|
|
2815
3420
|
* true when this call dropped + started the init-runner.
|
|
@@ -2819,7 +3424,7 @@ async function ensureInitRunner(transport, targetAgentId, deps = {}) {
|
|
|
2819
3424
|
if (!material.broughtUp) return { broughtUp: false };
|
|
2820
3425
|
const { reach, privateKey, bootstrapToken, orchestratorUrl, labels } = material;
|
|
2821
3426
|
if (!reach || !privateKey || !bootstrapToken || !orchestratorUrl || !labels) throw new Error(`orchestrator returned incomplete bring-up material for ${targetAgentId}`);
|
|
2822
|
-
const agentCommand =
|
|
3427
|
+
const agentCommand = await resolveAgentCommand(transport, reach, privateKey, material, deps);
|
|
2823
3428
|
await sshPush(reach, privateKey, buildLauncher({
|
|
2824
3429
|
bootstrapToken,
|
|
2825
3430
|
targetAgentId,
|
|
@@ -2830,10 +3435,11 @@ async function ensureInitRunner(transport, targetAgentId, deps = {}) {
|
|
|
2830
3435
|
if (run.exitCode !== 0) throw new Error(`init-runner launch on ${targetAgentId} failed: exit ${run.exitCode}${run.stderr ? `\n${run.stderr}` : ""}`);
|
|
2831
3436
|
return { broughtUp: true };
|
|
2832
3437
|
}
|
|
2833
|
-
var
|
|
3438
|
+
var LAUNCHER_REMOTE_PATH;
|
|
2834
3439
|
var init_ensure_init_runner = __esmMin((() => {
|
|
2835
3440
|
init_ssh_exec();
|
|
2836
|
-
|
|
3441
|
+
init_probe_platform();
|
|
3442
|
+
init_stage_agent_payload();
|
|
2837
3443
|
LAUNCHER_REMOTE_PATH = "/tmp/kici-init-runner.sh";
|
|
2838
3444
|
}));
|
|
2839
3445
|
//#endregion
|
|
@@ -2860,6 +3466,141 @@ var init_pre_boot_send = __esmMin((() => {
|
|
|
2860
3466
|
init_ssh_exec();
|
|
2861
3467
|
}));
|
|
2862
3468
|
//#endregion
|
|
3469
|
+
//#region src/bootstrap/restage-agent.ts
|
|
3470
|
+
/**
|
|
3471
|
+
* External-actor agent re-stage + restart (fleet auto-upgrade apply).
|
|
3472
|
+
*
|
|
3473
|
+
* Runs in the OPS agent (the one holding `kici:capability:ssh-transport`) —
|
|
3474
|
+
* never on the target host itself, so there is NO self-update-handoff: an
|
|
3475
|
+
* external actor swaps the bytes and restarts the target's agent, which
|
|
3476
|
+
* reconnects on its OWN persistent token (this function never mints or handles
|
|
3477
|
+
* a token). The install is folder-anchored (mirroring the versioned-upgrade
|
|
3478
|
+
* layout): the target version is staged into `<installDir>/kici-agent-<version>`
|
|
3479
|
+
* and an atomic symlink swap makes it current, so a stage failure aborts BEFORE
|
|
3480
|
+
* the swap and never leaves a half-upgraded install. Idempotent — a box already
|
|
3481
|
+
* on the target version is a no-op.
|
|
3482
|
+
*/
|
|
3483
|
+
/** Single-quote a value for safe embedding in a remote shell command. */
|
|
3484
|
+
function shQuote(v) {
|
|
3485
|
+
return `'${v.replace(/'/g, `'\\''`)}'`;
|
|
3486
|
+
}
|
|
3487
|
+
/** The versioned payload directory for a version under the install base. */
|
|
3488
|
+
function versionDir(installDir, version) {
|
|
3489
|
+
return path.posix.join(installDir, `kici-agent-${version}`);
|
|
3490
|
+
}
|
|
3491
|
+
/**
|
|
3492
|
+
* Read the version the `kici-agent` symlink currently points at, or null when
|
|
3493
|
+
* no install is present. The symlink target is `kici-agent-<version>`.
|
|
3494
|
+
*/
|
|
3495
|
+
async function readCurrentVersion(reach, privateKey, installDir, deps) {
|
|
3496
|
+
const target = (await sshExec(reach, privateKey, `readlink ${shQuote(path.posix.join(installDir, "kici-agent"))} 2>/dev/null || true`, {}, deps)).stdout.trim();
|
|
3497
|
+
const match = /(?:^|\/)kici-agent-(.+)$/.exec(target);
|
|
3498
|
+
return match ? match[1] : null;
|
|
3499
|
+
}
|
|
3500
|
+
/**
|
|
3501
|
+
* Atomically point `<installDir>/kici-agent` at the target version directory.
|
|
3502
|
+
* `ln -sfn` onto a temp name + `mv -T` is a single rename, so a reader never
|
|
3503
|
+
* sees a missing symlink. The target is stored relative so the tree is movable.
|
|
3504
|
+
*/
|
|
3505
|
+
async function swapInstall(reach, privateKey, installDir, version, deps) {
|
|
3506
|
+
const link = path.posix.join(installDir, "kici-agent");
|
|
3507
|
+
const tmp = `${link}.swap`;
|
|
3508
|
+
const res = await sshExec(reach, privateKey, [`ln -sfn ${shQuote(`kici-agent-${version}`)} ${shQuote(tmp)}`, `mv -T ${shQuote(tmp)} ${shQuote(link)}`].join(" && "), {}, deps);
|
|
3509
|
+
if (res.exitCode !== 0) throw new Error(`install swap on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3510
|
+
}
|
|
3511
|
+
/** Run the drain (stop) then the restart (start) over SSH. */
|
|
3512
|
+
async function drainAndRestart(reach, privateKey, restart, deps) {
|
|
3513
|
+
const stopRes = await sshExec(reach, privateKey, restart.stop, {}, deps);
|
|
3514
|
+
if (stopRes.exitCode !== 0) throw new Error(`agent drain on ${reach.agentId} failed: exit ${stopRes.exitCode}${stopRes.stderr ? `\n${stopRes.stderr}` : ""}`);
|
|
3515
|
+
const startRes = await sshExec(reach, privateKey, restart.start, {}, deps);
|
|
3516
|
+
if (startRes.exitCode !== 0) throw new Error(`agent restart on ${reach.agentId} failed: exit ${startRes.exitCode}${startRes.stderr ? `\n${startRes.stderr}` : ""}`);
|
|
3517
|
+
}
|
|
3518
|
+
/**
|
|
3519
|
+
* Re-stage the target version onto `reach` and restart its agent. Returns
|
|
3520
|
+
* `{ restaged: false }` when the host is already on the target version (no
|
|
3521
|
+
* stage, no swap, no restart), `{ restaged: true }` when this call swapped it.
|
|
3522
|
+
*/
|
|
3523
|
+
async function restageAgent(reach, privateKey, opts, deps) {
|
|
3524
|
+
const installDir = opts.installDir ?? DEFAULT_INSTALL_DIR;
|
|
3525
|
+
if (await readCurrentVersion(reach, privateKey, installDir, deps) === opts.version) return { restaged: false };
|
|
3526
|
+
await stageAgentPayload(reach, privateKey, {
|
|
3527
|
+
platform: opts.platform ?? await probeTargetPlatform(reach, privateKey, deps),
|
|
3528
|
+
version: opts.version,
|
|
3529
|
+
delivery: opts.delivery
|
|
3530
|
+
}, {
|
|
3531
|
+
spawnFn: deps.spawnFn,
|
|
3532
|
+
payloadSource: deps.payloadSource,
|
|
3533
|
+
extractDir: versionDir(installDir, opts.version),
|
|
3534
|
+
hashLocalFile: deps.hashLocalFile
|
|
3535
|
+
});
|
|
3536
|
+
await swapInstall(reach, privateKey, installDir, opts.version, deps);
|
|
3537
|
+
await drainAndRestart(reach, privateKey, opts.restart, deps);
|
|
3538
|
+
return { restaged: true };
|
|
3539
|
+
}
|
|
3540
|
+
var DEFAULT_INSTALL_DIR;
|
|
3541
|
+
var init_restage_agent = __esmMin((() => {
|
|
3542
|
+
init_ssh_exec();
|
|
3543
|
+
init_probe_platform();
|
|
3544
|
+
init_stage_agent_payload();
|
|
3545
|
+
DEFAULT_INSTALL_DIR = "/opt/kici-agent";
|
|
3546
|
+
}));
|
|
3547
|
+
//#endregion
|
|
3548
|
+
//#region src/bootstrap/run-restage.ts
|
|
3549
|
+
/**
|
|
3550
|
+
* Agent-process driver for the `kici.restageAgent` fleet-upgrade apply.
|
|
3551
|
+
*
|
|
3552
|
+
* Intercepted in the ops agent (like `ensureInitRunner`): the privileged
|
|
3553
|
+
* resolve — capability gate, availability gate, reach + SSH key, restart spec —
|
|
3554
|
+
* runs on the orchestrator; this driver performs the SSH transport (probe →
|
|
3555
|
+
* stage → swap → restart) so the bring-up key never reaches user workflow code.
|
|
3556
|
+
* The re-staged permanent agent reconnects on its own persistent credential, so
|
|
3557
|
+
* no token is minted or handled here (no self-update-handoff).
|
|
3558
|
+
*/
|
|
3559
|
+
/**
|
|
3560
|
+
* Resolve the delivery mode for the re-stage. `ssh-push` needs a local payload
|
|
3561
|
+
* source; `s3-direct` asks the orchestrator to mint a box-routable presigned URL
|
|
3562
|
+
* (keyed by the now-probed platform) and refuses a payload with no sha256
|
|
3563
|
+
* (fail-closed — never extract unverifiable bytes on the box).
|
|
3564
|
+
*/
|
|
3565
|
+
async function resolveDelivery(transport, targetAgentId, deliveryMode, platform) {
|
|
3566
|
+
if (deliveryMode !== AgentDeliveryMode.enum["s3-direct"]) return { mode: "ssh-push" };
|
|
3567
|
+
const presigned = await transport("kici.presignAgentPackage", {
|
|
3568
|
+
targetAgentId,
|
|
3569
|
+
platform
|
|
3570
|
+
});
|
|
3571
|
+
if (!presigned?.url) throw new Error(`orchestrator returned no presigned URL for re-staging ${targetAgentId}`);
|
|
3572
|
+
if (!presigned.sha256) throw new Error(`refusing s3-direct re-stage for ${targetAgentId}: no sha256 for the payload (cannot verify on the box)`);
|
|
3573
|
+
return {
|
|
3574
|
+
mode: "s3-direct",
|
|
3575
|
+
presignedUrl: presigned.url,
|
|
3576
|
+
sha256: presigned.sha256
|
|
3577
|
+
};
|
|
3578
|
+
}
|
|
3579
|
+
/**
|
|
3580
|
+
* Drive one external-actor re-stage: fetch the material, probe the target
|
|
3581
|
+
* platform, resolve delivery, then stage + swap + restart via {@link restageAgent}.
|
|
3582
|
+
* Returns `{ restaged: false }` when the host is already on the target version.
|
|
3583
|
+
*/
|
|
3584
|
+
async function runRestage(transport, targetAgentId, deps) {
|
|
3585
|
+
const { reach, privateKey, version, deliveryMode, restart } = await transport("kici.restageAgent", { targetAgentId });
|
|
3586
|
+
if (!reach || !privateKey || !version || !restart) throw new Error(`orchestrator returned incomplete re-stage material for ${targetAgentId}`);
|
|
3587
|
+
const platform = await probeTargetPlatform(reach, privateKey, deps);
|
|
3588
|
+
return restageAgent(reach, privateKey, {
|
|
3589
|
+
platform,
|
|
3590
|
+
version,
|
|
3591
|
+
delivery: await resolveDelivery(transport, targetAgentId, deliveryMode, platform),
|
|
3592
|
+
...restart.installDir ? { installDir: restart.installDir } : {},
|
|
3593
|
+
restart: {
|
|
3594
|
+
stop: restart.stop,
|
|
3595
|
+
start: restart.start
|
|
3596
|
+
}
|
|
3597
|
+
}, deps);
|
|
3598
|
+
}
|
|
3599
|
+
var init_run_restage = __esmMin((() => {
|
|
3600
|
+
init_probe_platform();
|
|
3601
|
+
init_restage_agent();
|
|
3602
|
+
}));
|
|
3603
|
+
//#endregion
|
|
2863
3604
|
//#region src/bootstrap/api-intercept.ts
|
|
2864
3605
|
/**
|
|
2865
3606
|
* Wrap the orchestrator API transport so the two bootstrap methods are handled
|
|
@@ -2878,17 +3619,187 @@ function withBootstrapInterception(relay, deps = {}) {
|
|
|
2878
3619
|
}, deps);
|
|
2879
3620
|
return;
|
|
2880
3621
|
}
|
|
3622
|
+
if (method === RESTAGE_AGENT) return runRestage(relay, String(params.targetAgentId ?? ""), deps);
|
|
2881
3623
|
return relay(method, params);
|
|
2882
3624
|
};
|
|
2883
3625
|
}
|
|
2884
|
-
var ENSURE_INIT_RUNNER, PRE_BOOT_SEND;
|
|
3626
|
+
var ENSURE_INIT_RUNNER, PRE_BOOT_SEND, RESTAGE_AGENT;
|
|
2885
3627
|
var init_api_intercept = __esmMin((() => {
|
|
2886
3628
|
init_ensure_init_runner();
|
|
2887
3629
|
init_pre_boot_send();
|
|
3630
|
+
init_run_restage();
|
|
2888
3631
|
ENSURE_INIT_RUNNER = "kici.ensureInitRunner";
|
|
2889
3632
|
PRE_BOOT_SEND = "kici.preBootSend";
|
|
3633
|
+
RESTAGE_AGENT = "kici.restageAgent";
|
|
2890
3634
|
}));
|
|
2891
3635
|
//#endregion
|
|
3636
|
+
//#region src/bootstrap/payload-source.ts
|
|
3637
|
+
/**
|
|
3638
|
+
* Agent payload source abstraction.
|
|
3639
|
+
*
|
|
3640
|
+
* A one-method plug so the place a self-contained agent+Node payload comes from
|
|
3641
|
+
* is swappable and overridable: the local-dir fallback here reads the tarballs
|
|
3642
|
+
* a `kici-admin agent package` producer wrote to disk; an S3-backed source
|
|
3643
|
+
* (fetching from the orchestrator cache bucket via presigned URL) plugs in the
|
|
3644
|
+
* same interface. Payloads are version-keyed and fail-fast — a missing version
|
|
3645
|
+
* throws loudly rather than silently staging stale bytes.
|
|
3646
|
+
*/
|
|
3647
|
+
/** The payload tarball name for a platform (matches the producer's layout). */
|
|
3648
|
+
function tarballName$1(platform) {
|
|
3649
|
+
return `kici-agent-${platform}.tar.gz`;
|
|
3650
|
+
}
|
|
3651
|
+
/** Extract the leading hex token from a `sha256sum`-style `<hex> <name>` line. */
|
|
3652
|
+
function parseSidecarHash(contents) {
|
|
3653
|
+
const first = contents.trim().split(/\s+/)[0];
|
|
3654
|
+
if (!first) throw new Error("empty .sha256 sidecar");
|
|
3655
|
+
return first;
|
|
3656
|
+
}
|
|
3657
|
+
var defaultPayloadFs, LocalDirPayloadSource;
|
|
3658
|
+
var init_payload_source = __esmMin((() => {
|
|
3659
|
+
defaultPayloadFs = {
|
|
3660
|
+
exists: async (filePath) => {
|
|
3661
|
+
try {
|
|
3662
|
+
await access(filePath);
|
|
3663
|
+
return true;
|
|
3664
|
+
} catch {
|
|
3665
|
+
return false;
|
|
3666
|
+
}
|
|
3667
|
+
},
|
|
3668
|
+
readFile: (filePath) => readFile(filePath, "utf8")
|
|
3669
|
+
};
|
|
3670
|
+
LocalDirPayloadSource = class {
|
|
3671
|
+
baseDir;
|
|
3672
|
+
fs;
|
|
3673
|
+
constructor(baseDir, fs = defaultPayloadFs) {
|
|
3674
|
+
this.baseDir = baseDir;
|
|
3675
|
+
this.fs = fs;
|
|
3676
|
+
}
|
|
3677
|
+
async resolve(platform, version) {
|
|
3678
|
+
const tarballPath = path.join(this.baseDir, version, tarballName$1(platform));
|
|
3679
|
+
if (!await this.fs.exists(tarballPath)) throw new Error(`no agent payload for version ${version} (${platform}) at ${tarballPath} — run \`kici-admin agent package --platform ${platform} --out ${this.baseDir}\``);
|
|
3680
|
+
const sidecarPath = `${tarballPath}.sha256`;
|
|
3681
|
+
return {
|
|
3682
|
+
tarballPath,
|
|
3683
|
+
sha256: await this.fs.exists(sidecarPath) ? parseSidecarHash(await this.fs.readFile(sidecarPath)) : null
|
|
3684
|
+
};
|
|
3685
|
+
}
|
|
3686
|
+
};
|
|
3687
|
+
}));
|
|
3688
|
+
//#endregion
|
|
3689
|
+
//#region src/bootstrap/s3-payload-source.ts
|
|
3690
|
+
/**
|
|
3691
|
+
* Object-storage agent payload source — the ops agent's PRIMARY source.
|
|
3692
|
+
*
|
|
3693
|
+
* Resolves a version-keyed self-contained agent+Node payload from the
|
|
3694
|
+
* orchestrator's own cache bucket via a presigned GET URL the orchestrator
|
|
3695
|
+
* mints (`kici.presignAgentPackage`), fetches it once into a local cache dir,
|
|
3696
|
+
* and returns it for the SSH-push delivery path. No standing S3 credential ever
|
|
3697
|
+
* reaches the ops agent — only a time-limited presigned URL. Payloads are
|
|
3698
|
+
* version-keyed and fail-fast: a missing object throws loudly (names the
|
|
3699
|
+
* version + platform) rather than staging stale bytes.
|
|
3700
|
+
*
|
|
3701
|
+
* For the `s3-direct` delivery path the box pulls the presigned URL itself, so
|
|
3702
|
+
* this source is not involved there; it backs `ssh-push` (box cannot reach
|
|
3703
|
+
* object storage). `LocalDirPayloadSource` remains the air-gap fallback.
|
|
3704
|
+
*/
|
|
3705
|
+
/** The payload tarball name for a platform (matches the producer's layout). */
|
|
3706
|
+
function tarballName(platform) {
|
|
3707
|
+
return `kici-agent-${platform}.tar.gz`;
|
|
3708
|
+
}
|
|
3709
|
+
/** Default cache root the ops agent stores pulled payloads under (cache-once). */
|
|
3710
|
+
function defaultPayloadCacheDir() {
|
|
3711
|
+
return path.join(kiciTmpBase(), "kici-agent-payloads");
|
|
3712
|
+
}
|
|
3713
|
+
/** Default existence check for the cache-once short-circuit. */
|
|
3714
|
+
async function payloadFileExists(filePath) {
|
|
3715
|
+
try {
|
|
3716
|
+
await access(filePath);
|
|
3717
|
+
return true;
|
|
3718
|
+
} catch {
|
|
3719
|
+
return false;
|
|
3720
|
+
}
|
|
3721
|
+
}
|
|
3722
|
+
/** Stream a (presigned) URL into a local file, binary-safe (no in-memory buffer). */
|
|
3723
|
+
async function fetchUrlToFile(url, dest) {
|
|
3724
|
+
await mkdir(path.dirname(dest), { recursive: true });
|
|
3725
|
+
const res = await fetch(url);
|
|
3726
|
+
if (!res.ok || !res.body) throw new Error(`fetch payload failed: HTTP ${res.status} ${res.statusText}`);
|
|
3727
|
+
await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
|
|
3728
|
+
}
|
|
3729
|
+
var S3PayloadSource;
|
|
3730
|
+
var init_s3_payload_source = __esmMin((() => {
|
|
3731
|
+
S3PayloadSource = class {
|
|
3732
|
+
deps;
|
|
3733
|
+
constructor(deps) {
|
|
3734
|
+
this.deps = deps;
|
|
3735
|
+
}
|
|
3736
|
+
async resolve(platform, version) {
|
|
3737
|
+
const presigned = await this.deps.presign(platform, version);
|
|
3738
|
+
if (!presigned) throw new Error(`no agent payload for version ${version} (${platform}) in the orchestrator cache bucket — run \`kici-admin agent package --platform ${platform} --upload\` to publish it`);
|
|
3739
|
+
const tarballPath = path.join(this.deps.cacheDir, version, tarballName(platform));
|
|
3740
|
+
if (!(this.deps.exists ? await this.deps.exists(tarballPath) : false)) await this.deps.fetchToFile(presigned.url, tarballPath);
|
|
3741
|
+
return {
|
|
3742
|
+
tarballPath,
|
|
3743
|
+
sha256: presigned.sha256
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
};
|
|
3747
|
+
}));
|
|
3748
|
+
//#endregion
|
|
3749
|
+
//#region src/execution/streaming-zx-log.ts
|
|
3750
|
+
/**
|
|
3751
|
+
* Shared factory for the zx `log` callback that streams a subprocess's
|
|
3752
|
+
* stdout/stderr into the captured/streamed run log, line by line, via `emit`.
|
|
3753
|
+
*
|
|
3754
|
+
* zx does NOT write child stdout/stderr to `process.stdout`; it pipes the
|
|
3755
|
+
* child stdio to an internal VoidStream and surfaces each chunk through the
|
|
3756
|
+
* shell's `log` callback as `{ kind: 'stdout' | 'stderr', data, verbose }`.
|
|
3757
|
+
* The `verbose` flag is zx's per-invocation quiet/verbose decision:
|
|
3758
|
+
*
|
|
3759
|
+
* - stdout entries: `verbose = !piped && (snapshot.verbose && !snapshot.quiet)`
|
|
3760
|
+
* - stderr entries: `verbose = !snapshot.quiet`
|
|
3761
|
+
*
|
|
3762
|
+
* so a step that opts into `$({ quiet: true })` (e.g. a `sops -d` decrypt of a
|
|
3763
|
+
* credential) produces `verbose: false` entries. This factory HONORS that flag
|
|
3764
|
+
* — it skips `verbose: false` entries — which is what makes `{ quiet: true }`
|
|
3765
|
+
* actually suppress sensitive output from the run log. Without the gate, a
|
|
3766
|
+
* decrypted-secret line leaks into the persisted/streamed log (zx's own default
|
|
3767
|
+
* log function gates on the same flag: `if (!entry.verbose) return`).
|
|
3768
|
+
*
|
|
3769
|
+
* IMPORTANT: the shell that installs this callback MUST be constructed with
|
|
3770
|
+
* `verbose: true`. With `verbose: true` zx flags ordinary (non-quiet)
|
|
3771
|
+
* subprocess output `verbose: true` (captured) and a `{ quiet: true }` call
|
|
3772
|
+
* `verbose: false` (suppressed). A `verbose: false` base would flag ordinary
|
|
3773
|
+
* output `verbose: false` too, and this gate would then drop every line.
|
|
3774
|
+
*
|
|
3775
|
+
* The returned callback owns one line buffer PER STREAM, so partial chunks are
|
|
3776
|
+
* coalesced into whole lines before `emit` is called. The buffers are separate
|
|
3777
|
+
* because the two streams are independent pipes: a stdout chunk ending mid-line
|
|
3778
|
+
* and a stderr chunk arriving next would otherwise concatenate into a single
|
|
3779
|
+
* spliced line attributed to whichever kind completed it.
|
|
3780
|
+
*
|
|
3781
|
+
* `emit` receives the originating stream alongside the line so a diagnostic
|
|
3782
|
+
* written to stderr stays distinguishable from ordinary progress output all the
|
|
3783
|
+
* way to the persisted run log.
|
|
3784
|
+
*/
|
|
3785
|
+
function makeStreamingZxLog(emit) {
|
|
3786
|
+
const lineBufs = {
|
|
3787
|
+
[LogStream.enum.stdout]: "",
|
|
3788
|
+
[LogStream.enum.stderr]: ""
|
|
3789
|
+
};
|
|
3790
|
+
return (entry) => {
|
|
3791
|
+
const e = entry;
|
|
3792
|
+
if (e.kind !== "stdout" && e.kind !== "stderr") return;
|
|
3793
|
+
if (!e.verbose) return;
|
|
3794
|
+
const stream = e.kind === "stderr" ? LogStream.enum.stderr : LogStream.enum.stdout;
|
|
3795
|
+
const text = typeof e.data === "string" ? e.data : String(e.data ?? "");
|
|
3796
|
+
const lines = (lineBufs[stream] + text).split("\n");
|
|
3797
|
+
lineBufs[stream] = lines.pop();
|
|
3798
|
+
for (const line of lines) if (line) emit(line, stream);
|
|
3799
|
+
};
|
|
3800
|
+
}
|
|
3801
|
+
var init_streaming_zx_log = __esmMin((() => {}));
|
|
3802
|
+
//#endregion
|
|
2892
3803
|
//#region src/execution/dynamic-job-serializer.ts
|
|
2893
3804
|
/**
|
|
2894
3805
|
* Convert an array of SDK Job objects into LockJob format for the orchestrator.
|
|
@@ -2912,8 +3823,8 @@ async function serializeJobsToLock(jobs, ctx, staticNames, allowedGroups) {
|
|
|
2912
3823
|
}
|
|
2913
3824
|
async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups) {
|
|
2914
3825
|
const { include: runsOn, exclude: excludeLabels } = normalizeRunsOnToMatchers(job.runsOn, `generated job '${job.name}' runsOn`);
|
|
2915
|
-
const envRefs = job.
|
|
2916
|
-
let
|
|
3826
|
+
const envRefs = job.contexts ?? (job.context !== void 0 ? [job.context] : void 0);
|
|
3827
|
+
let resolvedContexts;
|
|
2917
3828
|
if (envRefs !== void 0 && envRefs.length > 0) {
|
|
2918
3829
|
const resolved = [];
|
|
2919
3830
|
for (const ref of envRefs) if (typeof ref === "function") {
|
|
@@ -2926,7 +3837,7 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
|
|
|
2926
3837
|
value: ref,
|
|
2927
3838
|
dynamic: false
|
|
2928
3839
|
});
|
|
2929
|
-
if (resolved.length > 0)
|
|
3840
|
+
if (resolved.length > 0) resolvedContexts = resolved;
|
|
2930
3841
|
}
|
|
2931
3842
|
let resolvedEnv;
|
|
2932
3843
|
if (typeof job.env === "function") {
|
|
@@ -2953,7 +3864,7 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
|
|
|
2953
3864
|
...job.include ? { include: job.include } : {},
|
|
2954
3865
|
...job.exclude ? { exclude: job.exclude } : {},
|
|
2955
3866
|
...job.description ? { description: job.description } : {},
|
|
2956
|
-
...
|
|
3867
|
+
...resolvedContexts !== void 0 ? { contexts: resolvedContexts } : {},
|
|
2957
3868
|
...resolvedEnv !== void 0 ? { env: resolvedEnv } : {},
|
|
2958
3869
|
...resolvedConcurrencyGroup !== void 0 ? { concurrencyGroup: resolvedConcurrencyGroup } : {}
|
|
2959
3870
|
};
|
|
@@ -3113,6 +4024,8 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3113
4024
|
flushTimer = null;
|
|
3114
4025
|
totalBytes = 0;
|
|
3115
4026
|
truncated = false;
|
|
4027
|
+
/** Which stream the currently-buffered lines came from. */
|
|
4028
|
+
bufferStream = LogStream.enum.stdout;
|
|
3116
4029
|
/** Number of lines dropped due to backpressure (drop mode). */
|
|
3117
4030
|
droppedCount = 0;
|
|
3118
4031
|
/** Whether we are currently in a backpressured state (pause mode). */
|
|
@@ -3152,9 +4065,24 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3152
4065
|
/**
|
|
3153
4066
|
* Add a line to the buffer. Triggers flush if threshold reached,
|
|
3154
4067
|
* otherwise schedules a timer-based flush.
|
|
4068
|
+
*
|
|
4069
|
+
* A chunk carries a single stream, so a kind flip closes the pending chunk
|
|
4070
|
+
* before the new line is buffered. Chunks are delivered in order, so the
|
|
4071
|
+
* stdout/stderr interleaving is preserved across that boundary.
|
|
4072
|
+
*
|
|
4073
|
+
* `bufferStream` is only advanced once the buffer is actually empty. Under
|
|
4074
|
+
* pause-mode backpressure `flush()` deliberately leaves the buffer intact
|
|
4075
|
+
* until the socket drains, so the flip cannot close the pending chunk;
|
|
4076
|
+
* advancing the tag there would relabel the already-buffered lines as the
|
|
4077
|
+
* newly-arrived stream. The buffer keeps the tag of its first lines instead,
|
|
4078
|
+
* which means a chunk assembled while backpressured reports one stream for
|
|
4079
|
+
* lines that came from both — stderr in such a chunk is under-reported as
|
|
4080
|
+
* stdout. Separating them needs a queue of pending per-stream chunks.
|
|
3155
4081
|
*/
|
|
3156
|
-
addLine(line) {
|
|
4082
|
+
addLine(line, stream = LogStream.enum.stdout) {
|
|
3157
4083
|
if (this.truncated) return;
|
|
4084
|
+
if (this.buffer.length > 0 && stream !== this.bufferStream) this.flush();
|
|
4085
|
+
if (this.buffer.length === 0) this.bufferStream = stream;
|
|
3158
4086
|
if (this.totalBytes >= this.maxLogSizeBytes) {
|
|
3159
4087
|
this.buffer.push(`[TRUNCATED: log output exceeded ${this.maxLogSizeBytes} bytes]`);
|
|
3160
4088
|
this.truncated = true;
|
|
@@ -3209,7 +4137,8 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3209
4137
|
jobId: this.jobId,
|
|
3210
4138
|
stepIndex: this.stepIndex,
|
|
3211
4139
|
lines,
|
|
3212
|
-
timestamp: Date.now()
|
|
4140
|
+
timestamp: Date.now(),
|
|
4141
|
+
stream: this.bufferStream
|
|
3213
4142
|
});
|
|
3214
4143
|
}
|
|
3215
4144
|
/**
|
|
@@ -3271,7 +4200,8 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3271
4200
|
jobId: this.jobId,
|
|
3272
4201
|
stepIndex: this.stepIndex,
|
|
3273
4202
|
lines,
|
|
3274
|
-
timestamp: Date.now()
|
|
4203
|
+
timestamp: Date.now(),
|
|
4204
|
+
stream: this.bufferStream
|
|
3275
4205
|
});
|
|
3276
4206
|
}
|
|
3277
4207
|
/**
|
|
@@ -3366,7 +4296,7 @@ function decryptBuffer(encrypted, aesKey) {
|
|
|
3366
4296
|
*/
|
|
3367
4297
|
async function applyOverlay(config) {
|
|
3368
4298
|
const { tarballUrl, cliPublicKey, orchestratorPrivateKey, repoDir } = config;
|
|
3369
|
-
const tmpDir = await
|
|
4299
|
+
const { path: tmpDir, cleanup } = await makeTempDir("overlay");
|
|
3370
4300
|
try {
|
|
3371
4301
|
logger$7.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
|
|
3372
4302
|
let encryptedData;
|
|
@@ -3440,10 +4370,7 @@ async function applyOverlay(config) {
|
|
|
3440
4370
|
verified: true
|
|
3441
4371
|
};
|
|
3442
4372
|
} finally {
|
|
3443
|
-
await
|
|
3444
|
-
recursive: true,
|
|
3445
|
-
force: true
|
|
3446
|
-
}).catch(() => {});
|
|
4373
|
+
await cleanup().catch(() => {});
|
|
3447
4374
|
}
|
|
3448
4375
|
}
|
|
3449
4376
|
var logger$7, IV_LENGTH$1, AUTH_TAG_LENGTH;
|
|
@@ -3626,7 +4553,7 @@ async function applyYarnrcBerryConfig(args) {
|
|
|
3626
4553
|
const hasPrivateRegistry = registries.length > 0 || Object.keys(installEnvSecrets).length > 0;
|
|
3627
4554
|
const yarnrcPath = join(args.kiciDir, ".yarnrc.yml");
|
|
3628
4555
|
const { raw: original, doc } = await readOriginalYarnrc(yarnrcPath);
|
|
3629
|
-
const cacheFolder = await
|
|
4556
|
+
const { path: cacheFolder, cleanup: cleanupCache } = await makeTempDir("yarn-berry-cache");
|
|
3630
4557
|
const merged = {
|
|
3631
4558
|
...doc,
|
|
3632
4559
|
nodeLinker: "node-modules",
|
|
@@ -3663,10 +4590,7 @@ async function applyYarnrcBerryConfig(args) {
|
|
|
3663
4590
|
if (original === null) await unlink(yarnrcPath).catch(() => {});
|
|
3664
4591
|
else await writeFile(yarnrcPath, original, { encoding: "utf8" });
|
|
3665
4592
|
} catch {}
|
|
3666
|
-
await
|
|
3667
|
-
recursive: true,
|
|
3668
|
-
force: true
|
|
3669
|
-
}).catch(() => {});
|
|
4593
|
+
await cleanupCache().catch(() => {});
|
|
3670
4594
|
};
|
|
3671
4595
|
return {
|
|
3672
4596
|
extraEnv: {
|
|
@@ -4009,9 +4933,9 @@ async function detectKiciYarnFlavor(repoRoot, kiciDir) {
|
|
|
4009
4933
|
* Install `.kici/` dependencies inline with the repo's package manager.
|
|
4010
4934
|
*
|
|
4011
4935
|
* Falls back to this when the dep cache is unavailable or a download fails.
|
|
4012
|
-
* The install runs with an isolated cache/store directory (
|
|
4013
|
-
* `
|
|
4014
|
-
* is removed after installation.
|
|
4936
|
+
* The install runs with an isolated cache/store directory (allocated under the
|
|
4937
|
+
* global temp base, which honors `KICI_TMPDIR`) to prevent cache poisoning
|
|
4938
|
+
* between build jobs; the directory is removed after installation.
|
|
4015
4939
|
*
|
|
4016
4940
|
* If `opts.npmRegistries` / `opts.installEnvSecrets` is provided, a job-scoped
|
|
4017
4941
|
* `.kici/.npmrc` overlay is synthesized for the install, restored in `finally`,
|
|
@@ -4100,7 +5024,7 @@ function envWithNodeOnPath(extraEnv, nodeDir) {
|
|
|
4100
5024
|
/** Run `npm install` in `.kici/` with an isolated cache directory. */
|
|
4101
5025
|
async function runNpmInstall(args) {
|
|
4102
5026
|
const { npmCliPath, nodeExe, nodeDir } = resolveNpm();
|
|
4103
|
-
const cacheDir = await
|
|
5027
|
+
const { path: cacheDir, cleanup } = await makeTempDir("npm-cache");
|
|
4104
5028
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
4105
5029
|
const buildArgs = (...prefix) => {
|
|
4106
5030
|
const a = [
|
|
@@ -4125,10 +5049,7 @@ async function runNpmInstall(args) {
|
|
|
4125
5049
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
4126
5050
|
});
|
|
4127
5051
|
} finally {
|
|
4128
|
-
await
|
|
4129
|
-
recursive: true,
|
|
4130
|
-
force: true
|
|
4131
|
-
}).catch(() => {});
|
|
5052
|
+
await cleanup().catch(() => {});
|
|
4132
5053
|
}
|
|
4133
5054
|
}
|
|
4134
5055
|
/**
|
|
@@ -4142,7 +5063,7 @@ async function runNpmInstall(args) {
|
|
|
4142
5063
|
async function runPnpmInstall(args) {
|
|
4143
5064
|
await assertPnpmAvailable();
|
|
4144
5065
|
const { nodeDir } = resolveNpm();
|
|
4145
|
-
const storeDir = await
|
|
5066
|
+
const { path: storeDir, cleanup } = await makeTempDir("pnpm-store");
|
|
4146
5067
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
4147
5068
|
const argv = [
|
|
4148
5069
|
"install",
|
|
@@ -4162,10 +5083,7 @@ async function runPnpmInstall(args) {
|
|
|
4162
5083
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
4163
5084
|
});
|
|
4164
5085
|
} finally {
|
|
4165
|
-
await
|
|
4166
|
-
recursive: true,
|
|
4167
|
-
force: true
|
|
4168
|
-
}).catch(() => {});
|
|
5086
|
+
await cleanup().catch(() => {});
|
|
4169
5087
|
}
|
|
4170
5088
|
}
|
|
4171
5089
|
/** Pure: argv for `yarn install` with an isolated cache folder. */
|
|
@@ -4191,7 +5109,7 @@ function buildYarnInstallArgs(cacheDir, hasPrivateRegistry) {
|
|
|
4191
5109
|
async function runYarnInstall(args) {
|
|
4192
5110
|
await assertYarnAvailable();
|
|
4193
5111
|
const { nodeDir } = resolveNpm();
|
|
4194
|
-
const cacheDir = await
|
|
5112
|
+
const { path: cacheDir, cleanup } = await makeTempDir("yarn-cache");
|
|
4195
5113
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
4196
5114
|
const argv = buildYarnInstallArgs(cacheDir, args.hasPrivateRegistry);
|
|
4197
5115
|
try {
|
|
@@ -4203,10 +5121,7 @@ async function runYarnInstall(args) {
|
|
|
4203
5121
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
4204
5122
|
});
|
|
4205
5123
|
} finally {
|
|
4206
|
-
await
|
|
4207
|
-
recursive: true,
|
|
4208
|
-
force: true
|
|
4209
|
-
}).catch(() => {});
|
|
5124
|
+
await cleanup().catch(() => {});
|
|
4210
5125
|
}
|
|
4211
5126
|
}
|
|
4212
5127
|
/** Pure: argv for a berry `yarn install`. Cache + linker live in .yarnrc.yml. */
|
|
@@ -4471,8 +5386,8 @@ var init_dep_packer = __esmMin((() => {
|
|
|
4471
5386
|
* 1. Allowed system vars from process.env
|
|
4472
5387
|
* 2. Sandbox defaults (FORCE_COLOR=1, etc.)
|
|
4473
5388
|
* 3. KICI_* system vars (orchestrator-generated, from userEnv)
|
|
4474
|
-
* 4. Org-level environment vars (from orchestrator via
|
|
4475
|
-
* 5. Source-level environment overrides (merged into
|
|
5389
|
+
* 4. Org-level environment vars (from orchestrator via contextVars)
|
|
5390
|
+
* 5. Source-level environment overrides (merged into contextVars by orchestrator)
|
|
4476
5391
|
* 6. Job env (from lock file env field, evaluated by orchestrator)
|
|
4477
5392
|
* 7. setEnv() calls (runtime -- applied at step execution, not here)
|
|
4478
5393
|
*/
|
|
@@ -4494,19 +5409,28 @@ var init_dep_packer = __esmMin((() => {
|
|
|
4494
5409
|
* PLATFORM_TOKEN, WEBHOOK_SECRET, etc.) are never included because they
|
|
4495
5410
|
* are not in the allowlist.
|
|
4496
5411
|
*
|
|
5412
|
+
* The `trustedEnv` option (default false) selects the trusted fleet-agent
|
|
5413
|
+
* profile: Layer 1 passes the ambient host env through instead of restricting
|
|
5414
|
+
* to `ALLOWED_SYSTEM_VARS`, minus the agent's own KiCI identity/operational
|
|
5415
|
+
* secrets (the whole `KICI_*` namespace + a small non-`KICI_` infra denylist).
|
|
5416
|
+
* It is read ONLY from agent config (`KICI_TRUSTED_ENV`), never from a dispatch
|
|
5417
|
+
* payload, so a workflow cannot request it. With it OFF the output is
|
|
5418
|
+
* byte-identical to the allowlist-only behavior.
|
|
5419
|
+
*
|
|
4497
5420
|
* @param userEnv - Environment variables from workflow config and orchestrator
|
|
4498
5421
|
* @param options - Optional extended options for environment layers
|
|
4499
5422
|
* @returns A new Record with only safe environment variables
|
|
4500
5423
|
*/
|
|
4501
5424
|
function buildSanitizedEnv(userEnv, options) {
|
|
4502
5425
|
const sanitized = {};
|
|
4503
|
-
|
|
5426
|
+
if (options?.trustedEnv) Object.assign(sanitized, buildTrustedPassthroughEnv(process.env));
|
|
5427
|
+
else for (const key of ALLOWED_SYSTEM_VARS) {
|
|
4504
5428
|
const value = process.env[key];
|
|
4505
5429
|
if (value !== void 0) sanitized[key] = value;
|
|
4506
5430
|
}
|
|
4507
5431
|
Object.assign(sanitized, SANDBOX_DEFAULT_VARS);
|
|
4508
5432
|
Object.assign(sanitized, userEnv);
|
|
4509
|
-
if (options?.
|
|
5433
|
+
if (options?.contextVars) Object.assign(sanitized, options.contextVars);
|
|
4510
5434
|
if (options?.jobEnv) Object.assign(sanitized, options.jobEnv);
|
|
4511
5435
|
return sanitized;
|
|
4512
5436
|
}
|
|
@@ -4639,8 +5563,8 @@ function buildRequest(dispatch, workDir) {
|
|
|
4639
5563
|
cliPublicKey: jobConfig.cliPublicKey,
|
|
4640
5564
|
orchestratorPrivateKey: jobConfig.orchestratorPrivateKey,
|
|
4641
5565
|
runPublicKey: dispatch.runPublicKey,
|
|
4642
|
-
|
|
4643
|
-
|
|
5566
|
+
context: jobConfig.context,
|
|
5567
|
+
contextVars: jobConfig.contextVars,
|
|
4644
5568
|
jobEnv: jobConfig.jobEnv,
|
|
4645
5569
|
isGlobalWorkflow: jobConfig.isGlobalWorkflow,
|
|
4646
5570
|
workflowRepoUrl: jobConfig.workflowRepoUrl,
|
|
@@ -4649,6 +5573,7 @@ function buildRequest(dispatch, workDir) {
|
|
|
4649
5573
|
workflowRepoIdentifier: jobConfig.workflowRepoIdentifier,
|
|
4650
5574
|
hasConcurrencyGroup: jobConfig.hasConcurrencyGroup ?? false,
|
|
4651
5575
|
concurrencyEvaluationTimeoutMs: jobConfig.concurrencyEvaluationTimeoutMs,
|
|
5576
|
+
concurrencyWaitTimeoutMs: dispatch.concurrencyWaitTimeoutMs,
|
|
4652
5577
|
branch: dispatch.ref,
|
|
4653
5578
|
upstreamJobOutputs: dispatch.upstreamJobOutputs,
|
|
4654
5579
|
upstreamJobStatuses: dispatch.upstreamJobStatuses,
|
|
@@ -4716,6 +5641,7 @@ function buildBwrapArgs(workDir, nodeExecPath, networkIsolation = false, runnerP
|
|
|
4716
5641
|
const libIdx = args.indexOf("/lib", args.indexOf("--ro-bind") + 1);
|
|
4717
5642
|
if (libIdx !== -1) args.splice(libIdx + 2, 0, "--ro-bind", "/lib64", "/lib64");
|
|
4718
5643
|
}
|
|
5644
|
+
for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) args.push("--ro-bind", nssFile, nssFile);
|
|
4719
5645
|
const nodeDir = dirname(nodeExecPath);
|
|
4720
5646
|
const nodeInstallRoot = dirname(nodeDir);
|
|
4721
5647
|
if (!nodeInstallRoot.startsWith("/usr") && !nodeInstallRoot.startsWith("/bin") && nodeInstallRoot !== "/" && nodeInstallRoot !== "") args.push("--ro-bind", nodeInstallRoot, nodeInstallRoot);
|
|
@@ -4760,6 +5686,28 @@ function buildBwrapArgs(workDir, nodeExecPath, networkIsolation = false, runnerP
|
|
|
4760
5686
|
return args;
|
|
4761
5687
|
}
|
|
4762
5688
|
/**
|
|
5689
|
+
* Derive the read-only bind path(s) for a `file://` clone source. The workflow
|
|
5690
|
+
* runner clones the repo from inside the sandbox, so a local `file://` URL's
|
|
5691
|
+
* source directory must be exposed read-only or `git clone` fails with
|
|
5692
|
+
* `does not appear to be a git repository`. Returns `[]` for non-`file://`
|
|
5693
|
+
* remotes (https/ssh need no host bind) and for a malformed `file://` URL (the
|
|
5694
|
+
* clone then surfaces the real error rather than being masked here).
|
|
5695
|
+
*
|
|
5696
|
+
* Shared by both sandboxes that clone a local source: the bare-metal (bwrap)
|
|
5697
|
+
* backend threads the result into `buildBwrapArgs`'s `extraReadOnlyBinds`, and
|
|
5698
|
+
* the container backend binds each `<dir>:<dir>:ro` into the job container —
|
|
5699
|
+
* the same clone-source affordance across both isolation models.
|
|
5700
|
+
*/
|
|
5701
|
+
function fileCloneSourceBinds(repoUrl) {
|
|
5702
|
+
if (typeof repoUrl !== "string" || !repoUrl.startsWith("file://")) return [];
|
|
5703
|
+
try {
|
|
5704
|
+
const url = new URL(repoUrl);
|
|
5705
|
+
return url.pathname && url.pathname !== "/" ? [url.pathname] : [];
|
|
5706
|
+
} catch {
|
|
5707
|
+
return [];
|
|
5708
|
+
}
|
|
5709
|
+
}
|
|
5710
|
+
/**
|
|
4763
5711
|
* Spawn the workflow-runner child process. Returns the child handle and a
|
|
4764
5712
|
* boolean indicating whether the spawn produced a usable PID. On Windows
|
|
4765
5713
|
* services (shawl) fork() can fail silently when IPC pipes cannot be set up;
|
|
@@ -4908,6 +5856,24 @@ function relayProvenanceRequest$1(msg, ctx) {
|
|
|
4908
5856
|
error: toErrorMessage(err)
|
|
4909
5857
|
}));
|
|
4910
5858
|
}
|
|
5859
|
+
/** Relay `artifacts.request` and pipe the orchestrator response (or an error
|
|
5860
|
+
* response, or a "not configured" response when the callback isn't wired) back
|
|
5861
|
+
* into the sandbox runner. */
|
|
5862
|
+
function relayArtifactRequest$1(msg, ctx) {
|
|
5863
|
+
if (!ctx.execOptions.onArtifactRequest) {
|
|
5864
|
+
safeSendToChild(ctx.child, {
|
|
5865
|
+
type: "artifacts.response",
|
|
5866
|
+
requestId: msg.requestId,
|
|
5867
|
+
error: "Artifacts not available in this agent configuration"
|
|
5868
|
+
});
|
|
5869
|
+
return;
|
|
5870
|
+
}
|
|
5871
|
+
ctx.execOptions.onArtifactRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
|
|
5872
|
+
type: "artifacts.response",
|
|
5873
|
+
requestId: msg.requestId,
|
|
5874
|
+
error: toErrorMessage(err)
|
|
5875
|
+
}));
|
|
5876
|
+
}
|
|
4911
5877
|
/** Relay `approval.request` and pipe the orchestrator's resolution (or a
|
|
4912
5878
|
* fail-closed reject when the callback isn't wired or the relay throws) back
|
|
4913
5879
|
* into the sandbox runner. */
|
|
@@ -4962,7 +5928,7 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
4962
5928
|
});
|
|
4963
5929
|
return;
|
|
4964
5930
|
case "log.line":
|
|
4965
|
-
ctx.execOptions.onLogLine(msg.stepIndex, msg.line);
|
|
5931
|
+
ctx.execOptions.onLogLine(msg.stepIndex, msg.line, msg.stream);
|
|
4966
5932
|
return;
|
|
4967
5933
|
case "step.start": {
|
|
4968
5934
|
ctx.stepNames.set(msg.stepIndex, msg.stepName);
|
|
@@ -5010,6 +5976,9 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
5010
5976
|
case "cache.request":
|
|
5011
5977
|
relayCacheRequest$1(msg, ctx);
|
|
5012
5978
|
return;
|
|
5979
|
+
case "artifacts.request":
|
|
5980
|
+
relayArtifactRequest$1(msg, ctx);
|
|
5981
|
+
return;
|
|
5013
5982
|
case "provenance.request":
|
|
5014
5983
|
relayProvenanceRequest$1(msg, ctx);
|
|
5015
5984
|
return;
|
|
@@ -5227,14 +6196,7 @@ var init_bare_metal_sandbox = __esmMin((() => {
|
|
|
5227
6196
|
* Execute a job by forking the workflow runner with sanitized environment.
|
|
5228
6197
|
*/
|
|
5229
6198
|
async executeJob(options) {
|
|
5230
|
-
const extraReadOnlyBinds = [];
|
|
5231
|
-
if (this.useBwrap) {
|
|
5232
|
-
const repoUrl = options.dispatch.repoUrl;
|
|
5233
|
-
if (typeof repoUrl === "string" && repoUrl.startsWith("file://")) try {
|
|
5234
|
-
const url = new URL(repoUrl);
|
|
5235
|
-
if (url.pathname) extraReadOnlyBinds.push(url.pathname);
|
|
5236
|
-
} catch {}
|
|
5237
|
-
}
|
|
6199
|
+
const extraReadOnlyBinds = this.useBwrap ? fileCloneSourceBinds(options.dispatch.repoUrl) : [];
|
|
5238
6200
|
this.runner = createForkRunner({
|
|
5239
6201
|
runnerPath: this.runnerPath,
|
|
5240
6202
|
env: this.env,
|
|
@@ -5321,6 +6283,43 @@ var init_firecracker_sandbox = __esmMin((() => {
|
|
|
5321
6283
|
};
|
|
5322
6284
|
}));
|
|
5323
6285
|
//#endregion
|
|
6286
|
+
//#region src/execution/sandbox/container-hardening.ts
|
|
6287
|
+
/**
|
|
6288
|
+
* Build the hardened HostConfig fragment for a job sandbox container.
|
|
6289
|
+
*
|
|
6290
|
+
* Pure and side-effect-free: given resolved inputs it returns the fields the
|
|
6291
|
+
* container-sandbox merges into `docker.createContainer`. When `hardened` is
|
|
6292
|
+
* false it returns an empty posture (the rollback affordance).
|
|
6293
|
+
*/
|
|
6294
|
+
function buildContainerHardening(opts) {
|
|
6295
|
+
if (!opts.hardened) return { hostConfig: {} };
|
|
6296
|
+
const grant = opts.grant;
|
|
6297
|
+
const hostConfig = {
|
|
6298
|
+
CapDrop: [...CAP_DROP_ALL],
|
|
6299
|
+
SecurityOpt: [...NO_NEW_PRIVILEGES],
|
|
6300
|
+
PidsLimit: opts.pidsLimit,
|
|
6301
|
+
Memory: opts.memoryBytes,
|
|
6302
|
+
MemorySwap: opts.memoryBytes,
|
|
6303
|
+
NanoCpus: opts.nanoCpus,
|
|
6304
|
+
Tmpfs: { "/tmp": "rw,exec,nosuid,nodev" }
|
|
6305
|
+
};
|
|
6306
|
+
if (grant?.capabilities && grant.capabilities.length > 0) hostConfig.CapAdd = [...grant.capabilities];
|
|
6307
|
+
if (grant?.readonlyRootfs ?? opts.readonlyRootfs) hostConfig.ReadonlyRootfs = true;
|
|
6308
|
+
const network = grant?.network ?? opts.networkMode;
|
|
6309
|
+
if (network === "none") hostConfig.NetworkMode = "none";
|
|
6310
|
+
else if (network === "host") hostConfig.NetworkMode = "host";
|
|
6311
|
+
const user = grant?.user ?? opts.user;
|
|
6312
|
+
return user ? {
|
|
6313
|
+
hostConfig,
|
|
6314
|
+
user
|
|
6315
|
+
} : { hostConfig };
|
|
6316
|
+
}
|
|
6317
|
+
var CAP_DROP_ALL, NO_NEW_PRIVILEGES;
|
|
6318
|
+
var init_container_hardening = __esmMin((() => {
|
|
6319
|
+
CAP_DROP_ALL = ["ALL"];
|
|
6320
|
+
NO_NEW_PRIVILEGES = ["no-new-privileges"];
|
|
6321
|
+
}));
|
|
6322
|
+
//#endregion
|
|
5324
6323
|
//#region src/execution/sandbox/container-sandbox.ts
|
|
5325
6324
|
/**
|
|
5326
6325
|
* Container execution sandbox implementation.
|
|
@@ -5463,6 +6462,32 @@ function relayProvenanceRequest(stream, options, provMsg) {
|
|
|
5463
6462
|
}));
|
|
5464
6463
|
}
|
|
5465
6464
|
/**
|
|
6465
|
+
* Relay artifacts.request from the container runner to the orchestrator via
|
|
6466
|
+
* options.onArtifactRequest, then write the response back through `stream`. If
|
|
6467
|
+
* the agent doesn't expose an artifacts relay, write a structured error so the
|
|
6468
|
+
* runner doesn't hang.
|
|
6469
|
+
*/
|
|
6470
|
+
function relayArtifactRequest(stream, options, artMsg) {
|
|
6471
|
+
const writeResponse = (response) => {
|
|
6472
|
+
try {
|
|
6473
|
+
stream.write(JSON.stringify(response) + "\n");
|
|
6474
|
+
} catch {}
|
|
6475
|
+
};
|
|
6476
|
+
if (!options.onArtifactRequest) {
|
|
6477
|
+
writeResponse({
|
|
6478
|
+
type: "artifacts.response",
|
|
6479
|
+
requestId: artMsg.requestId,
|
|
6480
|
+
error: "Artifacts not available in this agent configuration"
|
|
6481
|
+
});
|
|
6482
|
+
return;
|
|
6483
|
+
}
|
|
6484
|
+
options.onArtifactRequest(artMsg).then((response) => writeResponse(response), (err) => writeResponse({
|
|
6485
|
+
type: "artifacts.response",
|
|
6486
|
+
requestId: artMsg.requestId,
|
|
6487
|
+
error: toErrorMessage(err)
|
|
6488
|
+
}));
|
|
6489
|
+
}
|
|
6490
|
+
/**
|
|
5466
6491
|
* Relay approval.request from the container runner to the orchestrator via
|
|
5467
6492
|
* options.onApprovalRequest, then write the resolution back through `stream`.
|
|
5468
6493
|
* If the agent doesn't expose an approval relay (or it throws), write a
|
|
@@ -5506,22 +6531,30 @@ function applyJobComplete(msg, stepResults, state, options) {
|
|
|
5506
6531
|
logger$3.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
|
|
5507
6532
|
}
|
|
5508
6533
|
}
|
|
5509
|
-
var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, ContainerSandbox;
|
|
6534
|
+
var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, HOOK_MOUNT_PATH, INTERNAL_ENV, ContainerSandbox;
|
|
5510
6535
|
var init_container_sandbox = __esmMin((() => {
|
|
5511
6536
|
init_fork_runner();
|
|
5512
6537
|
init_secret_encryption();
|
|
6538
|
+
init_container_hardening();
|
|
5513
6539
|
logger$3 = createLogger({ prefix: "container-sandbox" });
|
|
5514
6540
|
MAX_STDERR_LINES = 20;
|
|
5515
6541
|
ABORT_GRACE_MS = 1e4;
|
|
5516
6542
|
CONTAINER_STOP_TIMEOUT = 10;
|
|
6543
|
+
HOOK_MOUNT_PATH = "/opt/kici/ts-loader-hook.js";
|
|
6544
|
+
INTERNAL_ENV = [`KICI_TS_LOADER_HOOK_PATH=${HOOK_MOUNT_PATH}`, "KICI_LOG_STDERR=1"];
|
|
5517
6545
|
ContainerSandbox = class {
|
|
5518
6546
|
docker;
|
|
5519
6547
|
image;
|
|
5520
6548
|
runnerPath;
|
|
5521
6549
|
runnerMountPath;
|
|
6550
|
+
/** Host path to the pure-JS container loader-hook bundle (bind-mounted :ro). */
|
|
6551
|
+
hookHostPath;
|
|
5522
6552
|
env;
|
|
5523
6553
|
keepFailed;
|
|
5524
6554
|
jobId;
|
|
6555
|
+
hardening;
|
|
6556
|
+
/** Resolved container user (image-user override / grant), applied to createContainer + each exec. */
|
|
6557
|
+
resolvedUser;
|
|
5525
6558
|
/** The running container instance (set during setup). */
|
|
5526
6559
|
container = null;
|
|
5527
6560
|
/** The active exec stream (set during executeJob, used for abort). */
|
|
@@ -5535,29 +6568,66 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5535
6568
|
this.image = options.image;
|
|
5536
6569
|
this.runnerPath = options.runnerPath;
|
|
5537
6570
|
this.runnerMountPath = options.runnerMountPath ?? "/opt/kici/workflow-runner.js";
|
|
6571
|
+
this.hookHostPath = options.hookPath ?? join(dirname(options.runnerPath), "container-ts-loader-hook.js");
|
|
5538
6572
|
this.env = options.env;
|
|
5539
6573
|
this.keepFailed = options.keepFailed ?? false;
|
|
5540
6574
|
this.jobId = options.jobId ?? `unknown-${Date.now()}`;
|
|
6575
|
+
this.hardening = options.hardening;
|
|
6576
|
+
}
|
|
6577
|
+
/**
|
|
6578
|
+
* Ensure the job's container image is present locally, pulling it on demand
|
|
6579
|
+
* when it is not.
|
|
6580
|
+
*
|
|
6581
|
+
* dockerode's `createContainer` — unlike `docker run` / `podman run` — never
|
|
6582
|
+
* auto-pulls a missing image; it fails with `(HTTP code 404) ... No such
|
|
6583
|
+
* image`. A bare-metal executor that aggressively prunes unused images under
|
|
6584
|
+
* disk pressure can leave a container job with nothing to run, so the agent
|
|
6585
|
+
* pulls the image itself. Already-present images (the common case, and how
|
|
6586
|
+
* private images pre-pulled with registry auth stay working) skip the pull.
|
|
6587
|
+
*/
|
|
6588
|
+
async ensureImagePresent() {
|
|
6589
|
+
try {
|
|
6590
|
+
await this.docker.getImage(this.image).inspect();
|
|
6591
|
+
return;
|
|
6592
|
+
} catch {}
|
|
6593
|
+
logger$3.info("Pulling sandbox image (not present locally)", { image: this.image });
|
|
6594
|
+
const stream = await this.docker.pull(this.image);
|
|
6595
|
+
await new Promise((resolve, reject) => {
|
|
6596
|
+
this.docker.modem.followProgress(stream, (err) => err ? reject(err) : resolve());
|
|
6597
|
+
});
|
|
6598
|
+
logger$3.info("Sandbox image pulled", { image: this.image });
|
|
5541
6599
|
}
|
|
5542
6600
|
async setup(options) {
|
|
5543
6601
|
this.containerName = `kici-sandbox-${this.jobId}-${Date.now()}`;
|
|
5544
|
-
const envArray = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
|
|
6602
|
+
const envArray = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
|
|
5545
6603
|
logger$3.info("Creating sandbox container", {
|
|
5546
6604
|
name: this.containerName,
|
|
5547
6605
|
image: this.image,
|
|
5548
6606
|
workDir: options.workDir
|
|
5549
6607
|
});
|
|
6608
|
+
const hardened = this.hardening ? buildContainerHardening(this.hardening) : {
|
|
6609
|
+
hostConfig: {},
|
|
6610
|
+
user: void 0
|
|
6611
|
+
};
|
|
6612
|
+
this.resolvedUser = hardened.user;
|
|
6613
|
+
const binds = this.buildBinds(options, hardened.hostConfig);
|
|
6614
|
+
await this.ensureImagePresent();
|
|
5550
6615
|
this.container = await this.docker.createContainer({
|
|
5551
6616
|
Image: this.image,
|
|
5552
6617
|
name: this.containerName,
|
|
5553
6618
|
Cmd: ["sleep", "infinity"],
|
|
5554
6619
|
Env: envArray,
|
|
5555
6620
|
WorkingDir: "/workspace",
|
|
6621
|
+
Volumes: { "/workspace": {} },
|
|
6622
|
+
...hardened.user ? { User: hardened.user } : {},
|
|
5556
6623
|
Labels: {
|
|
5557
6624
|
"kici-sandbox": "true",
|
|
5558
6625
|
"kici-job-id": this.jobId
|
|
5559
6626
|
},
|
|
5560
|
-
HostConfig: {
|
|
6627
|
+
HostConfig: {
|
|
6628
|
+
Binds: binds,
|
|
6629
|
+
...hardened.hostConfig
|
|
6630
|
+
}
|
|
5561
6631
|
});
|
|
5562
6632
|
await this.container.start();
|
|
5563
6633
|
logger$3.info("Sandbox container started", {
|
|
@@ -5565,6 +6635,36 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5565
6635
|
containerId: this.container.id.slice(0, 12)
|
|
5566
6636
|
});
|
|
5567
6637
|
}
|
|
6638
|
+
/**
|
|
6639
|
+
* Build the container's read-only bind list.
|
|
6640
|
+
*
|
|
6641
|
+
* The workspace is NOT bound here — it is a container-owned anonymous volume
|
|
6642
|
+
* (`Volumes: { '/workspace': {} }` on the container config), so the container
|
|
6643
|
+
* user can write it on every runtime with `CapDrop: ['ALL']` intact. This
|
|
6644
|
+
* method binds the workflow runner (read-only) plus two parity affordances
|
|
6645
|
+
* that mirror the bare-metal bwrap sandbox — both strictly additive and gated,
|
|
6646
|
+
* so the default posture for a production (https-source, bridge-network) job is
|
|
6647
|
+
* unchanged:
|
|
6648
|
+
*
|
|
6649
|
+
* - **`file://` clone-source dir(s)** (`options.extraReadOnlyBinds`): the
|
|
6650
|
+
* workflow runner clones the repo from inside the container, so a local
|
|
6651
|
+
* `file://` source dir must be exposed read-only or `git clone` fails.
|
|
6652
|
+
* Empty for https/ssh remotes — mirrors fork-runner's `extraReadOnlyBinds`.
|
|
6653
|
+
* - **Host name-resolution files under host networking**: when the effective
|
|
6654
|
+
* network posture is `host` (`KICI_SANDBOX_NETWORK=host`, or a per-job host
|
|
6655
|
+
* grant), bind the host's `/etc/hosts` (+ `/etc/nsswitch.conf`) read-only so
|
|
6656
|
+
* an `/etc/hosts`-only name the host resolves — e.g. a private registry —
|
|
6657
|
+
* resolves inside the container too. Mirrors fork-runner's
|
|
6658
|
+
* `--ro-bind /etc/hosts` for the bwrap host-network mode.
|
|
6659
|
+
*/
|
|
6660
|
+
buildBinds(options, hostConfig) {
|
|
6661
|
+
const binds = [`${this.runnerPath}:${this.runnerMountPath}:ro`, `${this.hookHostPath}:${HOOK_MOUNT_PATH}:ro`];
|
|
6662
|
+
for (const dir of options.extraReadOnlyBinds ?? []) if (dir) binds.push(`${dir}:${dir}:ro`);
|
|
6663
|
+
if (hostConfig.NetworkMode === "host") {
|
|
6664
|
+
for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) binds.push(`${nssFile}:${nssFile}:ro`);
|
|
6665
|
+
}
|
|
6666
|
+
return binds;
|
|
6667
|
+
}
|
|
5568
6668
|
async executeJob(options) {
|
|
5569
6669
|
if (!this.container) throw new Error("ContainerSandbox.executeJob() called before setup()");
|
|
5570
6670
|
const startTime = Date.now();
|
|
@@ -5596,14 +6696,15 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5596
6696
|
* stderr lines for crash diagnostics, and install the abort listener.
|
|
5597
6697
|
*/
|
|
5598
6698
|
async attachExecStream(options) {
|
|
5599
|
-
const execEnv = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
|
|
6699
|
+
const execEnv = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
|
|
5600
6700
|
const stream = await (await this.container.exec({
|
|
5601
6701
|
Cmd: ["node", this.runnerMountPath],
|
|
5602
6702
|
AttachStdin: true,
|
|
5603
6703
|
AttachStdout: true,
|
|
5604
6704
|
AttachStderr: true,
|
|
5605
6705
|
Env: execEnv,
|
|
5606
|
-
WorkingDir: "/workspace"
|
|
6706
|
+
WorkingDir: "/workspace",
|
|
6707
|
+
...this.resolvedUser ? { User: this.resolvedUser } : {}
|
|
5607
6708
|
})).start({
|
|
5608
6709
|
hijack: true,
|
|
5609
6710
|
stdin: true
|
|
@@ -5734,7 +6835,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5734
6835
|
return false;
|
|
5735
6836
|
}
|
|
5736
6837
|
case "log.line":
|
|
5737
|
-
options.onLogLine(msg.stepIndex, msg.line);
|
|
6838
|
+
options.onLogLine(msg.stepIndex, msg.line, msg.stream);
|
|
5738
6839
|
return false;
|
|
5739
6840
|
case "step.secret_mount":
|
|
5740
6841
|
options.onSecretMount?.({
|
|
@@ -5757,6 +6858,9 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5757
6858
|
case "cache.request":
|
|
5758
6859
|
relayCacheRequest(stream, options, msg);
|
|
5759
6860
|
return false;
|
|
6861
|
+
case "artifacts.request":
|
|
6862
|
+
relayArtifactRequest(stream, options, msg);
|
|
6863
|
+
return false;
|
|
5760
6864
|
case "provenance.request":
|
|
5761
6865
|
relayProvenanceRequest(stream, options, msg);
|
|
5762
6866
|
return false;
|
|
@@ -5806,7 +6910,10 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5806
6910
|
await this.container.stop({ t: CONTAINER_STOP_TIMEOUT });
|
|
5807
6911
|
} catch {}
|
|
5808
6912
|
try {
|
|
5809
|
-
await this.container.remove({
|
|
6913
|
+
await this.container.remove({
|
|
6914
|
+
force: true,
|
|
6915
|
+
v: true
|
|
6916
|
+
});
|
|
5810
6917
|
} catch {}
|
|
5811
6918
|
this.container = null;
|
|
5812
6919
|
}
|
|
@@ -5850,12 +6957,15 @@ var init_sandbox = __esmMin((() => {
|
|
|
5850
6957
|
init_bare_metal_sandbox();
|
|
5851
6958
|
init_firecracker_sandbox();
|
|
5852
6959
|
init_container_sandbox();
|
|
6960
|
+
init_fork_runner();
|
|
5853
6961
|
}));
|
|
5854
6962
|
//#endregion
|
|
5855
6963
|
//#region src/execution/job-runner.ts
|
|
5856
6964
|
var job_runner_exports = /* @__PURE__ */ __exportAll({
|
|
5857
6965
|
JobRunner: () => JobRunner$1,
|
|
5858
|
-
buildEvalNeedsContext: () => buildEvalNeedsContext
|
|
6966
|
+
buildEvalNeedsContext: () => buildEvalNeedsContext,
|
|
6967
|
+
resolveJobWorkDir: () => resolveJobWorkDir,
|
|
6968
|
+
resolveRunnerBundlePath: () => resolveRunnerBundlePath
|
|
5859
6969
|
});
|
|
5860
6970
|
/**
|
|
5861
6971
|
* Check if a file exists at the given path.
|
|
@@ -5890,6 +7000,19 @@ function resolveRunnerPath() {
|
|
|
5890
7000
|
return bundlePath;
|
|
5891
7001
|
}
|
|
5892
7002
|
/**
|
|
7003
|
+
* Derive the self-contained runner bundle path from the resolved runner path.
|
|
7004
|
+
*
|
|
7005
|
+
* The container backend mounts the runner as a single file into the customer
|
|
7006
|
+
* job container, so it must run `workflow-runner-bundle.js` (zx + `@kici-dev/*`
|
|
7007
|
+
* inlined) rather than the external `workflow-runner.js`, which cannot resolve
|
|
7008
|
+
* its bare imports without the agent's node_modules / pnpm workspace. The
|
|
7009
|
+
* bundle is a flat sibling emitted alongside the runner by build-service.mjs.
|
|
7010
|
+
* bwrap / firecracker keep the external runner (they bind the workspace).
|
|
7011
|
+
*/
|
|
7012
|
+
function resolveRunnerBundlePath(runnerPath) {
|
|
7013
|
+
return join(dirname(runnerPath), "workflow-runner-bundle.js");
|
|
7014
|
+
}
|
|
7015
|
+
/**
|
|
5893
7016
|
* Determine the execution mode from agent config and job config.
|
|
5894
7017
|
*
|
|
5895
7018
|
* Priority:
|
|
@@ -5912,6 +7035,33 @@ function buildEvalNeedsContext(config) {
|
|
|
5912
7035
|
if (!config.resultAware || !config.upstreamSnapshot) return void 0;
|
|
5913
7036
|
return buildNeedsContext(config.upstreamSnapshot, config.declaredNeeds ?? []);
|
|
5914
7037
|
}
|
|
7038
|
+
/**
|
|
7039
|
+
* Resolve a job's workDir and its cleanup.
|
|
7040
|
+
*
|
|
7041
|
+
* - Default: a fresh `mkdtemp` the agent clones into and removes after the job.
|
|
7042
|
+
* - **In-place profile** (`inPlace` config + a `file://` source): the source's
|
|
7043
|
+
* real repo path used directly as the workDir, with **no clone** and **no
|
|
7044
|
+
* removal**. This is the routed `deploy:stg` profile — the operator runs their
|
|
7045
|
+
* own already-built working tree (module-relative `MONOREPO_ROOT`,
|
|
7046
|
+
* `node_modules`, `dist` all present). Gated to `file://` so a
|
|
7047
|
+
* Platform-connected agent (https sources) can never be pushed onto a tree,
|
|
7048
|
+
* and read only from agent config (never a dispatch/wire value).
|
|
7049
|
+
*/
|
|
7050
|
+
async function resolveJobWorkDir(inPlace, repoUrl) {
|
|
7051
|
+
if (inPlace && repoUrl && repoUrl.startsWith("file://")) return {
|
|
7052
|
+
workDir: fileURLToPath(repoUrl),
|
|
7053
|
+
cleanup: async () => {},
|
|
7054
|
+
inPlace: true
|
|
7055
|
+
};
|
|
7056
|
+
const { path: workDir, cleanup } = await makeTempDir("workdir");
|
|
7057
|
+
return {
|
|
7058
|
+
workDir,
|
|
7059
|
+
cleanup: async () => {
|
|
7060
|
+
await cleanup().catch(() => {});
|
|
7061
|
+
},
|
|
7062
|
+
inPlace: false
|
|
7063
|
+
};
|
|
7064
|
+
}
|
|
5915
7065
|
var logger$2, JobRunner$1;
|
|
5916
7066
|
var init_job_runner = __esmMin((() => {
|
|
5917
7067
|
init_git_clone();
|
|
@@ -5921,7 +7071,10 @@ var init_job_runner = __esmMin((() => {
|
|
|
5921
7071
|
init_init_runner();
|
|
5922
7072
|
init_api_intercept();
|
|
5923
7073
|
init_ensure_init_runner();
|
|
7074
|
+
init_payload_source();
|
|
7075
|
+
init_s3_payload_source();
|
|
5924
7076
|
init_timeout_util();
|
|
7077
|
+
init_streaming_zx_log();
|
|
5925
7078
|
init_dynamic_job_serializer();
|
|
5926
7079
|
init_log_streamer();
|
|
5927
7080
|
init_overlay_applier();
|
|
@@ -5934,7 +7087,6 @@ var init_job_runner = __esmMin((() => {
|
|
|
5934
7087
|
logger$2 = createLogger({ prefix: "job-runner" });
|
|
5935
7088
|
JobRunner$1 = class {
|
|
5936
7089
|
send;
|
|
5937
|
-
sendDirect;
|
|
5938
7090
|
config;
|
|
5939
7091
|
requestUploadUrl;
|
|
5940
7092
|
sendUploadComplete;
|
|
@@ -5947,6 +7099,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5947
7099
|
_sendApiRequest;
|
|
5948
7100
|
_requestUserCache;
|
|
5949
7101
|
_relayProvenance;
|
|
7102
|
+
_requestUserArtifact;
|
|
5950
7103
|
_sendStepApproval;
|
|
5951
7104
|
/** Tracks running jobs for concurrency and cancellation */
|
|
5952
7105
|
activeJobs = /* @__PURE__ */ new Map();
|
|
@@ -5954,7 +7107,6 @@ var init_job_runner = __esmMin((() => {
|
|
|
5954
7107
|
activeSandbox = null;
|
|
5955
7108
|
constructor(deps) {
|
|
5956
7109
|
this.send = deps.send;
|
|
5957
|
-
this.sendDirect = deps.sendDirect;
|
|
5958
7110
|
this.config = deps.config;
|
|
5959
7111
|
this.requestUploadUrl = deps.requestUploadUrl;
|
|
5960
7112
|
this.sendUploadComplete = deps.sendUploadComplete;
|
|
@@ -5967,6 +7119,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
5967
7119
|
this._sendApiRequest = deps.sendApiRequest;
|
|
5968
7120
|
this._requestUserCache = deps.requestUserCache;
|
|
5969
7121
|
this._relayProvenance = deps.relayProvenance;
|
|
7122
|
+
this._requestUserArtifact = deps.requestUserArtifact;
|
|
5970
7123
|
this._sendStepApproval = deps.sendStepApproval;
|
|
5971
7124
|
}
|
|
5972
7125
|
/**
|
|
@@ -5978,14 +7131,12 @@ var init_job_runner = __esmMin((() => {
|
|
|
5978
7131
|
async execute(dispatch) {
|
|
5979
7132
|
const { runId: _runId, jobId, jobConfig: _jobConfig } = dispatch;
|
|
5980
7133
|
const abortController = new AbortController();
|
|
5981
|
-
const workDir = await
|
|
7134
|
+
const { workDir, cleanup, inPlace } = await resolveJobWorkDir(this.config.inPlace, dispatch.repoUrl);
|
|
7135
|
+
if (inPlace) dispatch.jobConfig.checkout = false;
|
|
5982
7136
|
const completionPromise = this.runJob(dispatch, workDir, abortController).finally(async () => {
|
|
5983
7137
|
this.activeJobs.delete(jobId);
|
|
5984
7138
|
this.activeSandbox = null;
|
|
5985
|
-
await
|
|
5986
|
-
recursive: true,
|
|
5987
|
-
force: true
|
|
5988
|
-
}).catch(() => {});
|
|
7139
|
+
await cleanup();
|
|
5989
7140
|
});
|
|
5990
7141
|
this.activeJobs.set(jobId, {
|
|
5991
7142
|
abortController,
|
|
@@ -6072,13 +7223,15 @@ var init_job_runner = __esmMin((() => {
|
|
|
6072
7223
|
});
|
|
6073
7224
|
}, this.config.jobHeartbeatIntervalMs);
|
|
6074
7225
|
let sandbox;
|
|
7226
|
+
const logStreamers = /* @__PURE__ */ new Map();
|
|
6075
7227
|
try {
|
|
6076
7228
|
const setupResult = await this.setupSandboxForExecution(dispatch, workDir, abortController);
|
|
6077
7229
|
if (!setupResult) return;
|
|
6078
7230
|
sandbox = setupResult.sandbox;
|
|
6079
|
-
const
|
|
7231
|
+
const result = await this.runSandboxExecution(dispatch, sandbox, abortController, logStreamers);
|
|
6080
7232
|
this.reportExecutionResult(dispatch, result, logStreamers);
|
|
6081
7233
|
} catch (error) {
|
|
7234
|
+
for (const streamer of logStreamers.values()) streamer.destroy();
|
|
6082
7235
|
const errorMsg = toErrorMessage(error);
|
|
6083
7236
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, { error: errorMsg });
|
|
6084
7237
|
} finally {
|
|
@@ -6111,8 +7264,9 @@ var init_job_runner = __esmMin((() => {
|
|
|
6111
7264
|
const runnerPath = resolveRunnerPath();
|
|
6112
7265
|
const typedConfig = jobConfig;
|
|
6113
7266
|
const sanitizedEnv = buildSanitizedEnv(typedConfig.env ?? {}, {
|
|
6114
|
-
|
|
6115
|
-
jobEnv: typedConfig.jobEnv ?? void 0
|
|
7267
|
+
contextVars: typedConfig.contextVars ?? void 0,
|
|
7268
|
+
jobEnv: typedConfig.jobEnv ?? void 0,
|
|
7269
|
+
trustedEnv: this.config.trustedEnv
|
|
6116
7270
|
});
|
|
6117
7271
|
logger$2.info("Creating execution sandbox", {
|
|
6118
7272
|
executionMode,
|
|
@@ -6128,7 +7282,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
6128
7282
|
this.activeSandbox = sandbox;
|
|
6129
7283
|
await sandbox.setup({
|
|
6130
7284
|
workDir,
|
|
6131
|
-
env: sanitizedEnv
|
|
7285
|
+
env: sanitizedEnv,
|
|
7286
|
+
extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl)
|
|
6132
7287
|
});
|
|
6133
7288
|
if (abortController.signal.aborted) {
|
|
6134
7289
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.cancelled);
|
|
@@ -6153,13 +7308,13 @@ var init_job_runner = __esmMin((() => {
|
|
|
6153
7308
|
/**
|
|
6154
7309
|
* Drive `sandbox.executeJob` with IPC callbacks wired to the WS pipeline.
|
|
6155
7310
|
*
|
|
6156
|
-
* Lazily
|
|
7311
|
+
* Lazily populates `logStreamers` (owned by the caller so a throw still
|
|
7312
|
+
* leaves the buffered output flushable), forwards step + log + event-emit +
|
|
6157
7313
|
* concurrency-report + api-request messages, and emits the
|
|
6158
7314
|
* `agent.execution.start` / `agent.execution.end` lifecycle events.
|
|
6159
7315
|
*/
|
|
6160
|
-
async runSandboxExecution(dispatch, sandbox, abortController) {
|
|
7316
|
+
async runSandboxExecution(dispatch, sandbox, abortController, logStreamers) {
|
|
6161
7317
|
const { runId, jobId } = dispatch;
|
|
6162
|
-
const logStreamers = /* @__PURE__ */ new Map();
|
|
6163
7318
|
const maxLogSizeBytes = dispatch.maxLogSizeBytes ?? this.config.maxLogSizeBytes;
|
|
6164
7319
|
const getOrCreateLogStreamer = (stepIndex) => {
|
|
6165
7320
|
let streamer = logStreamers.get(stepIndex);
|
|
@@ -6188,8 +7343,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
6188
7343
|
this.maybeEmitCacheRunEvent(runId, jobId, stepIndex, state, data);
|
|
6189
7344
|
this.sendStepStatus(dispatch, stepIndex, stepName, state, data, logBytesStreamed);
|
|
6190
7345
|
},
|
|
6191
|
-
onLogLine: (stepIndex, line) => {
|
|
6192
|
-
getOrCreateLogStreamer(stepIndex).addLine(line);
|
|
7346
|
+
onLogLine: (stepIndex, line, stream) => {
|
|
7347
|
+
getOrCreateLogStreamer(stepIndex).addLine(line, stream);
|
|
6193
7348
|
},
|
|
6194
7349
|
signal: abortController.signal,
|
|
6195
7350
|
onEventEmit: async (request) => {
|
|
@@ -6212,6 +7367,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6212
7367
|
onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => this._sendApiRequest(method, params)) : void 0,
|
|
6213
7368
|
onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
|
|
6214
7369
|
onProvenanceRequest: this._relayProvenance ? async (request) => this._relayProvenance(jobId, request) : void 0,
|
|
7370
|
+
onArtifactRequest: this._requestUserArtifact ? async (request) => this._requestUserArtifact(jobId, request) : void 0,
|
|
6215
7371
|
onApprovalRequest: this._sendStepApproval ? async (request) => this._sendStepApproval(dispatch.runId, dispatch.jobId, request) : void 0,
|
|
6216
7372
|
onSecretMount: (event) => {
|
|
6217
7373
|
this.emitRunEvent(runId, "step.secret_mount", {
|
|
@@ -6231,10 +7387,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6231
7387
|
durationMs: Date.now() - executionStartMs,
|
|
6232
7388
|
metadata: { status: result.status }
|
|
6233
7389
|
});
|
|
6234
|
-
return
|
|
6235
|
-
result,
|
|
6236
|
-
logStreamers
|
|
6237
|
-
};
|
|
7390
|
+
return result;
|
|
6238
7391
|
}
|
|
6239
7392
|
/**
|
|
6240
7393
|
* Tear down log streamers, record step Prometheus metrics, log sandbox
|
|
@@ -6296,7 +7449,19 @@ var init_job_runner = __esmMin((() => {
|
|
|
6296
7449
|
if (!targetAgentId) throw new Error("bring-up job missing bringupTarget");
|
|
6297
7450
|
if (!this._sendApiRequest) throw new Error("bring-up job requires an orchestrator API transport");
|
|
6298
7451
|
streamer.addLine(`Bringing up init-runner on ${targetAgentId}…`);
|
|
6299
|
-
const
|
|
7452
|
+
const transport = async (method, params) => this._sendApiRequest(method, params);
|
|
7453
|
+
const result = await ensureInitRunner(transport, targetAgentId, {
|
|
7454
|
+
payloadSource: this.config.agentPayloadDir ? new LocalDirPayloadSource(this.config.agentPayloadDir) : new S3PayloadSource({
|
|
7455
|
+
presign: async (platform) => await transport("kici.presignAgentPackage", {
|
|
7456
|
+
targetAgentId,
|
|
7457
|
+
platform
|
|
7458
|
+
}),
|
|
7459
|
+
fetchToFile: fetchUrlToFile,
|
|
7460
|
+
cacheDir: defaultPayloadCacheDir(),
|
|
7461
|
+
exists: payloadFileExists
|
|
7462
|
+
}),
|
|
7463
|
+
agentCommand: this.config.agentCommand
|
|
7464
|
+
});
|
|
6300
7465
|
streamer.addLine(result.broughtUp ? `Init-runner brought up on ${targetAgentId}.` : `${targetAgentId} already has a live agent — no bring-up needed.`);
|
|
6301
7466
|
await streamer.flush();
|
|
6302
7467
|
this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.success, void 0, streamer.getTotalBytes());
|
|
@@ -6583,9 +7748,9 @@ var init_job_runner = __esmMin((() => {
|
|
|
6583
7748
|
const initResult = await runCaptured(initSink, async () => {
|
|
6584
7749
|
const { module } = await loadWorkflowSource(workDir, config.source, config.contentHash, config.resolvedHashFiles);
|
|
6585
7750
|
const workflow = extractWorkflow(module, config.workflowName);
|
|
6586
|
-
initLog(`Evaluating dynamic fields for job '${config.targetJobName}' (env=${config.dynamicEnv}
|
|
7751
|
+
initLog(`Evaluating dynamic fields for job '${config.targetJobName}' (env=${config.dynamicEnv} context=${config.dynamicContext} concurrencyGroup=${config.dynamicConcurrencyGroup} matrix=${config.dynamicMatrix ?? false})`);
|
|
6587
7752
|
return evaluateDynamicFields(workflow, config.targetJobName, config.event, {
|
|
6588
|
-
|
|
7753
|
+
dynamicContext: config.dynamicContext,
|
|
6589
7754
|
dynamicEnv: config.dynamicEnv,
|
|
6590
7755
|
dynamicConcurrencyGroup: config.dynamicConcurrencyGroup,
|
|
6591
7756
|
dynamicMatrix: config.dynamicMatrix ?? false
|
|
@@ -6593,7 +7758,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6593
7758
|
});
|
|
6594
7759
|
logger$2.info("Init job completed successfully", {
|
|
6595
7760
|
jobId,
|
|
6596
|
-
|
|
7761
|
+
hasContext: initResult.contextNames !== void 0,
|
|
6597
7762
|
hasEnv: initResult.env !== void 0,
|
|
6598
7763
|
hasConcurrencyGroup: initResult.concurrencyGroup !== void 0
|
|
6599
7764
|
});
|
|
@@ -6688,20 +7853,12 @@ var init_job_runner = __esmMin((() => {
|
|
|
6688
7853
|
});
|
|
6689
7854
|
}
|
|
6690
7855
|
const { $: zx$ } = await import("zx");
|
|
6691
|
-
let zxLineBuf = "";
|
|
6692
7856
|
const scopedDollar = zx$({
|
|
6693
7857
|
cwd: workDir,
|
|
6694
7858
|
env: { ...process.env },
|
|
6695
|
-
verbose:
|
|
7859
|
+
verbose: true,
|
|
6696
7860
|
quiet: false,
|
|
6697
|
-
log: ((
|
|
6698
|
-
if (entry.kind !== "stdout" && entry.kind !== "stderr") return;
|
|
6699
|
-
const text = typeof entry.data === "string" ? entry.data : String(entry.data ?? "");
|
|
6700
|
-
zxLineBuf += text;
|
|
6701
|
-
const lines = zxLineBuf.split("\n");
|
|
6702
|
-
zxLineBuf = lines.pop();
|
|
6703
|
-
for (const line of lines) if (line) evalStreamer.addLine(line);
|
|
6704
|
-
})
|
|
7861
|
+
log: makeStreamingZxLog((line, stream) => evalStreamer.addLine(line, stream))
|
|
6705
7862
|
});
|
|
6706
7863
|
const evalSink = { addLine: (line) => evalStreamer.addLine(line) };
|
|
6707
7864
|
const dynamicJobLogger = {
|
|
@@ -6786,10 +7943,20 @@ var init_job_runner = __esmMin((() => {
|
|
|
6786
7943
|
return new ContainerSandbox({
|
|
6787
7944
|
docker: new Docker(),
|
|
6788
7945
|
image,
|
|
6789
|
-
runnerPath: opts.runnerPath,
|
|
7946
|
+
runnerPath: resolveRunnerBundlePath(opts.runnerPath),
|
|
6790
7947
|
env: opts.env,
|
|
6791
7948
|
keepFailed: this.config.dockerKeepFailed,
|
|
6792
|
-
jobId: opts.jobId
|
|
7949
|
+
jobId: opts.jobId,
|
|
7950
|
+
hardening: {
|
|
7951
|
+
hardened: this.config.sandboxHardened,
|
|
7952
|
+
readonlyRootfs: this.config.sandboxReadonlyRootfs,
|
|
7953
|
+
user: this.config.sandboxUser,
|
|
7954
|
+
pidsLimit: this.config.sandboxPidsLimit,
|
|
7955
|
+
memoryBytes: this.config.sandboxMemoryBytes,
|
|
7956
|
+
nanoCpus: this.config.sandboxNanoCpus,
|
|
7957
|
+
networkMode: this.config.sandboxNetwork === "host" ? "host" : "default",
|
|
7958
|
+
grant: opts.jobConfig.sandboxGrant
|
|
7959
|
+
}
|
|
6793
7960
|
});
|
|
6794
7961
|
}
|
|
6795
7962
|
case "firecracker": return new FirecrackerSandbox({
|
|
@@ -6878,7 +8045,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6878
8045
|
* they are included as a top-level field on the WS message (not nested in data).
|
|
6879
8046
|
*/
|
|
6880
8047
|
sendJobStatus(dispatch, state, data, secretOutputs) {
|
|
6881
|
-
this.
|
|
8048
|
+
this.send({
|
|
6882
8049
|
type: "job.status",
|
|
6883
8050
|
messageId: randomUUID(),
|
|
6884
8051
|
runId: dispatch.runId,
|
|
@@ -6904,7 +8071,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6904
8071
|
const groupId = data?.groupId;
|
|
6905
8072
|
const { secretsAccessed: _s, concurrencyKind: _c, groupId: _g, ...restData } = data ?? {};
|
|
6906
8073
|
const hasRestData = Object.keys(restData).length > 0;
|
|
6907
|
-
this.
|
|
8074
|
+
this.send({
|
|
6908
8075
|
type: "step.status",
|
|
6909
8076
|
messageId: randomUUID(),
|
|
6910
8077
|
runId: dispatch.runId,
|
|
@@ -6930,7 +8097,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6930
8097
|
* Startup sequence:
|
|
6931
8098
|
* 1. Load config
|
|
6932
8099
|
* 2. Create logger
|
|
6933
|
-
* 3. Create JobRunner with send
|
|
8100
|
+
* 3. Create JobRunner with send callbacks
|
|
6934
8101
|
* 4. Create OrchestratorClient with dispatch and cancel handlers
|
|
6935
8102
|
* 5. Add WS log transport (if not scaler-managed)
|
|
6936
8103
|
* 6. Connect OrchestratorClient
|
|
@@ -6944,14 +8111,14 @@ var init_job_runner = __esmMin((() => {
|
|
|
6944
8111
|
*/
|
|
6945
8112
|
init_console_capture();
|
|
6946
8113
|
init_npm_resolver();
|
|
6947
|
-
const AGENT_VERSION = "0.
|
|
6948
|
-
const BUILD_COMMIT = "
|
|
6949
|
-
const SDK_VERSION = "0.
|
|
6950
|
-
const SDK_BUNDLE_HASH = "
|
|
6951
|
-
const SHARED_VERSION = "0.
|
|
6952
|
-
const SHARED_BUNDLE_HASH = "
|
|
6953
|
-
const ENGINE_VERSION = "0.
|
|
6954
|
-
const ENGINE_BUNDLE_HASH = "
|
|
8114
|
+
const AGENT_VERSION = "0.2.0";
|
|
8115
|
+
const BUILD_COMMIT = "15d5e4447";
|
|
8116
|
+
const SDK_VERSION = "0.2.0";
|
|
8117
|
+
const SDK_BUNDLE_HASH = "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
|
|
8118
|
+
const SHARED_VERSION = "0.2.0";
|
|
8119
|
+
const SHARED_BUNDLE_HASH = "f16fa528a8a9ef1bed8df369f0e88efafca8a9954a749005299fdbcdb93fa188";
|
|
8120
|
+
const ENGINE_VERSION = "0.2.0";
|
|
8121
|
+
const ENGINE_BUNDLE_HASH = "c579e54f7d0b0d5b16587a0528a3e66d32f63e2434bb471a93a01366296efd58";
|
|
6955
8122
|
initTelemetry({
|
|
6956
8123
|
serviceName: "kici-agent",
|
|
6957
8124
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
@@ -7021,7 +8188,6 @@ await guardStartup(logger$1, async () => {
|
|
|
7021
8188
|
let client;
|
|
7022
8189
|
const jobRunner = new JobRunner({
|
|
7023
8190
|
send: (msg) => client.send(msg),
|
|
7024
|
-
sendDirect: (msg) => client.sendDirect(msg),
|
|
7025
8191
|
config,
|
|
7026
8192
|
requestUploadUrl: (jobId, cacheType, key) => client.requestUploadUrl(jobId, cacheType, key),
|
|
7027
8193
|
sendUploadComplete: (jobId, cacheType, key) => client.sendUploadComplete(jobId, cacheType, key),
|
|
@@ -7042,6 +8208,7 @@ await guardStartup(logger$1, async () => {
|
|
|
7042
8208
|
},
|
|
7043
8209
|
requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
|
|
7044
8210
|
relayProvenance: (jobId, request) => client.relayProvenance(jobId, request),
|
|
8211
|
+
requestUserArtifact: (jobId, request) => client.requestUserArtifact(jobId, request),
|
|
7045
8212
|
sendStepApproval: (runId, jobId, request) => client.sendStepApproval(runId, jobId, request)
|
|
7046
8213
|
});
|
|
7047
8214
|
/** Build and send an agent.status message with dynamic OS metadata. */
|