@kici-dev/agent 0.1.27 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/ensure-init-runner.d.ts +23 -22
- package/dist/bootstrap/payload-source.d.ts +32 -0
- package/dist/bootstrap/probe-platform.d.ts +21 -0
- package/dist/bootstrap/restage-agent.d.ts +43 -0
- package/dist/bootstrap/run-restage.d.ts +12 -0
- package/dist/bootstrap/s3-payload-source.d.ts +35 -0
- package/dist/bootstrap/ssh-exec.d.ts +14 -0
- package/dist/bootstrap/stage-agent-payload.d.ts +41 -0
- package/dist/checkout/changed-files.d.ts +34 -0
- package/dist/config.d.ts +36 -14
- package/dist/container-ts-loader-hook.js +147710 -0
- package/dist/execution/artifacts/artifact-engine.d.ts +51 -0
- package/dist/execution/dep-installer.d.ts +3 -3
- package/dist/execution/job-runner.d.ts +24 -6
- package/dist/execution/log-streamer.d.ts +17 -2
- package/dist/execution/rule-evaluator.d.ts +1 -12
- package/dist/execution/sandbox/container-hardening.d.ts +80 -0
- package/dist/execution/sandbox/container-sandbox.d.ts +54 -0
- package/dist/execution/sandbox/container-ts-loader-hook.d.ts +26 -0
- package/dist/execution/sandbox/fork-runner.d.ts +14 -0
- package/dist/execution/sandbox/index.d.ts +2 -1
- package/dist/execution/sandbox/ipc-protocol.d.ts +79 -3
- package/dist/execution/sandbox/step-loop.d.ts +4 -0
- package/dist/execution/sandbox/types.d.ts +25 -4
- package/dist/execution/sandbox/workflow-runner.d.ts +111 -5
- package/dist/execution/streaming-zx-log.d.ts +11 -3
- package/dist/execution/tmp-gc.d.ts +22 -7
- package/dist/execution/workflow-loader.d.ts +32 -1
- package/dist/index.js +44 -45
- package/dist/provenance/statement-builder.d.ts +3 -2
- package/dist/server.d.ts +1 -1
- package/dist/server.js +1256 -171
- package/dist/workflow-runner-bundle.js +215393 -0
- package/dist/workflow-runner.js +886 -244
- package/dist/ws/orchestrator-client.d.ts +105 -1
- package/package.json +14 -12
- package/sbom.spdx.json +1090 -1721
package/dist/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
|
|
2
2
|
import { dirname as __cjs_dirname } from "node:path";
|
|
3
3
|
__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
|
|
4
|
-
import { register } from "node:module";
|
|
4
|
+
import { createRequire, register } from "node:module";
|
|
5
5
|
import crypto$1, { createCipheriv, createHash, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
|
6
6
|
import * as os$1 from "node:os";
|
|
7
7
|
import os, { hostname, tmpdir } from "node:os";
|
|
@@ -9,23 +9,25 @@ import { PassThrough, Readable, Transform, Writable } from "node:stream";
|
|
|
9
9
|
import { serve } from "@hono/node-server";
|
|
10
10
|
import { Hono } from "hono";
|
|
11
11
|
import winston from "winston";
|
|
12
|
-
import { RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
12
|
+
import { AgentDeliveryMode, AgentPlatform, RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, kiciTmpBase, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
13
13
|
import { z } from "zod";
|
|
14
14
|
import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
|
|
15
|
-
import { ALLOWED_SYSTEM_VARS, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
|
|
15
|
+
import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
|
|
16
16
|
import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
|
|
17
17
|
import WebSocket from "ws";
|
|
18
18
|
import * as fs$1 from "node:fs";
|
|
19
|
-
import fs, { existsSync } from "node:fs";
|
|
19
|
+
import fs, { createReadStream, createWriteStream, existsSync } from "node:fs";
|
|
20
20
|
import { ZipArchive } from "archiver";
|
|
21
21
|
import path, { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
22
22
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
23
23
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
24
24
|
import { format, promisify } from "node:util";
|
|
25
|
+
import { OTEL_DATA_POINT_TYPE, mapDataPointTypeToWireKind } from "@kici-dev/engine/metrics/metric-kind-compat";
|
|
25
26
|
import { gcStaleTmpDirs } from "@kici-dev/core/tmp-gc";
|
|
26
|
-
import fsPromises, { access, lstat, mkdir,
|
|
27
|
+
import fsPromises, { access, lstat, mkdir, readFile, readdir, realpath, unlink, writeFile } from "node:fs/promises";
|
|
28
|
+
import { makeTempDir } from "@kici-dev/core/tmp";
|
|
27
29
|
import Docker from "dockerode";
|
|
28
|
-
import { buildKiciApi, buildNeedsContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject } from "@kici-dev/sdk";
|
|
30
|
+
import { buildKiciApi, buildNeedsContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject, setJobOutputsMap, setStepOutputsMap, setStepRefMap } from "@kici-dev/sdk";
|
|
29
31
|
import { c, x } from "tar";
|
|
30
32
|
import https from "node:https";
|
|
31
33
|
import http from "node:http";
|
|
@@ -101,10 +103,18 @@ const envDef = defineEnv({
|
|
|
101
103
|
dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
|
|
102
104
|
jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
|
|
103
105
|
backpressureMode: z.enum(["pause", "drop"]).default("pause"),
|
|
106
|
+
agentPayloadDir: z.string().optional(),
|
|
107
|
+
agentCommand: z.string().optional(),
|
|
104
108
|
sandbox: z.string().default("false").transform((s) => s === "true"),
|
|
105
109
|
trustedEnv: z.string().default("false").transform((s) => s === "true"),
|
|
106
110
|
inPlace: z.string().default("false").transform((s) => s === "true"),
|
|
107
111
|
sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
|
|
112
|
+
sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
|
|
113
|
+
sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
|
|
114
|
+
sandboxUser: z.string().optional(),
|
|
115
|
+
sandboxPidsLimit: z.coerce.number().int().positive().default(512),
|
|
116
|
+
sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
|
|
117
|
+
sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
|
|
108
118
|
scalerManaged: z.string().optional().transform((s) => s === "1"),
|
|
109
119
|
scalerIdleTimeoutMs: z.coerce.number().default(5e3),
|
|
110
120
|
scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
|
|
@@ -129,10 +139,18 @@ const envDef = defineEnv({
|
|
|
129
139
|
dockerKeepFailed: "KICI_DOCKER_KEEP_FAILED",
|
|
130
140
|
jobHeartbeatIntervalMs: "KICI_JOB_HEARTBEAT_INTERVAL_MS",
|
|
131
141
|
backpressureMode: "KICI_BACKPRESSURE_MODE",
|
|
142
|
+
agentPayloadDir: "KICI_AGENT_PAYLOAD_DIR",
|
|
143
|
+
agentCommand: "KICI_AGENT_COMMAND",
|
|
132
144
|
sandbox: "KICI_SANDBOX",
|
|
133
145
|
trustedEnv: "KICI_TRUSTED_ENV",
|
|
134
146
|
inPlace: "KICI_IN_PLACE",
|
|
135
147
|
sandboxNetwork: "KICI_SANDBOX_NETWORK",
|
|
148
|
+
sandboxHardened: "KICI_SANDBOX_HARDENED",
|
|
149
|
+
sandboxReadonlyRootfs: "KICI_SANDBOX_READONLY_ROOTFS",
|
|
150
|
+
sandboxUser: "KICI_SANDBOX_USER",
|
|
151
|
+
sandboxPidsLimit: "KICI_SANDBOX_PIDS_LIMIT",
|
|
152
|
+
sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
|
|
153
|
+
sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
|
|
136
154
|
scalerManaged: "KICI_SCALER_MANAGED",
|
|
137
155
|
scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
|
|
138
156
|
scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
|
|
@@ -162,7 +180,13 @@ const envDef = defineEnv({
|
|
|
162
180
|
* - KICI_SANDBOX (default: false) — enable bubblewrap (bwrap) namespace isolation for bare-metal execution
|
|
163
181
|
* - KICI_TRUSTED_ENV (default: false) — trusted fleet-agent profile: pass the ambient host env (minus the agent's own KiCI identity secrets) through to steps
|
|
164
182
|
* - KICI_IN_PLACE (default: false) — in-place no-clone profile: for a file:// source, use the real repo path as workDir and skip the clone (the routed deploy:stg profile)
|
|
165
|
-
* - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — when sandbox=true
|
|
183
|
+
* - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — sandbox network posture for BOTH backends: the bwrap network namespace (when sandbox=true) and the container job network. `host` shares the host network (container backend also binds host /etc/hosts read-only for name resolution); applies to the container backend under the default hardened posture (KICI_SANDBOX_HARDENED=true)
|
|
184
|
+
* - KICI_SANDBOX_HARDENED (default: true) — hardened-by-default job containers (CapDrop ALL, no-new-privileges, cgroup caps, tmpfs /tmp); set false to roll back to the legacy unhardened posture
|
|
185
|
+
* - KICI_SANDBOX_READONLY_ROOTFS (default: false) — opt-in read-only container rootfs (/tmp stays a writable tmpfs)
|
|
186
|
+
* - KICI_SANDBOX_USER (optional) — container user override (uid, uid:gid, or name); honors the image user when unset
|
|
187
|
+
* - KICI_SANDBOX_PIDS_LIMIT (default: 512) — max PIDs in the job container cgroup
|
|
188
|
+
* - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
|
|
189
|
+
* - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
|
|
166
190
|
* - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
|
|
167
191
|
* - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
|
|
168
192
|
* - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
|
|
@@ -171,7 +195,7 @@ const envDef = defineEnv({
|
|
|
171
195
|
*/
|
|
172
196
|
function loadConfig() {
|
|
173
197
|
const data = envDef.parse();
|
|
174
|
-
if (!data.scalerManaged) validateNoReservedLabels(data.labels, "KICI_LABELS");
|
|
198
|
+
if (!data.scalerManaged && !data.agentToken) validateNoReservedLabels(data.labels, "KICI_LABELS");
|
|
175
199
|
validateUnknownKiciVars([...envDef.listKnownEnvVars(), ...LOGGER_ENV_VARS]);
|
|
176
200
|
return {
|
|
177
201
|
...data,
|
|
@@ -302,6 +326,22 @@ var LogBuffer = class extends RingBuffer {
|
|
|
302
326
|
//#region src/ws/orchestrator-client.ts
|
|
303
327
|
const logger$12 = createLogger({ prefix: "orchestrator-client" });
|
|
304
328
|
/**
|
|
329
|
+
* How long to wait for `artifacts.upload.complete.ack` before failing the step.
|
|
330
|
+
*
|
|
331
|
+
* Deliberately shorter than the sandbox's own 30s artifact-request timeout
|
|
332
|
+
* (`CACHE_RESPONSE_TIMEOUT_MS` in the workflow runner) so a lost ack surfaces
|
|
333
|
+
* this specific reason rather than the sandbox's generic "artifact request
|
|
334
|
+
* timed out". The orchestrator's bounded commit retry finishes in well under a
|
|
335
|
+
* second, so the margin costs nothing.
|
|
336
|
+
*/
|
|
337
|
+
const ARTIFACT_COMPLETE_ACK_TIMEOUT_MS = 25e3;
|
|
338
|
+
/**
|
|
339
|
+
* Tail shared by both give-up messages for an unacknowledged commit. The
|
|
340
|
+
* operator must not be sent hunting for a lost artifact that in fact landed:
|
|
341
|
+
* the orchestrator may well have committed it and only the ack was lost.
|
|
342
|
+
*/
|
|
343
|
+
const COMMIT_MAY_HAVE_LANDED = "the artifact may have been committed";
|
|
344
|
+
/**
|
|
305
345
|
* WebSocket client that connects the agent to the customer orchestrator.
|
|
306
346
|
*
|
|
307
347
|
* Handles:
|
|
@@ -323,6 +363,13 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
323
363
|
reconnectTimer = null;
|
|
324
364
|
reconnectAttempts = 0;
|
|
325
365
|
intentionalDisconnect = false;
|
|
366
|
+
/**
|
|
367
|
+
* Agent-facing capabilities advertised by the orchestrator on `register.ack`.
|
|
368
|
+
* Undefined until the first ack arrives, and on a pre-capability orchestrator
|
|
369
|
+
* that never advertises — both cases mean "assume unsupported" for every
|
|
370
|
+
* optional feature gated on it.
|
|
371
|
+
*/
|
|
372
|
+
orchCapabilities;
|
|
326
373
|
pendingLogBatch = [];
|
|
327
374
|
logFlushTimer = null;
|
|
328
375
|
static LOG_BATCH_SIZE = 50;
|
|
@@ -335,6 +382,26 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
335
382
|
pendingApiRequests = /* @__PURE__ */ new Map();
|
|
336
383
|
/** Pending user-cache restore/save requests awaiting orchestrator response. */
|
|
337
384
|
pendingUserCacheRequests = /* @__PURE__ */ new Map();
|
|
385
|
+
/** Pending user-artifact upload/download requests awaiting orchestrator response. */
|
|
386
|
+
pendingUserArtifactRequests = /* @__PURE__ */ new Map();
|
|
387
|
+
/**
|
|
388
|
+
* In-flight `artifacts.upload.complete` frames that are safe to re-send after a
|
|
389
|
+
* reconnect, keyed by their current `messageId`.
|
|
390
|
+
*
|
|
391
|
+
* The commit is idempotent server-side (the storage key is derived server-side,
|
|
392
|
+
* `initMeta` is a re-writable sidecar, and the DB insert is
|
|
393
|
+
* `onConflict.doNothing()`), so a dropped ack means "we did not hear the
|
|
394
|
+
* answer", not "it did not happen". `beginUpload` / `download` are deliberately
|
|
395
|
+
* absent: `beginUpload` mints a presigned PUT and is not idempotent the same
|
|
396
|
+
* way, and `download` is read-only and cheap to fail.
|
|
397
|
+
*/
|
|
398
|
+
resendableCompletes = /* @__PURE__ */ new Map();
|
|
399
|
+
/**
|
|
400
|
+
* Completes taken off `pendingUserArtifactRequests` by the disconnect sweep and
|
|
401
|
+
* parked until the next `register.ack` re-sends them. Their ack timers keep
|
|
402
|
+
* running — parking does not extend the deadline.
|
|
403
|
+
*/
|
|
404
|
+
heldCompletes = /* @__PURE__ */ new Set();
|
|
338
405
|
/**
|
|
339
406
|
* Pending step-approval requests awaiting the orchestrator's resolution.
|
|
340
407
|
* No client-side timeout: the orchestrator owns the (org-/SDK-configured)
|
|
@@ -430,6 +497,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
430
497
|
this.intentionalDisconnect = true;
|
|
431
498
|
this.stopHeartbeat();
|
|
432
499
|
this.cancelReconnect();
|
|
500
|
+
this.rejectHeldCompletes("the agent disconnected");
|
|
433
501
|
this.drainPendingLogBatch();
|
|
434
502
|
if (this.ws) {
|
|
435
503
|
this.ws.close(1e3, "Agent disconnect");
|
|
@@ -721,6 +789,105 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
721
789
|
});
|
|
722
790
|
}
|
|
723
791
|
/**
|
|
792
|
+
* Relay a user-facing artifact request from the sandbox to the orchestrator.
|
|
793
|
+
*
|
|
794
|
+
* Translates the sandbox `artifacts.request` IPC into the matching
|
|
795
|
+
* `artifacts.*` WS message and resolves with the orchestrator's response
|
|
796
|
+
* mapped onto the IPC response shape:
|
|
797
|
+
*
|
|
798
|
+
* - `beginUpload` -> `artifacts.upload.request`, awaits `artifacts.upload.response`.
|
|
799
|
+
* - `completeUpload` -> `artifacts.upload.complete`, awaits
|
|
800
|
+
* `artifacts.upload.complete.ack` when the orchestrator advertises the
|
|
801
|
+
* `artifactCompleteAck` capability, and rejects when the commit failed so
|
|
802
|
+
* the workflow step fails instead of losing the artifact behind a green run.
|
|
803
|
+
* Against an orchestrator that never advertises it, the message stays
|
|
804
|
+
* fire-and-forget and resolves immediately.
|
|
805
|
+
* - `download` -> `artifacts.download.request`, awaits `artifacts.download.response`.
|
|
806
|
+
*
|
|
807
|
+
* Times out after 30 seconds for the round-trip ops.
|
|
808
|
+
*/
|
|
809
|
+
async requestUserArtifact(jobId, request) {
|
|
810
|
+
if (request.op === "completeUpload") {
|
|
811
|
+
const completeId = randomUUID();
|
|
812
|
+
const complete = {
|
|
813
|
+
type: "artifacts.upload.complete",
|
|
814
|
+
messageId: completeId,
|
|
815
|
+
jobId,
|
|
816
|
+
name: request.name,
|
|
817
|
+
sizeBytes: request.sizeBytes,
|
|
818
|
+
sha256: request.sha256,
|
|
819
|
+
storageKey: request.storageKey
|
|
820
|
+
};
|
|
821
|
+
if (!hasOrchAgentCapability(this.orchCapabilities, "artifactCompleteAck")) {
|
|
822
|
+
this.sendDirect(complete);
|
|
823
|
+
return {
|
|
824
|
+
type: "artifacts.response",
|
|
825
|
+
requestId: request.requestId
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
return new Promise((resolve, reject) => {
|
|
829
|
+
const record = {
|
|
830
|
+
messageId: completeId,
|
|
831
|
+
frame: complete,
|
|
832
|
+
attempts: 0,
|
|
833
|
+
pending: {
|
|
834
|
+
resolve: () => {
|
|
835
|
+
clearTimeout(timer);
|
|
836
|
+
resolve({
|
|
837
|
+
type: "artifacts.response",
|
|
838
|
+
requestId: request.requestId
|
|
839
|
+
});
|
|
840
|
+
},
|
|
841
|
+
reject: (err) => {
|
|
842
|
+
clearTimeout(timer);
|
|
843
|
+
reject(err);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
const timer = setTimeout(() => {
|
|
848
|
+
this.forgetComplete(record);
|
|
849
|
+
reject(/* @__PURE__ */ new Error(`Artifact upload-complete ack timed out (${ARTIFACT_COMPLETE_ACK_TIMEOUT_MS}ms)`));
|
|
850
|
+
}, ARTIFACT_COMPLETE_ACK_TIMEOUT_MS);
|
|
851
|
+
this.pendingUserArtifactRequests.set(completeId, record.pending);
|
|
852
|
+
this.resendableCompletes.set(completeId, record);
|
|
853
|
+
this.sendDirect(complete);
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
const messageId = randomUUID();
|
|
857
|
+
return new Promise((resolve, reject) => {
|
|
858
|
+
const timer = setTimeout(() => {
|
|
859
|
+
this.pendingUserArtifactRequests.delete(messageId);
|
|
860
|
+
reject(/* @__PURE__ */ new Error("User-artifact request timed out (30s)"));
|
|
861
|
+
}, 3e4);
|
|
862
|
+
this.pendingUserArtifactRequests.set(messageId, {
|
|
863
|
+
resolve: (response) => {
|
|
864
|
+
clearTimeout(timer);
|
|
865
|
+
resolve({
|
|
866
|
+
...response,
|
|
867
|
+
requestId: request.requestId
|
|
868
|
+
});
|
|
869
|
+
},
|
|
870
|
+
reject: (err) => {
|
|
871
|
+
clearTimeout(timer);
|
|
872
|
+
reject(err);
|
|
873
|
+
}
|
|
874
|
+
});
|
|
875
|
+
if (request.op === "beginUpload") this.sendDirect({
|
|
876
|
+
type: "artifacts.upload.request",
|
|
877
|
+
messageId,
|
|
878
|
+
jobId,
|
|
879
|
+
name: request.name,
|
|
880
|
+
declaredSizeBytes: request.declaredSizeBytes
|
|
881
|
+
});
|
|
882
|
+
else this.sendDirect({
|
|
883
|
+
type: "artifacts.download.request",
|
|
884
|
+
messageId,
|
|
885
|
+
jobId,
|
|
886
|
+
name: request.name
|
|
887
|
+
});
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
/**
|
|
724
891
|
* Relay a provenance bundle upload operation to the orchestrator. Maps the
|
|
725
892
|
* IPC `provenance.request` onto `requestProvenanceUploadUrl` (returns the
|
|
726
893
|
* presigned URL) or `sendProvenanceUploadComplete` (fire-and-forget) and
|
|
@@ -935,6 +1102,17 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
935
1102
|
this.pendingApiRequests.clear();
|
|
936
1103
|
for (const [_id, pending] of this.pendingUserCacheRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
937
1104
|
this.pendingUserCacheRequests.clear();
|
|
1105
|
+
for (const [id, pending] of this.pendingUserArtifactRequests) {
|
|
1106
|
+
const record = this.resendableCompletes.get(id);
|
|
1107
|
+
if (record && !this.intentionalDisconnect) {
|
|
1108
|
+
this.heldCompletes.add(record);
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
1111
|
+
pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
1112
|
+
}
|
|
1113
|
+
this.pendingUserArtifactRequests.clear();
|
|
1114
|
+
this.resendableCompletes.clear();
|
|
1115
|
+
if (this.intentionalDisconnect) this.rejectHeldCompletes("the agent disconnected");
|
|
938
1116
|
for (const [_id, pending] of this.pendingConcurrencyRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
939
1117
|
this.pendingConcurrencyRequests.clear();
|
|
940
1118
|
for (const [_id, pending] of this.pendingStepApprovals) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
@@ -946,6 +1124,132 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
946
1124
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
947
1125
|
});
|
|
948
1126
|
}
|
|
1127
|
+
/**
|
|
1128
|
+
* Resolve a pending user-artifact request from an `artifacts.upload.response`
|
|
1129
|
+
* / `artifacts.download.response` WS message, mapping the wire fields onto the
|
|
1130
|
+
* IPC response shape. No-op when no request is pending for the requestId.
|
|
1131
|
+
*/
|
|
1132
|
+
resolveArtifactResponse(type, raw) {
|
|
1133
|
+
const artMsg = raw;
|
|
1134
|
+
const pending = this.pendingUserArtifactRequests.get(artMsg.requestId);
|
|
1135
|
+
if (!pending) return;
|
|
1136
|
+
this.pendingUserArtifactRequests.delete(artMsg.requestId);
|
|
1137
|
+
const isUpload = type === "artifacts.upload.response";
|
|
1138
|
+
pending.resolve({
|
|
1139
|
+
type: "artifacts.response",
|
|
1140
|
+
requestId: artMsg.requestId,
|
|
1141
|
+
...isUpload && artMsg.outcome !== void 0 && { uploadOutcome: artMsg.outcome },
|
|
1142
|
+
...!isUpload && artMsg.outcome !== void 0 && { downloadOutcome: artMsg.outcome },
|
|
1143
|
+
...artMsg.uploadUrl && { uploadUrl: artMsg.uploadUrl },
|
|
1144
|
+
...artMsg.storageKey && { storageKey: artMsg.storageKey },
|
|
1145
|
+
...artMsg.reason && { reason: artMsg.reason },
|
|
1146
|
+
...artMsg.error && { rejectionDetail: artMsg.error },
|
|
1147
|
+
...artMsg.downloadUrl && { downloadUrl: artMsg.downloadUrl },
|
|
1148
|
+
...artMsg.sizeBytes !== void 0 && { sizeBytes: artMsg.sizeBytes },
|
|
1149
|
+
...artMsg.sha256 && { sha256: artMsg.sha256 }
|
|
1150
|
+
});
|
|
1151
|
+
}
|
|
1152
|
+
/**
|
|
1153
|
+
* Resolve or reject a pending completeUpload from an
|
|
1154
|
+
* `artifacts.upload.complete.ack`. `committed` resolves the relayed IPC
|
|
1155
|
+
* request; `failed` rejects it (carrying the orchestrator's reason), which
|
|
1156
|
+
* propagates through the relay's `response.error` and fails the workflow step
|
|
1157
|
+
* rather than losing the artifact silently. No-op when nothing is pending for
|
|
1158
|
+
* the requestId (an ack for an already-timed-out complete).
|
|
1159
|
+
*/
|
|
1160
|
+
resolveArtifactCompleteAck(raw) {
|
|
1161
|
+
const ack = raw;
|
|
1162
|
+
const pending = this.pendingUserArtifactRequests.get(ack.requestId);
|
|
1163
|
+
if (!pending) return;
|
|
1164
|
+
this.pendingUserArtifactRequests.delete(ack.requestId);
|
|
1165
|
+
this.resendableCompletes.delete(ack.requestId);
|
|
1166
|
+
if (ack.outcome === ArtifactCompleteAckOutcome.enum.committed) pending.resolve({
|
|
1167
|
+
type: "artifacts.response",
|
|
1168
|
+
requestId: ack.requestId
|
|
1169
|
+
});
|
|
1170
|
+
else pending.reject(new Error(ack.reason ?? "artifact upload-complete failed"));
|
|
1171
|
+
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Drop every trace of an in-flight upload-complete, whichever key it currently
|
|
1174
|
+
* lives under and whether or not it is parked awaiting a resend.
|
|
1175
|
+
*
|
|
1176
|
+
* Called from the ack timer, which fires on the original schedule regardless of
|
|
1177
|
+
* how many times the complete has been re-keyed since.
|
|
1178
|
+
*/
|
|
1179
|
+
forgetComplete(record) {
|
|
1180
|
+
this.pendingUserArtifactRequests.delete(record.messageId);
|
|
1181
|
+
this.resendableCompletes.delete(record.messageId);
|
|
1182
|
+
this.heldCompletes.delete(record);
|
|
1183
|
+
}
|
|
1184
|
+
/**
|
|
1185
|
+
* Fail every parked upload-complete closed, for the paths where the reconnect
|
|
1186
|
+
* the park was betting on will never arrive (a deliberate shutdown, a
|
|
1187
|
+
* permanently rejected token). Naming the ambiguity keeps the operator from
|
|
1188
|
+
* hunting for an artifact that in fact landed.
|
|
1189
|
+
*/
|
|
1190
|
+
rejectHeldCompletes(cause) {
|
|
1191
|
+
if (this.heldCompletes.size === 0) return;
|
|
1192
|
+
const held = [...this.heldCompletes];
|
|
1193
|
+
this.heldCompletes.clear();
|
|
1194
|
+
for (const record of held) record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete could not be acknowledged before ${cause} — the artifact may have been committed`));
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Re-send every upload-complete parked by the disconnect sweep.
|
|
1198
|
+
*
|
|
1199
|
+
* Called from `register.ack`, once the new connection's capabilities are known.
|
|
1200
|
+
* Each frame is re-keyed with a fresh `messageId` before it goes out: the id is
|
|
1201
|
+
* the correlation key, so reusing it would let a late ack for the pre-disconnect
|
|
1202
|
+
* send resolve the new pending.
|
|
1203
|
+
*
|
|
1204
|
+
* The ack timer created when the complete was first sent is deliberately left
|
|
1205
|
+
* alone, so the hold plus every resend stays inside the original
|
|
1206
|
+
* ARTIFACT_COMPLETE_ACK_TIMEOUT_MS budget.
|
|
1207
|
+
*/
|
|
1208
|
+
resendHeldCompletes() {
|
|
1209
|
+
if (this.heldCompletes.size === 0) return;
|
|
1210
|
+
const held = [...this.heldCompletes];
|
|
1211
|
+
this.heldCompletes.clear();
|
|
1212
|
+
const canAck = hasOrchAgentCapability(this.orchCapabilities, "artifactCompleteAck");
|
|
1213
|
+
for (const record of held) {
|
|
1214
|
+
if (!canAck) {
|
|
1215
|
+
record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete cannot be acknowledged by the reconnected orchestrator — ${COMMIT_MAY_HAVE_LANDED}`));
|
|
1216
|
+
continue;
|
|
1217
|
+
}
|
|
1218
|
+
if (record.attempts >= 2) {
|
|
1219
|
+
record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete could not be acknowledged after 2 reconnects — ${COMMIT_MAY_HAVE_LANDED}`));
|
|
1220
|
+
continue;
|
|
1221
|
+
}
|
|
1222
|
+
record.messageId = randomUUID();
|
|
1223
|
+
record.attempts += 1;
|
|
1224
|
+
record.frame = {
|
|
1225
|
+
...record.frame,
|
|
1226
|
+
messageId: record.messageId
|
|
1227
|
+
};
|
|
1228
|
+
this.pendingUserArtifactRequests.set(record.messageId, record.pending);
|
|
1229
|
+
this.resendableCompletes.set(record.messageId, record);
|
|
1230
|
+
this.sendDirect(record.frame);
|
|
1231
|
+
logger$12.info("Re-sent artifact upload-complete after reconnect", {
|
|
1232
|
+
messageId: record.messageId,
|
|
1233
|
+
attempt: record.attempts
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Route the orchestrator's replies to a relayed user-artifact request, which
|
|
1239
|
+
* are dispatched off the raw frame rather than the parsed protocol union.
|
|
1240
|
+
* Returns true when the message was consumed.
|
|
1241
|
+
*/
|
|
1242
|
+
handleArtifactReply(type, raw) {
|
|
1243
|
+
if (type === "artifacts.upload.response" || type === "artifacts.download.response") {
|
|
1244
|
+
this.resolveArtifactResponse(type, raw);
|
|
1245
|
+
return true;
|
|
1246
|
+
}
|
|
1247
|
+
if (type === "artifacts.upload.complete.ack") {
|
|
1248
|
+
this.resolveArtifactCompleteAck(raw);
|
|
1249
|
+
return true;
|
|
1250
|
+
}
|
|
1251
|
+
return false;
|
|
1252
|
+
}
|
|
949
1253
|
handleMessage(data) {
|
|
950
1254
|
let raw;
|
|
951
1255
|
try {
|
|
@@ -1003,6 +1307,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1003
1307
|
}
|
|
1004
1308
|
return;
|
|
1005
1309
|
}
|
|
1310
|
+
if (this.handleArtifactReply(rawMsg.type, raw)) return;
|
|
1006
1311
|
const parsed = orchestratorToAgentMessageSchema.safeParse(raw);
|
|
1007
1312
|
if (parsed.success) {
|
|
1008
1313
|
const msg = parsed.data;
|
|
@@ -1031,10 +1336,12 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1031
1336
|
scalerManaged: msg.scalerManaged,
|
|
1032
1337
|
pendingDispatch: msg.pendingDispatch ?? false
|
|
1033
1338
|
});
|
|
1339
|
+
this.orchCapabilities = msg.capabilities;
|
|
1034
1340
|
this._state = "registered";
|
|
1035
1341
|
this.reconnectAttempts = 0;
|
|
1036
1342
|
this.startHeartbeat();
|
|
1037
1343
|
this.flushBuffer();
|
|
1344
|
+
this.resendHeldCompletes();
|
|
1038
1345
|
this.onRegistered?.({ pendingDispatch: msg.pendingDispatch ?? false });
|
|
1039
1346
|
if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
|
|
1040
1347
|
this.sendConfigAck(msg.agentId);
|
|
@@ -1348,14 +1655,14 @@ var init_console_capture = __esmMin((() => {
|
|
|
1348
1655
|
init_console_capture();
|
|
1349
1656
|
function safe(name, fallback = "unknown") {
|
|
1350
1657
|
switch (name) {
|
|
1351
|
-
case "version": return "0.
|
|
1352
|
-
case "buildCommit": return "
|
|
1353
|
-
case "sdkVersion": return "0.
|
|
1354
|
-
case "sdkBundleHash": return "
|
|
1355
|
-
case "sharedVersion": return "0.
|
|
1356
|
-
case "sharedBundleHash": return "
|
|
1357
|
-
case "engineVersion": return "0.
|
|
1358
|
-
case "engineBundleHash": return "
|
|
1658
|
+
case "version": return "0.2.0";
|
|
1659
|
+
case "buildCommit": return "15d5e4447";
|
|
1660
|
+
case "sdkVersion": return "0.2.0";
|
|
1661
|
+
case "sdkBundleHash": return "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
|
|
1662
|
+
case "sharedVersion": return "0.2.0";
|
|
1663
|
+
case "sharedBundleHash": return "f16fa528a8a9ef1bed8df369f0e88efafca8a9954a749005299fdbcdb93fa188";
|
|
1664
|
+
case "engineVersion": return "0.2.0";
|
|
1665
|
+
case "engineBundleHash": return "c579e54f7d0b0d5b16587a0528a3e66d32f63e2434bb471a93a01366296efd58";
|
|
1359
1666
|
default: return fallback;
|
|
1360
1667
|
}
|
|
1361
1668
|
}
|
|
@@ -1406,15 +1713,6 @@ function createHealthRoutes$1(deps) {
|
|
|
1406
1713
|
* The orchestrator aggregates these metrics from all agents and exposes
|
|
1407
1714
|
* them on its `/metrics` endpoint for Prometheus scraping.
|
|
1408
1715
|
*/
|
|
1409
|
-
/**
|
|
1410
|
-
* OTel DataPointType enum values (from @opentelemetry/sdk-metrics).
|
|
1411
|
-
* We duplicate them here to avoid a direct runtime dependency on sdk-metrics.
|
|
1412
|
-
*/
|
|
1413
|
-
const DataPointType = {
|
|
1414
|
-
HISTOGRAM: 0,
|
|
1415
|
-
GAUGE: 2,
|
|
1416
|
-
SUM: 3
|
|
1417
|
-
};
|
|
1418
1716
|
var MetricsReporter = class {
|
|
1419
1717
|
timer;
|
|
1420
1718
|
agentId;
|
|
@@ -1462,7 +1760,7 @@ var MetricsReporter = class {
|
|
|
1462
1760
|
const wireType = this.mapDataPointType(metricData.dataPointType, metricData.isMonotonic);
|
|
1463
1761
|
for (const dp of metricData.dataPoints) {
|
|
1464
1762
|
const labels = this.extractLabels(dp.attributes);
|
|
1465
|
-
if (metricData.dataPointType ===
|
|
1763
|
+
if (metricData.dataPointType === OTEL_DATA_POINT_TYPE.HISTOGRAM) {
|
|
1466
1764
|
const histValue = dp.value;
|
|
1467
1765
|
const boundaries = histValue.buckets.boundaries;
|
|
1468
1766
|
const counts = histValue.buckets.counts;
|
|
@@ -1495,12 +1793,7 @@ var MetricsReporter = class {
|
|
|
1495
1793
|
}
|
|
1496
1794
|
/** Map OTel DataPointType to wire format type string. */
|
|
1497
1795
|
mapDataPointType(dataPointType, isMonotonic) {
|
|
1498
|
-
|
|
1499
|
-
case DataPointType.HISTOGRAM: return "histogram";
|
|
1500
|
-
case DataPointType.GAUGE: return "gauge";
|
|
1501
|
-
case DataPointType.SUM: return isMonotonic === false ? "upDownCounter" : "counter";
|
|
1502
|
-
default: return "gauge";
|
|
1503
|
-
}
|
|
1796
|
+
return mapDataPointTypeToWireKind(dataPointType, isMonotonic);
|
|
1504
1797
|
}
|
|
1505
1798
|
/** Extract string labels from OTel Attributes. */
|
|
1506
1799
|
extractLabels(attributes) {
|
|
@@ -1581,28 +1874,57 @@ init_npm_resolver();
|
|
|
1581
1874
|
/**
|
|
1582
1875
|
* Startup garbage collection for this agent's own temp-directory families.
|
|
1583
1876
|
*
|
|
1584
|
-
*
|
|
1585
|
-
*
|
|
1586
|
-
*
|
|
1587
|
-
*
|
|
1588
|
-
*
|
|
1589
|
-
*
|
|
1590
|
-
*
|
|
1877
|
+
* Bare job workdirs (`kici-<6 random chars>`, see job-runner.ts), labeled
|
|
1878
|
+
* allocator dirs (`kici-<label>-<6 random chars>`, minted by the global temp
|
|
1879
|
+
* allocator `@kici-dev/core/tmp`, which guarantees every allocation carries
|
|
1880
|
+
* the `kici-` prefix), and isolated pnpm stores (`kici-pnpm-store-*`, see
|
|
1881
|
+
* dep-installer.ts) clean themselves up in `finally` blocks — but a hard
|
|
1882
|
+
* process death (SIGKILL, OOM kill) skips those, and on a long-lived
|
|
1883
|
+
* bare-metal agent the leftovers then accumulate forever. Collecting anything
|
|
1884
|
+
* older than a day at startup is safe on shared hosts: no job lives remotely
|
|
1885
|
+
* that long (job timeouts are minutes), so a concurrent agent's in-flight
|
|
1886
|
+
* dirs are never eligible.
|
|
1887
|
+
*
|
|
1888
|
+
* The deterministic persistent caches (`kici-agent-payloads`, `kici-data`,
|
|
1889
|
+
* `kici-scaler-ledger`) also live directly under the temp root and can be far
|
|
1890
|
+
* older than a day, so they are explicitly excluded — `kici-scaler-ledger`
|
|
1891
|
+
* even structurally matches the allocator pattern (`ledger` is a 6-char label
|
|
1892
|
+
* suffix), which a regex alone cannot distinguish.
|
|
1591
1893
|
*/
|
|
1592
1894
|
const AGENT_TMP_GC_MAX_AGE_MS = 1440 * 60 * 1e3;
|
|
1593
|
-
/**
|
|
1594
|
-
|
|
1895
|
+
/**
|
|
1896
|
+
* Bare `kici-<6 chars>` workdirs and labeled `kici-<label>-<6 chars>`
|
|
1897
|
+
* allocator dirs. The optional label group makes both families eligible.
|
|
1898
|
+
*/
|
|
1899
|
+
const AGENT_WORKDIR_PATTERN = /^kici-([a-z0-9-]+-)?[A-Za-z0-9]{6}$/;
|
|
1595
1900
|
const PNPM_STORE_PATTERN = /^kici-pnpm-store-/;
|
|
1596
1901
|
/**
|
|
1902
|
+
* Deterministic persistent caches that share the `kici-` prefix but must
|
|
1903
|
+
* never be collected — `kici-scaler-ledger` even matches the allocator
|
|
1904
|
+
* pattern, so a basename exclude (not a regex) is the only correct guard.
|
|
1905
|
+
*/
|
|
1906
|
+
const PERSISTENT_CACHES = /* @__PURE__ */ new Set([
|
|
1907
|
+
"kici-agent-payloads",
|
|
1908
|
+
"kici-data",
|
|
1909
|
+
"kici-scaler-ledger"
|
|
1910
|
+
]);
|
|
1911
|
+
/**
|
|
1597
1912
|
* Collect this agent's stale temp dirs. `base` is overridable for tests;
|
|
1598
1913
|
* production callers use the default temp root. Never throws.
|
|
1914
|
+
*
|
|
1915
|
+
* The default base is `kiciTmpBase()`, not the OS temp root, so it scans the
|
|
1916
|
+
* exact directory the agent now writes payloads/clones/pnpm stores into: the
|
|
1917
|
+
* global temp allocator and the payload cache both honor `KICI_TMPDIR` via the
|
|
1918
|
+
* same helper. If the GC scanned a different root, stale-temp reaping would
|
|
1919
|
+
* silently stop working whenever `KICI_TMPDIR` is set.
|
|
1599
1920
|
*/
|
|
1600
|
-
async function gcStaleAgentTmpDirs(base =
|
|
1921
|
+
async function gcStaleAgentTmpDirs(base = kiciTmpBase()) {
|
|
1601
1922
|
const log = (m) => logger.info(m);
|
|
1602
1923
|
return [...await gcStaleTmpDirs({
|
|
1603
1924
|
base,
|
|
1604
1925
|
pattern: AGENT_WORKDIR_PATTERN,
|
|
1605
1926
|
maxAgeMs: AGENT_TMP_GC_MAX_AGE_MS,
|
|
1927
|
+
exclude: PERSISTENT_CACHES,
|
|
1606
1928
|
log
|
|
1607
1929
|
}), ...await gcStaleTmpDirs({
|
|
1608
1930
|
base,
|
|
@@ -1758,7 +2080,7 @@ var init_prometheus = __esmMin((() => {
|
|
|
1758
2080
|
*/
|
|
1759
2081
|
async function setupSshAuth(opts) {
|
|
1760
2082
|
if (opts.hostKeyPolicy === "pinned" && !opts.knownHosts) throw new Error("pinned hostKeyPolicy requires knownHosts content");
|
|
1761
|
-
const tempDir = await
|
|
2083
|
+
const { path: tempDir, cleanup } = await makeTempDir("ssh");
|
|
1762
2084
|
const keyPath = join(tempDir, "id");
|
|
1763
2085
|
await writeFile(keyPath, opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`, { mode: 384 });
|
|
1764
2086
|
const knownHostsPath = join(tempDir, "known_hosts");
|
|
@@ -1779,12 +2101,7 @@ async function setupSshAuth(opts) {
|
|
|
1779
2101
|
return {
|
|
1780
2102
|
gitSshCommand: parts.join(" "),
|
|
1781
2103
|
tempDir,
|
|
1782
|
-
|
|
1783
|
-
await rm(tempDir, {
|
|
1784
|
-
recursive: true,
|
|
1785
|
-
force: true
|
|
1786
|
-
});
|
|
1787
|
-
}
|
|
2104
|
+
cleanup
|
|
1788
2105
|
};
|
|
1789
2106
|
}
|
|
1790
2107
|
/**
|
|
@@ -1836,19 +2153,16 @@ async function gitClone(options) {
|
|
|
1836
2153
|
let needsCustomEnv = false;
|
|
1837
2154
|
let safeDirCleanup;
|
|
1838
2155
|
if (repoUrl.startsWith("file://")) {
|
|
1839
|
-
const {
|
|
1840
|
-
const { tmpdir } = await import("node:os");
|
|
2156
|
+
const { writeFile } = await import("node:fs/promises");
|
|
1841
2157
|
const path = await import("node:path");
|
|
1842
|
-
const
|
|
2158
|
+
const { makeTempDir } = await import("@kici-dev/core/tmp");
|
|
2159
|
+
const { path: dir, cleanup } = await makeTempDir("gitcfg");
|
|
1843
2160
|
const cfgPath = path.join(dir, "config");
|
|
1844
2161
|
await writeFile(cfgPath, "[safe]\n directory = *\n", { mode: 384 });
|
|
1845
2162
|
envEntries.GIT_CONFIG_GLOBAL = cfgPath;
|
|
1846
2163
|
needsCustomEnv = true;
|
|
1847
2164
|
safeDirCleanup = async () => {
|
|
1848
|
-
await
|
|
1849
|
-
recursive: true,
|
|
1850
|
-
force: true
|
|
1851
|
-
}).catch(() => {});
|
|
2165
|
+
await cleanup().catch(() => {});
|
|
1852
2166
|
};
|
|
1853
2167
|
}
|
|
1854
2168
|
let sshSetup;
|
|
@@ -1961,11 +2275,46 @@ var workflow_loader_exports = /* @__PURE__ */ __exportAll({
|
|
|
1961
2275
|
extractSteps: () => extractSteps,
|
|
1962
2276
|
extractStepsFromDynamicJob: () => extractStepsFromDynamicJob,
|
|
1963
2277
|
extractWorkflow: () => extractWorkflow,
|
|
1964
|
-
loadWorkflowSource: () => loadWorkflowSource
|
|
2278
|
+
loadWorkflowSource: () => loadWorkflowSource,
|
|
2279
|
+
resolveWorkflowSdkSetters: () => resolveWorkflowSdkSetters
|
|
1965
2280
|
});
|
|
2281
|
+
/**
|
|
2282
|
+
* Resolve the `@kici-dev/sdk` instance the workflow module itself imports.
|
|
2283
|
+
*
|
|
2284
|
+
* The workflow's `.result` proxies read the module-global step-outputs map of
|
|
2285
|
+
* whichever SDK copy the workflow file resolves — which is generally a
|
|
2286
|
+
* different physical module than the agent's bundled SDK (the workflow is
|
|
2287
|
+
* imported from the cloned source tree and resolves its deps against that
|
|
2288
|
+
* tree's `node_modules`). Resolving via `createRequire(workflowFilePath)` walks
|
|
2289
|
+
* `node_modules` from the workflow file exactly the way the workflow's own
|
|
2290
|
+
* `import '@kici-dev/sdk'` does — including any hoisted copy — so the returned
|
|
2291
|
+
* setters mutate the SAME module-global map object the proxies read. Node caches
|
|
2292
|
+
* ESM modules by resolved URL, so importing that path yields the workflow's live
|
|
2293
|
+
* singleton, not a fresh copy.
|
|
2294
|
+
*
|
|
2295
|
+
* Falls back to the agent's bundled setters when resolution fails (mirrors
|
|
2296
|
+
* `resolveSdkSetters` in the compiler's test runner).
|
|
2297
|
+
*/
|
|
2298
|
+
async function resolveWorkflowSdkSetters(workflowFilePath) {
|
|
2299
|
+
try {
|
|
2300
|
+
const sdk = await import(pathToFileURL(createRequire(workflowFilePath).resolve("@kici-dev/sdk")).href);
|
|
2301
|
+
if (typeof sdk.setStepOutputsMap === "function" && typeof sdk.setStepRefMap === "function" && typeof sdk.setJobOutputsMap === "function") return {
|
|
2302
|
+
setStepOutputsMap: sdk.setStepOutputsMap,
|
|
2303
|
+
setStepRefMap: sdk.setStepRefMap,
|
|
2304
|
+
setJobOutputsMap: sdk.setJobOutputsMap
|
|
2305
|
+
};
|
|
2306
|
+
} catch {}
|
|
2307
|
+
return {
|
|
2308
|
+
setStepOutputsMap,
|
|
2309
|
+
setStepRefMap,
|
|
2310
|
+
setJobOutputsMap
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
1966
2313
|
function ensureLoaderHookRegistered() {
|
|
1967
2314
|
if (hookRegistered) return;
|
|
1968
|
-
|
|
2315
|
+
const hookPath = process.env.KICI_TS_LOADER_HOOK_PATH;
|
|
2316
|
+
if (hookPath) register(pathToFileURL(hookPath).href, import.meta.url);
|
|
2317
|
+
else register("@kici-dev/core/ts-loader-hook", import.meta.url);
|
|
1969
2318
|
hookRegistered = true;
|
|
1970
2319
|
}
|
|
1971
2320
|
/**
|
|
@@ -2020,7 +2369,10 @@ async function loadWorkflowSource(workDir, sourceFile, expectedContentHash, reso
|
|
|
2020
2369
|
const actualHash = computeContentHash(rawSource, assetDigest);
|
|
2021
2370
|
if (actualHash !== expectedContentHash) throw new Error(`Lock file is out of date: workflow source changed without regenerating kici.lock.json (expected contentHash ${expectedContentHash}, got ${actualHash}, agent baked @kici-dev/sdk@${AGENT_SDK_VERSION} bundleHash=${AGENT_SDK_BUNDLE_HASH}). Run 'kici compile' and commit the updated lock file.`);
|
|
2022
2371
|
}
|
|
2023
|
-
return {
|
|
2372
|
+
return {
|
|
2373
|
+
module: await import(pathToFileURL(filePath).href + `?t=${Date.now()}`),
|
|
2374
|
+
sdkSetters: await resolveWorkflowSdkSetters(filePath)
|
|
2375
|
+
};
|
|
2024
2376
|
}
|
|
2025
2377
|
/**
|
|
2026
2378
|
* Type guard for Workflow shape (discriminant: `_tag === 'Workflow'`).
|
|
@@ -2118,8 +2470,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
|
|
|
2118
2470
|
}
|
|
2119
2471
|
var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
|
|
2120
2472
|
var init_workflow_loader = __esmMin((() => {
|
|
2121
|
-
AGENT_SDK_VERSION = "0.
|
|
2122
|
-
AGENT_SDK_BUNDLE_HASH = "
|
|
2473
|
+
AGENT_SDK_VERSION = "0.2.0";
|
|
2474
|
+
AGENT_SDK_BUNDLE_HASH = "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
|
|
2123
2475
|
hookRegistered = false;
|
|
2124
2476
|
}));
|
|
2125
2477
|
//#endregion
|
|
@@ -2582,7 +2934,16 @@ async function evaluateDynamicFields(workflow, jobName, event, flags, timeoutMs
|
|
|
2582
2934
|
},
|
|
2583
2935
|
env: { ...process.env }
|
|
2584
2936
|
};
|
|
2585
|
-
|
|
2937
|
+
const resolved = await withTimeout(() => job.matrix(matrixContext), timeoutMs, `dynamicMatrix for job '${jobName}'`);
|
|
2938
|
+
let combos;
|
|
2939
|
+
try {
|
|
2940
|
+
const rawCount = matrixCombinationCount(resolved);
|
|
2941
|
+
if (rawCount > MAX_MATRIX_MATERIALIZATION) throw new MatrixShapeError(`matrix is too large to expand: ${rawCount} raw combinations (max ${MAX_MATRIX_MATERIALIZATION})`);
|
|
2942
|
+
combos = expandMatrix(resolved);
|
|
2943
|
+
} catch (err) {
|
|
2944
|
+
if (err instanceof MatrixShapeError) throw new MatrixShapeError(`dynamicMatrix for job '${jobName}': ${err.message}`);
|
|
2945
|
+
throw err;
|
|
2946
|
+
}
|
|
2586
2947
|
if (job.include || job.exclude) combos = applyIncludeExclude(combos, job.include, job.exclude);
|
|
2587
2948
|
result.matrixValues = combos;
|
|
2588
2949
|
}
|
|
@@ -2665,13 +3026,14 @@ async function sshExec(reach, privateKey, command, opts = {}, deps = {}) {
|
|
|
2665
3026
|
const hostKeyMode = opts.hostKeyMode ?? "accept-new";
|
|
2666
3027
|
const { dest, port } = resolveTarget(reach, opts.port);
|
|
2667
3028
|
return withEphemeralAgent(privateKey, spawnFn, async (env) => {
|
|
2668
|
-
|
|
3029
|
+
const args = [
|
|
2669
3030
|
...baseSshOptions(hostKeyMode),
|
|
2670
3031
|
"-p",
|
|
2671
3032
|
String(port),
|
|
2672
3033
|
dest,
|
|
2673
3034
|
command
|
|
2674
|
-
]
|
|
3035
|
+
];
|
|
3036
|
+
return spawnFn("ssh", args, {
|
|
2675
3037
|
env,
|
|
2676
3038
|
stdin: opts.stdin
|
|
2677
3039
|
});
|
|
@@ -2687,13 +3049,14 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
|
|
|
2687
3049
|
const hostKeyMode = opts.hostKeyMode ?? "accept-new";
|
|
2688
3050
|
const { dest, port } = resolveTarget(reach, opts.port);
|
|
2689
3051
|
const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
|
|
2690
|
-
|
|
3052
|
+
const args = [
|
|
2691
3053
|
...baseSshOptions(hostKeyMode),
|
|
2692
3054
|
"-p",
|
|
2693
3055
|
String(port),
|
|
2694
3056
|
dest,
|
|
2695
3057
|
`cat > '${remotePath.replace(/'/g, `'\\''`)}'`
|
|
2696
|
-
]
|
|
3058
|
+
];
|
|
3059
|
+
return spawnFn("ssh", args, {
|
|
2697
3060
|
env,
|
|
2698
3061
|
stdin: localBytes
|
|
2699
3062
|
});
|
|
@@ -2701,6 +3064,32 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
|
|
|
2701
3064
|
if (result.exitCode !== 0) throw new Error(`sshPush(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
|
|
2702
3065
|
}
|
|
2703
3066
|
/**
|
|
3067
|
+
* Ship a local FILE to a remote path over `scp` (a binary channel), using the
|
|
3068
|
+
* same ephemeral-key discipline as `sshExec`/`sshPush`.
|
|
3069
|
+
*
|
|
3070
|
+
* Unlike `sshPush` — which pipes a JS string through `ssh 'cat > path'` and so
|
|
3071
|
+
* corrupts binary payloads (a ~50 MB tarball) — `scp` streams the file's raw
|
|
3072
|
+
* bytes untouched. This is the ONLY sanctioned path for a binary payload
|
|
3073
|
+
* transfer; the string `cat >` path is for text (launcher scripts) only. Throws
|
|
3074
|
+
* on a non-zero exit (a payload push must succeed end-to-end).
|
|
3075
|
+
*/
|
|
3076
|
+
async function sshPushFile(reach, privateKey, localFilePath, remotePath, opts = {}, deps = {}) {
|
|
3077
|
+
const spawnFn = deps.spawnFn ?? defaultSpawn;
|
|
3078
|
+
const hostKeyMode = opts.hostKeyMode ?? "accept-new";
|
|
3079
|
+
const { dest, port } = resolveTarget(reach, opts.port);
|
|
3080
|
+
const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
|
|
3081
|
+
const args = [
|
|
3082
|
+
...baseSshOptions(hostKeyMode),
|
|
3083
|
+
"-P",
|
|
3084
|
+
String(port),
|
|
3085
|
+
localFilePath,
|
|
3086
|
+
`${dest}:${remotePath}`
|
|
3087
|
+
];
|
|
3088
|
+
return spawnFn("scp", args, { env });
|
|
3089
|
+
});
|
|
3090
|
+
if (result.exitCode !== 0) throw new Error(`sshPushFile(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
|
|
3091
|
+
}
|
|
3092
|
+
/**
|
|
2704
3093
|
* Start a per-call ephemeral ssh-agent, load the key via stdin (never a file),
|
|
2705
3094
|
* run `body` with `SSH_AUTH_SOCK` in env, and kill the agent in `finally`.
|
|
2706
3095
|
*
|
|
@@ -2716,7 +3105,7 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
|
|
|
2716
3105
|
*/
|
|
2717
3106
|
async function withEphemeralAgent(privateKey, spawnFn, body) {
|
|
2718
3107
|
const baseEnv = { ...process.env };
|
|
2719
|
-
const agentDir = await
|
|
3108
|
+
const { path: agentDir, cleanup } = await makeTempDir("bootstrap-ssh");
|
|
2720
3109
|
const sock = join(agentDir, "agent.sock");
|
|
2721
3110
|
try {
|
|
2722
3111
|
const start = await spawnFn("ssh-agent", [
|
|
@@ -2729,10 +3118,11 @@ async function withEphemeralAgent(privateKey, spawnFn, body) {
|
|
|
2729
3118
|
const agentEnv = {
|
|
2730
3119
|
...baseEnv,
|
|
2731
3120
|
SSH_AUTH_SOCK: sock,
|
|
2732
|
-
...pid ? { SSH_AGENT_PID: pid } : {},
|
|
2733
3121
|
SSH_ASKPASS: "/bin/false",
|
|
2734
3122
|
DISPLAY: ""
|
|
2735
3123
|
};
|
|
3124
|
+
if (pid) agentEnv.SSH_AGENT_PID = pid;
|
|
3125
|
+
else delete agentEnv.SSH_AGENT_PID;
|
|
2736
3126
|
try {
|
|
2737
3127
|
const add = await spawnFn("ssh-add", ["-"], {
|
|
2738
3128
|
env: agentEnv,
|
|
@@ -2744,10 +3134,7 @@ async function withEphemeralAgent(privateKey, spawnFn, body) {
|
|
|
2744
3134
|
await spawnFn("ssh-agent", ["-k"], { env: agentEnv }).catch(() => {});
|
|
2745
3135
|
}
|
|
2746
3136
|
} finally {
|
|
2747
|
-
await
|
|
2748
|
-
recursive: true,
|
|
2749
|
-
force: true
|
|
2750
|
-
}).catch(() => {});
|
|
3137
|
+
await cleanup().catch(() => {});
|
|
2751
3138
|
}
|
|
2752
3139
|
}
|
|
2753
3140
|
/** Extract `SSH_AGENT_PID=<n>;` from `ssh-agent -s` output (best-effort). */
|
|
@@ -2779,7 +3166,10 @@ var init_ssh_exec = __esmMin((() => {
|
|
|
2779
3166
|
stdout,
|
|
2780
3167
|
stderr
|
|
2781
3168
|
}));
|
|
2782
|
-
if (opts.stdin !== void 0)
|
|
3169
|
+
if (opts.stdin !== void 0) {
|
|
3170
|
+
child.stdin?.on("error", () => {});
|
|
3171
|
+
child.stdin?.end(opts.stdin);
|
|
3172
|
+
}
|
|
2783
3173
|
});
|
|
2784
3174
|
SSH_USER_DEFAULT = "root";
|
|
2785
3175
|
SSH_PORT_DEFAULT = 22;
|
|
@@ -2789,8 +3179,166 @@ var init_ssh_exec = __esmMin((() => {
|
|
|
2789
3179
|
};
|
|
2790
3180
|
}));
|
|
2791
3181
|
//#endregion
|
|
3182
|
+
//#region src/bootstrap/probe-platform.ts
|
|
3183
|
+
/**
|
|
3184
|
+
* Probe a bring-up target's platform over SSH.
|
|
3185
|
+
*
|
|
3186
|
+
* A stock rescue box has nothing but sshd — no `kici-agent`, no Node — so we
|
|
3187
|
+
* cannot ask the box what to run; we must ask what it IS, then stage the
|
|
3188
|
+
* matching self-contained payload. One `uname -s -m` (+ a libc check) maps to
|
|
3189
|
+
* an `AgentPlatform`; an unsupported target (musl-only, non-Linux, unknown
|
|
3190
|
+
* arch) throws early, naming what was detected, rather than staging bytes that
|
|
3191
|
+
* won't boot.
|
|
3192
|
+
*/
|
|
3193
|
+
/**
|
|
3194
|
+
* Map `uname -s -m` (+ libc line) output to an `AgentPlatform`. Pure + directly
|
|
3195
|
+
* unit-tested. Throws a self-describing error for any unsupported target.
|
|
3196
|
+
*/
|
|
3197
|
+
function parseUname(stdout) {
|
|
3198
|
+
const lines = stdout.split("\n");
|
|
3199
|
+
const unameLine = (lines[0] ?? "").trim();
|
|
3200
|
+
const libc = lines.slice(1).join("\n").toLowerCase();
|
|
3201
|
+
const [os, arch] = unameLine.split(/\s+/);
|
|
3202
|
+
if (os !== "Linux") throw new Error(`unsupported bring-up target OS "${os}" (uname: "${unameLine}") — Linux only`);
|
|
3203
|
+
if (libc.includes("musl") || libc.includes("ld-musl")) throw new Error(`unsupported bring-up target: musl libc detected (uname: "${unameLine}") — glibc Linux only`);
|
|
3204
|
+
if (arch === "x86_64") return AgentPlatform.enum["linux-x64"];
|
|
3205
|
+
if (arch === "aarch64") return AgentPlatform.enum["linux-arm64"];
|
|
3206
|
+
throw new Error(`unsupported bring-up target arch "${arch}" (uname: "${unameLine}") — x86_64 or aarch64 only`);
|
|
3207
|
+
}
|
|
3208
|
+
/** SSH into `reach` and detect its `AgentPlatform` (one probe, fail-fast on unsupported). */
|
|
3209
|
+
async function probeTargetPlatform(reach, privateKey, deps = {}) {
|
|
3210
|
+
const res = await sshExec(reach, privateKey, PROBE_COMMAND, {}, deps);
|
|
3211
|
+
if (res.exitCode !== 0) throw new Error(`platform probe on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3212
|
+
return parseUname(res.stdout);
|
|
3213
|
+
}
|
|
3214
|
+
var PROBE_COMMAND;
|
|
3215
|
+
var init_probe_platform = __esmMin((() => {
|
|
3216
|
+
init_ssh_exec();
|
|
3217
|
+
PROBE_COMMAND = "uname -s -m; (ldd --version 2>&1 | head -1) || true";
|
|
3218
|
+
}));
|
|
3219
|
+
//#endregion
|
|
3220
|
+
//#region src/bootstrap/stage-agent-payload.ts
|
|
3221
|
+
/**
|
|
3222
|
+
* Stage a self-contained agent payload onto a bring-up target.
|
|
3223
|
+
*
|
|
3224
|
+
* The reusable core mechanism behind fresh-box bring-up: resolve a version-keyed
|
|
3225
|
+
* payload from an `AgentPayloadSource`, verify it, deliver it to the box, verify
|
|
3226
|
+
* it again ON the box before extracting, and return the launcher path. The
|
|
3227
|
+
* launcher boots the agent on its VENDORED Node, so a stock rescue box (no Node)
|
|
3228
|
+
* becomes a connected agent.
|
|
3229
|
+
*
|
|
3230
|
+
* Two delivery modes: `ssh-push` (the ops agent pulls the payload from its
|
|
3231
|
+
* source, then streams it to the box over a binary-safe scp) and `s3-direct`
|
|
3232
|
+
* (the box pulls a presigned URL itself, so no 50 MB transits the ops agent).
|
|
3233
|
+
* Every path is fail-closed: a hash it cannot verify is never extracted.
|
|
3234
|
+
*/
|
|
3235
|
+
/** Streamed sha256 of a local file (the default ops-agent-side verify). */
|
|
3236
|
+
function hashFileSha256(filePath) {
|
|
3237
|
+
return new Promise((resolve, reject) => {
|
|
3238
|
+
const hash = createHash("sha256");
|
|
3239
|
+
createReadStream(filePath).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve(hash.digest("hex")));
|
|
3240
|
+
});
|
|
3241
|
+
}
|
|
3242
|
+
/** Single-quote a value for safe embedding in a remote shell command. */
|
|
3243
|
+
function shQuote$2(v) {
|
|
3244
|
+
return `'${v.replace(/'/g, `'\\''`)}'`;
|
|
3245
|
+
}
|
|
3246
|
+
/**
|
|
3247
|
+
* Verify the resolved payload on the ops agent BEFORE any transfer. A payload
|
|
3248
|
+
* with no sidecar hash is unverifiable and refused (fail-closed); a hash
|
|
3249
|
+
* mismatch throws so corrupt/tampered bytes never leave this process.
|
|
3250
|
+
*/
|
|
3251
|
+
async function verifyLocalPayload(tarballPath, expectedSha256, deps) {
|
|
3252
|
+
if (!expectedSha256) throw new Error(`refusing to stage ${tarballPath}: no sha256 sidecar — cannot verify the payload (run \`kici-admin agent package\` to (re)generate it with a hash)`);
|
|
3253
|
+
const actual = await (deps.hashLocalFile ?? hashFileSha256)(tarballPath);
|
|
3254
|
+
if (actual !== expectedSha256) throw new Error(`payload hash mismatch for ${tarballPath}: expected ${expectedSha256}, got ${actual}`);
|
|
3255
|
+
return expectedSha256;
|
|
3256
|
+
}
|
|
3257
|
+
/**
|
|
3258
|
+
* On the box: verify the pushed tarball's hash, then extract it into
|
|
3259
|
+
* `extractDir`. `sha256sum -c` fails the whole `&&` chain if the hash differs,
|
|
3260
|
+
* so extraction only ever runs on verified bytes.
|
|
3261
|
+
*/
|
|
3262
|
+
async function verifyAndExtractRemote(reach, privateKey, sha256, extractDir, deps) {
|
|
3263
|
+
const remoteBase = path.posix.basename(REMOTE_TARBALL_PATH);
|
|
3264
|
+
const res = await sshExec(reach, privateKey, [
|
|
3265
|
+
`cd ${shQuote$2(path.posix.dirname(REMOTE_TARBALL_PATH))}`,
|
|
3266
|
+
`echo ${shQuote$2(`${sha256} ${remoteBase}`)} | sha256sum -c -`,
|
|
3267
|
+
`mkdir -p ${shQuote$2(extractDir)}`,
|
|
3268
|
+
`tar xzf ${shQuote$2(remoteBase)} -C ${shQuote$2(extractDir)}`
|
|
3269
|
+
].join(" && "), {}, deps);
|
|
3270
|
+
if (res.exitCode !== 0) throw new Error(`on-box verify/extract on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3271
|
+
}
|
|
3272
|
+
/** Resolve → verify locally → push → verify+extract on box. */
|
|
3273
|
+
async function stageSshPush(reach, privateKey, opts, extractDir, deps) {
|
|
3274
|
+
if (!deps.payloadSource) throw new Error("ssh-push delivery requires a payload source (KICI_AGENT_BINARY_SOURCE / KICI_AGENT_PAYLOAD_DIR)");
|
|
3275
|
+
const staged = await deps.payloadSource.resolve(opts.platform, opts.version);
|
|
3276
|
+
const sha256 = await verifyLocalPayload(staged.tarballPath, staged.sha256, deps);
|
|
3277
|
+
await sshPushFile(reach, privateKey, staged.tarballPath, REMOTE_TARBALL_PATH, {}, deps);
|
|
3278
|
+
await verifyAndExtractRemote(reach, privateKey, sha256, extractDir, deps);
|
|
3279
|
+
}
|
|
3280
|
+
/**
|
|
3281
|
+
* `s3-direct`: the box pulls the presigned URL itself, verifies the sha256, and
|
|
3282
|
+
* extracts — all in one `sshExec`, so the 50 MB payload never transits the ops
|
|
3283
|
+
* agent. Fail-closed: `sha256sum -c` breaks the `&&` chain on mismatch, so the
|
|
3284
|
+
* box only ever extracts verified bytes, and `curl -f` fails the chain on a
|
|
3285
|
+
* non-2xx (expired/absent presign).
|
|
3286
|
+
*/
|
|
3287
|
+
async function stageS3Direct(reach, privateKey, presignedUrl, sha256, extractDir, deps) {
|
|
3288
|
+
const remoteBase = path.posix.basename(REMOTE_TARBALL_PATH);
|
|
3289
|
+
const res = await sshExec(reach, privateKey, [
|
|
3290
|
+
`cd ${shQuote$2(path.posix.dirname(REMOTE_TARBALL_PATH))}`,
|
|
3291
|
+
`curl -fsSL ${shQuote$2(presignedUrl)} -o ${shQuote$2(remoteBase)}`,
|
|
3292
|
+
`echo ${shQuote$2(`${sha256} ${remoteBase}`)} | sha256sum -c -`,
|
|
3293
|
+
`mkdir -p ${shQuote$2(extractDir)}`,
|
|
3294
|
+
`tar xzf ${shQuote$2(remoteBase)} -C ${shQuote$2(extractDir)}`
|
|
3295
|
+
].join(" && "), {}, deps);
|
|
3296
|
+
if (res.exitCode !== 0) throw new Error(`s3-direct pull/verify/extract on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3297
|
+
}
|
|
3298
|
+
/**
|
|
3299
|
+
* Stage a payload onto `reach` and return the launcher path the caller runs as
|
|
3300
|
+
* the init-runner's `agentCommand`.
|
|
3301
|
+
*/
|
|
3302
|
+
async function stageAgentPayload(reach, privateKey, opts, deps) {
|
|
3303
|
+
const extractDir = deps.extractDir ?? DEFAULT_EXTRACT_DIR;
|
|
3304
|
+
switch (opts.delivery.mode) {
|
|
3305
|
+
case "ssh-push":
|
|
3306
|
+
await stageSshPush(reach, privateKey, opts, extractDir, deps);
|
|
3307
|
+
break;
|
|
3308
|
+
case "s3-direct":
|
|
3309
|
+
await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
|
|
3310
|
+
break;
|
|
3311
|
+
}
|
|
3312
|
+
return { launcherPath: path.posix.join(extractDir, "kici-agent") };
|
|
3313
|
+
}
|
|
3314
|
+
var DEFAULT_EXTRACT_DIR, REMOTE_TARBALL_PATH;
|
|
3315
|
+
var init_stage_agent_payload = __esmMin((() => {
|
|
3316
|
+
init_ssh_exec();
|
|
3317
|
+
DEFAULT_EXTRACT_DIR = "/opt/kici-init";
|
|
3318
|
+
REMOTE_TARBALL_PATH = "/tmp/kici-agent-payload.tar.gz";
|
|
3319
|
+
}));
|
|
3320
|
+
//#endregion
|
|
2792
3321
|
//#region src/bootstrap/ensure-init-runner.ts
|
|
2793
3322
|
/**
|
|
3323
|
+
* Agent-side init-runner bring-up.
|
|
3324
|
+
*
|
|
3325
|
+
* Runs in the AGENT process (never the workflow sandbox), so the bring-up SSH
|
|
3326
|
+
* key and bootstrap token the orchestrator hands back never reach user
|
|
3327
|
+
* workflow code. The flow:
|
|
3328
|
+
*
|
|
3329
|
+
* 1. Call the orchestrator's privileged `kici.ensureInitRunner` handler — it
|
|
3330
|
+
* gates on this agent's `kici:capability:ssh-transport` capability,
|
|
3331
|
+
* resolves the target's reach + SSH key, mints a single-use bootstrap
|
|
3332
|
+
* token, audits, and returns the material (or `{ broughtUp: false }` when
|
|
3333
|
+
* the target already has a live agent).
|
|
3334
|
+
* 2. Over SSH (ephemeral key, never on disk): drop a launcher onto the target
|
|
3335
|
+
* that starts `kici-agent` with the bootstrap env (token + agent id +
|
|
3336
|
+
* orchestrator URL + labels), and start it detached.
|
|
3337
|
+
*
|
|
3338
|
+
* The init-runner then connects → `auth.request` (bootstrap token) →
|
|
3339
|
+
* `agent.register` auto-enroll as a temporary `kici:init` agent.
|
|
3340
|
+
*/
|
|
3341
|
+
/**
|
|
2794
3342
|
* Build the launcher script that starts the init-runner on the target with its
|
|
2795
3343
|
* bootstrap env. Detached (`setsid … &`) so the SSH session can return while
|
|
2796
3344
|
* the agent keeps running and dials the orchestrator.
|
|
@@ -2800,10 +3348,11 @@ function buildLauncher(material, agentCommand) {
|
|
|
2800
3348
|
"#!/usr/bin/env bash",
|
|
2801
3349
|
"set -euo pipefail",
|
|
2802
3350
|
`setsid env ${[
|
|
2803
|
-
`KICI_AGENT_TOKEN=${shQuote(material.bootstrapToken)}`,
|
|
2804
|
-
`KICI_AGENT_ID=${shQuote(material.targetAgentId)}`,
|
|
2805
|
-
`KICI_ORCHESTRATOR_URL=${shQuote(material.orchestratorUrl)}`,
|
|
2806
|
-
`KICI_LABELS=${shQuote(material.labels.join(","))}`,
|
|
3351
|
+
`KICI_AGENT_TOKEN=${shQuote$1(material.bootstrapToken)}`,
|
|
3352
|
+
`KICI_AGENT_ID=${shQuote$1(material.targetAgentId)}`,
|
|
3353
|
+
`KICI_ORCHESTRATOR_URL=${shQuote$1(material.orchestratorUrl)}`,
|
|
3354
|
+
`KICI_LABELS=${shQuote$1(material.labels.join(","))}`,
|
|
3355
|
+
"KICI_ROLES=",
|
|
2807
3356
|
"KICI_EXECUTION_MODE=bare-metal",
|
|
2808
3357
|
"KICI_PORT=0"
|
|
2809
3358
|
].join(" \\\n ")} \\`,
|
|
@@ -2812,10 +3361,60 @@ function buildLauncher(material, agentCommand) {
|
|
|
2812
3361
|
].join("\n");
|
|
2813
3362
|
}
|
|
2814
3363
|
/** Single-quote a value for safe embedding in the launcher's env assignment. */
|
|
2815
|
-
function shQuote(v) {
|
|
3364
|
+
function shQuote$1(v) {
|
|
2816
3365
|
return `'${v.replace(/'/g, `'\\''`)}'`;
|
|
2817
3366
|
}
|
|
2818
3367
|
/**
|
|
3368
|
+
* Resolve the command the launcher invokes to start the init-runner.
|
|
3369
|
+
*
|
|
3370
|
+
* - A `deps.agentCommand` override short-circuits staging (golden image).
|
|
3371
|
+
* - Otherwise a payload source is required: probe the target's platform, stage
|
|
3372
|
+
* the version-keyed self-contained payload over SSH, and return the staged
|
|
3373
|
+
* launcher path (which boots the agent on its vendored Node — no system Node).
|
|
3374
|
+
* - No override AND no source is a misconfiguration, not a silent Node
|
|
3375
|
+
* assumption — throw a clear error.
|
|
3376
|
+
*/
|
|
3377
|
+
async function resolveAgentCommand(transport, reach, privateKey, material, deps) {
|
|
3378
|
+
if (deps.agentCommand) return deps.agentCommand;
|
|
3379
|
+
if (!material.version) throw new Error(`orchestrator returned no version for ${material.targetAgentId} — cannot stage a payload`);
|
|
3380
|
+
if (!(deps.delivery?.mode === AgentDeliveryMode.enum["s3-direct"] || material.deliveryMode === AgentDeliveryMode.enum["s3-direct"]) && !deps.payloadSource) throw new Error(`no agent payload source configured for bringing up ${material.targetAgentId}: set KICI_AGENT_BINARY_SOURCE (object storage), KICI_AGENT_PAYLOAD_DIR (air-gap), or KICI_AGENT_COMMAND (golden image)`);
|
|
3381
|
+
const platform = await probeTargetPlatform(reach, privateKey, deps);
|
|
3382
|
+
const delivery = await resolveDelivery$1(transport, material, platform, deps);
|
|
3383
|
+
const { launcherPath } = await stageAgentPayload(reach, privateKey, {
|
|
3384
|
+
platform,
|
|
3385
|
+
version: material.version,
|
|
3386
|
+
delivery
|
|
3387
|
+
}, {
|
|
3388
|
+
spawnFn: deps.spawnFn,
|
|
3389
|
+
payloadSource: deps.payloadSource,
|
|
3390
|
+
extractDir: deps.extractDir,
|
|
3391
|
+
hashLocalFile: deps.hashLocalFile
|
|
3392
|
+
});
|
|
3393
|
+
return launcherPath;
|
|
3394
|
+
}
|
|
3395
|
+
/**
|
|
3396
|
+
* Decide how the payload reaches the box. An explicit `deps.delivery` override
|
|
3397
|
+
* wins (test/escape hatch); otherwise honor the orchestrator's per-host choice.
|
|
3398
|
+
* For `s3-direct` we ask the orchestrator (which knows the probed platform now)
|
|
3399
|
+
* to mint a box-routable presigned URL via `kici.presignAgentPackage`; a payload
|
|
3400
|
+
* with no sha256 is refused (fail-closed — never extract unverifiable bytes).
|
|
3401
|
+
*/
|
|
3402
|
+
async function resolveDelivery$1(transport, material, platform, deps) {
|
|
3403
|
+
if (deps.delivery) return deps.delivery;
|
|
3404
|
+
if (material.deliveryMode !== AgentDeliveryMode.enum["s3-direct"]) return { mode: "ssh-push" };
|
|
3405
|
+
const presigned = await transport("kici.presignAgentPackage", {
|
|
3406
|
+
targetAgentId: material.targetAgentId,
|
|
3407
|
+
platform
|
|
3408
|
+
});
|
|
3409
|
+
if (!presigned?.url) throw new Error(`orchestrator returned no presigned URL for ${material.targetAgentId}`);
|
|
3410
|
+
if (!presigned.sha256) throw new Error(`refusing s3-direct delivery for ${material.targetAgentId}: no sha256 for the payload (cannot verify on the box)`);
|
|
3411
|
+
return {
|
|
3412
|
+
mode: "s3-direct",
|
|
3413
|
+
presignedUrl: presigned.url,
|
|
3414
|
+
sha256: presigned.sha256
|
|
3415
|
+
};
|
|
3416
|
+
}
|
|
3417
|
+
/**
|
|
2819
3418
|
* Bring up a temporary init-runner on `targetAgentId`. Returns `{ broughtUp }`:
|
|
2820
3419
|
* false when the target already had a live agent (the orchestrator no-op'd),
|
|
2821
3420
|
* true when this call dropped + started the init-runner.
|
|
@@ -2825,7 +3424,7 @@ async function ensureInitRunner(transport, targetAgentId, deps = {}) {
|
|
|
2825
3424
|
if (!material.broughtUp) return { broughtUp: false };
|
|
2826
3425
|
const { reach, privateKey, bootstrapToken, orchestratorUrl, labels } = material;
|
|
2827
3426
|
if (!reach || !privateKey || !bootstrapToken || !orchestratorUrl || !labels) throw new Error(`orchestrator returned incomplete bring-up material for ${targetAgentId}`);
|
|
2828
|
-
const agentCommand =
|
|
3427
|
+
const agentCommand = await resolveAgentCommand(transport, reach, privateKey, material, deps);
|
|
2829
3428
|
await sshPush(reach, privateKey, buildLauncher({
|
|
2830
3429
|
bootstrapToken,
|
|
2831
3430
|
targetAgentId,
|
|
@@ -2836,10 +3435,11 @@ async function ensureInitRunner(transport, targetAgentId, deps = {}) {
|
|
|
2836
3435
|
if (run.exitCode !== 0) throw new Error(`init-runner launch on ${targetAgentId} failed: exit ${run.exitCode}${run.stderr ? `\n${run.stderr}` : ""}`);
|
|
2837
3436
|
return { broughtUp: true };
|
|
2838
3437
|
}
|
|
2839
|
-
var
|
|
3438
|
+
var LAUNCHER_REMOTE_PATH;
|
|
2840
3439
|
var init_ensure_init_runner = __esmMin((() => {
|
|
2841
3440
|
init_ssh_exec();
|
|
2842
|
-
|
|
3441
|
+
init_probe_platform();
|
|
3442
|
+
init_stage_agent_payload();
|
|
2843
3443
|
LAUNCHER_REMOTE_PATH = "/tmp/kici-init-runner.sh";
|
|
2844
3444
|
}));
|
|
2845
3445
|
//#endregion
|
|
@@ -2866,6 +3466,141 @@ var init_pre_boot_send = __esmMin((() => {
|
|
|
2866
3466
|
init_ssh_exec();
|
|
2867
3467
|
}));
|
|
2868
3468
|
//#endregion
|
|
3469
|
+
//#region src/bootstrap/restage-agent.ts
|
|
3470
|
+
/**
|
|
3471
|
+
* External-actor agent re-stage + restart (fleet auto-upgrade apply).
|
|
3472
|
+
*
|
|
3473
|
+
* Runs in the OPS agent (the one holding `kici:capability:ssh-transport`) —
|
|
3474
|
+
* never on the target host itself, so there is NO self-update-handoff: an
|
|
3475
|
+
* external actor swaps the bytes and restarts the target's agent, which
|
|
3476
|
+
* reconnects on its OWN persistent token (this function never mints or handles
|
|
3477
|
+
* a token). The install is folder-anchored (mirroring the versioned-upgrade
|
|
3478
|
+
* layout): the target version is staged into `<installDir>/kici-agent-<version>`
|
|
3479
|
+
* and an atomic symlink swap makes it current, so a stage failure aborts BEFORE
|
|
3480
|
+
* the swap and never leaves a half-upgraded install. Idempotent — a box already
|
|
3481
|
+
* on the target version is a no-op.
|
|
3482
|
+
*/
|
|
3483
|
+
/** Single-quote a value for safe embedding in a remote shell command. */
|
|
3484
|
+
function shQuote(v) {
|
|
3485
|
+
return `'${v.replace(/'/g, `'\\''`)}'`;
|
|
3486
|
+
}
|
|
3487
|
+
/** The versioned payload directory for a version under the install base. */
|
|
3488
|
+
function versionDir(installDir, version) {
|
|
3489
|
+
return path.posix.join(installDir, `kici-agent-${version}`);
|
|
3490
|
+
}
|
|
3491
|
+
/**
|
|
3492
|
+
* Read the version the `kici-agent` symlink currently points at, or null when
|
|
3493
|
+
* no install is present. The symlink target is `kici-agent-<version>`.
|
|
3494
|
+
*/
|
|
3495
|
+
async function readCurrentVersion(reach, privateKey, installDir, deps) {
|
|
3496
|
+
const target = (await sshExec(reach, privateKey, `readlink ${shQuote(path.posix.join(installDir, "kici-agent"))} 2>/dev/null || true`, {}, deps)).stdout.trim();
|
|
3497
|
+
const match = /(?:^|\/)kici-agent-(.+)$/.exec(target);
|
|
3498
|
+
return match ? match[1] : null;
|
|
3499
|
+
}
|
|
3500
|
+
/**
|
|
3501
|
+
* Atomically point `<installDir>/kici-agent` at the target version directory.
|
|
3502
|
+
* `ln -sfn` onto a temp name + `mv -T` is a single rename, so a reader never
|
|
3503
|
+
* sees a missing symlink. The target is stored relative so the tree is movable.
|
|
3504
|
+
*/
|
|
3505
|
+
async function swapInstall(reach, privateKey, installDir, version, deps) {
|
|
3506
|
+
const link = path.posix.join(installDir, "kici-agent");
|
|
3507
|
+
const tmp = `${link}.swap`;
|
|
3508
|
+
const res = await sshExec(reach, privateKey, [`ln -sfn ${shQuote(`kici-agent-${version}`)} ${shQuote(tmp)}`, `mv -T ${shQuote(tmp)} ${shQuote(link)}`].join(" && "), {}, deps);
|
|
3509
|
+
if (res.exitCode !== 0) throw new Error(`install swap on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
|
|
3510
|
+
}
|
|
3511
|
+
/** Run the drain (stop) then the restart (start) over SSH. */
|
|
3512
|
+
async function drainAndRestart(reach, privateKey, restart, deps) {
|
|
3513
|
+
const stopRes = await sshExec(reach, privateKey, restart.stop, {}, deps);
|
|
3514
|
+
if (stopRes.exitCode !== 0) throw new Error(`agent drain on ${reach.agentId} failed: exit ${stopRes.exitCode}${stopRes.stderr ? `\n${stopRes.stderr}` : ""}`);
|
|
3515
|
+
const startRes = await sshExec(reach, privateKey, restart.start, {}, deps);
|
|
3516
|
+
if (startRes.exitCode !== 0) throw new Error(`agent restart on ${reach.agentId} failed: exit ${startRes.exitCode}${startRes.stderr ? `\n${startRes.stderr}` : ""}`);
|
|
3517
|
+
}
|
|
3518
|
+
/**
|
|
3519
|
+
* Re-stage the target version onto `reach` and restart its agent. Returns
|
|
3520
|
+
* `{ restaged: false }` when the host is already on the target version (no
|
|
3521
|
+
* stage, no swap, no restart), `{ restaged: true }` when this call swapped it.
|
|
3522
|
+
*/
|
|
3523
|
+
async function restageAgent(reach, privateKey, opts, deps) {
|
|
3524
|
+
const installDir = opts.installDir ?? DEFAULT_INSTALL_DIR;
|
|
3525
|
+
if (await readCurrentVersion(reach, privateKey, installDir, deps) === opts.version) return { restaged: false };
|
|
3526
|
+
await stageAgentPayload(reach, privateKey, {
|
|
3527
|
+
platform: opts.platform ?? await probeTargetPlatform(reach, privateKey, deps),
|
|
3528
|
+
version: opts.version,
|
|
3529
|
+
delivery: opts.delivery
|
|
3530
|
+
}, {
|
|
3531
|
+
spawnFn: deps.spawnFn,
|
|
3532
|
+
payloadSource: deps.payloadSource,
|
|
3533
|
+
extractDir: versionDir(installDir, opts.version),
|
|
3534
|
+
hashLocalFile: deps.hashLocalFile
|
|
3535
|
+
});
|
|
3536
|
+
await swapInstall(reach, privateKey, installDir, opts.version, deps);
|
|
3537
|
+
await drainAndRestart(reach, privateKey, opts.restart, deps);
|
|
3538
|
+
return { restaged: true };
|
|
3539
|
+
}
|
|
3540
|
+
var DEFAULT_INSTALL_DIR;
|
|
3541
|
+
var init_restage_agent = __esmMin((() => {
|
|
3542
|
+
init_ssh_exec();
|
|
3543
|
+
init_probe_platform();
|
|
3544
|
+
init_stage_agent_payload();
|
|
3545
|
+
DEFAULT_INSTALL_DIR = "/opt/kici-agent";
|
|
3546
|
+
}));
|
|
3547
|
+
//#endregion
|
|
3548
|
+
//#region src/bootstrap/run-restage.ts
|
|
3549
|
+
/**
|
|
3550
|
+
* Agent-process driver for the `kici.restageAgent` fleet-upgrade apply.
|
|
3551
|
+
*
|
|
3552
|
+
* Intercepted in the ops agent (like `ensureInitRunner`): the privileged
|
|
3553
|
+
* resolve — capability gate, availability gate, reach + SSH key, restart spec —
|
|
3554
|
+
* runs on the orchestrator; this driver performs the SSH transport (probe →
|
|
3555
|
+
* stage → swap → restart) so the bring-up key never reaches user workflow code.
|
|
3556
|
+
* The re-staged permanent agent reconnects on its own persistent credential, so
|
|
3557
|
+
* no token is minted or handled here (no self-update-handoff).
|
|
3558
|
+
*/
|
|
3559
|
+
/**
|
|
3560
|
+
* Resolve the delivery mode for the re-stage. `ssh-push` needs a local payload
|
|
3561
|
+
* source; `s3-direct` asks the orchestrator to mint a box-routable presigned URL
|
|
3562
|
+
* (keyed by the now-probed platform) and refuses a payload with no sha256
|
|
3563
|
+
* (fail-closed — never extract unverifiable bytes on the box).
|
|
3564
|
+
*/
|
|
3565
|
+
async function resolveDelivery(transport, targetAgentId, deliveryMode, platform) {
|
|
3566
|
+
if (deliveryMode !== AgentDeliveryMode.enum["s3-direct"]) return { mode: "ssh-push" };
|
|
3567
|
+
const presigned = await transport("kici.presignAgentPackage", {
|
|
3568
|
+
targetAgentId,
|
|
3569
|
+
platform
|
|
3570
|
+
});
|
|
3571
|
+
if (!presigned?.url) throw new Error(`orchestrator returned no presigned URL for re-staging ${targetAgentId}`);
|
|
3572
|
+
if (!presigned.sha256) throw new Error(`refusing s3-direct re-stage for ${targetAgentId}: no sha256 for the payload (cannot verify on the box)`);
|
|
3573
|
+
return {
|
|
3574
|
+
mode: "s3-direct",
|
|
3575
|
+
presignedUrl: presigned.url,
|
|
3576
|
+
sha256: presigned.sha256
|
|
3577
|
+
};
|
|
3578
|
+
}
|
|
3579
|
+
/**
|
|
3580
|
+
* Drive one external-actor re-stage: fetch the material, probe the target
|
|
3581
|
+
* platform, resolve delivery, then stage + swap + restart via {@link restageAgent}.
|
|
3582
|
+
* Returns `{ restaged: false }` when the host is already on the target version.
|
|
3583
|
+
*/
|
|
3584
|
+
async function runRestage(transport, targetAgentId, deps) {
|
|
3585
|
+
const { reach, privateKey, version, deliveryMode, restart } = await transport("kici.restageAgent", { targetAgentId });
|
|
3586
|
+
if (!reach || !privateKey || !version || !restart) throw new Error(`orchestrator returned incomplete re-stage material for ${targetAgentId}`);
|
|
3587
|
+
const platform = await probeTargetPlatform(reach, privateKey, deps);
|
|
3588
|
+
return restageAgent(reach, privateKey, {
|
|
3589
|
+
platform,
|
|
3590
|
+
version,
|
|
3591
|
+
delivery: await resolveDelivery(transport, targetAgentId, deliveryMode, platform),
|
|
3592
|
+
...restart.installDir ? { installDir: restart.installDir } : {},
|
|
3593
|
+
restart: {
|
|
3594
|
+
stop: restart.stop,
|
|
3595
|
+
start: restart.start
|
|
3596
|
+
}
|
|
3597
|
+
}, deps);
|
|
3598
|
+
}
|
|
3599
|
+
var init_run_restage = __esmMin((() => {
|
|
3600
|
+
init_probe_platform();
|
|
3601
|
+
init_restage_agent();
|
|
3602
|
+
}));
|
|
3603
|
+
//#endregion
|
|
2869
3604
|
//#region src/bootstrap/api-intercept.ts
|
|
2870
3605
|
/**
|
|
2871
3606
|
* Wrap the orchestrator API transport so the two bootstrap methods are handled
|
|
@@ -2884,15 +3619,131 @@ function withBootstrapInterception(relay, deps = {}) {
|
|
|
2884
3619
|
}, deps);
|
|
2885
3620
|
return;
|
|
2886
3621
|
}
|
|
3622
|
+
if (method === RESTAGE_AGENT) return runRestage(relay, String(params.targetAgentId ?? ""), deps);
|
|
2887
3623
|
return relay(method, params);
|
|
2888
3624
|
};
|
|
2889
3625
|
}
|
|
2890
|
-
var ENSURE_INIT_RUNNER, PRE_BOOT_SEND;
|
|
3626
|
+
var ENSURE_INIT_RUNNER, PRE_BOOT_SEND, RESTAGE_AGENT;
|
|
2891
3627
|
var init_api_intercept = __esmMin((() => {
|
|
2892
3628
|
init_ensure_init_runner();
|
|
2893
3629
|
init_pre_boot_send();
|
|
3630
|
+
init_run_restage();
|
|
2894
3631
|
ENSURE_INIT_RUNNER = "kici.ensureInitRunner";
|
|
2895
3632
|
PRE_BOOT_SEND = "kici.preBootSend";
|
|
3633
|
+
RESTAGE_AGENT = "kici.restageAgent";
|
|
3634
|
+
}));
|
|
3635
|
+
//#endregion
|
|
3636
|
+
//#region src/bootstrap/payload-source.ts
|
|
3637
|
+
/**
|
|
3638
|
+
* Agent payload source abstraction.
|
|
3639
|
+
*
|
|
3640
|
+
* A one-method plug so the place a self-contained agent+Node payload comes from
|
|
3641
|
+
* is swappable and overridable: the local-dir fallback here reads the tarballs
|
|
3642
|
+
* a `kici-admin agent package` producer wrote to disk; an S3-backed source
|
|
3643
|
+
* (fetching from the orchestrator cache bucket via presigned URL) plugs in the
|
|
3644
|
+
* same interface. Payloads are version-keyed and fail-fast — a missing version
|
|
3645
|
+
* throws loudly rather than silently staging stale bytes.
|
|
3646
|
+
*/
|
|
3647
|
+
/** The payload tarball name for a platform (matches the producer's layout). */
|
|
3648
|
+
function tarballName$1(platform) {
|
|
3649
|
+
return `kici-agent-${platform}.tar.gz`;
|
|
3650
|
+
}
|
|
3651
|
+
/** Extract the leading hex token from a `sha256sum`-style `<hex> <name>` line. */
|
|
3652
|
+
function parseSidecarHash(contents) {
|
|
3653
|
+
const first = contents.trim().split(/\s+/)[0];
|
|
3654
|
+
if (!first) throw new Error("empty .sha256 sidecar");
|
|
3655
|
+
return first;
|
|
3656
|
+
}
|
|
3657
|
+
var defaultPayloadFs, LocalDirPayloadSource;
|
|
3658
|
+
var init_payload_source = __esmMin((() => {
|
|
3659
|
+
defaultPayloadFs = {
|
|
3660
|
+
exists: async (filePath) => {
|
|
3661
|
+
try {
|
|
3662
|
+
await access(filePath);
|
|
3663
|
+
return true;
|
|
3664
|
+
} catch {
|
|
3665
|
+
return false;
|
|
3666
|
+
}
|
|
3667
|
+
},
|
|
3668
|
+
readFile: (filePath) => readFile(filePath, "utf8")
|
|
3669
|
+
};
|
|
3670
|
+
LocalDirPayloadSource = class {
|
|
3671
|
+
baseDir;
|
|
3672
|
+
fs;
|
|
3673
|
+
constructor(baseDir, fs = defaultPayloadFs) {
|
|
3674
|
+
this.baseDir = baseDir;
|
|
3675
|
+
this.fs = fs;
|
|
3676
|
+
}
|
|
3677
|
+
async resolve(platform, version) {
|
|
3678
|
+
const tarballPath = path.join(this.baseDir, version, tarballName$1(platform));
|
|
3679
|
+
if (!await this.fs.exists(tarballPath)) throw new Error(`no agent payload for version ${version} (${platform}) at ${tarballPath} — run \`kici-admin agent package --platform ${platform} --out ${this.baseDir}\``);
|
|
3680
|
+
const sidecarPath = `${tarballPath}.sha256`;
|
|
3681
|
+
return {
|
|
3682
|
+
tarballPath,
|
|
3683
|
+
sha256: await this.fs.exists(sidecarPath) ? parseSidecarHash(await this.fs.readFile(sidecarPath)) : null
|
|
3684
|
+
};
|
|
3685
|
+
}
|
|
3686
|
+
};
|
|
3687
|
+
}));
|
|
3688
|
+
//#endregion
|
|
3689
|
+
//#region src/bootstrap/s3-payload-source.ts
|
|
3690
|
+
/**
|
|
3691
|
+
* Object-storage agent payload source — the ops agent's PRIMARY source.
|
|
3692
|
+
*
|
|
3693
|
+
* Resolves a version-keyed self-contained agent+Node payload from the
|
|
3694
|
+
* orchestrator's own cache bucket via a presigned GET URL the orchestrator
|
|
3695
|
+
* mints (`kici.presignAgentPackage`), fetches it once into a local cache dir,
|
|
3696
|
+
* and returns it for the SSH-push delivery path. No standing S3 credential ever
|
|
3697
|
+
* reaches the ops agent — only a time-limited presigned URL. Payloads are
|
|
3698
|
+
* version-keyed and fail-fast: a missing object throws loudly (names the
|
|
3699
|
+
* version + platform) rather than staging stale bytes.
|
|
3700
|
+
*
|
|
3701
|
+
* For the `s3-direct` delivery path the box pulls the presigned URL itself, so
|
|
3702
|
+
* this source is not involved there; it backs `ssh-push` (box cannot reach
|
|
3703
|
+
* object storage). `LocalDirPayloadSource` remains the air-gap fallback.
|
|
3704
|
+
*/
|
|
3705
|
+
/** The payload tarball name for a platform (matches the producer's layout). */
|
|
3706
|
+
function tarballName(platform) {
|
|
3707
|
+
return `kici-agent-${platform}.tar.gz`;
|
|
3708
|
+
}
|
|
3709
|
+
/** Default cache root the ops agent stores pulled payloads under (cache-once). */
|
|
3710
|
+
function defaultPayloadCacheDir() {
|
|
3711
|
+
return path.join(kiciTmpBase(), "kici-agent-payloads");
|
|
3712
|
+
}
|
|
3713
|
+
/** Default existence check for the cache-once short-circuit. */
|
|
3714
|
+
async function payloadFileExists(filePath) {
|
|
3715
|
+
try {
|
|
3716
|
+
await access(filePath);
|
|
3717
|
+
return true;
|
|
3718
|
+
} catch {
|
|
3719
|
+
return false;
|
|
3720
|
+
}
|
|
3721
|
+
}
|
|
3722
|
+
/** Stream a (presigned) URL into a local file, binary-safe (no in-memory buffer). */
|
|
3723
|
+
async function fetchUrlToFile(url, dest) {
|
|
3724
|
+
await mkdir(path.dirname(dest), { recursive: true });
|
|
3725
|
+
const res = await fetch(url);
|
|
3726
|
+
if (!res.ok || !res.body) throw new Error(`fetch payload failed: HTTP ${res.status} ${res.statusText}`);
|
|
3727
|
+
await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
|
|
3728
|
+
}
|
|
3729
|
+
var S3PayloadSource;
|
|
3730
|
+
var init_s3_payload_source = __esmMin((() => {
|
|
3731
|
+
S3PayloadSource = class {
|
|
3732
|
+
deps;
|
|
3733
|
+
constructor(deps) {
|
|
3734
|
+
this.deps = deps;
|
|
3735
|
+
}
|
|
3736
|
+
async resolve(platform, version) {
|
|
3737
|
+
const presigned = await this.deps.presign(platform, version);
|
|
3738
|
+
if (!presigned) throw new Error(`no agent payload for version ${version} (${platform}) in the orchestrator cache bucket — run \`kici-admin agent package --platform ${platform} --upload\` to publish it`);
|
|
3739
|
+
const tarballPath = path.join(this.deps.cacheDir, version, tarballName(platform));
|
|
3740
|
+
if (!(this.deps.exists ? await this.deps.exists(tarballPath) : false)) await this.deps.fetchToFile(presigned.url, tarballPath);
|
|
3741
|
+
return {
|
|
3742
|
+
tarballPath,
|
|
3743
|
+
sha256: presigned.sha256
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
};
|
|
2896
3747
|
}));
|
|
2897
3748
|
//#endregion
|
|
2898
3749
|
//#region src/execution/streaming-zx-log.ts
|
|
@@ -2921,20 +3772,30 @@ var init_api_intercept = __esmMin((() => {
|
|
|
2921
3772
|
* `verbose: false` (suppressed). A `verbose: false` base would flag ordinary
|
|
2922
3773
|
* output `verbose: false` too, and this gate would then drop every line.
|
|
2923
3774
|
*
|
|
2924
|
-
* The returned callback owns
|
|
2925
|
-
* coalesced into whole lines before `emit` is called.
|
|
3775
|
+
* The returned callback owns one line buffer PER STREAM, so partial chunks are
|
|
3776
|
+
* coalesced into whole lines before `emit` is called. The buffers are separate
|
|
3777
|
+
* because the two streams are independent pipes: a stdout chunk ending mid-line
|
|
3778
|
+
* and a stderr chunk arriving next would otherwise concatenate into a single
|
|
3779
|
+
* spliced line attributed to whichever kind completed it.
|
|
3780
|
+
*
|
|
3781
|
+
* `emit` receives the originating stream alongside the line so a diagnostic
|
|
3782
|
+
* written to stderr stays distinguishable from ordinary progress output all the
|
|
3783
|
+
* way to the persisted run log.
|
|
2926
3784
|
*/
|
|
2927
3785
|
function makeStreamingZxLog(emit) {
|
|
2928
|
-
|
|
3786
|
+
const lineBufs = {
|
|
3787
|
+
[LogStream.enum.stdout]: "",
|
|
3788
|
+
[LogStream.enum.stderr]: ""
|
|
3789
|
+
};
|
|
2929
3790
|
return (entry) => {
|
|
2930
3791
|
const e = entry;
|
|
2931
3792
|
if (e.kind !== "stdout" && e.kind !== "stderr") return;
|
|
2932
3793
|
if (!e.verbose) return;
|
|
3794
|
+
const stream = e.kind === "stderr" ? LogStream.enum.stderr : LogStream.enum.stdout;
|
|
2933
3795
|
const text = typeof e.data === "string" ? e.data : String(e.data ?? "");
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
for (const line of lines) if (line) emit(line);
|
|
3796
|
+
const lines = (lineBufs[stream] + text).split("\n");
|
|
3797
|
+
lineBufs[stream] = lines.pop();
|
|
3798
|
+
for (const line of lines) if (line) emit(line, stream);
|
|
2938
3799
|
};
|
|
2939
3800
|
}
|
|
2940
3801
|
var init_streaming_zx_log = __esmMin((() => {}));
|
|
@@ -3163,6 +4024,8 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3163
4024
|
flushTimer = null;
|
|
3164
4025
|
totalBytes = 0;
|
|
3165
4026
|
truncated = false;
|
|
4027
|
+
/** Which stream the currently-buffered lines came from. */
|
|
4028
|
+
bufferStream = LogStream.enum.stdout;
|
|
3166
4029
|
/** Number of lines dropped due to backpressure (drop mode). */
|
|
3167
4030
|
droppedCount = 0;
|
|
3168
4031
|
/** Whether we are currently in a backpressured state (pause mode). */
|
|
@@ -3202,9 +4065,24 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3202
4065
|
/**
|
|
3203
4066
|
* Add a line to the buffer. Triggers flush if threshold reached,
|
|
3204
4067
|
* otherwise schedules a timer-based flush.
|
|
4068
|
+
*
|
|
4069
|
+
* A chunk carries a single stream, so a kind flip closes the pending chunk
|
|
4070
|
+
* before the new line is buffered. Chunks are delivered in order, so the
|
|
4071
|
+
* stdout/stderr interleaving is preserved across that boundary.
|
|
4072
|
+
*
|
|
4073
|
+
* `bufferStream` is only advanced once the buffer is actually empty. Under
|
|
4074
|
+
* pause-mode backpressure `flush()` deliberately leaves the buffer intact
|
|
4075
|
+
* until the socket drains, so the flip cannot close the pending chunk;
|
|
4076
|
+
* advancing the tag there would relabel the already-buffered lines as the
|
|
4077
|
+
* newly-arrived stream. The buffer keeps the tag of its first lines instead,
|
|
4078
|
+
* which means a chunk assembled while backpressured reports one stream for
|
|
4079
|
+
* lines that came from both — stderr in such a chunk is under-reported as
|
|
4080
|
+
* stdout. Separating them needs a queue of pending per-stream chunks.
|
|
3205
4081
|
*/
|
|
3206
|
-
addLine(line) {
|
|
4082
|
+
addLine(line, stream = LogStream.enum.stdout) {
|
|
3207
4083
|
if (this.truncated) return;
|
|
4084
|
+
if (this.buffer.length > 0 && stream !== this.bufferStream) this.flush();
|
|
4085
|
+
if (this.buffer.length === 0) this.bufferStream = stream;
|
|
3208
4086
|
if (this.totalBytes >= this.maxLogSizeBytes) {
|
|
3209
4087
|
this.buffer.push(`[TRUNCATED: log output exceeded ${this.maxLogSizeBytes} bytes]`);
|
|
3210
4088
|
this.truncated = true;
|
|
@@ -3259,7 +4137,8 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3259
4137
|
jobId: this.jobId,
|
|
3260
4138
|
stepIndex: this.stepIndex,
|
|
3261
4139
|
lines,
|
|
3262
|
-
timestamp: Date.now()
|
|
4140
|
+
timestamp: Date.now(),
|
|
4141
|
+
stream: this.bufferStream
|
|
3263
4142
|
});
|
|
3264
4143
|
}
|
|
3265
4144
|
/**
|
|
@@ -3321,7 +4200,8 @@ var init_log_streamer = __esmMin((() => {
|
|
|
3321
4200
|
jobId: this.jobId,
|
|
3322
4201
|
stepIndex: this.stepIndex,
|
|
3323
4202
|
lines,
|
|
3324
|
-
timestamp: Date.now()
|
|
4203
|
+
timestamp: Date.now(),
|
|
4204
|
+
stream: this.bufferStream
|
|
3325
4205
|
});
|
|
3326
4206
|
}
|
|
3327
4207
|
/**
|
|
@@ -3416,7 +4296,7 @@ function decryptBuffer(encrypted, aesKey) {
|
|
|
3416
4296
|
*/
|
|
3417
4297
|
async function applyOverlay(config) {
|
|
3418
4298
|
const { tarballUrl, cliPublicKey, orchestratorPrivateKey, repoDir } = config;
|
|
3419
|
-
const tmpDir = await
|
|
4299
|
+
const { path: tmpDir, cleanup } = await makeTempDir("overlay");
|
|
3420
4300
|
try {
|
|
3421
4301
|
logger$7.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
|
|
3422
4302
|
let encryptedData;
|
|
@@ -3490,10 +4370,7 @@ async function applyOverlay(config) {
|
|
|
3490
4370
|
verified: true
|
|
3491
4371
|
};
|
|
3492
4372
|
} finally {
|
|
3493
|
-
await
|
|
3494
|
-
recursive: true,
|
|
3495
|
-
force: true
|
|
3496
|
-
}).catch(() => {});
|
|
4373
|
+
await cleanup().catch(() => {});
|
|
3497
4374
|
}
|
|
3498
4375
|
}
|
|
3499
4376
|
var logger$7, IV_LENGTH$1, AUTH_TAG_LENGTH;
|
|
@@ -3676,7 +4553,7 @@ async function applyYarnrcBerryConfig(args) {
|
|
|
3676
4553
|
const hasPrivateRegistry = registries.length > 0 || Object.keys(installEnvSecrets).length > 0;
|
|
3677
4554
|
const yarnrcPath = join(args.kiciDir, ".yarnrc.yml");
|
|
3678
4555
|
const { raw: original, doc } = await readOriginalYarnrc(yarnrcPath);
|
|
3679
|
-
const cacheFolder = await
|
|
4556
|
+
const { path: cacheFolder, cleanup: cleanupCache } = await makeTempDir("yarn-berry-cache");
|
|
3680
4557
|
const merged = {
|
|
3681
4558
|
...doc,
|
|
3682
4559
|
nodeLinker: "node-modules",
|
|
@@ -3713,10 +4590,7 @@ async function applyYarnrcBerryConfig(args) {
|
|
|
3713
4590
|
if (original === null) await unlink(yarnrcPath).catch(() => {});
|
|
3714
4591
|
else await writeFile(yarnrcPath, original, { encoding: "utf8" });
|
|
3715
4592
|
} catch {}
|
|
3716
|
-
await
|
|
3717
|
-
recursive: true,
|
|
3718
|
-
force: true
|
|
3719
|
-
}).catch(() => {});
|
|
4593
|
+
await cleanupCache().catch(() => {});
|
|
3720
4594
|
};
|
|
3721
4595
|
return {
|
|
3722
4596
|
extraEnv: {
|
|
@@ -4059,9 +4933,9 @@ async function detectKiciYarnFlavor(repoRoot, kiciDir) {
|
|
|
4059
4933
|
* Install `.kici/` dependencies inline with the repo's package manager.
|
|
4060
4934
|
*
|
|
4061
4935
|
* Falls back to this when the dep cache is unavailable or a download fails.
|
|
4062
|
-
* The install runs with an isolated cache/store directory (
|
|
4063
|
-
* `
|
|
4064
|
-
* is removed after installation.
|
|
4936
|
+
* The install runs with an isolated cache/store directory (allocated under the
|
|
4937
|
+
* global temp base, which honors `KICI_TMPDIR`) to prevent cache poisoning
|
|
4938
|
+
* between build jobs; the directory is removed after installation.
|
|
4065
4939
|
*
|
|
4066
4940
|
* If `opts.npmRegistries` / `opts.installEnvSecrets` is provided, a job-scoped
|
|
4067
4941
|
* `.kici/.npmrc` overlay is synthesized for the install, restored in `finally`,
|
|
@@ -4150,7 +5024,7 @@ function envWithNodeOnPath(extraEnv, nodeDir) {
|
|
|
4150
5024
|
/** Run `npm install` in `.kici/` with an isolated cache directory. */
|
|
4151
5025
|
async function runNpmInstall(args) {
|
|
4152
5026
|
const { npmCliPath, nodeExe, nodeDir } = resolveNpm();
|
|
4153
|
-
const cacheDir = await
|
|
5027
|
+
const { path: cacheDir, cleanup } = await makeTempDir("npm-cache");
|
|
4154
5028
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
4155
5029
|
const buildArgs = (...prefix) => {
|
|
4156
5030
|
const a = [
|
|
@@ -4175,10 +5049,7 @@ async function runNpmInstall(args) {
|
|
|
4175
5049
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
4176
5050
|
});
|
|
4177
5051
|
} finally {
|
|
4178
|
-
await
|
|
4179
|
-
recursive: true,
|
|
4180
|
-
force: true
|
|
4181
|
-
}).catch(() => {});
|
|
5052
|
+
await cleanup().catch(() => {});
|
|
4182
5053
|
}
|
|
4183
5054
|
}
|
|
4184
5055
|
/**
|
|
@@ -4192,7 +5063,7 @@ async function runNpmInstall(args) {
|
|
|
4192
5063
|
async function runPnpmInstall(args) {
|
|
4193
5064
|
await assertPnpmAvailable();
|
|
4194
5065
|
const { nodeDir } = resolveNpm();
|
|
4195
|
-
const storeDir = await
|
|
5066
|
+
const { path: storeDir, cleanup } = await makeTempDir("pnpm-store");
|
|
4196
5067
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
4197
5068
|
const argv = [
|
|
4198
5069
|
"install",
|
|
@@ -4212,10 +5083,7 @@ async function runPnpmInstall(args) {
|
|
|
4212
5083
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
4213
5084
|
});
|
|
4214
5085
|
} finally {
|
|
4215
|
-
await
|
|
4216
|
-
recursive: true,
|
|
4217
|
-
force: true
|
|
4218
|
-
}).catch(() => {});
|
|
5086
|
+
await cleanup().catch(() => {});
|
|
4219
5087
|
}
|
|
4220
5088
|
}
|
|
4221
5089
|
/** Pure: argv for `yarn install` with an isolated cache folder. */
|
|
@@ -4241,7 +5109,7 @@ function buildYarnInstallArgs(cacheDir, hasPrivateRegistry) {
|
|
|
4241
5109
|
async function runYarnInstall(args) {
|
|
4242
5110
|
await assertYarnAvailable();
|
|
4243
5111
|
const { nodeDir } = resolveNpm();
|
|
4244
|
-
const cacheDir = await
|
|
5112
|
+
const { path: cacheDir, cleanup } = await makeTempDir("yarn-cache");
|
|
4245
5113
|
const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
|
|
4246
5114
|
const argv = buildYarnInstallArgs(cacheDir, args.hasPrivateRegistry);
|
|
4247
5115
|
try {
|
|
@@ -4253,10 +5121,7 @@ async function runYarnInstall(args) {
|
|
|
4253
5121
|
maxBuffer: INSTALL_MAX_BUFFER
|
|
4254
5122
|
});
|
|
4255
5123
|
} finally {
|
|
4256
|
-
await
|
|
4257
|
-
recursive: true,
|
|
4258
|
-
force: true
|
|
4259
|
-
}).catch(() => {});
|
|
5124
|
+
await cleanup().catch(() => {});
|
|
4260
5125
|
}
|
|
4261
5126
|
}
|
|
4262
5127
|
/** Pure: argv for a berry `yarn install`. Cache + linker live in .yarnrc.yml. */
|
|
@@ -4708,6 +5573,7 @@ function buildRequest(dispatch, workDir) {
|
|
|
4708
5573
|
workflowRepoIdentifier: jobConfig.workflowRepoIdentifier,
|
|
4709
5574
|
hasConcurrencyGroup: jobConfig.hasConcurrencyGroup ?? false,
|
|
4710
5575
|
concurrencyEvaluationTimeoutMs: jobConfig.concurrencyEvaluationTimeoutMs,
|
|
5576
|
+
concurrencyWaitTimeoutMs: dispatch.concurrencyWaitTimeoutMs,
|
|
4711
5577
|
branch: dispatch.ref,
|
|
4712
5578
|
upstreamJobOutputs: dispatch.upstreamJobOutputs,
|
|
4713
5579
|
upstreamJobStatuses: dispatch.upstreamJobStatuses,
|
|
@@ -4775,6 +5641,7 @@ function buildBwrapArgs(workDir, nodeExecPath, networkIsolation = false, runnerP
|
|
|
4775
5641
|
const libIdx = args.indexOf("/lib", args.indexOf("--ro-bind") + 1);
|
|
4776
5642
|
if (libIdx !== -1) args.splice(libIdx + 2, 0, "--ro-bind", "/lib64", "/lib64");
|
|
4777
5643
|
}
|
|
5644
|
+
for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) args.push("--ro-bind", nssFile, nssFile);
|
|
4778
5645
|
const nodeDir = dirname(nodeExecPath);
|
|
4779
5646
|
const nodeInstallRoot = dirname(nodeDir);
|
|
4780
5647
|
if (!nodeInstallRoot.startsWith("/usr") && !nodeInstallRoot.startsWith("/bin") && nodeInstallRoot !== "/" && nodeInstallRoot !== "") args.push("--ro-bind", nodeInstallRoot, nodeInstallRoot);
|
|
@@ -4819,6 +5686,28 @@ function buildBwrapArgs(workDir, nodeExecPath, networkIsolation = false, runnerP
|
|
|
4819
5686
|
return args;
|
|
4820
5687
|
}
|
|
4821
5688
|
/**
|
|
5689
|
+
* Derive the read-only bind path(s) for a `file://` clone source. The workflow
|
|
5690
|
+
* runner clones the repo from inside the sandbox, so a local `file://` URL's
|
|
5691
|
+
* source directory must be exposed read-only or `git clone` fails with
|
|
5692
|
+
* `does not appear to be a git repository`. Returns `[]` for non-`file://`
|
|
5693
|
+
* remotes (https/ssh need no host bind) and for a malformed `file://` URL (the
|
|
5694
|
+
* clone then surfaces the real error rather than being masked here).
|
|
5695
|
+
*
|
|
5696
|
+
* Shared by both sandboxes that clone a local source: the bare-metal (bwrap)
|
|
5697
|
+
* backend threads the result into `buildBwrapArgs`'s `extraReadOnlyBinds`, and
|
|
5698
|
+
* the container backend binds each `<dir>:<dir>:ro` into the job container —
|
|
5699
|
+
* the same clone-source affordance across both isolation models.
|
|
5700
|
+
*/
|
|
5701
|
+
function fileCloneSourceBinds(repoUrl) {
|
|
5702
|
+
if (typeof repoUrl !== "string" || !repoUrl.startsWith("file://")) return [];
|
|
5703
|
+
try {
|
|
5704
|
+
const url = new URL(repoUrl);
|
|
5705
|
+
return url.pathname && url.pathname !== "/" ? [url.pathname] : [];
|
|
5706
|
+
} catch {
|
|
5707
|
+
return [];
|
|
5708
|
+
}
|
|
5709
|
+
}
|
|
5710
|
+
/**
|
|
4822
5711
|
* Spawn the workflow-runner child process. Returns the child handle and a
|
|
4823
5712
|
* boolean indicating whether the spawn produced a usable PID. On Windows
|
|
4824
5713
|
* services (shawl) fork() can fail silently when IPC pipes cannot be set up;
|
|
@@ -4967,6 +5856,24 @@ function relayProvenanceRequest$1(msg, ctx) {
|
|
|
4967
5856
|
error: toErrorMessage(err)
|
|
4968
5857
|
}));
|
|
4969
5858
|
}
|
|
5859
|
+
/** Relay `artifacts.request` and pipe the orchestrator response (or an error
|
|
5860
|
+
* response, or a "not configured" response when the callback isn't wired) back
|
|
5861
|
+
* into the sandbox runner. */
|
|
5862
|
+
function relayArtifactRequest$1(msg, ctx) {
|
|
5863
|
+
if (!ctx.execOptions.onArtifactRequest) {
|
|
5864
|
+
safeSendToChild(ctx.child, {
|
|
5865
|
+
type: "artifacts.response",
|
|
5866
|
+
requestId: msg.requestId,
|
|
5867
|
+
error: "Artifacts not available in this agent configuration"
|
|
5868
|
+
});
|
|
5869
|
+
return;
|
|
5870
|
+
}
|
|
5871
|
+
ctx.execOptions.onArtifactRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
|
|
5872
|
+
type: "artifacts.response",
|
|
5873
|
+
requestId: msg.requestId,
|
|
5874
|
+
error: toErrorMessage(err)
|
|
5875
|
+
}));
|
|
5876
|
+
}
|
|
4970
5877
|
/** Relay `approval.request` and pipe the orchestrator's resolution (or a
|
|
4971
5878
|
* fail-closed reject when the callback isn't wired or the relay throws) back
|
|
4972
5879
|
* into the sandbox runner. */
|
|
@@ -5021,7 +5928,7 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
5021
5928
|
});
|
|
5022
5929
|
return;
|
|
5023
5930
|
case "log.line":
|
|
5024
|
-
ctx.execOptions.onLogLine(msg.stepIndex, msg.line);
|
|
5931
|
+
ctx.execOptions.onLogLine(msg.stepIndex, msg.line, msg.stream);
|
|
5025
5932
|
return;
|
|
5026
5933
|
case "step.start": {
|
|
5027
5934
|
ctx.stepNames.set(msg.stepIndex, msg.stepName);
|
|
@@ -5069,6 +5976,9 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
5069
5976
|
case "cache.request":
|
|
5070
5977
|
relayCacheRequest$1(msg, ctx);
|
|
5071
5978
|
return;
|
|
5979
|
+
case "artifacts.request":
|
|
5980
|
+
relayArtifactRequest$1(msg, ctx);
|
|
5981
|
+
return;
|
|
5072
5982
|
case "provenance.request":
|
|
5073
5983
|
relayProvenanceRequest$1(msg, ctx);
|
|
5074
5984
|
return;
|
|
@@ -5286,14 +6196,7 @@ var init_bare_metal_sandbox = __esmMin((() => {
|
|
|
5286
6196
|
* Execute a job by forking the workflow runner with sanitized environment.
|
|
5287
6197
|
*/
|
|
5288
6198
|
async executeJob(options) {
|
|
5289
|
-
const extraReadOnlyBinds = [];
|
|
5290
|
-
if (this.useBwrap) {
|
|
5291
|
-
const repoUrl = options.dispatch.repoUrl;
|
|
5292
|
-
if (typeof repoUrl === "string" && repoUrl.startsWith("file://")) try {
|
|
5293
|
-
const url = new URL(repoUrl);
|
|
5294
|
-
if (url.pathname) extraReadOnlyBinds.push(url.pathname);
|
|
5295
|
-
} catch {}
|
|
5296
|
-
}
|
|
6199
|
+
const extraReadOnlyBinds = this.useBwrap ? fileCloneSourceBinds(options.dispatch.repoUrl) : [];
|
|
5297
6200
|
this.runner = createForkRunner({
|
|
5298
6201
|
runnerPath: this.runnerPath,
|
|
5299
6202
|
env: this.env,
|
|
@@ -5380,6 +6283,43 @@ var init_firecracker_sandbox = __esmMin((() => {
|
|
|
5380
6283
|
};
|
|
5381
6284
|
}));
|
|
5382
6285
|
//#endregion
|
|
6286
|
+
//#region src/execution/sandbox/container-hardening.ts
|
|
6287
|
+
/**
|
|
6288
|
+
* Build the hardened HostConfig fragment for a job sandbox container.
|
|
6289
|
+
*
|
|
6290
|
+
* Pure and side-effect-free: given resolved inputs it returns the fields the
|
|
6291
|
+
* container-sandbox merges into `docker.createContainer`. When `hardened` is
|
|
6292
|
+
* false it returns an empty posture (the rollback affordance).
|
|
6293
|
+
*/
|
|
6294
|
+
function buildContainerHardening(opts) {
|
|
6295
|
+
if (!opts.hardened) return { hostConfig: {} };
|
|
6296
|
+
const grant = opts.grant;
|
|
6297
|
+
const hostConfig = {
|
|
6298
|
+
CapDrop: [...CAP_DROP_ALL],
|
|
6299
|
+
SecurityOpt: [...NO_NEW_PRIVILEGES],
|
|
6300
|
+
PidsLimit: opts.pidsLimit,
|
|
6301
|
+
Memory: opts.memoryBytes,
|
|
6302
|
+
MemorySwap: opts.memoryBytes,
|
|
6303
|
+
NanoCpus: opts.nanoCpus,
|
|
6304
|
+
Tmpfs: { "/tmp": "rw,exec,nosuid,nodev" }
|
|
6305
|
+
};
|
|
6306
|
+
if (grant?.capabilities && grant.capabilities.length > 0) hostConfig.CapAdd = [...grant.capabilities];
|
|
6307
|
+
if (grant?.readonlyRootfs ?? opts.readonlyRootfs) hostConfig.ReadonlyRootfs = true;
|
|
6308
|
+
const network = grant?.network ?? opts.networkMode;
|
|
6309
|
+
if (network === "none") hostConfig.NetworkMode = "none";
|
|
6310
|
+
else if (network === "host") hostConfig.NetworkMode = "host";
|
|
6311
|
+
const user = grant?.user ?? opts.user;
|
|
6312
|
+
return user ? {
|
|
6313
|
+
hostConfig,
|
|
6314
|
+
user
|
|
6315
|
+
} : { hostConfig };
|
|
6316
|
+
}
|
|
6317
|
+
var CAP_DROP_ALL, NO_NEW_PRIVILEGES;
|
|
6318
|
+
var init_container_hardening = __esmMin((() => {
|
|
6319
|
+
CAP_DROP_ALL = ["ALL"];
|
|
6320
|
+
NO_NEW_PRIVILEGES = ["no-new-privileges"];
|
|
6321
|
+
}));
|
|
6322
|
+
//#endregion
|
|
5383
6323
|
//#region src/execution/sandbox/container-sandbox.ts
|
|
5384
6324
|
/**
|
|
5385
6325
|
* Container execution sandbox implementation.
|
|
@@ -5522,6 +6462,32 @@ function relayProvenanceRequest(stream, options, provMsg) {
|
|
|
5522
6462
|
}));
|
|
5523
6463
|
}
|
|
5524
6464
|
/**
|
|
6465
|
+
* Relay artifacts.request from the container runner to the orchestrator via
|
|
6466
|
+
* options.onArtifactRequest, then write the response back through `stream`. If
|
|
6467
|
+
* the agent doesn't expose an artifacts relay, write a structured error so the
|
|
6468
|
+
* runner doesn't hang.
|
|
6469
|
+
*/
|
|
6470
|
+
function relayArtifactRequest(stream, options, artMsg) {
|
|
6471
|
+
const writeResponse = (response) => {
|
|
6472
|
+
try {
|
|
6473
|
+
stream.write(JSON.stringify(response) + "\n");
|
|
6474
|
+
} catch {}
|
|
6475
|
+
};
|
|
6476
|
+
if (!options.onArtifactRequest) {
|
|
6477
|
+
writeResponse({
|
|
6478
|
+
type: "artifacts.response",
|
|
6479
|
+
requestId: artMsg.requestId,
|
|
6480
|
+
error: "Artifacts not available in this agent configuration"
|
|
6481
|
+
});
|
|
6482
|
+
return;
|
|
6483
|
+
}
|
|
6484
|
+
options.onArtifactRequest(artMsg).then((response) => writeResponse(response), (err) => writeResponse({
|
|
6485
|
+
type: "artifacts.response",
|
|
6486
|
+
requestId: artMsg.requestId,
|
|
6487
|
+
error: toErrorMessage(err)
|
|
6488
|
+
}));
|
|
6489
|
+
}
|
|
6490
|
+
/**
|
|
5525
6491
|
* Relay approval.request from the container runner to the orchestrator via
|
|
5526
6492
|
* options.onApprovalRequest, then write the resolution back through `stream`.
|
|
5527
6493
|
* If the agent doesn't expose an approval relay (or it throws), write a
|
|
@@ -5565,22 +6531,30 @@ function applyJobComplete(msg, stepResults, state, options) {
|
|
|
5565
6531
|
logger$3.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
|
|
5566
6532
|
}
|
|
5567
6533
|
}
|
|
5568
|
-
var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, ContainerSandbox;
|
|
6534
|
+
var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, HOOK_MOUNT_PATH, INTERNAL_ENV, ContainerSandbox;
|
|
5569
6535
|
var init_container_sandbox = __esmMin((() => {
|
|
5570
6536
|
init_fork_runner();
|
|
5571
6537
|
init_secret_encryption();
|
|
6538
|
+
init_container_hardening();
|
|
5572
6539
|
logger$3 = createLogger({ prefix: "container-sandbox" });
|
|
5573
6540
|
MAX_STDERR_LINES = 20;
|
|
5574
6541
|
ABORT_GRACE_MS = 1e4;
|
|
5575
6542
|
CONTAINER_STOP_TIMEOUT = 10;
|
|
6543
|
+
HOOK_MOUNT_PATH = "/opt/kici/ts-loader-hook.js";
|
|
6544
|
+
INTERNAL_ENV = [`KICI_TS_LOADER_HOOK_PATH=${HOOK_MOUNT_PATH}`, "KICI_LOG_STDERR=1"];
|
|
5576
6545
|
ContainerSandbox = class {
|
|
5577
6546
|
docker;
|
|
5578
6547
|
image;
|
|
5579
6548
|
runnerPath;
|
|
5580
6549
|
runnerMountPath;
|
|
6550
|
+
/** Host path to the pure-JS container loader-hook bundle (bind-mounted :ro). */
|
|
6551
|
+
hookHostPath;
|
|
5581
6552
|
env;
|
|
5582
6553
|
keepFailed;
|
|
5583
6554
|
jobId;
|
|
6555
|
+
hardening;
|
|
6556
|
+
/** Resolved container user (image-user override / grant), applied to createContainer + each exec. */
|
|
6557
|
+
resolvedUser;
|
|
5584
6558
|
/** The running container instance (set during setup). */
|
|
5585
6559
|
container = null;
|
|
5586
6560
|
/** The active exec stream (set during executeJob, used for abort). */
|
|
@@ -5594,29 +6568,66 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5594
6568
|
this.image = options.image;
|
|
5595
6569
|
this.runnerPath = options.runnerPath;
|
|
5596
6570
|
this.runnerMountPath = options.runnerMountPath ?? "/opt/kici/workflow-runner.js";
|
|
6571
|
+
this.hookHostPath = options.hookPath ?? join(dirname(options.runnerPath), "container-ts-loader-hook.js");
|
|
5597
6572
|
this.env = options.env;
|
|
5598
6573
|
this.keepFailed = options.keepFailed ?? false;
|
|
5599
6574
|
this.jobId = options.jobId ?? `unknown-${Date.now()}`;
|
|
6575
|
+
this.hardening = options.hardening;
|
|
6576
|
+
}
|
|
6577
|
+
/**
|
|
6578
|
+
* Ensure the job's container image is present locally, pulling it on demand
|
|
6579
|
+
* when it is not.
|
|
6580
|
+
*
|
|
6581
|
+
* dockerode's `createContainer` — unlike `docker run` / `podman run` — never
|
|
6582
|
+
* auto-pulls a missing image; it fails with `(HTTP code 404) ... No such
|
|
6583
|
+
* image`. A bare-metal executor that aggressively prunes unused images under
|
|
6584
|
+
* disk pressure can leave a container job with nothing to run, so the agent
|
|
6585
|
+
* pulls the image itself. Already-present images (the common case, and how
|
|
6586
|
+
* private images pre-pulled with registry auth stay working) skip the pull.
|
|
6587
|
+
*/
|
|
6588
|
+
async ensureImagePresent() {
|
|
6589
|
+
try {
|
|
6590
|
+
await this.docker.getImage(this.image).inspect();
|
|
6591
|
+
return;
|
|
6592
|
+
} catch {}
|
|
6593
|
+
logger$3.info("Pulling sandbox image (not present locally)", { image: this.image });
|
|
6594
|
+
const stream = await this.docker.pull(this.image);
|
|
6595
|
+
await new Promise((resolve, reject) => {
|
|
6596
|
+
this.docker.modem.followProgress(stream, (err) => err ? reject(err) : resolve());
|
|
6597
|
+
});
|
|
6598
|
+
logger$3.info("Sandbox image pulled", { image: this.image });
|
|
5600
6599
|
}
|
|
5601
6600
|
async setup(options) {
|
|
5602
6601
|
this.containerName = `kici-sandbox-${this.jobId}-${Date.now()}`;
|
|
5603
|
-
const envArray = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
|
|
6602
|
+
const envArray = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
|
|
5604
6603
|
logger$3.info("Creating sandbox container", {
|
|
5605
6604
|
name: this.containerName,
|
|
5606
6605
|
image: this.image,
|
|
5607
6606
|
workDir: options.workDir
|
|
5608
6607
|
});
|
|
6608
|
+
const hardened = this.hardening ? buildContainerHardening(this.hardening) : {
|
|
6609
|
+
hostConfig: {},
|
|
6610
|
+
user: void 0
|
|
6611
|
+
};
|
|
6612
|
+
this.resolvedUser = hardened.user;
|
|
6613
|
+
const binds = this.buildBinds(options, hardened.hostConfig);
|
|
6614
|
+
await this.ensureImagePresent();
|
|
5609
6615
|
this.container = await this.docker.createContainer({
|
|
5610
6616
|
Image: this.image,
|
|
5611
6617
|
name: this.containerName,
|
|
5612
6618
|
Cmd: ["sleep", "infinity"],
|
|
5613
6619
|
Env: envArray,
|
|
5614
6620
|
WorkingDir: "/workspace",
|
|
6621
|
+
Volumes: { "/workspace": {} },
|
|
6622
|
+
...hardened.user ? { User: hardened.user } : {},
|
|
5615
6623
|
Labels: {
|
|
5616
6624
|
"kici-sandbox": "true",
|
|
5617
6625
|
"kici-job-id": this.jobId
|
|
5618
6626
|
},
|
|
5619
|
-
HostConfig: {
|
|
6627
|
+
HostConfig: {
|
|
6628
|
+
Binds: binds,
|
|
6629
|
+
...hardened.hostConfig
|
|
6630
|
+
}
|
|
5620
6631
|
});
|
|
5621
6632
|
await this.container.start();
|
|
5622
6633
|
logger$3.info("Sandbox container started", {
|
|
@@ -5624,6 +6635,36 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5624
6635
|
containerId: this.container.id.slice(0, 12)
|
|
5625
6636
|
});
|
|
5626
6637
|
}
|
|
6638
|
+
/**
|
|
6639
|
+
* Build the container's read-only bind list.
|
|
6640
|
+
*
|
|
6641
|
+
* The workspace is NOT bound here — it is a container-owned anonymous volume
|
|
6642
|
+
* (`Volumes: { '/workspace': {} }` on the container config), so the container
|
|
6643
|
+
* user can write it on every runtime with `CapDrop: ['ALL']` intact. This
|
|
6644
|
+
* method binds the workflow runner (read-only) plus two parity affordances
|
|
6645
|
+
* that mirror the bare-metal bwrap sandbox — both strictly additive and gated,
|
|
6646
|
+
* so the default posture for a production (https-source, bridge-network) job is
|
|
6647
|
+
* unchanged:
|
|
6648
|
+
*
|
|
6649
|
+
* - **`file://` clone-source dir(s)** (`options.extraReadOnlyBinds`): the
|
|
6650
|
+
* workflow runner clones the repo from inside the container, so a local
|
|
6651
|
+
* `file://` source dir must be exposed read-only or `git clone` fails.
|
|
6652
|
+
* Empty for https/ssh remotes — mirrors fork-runner's `extraReadOnlyBinds`.
|
|
6653
|
+
* - **Host name-resolution files under host networking**: when the effective
|
|
6654
|
+
* network posture is `host` (`KICI_SANDBOX_NETWORK=host`, or a per-job host
|
|
6655
|
+
* grant), bind the host's `/etc/hosts` (+ `/etc/nsswitch.conf`) read-only so
|
|
6656
|
+
* an `/etc/hosts`-only name the host resolves — e.g. a private registry —
|
|
6657
|
+
* resolves inside the container too. Mirrors fork-runner's
|
|
6658
|
+
* `--ro-bind /etc/hosts` for the bwrap host-network mode.
|
|
6659
|
+
*/
|
|
6660
|
+
buildBinds(options, hostConfig) {
|
|
6661
|
+
const binds = [`${this.runnerPath}:${this.runnerMountPath}:ro`, `${this.hookHostPath}:${HOOK_MOUNT_PATH}:ro`];
|
|
6662
|
+
for (const dir of options.extraReadOnlyBinds ?? []) if (dir) binds.push(`${dir}:${dir}:ro`);
|
|
6663
|
+
if (hostConfig.NetworkMode === "host") {
|
|
6664
|
+
for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) binds.push(`${nssFile}:${nssFile}:ro`);
|
|
6665
|
+
}
|
|
6666
|
+
return binds;
|
|
6667
|
+
}
|
|
5627
6668
|
async executeJob(options) {
|
|
5628
6669
|
if (!this.container) throw new Error("ContainerSandbox.executeJob() called before setup()");
|
|
5629
6670
|
const startTime = Date.now();
|
|
@@ -5655,14 +6696,15 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5655
6696
|
* stderr lines for crash diagnostics, and install the abort listener.
|
|
5656
6697
|
*/
|
|
5657
6698
|
async attachExecStream(options) {
|
|
5658
|
-
const execEnv = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
|
|
6699
|
+
const execEnv = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
|
|
5659
6700
|
const stream = await (await this.container.exec({
|
|
5660
6701
|
Cmd: ["node", this.runnerMountPath],
|
|
5661
6702
|
AttachStdin: true,
|
|
5662
6703
|
AttachStdout: true,
|
|
5663
6704
|
AttachStderr: true,
|
|
5664
6705
|
Env: execEnv,
|
|
5665
|
-
WorkingDir: "/workspace"
|
|
6706
|
+
WorkingDir: "/workspace",
|
|
6707
|
+
...this.resolvedUser ? { User: this.resolvedUser } : {}
|
|
5666
6708
|
})).start({
|
|
5667
6709
|
hijack: true,
|
|
5668
6710
|
stdin: true
|
|
@@ -5793,7 +6835,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5793
6835
|
return false;
|
|
5794
6836
|
}
|
|
5795
6837
|
case "log.line":
|
|
5796
|
-
options.onLogLine(msg.stepIndex, msg.line);
|
|
6838
|
+
options.onLogLine(msg.stepIndex, msg.line, msg.stream);
|
|
5797
6839
|
return false;
|
|
5798
6840
|
case "step.secret_mount":
|
|
5799
6841
|
options.onSecretMount?.({
|
|
@@ -5816,6 +6858,9 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5816
6858
|
case "cache.request":
|
|
5817
6859
|
relayCacheRequest(stream, options, msg);
|
|
5818
6860
|
return false;
|
|
6861
|
+
case "artifacts.request":
|
|
6862
|
+
relayArtifactRequest(stream, options, msg);
|
|
6863
|
+
return false;
|
|
5819
6864
|
case "provenance.request":
|
|
5820
6865
|
relayProvenanceRequest(stream, options, msg);
|
|
5821
6866
|
return false;
|
|
@@ -5865,7 +6910,10 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
5865
6910
|
await this.container.stop({ t: CONTAINER_STOP_TIMEOUT });
|
|
5866
6911
|
} catch {}
|
|
5867
6912
|
try {
|
|
5868
|
-
await this.container.remove({
|
|
6913
|
+
await this.container.remove({
|
|
6914
|
+
force: true,
|
|
6915
|
+
v: true
|
|
6916
|
+
});
|
|
5869
6917
|
} catch {}
|
|
5870
6918
|
this.container = null;
|
|
5871
6919
|
}
|
|
@@ -5909,13 +6957,15 @@ var init_sandbox = __esmMin((() => {
|
|
|
5909
6957
|
init_bare_metal_sandbox();
|
|
5910
6958
|
init_firecracker_sandbox();
|
|
5911
6959
|
init_container_sandbox();
|
|
6960
|
+
init_fork_runner();
|
|
5912
6961
|
}));
|
|
5913
6962
|
//#endregion
|
|
5914
6963
|
//#region src/execution/job-runner.ts
|
|
5915
6964
|
var job_runner_exports = /* @__PURE__ */ __exportAll({
|
|
5916
6965
|
JobRunner: () => JobRunner$1,
|
|
5917
6966
|
buildEvalNeedsContext: () => buildEvalNeedsContext,
|
|
5918
|
-
resolveJobWorkDir: () => resolveJobWorkDir
|
|
6967
|
+
resolveJobWorkDir: () => resolveJobWorkDir,
|
|
6968
|
+
resolveRunnerBundlePath: () => resolveRunnerBundlePath
|
|
5919
6969
|
});
|
|
5920
6970
|
/**
|
|
5921
6971
|
* Check if a file exists at the given path.
|
|
@@ -5950,6 +7000,19 @@ function resolveRunnerPath() {
|
|
|
5950
7000
|
return bundlePath;
|
|
5951
7001
|
}
|
|
5952
7002
|
/**
|
|
7003
|
+
* Derive the self-contained runner bundle path from the resolved runner path.
|
|
7004
|
+
*
|
|
7005
|
+
* The container backend mounts the runner as a single file into the customer
|
|
7006
|
+
* job container, so it must run `workflow-runner-bundle.js` (zx + `@kici-dev/*`
|
|
7007
|
+
* inlined) rather than the external `workflow-runner.js`, which cannot resolve
|
|
7008
|
+
* its bare imports without the agent's node_modules / pnpm workspace. The
|
|
7009
|
+
* bundle is a flat sibling emitted alongside the runner by build-service.mjs.
|
|
7010
|
+
* bwrap / firecracker keep the external runner (they bind the workspace).
|
|
7011
|
+
*/
|
|
7012
|
+
function resolveRunnerBundlePath(runnerPath) {
|
|
7013
|
+
return join(dirname(runnerPath), "workflow-runner-bundle.js");
|
|
7014
|
+
}
|
|
7015
|
+
/**
|
|
5953
7016
|
* Determine the execution mode from agent config and job config.
|
|
5954
7017
|
*
|
|
5955
7018
|
* Priority:
|
|
@@ -5990,14 +7053,11 @@ async function resolveJobWorkDir(inPlace, repoUrl) {
|
|
|
5990
7053
|
cleanup: async () => {},
|
|
5991
7054
|
inPlace: true
|
|
5992
7055
|
};
|
|
5993
|
-
const workDir = await
|
|
7056
|
+
const { path: workDir, cleanup } = await makeTempDir("workdir");
|
|
5994
7057
|
return {
|
|
5995
7058
|
workDir,
|
|
5996
7059
|
cleanup: async () => {
|
|
5997
|
-
await
|
|
5998
|
-
recursive: true,
|
|
5999
|
-
force: true
|
|
6000
|
-
}).catch(() => {});
|
|
7060
|
+
await cleanup().catch(() => {});
|
|
6001
7061
|
},
|
|
6002
7062
|
inPlace: false
|
|
6003
7063
|
};
|
|
@@ -6011,6 +7071,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
6011
7071
|
init_init_runner();
|
|
6012
7072
|
init_api_intercept();
|
|
6013
7073
|
init_ensure_init_runner();
|
|
7074
|
+
init_payload_source();
|
|
7075
|
+
init_s3_payload_source();
|
|
6014
7076
|
init_timeout_util();
|
|
6015
7077
|
init_streaming_zx_log();
|
|
6016
7078
|
init_dynamic_job_serializer();
|
|
@@ -6025,7 +7087,6 @@ var init_job_runner = __esmMin((() => {
|
|
|
6025
7087
|
logger$2 = createLogger({ prefix: "job-runner" });
|
|
6026
7088
|
JobRunner$1 = class {
|
|
6027
7089
|
send;
|
|
6028
|
-
sendDirect;
|
|
6029
7090
|
config;
|
|
6030
7091
|
requestUploadUrl;
|
|
6031
7092
|
sendUploadComplete;
|
|
@@ -6038,6 +7099,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6038
7099
|
_sendApiRequest;
|
|
6039
7100
|
_requestUserCache;
|
|
6040
7101
|
_relayProvenance;
|
|
7102
|
+
_requestUserArtifact;
|
|
6041
7103
|
_sendStepApproval;
|
|
6042
7104
|
/** Tracks running jobs for concurrency and cancellation */
|
|
6043
7105
|
activeJobs = /* @__PURE__ */ new Map();
|
|
@@ -6045,7 +7107,6 @@ var init_job_runner = __esmMin((() => {
|
|
|
6045
7107
|
activeSandbox = null;
|
|
6046
7108
|
constructor(deps) {
|
|
6047
7109
|
this.send = deps.send;
|
|
6048
|
-
this.sendDirect = deps.sendDirect;
|
|
6049
7110
|
this.config = deps.config;
|
|
6050
7111
|
this.requestUploadUrl = deps.requestUploadUrl;
|
|
6051
7112
|
this.sendUploadComplete = deps.sendUploadComplete;
|
|
@@ -6058,6 +7119,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6058
7119
|
this._sendApiRequest = deps.sendApiRequest;
|
|
6059
7120
|
this._requestUserCache = deps.requestUserCache;
|
|
6060
7121
|
this._relayProvenance = deps.relayProvenance;
|
|
7122
|
+
this._requestUserArtifact = deps.requestUserArtifact;
|
|
6061
7123
|
this._sendStepApproval = deps.sendStepApproval;
|
|
6062
7124
|
}
|
|
6063
7125
|
/**
|
|
@@ -6161,13 +7223,15 @@ var init_job_runner = __esmMin((() => {
|
|
|
6161
7223
|
});
|
|
6162
7224
|
}, this.config.jobHeartbeatIntervalMs);
|
|
6163
7225
|
let sandbox;
|
|
7226
|
+
const logStreamers = /* @__PURE__ */ new Map();
|
|
6164
7227
|
try {
|
|
6165
7228
|
const setupResult = await this.setupSandboxForExecution(dispatch, workDir, abortController);
|
|
6166
7229
|
if (!setupResult) return;
|
|
6167
7230
|
sandbox = setupResult.sandbox;
|
|
6168
|
-
const
|
|
7231
|
+
const result = await this.runSandboxExecution(dispatch, sandbox, abortController, logStreamers);
|
|
6169
7232
|
this.reportExecutionResult(dispatch, result, logStreamers);
|
|
6170
7233
|
} catch (error) {
|
|
7234
|
+
for (const streamer of logStreamers.values()) streamer.destroy();
|
|
6171
7235
|
const errorMsg = toErrorMessage(error);
|
|
6172
7236
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, { error: errorMsg });
|
|
6173
7237
|
} finally {
|
|
@@ -6218,7 +7282,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
6218
7282
|
this.activeSandbox = sandbox;
|
|
6219
7283
|
await sandbox.setup({
|
|
6220
7284
|
workDir,
|
|
6221
|
-
env: sanitizedEnv
|
|
7285
|
+
env: sanitizedEnv,
|
|
7286
|
+
extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl)
|
|
6222
7287
|
});
|
|
6223
7288
|
if (abortController.signal.aborted) {
|
|
6224
7289
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.cancelled);
|
|
@@ -6243,13 +7308,13 @@ var init_job_runner = __esmMin((() => {
|
|
|
6243
7308
|
/**
|
|
6244
7309
|
* Drive `sandbox.executeJob` with IPC callbacks wired to the WS pipeline.
|
|
6245
7310
|
*
|
|
6246
|
-
* Lazily
|
|
7311
|
+
* Lazily populates `logStreamers` (owned by the caller so a throw still
|
|
7312
|
+
* leaves the buffered output flushable), forwards step + log + event-emit +
|
|
6247
7313
|
* concurrency-report + api-request messages, and emits the
|
|
6248
7314
|
* `agent.execution.start` / `agent.execution.end` lifecycle events.
|
|
6249
7315
|
*/
|
|
6250
|
-
async runSandboxExecution(dispatch, sandbox, abortController) {
|
|
7316
|
+
async runSandboxExecution(dispatch, sandbox, abortController, logStreamers) {
|
|
6251
7317
|
const { runId, jobId } = dispatch;
|
|
6252
|
-
const logStreamers = /* @__PURE__ */ new Map();
|
|
6253
7318
|
const maxLogSizeBytes = dispatch.maxLogSizeBytes ?? this.config.maxLogSizeBytes;
|
|
6254
7319
|
const getOrCreateLogStreamer = (stepIndex) => {
|
|
6255
7320
|
let streamer = logStreamers.get(stepIndex);
|
|
@@ -6278,8 +7343,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
6278
7343
|
this.maybeEmitCacheRunEvent(runId, jobId, stepIndex, state, data);
|
|
6279
7344
|
this.sendStepStatus(dispatch, stepIndex, stepName, state, data, logBytesStreamed);
|
|
6280
7345
|
},
|
|
6281
|
-
onLogLine: (stepIndex, line) => {
|
|
6282
|
-
getOrCreateLogStreamer(stepIndex).addLine(line);
|
|
7346
|
+
onLogLine: (stepIndex, line, stream) => {
|
|
7347
|
+
getOrCreateLogStreamer(stepIndex).addLine(line, stream);
|
|
6283
7348
|
},
|
|
6284
7349
|
signal: abortController.signal,
|
|
6285
7350
|
onEventEmit: async (request) => {
|
|
@@ -6302,6 +7367,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6302
7367
|
onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => this._sendApiRequest(method, params)) : void 0,
|
|
6303
7368
|
onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
|
|
6304
7369
|
onProvenanceRequest: this._relayProvenance ? async (request) => this._relayProvenance(jobId, request) : void 0,
|
|
7370
|
+
onArtifactRequest: this._requestUserArtifact ? async (request) => this._requestUserArtifact(jobId, request) : void 0,
|
|
6305
7371
|
onApprovalRequest: this._sendStepApproval ? async (request) => this._sendStepApproval(dispatch.runId, dispatch.jobId, request) : void 0,
|
|
6306
7372
|
onSecretMount: (event) => {
|
|
6307
7373
|
this.emitRunEvent(runId, "step.secret_mount", {
|
|
@@ -6321,10 +7387,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6321
7387
|
durationMs: Date.now() - executionStartMs,
|
|
6322
7388
|
metadata: { status: result.status }
|
|
6323
7389
|
});
|
|
6324
|
-
return
|
|
6325
|
-
result,
|
|
6326
|
-
logStreamers
|
|
6327
|
-
};
|
|
7390
|
+
return result;
|
|
6328
7391
|
}
|
|
6329
7392
|
/**
|
|
6330
7393
|
* Tear down log streamers, record step Prometheus metrics, log sandbox
|
|
@@ -6386,7 +7449,19 @@ var init_job_runner = __esmMin((() => {
|
|
|
6386
7449
|
if (!targetAgentId) throw new Error("bring-up job missing bringupTarget");
|
|
6387
7450
|
if (!this._sendApiRequest) throw new Error("bring-up job requires an orchestrator API transport");
|
|
6388
7451
|
streamer.addLine(`Bringing up init-runner on ${targetAgentId}…`);
|
|
6389
|
-
const
|
|
7452
|
+
const transport = async (method, params) => this._sendApiRequest(method, params);
|
|
7453
|
+
const result = await ensureInitRunner(transport, targetAgentId, {
|
|
7454
|
+
payloadSource: this.config.agentPayloadDir ? new LocalDirPayloadSource(this.config.agentPayloadDir) : new S3PayloadSource({
|
|
7455
|
+
presign: async (platform) => await transport("kici.presignAgentPackage", {
|
|
7456
|
+
targetAgentId,
|
|
7457
|
+
platform
|
|
7458
|
+
}),
|
|
7459
|
+
fetchToFile: fetchUrlToFile,
|
|
7460
|
+
cacheDir: defaultPayloadCacheDir(),
|
|
7461
|
+
exists: payloadFileExists
|
|
7462
|
+
}),
|
|
7463
|
+
agentCommand: this.config.agentCommand
|
|
7464
|
+
});
|
|
6390
7465
|
streamer.addLine(result.broughtUp ? `Init-runner brought up on ${targetAgentId}.` : `${targetAgentId} already has a live agent — no bring-up needed.`);
|
|
6391
7466
|
await streamer.flush();
|
|
6392
7467
|
this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.success, void 0, streamer.getTotalBytes());
|
|
@@ -6783,7 +7858,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6783
7858
|
env: { ...process.env },
|
|
6784
7859
|
verbose: true,
|
|
6785
7860
|
quiet: false,
|
|
6786
|
-
log: makeStreamingZxLog((line) => evalStreamer.addLine(line))
|
|
7861
|
+
log: makeStreamingZxLog((line, stream) => evalStreamer.addLine(line, stream))
|
|
6787
7862
|
});
|
|
6788
7863
|
const evalSink = { addLine: (line) => evalStreamer.addLine(line) };
|
|
6789
7864
|
const dynamicJobLogger = {
|
|
@@ -6868,10 +7943,20 @@ var init_job_runner = __esmMin((() => {
|
|
|
6868
7943
|
return new ContainerSandbox({
|
|
6869
7944
|
docker: new Docker(),
|
|
6870
7945
|
image,
|
|
6871
|
-
runnerPath: opts.runnerPath,
|
|
7946
|
+
runnerPath: resolveRunnerBundlePath(opts.runnerPath),
|
|
6872
7947
|
env: opts.env,
|
|
6873
7948
|
keepFailed: this.config.dockerKeepFailed,
|
|
6874
|
-
jobId: opts.jobId
|
|
7949
|
+
jobId: opts.jobId,
|
|
7950
|
+
hardening: {
|
|
7951
|
+
hardened: this.config.sandboxHardened,
|
|
7952
|
+
readonlyRootfs: this.config.sandboxReadonlyRootfs,
|
|
7953
|
+
user: this.config.sandboxUser,
|
|
7954
|
+
pidsLimit: this.config.sandboxPidsLimit,
|
|
7955
|
+
memoryBytes: this.config.sandboxMemoryBytes,
|
|
7956
|
+
nanoCpus: this.config.sandboxNanoCpus,
|
|
7957
|
+
networkMode: this.config.sandboxNetwork === "host" ? "host" : "default",
|
|
7958
|
+
grant: opts.jobConfig.sandboxGrant
|
|
7959
|
+
}
|
|
6875
7960
|
});
|
|
6876
7961
|
}
|
|
6877
7962
|
case "firecracker": return new FirecrackerSandbox({
|
|
@@ -6960,7 +8045,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6960
8045
|
* they are included as a top-level field on the WS message (not nested in data).
|
|
6961
8046
|
*/
|
|
6962
8047
|
sendJobStatus(dispatch, state, data, secretOutputs) {
|
|
6963
|
-
this.
|
|
8048
|
+
this.send({
|
|
6964
8049
|
type: "job.status",
|
|
6965
8050
|
messageId: randomUUID(),
|
|
6966
8051
|
runId: dispatch.runId,
|
|
@@ -6986,7 +8071,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
6986
8071
|
const groupId = data?.groupId;
|
|
6987
8072
|
const { secretsAccessed: _s, concurrencyKind: _c, groupId: _g, ...restData } = data ?? {};
|
|
6988
8073
|
const hasRestData = Object.keys(restData).length > 0;
|
|
6989
|
-
this.
|
|
8074
|
+
this.send({
|
|
6990
8075
|
type: "step.status",
|
|
6991
8076
|
messageId: randomUUID(),
|
|
6992
8077
|
runId: dispatch.runId,
|
|
@@ -7012,7 +8097,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
7012
8097
|
* Startup sequence:
|
|
7013
8098
|
* 1. Load config
|
|
7014
8099
|
* 2. Create logger
|
|
7015
|
-
* 3. Create JobRunner with send
|
|
8100
|
+
* 3. Create JobRunner with send callbacks
|
|
7016
8101
|
* 4. Create OrchestratorClient with dispatch and cancel handlers
|
|
7017
8102
|
* 5. Add WS log transport (if not scaler-managed)
|
|
7018
8103
|
* 6. Connect OrchestratorClient
|
|
@@ -7026,14 +8111,14 @@ var init_job_runner = __esmMin((() => {
|
|
|
7026
8111
|
*/
|
|
7027
8112
|
init_console_capture();
|
|
7028
8113
|
init_npm_resolver();
|
|
7029
|
-
const AGENT_VERSION = "0.
|
|
7030
|
-
const BUILD_COMMIT = "
|
|
7031
|
-
const SDK_VERSION = "0.
|
|
7032
|
-
const SDK_BUNDLE_HASH = "
|
|
7033
|
-
const SHARED_VERSION = "0.
|
|
7034
|
-
const SHARED_BUNDLE_HASH = "
|
|
7035
|
-
const ENGINE_VERSION = "0.
|
|
7036
|
-
const ENGINE_BUNDLE_HASH = "
|
|
8114
|
+
const AGENT_VERSION = "0.2.0";
|
|
8115
|
+
const BUILD_COMMIT = "15d5e4447";
|
|
8116
|
+
const SDK_VERSION = "0.2.0";
|
|
8117
|
+
const SDK_BUNDLE_HASH = "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
|
|
8118
|
+
const SHARED_VERSION = "0.2.0";
|
|
8119
|
+
const SHARED_BUNDLE_HASH = "f16fa528a8a9ef1bed8df369f0e88efafca8a9954a749005299fdbcdb93fa188";
|
|
8120
|
+
const ENGINE_VERSION = "0.2.0";
|
|
8121
|
+
const ENGINE_BUNDLE_HASH = "c579e54f7d0b0d5b16587a0528a3e66d32f63e2434bb471a93a01366296efd58";
|
|
7037
8122
|
initTelemetry({
|
|
7038
8123
|
serviceName: "kici-agent",
|
|
7039
8124
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
@@ -7103,7 +8188,6 @@ await guardStartup(logger$1, async () => {
|
|
|
7103
8188
|
let client;
|
|
7104
8189
|
const jobRunner = new JobRunner({
|
|
7105
8190
|
send: (msg) => client.send(msg),
|
|
7106
|
-
sendDirect: (msg) => client.sendDirect(msg),
|
|
7107
8191
|
config,
|
|
7108
8192
|
requestUploadUrl: (jobId, cacheType, key) => client.requestUploadUrl(jobId, cacheType, key),
|
|
7109
8193
|
sendUploadComplete: (jobId, cacheType, key) => client.sendUploadComplete(jobId, cacheType, key),
|
|
@@ -7124,6 +8208,7 @@ await guardStartup(logger$1, async () => {
|
|
|
7124
8208
|
},
|
|
7125
8209
|
requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
|
|
7126
8210
|
relayProvenance: (jobId, request) => client.relayProvenance(jobId, request),
|
|
8211
|
+
requestUserArtifact: (jobId, request) => client.requestUserArtifact(jobId, request),
|
|
7127
8212
|
sendStepApproval: (runId, jobId, request) => client.sendStepApproval(runId, jobId, request)
|
|
7128
8213
|
});
|
|
7129
8214
|
/** Build and send an agent.status message with dynamic OS metadata. */
|