@kici-dev/agent 0.1.27 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/bootstrap/ensure-init-runner.d.ts +23 -22
  2. package/dist/bootstrap/payload-source.d.ts +32 -0
  3. package/dist/bootstrap/probe-platform.d.ts +21 -0
  4. package/dist/bootstrap/restage-agent.d.ts +43 -0
  5. package/dist/bootstrap/run-restage.d.ts +12 -0
  6. package/dist/bootstrap/s3-payload-source.d.ts +35 -0
  7. package/dist/bootstrap/ssh-exec.d.ts +14 -0
  8. package/dist/bootstrap/stage-agent-payload.d.ts +41 -0
  9. package/dist/checkout/changed-files.d.ts +34 -0
  10. package/dist/config.d.ts +36 -14
  11. package/dist/container-ts-loader-hook.js +147710 -0
  12. package/dist/execution/artifacts/artifact-engine.d.ts +51 -0
  13. package/dist/execution/dep-installer.d.ts +3 -3
  14. package/dist/execution/job-runner.d.ts +24 -6
  15. package/dist/execution/log-streamer.d.ts +17 -2
  16. package/dist/execution/rule-evaluator.d.ts +1 -12
  17. package/dist/execution/sandbox/container-hardening.d.ts +80 -0
  18. package/dist/execution/sandbox/container-sandbox.d.ts +54 -0
  19. package/dist/execution/sandbox/container-ts-loader-hook.d.ts +26 -0
  20. package/dist/execution/sandbox/fork-runner.d.ts +14 -0
  21. package/dist/execution/sandbox/index.d.ts +2 -1
  22. package/dist/execution/sandbox/ipc-protocol.d.ts +79 -3
  23. package/dist/execution/sandbox/step-loop.d.ts +4 -0
  24. package/dist/execution/sandbox/types.d.ts +25 -4
  25. package/dist/execution/sandbox/workflow-runner.d.ts +111 -5
  26. package/dist/execution/streaming-zx-log.d.ts +11 -3
  27. package/dist/execution/tmp-gc.d.ts +22 -7
  28. package/dist/execution/workflow-loader.d.ts +32 -1
  29. package/dist/index.js +44 -45
  30. package/dist/provenance/statement-builder.d.ts +3 -2
  31. package/dist/server.d.ts +1 -1
  32. package/dist/server.js +1256 -171
  33. package/dist/workflow-runner-bundle.js +215393 -0
  34. package/dist/workflow-runner.js +886 -244
  35. package/dist/ws/orchestrator-client.d.ts +105 -1
  36. package/package.json +14 -12
  37. package/sbom.spdx.json +1090 -1721
package/dist/server.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
2
2
  import { dirname as __cjs_dirname } from "node:path";
3
3
  __cjs_dirname(__cjs_fileURLToPath(import.meta.url));
4
- import { register } from "node:module";
4
+ import { createRequire, register } from "node:module";
5
5
  import crypto$1, { createCipheriv, createHash, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
6
6
  import * as os$1 from "node:os";
7
7
  import os, { hostname, tmpdir } from "node:os";
@@ -9,23 +9,25 @@ import { PassThrough, Readable, Transform, Writable } from "node:stream";
9
9
  import { serve } from "@hono/node-server";
10
10
  import { Hono } from "hono";
11
11
  import winston from "winston";
12
- import { RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
12
+ import { AgentDeliveryMode, AgentPlatform, RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, kiciTmpBase, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
13
13
  import { z } from "zod";
14
14
  import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
15
- import { ALLOWED_SYSTEM_VARS, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
15
+ import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
16
16
  import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
17
17
  import WebSocket from "ws";
18
18
  import * as fs$1 from "node:fs";
19
- import fs, { existsSync } from "node:fs";
19
+ import fs, { createReadStream, createWriteStream, existsSync } from "node:fs";
20
20
  import { ZipArchive } from "archiver";
21
21
  import path, { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
22
22
  import { fileURLToPath, pathToFileURL } from "node:url";
23
23
  import { AsyncLocalStorage } from "node:async_hooks";
24
24
  import { format, promisify } from "node:util";
25
+ import { OTEL_DATA_POINT_TYPE, mapDataPointTypeToWireKind } from "@kici-dev/engine/metrics/metric-kind-compat";
25
26
  import { gcStaleTmpDirs } from "@kici-dev/core/tmp-gc";
26
- import fsPromises, { access, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, unlink, writeFile } from "node:fs/promises";
27
+ import fsPromises, { access, lstat, mkdir, readFile, readdir, realpath, unlink, writeFile } from "node:fs/promises";
28
+ import { makeTempDir } from "@kici-dev/core/tmp";
27
29
  import Docker from "dockerode";
28
- import { buildKiciApi, buildNeedsContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject } from "@kici-dev/sdk";
30
+ import { buildKiciApi, buildNeedsContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject, setJobOutputsMap, setStepOutputsMap, setStepRefMap } from "@kici-dev/sdk";
29
31
  import { c, x } from "tar";
30
32
  import https from "node:https";
31
33
  import http from "node:http";
@@ -101,10 +103,18 @@ const envDef = defineEnv({
101
103
  dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
102
104
  jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
103
105
  backpressureMode: z.enum(["pause", "drop"]).default("pause"),
106
+ agentPayloadDir: z.string().optional(),
107
+ agentCommand: z.string().optional(),
104
108
  sandbox: z.string().default("false").transform((s) => s === "true"),
105
109
  trustedEnv: z.string().default("false").transform((s) => s === "true"),
106
110
  inPlace: z.string().default("false").transform((s) => s === "true"),
107
111
  sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
112
+ sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
113
+ sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
114
+ sandboxUser: z.string().optional(),
115
+ sandboxPidsLimit: z.coerce.number().int().positive().default(512),
116
+ sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
117
+ sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
108
118
  scalerManaged: z.string().optional().transform((s) => s === "1"),
109
119
  scalerIdleTimeoutMs: z.coerce.number().default(5e3),
110
120
  scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
@@ -129,10 +139,18 @@ const envDef = defineEnv({
129
139
  dockerKeepFailed: "KICI_DOCKER_KEEP_FAILED",
130
140
  jobHeartbeatIntervalMs: "KICI_JOB_HEARTBEAT_INTERVAL_MS",
131
141
  backpressureMode: "KICI_BACKPRESSURE_MODE",
142
+ agentPayloadDir: "KICI_AGENT_PAYLOAD_DIR",
143
+ agentCommand: "KICI_AGENT_COMMAND",
132
144
  sandbox: "KICI_SANDBOX",
133
145
  trustedEnv: "KICI_TRUSTED_ENV",
134
146
  inPlace: "KICI_IN_PLACE",
135
147
  sandboxNetwork: "KICI_SANDBOX_NETWORK",
148
+ sandboxHardened: "KICI_SANDBOX_HARDENED",
149
+ sandboxReadonlyRootfs: "KICI_SANDBOX_READONLY_ROOTFS",
150
+ sandboxUser: "KICI_SANDBOX_USER",
151
+ sandboxPidsLimit: "KICI_SANDBOX_PIDS_LIMIT",
152
+ sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
153
+ sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
136
154
  scalerManaged: "KICI_SCALER_MANAGED",
137
155
  scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
138
156
  scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
@@ -162,7 +180,13 @@ const envDef = defineEnv({
162
180
  * - KICI_SANDBOX (default: false) — enable bubblewrap (bwrap) namespace isolation for bare-metal execution
163
181
  * - KICI_TRUSTED_ENV (default: false) — trusted fleet-agent profile: pass the ambient host env (minus the agent's own KiCI identity secrets) through to steps
164
182
  * - KICI_IN_PLACE (default: false) — in-place no-clone profile: for a file:// source, use the real repo path as workDir and skip the clone (the routed deploy:stg profile)
165
- * - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — when sandbox=true, controls bwrap network namespace
183
+ * - KICI_SANDBOX_NETWORK (default: isolated, options: isolated | host) — sandbox network posture for BOTH backends: the bwrap network namespace (when sandbox=true) and the container job network. `host` shares the host network (container backend also binds host /etc/hosts read-only for name resolution); applies to the container backend under the default hardened posture (KICI_SANDBOX_HARDENED=true)
184
+ * - KICI_SANDBOX_HARDENED (default: true) — hardened-by-default job containers (CapDrop ALL, no-new-privileges, cgroup caps, tmpfs /tmp); set false to roll back to the legacy unhardened posture
185
+ * - KICI_SANDBOX_READONLY_ROOTFS (default: false) — opt-in read-only container rootfs (/tmp stays a writable tmpfs)
186
+ * - KICI_SANDBOX_USER (optional) — container user override (uid, uid:gid, or name); honors the image user when unset
187
+ * - KICI_SANDBOX_PIDS_LIMIT (default: 512) — max PIDs in the job container cgroup
188
+ * - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
189
+ * - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
166
190
  * - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
167
191
  * - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
168
192
  * - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
@@ -171,7 +195,7 @@ const envDef = defineEnv({
171
195
  */
172
196
  function loadConfig() {
173
197
  const data = envDef.parse();
174
- if (!data.scalerManaged) validateNoReservedLabels(data.labels, "KICI_LABELS");
198
+ if (!data.scalerManaged && !data.agentToken) validateNoReservedLabels(data.labels, "KICI_LABELS");
175
199
  validateUnknownKiciVars([...envDef.listKnownEnvVars(), ...LOGGER_ENV_VARS]);
176
200
  return {
177
201
  ...data,
@@ -302,6 +326,22 @@ var LogBuffer = class extends RingBuffer {
302
326
  //#region src/ws/orchestrator-client.ts
303
327
  const logger$12 = createLogger({ prefix: "orchestrator-client" });
304
328
  /**
329
+ * How long to wait for `artifacts.upload.complete.ack` before failing the step.
330
+ *
331
+ * Deliberately shorter than the sandbox's own 30s artifact-request timeout
332
+ * (`CACHE_RESPONSE_TIMEOUT_MS` in the workflow runner) so a lost ack surfaces
333
+ * this specific reason rather than the sandbox's generic "artifact request
334
+ * timed out". The orchestrator's bounded commit retry finishes in well under a
335
+ * second, so the margin costs nothing.
336
+ */
337
+ const ARTIFACT_COMPLETE_ACK_TIMEOUT_MS = 25e3;
338
+ /**
339
+ * Tail shared by both give-up messages for an unacknowledged commit. The
340
+ * operator must not be sent hunting for a lost artifact that in fact landed:
341
+ * the orchestrator may well have committed it and only the ack was lost.
342
+ */
343
+ const COMMIT_MAY_HAVE_LANDED = "the artifact may have been committed";
344
+ /**
305
345
  * WebSocket client that connects the agent to the customer orchestrator.
306
346
  *
307
347
  * Handles:
@@ -323,6 +363,13 @@ var OrchestratorClient = class OrchestratorClient {
323
363
  reconnectTimer = null;
324
364
  reconnectAttempts = 0;
325
365
  intentionalDisconnect = false;
366
+ /**
367
+ * Agent-facing capabilities advertised by the orchestrator on `register.ack`.
368
+ * Undefined until the first ack arrives, and on a pre-capability orchestrator
369
+ * that never advertises — both cases mean "assume unsupported" for every
370
+ * optional feature gated on it.
371
+ */
372
+ orchCapabilities;
326
373
  pendingLogBatch = [];
327
374
  logFlushTimer = null;
328
375
  static LOG_BATCH_SIZE = 50;
@@ -335,6 +382,26 @@ var OrchestratorClient = class OrchestratorClient {
335
382
  pendingApiRequests = /* @__PURE__ */ new Map();
336
383
  /** Pending user-cache restore/save requests awaiting orchestrator response. */
337
384
  pendingUserCacheRequests = /* @__PURE__ */ new Map();
385
+ /** Pending user-artifact upload/download requests awaiting orchestrator response. */
386
+ pendingUserArtifactRequests = /* @__PURE__ */ new Map();
387
+ /**
388
+ * In-flight `artifacts.upload.complete` frames that are safe to re-send after a
389
+ * reconnect, keyed by their current `messageId`.
390
+ *
391
+ * The commit is idempotent server-side (the storage key is derived server-side,
392
+ * `initMeta` is a re-writable sidecar, and the DB insert is
393
+ * `onConflict.doNothing()`), so a dropped ack means "we did not hear the
394
+ * answer", not "it did not happen". `beginUpload` / `download` are deliberately
395
+ * absent: `beginUpload` mints a presigned PUT and is not idempotent the same
396
+ * way, and `download` is read-only and cheap to fail.
397
+ */
398
+ resendableCompletes = /* @__PURE__ */ new Map();
399
+ /**
400
+ * Completes taken off `pendingUserArtifactRequests` by the disconnect sweep and
401
+ * parked until the next `register.ack` re-sends them. Their ack timers keep
402
+ * running — parking does not extend the deadline.
403
+ */
404
+ heldCompletes = /* @__PURE__ */ new Set();
338
405
  /**
339
406
  * Pending step-approval requests awaiting the orchestrator's resolution.
340
407
  * No client-side timeout: the orchestrator owns the (org-/SDK-configured)
@@ -430,6 +497,7 @@ var OrchestratorClient = class OrchestratorClient {
430
497
  this.intentionalDisconnect = true;
431
498
  this.stopHeartbeat();
432
499
  this.cancelReconnect();
500
+ this.rejectHeldCompletes("the agent disconnected");
433
501
  this.drainPendingLogBatch();
434
502
  if (this.ws) {
435
503
  this.ws.close(1e3, "Agent disconnect");
@@ -721,6 +789,105 @@ var OrchestratorClient = class OrchestratorClient {
721
789
  });
722
790
  }
723
791
  /**
792
+ * Relay a user-facing artifact request from the sandbox to the orchestrator.
793
+ *
794
+ * Translates the sandbox `artifacts.request` IPC into the matching
795
+ * `artifacts.*` WS message and resolves with the orchestrator's response
796
+ * mapped onto the IPC response shape:
797
+ *
798
+ * - `beginUpload` -> `artifacts.upload.request`, awaits `artifacts.upload.response`.
799
+ * - `completeUpload` -> `artifacts.upload.complete`, awaits
800
+ * `artifacts.upload.complete.ack` when the orchestrator advertises the
801
+ * `artifactCompleteAck` capability, and rejects when the commit failed so
802
+ * the workflow step fails instead of losing the artifact behind a green run.
803
+ * Against an orchestrator that never advertises it, the message stays
804
+ * fire-and-forget and resolves immediately.
805
+ * - `download` -> `artifacts.download.request`, awaits `artifacts.download.response`.
806
+ *
807
+ * Times out after 30 seconds for the round-trip ops.
808
+ */
809
+ async requestUserArtifact(jobId, request) {
810
+ if (request.op === "completeUpload") {
811
+ const completeId = randomUUID();
812
+ const complete = {
813
+ type: "artifacts.upload.complete",
814
+ messageId: completeId,
815
+ jobId,
816
+ name: request.name,
817
+ sizeBytes: request.sizeBytes,
818
+ sha256: request.sha256,
819
+ storageKey: request.storageKey
820
+ };
821
+ if (!hasOrchAgentCapability(this.orchCapabilities, "artifactCompleteAck")) {
822
+ this.sendDirect(complete);
823
+ return {
824
+ type: "artifacts.response",
825
+ requestId: request.requestId
826
+ };
827
+ }
828
+ return new Promise((resolve, reject) => {
829
+ const record = {
830
+ messageId: completeId,
831
+ frame: complete,
832
+ attempts: 0,
833
+ pending: {
834
+ resolve: () => {
835
+ clearTimeout(timer);
836
+ resolve({
837
+ type: "artifacts.response",
838
+ requestId: request.requestId
839
+ });
840
+ },
841
+ reject: (err) => {
842
+ clearTimeout(timer);
843
+ reject(err);
844
+ }
845
+ }
846
+ };
847
+ const timer = setTimeout(() => {
848
+ this.forgetComplete(record);
849
+ reject(/* @__PURE__ */ new Error(`Artifact upload-complete ack timed out (${ARTIFACT_COMPLETE_ACK_TIMEOUT_MS}ms)`));
850
+ }, ARTIFACT_COMPLETE_ACK_TIMEOUT_MS);
851
+ this.pendingUserArtifactRequests.set(completeId, record.pending);
852
+ this.resendableCompletes.set(completeId, record);
853
+ this.sendDirect(complete);
854
+ });
855
+ }
856
+ const messageId = randomUUID();
857
+ return new Promise((resolve, reject) => {
858
+ const timer = setTimeout(() => {
859
+ this.pendingUserArtifactRequests.delete(messageId);
860
+ reject(/* @__PURE__ */ new Error("User-artifact request timed out (30s)"));
861
+ }, 3e4);
862
+ this.pendingUserArtifactRequests.set(messageId, {
863
+ resolve: (response) => {
864
+ clearTimeout(timer);
865
+ resolve({
866
+ ...response,
867
+ requestId: request.requestId
868
+ });
869
+ },
870
+ reject: (err) => {
871
+ clearTimeout(timer);
872
+ reject(err);
873
+ }
874
+ });
875
+ if (request.op === "beginUpload") this.sendDirect({
876
+ type: "artifacts.upload.request",
877
+ messageId,
878
+ jobId,
879
+ name: request.name,
880
+ declaredSizeBytes: request.declaredSizeBytes
881
+ });
882
+ else this.sendDirect({
883
+ type: "artifacts.download.request",
884
+ messageId,
885
+ jobId,
886
+ name: request.name
887
+ });
888
+ });
889
+ }
890
+ /**
724
891
  * Relay a provenance bundle upload operation to the orchestrator. Maps the
725
892
  * IPC `provenance.request` onto `requestProvenanceUploadUrl` (returns the
726
893
  * presigned URL) or `sendProvenanceUploadComplete` (fire-and-forget) and
@@ -935,6 +1102,17 @@ var OrchestratorClient = class OrchestratorClient {
935
1102
  this.pendingApiRequests.clear();
936
1103
  for (const [_id, pending] of this.pendingUserCacheRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
937
1104
  this.pendingUserCacheRequests.clear();
1105
+ for (const [id, pending] of this.pendingUserArtifactRequests) {
1106
+ const record = this.resendableCompletes.get(id);
1107
+ if (record && !this.intentionalDisconnect) {
1108
+ this.heldCompletes.add(record);
1109
+ continue;
1110
+ }
1111
+ pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
1112
+ }
1113
+ this.pendingUserArtifactRequests.clear();
1114
+ this.resendableCompletes.clear();
1115
+ if (this.intentionalDisconnect) this.rejectHeldCompletes("the agent disconnected");
938
1116
  for (const [_id, pending] of this.pendingConcurrencyRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
939
1117
  this.pendingConcurrencyRequests.clear();
940
1118
  for (const [_id, pending] of this.pendingStepApprovals) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
@@ -946,6 +1124,132 @@ var OrchestratorClient = class OrchestratorClient {
946
1124
  if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
947
1125
  });
948
1126
  }
1127
+ /**
1128
+ * Resolve a pending user-artifact request from an `artifacts.upload.response`
1129
+ * / `artifacts.download.response` WS message, mapping the wire fields onto the
1130
+ * IPC response shape. No-op when no request is pending for the requestId.
1131
+ */
1132
+ resolveArtifactResponse(type, raw) {
1133
+ const artMsg = raw;
1134
+ const pending = this.pendingUserArtifactRequests.get(artMsg.requestId);
1135
+ if (!pending) return;
1136
+ this.pendingUserArtifactRequests.delete(artMsg.requestId);
1137
+ const isUpload = type === "artifacts.upload.response";
1138
+ pending.resolve({
1139
+ type: "artifacts.response",
1140
+ requestId: artMsg.requestId,
1141
+ ...isUpload && artMsg.outcome !== void 0 && { uploadOutcome: artMsg.outcome },
1142
+ ...!isUpload && artMsg.outcome !== void 0 && { downloadOutcome: artMsg.outcome },
1143
+ ...artMsg.uploadUrl && { uploadUrl: artMsg.uploadUrl },
1144
+ ...artMsg.storageKey && { storageKey: artMsg.storageKey },
1145
+ ...artMsg.reason && { reason: artMsg.reason },
1146
+ ...artMsg.error && { rejectionDetail: artMsg.error },
1147
+ ...artMsg.downloadUrl && { downloadUrl: artMsg.downloadUrl },
1148
+ ...artMsg.sizeBytes !== void 0 && { sizeBytes: artMsg.sizeBytes },
1149
+ ...artMsg.sha256 && { sha256: artMsg.sha256 }
1150
+ });
1151
+ }
1152
+ /**
1153
+ * Resolve or reject a pending completeUpload from an
1154
+ * `artifacts.upload.complete.ack`. `committed` resolves the relayed IPC
1155
+ * request; `failed` rejects it (carrying the orchestrator's reason), which
1156
+ * propagates through the relay's `response.error` and fails the workflow step
1157
+ * rather than losing the artifact silently. No-op when nothing is pending for
1158
+ * the requestId (an ack for an already-timed-out complete).
1159
+ */
1160
+ resolveArtifactCompleteAck(raw) {
1161
+ const ack = raw;
1162
+ const pending = this.pendingUserArtifactRequests.get(ack.requestId);
1163
+ if (!pending) return;
1164
+ this.pendingUserArtifactRequests.delete(ack.requestId);
1165
+ this.resendableCompletes.delete(ack.requestId);
1166
+ if (ack.outcome === ArtifactCompleteAckOutcome.enum.committed) pending.resolve({
1167
+ type: "artifacts.response",
1168
+ requestId: ack.requestId
1169
+ });
1170
+ else pending.reject(new Error(ack.reason ?? "artifact upload-complete failed"));
1171
+ }
1172
+ /**
1173
+ * Drop every trace of an in-flight upload-complete, whichever key it currently
1174
+ * lives under and whether or not it is parked awaiting a resend.
1175
+ *
1176
+ * Called from the ack timer, which fires on the original schedule regardless of
1177
+ * how many times the complete has been re-keyed since.
1178
+ */
1179
+ forgetComplete(record) {
1180
+ this.pendingUserArtifactRequests.delete(record.messageId);
1181
+ this.resendableCompletes.delete(record.messageId);
1182
+ this.heldCompletes.delete(record);
1183
+ }
1184
+ /**
1185
+ * Fail every parked upload-complete closed, for the paths where the reconnect
1186
+ * the park was betting on will never arrive (a deliberate shutdown, a
1187
+ * permanently rejected token). Naming the ambiguity keeps the operator from
1188
+ * hunting for an artifact that in fact landed.
1189
+ */
1190
+ rejectHeldCompletes(cause) {
1191
+ if (this.heldCompletes.size === 0) return;
1192
+ const held = [...this.heldCompletes];
1193
+ this.heldCompletes.clear();
1194
+ for (const record of held) record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete could not be acknowledged before ${cause} — the artifact may have been committed`));
1195
+ }
1196
+ /**
1197
+ * Re-send every upload-complete parked by the disconnect sweep.
1198
+ *
1199
+ * Called from `register.ack`, once the new connection's capabilities are known.
1200
+ * Each frame is re-keyed with a fresh `messageId` before it goes out: the id is
1201
+ * the correlation key, so reusing it would let a late ack for the pre-disconnect
1202
+ * send resolve the new pending.
1203
+ *
1204
+ * The ack timer created when the complete was first sent is deliberately left
1205
+ * alone, so the hold plus every resend stays inside the original
1206
+ * ARTIFACT_COMPLETE_ACK_TIMEOUT_MS budget.
1207
+ */
1208
+ resendHeldCompletes() {
1209
+ if (this.heldCompletes.size === 0) return;
1210
+ const held = [...this.heldCompletes];
1211
+ this.heldCompletes.clear();
1212
+ const canAck = hasOrchAgentCapability(this.orchCapabilities, "artifactCompleteAck");
1213
+ for (const record of held) {
1214
+ if (!canAck) {
1215
+ record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete cannot be acknowledged by the reconnected orchestrator — ${COMMIT_MAY_HAVE_LANDED}`));
1216
+ continue;
1217
+ }
1218
+ if (record.attempts >= 2) {
1219
+ record.pending.reject(/* @__PURE__ */ new Error(`artifact upload-complete could not be acknowledged after 2 reconnects — ${COMMIT_MAY_HAVE_LANDED}`));
1220
+ continue;
1221
+ }
1222
+ record.messageId = randomUUID();
1223
+ record.attempts += 1;
1224
+ record.frame = {
1225
+ ...record.frame,
1226
+ messageId: record.messageId
1227
+ };
1228
+ this.pendingUserArtifactRequests.set(record.messageId, record.pending);
1229
+ this.resendableCompletes.set(record.messageId, record);
1230
+ this.sendDirect(record.frame);
1231
+ logger$12.info("Re-sent artifact upload-complete after reconnect", {
1232
+ messageId: record.messageId,
1233
+ attempt: record.attempts
1234
+ });
1235
+ }
1236
+ }
1237
+ /**
1238
+ * Route the orchestrator's replies to a relayed user-artifact request, which
1239
+ * are dispatched off the raw frame rather than the parsed protocol union.
1240
+ * Returns true when the message was consumed.
1241
+ */
1242
+ handleArtifactReply(type, raw) {
1243
+ if (type === "artifacts.upload.response" || type === "artifacts.download.response") {
1244
+ this.resolveArtifactResponse(type, raw);
1245
+ return true;
1246
+ }
1247
+ if (type === "artifacts.upload.complete.ack") {
1248
+ this.resolveArtifactCompleteAck(raw);
1249
+ return true;
1250
+ }
1251
+ return false;
1252
+ }
949
1253
  handleMessage(data) {
950
1254
  let raw;
951
1255
  try {
@@ -1003,6 +1307,7 @@ var OrchestratorClient = class OrchestratorClient {
1003
1307
  }
1004
1308
  return;
1005
1309
  }
1310
+ if (this.handleArtifactReply(rawMsg.type, raw)) return;
1006
1311
  const parsed = orchestratorToAgentMessageSchema.safeParse(raw);
1007
1312
  if (parsed.success) {
1008
1313
  const msg = parsed.data;
@@ -1031,10 +1336,12 @@ var OrchestratorClient = class OrchestratorClient {
1031
1336
  scalerManaged: msg.scalerManaged,
1032
1337
  pendingDispatch: msg.pendingDispatch ?? false
1033
1338
  });
1339
+ this.orchCapabilities = msg.capabilities;
1034
1340
  this._state = "registered";
1035
1341
  this.reconnectAttempts = 0;
1036
1342
  this.startHeartbeat();
1037
1343
  this.flushBuffer();
1344
+ this.resendHeldCompletes();
1038
1345
  this.onRegistered?.({ pendingDispatch: msg.pendingDispatch ?? false });
1039
1346
  if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
1040
1347
  this.sendConfigAck(msg.agentId);
@@ -1348,14 +1655,14 @@ var init_console_capture = __esmMin((() => {
1348
1655
  init_console_capture();
1349
1656
  function safe(name, fallback = "unknown") {
1350
1657
  switch (name) {
1351
- case "version": return "0.1.27";
1352
- case "buildCommit": return "f0747470b";
1353
- case "sdkVersion": return "0.1.27";
1354
- case "sdkBundleHash": return "e06bd1acf349ba365bdb357f3d865e0cbb4ad4604a0270b707a681a4f440954e";
1355
- case "sharedVersion": return "0.1.27";
1356
- case "sharedBundleHash": return "f8e1f0117cf41fca5a74377633e2ec89e54ce4eda47d9f476cdb83336f1ed962";
1357
- case "engineVersion": return "0.1.27";
1358
- case "engineBundleHash": return "ce675fb4790406ed2a938f8f1783db37b283cffd7d8fc1eb12a878e0bfb919b3";
1658
+ case "version": return "0.2.0";
1659
+ case "buildCommit": return "15d5e4447";
1660
+ case "sdkVersion": return "0.2.0";
1661
+ case "sdkBundleHash": return "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
1662
+ case "sharedVersion": return "0.2.0";
1663
+ case "sharedBundleHash": return "f16fa528a8a9ef1bed8df369f0e88efafca8a9954a749005299fdbcdb93fa188";
1664
+ case "engineVersion": return "0.2.0";
1665
+ case "engineBundleHash": return "c579e54f7d0b0d5b16587a0528a3e66d32f63e2434bb471a93a01366296efd58";
1359
1666
  default: return fallback;
1360
1667
  }
1361
1668
  }
@@ -1406,15 +1713,6 @@ function createHealthRoutes$1(deps) {
1406
1713
  * The orchestrator aggregates these metrics from all agents and exposes
1407
1714
  * them on its `/metrics` endpoint for Prometheus scraping.
1408
1715
  */
1409
- /**
1410
- * OTel DataPointType enum values (from @opentelemetry/sdk-metrics).
1411
- * We duplicate them here to avoid a direct runtime dependency on sdk-metrics.
1412
- */
1413
- const DataPointType = {
1414
- HISTOGRAM: 0,
1415
- GAUGE: 2,
1416
- SUM: 3
1417
- };
1418
1716
  var MetricsReporter = class {
1419
1717
  timer;
1420
1718
  agentId;
@@ -1462,7 +1760,7 @@ var MetricsReporter = class {
1462
1760
  const wireType = this.mapDataPointType(metricData.dataPointType, metricData.isMonotonic);
1463
1761
  for (const dp of metricData.dataPoints) {
1464
1762
  const labels = this.extractLabels(dp.attributes);
1465
- if (metricData.dataPointType === DataPointType.HISTOGRAM) {
1763
+ if (metricData.dataPointType === OTEL_DATA_POINT_TYPE.HISTOGRAM) {
1466
1764
  const histValue = dp.value;
1467
1765
  const boundaries = histValue.buckets.boundaries;
1468
1766
  const counts = histValue.buckets.counts;
@@ -1495,12 +1793,7 @@ var MetricsReporter = class {
1495
1793
  }
1496
1794
  /** Map OTel DataPointType to wire format type string. */
1497
1795
  mapDataPointType(dataPointType, isMonotonic) {
1498
- switch (dataPointType) {
1499
- case DataPointType.HISTOGRAM: return "histogram";
1500
- case DataPointType.GAUGE: return "gauge";
1501
- case DataPointType.SUM: return isMonotonic === false ? "upDownCounter" : "counter";
1502
- default: return "gauge";
1503
- }
1796
+ return mapDataPointTypeToWireKind(dataPointType, isMonotonic);
1504
1797
  }
1505
1798
  /** Extract string labels from OTel Attributes. */
1506
1799
  extractLabels(attributes) {
@@ -1581,28 +1874,57 @@ init_npm_resolver();
1581
1874
  /**
1582
1875
  * Startup garbage collection for this agent's own temp-directory families.
1583
1876
  *
1584
- * Job workdirs (`kici-<6 random chars>`, see job-runner.ts) and isolated
1585
- * pnpm stores (`kici-pnpm-store-*`, see dep-installer.ts) clean themselves
1586
- * up in `finally` blocks but a hard process death (SIGKILL, OOM kill)
1587
- * skips those, and on a long-lived bare-metal agent the leftovers then
1588
- * accumulate forever. Collecting anything older than a day at startup is
1589
- * safe on shared hosts: no job lives remotely that long (job timeouts are
1590
- * minutes), so a concurrent agent's in-flight dirs are never eligible.
1877
+ * Bare job workdirs (`kici-<6 random chars>`, see job-runner.ts), labeled
1878
+ * allocator dirs (`kici-<label>-<6 random chars>`, minted by the global temp
1879
+ * allocator `@kici-dev/core/tmp`, which guarantees every allocation carries
1880
+ * the `kici-` prefix), and isolated pnpm stores (`kici-pnpm-store-*`, see
1881
+ * dep-installer.ts) clean themselves up in `finally` blocks but a hard
1882
+ * process death (SIGKILL, OOM kill) skips those, and on a long-lived
1883
+ * bare-metal agent the leftovers then accumulate forever. Collecting anything
1884
+ * older than a day at startup is safe on shared hosts: no job lives remotely
1885
+ * that long (job timeouts are minutes), so a concurrent agent's in-flight
1886
+ * dirs are never eligible.
1887
+ *
1888
+ * The deterministic persistent caches (`kici-agent-payloads`, `kici-data`,
1889
+ * `kici-scaler-ledger`) also live directly under the temp root and can be far
1890
+ * older than a day, so they are explicitly excluded — `kici-scaler-ledger`
1891
+ * even structurally matches the allocator pattern (`ledger` is a 6-char label
1892
+ * suffix), which a regex alone cannot distinguish.
1591
1893
  */
1592
1894
  const AGENT_TMP_GC_MAX_AGE_MS = 1440 * 60 * 1e3;
1593
- /** mkdtemp's 6-char suffix on the bare `kici-` prefix — job workdirs only. */
1594
- const AGENT_WORKDIR_PATTERN = /^kici-[A-Za-z0-9]{6}$/;
1895
+ /**
1896
+ * Bare `kici-<6 chars>` workdirs and labeled `kici-<label>-<6 chars>`
1897
+ * allocator dirs. The optional label group makes both families eligible.
1898
+ */
1899
+ const AGENT_WORKDIR_PATTERN = /^kici-([a-z0-9-]+-)?[A-Za-z0-9]{6}$/;
1595
1900
  const PNPM_STORE_PATTERN = /^kici-pnpm-store-/;
1596
1901
  /**
1902
+ * Deterministic persistent caches that share the `kici-` prefix but must
1903
+ * never be collected — `kici-scaler-ledger` even matches the allocator
1904
+ * pattern, so a basename exclude (not a regex) is the only correct guard.
1905
+ */
1906
+ const PERSISTENT_CACHES = /* @__PURE__ */ new Set([
1907
+ "kici-agent-payloads",
1908
+ "kici-data",
1909
+ "kici-scaler-ledger"
1910
+ ]);
1911
+ /**
1597
1912
  * Collect this agent's stale temp dirs. `base` is overridable for tests;
1598
1913
  * production callers use the default temp root. Never throws.
1914
+ *
1915
+ * The default base is `kiciTmpBase()`, not the OS temp root, so it scans the
1916
+ * exact directory the agent now writes payloads/clones/pnpm stores into: the
1917
+ * global temp allocator and the payload cache both honor `KICI_TMPDIR` via the
1918
+ * same helper. If the GC scanned a different root, stale-temp reaping would
1919
+ * silently stop working whenever `KICI_TMPDIR` is set.
1599
1920
  */
1600
- async function gcStaleAgentTmpDirs(base = tmpdir()) {
1921
+ async function gcStaleAgentTmpDirs(base = kiciTmpBase()) {
1601
1922
  const log = (m) => logger.info(m);
1602
1923
  return [...await gcStaleTmpDirs({
1603
1924
  base,
1604
1925
  pattern: AGENT_WORKDIR_PATTERN,
1605
1926
  maxAgeMs: AGENT_TMP_GC_MAX_AGE_MS,
1927
+ exclude: PERSISTENT_CACHES,
1606
1928
  log
1607
1929
  }), ...await gcStaleTmpDirs({
1608
1930
  base,
@@ -1758,7 +2080,7 @@ var init_prometheus = __esmMin((() => {
1758
2080
  */
1759
2081
  async function setupSshAuth(opts) {
1760
2082
  if (opts.hostKeyPolicy === "pinned" && !opts.knownHosts) throw new Error("pinned hostKeyPolicy requires knownHosts content");
1761
- const tempDir = await mkdtemp(join(tmpdir(), "kici-ssh-"));
2083
+ const { path: tempDir, cleanup } = await makeTempDir("ssh");
1762
2084
  const keyPath = join(tempDir, "id");
1763
2085
  await writeFile(keyPath, opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`, { mode: 384 });
1764
2086
  const knownHostsPath = join(tempDir, "known_hosts");
@@ -1779,12 +2101,7 @@ async function setupSshAuth(opts) {
1779
2101
  return {
1780
2102
  gitSshCommand: parts.join(" "),
1781
2103
  tempDir,
1782
- async cleanup() {
1783
- await rm(tempDir, {
1784
- recursive: true,
1785
- force: true
1786
- });
1787
- }
2104
+ cleanup
1788
2105
  };
1789
2106
  }
1790
2107
  /**
@@ -1836,19 +2153,16 @@ async function gitClone(options) {
1836
2153
  let needsCustomEnv = false;
1837
2154
  let safeDirCleanup;
1838
2155
  if (repoUrl.startsWith("file://")) {
1839
- const { mkdtemp, writeFile, rm } = await import("node:fs/promises");
1840
- const { tmpdir } = await import("node:os");
2156
+ const { writeFile } = await import("node:fs/promises");
1841
2157
  const path = await import("node:path");
1842
- const dir = await mkdtemp(path.join(tmpdir(), "kici-gitcfg-"));
2158
+ const { makeTempDir } = await import("@kici-dev/core/tmp");
2159
+ const { path: dir, cleanup } = await makeTempDir("gitcfg");
1843
2160
  const cfgPath = path.join(dir, "config");
1844
2161
  await writeFile(cfgPath, "[safe]\n directory = *\n", { mode: 384 });
1845
2162
  envEntries.GIT_CONFIG_GLOBAL = cfgPath;
1846
2163
  needsCustomEnv = true;
1847
2164
  safeDirCleanup = async () => {
1848
- await rm(dir, {
1849
- recursive: true,
1850
- force: true
1851
- }).catch(() => {});
2165
+ await cleanup().catch(() => {});
1852
2166
  };
1853
2167
  }
1854
2168
  let sshSetup;
@@ -1961,11 +2275,46 @@ var workflow_loader_exports = /* @__PURE__ */ __exportAll({
1961
2275
  extractSteps: () => extractSteps,
1962
2276
  extractStepsFromDynamicJob: () => extractStepsFromDynamicJob,
1963
2277
  extractWorkflow: () => extractWorkflow,
1964
- loadWorkflowSource: () => loadWorkflowSource
2278
+ loadWorkflowSource: () => loadWorkflowSource,
2279
+ resolveWorkflowSdkSetters: () => resolveWorkflowSdkSetters
1965
2280
  });
2281
+ /**
2282
+ * Resolve the `@kici-dev/sdk` instance the workflow module itself imports.
2283
+ *
2284
+ * The workflow's `.result` proxies read the module-global step-outputs map of
2285
+ * whichever SDK copy the workflow file resolves — which is generally a
2286
+ * different physical module than the agent's bundled SDK (the workflow is
2287
+ * imported from the cloned source tree and resolves its deps against that
2288
+ * tree's `node_modules`). Resolving via `createRequire(workflowFilePath)` walks
2289
+ * `node_modules` from the workflow file exactly the way the workflow's own
2290
+ * `import '@kici-dev/sdk'` does — including any hoisted copy — so the returned
2291
+ * setters mutate the SAME module-global map object the proxies read. Node caches
2292
+ * ESM modules by resolved URL, so importing that path yields the workflow's live
2293
+ * singleton, not a fresh copy.
2294
+ *
2295
+ * Falls back to the agent's bundled setters when resolution fails (mirrors
2296
+ * `resolveSdkSetters` in the compiler's test runner).
2297
+ */
2298
+ async function resolveWorkflowSdkSetters(workflowFilePath) {
2299
+ try {
2300
+ const sdk = await import(pathToFileURL(createRequire(workflowFilePath).resolve("@kici-dev/sdk")).href);
2301
+ if (typeof sdk.setStepOutputsMap === "function" && typeof sdk.setStepRefMap === "function" && typeof sdk.setJobOutputsMap === "function") return {
2302
+ setStepOutputsMap: sdk.setStepOutputsMap,
2303
+ setStepRefMap: sdk.setStepRefMap,
2304
+ setJobOutputsMap: sdk.setJobOutputsMap
2305
+ };
2306
+ } catch {}
2307
+ return {
2308
+ setStepOutputsMap,
2309
+ setStepRefMap,
2310
+ setJobOutputsMap
2311
+ };
2312
+ }
1966
2313
  function ensureLoaderHookRegistered() {
1967
2314
  if (hookRegistered) return;
1968
- register("@kici-dev/core/ts-loader-hook", import.meta.url);
2315
+ const hookPath = process.env.KICI_TS_LOADER_HOOK_PATH;
2316
+ if (hookPath) register(pathToFileURL(hookPath).href, import.meta.url);
2317
+ else register("@kici-dev/core/ts-loader-hook", import.meta.url);
1969
2318
  hookRegistered = true;
1970
2319
  }
1971
2320
  /**
@@ -2020,7 +2369,10 @@ async function loadWorkflowSource(workDir, sourceFile, expectedContentHash, reso
2020
2369
  const actualHash = computeContentHash(rawSource, assetDigest);
2021
2370
  if (actualHash !== expectedContentHash) throw new Error(`Lock file is out of date: workflow source changed without regenerating kici.lock.json (expected contentHash ${expectedContentHash}, got ${actualHash}, agent baked @kici-dev/sdk@${AGENT_SDK_VERSION} bundleHash=${AGENT_SDK_BUNDLE_HASH}). Run 'kici compile' and commit the updated lock file.`);
2022
2371
  }
2023
- return { module: await import(pathToFileURL(filePath).href + `?t=${Date.now()}`) };
2372
+ return {
2373
+ module: await import(pathToFileURL(filePath).href + `?t=${Date.now()}`),
2374
+ sdkSetters: await resolveWorkflowSdkSetters(filePath)
2375
+ };
2024
2376
  }
2025
2377
  /**
2026
2378
  * Type guard for Workflow shape (discriminant: `_tag === 'Workflow'`).
@@ -2118,8 +2470,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
2118
2470
  }
2119
2471
  var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
2120
2472
  var init_workflow_loader = __esmMin((() => {
2121
- AGENT_SDK_VERSION = "0.1.27";
2122
- AGENT_SDK_BUNDLE_HASH = "e06bd1acf349ba365bdb357f3d865e0cbb4ad4604a0270b707a681a4f440954e";
2473
+ AGENT_SDK_VERSION = "0.2.0";
2474
+ AGENT_SDK_BUNDLE_HASH = "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
2123
2475
  hookRegistered = false;
2124
2476
  }));
2125
2477
  //#endregion
@@ -2582,7 +2934,16 @@ async function evaluateDynamicFields(workflow, jobName, event, flags, timeoutMs
2582
2934
  },
2583
2935
  env: { ...process.env }
2584
2936
  };
2585
- let combos = expandMatrix(await withTimeout(() => job.matrix(matrixContext), timeoutMs, `dynamicMatrix for job '${jobName}'`));
2937
+ const resolved = await withTimeout(() => job.matrix(matrixContext), timeoutMs, `dynamicMatrix for job '${jobName}'`);
2938
+ let combos;
2939
+ try {
2940
+ const rawCount = matrixCombinationCount(resolved);
2941
+ if (rawCount > MAX_MATRIX_MATERIALIZATION) throw new MatrixShapeError(`matrix is too large to expand: ${rawCount} raw combinations (max ${MAX_MATRIX_MATERIALIZATION})`);
2942
+ combos = expandMatrix(resolved);
2943
+ } catch (err) {
2944
+ if (err instanceof MatrixShapeError) throw new MatrixShapeError(`dynamicMatrix for job '${jobName}': ${err.message}`);
2945
+ throw err;
2946
+ }
2586
2947
  if (job.include || job.exclude) combos = applyIncludeExclude(combos, job.include, job.exclude);
2587
2948
  result.matrixValues = combos;
2588
2949
  }
@@ -2665,13 +3026,14 @@ async function sshExec(reach, privateKey, command, opts = {}, deps = {}) {
2665
3026
  const hostKeyMode = opts.hostKeyMode ?? "accept-new";
2666
3027
  const { dest, port } = resolveTarget(reach, opts.port);
2667
3028
  return withEphemeralAgent(privateKey, spawnFn, async (env) => {
2668
- return spawnFn("ssh", [
3029
+ const args = [
2669
3030
  ...baseSshOptions(hostKeyMode),
2670
3031
  "-p",
2671
3032
  String(port),
2672
3033
  dest,
2673
3034
  command
2674
- ], {
3035
+ ];
3036
+ return spawnFn("ssh", args, {
2675
3037
  env,
2676
3038
  stdin: opts.stdin
2677
3039
  });
@@ -2687,13 +3049,14 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
2687
3049
  const hostKeyMode = opts.hostKeyMode ?? "accept-new";
2688
3050
  const { dest, port } = resolveTarget(reach, opts.port);
2689
3051
  const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
2690
- return spawnFn("ssh", [
3052
+ const args = [
2691
3053
  ...baseSshOptions(hostKeyMode),
2692
3054
  "-p",
2693
3055
  String(port),
2694
3056
  dest,
2695
3057
  `cat > '${remotePath.replace(/'/g, `'\\''`)}'`
2696
- ], {
3058
+ ];
3059
+ return spawnFn("ssh", args, {
2697
3060
  env,
2698
3061
  stdin: localBytes
2699
3062
  });
@@ -2701,6 +3064,32 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
2701
3064
  if (result.exitCode !== 0) throw new Error(`sshPush(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
2702
3065
  }
2703
3066
  /**
3067
+ * Ship a local FILE to a remote path over `scp` (a binary channel), using the
3068
+ * same ephemeral-key discipline as `sshExec`/`sshPush`.
3069
+ *
3070
+ * Unlike `sshPush` — which pipes a JS string through `ssh 'cat > path'` and so
3071
+ * corrupts binary payloads (a ~50 MB tarball) — `scp` streams the file's raw
3072
+ * bytes untouched. This is the ONLY sanctioned path for a binary payload
3073
+ * transfer; the string `cat >` path is for text (launcher scripts) only. Throws
3074
+ * on a non-zero exit (a payload push must succeed end-to-end).
3075
+ */
3076
+ async function sshPushFile(reach, privateKey, localFilePath, remotePath, opts = {}, deps = {}) {
3077
+ const spawnFn = deps.spawnFn ?? defaultSpawn;
3078
+ const hostKeyMode = opts.hostKeyMode ?? "accept-new";
3079
+ const { dest, port } = resolveTarget(reach, opts.port);
3080
+ const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
3081
+ const args = [
3082
+ ...baseSshOptions(hostKeyMode),
3083
+ "-P",
3084
+ String(port),
3085
+ localFilePath,
3086
+ `${dest}:${remotePath}`
3087
+ ];
3088
+ return spawnFn("scp", args, { env });
3089
+ });
3090
+ if (result.exitCode !== 0) throw new Error(`sshPushFile(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
3091
+ }
3092
+ /**
2704
3093
  * Start a per-call ephemeral ssh-agent, load the key via stdin (never a file),
2705
3094
  * run `body` with `SSH_AUTH_SOCK` in env, and kill the agent in `finally`.
2706
3095
  *
@@ -2716,7 +3105,7 @@ async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, dep
2716
3105
  */
2717
3106
  async function withEphemeralAgent(privateKey, spawnFn, body) {
2718
3107
  const baseEnv = { ...process.env };
2719
- const agentDir = await mkdtemp(join(tmpdir(), "kici-bootstrap-ssh-"));
3108
+ const { path: agentDir, cleanup } = await makeTempDir("bootstrap-ssh");
2720
3109
  const sock = join(agentDir, "agent.sock");
2721
3110
  try {
2722
3111
  const start = await spawnFn("ssh-agent", [
@@ -2729,10 +3118,11 @@ async function withEphemeralAgent(privateKey, spawnFn, body) {
2729
3118
  const agentEnv = {
2730
3119
  ...baseEnv,
2731
3120
  SSH_AUTH_SOCK: sock,
2732
- ...pid ? { SSH_AGENT_PID: pid } : {},
2733
3121
  SSH_ASKPASS: "/bin/false",
2734
3122
  DISPLAY: ""
2735
3123
  };
3124
+ if (pid) agentEnv.SSH_AGENT_PID = pid;
3125
+ else delete agentEnv.SSH_AGENT_PID;
2736
3126
  try {
2737
3127
  const add = await spawnFn("ssh-add", ["-"], {
2738
3128
  env: agentEnv,
@@ -2744,10 +3134,7 @@ async function withEphemeralAgent(privateKey, spawnFn, body) {
2744
3134
  await spawnFn("ssh-agent", ["-k"], { env: agentEnv }).catch(() => {});
2745
3135
  }
2746
3136
  } finally {
2747
- await rm(agentDir, {
2748
- recursive: true,
2749
- force: true
2750
- }).catch(() => {});
3137
+ await cleanup().catch(() => {});
2751
3138
  }
2752
3139
  }
2753
3140
  /** Extract `SSH_AGENT_PID=<n>;` from `ssh-agent -s` output (best-effort). */
@@ -2779,7 +3166,10 @@ var init_ssh_exec = __esmMin((() => {
2779
3166
  stdout,
2780
3167
  stderr
2781
3168
  }));
2782
- if (opts.stdin !== void 0) child.stdin?.end(opts.stdin);
3169
+ if (opts.stdin !== void 0) {
3170
+ child.stdin?.on("error", () => {});
3171
+ child.stdin?.end(opts.stdin);
3172
+ }
2783
3173
  });
2784
3174
  SSH_USER_DEFAULT = "root";
2785
3175
  SSH_PORT_DEFAULT = 22;
@@ -2789,8 +3179,166 @@ var init_ssh_exec = __esmMin((() => {
2789
3179
  };
2790
3180
  }));
2791
3181
  //#endregion
3182
+ //#region src/bootstrap/probe-platform.ts
3183
+ /**
3184
+ * Probe a bring-up target's platform over SSH.
3185
+ *
3186
+ * A stock rescue box has nothing but sshd — no `kici-agent`, no Node — so we
3187
+ * cannot ask the box what to run; we must ask what it IS, then stage the
3188
+ * matching self-contained payload. One `uname -s -m` (+ a libc check) maps to
3189
+ * an `AgentPlatform`; an unsupported target (musl-only, non-Linux, unknown
3190
+ * arch) throws early, naming what was detected, rather than staging bytes that
3191
+ * won't boot.
3192
+ */
3193
+ /**
3194
+ * Map `uname -s -m` (+ libc line) output to an `AgentPlatform`. Pure + directly
3195
+ * unit-tested. Throws a self-describing error for any unsupported target.
3196
+ */
3197
+ function parseUname(stdout) {
3198
+ const lines = stdout.split("\n");
3199
+ const unameLine = (lines[0] ?? "").trim();
3200
+ const libc = lines.slice(1).join("\n").toLowerCase();
3201
+ const [os, arch] = unameLine.split(/\s+/);
3202
+ if (os !== "Linux") throw new Error(`unsupported bring-up target OS "${os}" (uname: "${unameLine}") — Linux only`);
3203
+ if (libc.includes("musl") || libc.includes("ld-musl")) throw new Error(`unsupported bring-up target: musl libc detected (uname: "${unameLine}") — glibc Linux only`);
3204
+ if (arch === "x86_64") return AgentPlatform.enum["linux-x64"];
3205
+ if (arch === "aarch64") return AgentPlatform.enum["linux-arm64"];
3206
+ throw new Error(`unsupported bring-up target arch "${arch}" (uname: "${unameLine}") — x86_64 or aarch64 only`);
3207
+ }
3208
+ /** SSH into `reach` and detect its `AgentPlatform` (one probe, fail-fast on unsupported). */
3209
+ async function probeTargetPlatform(reach, privateKey, deps = {}) {
3210
+ const res = await sshExec(reach, privateKey, PROBE_COMMAND, {}, deps);
3211
+ if (res.exitCode !== 0) throw new Error(`platform probe on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
3212
+ return parseUname(res.stdout);
3213
+ }
3214
+ var PROBE_COMMAND;
3215
+ var init_probe_platform = __esmMin((() => {
3216
+ init_ssh_exec();
3217
+ PROBE_COMMAND = "uname -s -m; (ldd --version 2>&1 | head -1) || true";
3218
+ }));
3219
+ //#endregion
3220
+ //#region src/bootstrap/stage-agent-payload.ts
3221
+ /**
3222
+ * Stage a self-contained agent payload onto a bring-up target.
3223
+ *
3224
+ * The reusable core mechanism behind fresh-box bring-up: resolve a version-keyed
3225
+ * payload from an `AgentPayloadSource`, verify it, deliver it to the box, verify
3226
+ * it again ON the box before extracting, and return the launcher path. The
3227
+ * launcher boots the agent on its VENDORED Node, so a stock rescue box (no Node)
3228
+ * becomes a connected agent.
3229
+ *
3230
+ * Two delivery modes: `ssh-push` (the ops agent pulls the payload from its
3231
+ * source, then streams it to the box over a binary-safe scp) and `s3-direct`
3232
+ * (the box pulls a presigned URL itself, so no 50 MB transits the ops agent).
3233
+ * Every path is fail-closed: a hash it cannot verify is never extracted.
3234
+ */
3235
+ /** Streamed sha256 of a local file (the default ops-agent-side verify). */
3236
+ function hashFileSha256(filePath) {
3237
+ return new Promise((resolve, reject) => {
3238
+ const hash = createHash("sha256");
3239
+ createReadStream(filePath).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve(hash.digest("hex")));
3240
+ });
3241
+ }
3242
+ /** Single-quote a value for safe embedding in a remote shell command. */
3243
+ function shQuote$2(v) {
3244
+ return `'${v.replace(/'/g, `'\\''`)}'`;
3245
+ }
3246
+ /**
3247
+ * Verify the resolved payload on the ops agent BEFORE any transfer. A payload
3248
+ * with no sidecar hash is unverifiable and refused (fail-closed); a hash
3249
+ * mismatch throws so corrupt/tampered bytes never leave this process.
3250
+ */
3251
+ async function verifyLocalPayload(tarballPath, expectedSha256, deps) {
3252
+ if (!expectedSha256) throw new Error(`refusing to stage ${tarballPath}: no sha256 sidecar — cannot verify the payload (run \`kici-admin agent package\` to (re)generate it with a hash)`);
3253
+ const actual = await (deps.hashLocalFile ?? hashFileSha256)(tarballPath);
3254
+ if (actual !== expectedSha256) throw new Error(`payload hash mismatch for ${tarballPath}: expected ${expectedSha256}, got ${actual}`);
3255
+ return expectedSha256;
3256
+ }
3257
+ /**
3258
+ * On the box: verify the pushed tarball's hash, then extract it into
3259
+ * `extractDir`. `sha256sum -c` fails the whole `&&` chain if the hash differs,
3260
+ * so extraction only ever runs on verified bytes.
3261
+ */
3262
+ async function verifyAndExtractRemote(reach, privateKey, sha256, extractDir, deps) {
3263
+ const remoteBase = path.posix.basename(REMOTE_TARBALL_PATH);
3264
+ const res = await sshExec(reach, privateKey, [
3265
+ `cd ${shQuote$2(path.posix.dirname(REMOTE_TARBALL_PATH))}`,
3266
+ `echo ${shQuote$2(`${sha256} ${remoteBase}`)} | sha256sum -c -`,
3267
+ `mkdir -p ${shQuote$2(extractDir)}`,
3268
+ `tar xzf ${shQuote$2(remoteBase)} -C ${shQuote$2(extractDir)}`
3269
+ ].join(" && "), {}, deps);
3270
+ if (res.exitCode !== 0) throw new Error(`on-box verify/extract on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
3271
+ }
3272
+ /** Resolve → verify locally → push → verify+extract on box. */
3273
+ async function stageSshPush(reach, privateKey, opts, extractDir, deps) {
3274
+ if (!deps.payloadSource) throw new Error("ssh-push delivery requires a payload source (KICI_AGENT_BINARY_SOURCE / KICI_AGENT_PAYLOAD_DIR)");
3275
+ const staged = await deps.payloadSource.resolve(opts.platform, opts.version);
3276
+ const sha256 = await verifyLocalPayload(staged.tarballPath, staged.sha256, deps);
3277
+ await sshPushFile(reach, privateKey, staged.tarballPath, REMOTE_TARBALL_PATH, {}, deps);
3278
+ await verifyAndExtractRemote(reach, privateKey, sha256, extractDir, deps);
3279
+ }
3280
+ /**
3281
+ * `s3-direct`: the box pulls the presigned URL itself, verifies the sha256, and
3282
+ * extracts — all in one `sshExec`, so the 50 MB payload never transits the ops
3283
+ * agent. Fail-closed: `sha256sum -c` breaks the `&&` chain on mismatch, so the
3284
+ * box only ever extracts verified bytes, and `curl -f` fails the chain on a
3285
+ * non-2xx (expired/absent presign).
3286
+ */
3287
+ async function stageS3Direct(reach, privateKey, presignedUrl, sha256, extractDir, deps) {
3288
+ const remoteBase = path.posix.basename(REMOTE_TARBALL_PATH);
3289
+ const res = await sshExec(reach, privateKey, [
3290
+ `cd ${shQuote$2(path.posix.dirname(REMOTE_TARBALL_PATH))}`,
3291
+ `curl -fsSL ${shQuote$2(presignedUrl)} -o ${shQuote$2(remoteBase)}`,
3292
+ `echo ${shQuote$2(`${sha256} ${remoteBase}`)} | sha256sum -c -`,
3293
+ `mkdir -p ${shQuote$2(extractDir)}`,
3294
+ `tar xzf ${shQuote$2(remoteBase)} -C ${shQuote$2(extractDir)}`
3295
+ ].join(" && "), {}, deps);
3296
+ if (res.exitCode !== 0) throw new Error(`s3-direct pull/verify/extract on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
3297
+ }
3298
+ /**
3299
+ * Stage a payload onto `reach` and return the launcher path the caller runs as
3300
+ * the init-runner's `agentCommand`.
3301
+ */
3302
+ async function stageAgentPayload(reach, privateKey, opts, deps) {
3303
+ const extractDir = deps.extractDir ?? DEFAULT_EXTRACT_DIR;
3304
+ switch (opts.delivery.mode) {
3305
+ case "ssh-push":
3306
+ await stageSshPush(reach, privateKey, opts, extractDir, deps);
3307
+ break;
3308
+ case "s3-direct":
3309
+ await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
3310
+ break;
3311
+ }
3312
+ return { launcherPath: path.posix.join(extractDir, "kici-agent") };
3313
+ }
3314
+ var DEFAULT_EXTRACT_DIR, REMOTE_TARBALL_PATH;
3315
+ var init_stage_agent_payload = __esmMin((() => {
3316
+ init_ssh_exec();
3317
+ DEFAULT_EXTRACT_DIR = "/opt/kici-init";
3318
+ REMOTE_TARBALL_PATH = "/tmp/kici-agent-payload.tar.gz";
3319
+ }));
3320
+ //#endregion
2792
3321
  //#region src/bootstrap/ensure-init-runner.ts
2793
3322
  /**
3323
+ * Agent-side init-runner bring-up.
3324
+ *
3325
+ * Runs in the AGENT process (never the workflow sandbox), so the bring-up SSH
3326
+ * key and bootstrap token the orchestrator hands back never reach user
3327
+ * workflow code. The flow:
3328
+ *
3329
+ * 1. Call the orchestrator's privileged `kici.ensureInitRunner` handler — it
3330
+ * gates on this agent's `kici:capability:ssh-transport` capability,
3331
+ * resolves the target's reach + SSH key, mints a single-use bootstrap
3332
+ * token, audits, and returns the material (or `{ broughtUp: false }` when
3333
+ * the target already has a live agent).
3334
+ * 2. Over SSH (ephemeral key, never on disk): drop a launcher onto the target
3335
+ * that starts `kici-agent` with the bootstrap env (token + agent id +
3336
+ * orchestrator URL + labels), and start it detached.
3337
+ *
3338
+ * The init-runner then connects → `auth.request` (bootstrap token) →
3339
+ * `agent.register` auto-enroll as a temporary `kici:init` agent.
3340
+ */
3341
+ /**
2794
3342
  * Build the launcher script that starts the init-runner on the target with its
2795
3343
  * bootstrap env. Detached (`setsid … &`) so the SSH session can return while
2796
3344
  * the agent keeps running and dials the orchestrator.
@@ -2800,10 +3348,11 @@ function buildLauncher(material, agentCommand) {
2800
3348
  "#!/usr/bin/env bash",
2801
3349
  "set -euo pipefail",
2802
3350
  `setsid env ${[
2803
- `KICI_AGENT_TOKEN=${shQuote(material.bootstrapToken)}`,
2804
- `KICI_AGENT_ID=${shQuote(material.targetAgentId)}`,
2805
- `KICI_ORCHESTRATOR_URL=${shQuote(material.orchestratorUrl)}`,
2806
- `KICI_LABELS=${shQuote(material.labels.join(","))}`,
3351
+ `KICI_AGENT_TOKEN=${shQuote$1(material.bootstrapToken)}`,
3352
+ `KICI_AGENT_ID=${shQuote$1(material.targetAgentId)}`,
3353
+ `KICI_ORCHESTRATOR_URL=${shQuote$1(material.orchestratorUrl)}`,
3354
+ `KICI_LABELS=${shQuote$1(material.labels.join(","))}`,
3355
+ "KICI_ROLES=",
2807
3356
  "KICI_EXECUTION_MODE=bare-metal",
2808
3357
  "KICI_PORT=0"
2809
3358
  ].join(" \\\n ")} \\`,
@@ -2812,10 +3361,60 @@ function buildLauncher(material, agentCommand) {
2812
3361
  ].join("\n");
2813
3362
  }
2814
3363
  /** Single-quote a value for safe embedding in the launcher's env assignment. */
2815
- function shQuote(v) {
3364
+ function shQuote$1(v) {
2816
3365
  return `'${v.replace(/'/g, `'\\''`)}'`;
2817
3366
  }
2818
3367
  /**
3368
+ * Resolve the command the launcher invokes to start the init-runner.
3369
+ *
3370
+ * - A `deps.agentCommand` override short-circuits staging (golden image).
3371
+ * - Otherwise a payload source is required: probe the target's platform, stage
3372
+ * the version-keyed self-contained payload over SSH, and return the staged
3373
+ * launcher path (which boots the agent on its vendored Node — no system Node).
3374
+ * - No override AND no source is a misconfiguration, not a silent Node
3375
+ * assumption — throw a clear error.
3376
+ */
3377
+ async function resolveAgentCommand(transport, reach, privateKey, material, deps) {
3378
+ if (deps.agentCommand) return deps.agentCommand;
3379
+ if (!material.version) throw new Error(`orchestrator returned no version for ${material.targetAgentId} — cannot stage a payload`);
3380
+ if (!(deps.delivery?.mode === AgentDeliveryMode.enum["s3-direct"] || material.deliveryMode === AgentDeliveryMode.enum["s3-direct"]) && !deps.payloadSource) throw new Error(`no agent payload source configured for bringing up ${material.targetAgentId}: set KICI_AGENT_BINARY_SOURCE (object storage), KICI_AGENT_PAYLOAD_DIR (air-gap), or KICI_AGENT_COMMAND (golden image)`);
3381
+ const platform = await probeTargetPlatform(reach, privateKey, deps);
3382
+ const delivery = await resolveDelivery$1(transport, material, platform, deps);
3383
+ const { launcherPath } = await stageAgentPayload(reach, privateKey, {
3384
+ platform,
3385
+ version: material.version,
3386
+ delivery
3387
+ }, {
3388
+ spawnFn: deps.spawnFn,
3389
+ payloadSource: deps.payloadSource,
3390
+ extractDir: deps.extractDir,
3391
+ hashLocalFile: deps.hashLocalFile
3392
+ });
3393
+ return launcherPath;
3394
+ }
3395
+ /**
3396
+ * Decide how the payload reaches the box. An explicit `deps.delivery` override
3397
+ * wins (test/escape hatch); otherwise honor the orchestrator's per-host choice.
3398
+ * For `s3-direct` we ask the orchestrator (which knows the probed platform now)
3399
+ * to mint a box-routable presigned URL via `kici.presignAgentPackage`; a payload
3400
+ * with no sha256 is refused (fail-closed — never extract unverifiable bytes).
3401
+ */
3402
+ async function resolveDelivery$1(transport, material, platform, deps) {
3403
+ if (deps.delivery) return deps.delivery;
3404
+ if (material.deliveryMode !== AgentDeliveryMode.enum["s3-direct"]) return { mode: "ssh-push" };
3405
+ const presigned = await transport("kici.presignAgentPackage", {
3406
+ targetAgentId: material.targetAgentId,
3407
+ platform
3408
+ });
3409
+ if (!presigned?.url) throw new Error(`orchestrator returned no presigned URL for ${material.targetAgentId}`);
3410
+ if (!presigned.sha256) throw new Error(`refusing s3-direct delivery for ${material.targetAgentId}: no sha256 for the payload (cannot verify on the box)`);
3411
+ return {
3412
+ mode: "s3-direct",
3413
+ presignedUrl: presigned.url,
3414
+ sha256: presigned.sha256
3415
+ };
3416
+ }
3417
+ /**
2819
3418
  * Bring up a temporary init-runner on `targetAgentId`. Returns `{ broughtUp }`:
2820
3419
  * false when the target already had a live agent (the orchestrator no-op'd),
2821
3420
  * true when this call dropped + started the init-runner.
@@ -2825,7 +3424,7 @@ async function ensureInitRunner(transport, targetAgentId, deps = {}) {
2825
3424
  if (!material.broughtUp) return { broughtUp: false };
2826
3425
  const { reach, privateKey, bootstrapToken, orchestratorUrl, labels } = material;
2827
3426
  if (!reach || !privateKey || !bootstrapToken || !orchestratorUrl || !labels) throw new Error(`orchestrator returned incomplete bring-up material for ${targetAgentId}`);
2828
- const agentCommand = deps.agentCommand ?? DEFAULT_AGENT_COMMAND;
3427
+ const agentCommand = await resolveAgentCommand(transport, reach, privateKey, material, deps);
2829
3428
  await sshPush(reach, privateKey, buildLauncher({
2830
3429
  bootstrapToken,
2831
3430
  targetAgentId,
@@ -2836,10 +3435,11 @@ async function ensureInitRunner(transport, targetAgentId, deps = {}) {
2836
3435
  if (run.exitCode !== 0) throw new Error(`init-runner launch on ${targetAgentId} failed: exit ${run.exitCode}${run.stderr ? `\n${run.stderr}` : ""}`);
2837
3436
  return { broughtUp: true };
2838
3437
  }
2839
- var DEFAULT_AGENT_COMMAND, LAUNCHER_REMOTE_PATH;
3438
+ var LAUNCHER_REMOTE_PATH;
2840
3439
  var init_ensure_init_runner = __esmMin((() => {
2841
3440
  init_ssh_exec();
2842
- DEFAULT_AGENT_COMMAND = "kici-agent";
3441
+ init_probe_platform();
3442
+ init_stage_agent_payload();
2843
3443
  LAUNCHER_REMOTE_PATH = "/tmp/kici-init-runner.sh";
2844
3444
  }));
2845
3445
  //#endregion
@@ -2866,6 +3466,141 @@ var init_pre_boot_send = __esmMin((() => {
2866
3466
  init_ssh_exec();
2867
3467
  }));
2868
3468
  //#endregion
3469
+ //#region src/bootstrap/restage-agent.ts
3470
+ /**
3471
+ * External-actor agent re-stage + restart (fleet auto-upgrade apply).
3472
+ *
3473
+ * Runs in the OPS agent (the one holding `kici:capability:ssh-transport`) —
3474
+ * never on the target host itself, so there is NO self-update-handoff: an
3475
+ * external actor swaps the bytes and restarts the target's agent, which
3476
+ * reconnects on its OWN persistent token (this function never mints or handles
3477
+ * a token). The install is folder-anchored (mirroring the versioned-upgrade
3478
+ * layout): the target version is staged into `<installDir>/kici-agent-<version>`
3479
+ * and an atomic symlink swap makes it current, so a stage failure aborts BEFORE
3480
+ * the swap and never leaves a half-upgraded install. Idempotent — a box already
3481
+ * on the target version is a no-op.
3482
+ */
3483
+ /** Single-quote a value for safe embedding in a remote shell command. */
3484
+ function shQuote(v) {
3485
+ return `'${v.replace(/'/g, `'\\''`)}'`;
3486
+ }
3487
+ /** The versioned payload directory for a version under the install base. */
3488
+ function versionDir(installDir, version) {
3489
+ return path.posix.join(installDir, `kici-agent-${version}`);
3490
+ }
3491
+ /**
3492
+ * Read the version the `kici-agent` symlink currently points at, or null when
3493
+ * no install is present. The symlink target is `kici-agent-<version>`.
3494
+ */
3495
+ async function readCurrentVersion(reach, privateKey, installDir, deps) {
3496
+ const target = (await sshExec(reach, privateKey, `readlink ${shQuote(path.posix.join(installDir, "kici-agent"))} 2>/dev/null || true`, {}, deps)).stdout.trim();
3497
+ const match = /(?:^|\/)kici-agent-(.+)$/.exec(target);
3498
+ return match ? match[1] : null;
3499
+ }
3500
+ /**
3501
+ * Atomically point `<installDir>/kici-agent` at the target version directory.
3502
+ * `ln -sfn` onto a temp name + `mv -T` is a single rename, so a reader never
3503
+ * sees a missing symlink. The target is stored relative so the tree is movable.
3504
+ */
3505
+ async function swapInstall(reach, privateKey, installDir, version, deps) {
3506
+ const link = path.posix.join(installDir, "kici-agent");
3507
+ const tmp = `${link}.swap`;
3508
+ const res = await sshExec(reach, privateKey, [`ln -sfn ${shQuote(`kici-agent-${version}`)} ${shQuote(tmp)}`, `mv -T ${shQuote(tmp)} ${shQuote(link)}`].join(" && "), {}, deps);
3509
+ if (res.exitCode !== 0) throw new Error(`install swap on ${reach.agentId} failed: exit ${res.exitCode}${res.stderr ? `\n${res.stderr}` : ""}`);
3510
+ }
3511
+ /** Run the drain (stop) then the restart (start) over SSH. */
3512
+ async function drainAndRestart(reach, privateKey, restart, deps) {
3513
+ const stopRes = await sshExec(reach, privateKey, restart.stop, {}, deps);
3514
+ if (stopRes.exitCode !== 0) throw new Error(`agent drain on ${reach.agentId} failed: exit ${stopRes.exitCode}${stopRes.stderr ? `\n${stopRes.stderr}` : ""}`);
3515
+ const startRes = await sshExec(reach, privateKey, restart.start, {}, deps);
3516
+ if (startRes.exitCode !== 0) throw new Error(`agent restart on ${reach.agentId} failed: exit ${startRes.exitCode}${startRes.stderr ? `\n${startRes.stderr}` : ""}`);
3517
+ }
3518
+ /**
3519
+ * Re-stage the target version onto `reach` and restart its agent. Returns
3520
+ * `{ restaged: false }` when the host is already on the target version (no
3521
+ * stage, no swap, no restart), `{ restaged: true }` when this call swapped it.
3522
+ */
3523
+ async function restageAgent(reach, privateKey, opts, deps) {
3524
+ const installDir = opts.installDir ?? DEFAULT_INSTALL_DIR;
3525
+ if (await readCurrentVersion(reach, privateKey, installDir, deps) === opts.version) return { restaged: false };
3526
+ await stageAgentPayload(reach, privateKey, {
3527
+ platform: opts.platform ?? await probeTargetPlatform(reach, privateKey, deps),
3528
+ version: opts.version,
3529
+ delivery: opts.delivery
3530
+ }, {
3531
+ spawnFn: deps.spawnFn,
3532
+ payloadSource: deps.payloadSource,
3533
+ extractDir: versionDir(installDir, opts.version),
3534
+ hashLocalFile: deps.hashLocalFile
3535
+ });
3536
+ await swapInstall(reach, privateKey, installDir, opts.version, deps);
3537
+ await drainAndRestart(reach, privateKey, opts.restart, deps);
3538
+ return { restaged: true };
3539
+ }
3540
+ var DEFAULT_INSTALL_DIR;
3541
+ var init_restage_agent = __esmMin((() => {
3542
+ init_ssh_exec();
3543
+ init_probe_platform();
3544
+ init_stage_agent_payload();
3545
+ DEFAULT_INSTALL_DIR = "/opt/kici-agent";
3546
+ }));
3547
+ //#endregion
3548
+ //#region src/bootstrap/run-restage.ts
3549
+ /**
3550
+ * Agent-process driver for the `kici.restageAgent` fleet-upgrade apply.
3551
+ *
3552
+ * Intercepted in the ops agent (like `ensureInitRunner`): the privileged
3553
+ * resolve — capability gate, availability gate, reach + SSH key, restart spec —
3554
+ * runs on the orchestrator; this driver performs the SSH transport (probe →
3555
+ * stage → swap → restart) so the bring-up key never reaches user workflow code.
3556
+ * The re-staged permanent agent reconnects on its own persistent credential, so
3557
+ * no token is minted or handled here (no self-update-handoff).
3558
+ */
3559
+ /**
3560
+ * Resolve the delivery mode for the re-stage. `ssh-push` needs a local payload
3561
+ * source; `s3-direct` asks the orchestrator to mint a box-routable presigned URL
3562
+ * (keyed by the now-probed platform) and refuses a payload with no sha256
3563
+ * (fail-closed — never extract unverifiable bytes on the box).
3564
+ */
3565
+ async function resolveDelivery(transport, targetAgentId, deliveryMode, platform) {
3566
+ if (deliveryMode !== AgentDeliveryMode.enum["s3-direct"]) return { mode: "ssh-push" };
3567
+ const presigned = await transport("kici.presignAgentPackage", {
3568
+ targetAgentId,
3569
+ platform
3570
+ });
3571
+ if (!presigned?.url) throw new Error(`orchestrator returned no presigned URL for re-staging ${targetAgentId}`);
3572
+ if (!presigned.sha256) throw new Error(`refusing s3-direct re-stage for ${targetAgentId}: no sha256 for the payload (cannot verify on the box)`);
3573
+ return {
3574
+ mode: "s3-direct",
3575
+ presignedUrl: presigned.url,
3576
+ sha256: presigned.sha256
3577
+ };
3578
+ }
3579
+ /**
3580
+ * Drive one external-actor re-stage: fetch the material, probe the target
3581
+ * platform, resolve delivery, then stage + swap + restart via {@link restageAgent}.
3582
+ * Returns `{ restaged: false }` when the host is already on the target version.
3583
+ */
3584
+ async function runRestage(transport, targetAgentId, deps) {
3585
+ const { reach, privateKey, version, deliveryMode, restart } = await transport("kici.restageAgent", { targetAgentId });
3586
+ if (!reach || !privateKey || !version || !restart) throw new Error(`orchestrator returned incomplete re-stage material for ${targetAgentId}`);
3587
+ const platform = await probeTargetPlatform(reach, privateKey, deps);
3588
+ return restageAgent(reach, privateKey, {
3589
+ platform,
3590
+ version,
3591
+ delivery: await resolveDelivery(transport, targetAgentId, deliveryMode, platform),
3592
+ ...restart.installDir ? { installDir: restart.installDir } : {},
3593
+ restart: {
3594
+ stop: restart.stop,
3595
+ start: restart.start
3596
+ }
3597
+ }, deps);
3598
+ }
3599
+ var init_run_restage = __esmMin((() => {
3600
+ init_probe_platform();
3601
+ init_restage_agent();
3602
+ }));
3603
+ //#endregion
2869
3604
  //#region src/bootstrap/api-intercept.ts
2870
3605
  /**
2871
3606
  * Wrap the orchestrator API transport so the two bootstrap methods are handled
@@ -2884,15 +3619,131 @@ function withBootstrapInterception(relay, deps = {}) {
2884
3619
  }, deps);
2885
3620
  return;
2886
3621
  }
3622
+ if (method === RESTAGE_AGENT) return runRestage(relay, String(params.targetAgentId ?? ""), deps);
2887
3623
  return relay(method, params);
2888
3624
  };
2889
3625
  }
2890
- var ENSURE_INIT_RUNNER, PRE_BOOT_SEND;
3626
+ var ENSURE_INIT_RUNNER, PRE_BOOT_SEND, RESTAGE_AGENT;
2891
3627
  var init_api_intercept = __esmMin((() => {
2892
3628
  init_ensure_init_runner();
2893
3629
  init_pre_boot_send();
3630
+ init_run_restage();
2894
3631
  ENSURE_INIT_RUNNER = "kici.ensureInitRunner";
2895
3632
  PRE_BOOT_SEND = "kici.preBootSend";
3633
+ RESTAGE_AGENT = "kici.restageAgent";
3634
+ }));
3635
+ //#endregion
3636
+ //#region src/bootstrap/payload-source.ts
3637
+ /**
3638
+ * Agent payload source abstraction.
3639
+ *
3640
+ * A one-method plug so the place a self-contained agent+Node payload comes from
3641
+ * is swappable and overridable: the local-dir fallback here reads the tarballs
3642
+ * a `kici-admin agent package` producer wrote to disk; an S3-backed source
3643
+ * (fetching from the orchestrator cache bucket via presigned URL) plugs in the
3644
+ * same interface. Payloads are version-keyed and fail-fast — a missing version
3645
+ * throws loudly rather than silently staging stale bytes.
3646
+ */
3647
+ /** The payload tarball name for a platform (matches the producer's layout). */
3648
+ function tarballName$1(platform) {
3649
+ return `kici-agent-${platform}.tar.gz`;
3650
+ }
3651
+ /** Extract the leading hex token from a `sha256sum`-style `<hex> <name>` line. */
3652
+ function parseSidecarHash(contents) {
3653
+ const first = contents.trim().split(/\s+/)[0];
3654
+ if (!first) throw new Error("empty .sha256 sidecar");
3655
+ return first;
3656
+ }
3657
+ var defaultPayloadFs, LocalDirPayloadSource;
3658
+ var init_payload_source = __esmMin((() => {
3659
+ defaultPayloadFs = {
3660
+ exists: async (filePath) => {
3661
+ try {
3662
+ await access(filePath);
3663
+ return true;
3664
+ } catch {
3665
+ return false;
3666
+ }
3667
+ },
3668
+ readFile: (filePath) => readFile(filePath, "utf8")
3669
+ };
3670
+ LocalDirPayloadSource = class {
3671
+ baseDir;
3672
+ fs;
3673
+ constructor(baseDir, fs = defaultPayloadFs) {
3674
+ this.baseDir = baseDir;
3675
+ this.fs = fs;
3676
+ }
3677
+ async resolve(platform, version) {
3678
+ const tarballPath = path.join(this.baseDir, version, tarballName$1(platform));
3679
+ if (!await this.fs.exists(tarballPath)) throw new Error(`no agent payload for version ${version} (${platform}) at ${tarballPath} — run \`kici-admin agent package --platform ${platform} --out ${this.baseDir}\``);
3680
+ const sidecarPath = `${tarballPath}.sha256`;
3681
+ return {
3682
+ tarballPath,
3683
+ sha256: await this.fs.exists(sidecarPath) ? parseSidecarHash(await this.fs.readFile(sidecarPath)) : null
3684
+ };
3685
+ }
3686
+ };
3687
+ }));
3688
+ //#endregion
3689
+ //#region src/bootstrap/s3-payload-source.ts
3690
+ /**
3691
+ * Object-storage agent payload source — the ops agent's PRIMARY source.
3692
+ *
3693
+ * Resolves a version-keyed self-contained agent+Node payload from the
3694
+ * orchestrator's own cache bucket via a presigned GET URL the orchestrator
3695
+ * mints (`kici.presignAgentPackage`), fetches it once into a local cache dir,
3696
+ * and returns it for the SSH-push delivery path. No standing S3 credential ever
3697
+ * reaches the ops agent — only a time-limited presigned URL. Payloads are
3698
+ * version-keyed and fail-fast: a missing object throws loudly (names the
3699
+ * version + platform) rather than staging stale bytes.
3700
+ *
3701
+ * For the `s3-direct` delivery path the box pulls the presigned URL itself, so
3702
+ * this source is not involved there; it backs `ssh-push` (box cannot reach
3703
+ * object storage). `LocalDirPayloadSource` remains the air-gap fallback.
3704
+ */
3705
+ /** The payload tarball name for a platform (matches the producer's layout). */
3706
+ function tarballName(platform) {
3707
+ return `kici-agent-${platform}.tar.gz`;
3708
+ }
3709
+ /** Default cache root the ops agent stores pulled payloads under (cache-once). */
3710
+ function defaultPayloadCacheDir() {
3711
+ return path.join(kiciTmpBase(), "kici-agent-payloads");
3712
+ }
3713
+ /** Default existence check for the cache-once short-circuit. */
3714
+ async function payloadFileExists(filePath) {
3715
+ try {
3716
+ await access(filePath);
3717
+ return true;
3718
+ } catch {
3719
+ return false;
3720
+ }
3721
+ }
3722
+ /** Stream a (presigned) URL into a local file, binary-safe (no in-memory buffer). */
3723
+ async function fetchUrlToFile(url, dest) {
3724
+ await mkdir(path.dirname(dest), { recursive: true });
3725
+ const res = await fetch(url);
3726
+ if (!res.ok || !res.body) throw new Error(`fetch payload failed: HTTP ${res.status} ${res.statusText}`);
3727
+ await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
3728
+ }
3729
+ var S3PayloadSource;
3730
+ var init_s3_payload_source = __esmMin((() => {
3731
+ S3PayloadSource = class {
3732
+ deps;
3733
+ constructor(deps) {
3734
+ this.deps = deps;
3735
+ }
3736
+ async resolve(platform, version) {
3737
+ const presigned = await this.deps.presign(platform, version);
3738
+ if (!presigned) throw new Error(`no agent payload for version ${version} (${platform}) in the orchestrator cache bucket — run \`kici-admin agent package --platform ${platform} --upload\` to publish it`);
3739
+ const tarballPath = path.join(this.deps.cacheDir, version, tarballName(platform));
3740
+ if (!(this.deps.exists ? await this.deps.exists(tarballPath) : false)) await this.deps.fetchToFile(presigned.url, tarballPath);
3741
+ return {
3742
+ tarballPath,
3743
+ sha256: presigned.sha256
3744
+ };
3745
+ }
3746
+ };
2896
3747
  }));
2897
3748
  //#endregion
2898
3749
  //#region src/execution/streaming-zx-log.ts
@@ -2921,20 +3772,30 @@ var init_api_intercept = __esmMin((() => {
2921
3772
  * `verbose: false` (suppressed). A `verbose: false` base would flag ordinary
2922
3773
  * output `verbose: false` too, and this gate would then drop every line.
2923
3774
  *
2924
- * The returned callback owns a per-shell line buffer, so partial chunks are
2925
- * coalesced into whole lines before `emit` is called.
3775
+ * The returned callback owns one line buffer PER STREAM, so partial chunks are
3776
+ * coalesced into whole lines before `emit` is called. The buffers are separate
3777
+ * because the two streams are independent pipes: a stdout chunk ending mid-line
3778
+ * and a stderr chunk arriving next would otherwise concatenate into a single
3779
+ * spliced line attributed to whichever kind completed it.
3780
+ *
3781
+ * `emit` receives the originating stream alongside the line so a diagnostic
3782
+ * written to stderr stays distinguishable from ordinary progress output all the
3783
+ * way to the persisted run log.
2926
3784
  */
2927
3785
  function makeStreamingZxLog(emit) {
2928
- let lineBuf = "";
3786
+ const lineBufs = {
3787
+ [LogStream.enum.stdout]: "",
3788
+ [LogStream.enum.stderr]: ""
3789
+ };
2929
3790
  return (entry) => {
2930
3791
  const e = entry;
2931
3792
  if (e.kind !== "stdout" && e.kind !== "stderr") return;
2932
3793
  if (!e.verbose) return;
3794
+ const stream = e.kind === "stderr" ? LogStream.enum.stderr : LogStream.enum.stdout;
2933
3795
  const text = typeof e.data === "string" ? e.data : String(e.data ?? "");
2934
- lineBuf += text;
2935
- const lines = lineBuf.split("\n");
2936
- lineBuf = lines.pop();
2937
- for (const line of lines) if (line) emit(line);
3796
+ const lines = (lineBufs[stream] + text).split("\n");
3797
+ lineBufs[stream] = lines.pop();
3798
+ for (const line of lines) if (line) emit(line, stream);
2938
3799
  };
2939
3800
  }
2940
3801
  var init_streaming_zx_log = __esmMin((() => {}));
@@ -3163,6 +4024,8 @@ var init_log_streamer = __esmMin((() => {
3163
4024
  flushTimer = null;
3164
4025
  totalBytes = 0;
3165
4026
  truncated = false;
4027
+ /** Which stream the currently-buffered lines came from. */
4028
+ bufferStream = LogStream.enum.stdout;
3166
4029
  /** Number of lines dropped due to backpressure (drop mode). */
3167
4030
  droppedCount = 0;
3168
4031
  /** Whether we are currently in a backpressured state (pause mode). */
@@ -3202,9 +4065,24 @@ var init_log_streamer = __esmMin((() => {
3202
4065
  /**
3203
4066
  * Add a line to the buffer. Triggers flush if threshold reached,
3204
4067
  * otherwise schedules a timer-based flush.
4068
+ *
4069
+ * A chunk carries a single stream, so a kind flip closes the pending chunk
4070
+ * before the new line is buffered. Chunks are delivered in order, so the
4071
+ * stdout/stderr interleaving is preserved across that boundary.
4072
+ *
4073
+ * `bufferStream` is only advanced once the buffer is actually empty. Under
4074
+ * pause-mode backpressure `flush()` deliberately leaves the buffer intact
4075
+ * until the socket drains, so the flip cannot close the pending chunk;
4076
+ * advancing the tag there would relabel the already-buffered lines as the
4077
+ * newly-arrived stream. The buffer keeps the tag of its first lines instead,
4078
+ * which means a chunk assembled while backpressured reports one stream for
4079
+ * lines that came from both — stderr in such a chunk is under-reported as
4080
+ * stdout. Separating them needs a queue of pending per-stream chunks.
3205
4081
  */
3206
- addLine(line) {
4082
+ addLine(line, stream = LogStream.enum.stdout) {
3207
4083
  if (this.truncated) return;
4084
+ if (this.buffer.length > 0 && stream !== this.bufferStream) this.flush();
4085
+ if (this.buffer.length === 0) this.bufferStream = stream;
3208
4086
  if (this.totalBytes >= this.maxLogSizeBytes) {
3209
4087
  this.buffer.push(`[TRUNCATED: log output exceeded ${this.maxLogSizeBytes} bytes]`);
3210
4088
  this.truncated = true;
@@ -3259,7 +4137,8 @@ var init_log_streamer = __esmMin((() => {
3259
4137
  jobId: this.jobId,
3260
4138
  stepIndex: this.stepIndex,
3261
4139
  lines,
3262
- timestamp: Date.now()
4140
+ timestamp: Date.now(),
4141
+ stream: this.bufferStream
3263
4142
  });
3264
4143
  }
3265
4144
  /**
@@ -3321,7 +4200,8 @@ var init_log_streamer = __esmMin((() => {
3321
4200
  jobId: this.jobId,
3322
4201
  stepIndex: this.stepIndex,
3323
4202
  lines,
3324
- timestamp: Date.now()
4203
+ timestamp: Date.now(),
4204
+ stream: this.bufferStream
3325
4205
  });
3326
4206
  }
3327
4207
  /**
@@ -3416,7 +4296,7 @@ function decryptBuffer(encrypted, aesKey) {
3416
4296
  */
3417
4297
  async function applyOverlay(config) {
3418
4298
  const { tarballUrl, cliPublicKey, orchestratorPrivateKey, repoDir } = config;
3419
- const tmpDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), "kici-overlay-"));
4299
+ const { path: tmpDir, cleanup } = await makeTempDir("overlay");
3420
4300
  try {
3421
4301
  logger$7.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
3422
4302
  let encryptedData;
@@ -3490,10 +4370,7 @@ async function applyOverlay(config) {
3490
4370
  verified: true
3491
4371
  };
3492
4372
  } finally {
3493
- await fsPromises.rm(tmpDir, {
3494
- recursive: true,
3495
- force: true
3496
- }).catch(() => {});
4373
+ await cleanup().catch(() => {});
3497
4374
  }
3498
4375
  }
3499
4376
  var logger$7, IV_LENGTH$1, AUTH_TAG_LENGTH;
@@ -3676,7 +4553,7 @@ async function applyYarnrcBerryConfig(args) {
3676
4553
  const hasPrivateRegistry = registries.length > 0 || Object.keys(installEnvSecrets).length > 0;
3677
4554
  const yarnrcPath = join(args.kiciDir, ".yarnrc.yml");
3678
4555
  const { raw: original, doc } = await readOriginalYarnrc(yarnrcPath);
3679
- const cacheFolder = await mkdtemp(join(tmpdir(), "kici-yarn-berry-cache-"));
4556
+ const { path: cacheFolder, cleanup: cleanupCache } = await makeTempDir("yarn-berry-cache");
3680
4557
  const merged = {
3681
4558
  ...doc,
3682
4559
  nodeLinker: "node-modules",
@@ -3713,10 +4590,7 @@ async function applyYarnrcBerryConfig(args) {
3713
4590
  if (original === null) await unlink(yarnrcPath).catch(() => {});
3714
4591
  else await writeFile(yarnrcPath, original, { encoding: "utf8" });
3715
4592
  } catch {}
3716
- await rm(cacheFolder, {
3717
- recursive: true,
3718
- force: true
3719
- }).catch(() => {});
4593
+ await cleanupCache().catch(() => {});
3720
4594
  };
3721
4595
  return {
3722
4596
  extraEnv: {
@@ -4059,9 +4933,9 @@ async function detectKiciYarnFlavor(repoRoot, kiciDir) {
4059
4933
  * Install `.kici/` dependencies inline with the repo's package manager.
4060
4934
  *
4061
4935
  * Falls back to this when the dep cache is unavailable or a download fails.
4062
- * The install runs with an isolated cache/store directory (created in
4063
- * `os.tmpdir()`) to prevent cache poisoning between build jobs; the directory
4064
- * is removed after installation.
4936
+ * The install runs with an isolated cache/store directory (allocated under the
4937
+ * global temp base, which honors `KICI_TMPDIR`) to prevent cache poisoning
4938
+ * between build jobs; the directory is removed after installation.
4065
4939
  *
4066
4940
  * If `opts.npmRegistries` / `opts.installEnvSecrets` is provided, a job-scoped
4067
4941
  * `.kici/.npmrc` overlay is synthesized for the install, restored in `finally`,
@@ -4150,7 +5024,7 @@ function envWithNodeOnPath(extraEnv, nodeDir) {
4150
5024
  /** Run `npm install` in `.kici/` with an isolated cache directory. */
4151
5025
  async function runNpmInstall(args) {
4152
5026
  const { npmCliPath, nodeExe, nodeDir } = resolveNpm();
4153
- const cacheDir = await mkdtemp(join(tmpdir(), "kici-npm-cache-"));
5027
+ const { path: cacheDir, cleanup } = await makeTempDir("npm-cache");
4154
5028
  const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
4155
5029
  const buildArgs = (...prefix) => {
4156
5030
  const a = [
@@ -4175,10 +5049,7 @@ async function runNpmInstall(args) {
4175
5049
  maxBuffer: INSTALL_MAX_BUFFER
4176
5050
  });
4177
5051
  } finally {
4178
- await rm(cacheDir, {
4179
- recursive: true,
4180
- force: true
4181
- }).catch(() => {});
5052
+ await cleanup().catch(() => {});
4182
5053
  }
4183
5054
  }
4184
5055
  /**
@@ -4192,7 +5063,7 @@ async function runNpmInstall(args) {
4192
5063
  async function runPnpmInstall(args) {
4193
5064
  await assertPnpmAvailable();
4194
5065
  const { nodeDir } = resolveNpm();
4195
- const storeDir = await mkdtemp(join(tmpdir(), "kici-pnpm-store-"));
5066
+ const { path: storeDir, cleanup } = await makeTempDir("pnpm-store");
4196
5067
  const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
4197
5068
  const argv = [
4198
5069
  "install",
@@ -4212,10 +5083,7 @@ async function runPnpmInstall(args) {
4212
5083
  maxBuffer: INSTALL_MAX_BUFFER
4213
5084
  });
4214
5085
  } finally {
4215
- await rm(storeDir, {
4216
- recursive: true,
4217
- force: true
4218
- }).catch(() => {});
5086
+ await cleanup().catch(() => {});
4219
5087
  }
4220
5088
  }
4221
5089
  /** Pure: argv for `yarn install` with an isolated cache folder. */
@@ -4241,7 +5109,7 @@ function buildYarnInstallArgs(cacheDir, hasPrivateRegistry) {
4241
5109
  async function runYarnInstall(args) {
4242
5110
  await assertYarnAvailable();
4243
5111
  const { nodeDir } = resolveNpm();
4244
- const cacheDir = await mkdtemp(join(tmpdir(), "kici-yarn-cache-"));
5112
+ const { path: cacheDir, cleanup } = await makeTempDir("yarn-cache");
4245
5113
  const env = envWithNodeOnPath(args.registryConfig.extraEnv, nodeDir);
4246
5114
  const argv = buildYarnInstallArgs(cacheDir, args.hasPrivateRegistry);
4247
5115
  try {
@@ -4253,10 +5121,7 @@ async function runYarnInstall(args) {
4253
5121
  maxBuffer: INSTALL_MAX_BUFFER
4254
5122
  });
4255
5123
  } finally {
4256
- await rm(cacheDir, {
4257
- recursive: true,
4258
- force: true
4259
- }).catch(() => {});
5124
+ await cleanup().catch(() => {});
4260
5125
  }
4261
5126
  }
4262
5127
  /** Pure: argv for a berry `yarn install`. Cache + linker live in .yarnrc.yml. */
@@ -4708,6 +5573,7 @@ function buildRequest(dispatch, workDir) {
4708
5573
  workflowRepoIdentifier: jobConfig.workflowRepoIdentifier,
4709
5574
  hasConcurrencyGroup: jobConfig.hasConcurrencyGroup ?? false,
4710
5575
  concurrencyEvaluationTimeoutMs: jobConfig.concurrencyEvaluationTimeoutMs,
5576
+ concurrencyWaitTimeoutMs: dispatch.concurrencyWaitTimeoutMs,
4711
5577
  branch: dispatch.ref,
4712
5578
  upstreamJobOutputs: dispatch.upstreamJobOutputs,
4713
5579
  upstreamJobStatuses: dispatch.upstreamJobStatuses,
@@ -4775,6 +5641,7 @@ function buildBwrapArgs(workDir, nodeExecPath, networkIsolation = false, runnerP
4775
5641
  const libIdx = args.indexOf("/lib", args.indexOf("--ro-bind") + 1);
4776
5642
  if (libIdx !== -1) args.splice(libIdx + 2, 0, "--ro-bind", "/lib64", "/lib64");
4777
5643
  }
5644
+ for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) args.push("--ro-bind", nssFile, nssFile);
4778
5645
  const nodeDir = dirname(nodeExecPath);
4779
5646
  const nodeInstallRoot = dirname(nodeDir);
4780
5647
  if (!nodeInstallRoot.startsWith("/usr") && !nodeInstallRoot.startsWith("/bin") && nodeInstallRoot !== "/" && nodeInstallRoot !== "") args.push("--ro-bind", nodeInstallRoot, nodeInstallRoot);
@@ -4819,6 +5686,28 @@ function buildBwrapArgs(workDir, nodeExecPath, networkIsolation = false, runnerP
4819
5686
  return args;
4820
5687
  }
4821
5688
  /**
5689
+ * Derive the read-only bind path(s) for a `file://` clone source. The workflow
5690
+ * runner clones the repo from inside the sandbox, so a local `file://` URL's
5691
+ * source directory must be exposed read-only or `git clone` fails with
5692
+ * `does not appear to be a git repository`. Returns `[]` for non-`file://`
5693
+ * remotes (https/ssh need no host bind) and for a malformed `file://` URL (the
5694
+ * clone then surfaces the real error rather than being masked here).
5695
+ *
5696
+ * Shared by both sandboxes that clone a local source: the bare-metal (bwrap)
5697
+ * backend threads the result into `buildBwrapArgs`'s `extraReadOnlyBinds`, and
5698
+ * the container backend binds each `<dir>:<dir>:ro` into the job container —
5699
+ * the same clone-source affordance across both isolation models.
5700
+ */
5701
+ function fileCloneSourceBinds(repoUrl) {
5702
+ if (typeof repoUrl !== "string" || !repoUrl.startsWith("file://")) return [];
5703
+ try {
5704
+ const url = new URL(repoUrl);
5705
+ return url.pathname && url.pathname !== "/" ? [url.pathname] : [];
5706
+ } catch {
5707
+ return [];
5708
+ }
5709
+ }
5710
+ /**
4822
5711
  * Spawn the workflow-runner child process. Returns the child handle and a
4823
5712
  * boolean indicating whether the spawn produced a usable PID. On Windows
4824
5713
  * services (shawl) fork() can fail silently when IPC pipes cannot be set up;
@@ -4967,6 +5856,24 @@ function relayProvenanceRequest$1(msg, ctx) {
4967
5856
  error: toErrorMessage(err)
4968
5857
  }));
4969
5858
  }
5859
+ /** Relay `artifacts.request` and pipe the orchestrator response (or an error
5860
+ * response, or a "not configured" response when the callback isn't wired) back
5861
+ * into the sandbox runner. */
5862
+ function relayArtifactRequest$1(msg, ctx) {
5863
+ if (!ctx.execOptions.onArtifactRequest) {
5864
+ safeSendToChild(ctx.child, {
5865
+ type: "artifacts.response",
5866
+ requestId: msg.requestId,
5867
+ error: "Artifacts not available in this agent configuration"
5868
+ });
5869
+ return;
5870
+ }
5871
+ ctx.execOptions.onArtifactRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
5872
+ type: "artifacts.response",
5873
+ requestId: msg.requestId,
5874
+ error: toErrorMessage(err)
5875
+ }));
5876
+ }
4970
5877
  /** Relay `approval.request` and pipe the orchestrator's resolution (or a
4971
5878
  * fail-closed reject when the callback isn't wired or the relay throws) back
4972
5879
  * into the sandbox runner. */
@@ -5021,7 +5928,7 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
5021
5928
  });
5022
5929
  return;
5023
5930
  case "log.line":
5024
- ctx.execOptions.onLogLine(msg.stepIndex, msg.line);
5931
+ ctx.execOptions.onLogLine(msg.stepIndex, msg.line, msg.stream);
5025
5932
  return;
5026
5933
  case "step.start": {
5027
5934
  ctx.stepNames.set(msg.stepIndex, msg.stepName);
@@ -5069,6 +5976,9 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
5069
5976
  case "cache.request":
5070
5977
  relayCacheRequest$1(msg, ctx);
5071
5978
  return;
5979
+ case "artifacts.request":
5980
+ relayArtifactRequest$1(msg, ctx);
5981
+ return;
5072
5982
  case "provenance.request":
5073
5983
  relayProvenanceRequest$1(msg, ctx);
5074
5984
  return;
@@ -5286,14 +6196,7 @@ var init_bare_metal_sandbox = __esmMin((() => {
5286
6196
  * Execute a job by forking the workflow runner with sanitized environment.
5287
6197
  */
5288
6198
  async executeJob(options) {
5289
- const extraReadOnlyBinds = [];
5290
- if (this.useBwrap) {
5291
- const repoUrl = options.dispatch.repoUrl;
5292
- if (typeof repoUrl === "string" && repoUrl.startsWith("file://")) try {
5293
- const url = new URL(repoUrl);
5294
- if (url.pathname) extraReadOnlyBinds.push(url.pathname);
5295
- } catch {}
5296
- }
6199
+ const extraReadOnlyBinds = this.useBwrap ? fileCloneSourceBinds(options.dispatch.repoUrl) : [];
5297
6200
  this.runner = createForkRunner({
5298
6201
  runnerPath: this.runnerPath,
5299
6202
  env: this.env,
@@ -5380,6 +6283,43 @@ var init_firecracker_sandbox = __esmMin((() => {
5380
6283
  };
5381
6284
  }));
5382
6285
  //#endregion
6286
+ //#region src/execution/sandbox/container-hardening.ts
6287
+ /**
6288
+ * Build the hardened HostConfig fragment for a job sandbox container.
6289
+ *
6290
+ * Pure and side-effect-free: given resolved inputs it returns the fields the
6291
+ * container-sandbox merges into `docker.createContainer`. When `hardened` is
6292
+ * false it returns an empty posture (the rollback affordance).
6293
+ */
6294
+ function buildContainerHardening(opts) {
6295
+ if (!opts.hardened) return { hostConfig: {} };
6296
+ const grant = opts.grant;
6297
+ const hostConfig = {
6298
+ CapDrop: [...CAP_DROP_ALL],
6299
+ SecurityOpt: [...NO_NEW_PRIVILEGES],
6300
+ PidsLimit: opts.pidsLimit,
6301
+ Memory: opts.memoryBytes,
6302
+ MemorySwap: opts.memoryBytes,
6303
+ NanoCpus: opts.nanoCpus,
6304
+ Tmpfs: { "/tmp": "rw,exec,nosuid,nodev" }
6305
+ };
6306
+ if (grant?.capabilities && grant.capabilities.length > 0) hostConfig.CapAdd = [...grant.capabilities];
6307
+ if (grant?.readonlyRootfs ?? opts.readonlyRootfs) hostConfig.ReadonlyRootfs = true;
6308
+ const network = grant?.network ?? opts.networkMode;
6309
+ if (network === "none") hostConfig.NetworkMode = "none";
6310
+ else if (network === "host") hostConfig.NetworkMode = "host";
6311
+ const user = grant?.user ?? opts.user;
6312
+ return user ? {
6313
+ hostConfig,
6314
+ user
6315
+ } : { hostConfig };
6316
+ }
6317
+ var CAP_DROP_ALL, NO_NEW_PRIVILEGES;
6318
+ var init_container_hardening = __esmMin((() => {
6319
+ CAP_DROP_ALL = ["ALL"];
6320
+ NO_NEW_PRIVILEGES = ["no-new-privileges"];
6321
+ }));
6322
+ //#endregion
5383
6323
  //#region src/execution/sandbox/container-sandbox.ts
5384
6324
  /**
5385
6325
  * Container execution sandbox implementation.
@@ -5522,6 +6462,32 @@ function relayProvenanceRequest(stream, options, provMsg) {
5522
6462
  }));
5523
6463
  }
5524
6464
  /**
6465
+ * Relay artifacts.request from the container runner to the orchestrator via
6466
+ * options.onArtifactRequest, then write the response back through `stream`. If
6467
+ * the agent doesn't expose an artifacts relay, write a structured error so the
6468
+ * runner doesn't hang.
6469
+ */
6470
+ function relayArtifactRequest(stream, options, artMsg) {
6471
+ const writeResponse = (response) => {
6472
+ try {
6473
+ stream.write(JSON.stringify(response) + "\n");
6474
+ } catch {}
6475
+ };
6476
+ if (!options.onArtifactRequest) {
6477
+ writeResponse({
6478
+ type: "artifacts.response",
6479
+ requestId: artMsg.requestId,
6480
+ error: "Artifacts not available in this agent configuration"
6481
+ });
6482
+ return;
6483
+ }
6484
+ options.onArtifactRequest(artMsg).then((response) => writeResponse(response), (err) => writeResponse({
6485
+ type: "artifacts.response",
6486
+ requestId: artMsg.requestId,
6487
+ error: toErrorMessage(err)
6488
+ }));
6489
+ }
6490
+ /**
5525
6491
  * Relay approval.request from the container runner to the orchestrator via
5526
6492
  * options.onApprovalRequest, then write the resolution back through `stream`.
5527
6493
  * If the agent doesn't expose an approval relay (or it throws), write a
@@ -5565,22 +6531,30 @@ function applyJobComplete(msg, stepResults, state, options) {
5565
6531
  logger$3.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
5566
6532
  }
5567
6533
  }
5568
- var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, ContainerSandbox;
6534
+ var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, HOOK_MOUNT_PATH, INTERNAL_ENV, ContainerSandbox;
5569
6535
  var init_container_sandbox = __esmMin((() => {
5570
6536
  init_fork_runner();
5571
6537
  init_secret_encryption();
6538
+ init_container_hardening();
5572
6539
  logger$3 = createLogger({ prefix: "container-sandbox" });
5573
6540
  MAX_STDERR_LINES = 20;
5574
6541
  ABORT_GRACE_MS = 1e4;
5575
6542
  CONTAINER_STOP_TIMEOUT = 10;
6543
+ HOOK_MOUNT_PATH = "/opt/kici/ts-loader-hook.js";
6544
+ INTERNAL_ENV = [`KICI_TS_LOADER_HOOK_PATH=${HOOK_MOUNT_PATH}`, "KICI_LOG_STDERR=1"];
5576
6545
  ContainerSandbox = class {
5577
6546
  docker;
5578
6547
  image;
5579
6548
  runnerPath;
5580
6549
  runnerMountPath;
6550
+ /** Host path to the pure-JS container loader-hook bundle (bind-mounted :ro). */
6551
+ hookHostPath;
5581
6552
  env;
5582
6553
  keepFailed;
5583
6554
  jobId;
6555
+ hardening;
6556
+ /** Resolved container user (image-user override / grant), applied to createContainer + each exec. */
6557
+ resolvedUser;
5584
6558
  /** The running container instance (set during setup). */
5585
6559
  container = null;
5586
6560
  /** The active exec stream (set during executeJob, used for abort). */
@@ -5594,29 +6568,66 @@ var init_container_sandbox = __esmMin((() => {
5594
6568
  this.image = options.image;
5595
6569
  this.runnerPath = options.runnerPath;
5596
6570
  this.runnerMountPath = options.runnerMountPath ?? "/opt/kici/workflow-runner.js";
6571
+ this.hookHostPath = options.hookPath ?? join(dirname(options.runnerPath), "container-ts-loader-hook.js");
5597
6572
  this.env = options.env;
5598
6573
  this.keepFailed = options.keepFailed ?? false;
5599
6574
  this.jobId = options.jobId ?? `unknown-${Date.now()}`;
6575
+ this.hardening = options.hardening;
6576
+ }
6577
+ /**
6578
+ * Ensure the job's container image is present locally, pulling it on demand
6579
+ * when it is not.
6580
+ *
6581
+ * dockerode's `createContainer` — unlike `docker run` / `podman run` — never
6582
+ * auto-pulls a missing image; it fails with `(HTTP code 404) ... No such
6583
+ * image`. A bare-metal executor that aggressively prunes unused images under
6584
+ * disk pressure can leave a container job with nothing to run, so the agent
6585
+ * pulls the image itself. Already-present images (the common case, and how
6586
+ * private images pre-pulled with registry auth stay working) skip the pull.
6587
+ */
6588
+ async ensureImagePresent() {
6589
+ try {
6590
+ await this.docker.getImage(this.image).inspect();
6591
+ return;
6592
+ } catch {}
6593
+ logger$3.info("Pulling sandbox image (not present locally)", { image: this.image });
6594
+ const stream = await this.docker.pull(this.image);
6595
+ await new Promise((resolve, reject) => {
6596
+ this.docker.modem.followProgress(stream, (err) => err ? reject(err) : resolve());
6597
+ });
6598
+ logger$3.info("Sandbox image pulled", { image: this.image });
5600
6599
  }
5601
6600
  async setup(options) {
5602
6601
  this.containerName = `kici-sandbox-${this.jobId}-${Date.now()}`;
5603
- const envArray = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
6602
+ const envArray = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
5604
6603
  logger$3.info("Creating sandbox container", {
5605
6604
  name: this.containerName,
5606
6605
  image: this.image,
5607
6606
  workDir: options.workDir
5608
6607
  });
6608
+ const hardened = this.hardening ? buildContainerHardening(this.hardening) : {
6609
+ hostConfig: {},
6610
+ user: void 0
6611
+ };
6612
+ this.resolvedUser = hardened.user;
6613
+ const binds = this.buildBinds(options, hardened.hostConfig);
6614
+ await this.ensureImagePresent();
5609
6615
  this.container = await this.docker.createContainer({
5610
6616
  Image: this.image,
5611
6617
  name: this.containerName,
5612
6618
  Cmd: ["sleep", "infinity"],
5613
6619
  Env: envArray,
5614
6620
  WorkingDir: "/workspace",
6621
+ Volumes: { "/workspace": {} },
6622
+ ...hardened.user ? { User: hardened.user } : {},
5615
6623
  Labels: {
5616
6624
  "kici-sandbox": "true",
5617
6625
  "kici-job-id": this.jobId
5618
6626
  },
5619
- HostConfig: { Binds: [`${options.workDir}:/workspace`, `${this.runnerPath}:${this.runnerMountPath}:ro`] }
6627
+ HostConfig: {
6628
+ Binds: binds,
6629
+ ...hardened.hostConfig
6630
+ }
5620
6631
  });
5621
6632
  await this.container.start();
5622
6633
  logger$3.info("Sandbox container started", {
@@ -5624,6 +6635,36 @@ var init_container_sandbox = __esmMin((() => {
5624
6635
  containerId: this.container.id.slice(0, 12)
5625
6636
  });
5626
6637
  }
6638
+ /**
6639
+ * Build the container's read-only bind list.
6640
+ *
6641
+ * The workspace is NOT bound here — it is a container-owned anonymous volume
6642
+ * (`Volumes: { '/workspace': {} }` on the container config), so the container
6643
+ * user can write it on every runtime with `CapDrop: ['ALL']` intact. This
6644
+ * method binds the workflow runner (read-only) plus two parity affordances
6645
+ * that mirror the bare-metal bwrap sandbox — both strictly additive and gated,
6646
+ * so the default posture for a production (https-source, bridge-network) job is
6647
+ * unchanged:
6648
+ *
6649
+ * - **`file://` clone-source dir(s)** (`options.extraReadOnlyBinds`): the
6650
+ * workflow runner clones the repo from inside the container, so a local
6651
+ * `file://` source dir must be exposed read-only or `git clone` fails.
6652
+ * Empty for https/ssh remotes — mirrors fork-runner's `extraReadOnlyBinds`.
6653
+ * - **Host name-resolution files under host networking**: when the effective
6654
+ * network posture is `host` (`KICI_SANDBOX_NETWORK=host`, or a per-job host
6655
+ * grant), bind the host's `/etc/hosts` (+ `/etc/nsswitch.conf`) read-only so
6656
+ * an `/etc/hosts`-only name the host resolves — e.g. a private registry —
6657
+ * resolves inside the container too. Mirrors fork-runner's
6658
+ * `--ro-bind /etc/hosts` for the bwrap host-network mode.
6659
+ */
6660
+ buildBinds(options, hostConfig) {
6661
+ const binds = [`${this.runnerPath}:${this.runnerMountPath}:ro`, `${this.hookHostPath}:${HOOK_MOUNT_PATH}:ro`];
6662
+ for (const dir of options.extraReadOnlyBinds ?? []) if (dir) binds.push(`${dir}:${dir}:ro`);
6663
+ if (hostConfig.NetworkMode === "host") {
6664
+ for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) binds.push(`${nssFile}:${nssFile}:ro`);
6665
+ }
6666
+ return binds;
6667
+ }
5627
6668
  async executeJob(options) {
5628
6669
  if (!this.container) throw new Error("ContainerSandbox.executeJob() called before setup()");
5629
6670
  const startTime = Date.now();
@@ -5655,14 +6696,15 @@ var init_container_sandbox = __esmMin((() => {
5655
6696
  * stderr lines for crash diagnostics, and install the abort listener.
5656
6697
  */
5657
6698
  async attachExecStream(options) {
5658
- const execEnv = Object.entries(this.env).map(([k, v]) => `${k}=${v}`);
6699
+ const execEnv = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
5659
6700
  const stream = await (await this.container.exec({
5660
6701
  Cmd: ["node", this.runnerMountPath],
5661
6702
  AttachStdin: true,
5662
6703
  AttachStdout: true,
5663
6704
  AttachStderr: true,
5664
6705
  Env: execEnv,
5665
- WorkingDir: "/workspace"
6706
+ WorkingDir: "/workspace",
6707
+ ...this.resolvedUser ? { User: this.resolvedUser } : {}
5666
6708
  })).start({
5667
6709
  hijack: true,
5668
6710
  stdin: true
@@ -5793,7 +6835,7 @@ var init_container_sandbox = __esmMin((() => {
5793
6835
  return false;
5794
6836
  }
5795
6837
  case "log.line":
5796
- options.onLogLine(msg.stepIndex, msg.line);
6838
+ options.onLogLine(msg.stepIndex, msg.line, msg.stream);
5797
6839
  return false;
5798
6840
  case "step.secret_mount":
5799
6841
  options.onSecretMount?.({
@@ -5816,6 +6858,9 @@ var init_container_sandbox = __esmMin((() => {
5816
6858
  case "cache.request":
5817
6859
  relayCacheRequest(stream, options, msg);
5818
6860
  return false;
6861
+ case "artifacts.request":
6862
+ relayArtifactRequest(stream, options, msg);
6863
+ return false;
5819
6864
  case "provenance.request":
5820
6865
  relayProvenanceRequest(stream, options, msg);
5821
6866
  return false;
@@ -5865,7 +6910,10 @@ var init_container_sandbox = __esmMin((() => {
5865
6910
  await this.container.stop({ t: CONTAINER_STOP_TIMEOUT });
5866
6911
  } catch {}
5867
6912
  try {
5868
- await this.container.remove({ force: true });
6913
+ await this.container.remove({
6914
+ force: true,
6915
+ v: true
6916
+ });
5869
6917
  } catch {}
5870
6918
  this.container = null;
5871
6919
  }
@@ -5909,13 +6957,15 @@ var init_sandbox = __esmMin((() => {
5909
6957
  init_bare_metal_sandbox();
5910
6958
  init_firecracker_sandbox();
5911
6959
  init_container_sandbox();
6960
+ init_fork_runner();
5912
6961
  }));
5913
6962
  //#endregion
5914
6963
  //#region src/execution/job-runner.ts
5915
6964
  var job_runner_exports = /* @__PURE__ */ __exportAll({
5916
6965
  JobRunner: () => JobRunner$1,
5917
6966
  buildEvalNeedsContext: () => buildEvalNeedsContext,
5918
- resolveJobWorkDir: () => resolveJobWorkDir
6967
+ resolveJobWorkDir: () => resolveJobWorkDir,
6968
+ resolveRunnerBundlePath: () => resolveRunnerBundlePath
5919
6969
  });
5920
6970
  /**
5921
6971
  * Check if a file exists at the given path.
@@ -5950,6 +7000,19 @@ function resolveRunnerPath() {
5950
7000
  return bundlePath;
5951
7001
  }
5952
7002
  /**
7003
+ * Derive the self-contained runner bundle path from the resolved runner path.
7004
+ *
7005
+ * The container backend mounts the runner as a single file into the customer
7006
+ * job container, so it must run `workflow-runner-bundle.js` (zx + `@kici-dev/*`
7007
+ * inlined) rather than the external `workflow-runner.js`, which cannot resolve
7008
+ * its bare imports without the agent's node_modules / pnpm workspace. The
7009
+ * bundle is a flat sibling emitted alongside the runner by build-service.mjs.
7010
+ * bwrap / firecracker keep the external runner (they bind the workspace).
7011
+ */
7012
+ function resolveRunnerBundlePath(runnerPath) {
7013
+ return join(dirname(runnerPath), "workflow-runner-bundle.js");
7014
+ }
7015
+ /**
5953
7016
  * Determine the execution mode from agent config and job config.
5954
7017
  *
5955
7018
  * Priority:
@@ -5990,14 +7053,11 @@ async function resolveJobWorkDir(inPlace, repoUrl) {
5990
7053
  cleanup: async () => {},
5991
7054
  inPlace: true
5992
7055
  };
5993
- const workDir = await fsPromises.mkdtemp(join(tmpdir(), "kici-"));
7056
+ const { path: workDir, cleanup } = await makeTempDir("workdir");
5994
7057
  return {
5995
7058
  workDir,
5996
7059
  cleanup: async () => {
5997
- await fsPromises.rm(workDir, {
5998
- recursive: true,
5999
- force: true
6000
- }).catch(() => {});
7060
+ await cleanup().catch(() => {});
6001
7061
  },
6002
7062
  inPlace: false
6003
7063
  };
@@ -6011,6 +7071,8 @@ var init_job_runner = __esmMin((() => {
6011
7071
  init_init_runner();
6012
7072
  init_api_intercept();
6013
7073
  init_ensure_init_runner();
7074
+ init_payload_source();
7075
+ init_s3_payload_source();
6014
7076
  init_timeout_util();
6015
7077
  init_streaming_zx_log();
6016
7078
  init_dynamic_job_serializer();
@@ -6025,7 +7087,6 @@ var init_job_runner = __esmMin((() => {
6025
7087
  logger$2 = createLogger({ prefix: "job-runner" });
6026
7088
  JobRunner$1 = class {
6027
7089
  send;
6028
- sendDirect;
6029
7090
  config;
6030
7091
  requestUploadUrl;
6031
7092
  sendUploadComplete;
@@ -6038,6 +7099,7 @@ var init_job_runner = __esmMin((() => {
6038
7099
  _sendApiRequest;
6039
7100
  _requestUserCache;
6040
7101
  _relayProvenance;
7102
+ _requestUserArtifact;
6041
7103
  _sendStepApproval;
6042
7104
  /** Tracks running jobs for concurrency and cancellation */
6043
7105
  activeJobs = /* @__PURE__ */ new Map();
@@ -6045,7 +7107,6 @@ var init_job_runner = __esmMin((() => {
6045
7107
  activeSandbox = null;
6046
7108
  constructor(deps) {
6047
7109
  this.send = deps.send;
6048
- this.sendDirect = deps.sendDirect;
6049
7110
  this.config = deps.config;
6050
7111
  this.requestUploadUrl = deps.requestUploadUrl;
6051
7112
  this.sendUploadComplete = deps.sendUploadComplete;
@@ -6058,6 +7119,7 @@ var init_job_runner = __esmMin((() => {
6058
7119
  this._sendApiRequest = deps.sendApiRequest;
6059
7120
  this._requestUserCache = deps.requestUserCache;
6060
7121
  this._relayProvenance = deps.relayProvenance;
7122
+ this._requestUserArtifact = deps.requestUserArtifact;
6061
7123
  this._sendStepApproval = deps.sendStepApproval;
6062
7124
  }
6063
7125
  /**
@@ -6161,13 +7223,15 @@ var init_job_runner = __esmMin((() => {
6161
7223
  });
6162
7224
  }, this.config.jobHeartbeatIntervalMs);
6163
7225
  let sandbox;
7226
+ const logStreamers = /* @__PURE__ */ new Map();
6164
7227
  try {
6165
7228
  const setupResult = await this.setupSandboxForExecution(dispatch, workDir, abortController);
6166
7229
  if (!setupResult) return;
6167
7230
  sandbox = setupResult.sandbox;
6168
- const { result, logStreamers } = await this.runSandboxExecution(dispatch, sandbox, abortController);
7231
+ const result = await this.runSandboxExecution(dispatch, sandbox, abortController, logStreamers);
6169
7232
  this.reportExecutionResult(dispatch, result, logStreamers);
6170
7233
  } catch (error) {
7234
+ for (const streamer of logStreamers.values()) streamer.destroy();
6171
7235
  const errorMsg = toErrorMessage(error);
6172
7236
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, { error: errorMsg });
6173
7237
  } finally {
@@ -6218,7 +7282,8 @@ var init_job_runner = __esmMin((() => {
6218
7282
  this.activeSandbox = sandbox;
6219
7283
  await sandbox.setup({
6220
7284
  workDir,
6221
- env: sanitizedEnv
7285
+ env: sanitizedEnv,
7286
+ extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl)
6222
7287
  });
6223
7288
  if (abortController.signal.aborted) {
6224
7289
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.cancelled);
@@ -6243,13 +7308,13 @@ var init_job_runner = __esmMin((() => {
6243
7308
  /**
6244
7309
  * Drive `sandbox.executeJob` with IPC callbacks wired to the WS pipeline.
6245
7310
  *
6246
- * Lazily creates per-step LogStreamers, forwards step + log + event-emit +
7311
+ * Lazily populates `logStreamers` (owned by the caller so a throw still
7312
+ * leaves the buffered output flushable), forwards step + log + event-emit +
6247
7313
  * concurrency-report + api-request messages, and emits the
6248
7314
  * `agent.execution.start` / `agent.execution.end` lifecycle events.
6249
7315
  */
6250
- async runSandboxExecution(dispatch, sandbox, abortController) {
7316
+ async runSandboxExecution(dispatch, sandbox, abortController, logStreamers) {
6251
7317
  const { runId, jobId } = dispatch;
6252
- const logStreamers = /* @__PURE__ */ new Map();
6253
7318
  const maxLogSizeBytes = dispatch.maxLogSizeBytes ?? this.config.maxLogSizeBytes;
6254
7319
  const getOrCreateLogStreamer = (stepIndex) => {
6255
7320
  let streamer = logStreamers.get(stepIndex);
@@ -6278,8 +7343,8 @@ var init_job_runner = __esmMin((() => {
6278
7343
  this.maybeEmitCacheRunEvent(runId, jobId, stepIndex, state, data);
6279
7344
  this.sendStepStatus(dispatch, stepIndex, stepName, state, data, logBytesStreamed);
6280
7345
  },
6281
- onLogLine: (stepIndex, line) => {
6282
- getOrCreateLogStreamer(stepIndex).addLine(line);
7346
+ onLogLine: (stepIndex, line, stream) => {
7347
+ getOrCreateLogStreamer(stepIndex).addLine(line, stream);
6283
7348
  },
6284
7349
  signal: abortController.signal,
6285
7350
  onEventEmit: async (request) => {
@@ -6302,6 +7367,7 @@ var init_job_runner = __esmMin((() => {
6302
7367
  onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => this._sendApiRequest(method, params)) : void 0,
6303
7368
  onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
6304
7369
  onProvenanceRequest: this._relayProvenance ? async (request) => this._relayProvenance(jobId, request) : void 0,
7370
+ onArtifactRequest: this._requestUserArtifact ? async (request) => this._requestUserArtifact(jobId, request) : void 0,
6305
7371
  onApprovalRequest: this._sendStepApproval ? async (request) => this._sendStepApproval(dispatch.runId, dispatch.jobId, request) : void 0,
6306
7372
  onSecretMount: (event) => {
6307
7373
  this.emitRunEvent(runId, "step.secret_mount", {
@@ -6321,10 +7387,7 @@ var init_job_runner = __esmMin((() => {
6321
7387
  durationMs: Date.now() - executionStartMs,
6322
7388
  metadata: { status: result.status }
6323
7389
  });
6324
- return {
6325
- result,
6326
- logStreamers
6327
- };
7390
+ return result;
6328
7391
  }
6329
7392
  /**
6330
7393
  * Tear down log streamers, record step Prometheus metrics, log sandbox
@@ -6386,7 +7449,19 @@ var init_job_runner = __esmMin((() => {
6386
7449
  if (!targetAgentId) throw new Error("bring-up job missing bringupTarget");
6387
7450
  if (!this._sendApiRequest) throw new Error("bring-up job requires an orchestrator API transport");
6388
7451
  streamer.addLine(`Bringing up init-runner on ${targetAgentId}…`);
6389
- const result = await ensureInitRunner(async (method, params) => this._sendApiRequest(method, params), targetAgentId);
7452
+ const transport = async (method, params) => this._sendApiRequest(method, params);
7453
+ const result = await ensureInitRunner(transport, targetAgentId, {
7454
+ payloadSource: this.config.agentPayloadDir ? new LocalDirPayloadSource(this.config.agentPayloadDir) : new S3PayloadSource({
7455
+ presign: async (platform) => await transport("kici.presignAgentPackage", {
7456
+ targetAgentId,
7457
+ platform
7458
+ }),
7459
+ fetchToFile: fetchUrlToFile,
7460
+ cacheDir: defaultPayloadCacheDir(),
7461
+ exists: payloadFileExists
7462
+ }),
7463
+ agentCommand: this.config.agentCommand
7464
+ });
6390
7465
  streamer.addLine(result.broughtUp ? `Init-runner brought up on ${targetAgentId}.` : `${targetAgentId} already has a live agent — no bring-up needed.`);
6391
7466
  await streamer.flush();
6392
7467
  this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.success, void 0, streamer.getTotalBytes());
@@ -6783,7 +7858,7 @@ var init_job_runner = __esmMin((() => {
6783
7858
  env: { ...process.env },
6784
7859
  verbose: true,
6785
7860
  quiet: false,
6786
- log: makeStreamingZxLog((line) => evalStreamer.addLine(line))
7861
+ log: makeStreamingZxLog((line, stream) => evalStreamer.addLine(line, stream))
6787
7862
  });
6788
7863
  const evalSink = { addLine: (line) => evalStreamer.addLine(line) };
6789
7864
  const dynamicJobLogger = {
@@ -6868,10 +7943,20 @@ var init_job_runner = __esmMin((() => {
6868
7943
  return new ContainerSandbox({
6869
7944
  docker: new Docker(),
6870
7945
  image,
6871
- runnerPath: opts.runnerPath,
7946
+ runnerPath: resolveRunnerBundlePath(opts.runnerPath),
6872
7947
  env: opts.env,
6873
7948
  keepFailed: this.config.dockerKeepFailed,
6874
- jobId: opts.jobId
7949
+ jobId: opts.jobId,
7950
+ hardening: {
7951
+ hardened: this.config.sandboxHardened,
7952
+ readonlyRootfs: this.config.sandboxReadonlyRootfs,
7953
+ user: this.config.sandboxUser,
7954
+ pidsLimit: this.config.sandboxPidsLimit,
7955
+ memoryBytes: this.config.sandboxMemoryBytes,
7956
+ nanoCpus: this.config.sandboxNanoCpus,
7957
+ networkMode: this.config.sandboxNetwork === "host" ? "host" : "default",
7958
+ grant: opts.jobConfig.sandboxGrant
7959
+ }
6875
7960
  });
6876
7961
  }
6877
7962
  case "firecracker": return new FirecrackerSandbox({
@@ -6960,7 +8045,7 @@ var init_job_runner = __esmMin((() => {
6960
8045
  * they are included as a top-level field on the WS message (not nested in data).
6961
8046
  */
6962
8047
  sendJobStatus(dispatch, state, data, secretOutputs) {
6963
- this.sendDirect({
8048
+ this.send({
6964
8049
  type: "job.status",
6965
8050
  messageId: randomUUID(),
6966
8051
  runId: dispatch.runId,
@@ -6986,7 +8071,7 @@ var init_job_runner = __esmMin((() => {
6986
8071
  const groupId = data?.groupId;
6987
8072
  const { secretsAccessed: _s, concurrencyKind: _c, groupId: _g, ...restData } = data ?? {};
6988
8073
  const hasRestData = Object.keys(restData).length > 0;
6989
- this.sendDirect({
8074
+ this.send({
6990
8075
  type: "step.status",
6991
8076
  messageId: randomUUID(),
6992
8077
  runId: dispatch.runId,
@@ -7012,7 +8097,7 @@ var init_job_runner = __esmMin((() => {
7012
8097
  * Startup sequence:
7013
8098
  * 1. Load config
7014
8099
  * 2. Create logger
7015
- * 3. Create JobRunner with send/sendDirect callbacks
8100
+ * 3. Create JobRunner with send callbacks
7016
8101
  * 4. Create OrchestratorClient with dispatch and cancel handlers
7017
8102
  * 5. Add WS log transport (if not scaler-managed)
7018
8103
  * 6. Connect OrchestratorClient
@@ -7026,14 +8111,14 @@ var init_job_runner = __esmMin((() => {
7026
8111
  */
7027
8112
  init_console_capture();
7028
8113
  init_npm_resolver();
7029
- const AGENT_VERSION = "0.1.27";
7030
- const BUILD_COMMIT = "f0747470b";
7031
- const SDK_VERSION = "0.1.27";
7032
- const SDK_BUNDLE_HASH = "e06bd1acf349ba365bdb357f3d865e0cbb4ad4604a0270b707a681a4f440954e";
7033
- const SHARED_VERSION = "0.1.27";
7034
- const SHARED_BUNDLE_HASH = "f8e1f0117cf41fca5a74377633e2ec89e54ce4eda47d9f476cdb83336f1ed962";
7035
- const ENGINE_VERSION = "0.1.27";
7036
- const ENGINE_BUNDLE_HASH = "ce675fb4790406ed2a938f8f1783db37b283cffd7d8fc1eb12a878e0bfb919b3";
8114
+ const AGENT_VERSION = "0.2.0";
8115
+ const BUILD_COMMIT = "15d5e4447";
8116
+ const SDK_VERSION = "0.2.0";
8117
+ const SDK_BUNDLE_HASH = "fcd0a836aa1cb92debd93e0254450614d123b3748511aa4148d956fd72a7c29f";
8118
+ const SHARED_VERSION = "0.2.0";
8119
+ const SHARED_BUNDLE_HASH = "f16fa528a8a9ef1bed8df369f0e88efafca8a9954a749005299fdbcdb93fa188";
8120
+ const ENGINE_VERSION = "0.2.0";
8121
+ const ENGINE_BUNDLE_HASH = "c579e54f7d0b0d5b16587a0528a3e66d32f63e2434bb471a93a01366296efd58";
7037
8122
  initTelemetry({
7038
8123
  serviceName: "kici-agent",
7039
8124
  otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
@@ -7103,7 +8188,6 @@ await guardStartup(logger$1, async () => {
7103
8188
  let client;
7104
8189
  const jobRunner = new JobRunner({
7105
8190
  send: (msg) => client.send(msg),
7106
- sendDirect: (msg) => client.sendDirect(msg),
7107
8191
  config,
7108
8192
  requestUploadUrl: (jobId, cacheType, key) => client.requestUploadUrl(jobId, cacheType, key),
7109
8193
  sendUploadComplete: (jobId, cacheType, key) => client.sendUploadComplete(jobId, cacheType, key),
@@ -7124,6 +8208,7 @@ await guardStartup(logger$1, async () => {
7124
8208
  },
7125
8209
  requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
7126
8210
  relayProvenance: (jobId, request) => client.relayProvenance(jobId, request),
8211
+ requestUserArtifact: (jobId, request) => client.requestUserArtifact(jobId, request),
7127
8212
  sendStepApproval: (runId, jobId, request) => client.sendStepApproval(runId, jobId, request)
7128
8213
  });
7129
8214
  /** Build and send an agent.status message with dynamic OS metadata. */