@kici-dev/agent 0.1.21 → 0.1.23

package/dist/server.js

@@ -12,7 +12,7 @@ import winston from "winston";
 import { RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
 import { z } from "zod";
 import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
-import { ALLOWED_SYSTEM_VARS, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, deriveOsArchLabels, expandMatrix, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, validateNoReservedLabels } from "@kici-dev/engine";
+import { ALLOWED_SYSTEM_VARS, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, deriveOsArchLabels, expandMatrix, heartbeatSchema, hostLabel, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
 import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
 import WebSocket from "ws";
 import * as fs$2 from "node:fs";
@@ -102,7 +102,8 @@ const envDef = defineEnv({
 		scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
 		executionMode: ExecutionMode.optional(),
 		otelExporterOtlpEndpoint: z.string().optional(),
-		concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5)
+		concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
+		isOrchestratorHost: z.string().optional().transform((v) => v === "true")
 	}),
 	envMap: {
 		orchestratorUrl: "KICI_ORCHESTRATOR_URL",
@@ -113,6 +114,7 @@ const envDef = defineEnv({
 		port: "KICI_PORT",
 		logLevel: "KICI_LOG_LEVEL",
 		agentToken: "KICI_AGENT_TOKEN",
+		isOrchestratorHost: "KICI_AGENT_IS_ORCHESTRATOR_HOST",
 		githubToken: "KICI_GITHUB_TOKEN",
 		maxLogSizeBytes: "KICI_MAX_LOG_SIZE_BYTES",
 		defaultStepTimeoutMs: "KICI_DEFAULT_STEP_TIMEOUT_MS",
@@ -286,7 +288,7 @@ var LogBuffer = class extends RingBuffer {
 };
 //#endregion
 //#region src/ws/orchestrator-client.ts
-const logger$11 = createLogger({ prefix: "orchestrator-client" });
+const logger$12 = createLogger({ prefix: "orchestrator-client" });
 /**
 * WebSocket client that connects the agent to the customer orchestrator.
 *
@@ -403,7 +405,7 @@ var OrchestratorClient = class OrchestratorClient {
 	*/
 	connect() {
 		if (this._state !== "disconnected") {
-			logger$11.warn("connect() called while not disconnected", { state: this._state });
+			logger$12.warn("connect() called while not disconnected", { state: this._state });
 			return;
 		}
 		this.intentionalDisconnect = false;
@@ -754,7 +756,8 @@ var OrchestratorClient = class OrchestratorClient {
 				stepName: request.stepName,
 				clauses: request.clauses,
 				reason: request.reason,
-				...request.timeoutSeconds !== void 0 && { timeoutSeconds: request.timeoutSeconds }
+				...request.timeoutSeconds !== void 0 && { timeoutSeconds: request.timeoutSeconds },
+				...request.payload !== void 0 && { payload: request.payload }
 			});
 		});
 	}
@@ -840,7 +843,7 @@ var OrchestratorClient = class OrchestratorClient {
 				}
 			});
 		} catch (err) {
-			logger$11.error("Failed to create WebSocket", { error: toErrorMessage(err) });
+			logger$12.error("Failed to create WebSocket", { error: toErrorMessage(err) });
 			this._state = "disconnected";
 			this.scheduleReconnect();
 			return;
@@ -848,7 +851,7 @@ var OrchestratorClient = class OrchestratorClient {
 		this.ws.on("open", () => {
 			if (this.token) {
 				this._state = "authenticating";
-				logger$11.info("Connected to orchestrator, sending auth.request", {
+				logger$12.info("Connected to orchestrator, sending auth.request", {
 					url: this.url,
 					agentId: this.agentId
 				});
@@ -859,7 +862,7 @@ var OrchestratorClient = class OrchestratorClient {
 				}));
 			} else {
 				this._state = "registering";
-				logger$11.info("Connected to orchestrator, sending agent.register (no token)", {
+				logger$12.info("Connected to orchestrator, sending agent.register (no token)", {
 					url: this.url,
 					agentId: this.agentId
 				});
@@ -870,12 +873,12 @@ var OrchestratorClient = class OrchestratorClient {
 			this.handleMessage(data);
 		});
 		this.ws.on("close", (code, reason) => {
-			logger$11.info("Orchestrator connection closed", {
+			logger$12.info("Orchestrator connection closed", {
 				code,
 				reason: reason.toString()
 			});
 			if (code === WS_CLOSE_AGENT_AUTH_FAILED) {
-				logger$11.error("Orchestrator closed with auth-failed code -- token is invalid or revoked. NOT retrying.", {
+				logger$12.error("Orchestrator closed with auth-failed code -- token is invalid or revoked. NOT retrying.", {
 					code,
 					reason: reason.toString()
 				});
@@ -901,7 +904,7 @@ var OrchestratorClient = class OrchestratorClient {
 			if (!this.intentionalDisconnect) this.scheduleReconnect();
 		});
 		this.ws.on("error", (err) => {
-			logger$11.error(`Orchestrator WebSocket error: ${err.message}`);
+			logger$12.error(`Orchestrator WebSocket error: ${err.message}`);
 			if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
 		});
 	}
@@ -910,7 +913,7 @@ var OrchestratorClient = class OrchestratorClient {
 		try {
 			raw = JSON.parse(data.toString());
 		} catch {
-			logger$11.warn("Malformed JSON received from orchestrator");
+			logger$12.warn("Malformed JSON received from orchestrator");
 			return;
 		}
 		const rawMsg = raw;
@@ -968,13 +971,13 @@ var OrchestratorClient = class OrchestratorClient {
 			switch (msg.type) {
 				case "auth.success":
 					if (this._state === "authenticating") {
-						logger$11.info("Authentication successful, sending agent.register", { connectionId: msg.connectionId });
+						logger$12.info("Authentication successful, sending agent.register", { connectionId: msg.connectionId });
 						this._state = "registering";
 						this.sendAgentRegister();
 					}
 					break;
 				case "auth.failure":
-					logger$11.error("Authentication FAILED -- token is invalid or expired. NOT retrying.", { reason: msg.reason });
+					logger$12.error("Authentication FAILED -- token is invalid or expired. NOT retrying.", { reason: msg.reason });
 					this.authFailed = true;
 					this.intentionalDisconnect = true;
 					if (this.ws) {
@@ -984,7 +987,7 @@ var OrchestratorClient = class OrchestratorClient {
 					this._state = "disconnected";
 					break;
 				case "register.ack":
-					logger$11.info("Registration acknowledged by orchestrator", {
+					logger$12.info("Registration acknowledged by orchestrator", {
 						agentId: msg.agentId,
 						labels: msg.labels,
 						scalerManaged: msg.scalerManaged,
@@ -999,14 +1002,14 @@ var OrchestratorClient = class OrchestratorClient {
 					this.sendConfigAck(msg.agentId);
 					break;
 				case "job.dispatch":
-					logger$11.info("Job dispatch received", {
+					logger$12.info("Job dispatch received", {
 						runId: msg.runId,
 						jobId: msg.jobId
 					});
 					this.onJobDispatch(msg);
 					break;
 				case "job.cancel":
-					logger$11.info("Job cancel received", {
+					logger$12.info("Job cancel received", {
 						runId: msg.runId,
 						jobId: msg.jobId,
 						reason: msg.reason
@@ -1014,7 +1017,7 @@ var OrchestratorClient = class OrchestratorClient {
 					this.onJobCancel(msg);
 					break;
 				case "job.concurrency.ack": {
-					logger$11.info("Concurrency ack received", {
+					logger$12.info("Concurrency ack received", {
 						requestId: msg.requestId,
 						action: msg.action
 					});
@@ -1029,7 +1032,7 @@ var OrchestratorClient = class OrchestratorClient {
 					break;
 				}
 				case "step.approval-resolved": {
-					logger$11.info("Step approval resolved", {
+					logger$12.info("Step approval resolved", {
 						requestId: msg.requestId,
 						runId: msg.runId,
 						jobId: msg.jobId,
@@ -1049,7 +1052,7 @@ var OrchestratorClient = class OrchestratorClient {
 					break;
 				}
 				case "fleet.logs.request":
-					logger$11.info("Fleet log collection requested", {
+					logger$12.info("Fleet log collection requested", {
 						requestId: msg.requestId,
 						logWindowHours: msg.logWindowHours
 					});
@@ -1059,7 +1062,7 @@ var OrchestratorClient = class OrchestratorClient {
 			return;
 		}
 		if (heartbeatSchema.safeParse(raw).success) return;
-		logger$11.warn("Invalid message from orchestrator", { errors: parsed.error.issues });
+		logger$12.warn("Invalid message from orchestrator", { errors: parsed.error.issues });
 	}
 	flushBuffer() {
 		const events = this.eventBuffer.flush();
@@ -1073,11 +1076,11 @@ var OrchestratorClient = class OrchestratorClient {
 		}
 		this.disconnectedAt = null;
 		if (events.length > 0) {
-			logger$11.info("Flushing event buffer", { count: events.length });
+			logger$12.info("Flushing event buffer", { count: events.length });
 			for (const msg of events) this.sendDirect(msg);
 		}
 		if (logLines.length > 0) {
-			logger$11.info("Flushing log buffer", { count: logLines.length });
+			logger$12.info("Flushing log buffer", { count: logLines.length });
 			for (let i = 0; i < logLines.length; i += OrchestratorClient.LOG_BATCH_SIZE) {
 				const batch = logLines.slice(i, i + OrchestratorClient.LOG_BATCH_SIZE);
 				this.sendAgentLogMessage(batch);
@@ -1130,14 +1133,14 @@ var OrchestratorClient = class OrchestratorClient {
 	*/
 	blockMmdsAccess() {
 		if (process.getuid?.() !== 0) {
-			logger$11.info("MMDS iptables block skipped (non-root) — network isolation handled by orchestrator");
+			logger$12.info("MMDS iptables block skipped (non-root) — network isolation handled by orchestrator");
 			return;
 		}
 		try {
 			execSync("iptables -A OUTPUT -d 169.254.169.254 -j DROP", { timeout: 5e3 });
-			logger$11.info("MMDS access blocked via iptables");
+			logger$12.info("MMDS access blocked via iptables");
 		} catch (err) {
-			logger$11.warn("Failed to block MMDS access via iptables", { error: toErrorMessage(err) });
+			logger$12.warn("Failed to block MMDS access via iptables", { error: toErrorMessage(err) });
 		}
 	}
 	/**
@@ -1151,7 +1154,7 @@ var OrchestratorClient = class OrchestratorClient {
 				messageId: `config-ack-${agentId}-${Date.now()}`,
 				agentId
 			}));
-			logger$11.info("Config ACK sent to orchestrator", { agentId });
+			logger$12.info("Config ACK sent to orchestrator", { agentId });
 		}
 	}
 	startHeartbeat() {
@@ -1217,12 +1220,12 @@ var OrchestratorClient = class OrchestratorClient {
 	scheduleReconnect() {
 		this.cancelReconnect();
 		if (this.authFailed) {
-			logger$11.error("Not reconnecting: authentication permanently failed");
+			logger$12.error("Not reconnecting: authentication permanently failed");
 			return;
 		}
 		const delay = this.getReconnectDelay();
 		this.reconnectAttempts++;
-		logger$11.info("Scheduling reconnect", {
+		logger$12.info("Scheduling reconnect", {
 			attempt: this.reconnectAttempts,
 			delayMs: Math.round(delay)
 		});
@@ -1307,14 +1310,14 @@ var init_console_capture = __esmMin((() => {
 init_console_capture();
 function safe(name, fallback = "unknown") {
 	switch (name) {
-		case "version": return "0.1.21";
-		case "buildCommit": return "8aeccc2c4";
-		case "sdkVersion": return "0.1.21";
-		case "sdkBundleHash": return "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
-		case "sharedVersion": return "0.1.21";
+		case "version": return "0.1.23";
+		case "buildCommit": return "4465935bb";
+		case "sdkVersion": return "0.1.23";
+		case "sdkBundleHash": return "3a83610d2d122b9f9b0f924b225a47050d0b35f00a8f3256a3e15f3ff3bf7ddc";
+		case "sharedVersion": return "0.1.23";
 		case "sharedBundleHash": return "991385c024392c395d3eb8a68946ef8ed3fcba96f2652f21c3164a54eafa1b1b";
-		case "engineVersion": return "0.1.21";
-		case "engineBundleHash": return "1b1f49acbbb66f045cfa8284406fce48de8975d9ec3a5685aff70313cd8232cc";
+		case "engineVersion": return "0.1.23";
+		case "engineBundleHash": return "4a0566c709180a6a8744ff3281640e596b8e661ddceb81327cd37d232d9768d5";
 		default: return fallback;
 	}
 }
@@ -1571,6 +1574,83 @@ async function gcStaleAgentTmpDirs(base = tmpdir()) {
 	})];
 }
 //#endregion
+//#region src/execution/reboot.ts
+/**
+* Cross-OS host reboot for the workflow-level `restartHost()` step.
+*
+* The agent runs ON the host it executes jobs for, so a `restartHost()` step
+* reboots that host. `rebootCommandFor` is the pure OS→command mapping (kept
+* pure for unit-testing); `issueReboot` spawns it detached with a tiny grace so
+* the job's final flush completes before the box goes down.
+*
+* Rebooting needs host privilege (root/admin). If the primitive is denied, the
+* spawn fails and the caller clears the orchestrator's reboot-pending flag and
+* surfaces the error — the deadline sweep is the backstop.
+*/
+const logger$11 = createLogger({ prefix: "reboot" });
+/** The OS reboot primitive for a Node platform string. */
+function rebootCommandFor(platform) {
+	switch (platform) {
+		case "darwin": return {
+			cmd: "shutdown",
+			args: ["-r", "now"]
+		};
+		case "win32": return {
+			cmd: "shutdown",
+			args: [
+				"/r",
+				"/t",
+				"0"
+			]
+		};
+		default: return {
+			cmd: "systemctl",
+			args: ["reboot"]
+		};
+	}
+}
+/**
+* Issue the OS reboot detached. Resolves once the child has been spawned (the
+* box is going down; there is nothing to await). Rejects synchronously if the
+* spawn itself fails (e.g. the binary is missing). A privilege denial usually
+* surfaces as a non-zero exit AFTER spawn — logged, not thrown, because by then
+* the box may already be on its way down.
+*/
+function issueReboot(platform = process.platform) {
+	const { cmd, args } = rebootCommandFor(platform);
+	return new Promise((resolve, reject) => {
+		try {
+			const child = spawn(cmd, args, {
+				detached: true,
+				stdio: "ignore"
+			});
+			child.on("error", (err) => {
+				logger$11.error("Reboot command failed to spawn", {
+					cmd,
+					args,
+					error: String(err)
+				});
+				reject(err);
+			});
+			child.on("exit", (code) => {
+				if (code !== 0 && code !== null) logger$11.warn("Reboot command exited non-zero (privilege denied?)", {
+					cmd,
+					args,
+					code
+				});
+			});
+			child.unref();
+			logger$11.info("Issued host reboot", {
+				cmd,
+				args
+			});
+			setImmediate(resolve);
+		} catch (err) {
+			reject(err);
+		}
+	});
+}
+//#endregion
 //#region src/metrics/prometheus.ts
 var prometheus_exports = /* @__PURE__ */ __exportAll({
 	cloneDurationSeconds: () => cloneDurationSeconds,
@@ -2000,8 +2080,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
 }
 var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
 var init_workflow_loader = __esmMin((() => {
-	AGENT_SDK_VERSION = "0.1.21";
-	AGENT_SDK_BUNDLE_HASH = "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
+	AGENT_SDK_VERSION = "0.1.23";
+	AGENT_SDK_BUNDLE_HASH = "3a83610d2d122b9f9b0f924b225a47050d0b35f00a8f3256a3e15f3ff3bf7ddc";
 	hookRegistered = false;
 }));
 //#endregion
@@ -2486,6 +2566,274 @@ var init_init_runner = __esmMin((() => {
 	init_timeout_util();
 }));
 //#endregion
+//#region src/bootstrap/ssh-exec.ts
+/**
+* Agent-side SSH helper for bootstrap bring-up.
+*
+* Lifts the discipline from `infra/prod/hw/ssh.sh` + the deploy-prod
+* `runOnBox` helper into the agent: the bring-up private key is piped into an
+* **ephemeral ssh-agent** (`ssh-add -` reads it from stdin) and the agent is
+* torn down in a `finally`, so the key never lands on disk and never enters a
+* long-lived process environment. The agent only holds the key for the
+* lifetime of the one `ssh` / `scp` invocation.
+*
+* `sshExec` runs a remote command; `sshPush` ships local bytes to a remote
+* path (`ssh 'cat > path'`). Both accept a resolved private key string
+* (supplied by the caller — the orchestrator resolves the scoped secret and
+* hands the key down). `sshExec` supports `{ stdin, port, hostKeyMode }` so it
+* can also drive a pre-boot dropbear / initramfs prompt (a forced-command
+* endpoint such as `cryptroot-unlock` on port 2222, which accepts the unlock
+* input on stdin and uses a transient host key distinct from the OS sshd key).
+*/
+/** Common `-o` flags every bootstrap SSH connection carries. */
+function baseSshOptions(hostKeyMode) {
+	return [
+		"-o",
+		`StrictHostKeyChecking=${HOST_KEY_FLAG[hostKeyMode]}`,
+		"-o",
+		"ConnectTimeout=10",
+		"-o",
+		"BatchMode=yes"
+	];
+}
+/** Resolve the `user@address` target + port from reach metadata. */
+function resolveTarget(reach, portOverride) {
+	if (!reach.address) throw new Error(`host ${reach.agentId} has no SSH reach address declared`);
+	const user = reach.sshUser ?? SSH_USER_DEFAULT;
+	const port = portOverride ?? reach.sshPort ?? SSH_PORT_DEFAULT;
+	return {
+		dest: `${user}@${reach.address}`,
+		port
+	};
+}
+/**
+* Run a command on the target over SSH using an ephemeral, in-memory key.
+*
+* The key is loaded into a per-call ssh-agent (never written to disk) and the
+* agent is killed in `finally`. The remote command's exit code / stdout /
+* stderr are returned verbatim — a non-zero exit is reported, not swallowed
+* (a pre-boot unlock legitimately drops the session, so the caller decides
+* what a "success" looks like).
+*/
+async function sshExec(reach, privateKey, command, opts = {}, deps = {}) {
+	const spawnFn = deps.spawnFn ?? defaultSpawn;
+	const hostKeyMode = opts.hostKeyMode ?? "accept-new";
+	const { dest, port } = resolveTarget(reach, opts.port);
+	return withEphemeralAgent(privateKey, spawnFn, async (env) => {
+		return spawnFn("ssh", [
+			...baseSshOptions(hostKeyMode),
+			"-p",
+			String(port),
+			dest,
+			command
+		], {
+			env,
+			stdin: opts.stdin
+		});
+	});
+}
+/**
+* Ship local bytes to a remote path over SSH (`ssh 'cat > path'`), using the
+* same ephemeral-key discipline. Throws on a non-zero exit (a push must
+* succeed end-to-end, unlike a pre-boot unlock).
+*/
+async function sshPush(reach, privateKey, localBytes, remotePath, opts = {}, deps = {}) {
+	const spawnFn = deps.spawnFn ?? defaultSpawn;
+	const hostKeyMode = opts.hostKeyMode ?? "accept-new";
+	const { dest, port } = resolveTarget(reach, opts.port);
+	const result = await withEphemeralAgent(privateKey, spawnFn, async (env) => {
+		return spawnFn("ssh", [
+			...baseSshOptions(hostKeyMode),
+			"-p",
+			String(port),
+			dest,
+			`cat > '${remotePath.replace(/'/g, `'\\''`)}'`
+		], {
+			env,
+			stdin: localBytes
+		});
+	});
+	if (result.exitCode !== 0) throw new Error(`sshPush(${reach.agentId}:${remotePath}): exit ${result.exitCode}${result.stderr ? `\n${result.stderr}` : ""}`);
+}
+/**
+* Start a per-call ephemeral ssh-agent, load the key via stdin (never a file),
+* run `body` with `SSH_AUTH_SOCK` in env, and kill the agent in `finally`.
+*/
+async function withEphemeralAgent(privateKey, spawnFn, body) {
+	const baseEnv = { ...process.env };
+	const start = await spawnFn("ssh-agent", ["-s"], { env: baseEnv });
+	if (start.exitCode !== 0) throw new Error(`ssh-agent start failed: exit ${start.exitCode}\n${start.stderr}`);
+	const sock = parseAgentSocket(start.stdout);
+	const pid = parseAgentPid(start.stdout);
+	const agentEnv = {
+		...baseEnv,
+		SSH_AUTH_SOCK: sock,
+		...pid ? { SSH_AGENT_PID: pid } : {},
+		SSH_ASKPASS: "/bin/false",
+		DISPLAY: ""
+	};
+	try {
+		const add = await spawnFn("ssh-add", ["-"], {
+			env: agentEnv,
+			stdin: privateKey.endsWith("\n") ? privateKey : `${privateKey}\n`
+		});
+		if (add.exitCode !== 0) throw new Error(`ssh-add failed: exit ${add.exitCode}\n${add.stderr}`);
+		return await body(agentEnv);
+	} finally {
+		await spawnFn("ssh-agent", ["-k"], { env: agentEnv }).catch(() => {});
+	}
+}
+/** Extract `SSH_AUTH_SOCK=<path>;` from `ssh-agent -s` output. */
+function parseAgentSocket(out) {
+	const m = out.match(/SSH_AUTH_SOCK=([^;\n]+)/);
+	if (!m) throw new Error("ssh-agent -s did not emit SSH_AUTH_SOCK");
+	return m[1];
+}
+/** Extract `SSH_AGENT_PID=<n>;` from `ssh-agent -s` output (best-effort). */
+function parseAgentPid(out) {
+	return out.match(/SSH_AGENT_PID=([^;\n]+)/)?.[1];
+}
+var defaultSpawn, SSH_USER_DEFAULT, SSH_PORT_DEFAULT, HOST_KEY_FLAG;
+var init_ssh_exec = __esmMin((() => {
+	defaultSpawn = (command, args, opts) => new Promise((resolve, reject) => {
+		const child = spawn(command, args, {
+			env: opts.env,
+			stdio: [
+				opts.stdin !== void 0 ? "pipe" : "ignore",
+				"pipe",
+				"pipe"
+			]
+		});
+		let stdout = "";
+		let stderr = "";
+		child.stdout?.on("data", (b) => {
+			stdout += b.toString();
+		});
+		child.stderr?.on("data", (b) => {
+			stderr += b.toString();
+		});
+		child.on("error", reject);
+		child.on("close", (code) => resolve({
+			exitCode: code ?? -1,
+			stdout,
+			stderr
+		}));
+		if (opts.stdin !== void 0) child.stdin?.end(opts.stdin);
+	});
+	SSH_USER_DEFAULT = "root";
+	SSH_PORT_DEFAULT = 22;
+	HOST_KEY_FLAG = {
+		"accept-new": "accept-new",
+		strict: "yes"
+	};
+}));
+//#endregion
+//#region src/bootstrap/ensure-init-runner.ts
+/**
+* Build the launcher script that starts the init-runner on the target with its
+* bootstrap env. Detached (`setsid … &`) so the SSH session can return while
+* the agent keeps running and dials the orchestrator.
+*/
+function buildLauncher(material, agentCommand) {
+	return [
+		"#!/usr/bin/env bash",
+		"set -euo pipefail",
+		`setsid env ${[
+			`KICI_AGENT_TOKEN=${shQuote(material.bootstrapToken)}`,
+			`KICI_AGENT_ID=${shQuote(material.targetAgentId)}`,
+			`KICI_ORCHESTRATOR_URL=${shQuote(material.orchestratorUrl)}`,
+			`KICI_LABELS=${shQuote(material.labels.join(","))}`,
+			"KICI_EXECUTION_MODE=bare-metal",
+			"KICI_PORT=0"
+		].join(" \\\n  ")} \\`,
+		`  ${agentCommand} >/tmp/kici-init-runner.log 2>&1 &`,
+		"echo \"init-runner started pid=$!\""
+	].join("\n");
+}
+/** Single-quote a value for safe embedding in the launcher's env assignment. */
+function shQuote(v) {
+	return `'${v.replace(/'/g, `'\\''`)}'`;
+}
+/**
+* Bring up a temporary init-runner on `targetAgentId`. Returns `{ broughtUp }`:
+* false when the target already had a live agent (the orchestrator no-op'd),
+* true when this call dropped + started the init-runner.
+*/
+async function ensureInitRunner(transport, targetAgentId, deps = {}) {
+	const material = await transport("kici.ensureInitRunner", { targetAgentId });
+	if (!material.broughtUp) return { broughtUp: false };
+	const { reach, privateKey, bootstrapToken, orchestratorUrl, labels } = material;
+	if (!reach || !privateKey || !bootstrapToken || !orchestratorUrl || !labels) throw new Error(`orchestrator returned incomplete bring-up material for ${targetAgentId}`);
+	const agentCommand = deps.agentCommand ?? DEFAULT_AGENT_COMMAND;
+	await sshPush(reach, privateKey, buildLauncher({
+		bootstrapToken,
+		targetAgentId,
+		orchestratorUrl,
+		labels
+	}, agentCommand), LAUNCHER_REMOTE_PATH, {}, deps);
+	const run = await sshExec(reach, privateKey, `chmod 0700 ${LAUNCHER_REMOTE_PATH} && ${LAUNCHER_REMOTE_PATH}`, {}, deps);
+	if (run.exitCode !== 0) throw new Error(`init-runner launch on ${targetAgentId} failed: exit ${run.exitCode}${run.stderr ? `\n${run.stderr}` : ""}`);
+	return { broughtUp: true };
+}
+var DEFAULT_AGENT_COMMAND, LAUNCHER_REMOTE_PATH;
+var init_ensure_init_runner = __esmMin((() => {
+	init_ssh_exec();
+	DEFAULT_AGENT_COMMAND = "kici-agent";
+	LAUNCHER_REMOTE_PATH = "/tmp/kici-init-runner.sh";
+}));
+//#endregion
+//#region src/bootstrap/pre-boot-send.ts
+/**
+* Ship a pre-boot input to the target's dropbear/initramfs SSH channel. The
+* input plaintext is resolved server-side and never logged. Resolves once the
+* send completes (the SSH session legitimately drops as the box boots).
+*/
+async function preBootSend(transport, targetAgentId, opts, deps = {}) {
+	const material = await transport("kici.preBootSend", {
+		targetAgentId,
+		inputSecret: opts.inputSecret,
+		...opts.port !== void 0 ? { port: opts.port } : {},
+		...opts.command !== void 0 ? { command: opts.command } : {}
+	});
+	await sshExec(material.reach, material.privateKey, material.command, {
+		stdin: material.input,
+		port: material.port,
+		hostKeyMode: "accept-new"
+	}, deps);
+}
+var init_pre_boot_send = __esmMin((() => {
+	init_ssh_exec();
+}));
+//#endregion
+//#region src/bootstrap/api-intercept.ts
+/**
+* Wrap the orchestrator API transport so the two bootstrap methods are handled
+* in-process (SSH transport here; privileged resolve relayed to the
+* orchestrator). `relay` is the raw orchestrator transport (the WS
+* `sendApiRequest`).
+*/
+function withBootstrapInterception(relay, deps = {}) {
+	return async (method, params = {}) => {
+		if (method === ENSURE_INIT_RUNNER) return ensureInitRunner(relay, String(params.targetAgentId ?? ""), deps);
+		if (method === PRE_BOOT_SEND) {
+			await preBootSend(relay, String(params.targetAgentId ?? ""), {
+				inputSecret: String(params.inputSecret ?? ""),
+				...typeof params.port === "number" ? { port: params.port } : {},
+				...typeof params.command === "string" ? { command: params.command } : {}
+			}, deps);
+			return;
+		}
+		return relay(method, params);
+	};
+}
+var ENSURE_INIT_RUNNER, PRE_BOOT_SEND;
+var init_api_intercept = __esmMin((() => {
+	init_ensure_init_runner();
+	init_pre_boot_send();
+	ENSURE_INIT_RUNNER = "kici.ensureInitRunner";
+	PRE_BOOT_SEND = "kici.preBootSend";
+}));
+//#endregion
 //#region src/execution/dynamic-job-serializer.ts
 /**
 * Convert an array of SDK Job objects into LockJob format for the orchestrator.
@@ -2550,7 +2898,8 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
 * Validates against generatedNames union staticNames union allowedGroups.
 *
 * Returns the lock file representation: strings for concrete refs,
-* NeedsEntry for { name, ifFailed }, NeedsGroupEntry for group refs.
+* NeedsEntry for { name, when }, NeedsGroupEntry for group refs (each `when`
+* normalized to a runOn status-set).
 */
 function resolveNeeds(needs, generatedNames, staticNames, allowedGroups) {
 	if (!needs || needs.length === 0) return [];
@@ -2565,7 +2914,7 @@ function resolveNeeds(needs, generatedNames, staticNames, allowedGroups) {
 			if (!allowedGroups.has(groupRef.group)) throw new Error(`Dynamic group '${groupRef.group}' not found in workflow (available groups: ${[...allowedGroups].join(", ") || "none"})`);
 			return {
 				group: groupRef.group,
-				ifFailed: groupRef.ifFailed ?? "skip"
+				runOn: resolveWhenToRunOn(groupRef.when)
 			};
 		}
 		if (typeof dep === "object" && dep !== null && "group" in dep) {
@@ -2573,7 +2922,7 @@ function resolveNeeds(needs, generatedNames, staticNames, allowedGroups) {
 			if (!allowedGroups.has(groupDep.group)) throw new Error(`Dynamic group '${groupDep.group}' not found in workflow (available groups: ${[...allowedGroups].join(", ") || "none"})`);
 			return {
 				group: groupDep.group,
-				ifFailed: groupDep.ifFailed ?? "skip"
+				runOn: resolveWhenToRunOn(groupDep.when)
 			};
 		}
 		if (typeof dep === "object" && dep !== null && "name" in dep && !("steps" in dep)) {
@@ -2581,7 +2930,7 @@ function resolveNeeds(needs, generatedNames, staticNames, allowedGroups) {
 			if (!allNames.has(namedDep.name)) throw new Error(`Job dependency '${namedDep.name}' not found in workflow jobs (checked: ${generatedNames.size} generated, ${staticNames.size} static)`);
 			return {
 				name: namedDep.name,
-				ifFailed: namedDep.ifFailed ?? "skip"
+				runOn: resolveWhenToRunOn(namedDep.when)
 			};
 		}
 		if (isDynamicJobFn(dep)) throw new Error("Job dependency cannot be a DynamicJobFn");
@@ -2606,7 +2955,13 @@ function serializeSteps(steps) {
 			name: step.name || `step-${index}`,
 			hasOutputs: !!step.outputs,
 			...step.continueOnError ? { continueOnError: true } : {},
-			...step.timeout ? { timeout: step.timeout } : {}
+			...step.timeout ? { timeout: step.timeout } : {},
+			...step.retry ? { retry: {
+				maxAttempts: step.retry.maxAttempts,
+				delayMs: step.retry.delayMs,
+				backoff: step.retry.backoff,
+				maxDelayMs: step.retry.maxDelayMs
+			} } : {}
 		};
 	});
 }
@@ -4182,6 +4537,9 @@ function buildRequest(dispatch, workDir) {
 		matrixValues: jobConfig.matrixValues,
 		host: jobConfig.host,
 		agent: jobConfig.agent,
+		dispatchInputs: jobConfig.dispatchInputs,
+		fanoutIndex: jobConfig.fanoutIndex,
+		fanoutTotal: jobConfig.fanoutTotal,
 		secrets: dispatch.secrets,
 		namespacedSecrets: dispatch.namespacedSecrets,
 		sourceFile: jobConfig.source?.file,
@@ -4212,6 +4570,8 @@ function buildRequest(dispatch, workDir) {
 		concurrencyEvaluationTimeoutMs: jobConfig.concurrencyEvaluationTimeoutMs,
 		branch: dispatch.ref,
 		upstreamJobOutputs: dispatch.upstreamJobOutputs,
+		upstreamJobStatuses: dispatch.upstreamJobStatuses,
+		jobNeeds: jobConfig.needs,
 		npmRegistries: dispatch.npmRegistries,
 		installEnvSecrets: dispatch.installEnvSecrets,
 		jobIdShort: dispatch.jobId.slice(0, 8),
@@ -5394,7 +5754,10 @@ var init_sandbox = __esmMin((() => {
 }));
 //#endregion
 //#region src/execution/job-runner.ts
-var job_runner_exports = /* @__PURE__ */ __exportAll({ JobRunner: () => JobRunner$1 });
+var job_runner_exports = /* @__PURE__ */ __exportAll({
+	JobRunner: () => JobRunner$1,
+	buildEvalNeedsContext: () => buildEvalNeedsContext
+});
 /**
 * Check if a file exists at the given path.
 */
@@ -5457,6 +5820,8 @@ var init_job_runner = __esmMin((() => {
 	init_source_packer();
 	init_source_restore();
 	init_init_runner();
+	init_api_intercept();
+	init_ensure_init_runner();
 	init_timeout_util();
 	init_dynamic_job_serializer();
 	init_log_streamer();
@@ -5569,6 +5934,10 @@ var init_job_runner = __esmMin((() => {
 				await this.handleDynamicJobFn(dispatch, workDir, abortController);
 				return true;
 			}
+			if (jobConfig.bringupOnly === true) {
+				await this.handleBringupJob(dispatch);
+				return true;
+			}
 			if (jobConfig.buildOnly === true) {
 				if (jobConfig.fullRepo) {
 					logger$2.warn("Build job received for fullRepo run -- skipping (should not happen)", {
@@ -5741,7 +6110,7 @@ var init_job_runner = __esmMin((() => {
 						reason: ack.reason
 					};
 				},
-				onApiRequest: this._sendApiRequest ? async (method, params) => this._sendApiRequest(method, params) : void 0,
+				onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => this._sendApiRequest(method, params)) : void 0,
 				onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
 				onProvenanceRequest: this._relayProvenance ? async (request) => this._relayProvenance(jobId, request) : void 0,
 				onApprovalRequest: this._sendStepApproval ? async (request) => this._sendStepApproval(dispatch.runId, dispatch.jobId, request) : void 0,
@@ -5803,6 +6172,53 @@ var init_job_runner = __esmMin((() => {
 			}, result.secretOutputs);
 		}
 		/**
+		* Handle a bring-up job: bring up a temporary init-runner on a declared-but-
+		* un-agented host over SSH (fresh-box bootstrap convergence). The orchestrator
+		* dispatches this synthetic `__bringup__` job to an agent holding the
+		* `kici:capability:ssh-transport` capability; here we run the agent-side
+		* `ensureInitRunner` helper (the privileged resolve is relayed to the
+		* orchestrator, the SSH transport happens in this agent process — never
+		* reaching workflow code). No clone, no sandbox — the init-runner then
+		* connects under the target's agent id and the orchestrator's pinned-hold
+		* drains the target's bootstrap steps onto it.
+		*/
+		async handleBringupJob(dispatch) {
+			const { runId, jobId, jobConfig } = dispatch;
+			const targetAgentId = String(jobConfig.bringupTarget ?? "");
+			logger$2.info("Starting bring-up job", {
+				jobId,
+				runId,
+				targetAgentId
+			});
+			this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
+			const streamer = this.createStepStreamer(dispatch, 0);
+			this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.running);
+			try {
+				if (!targetAgentId) throw new Error("bring-up job missing bringupTarget");
+				if (!this._sendApiRequest) throw new Error("bring-up job requires an orchestrator API transport");
+				streamer.addLine(`Bringing up init-runner on ${targetAgentId}…`);
+				const result = await ensureInitRunner(async (method, params) => this._sendApiRequest(method, params), targetAgentId);
+				streamer.addLine(result.broughtUp ? `Init-runner brought up on ${targetAgentId}.` : `${targetAgentId} already has a live agent — no bring-up needed.`);
+				await streamer.flush();
+				this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.success, void 0, streamer.getTotalBytes());
+				streamer.destroy();
+				this.sendJobStatus(dispatch, ExecutionJobStatus.enum.success);
+			} catch (err) {
+				const message = err instanceof Error ? err.message : String(err);
+				streamer.addLine(`Bring-up failed: ${message}`);
+				await streamer.flush();
+				this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.failed, void 0, streamer.getTotalBytes());
+				streamer.destroy();
+				this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, { error: message });
+				logger$2.warn("Bring-up job failed", {
+					jobId,
+					runId,
+					targetAgentId,
+					error: message
+				});
+			}
+		}
+		/**
 		* Handle a build-only job.
 		*
 		* Build jobs install dependencies, pack them into a tarball,
@@ -6195,7 +6611,7 @@ var init_job_runner = __esmMin((() => {
 					error: (msg, ..._args) => evalLog(`ERROR: ${msg}`),
 					debug: (msg, ..._args) => evalLog(`DEBUG: ${msg}`)
 				};
-				const kici = buildKiciApi(this._sendApiRequest ? (method, params) => this._sendApiRequest(method, params ?? {}) : () => Promise.reject(/* @__PURE__ */ new Error("Agent API not available")));
+				const kici = buildKiciApi(this._sendApiRequest ? withBootstrapInterception((method, params) => this._sendApiRequest(method, params ?? {})) : () => Promise.reject(/* @__PURE__ */ new Error("Agent API not available")));
 				const lockJobs = await runCaptured(evalSink, async () => {
 					const { module } = await loadWorkflowSource(workDir, config.source.file, config.contentHash, config.resolvedHashFiles);
 					evalLog("Workflow loaded");
@@ -6425,14 +6841,14 @@ var init_job_runner = __esmMin((() => {
 */
 init_console_capture();
 init_npm_resolver();
-const AGENT_VERSION = "0.1.21";
-const BUILD_COMMIT = "8aeccc2c4";
-const SDK_VERSION = "0.1.21";
-const SDK_BUNDLE_HASH = "8308089347c304e41b457d3867b17bbff11d6b5cd9706b6823e7abdbd849f33f";
-const SHARED_VERSION = "0.1.21";
+const AGENT_VERSION = "0.1.23";
+const BUILD_COMMIT = "4465935bb";
+const SDK_VERSION = "0.1.23";
+const SDK_BUNDLE_HASH = "3a83610d2d122b9f9b0f924b225a47050d0b35f00a8f3256a3e15f3ff3bf7ddc";
+const SHARED_VERSION = "0.1.23";
 const SHARED_BUNDLE_HASH = "991385c024392c395d3eb8a68946ef8ed3fcba96f2652f21c3164a54eafa1b1b";
-const ENGINE_VERSION = "0.1.21";
-const ENGINE_BUNDLE_HASH = "1b1f49acbbb66f045cfa8284406fce48de8975d9ec3a5685aff70313cd8232cc";
+const ENGINE_VERSION = "0.1.23";
+const ENGINE_BUNDLE_HASH = "4a0566c709180a6a8744ff3281640e596b8e661ddceb81327cd37d232d9768d5";
 initTelemetry({
 	serviceName: "kici-agent",
 	otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
@@ -6498,6 +6914,7 @@ await guardStartup(logger$1, async () => {
 	}
 	let isDraining = false;
 	let idleShutdownTimer;
+	let rebootIntent = false;
 	let client;
 	const jobRunner = new JobRunner({
 		send: (msg) => client.send(msg),
@@ -6511,7 +6928,15 @@ await guardStartup(logger$1, async () => {
 		sendJobContext: (runId, jobId, context) => client.sendJobContext(runId, jobId, context),
 		sendRunEvent: (runId, eventType, opts) => client.sendRunEvent(runId, eventType, opts),
 		sendConcurrencyReport: (runId, jobId, group) => client.sendConcurrencyReport(runId, jobId, group),
-		sendApiRequest: (method, params) => client.sendApiRequest(method, params ?? {}),
+		sendApiRequest: async (method, params) => {
+			if (method === "host.requestReboot") {
+				if (config.isOrchestratorHost) throw new Error("refusing to reboot the orchestrator host");
+				const result = await client.sendApiRequest(method, params ?? {});
+				rebootIntent = true;
+				return result;
+			}
+			return client.sendApiRequest(method, params ?? {});
+		},
 		requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
 		relayProvenance: (jobId, request) => client.relayProvenance(jobId, request),
 		sendStepApproval: (runId, jobId, request) => client.sendStepApproval(runId, jobId, request)
@@ -6600,6 +7025,13 @@ await guardStartup(logger$1, async () => {
 					jobsActive.add(-1);
 					metricsReporter.collectAndSend().catch(() => {});
 					sendAgentStatus();
+					if (rebootIntent) {
+						rebootIntent = false;
+						issueReboot().catch((err) => {
+							logger$1.error("Host reboot failed; cancelling reboot-pending flag", { error: toErrorMessage(err) });
+							client.sendApiRequest("host.cancelReboot", {}).catch(() => {});
+						});
+					}
 					if (config.scalerManaged && jobRunner.activeJobs.size === 0) {
 						if (client.state !== "registered") {
 							logger$1.info("Scaler-managed agent idle but disconnected, deferring shutdown until reconnected");