@kici-dev/agent 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/checkout/clone-job-repos.d.ts +53 -0
  2. package/dist/checkout/credential-helper-bin.d.ts +28 -0
  3. package/dist/checkout/credential-helper-host.d.ts +48 -0
  4. package/dist/checkout/credential-helper.d.ts +44 -0
  5. package/dist/checkout/git-clone.d.ts +26 -0
  6. package/dist/checkout/grant-table.d.ts +30 -0
  7. package/dist/checkout/job-git-credentials.d.ts +49 -0
  8. package/dist/checkout/write-elevation.d.ts +40 -0
  9. package/dist/config.d.ts +38 -0
  10. package/dist/container-ts-loader-hook.js +2047 -1814
  11. package/dist/execution/between-jobs-controller.d.ts +50 -0
  12. package/dist/execution/between-jobs-reset.d.ts +25 -0
  13. package/dist/execution/cleanup-rerun.d.ts +21 -0
  14. package/dist/execution/dynamic-job-serializer.d.ts +6 -2
  15. package/dist/execution/image-build/build-engine.d.ts +75 -0
  16. package/dist/execution/image-build/build-step.d.ts +57 -0
  17. package/dist/execution/image-build/resolve-build-spec.d.ts +41 -0
  18. package/dist/execution/image-build/runtime-facts.d.ts +31 -0
  19. package/dist/execution/job-runner.d.ts +48 -0
  20. package/dist/execution/sandbox/bare-metal-sandbox.d.ts +25 -1
  21. package/dist/execution/sandbox/container-sandbox.d.ts +97 -2
  22. package/dist/execution/sandbox/fork-runner.d.ts +55 -1
  23. package/dist/execution/sandbox/image-preflight.d.ts +38 -0
  24. package/dist/execution/sandbox/ipc-protocol.d.ts +96 -2
  25. package/dist/execution/sandbox/kici-runtime.d.ts +48 -0
  26. package/dist/execution/sandbox/step-loop.d.ts +7 -0
  27. package/dist/execution/sandbox/types.d.ts +55 -1
  28. package/dist/execution/sandbox/workflow-runner.d.ts +23 -3
  29. package/dist/idle-shutdown.d.ts +24 -0
  30. package/dist/index.js +133 -62
  31. package/dist/metrics/prometheus.d.ts +26 -0
  32. package/dist/server.js +2050 -312
  33. package/dist/workflow-runner-bundle.js +41961 -41319
  34. package/dist/workflow-runner.js +332 -136
  35. package/dist/ws/orchestrator-client.d.ts +95 -3
  36. package/package.json +10 -10
  37. package/sbom.spdx.json +460 -455
package/dist/server.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
2
2
  import { dirname as __cjs_dirname } from "node:path";
3
- __cjs_dirname(__cjs_fileURLToPath(import.meta.url));
3
+ const __filename = __cjs_fileURLToPath(import.meta.url);
4
+ __cjs_dirname(__filename);
4
5
  import { createRequire, register } from "node:module";
5
6
  import crypto$1, { createCipheriv, createHash, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
6
7
  import * as os$1 from "node:os";
@@ -12,21 +13,23 @@ import winston from "winston";
12
13
  import { AgentDeliveryMode, AgentPlatform, RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, kiciTmpBase, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
13
14
  import { z } from "zod";
14
15
  import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
15
- import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
16
+ import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, RuntimeFact, SANDBOX_DEFAULT_VARS, SetupStepType, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, runtimeLabel, validateNoReservedLabels } from "@kici-dev/engine";
16
17
  import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
17
- import WebSocket from "ws";
18
+ import fsPromises, { access, chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, unlink, writeFile } from "node:fs/promises";
18
19
  import * as fs$1 from "node:fs";
19
20
  import fs, { createReadStream, createWriteStream, existsSync } from "node:fs";
21
+ import path, { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
22
+ import WebSocket from "ws";
20
23
  import { ZipArchive } from "archiver";
21
- import path, { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
22
24
  import { fileURLToPath, pathToFileURL } from "node:url";
23
25
  import { AsyncLocalStorage } from "node:async_hooks";
24
26
  import { format, promisify } from "node:util";
25
27
  import { OTEL_DATA_POINT_TYPE, mapDataPointTypeToWireKind } from "@kici-dev/engine/metrics/metric-kind-compat";
26
28
  import { gcStaleTmpDirs } from "@kici-dev/core/tmp-gc";
27
- import fsPromises, { access, lstat, mkdir, readFile, readdir, realpath, unlink, writeFile } from "node:fs/promises";
28
29
  import { makeTempDir } from "@kici-dev/core/tmp";
29
30
  import Docker from "dockerode";
31
+ import { GIT_CREDENTIAL_REQUEST_METHOD } from "@kici-dev/engine/protocol/messages/git-credential-relay";
32
+ import { createServer } from "node:net";
30
33
  import { buildKiciApi, buildNeedsContext, createFilterContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject, setJobOutputsMap, setStepOutputsMap, setStepRefMap } from "@kici-dev/sdk";
31
34
  import { $ } from "zx";
32
35
  import { c, x } from "tar";
@@ -38,6 +41,7 @@ import { normalizeRunsOnToMatchers } from "@kici-dev/engine/labels/compile";
38
41
  import { PNPM_IGNORE_BUILD_GATE_ARG, PackageManager, YarnFlavor, detectPackageManagerFromManifests, detectYarnFlavor } from "@kici-dev/shared/package-manager";
39
42
  import { parse, stringify } from "yaml";
40
43
  import { createInterface } from "node:readline";
44
+ import { RuntimeSubtree, ensureRuntimeVolume } from "@kici-dev/shared/container-runtime";
41
45
  var __defProp = Object.defineProperty;
42
46
  var __esmMin = (fn, res, err) => () => {
43
47
  if (err) throw err[0];
@@ -57,13 +61,258 @@ var __exportAll = (all, no_symbols) => {
57
61
  return target;
58
62
  };
59
63
  //#endregion
64
+ //#region src/execution/image-build/build-engine.ts
65
+ /**
66
+ * Build a job's container image with the host's build CLI.
67
+ *
68
+ * The CLI is REQUIRED — there is deliberately no socket-API fallback. One build
69
+ * path means one set of Dockerfile semantics: `.dockerignore`, BuildKit and
70
+ * every directive behave as they do on the author's own machine, instead of
71
+ * depending on which agent happened to pick the job up. A host without a CLI
72
+ * cannot run a Dockerfile job, and says so in as many words.
73
+ */
74
+ /** Is `bin` executable somewhere on `PATH`? */
75
+ function binaryOnPath(bin) {
76
+ return (process.env.PATH ?? "").split(delimiter).filter(Boolean).some((dir) => existsSync(join(dir, bin)));
77
+ }
78
+ /**
79
+ * The container socket the SANDBOX will use.
80
+ *
81
+ * Mirrors what dockerode's `new Docker()` resolves, and exists so build and run
82
+ * provably agree. A host with both runtimes whose sandbox socket points at
83
+ * podman would otherwise build with docker and then start the job container on
84
+ * a daemon that has never heard of that image — a failure that surfaces as
85
+ * "no such image" and names nothing.
86
+ */
87
+ function sandboxSocketPath() {
88
+ return process.env.DOCKER_HOST ?? "/var/run/docker.sock";
89
+ }
90
+ function resolveBuildCli(args) {
91
+ const { configured } = args;
92
+ const onPath = args.onPath ?? binaryOnPath;
93
+ if (configured) {
94
+ if (!onPath(configured)) throw new Error(`KICI_CONTAINER_BUILD_CLI is set to '${configured}', but '${configured}' is not on PATH on this agent host.`);
95
+ return configured;
96
+ }
97
+ const found = CLI_PREFERENCE.find((bin) => onPath(bin));
98
+ if (!found) throw new Error("This job builds its container image from a Dockerfile, which needs a build CLI on the agent host. Neither 'docker' nor 'podman' is on PATH. Install one, or route the job to a pool whose hosts have one.");
99
+ return found;
100
+ }
101
+ /** Point the CLI at a specific daemon, in that CLI's own spelling. */
102
+ function socketFlags(cli, socketPath) {
103
+ const url = socketPath.includes("://") ? socketPath : `unix://${socketPath}`;
104
+ return cli === ContainerBuildCli.enum.docker ? ["-H", url] : ["--url", url];
105
+ }
106
+ function buildArgv(args) {
107
+ const { cli, spec, socketPath } = args;
108
+ const argv = [];
109
+ if (socketPath) argv.push(...socketFlags(cli, socketPath));
110
+ argv.push("build", "-f", spec.dockerfilePath);
111
+ if (spec.target !== void 0) argv.push("--target", spec.target);
112
+ for (const [k, v] of Object.entries(spec.args)) argv.push("--build-arg", `${k}=${v}`);
113
+ for (const [k, v] of Object.entries(spec.labels)) argv.push("--label", `${k}=${v}`);
114
+ argv.push("-t", spec.tag);
115
+ argv.push(spec.contextDir);
116
+ return argv;
117
+ }
118
+ /**
119
+ * Write `authconfig` into a throwaway config directory and return the env that
120
+ * points the CLI at it.
121
+ *
122
+ * A throwaway directory rather than the host's own credential file: the build
123
+ * runs on a shared agent host, and a job's registry token has no business being
124
+ * written where the next job (or the operator) can read it.
125
+ */
126
+ async function withTempAuth(cli, authconfig) {
127
+ const dir = await mkdtemp(join(tmpdir(), "kici-build-auth-"));
128
+ const encoded = Buffer.from(`${authconfig.username}:${authconfig.password}`).toString("base64");
129
+ const body = JSON.stringify({ auths: { [authconfig.serveraddress]: { auth: encoded } } });
130
+ if (cli === ContainerBuildCli.enum.docker) {
131
+ await writeFile(join(dir, "config.json"), body, { mode: 384 });
132
+ return {
133
+ env: { DOCKER_CONFIG: dir },
134
+ dispose: () => rm(dir, {
135
+ recursive: true,
136
+ force: true
137
+ })
138
+ };
139
+ }
140
+ const file = join(dir, "auth.json");
141
+ await writeFile(file, body, { mode: 384 });
142
+ return {
143
+ env: { REGISTRY_AUTH_FILE: file },
144
+ dispose: () => rm(dir, {
145
+ recursive: true,
146
+ force: true
147
+ })
148
+ };
149
+ }
150
+ /**
151
+ * Split a chunk stream into whole lines, carrying the remainder between chunks.
152
+ *
153
+ * `flush` matters: a builder whose last line has no trailing newline — which is
154
+ * exactly the shape of an error written just before exit — would otherwise leave
155
+ * that line in the carry, and the one line the author most needs would be the
156
+ * one that never reaches the run log.
157
+ */
158
+ function makeLineSplitter(onLine) {
159
+ let carry = "";
160
+ return {
161
+ write: (chunk) => {
162
+ carry += chunk.toString("utf-8");
163
+ const lines = carry.split("\n");
164
+ carry = lines.pop() ?? "";
165
+ for (const line of lines) onLine(line);
166
+ },
167
+ flush: () => {
168
+ if (carry.length === 0) return;
169
+ onLine(carry);
170
+ carry = "";
171
+ }
172
+ };
173
+ }
174
+ /**
175
+ * Run the build. Resolves when the image is tagged; rejects with the builder's
176
+ * own last words otherwise.
177
+ *
178
+ * The rejection carries real output rather than an exit code because the author
179
+ * is the one who has to act on it, and "exit 1" tells them nothing about which
180
+ * `RUN` failed.
181
+ */
182
+ async function buildJobImage(args) {
183
+ const { spec, cli, socketPath, authconfig, onLog, signal } = args;
184
+ if (signal?.aborted) throw signal.reason ?? /* @__PURE__ */ new Error("job image build aborted");
185
+ const auth = authconfig ? await withTempAuth(cli, authconfig) : void 0;
186
+ const argv = buildArgv({
187
+ cli,
188
+ spec,
189
+ ...socketPath ? { socketPath } : {}
190
+ });
191
+ try {
192
+ await new Promise((resolvePromise, reject) => {
193
+ const child = spawn(cli, argv, {
194
+ env: {
195
+ ...process.env,
196
+ ...auth?.env ?? {}
197
+ },
198
+ stdio: [
199
+ "ignore",
200
+ "pipe",
201
+ "pipe"
202
+ ]
203
+ });
204
+ const tail = [];
205
+ const record = (line) => {
206
+ onLog(line);
207
+ tail.push(line);
208
+ if (tail.length > FAILURE_TAIL_LINES) tail.shift();
209
+ };
210
+ const outSplitter = makeLineSplitter(record);
211
+ const errSplitter = makeLineSplitter(record);
212
+ child.stdout.on("data", outSplitter.write);
213
+ child.stderr.on("data", errSplitter.write);
214
+ const onAbort = () => child.kill("SIGKILL");
215
+ signal?.addEventListener("abort", onAbort, { once: true });
216
+ child.on("error", (err) => {
217
+ signal?.removeEventListener("abort", onAbort);
218
+ reject(/* @__PURE__ */ new Error(`failed to start '${cli} build': ${err.message}`));
219
+ });
220
+ child.on("close", (code) => {
221
+ signal?.removeEventListener("abort", onAbort);
222
+ outSplitter.flush();
223
+ errSplitter.flush();
224
+ if (code === 0) {
225
+ resolvePromise();
226
+ return;
227
+ }
228
+ const detail = tail.length > 0 ? `\n${tail.join("\n")}` : "";
229
+ reject(/* @__PURE__ */ new Error(`'${cli} build' failed for ${spec.dockerfilePath} (exit ${code ?? "signal"})${detail}`));
230
+ });
231
+ });
232
+ } finally {
233
+ await auth?.dispose();
234
+ }
235
+ }
236
+ var ContainerBuildCli, CLI_PREFERENCE, FAILURE_TAIL_LINES;
237
+ var init_build_engine = __esmMin((() => {
238
+ ContainerBuildCli = z.enum(["docker", "podman"]);
239
+ CLI_PREFERENCE = [ContainerBuildCli.enum.docker, ContainerBuildCli.enum.podman];
240
+ FAILURE_TAIL_LINES = 20;
241
+ }));
242
+ //#endregion
60
243
  //#region src/config.ts
244
+ init_build_engine();
61
245
  /** Execution mode for the agent's sandbox backend. Mirrors the runtime enum. */
62
246
  const ExecutionMode = z.enum([
63
247
  "container",
64
248
  "bare-metal",
65
249
  "firecracker"
66
250
  ]);
251
+ /** When the operator between-jobs reset command runs. */
252
+ const BetweenJobsRunOn = z.enum(["always", "on-failure"]);
253
+ const configSchema = z.object({
254
+ orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
255
+ agentId: z.string().optional(),
256
+ labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
257
+ properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
258
+ roles: z.string().optional().transform((s) => {
259
+ if (s === void 0) return void 0;
260
+ if (s === "") return [];
261
+ return s.split(",").filter(Boolean);
262
+ }).refine((roles) => {
263
+ if (roles === void 0) return true;
264
+ const validValues = [...KNOWN_ROLES, "all"];
265
+ return roles.every((r) => validValues.includes(r));
266
+ }, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
267
+ if (roles === void 0) return void 0;
268
+ if (roles.includes("all")) return void 0;
269
+ if (roles.length === 0) return [];
270
+ return roles.filter((r) => r !== "all");
271
+ }),
272
+ port: z.coerce.number().default(8080),
273
+ logLevel: z.enum([
274
+ "debug",
275
+ "info",
276
+ "warn",
277
+ "error"
278
+ ]).default("info"),
279
+ agentToken: z.string().optional(),
280
+ githubToken: z.string().optional(),
281
+ maxLogSizeBytes: z.coerce.number().default(10485760),
282
+ defaultStepTimeoutMs: z.coerce.number().default(18e5),
283
+ dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
284
+ jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
285
+ backpressureMode: z.enum(["pause", "drop"]).default("pause"),
286
+ agentPayloadDir: z.string().optional(),
287
+ agentCommand: z.string().optional(),
288
+ sandbox: z.string().default("false").transform((s) => s === "true"),
289
+ trustedEnv: z.string().default("false").transform((s) => s === "true"),
290
+ inPlace: z.string().default("false").transform((s) => s === "true"),
291
+ sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
292
+ sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
293
+ sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
294
+ sandboxUser: z.string().optional(),
295
+ sandboxPidsLimit: z.coerce.number().int().positive().default(512),
296
+ sandboxMemoryBytes: z.coerce.number().int().positive().default(2147483648),
297
+ sandboxNanoCpus: z.coerce.number().int().positive().default(2e9),
298
+ scalerManaged: z.string().optional().transform((s) => s === "1"),
299
+ jobImageAgent: z.string().optional().transform((s) => s === "1"),
300
+ runtimeImage: z.string().optional(),
301
+ runtimeNodeSource: z.string().optional(),
302
+ containerBuildCli: ContainerBuildCli.optional(),
303
+ scalerClaimCode: z.string().optional(),
304
+ scalerIdleTimeoutMs: z.coerce.number().default(5e3),
305
+ scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
306
+ executionMode: ExecutionMode.optional(),
307
+ otelExporterOtlpEndpoint: z.string().optional(),
308
+ concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
309
+ isOrchestratorHost: z.string().optional().transform((v) => v === "true"),
310
+ betweenJobsResetCommand: z.string().optional(),
311
+ betweenJobsResetTimeoutMs: z.coerce.number().int().positive().default(6e4),
312
+ betweenJobsResetRunOn: BetweenJobsRunOn.default("always"),
313
+ orphanCleanup: z.string().default("true").transform((s) => s !== "false"),
314
+ drainOnResetFailure: z.string().default("false").transform((s) => s === "true")
315
+ });
67
316
  /**
68
317
  * Env-var definition for the agent. Exported so the docs generator and the
69
318
  * deploy-stg pre-validator can inspect / re-parse without round-tripping
@@ -71,59 +320,7 @@ const ExecutionMode = z.enum([
71
320
  */
72
321
  const envDef = defineEnv({
73
322
  service: "agent",
74
- schema: z.object({
75
- orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
76
- agentId: z.string().optional(),
77
- labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
78
- properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
79
- roles: z.string().optional().transform((s) => {
80
- if (s === void 0) return void 0;
81
- if (s === "") return [];
82
- return s.split(",").filter(Boolean);
83
- }).refine((roles) => {
84
- if (roles === void 0) return true;
85
- const validValues = [...KNOWN_ROLES, "all"];
86
- return roles.every((r) => validValues.includes(r));
87
- }, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
88
- if (roles === void 0) return void 0;
89
- if (roles.includes("all")) return void 0;
90
- if (roles.length === 0) return [];
91
- return roles.filter((r) => r !== "all");
92
- }),
93
- port: z.coerce.number().default(8080),
94
- logLevel: z.enum([
95
- "debug",
96
- "info",
97
- "warn",
98
- "error"
99
- ]).default("info"),
100
- agentToken: z.string().optional(),
101
- githubToken: z.string().optional(),
102
- maxLogSizeBytes: z.coerce.number().default(10 * 1024 * 1024),
103
- defaultStepTimeoutMs: z.coerce.number().default(1800 * 1e3),
104
- dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
105
- jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
106
- backpressureMode: z.enum(["pause", "drop"]).default("pause"),
107
- agentPayloadDir: z.string().optional(),
108
- agentCommand: z.string().optional(),
109
- sandbox: z.string().default("false").transform((s) => s === "true"),
110
- trustedEnv: z.string().default("false").transform((s) => s === "true"),
111
- inPlace: z.string().default("false").transform((s) => s === "true"),
112
- sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
113
- sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
114
- sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
115
- sandboxUser: z.string().optional(),
116
- sandboxPidsLimit: z.coerce.number().int().positive().default(512),
117
- sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
118
- sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
119
- scalerManaged: z.string().optional().transform((s) => s === "1"),
120
- scalerIdleTimeoutMs: z.coerce.number().default(5e3),
121
- scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
122
- executionMode: ExecutionMode.optional(),
123
- otelExporterOtlpEndpoint: z.string().optional(),
124
- concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
125
- isOrchestratorHost: z.string().optional().transform((v) => v === "true")
126
- }),
323
+ schema: configSchema,
127
324
  envMap: {
128
325
  orchestratorUrl: "KICI_ORCHESTRATOR_URL",
129
326
  agentId: "KICI_AGENT_ID",
@@ -153,11 +350,21 @@ const envDef = defineEnv({
153
350
  sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
154
351
  sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
155
352
  scalerManaged: "KICI_SCALER_MANAGED",
353
+ jobImageAgent: "KICI_JOB_IMAGE_AGENT",
354
+ runtimeImage: "KICI_RUNTIME_IMAGE",
355
+ runtimeNodeSource: "KICI_RUNTIME_NODE_SOURCE",
356
+ containerBuildCli: "KICI_CONTAINER_BUILD_CLI",
357
+ scalerClaimCode: "KICI_SCALER_CLAIM_CODE",
156
358
  scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
157
359
  scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
158
360
  executionMode: "KICI_EXECUTION_MODE",
159
361
  otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
160
- concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS"
362
+ concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS",
363
+ betweenJobsResetCommand: "KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND",
364
+ betweenJobsResetTimeoutMs: "KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS",
365
+ betweenJobsResetRunOn: "KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON",
366
+ orphanCleanup: "KICI_AGENT_ORPHAN_CLEANUP",
367
+ drainOnResetFailure: "KICI_AGENT_DRAIN_ON_RESET_FAILURE"
161
368
  }
162
369
  });
163
370
  /**
@@ -189,10 +396,16 @@ const envDef = defineEnv({
189
396
  * - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
190
397
  * - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
191
398
  * - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
399
+ * - KICI_SCALER_CLAIM_CODE (optional single-use claim code the agent exchanges for its own ephemeral credentials before registering; ignored when KICI_AGENT_TOKEN is set)
192
400
  * - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
193
401
  * - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
194
402
  * - KICI_EXECUTION_MODE (optional, options: container | bare-metal | firecracker) — override the runner's mode-pick logic
195
403
  * - KICI_CONCURRENCY_WAIT_TIMEOUT_MS (default: 3_600_000) — workflow-runner timeout when long-polling for a slot-release follow-up `concurrency.ack`
404
+ * - KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND (optional) — host-reset command run between jobs on a reused agent (fail-open)
405
+ * - KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS (default: 60000) — reset command timeout
406
+ * - KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON (default: always, options: always | on-failure) — when the reset command runs
407
+ * - KICI_AGENT_ORPHAN_CLEANUP (default: true) — reap a finished job's leaked process tree (bare-metal); false = only signal the runner child
408
+ * - KICI_AGENT_DRAIN_ON_RESET_FAILURE (default: false) — drain the agent after repeated consecutive reset failures
196
409
  */
197
410
  function loadConfig() {
198
411
  const data = envDef.parse();
@@ -225,6 +438,68 @@ function agentClientConnectionOptions(config) {
225
438
  };
226
439
  }
227
440
  //#endregion
441
+ //#region src/execution/image-build/runtime-facts.ts
442
+ /**
443
+ * What this agent's own host can do with containers, discovered at startup.
444
+ *
445
+ * The orchestrator cannot answer this. An agent runs on its own machine, and
446
+ * whether that machine has a container runtime is the agent's fact — probing
447
+ * the orchestrator's filesystem answers a different question, and doing so once
448
+ * stranded container jobs that had been running fine, because the probe and the
449
+ * job ran in different places.
450
+ *
451
+ * Reported as `kici:runtime:*` labels, which the register-time scope gate
452
+ * accepts unchallenged as self-reported facts. Deliberately NOT
453
+ * `kici:capability:*` — that prefix grants a privilege and stays token-bound.
454
+ */
455
+ /** Sockets a container runtime answers on, in the order the agent would use them. */
456
+ function candidateSockets() {
457
+ const uid = typeof process.getuid === "function" ? process.getuid() : void 0;
458
+ const dockerHost = process.env.DOCKER_HOST?.replace(/^unix:\/\//, "");
459
+ return [
460
+ ...dockerHost ? [{
461
+ fact: RuntimeFact.enum.docker,
462
+ path: dockerHost
463
+ }] : [],
464
+ {
465
+ fact: RuntimeFact.enum.docker,
466
+ path: "/var/run/docker.sock"
467
+ },
468
+ {
469
+ fact: RuntimeFact.enum.podman,
470
+ path: "/run/podman/podman.sock"
471
+ },
472
+ ...uid !== void 0 ? [{
473
+ fact: RuntimeFact.enum.podman,
474
+ path: `/run/user/${uid}/podman/podman.sock`
475
+ }] : []
476
+ ];
477
+ }
478
+ /** Is `bin` executable somewhere on PATH? */
479
+ function onPath(bin) {
480
+ return (process.env.PATH ?? "").split(delimiter).filter(Boolean).some((dir) => existsSync(join(dir, bin)));
481
+ }
482
+ /**
483
+ * Discover this host's runtime facts.
484
+ *
485
+ * Presence of the socket FILE, not a handshake: registration must not block on
486
+ * a daemon that is slow or wedged, and a job that reaches a broken runtime
487
+ * still fails with the runtime's own error. The label answers "is there a
488
+ * runtime here at all", which is the routing question.
489
+ */
490
+ function detectRuntimeFacts(deps = {}) {
491
+ const pathExists = deps.pathExists ?? existsSync;
492
+ const hasBinary = deps.binaryOnPath ?? onPath;
493
+ const facts = /* @__PURE__ */ new Set();
494
+ for (const { fact, path } of candidateSockets()) if (pathExists(path)) facts.add(fact);
495
+ if (hasBinary("docker") || hasBinary("podman")) facts.add(RuntimeFact.enum["container-build"]);
496
+ return [...facts];
497
+ }
498
+ /** The `kici:runtime:*` labels this host should register with. */
499
+ function runtimeFactLabels(deps = {}) {
500
+ return detectRuntimeFacts(deps).map(runtimeLabel);
501
+ }
502
+ //#endregion
228
503
  //#region src/diagnostics/mini-bundle.ts
229
504
  /**
230
505
  * Agent fleet mini-bundle assembler.
@@ -325,7 +600,7 @@ var LogBuffer = class extends RingBuffer {
325
600
  };
326
601
  //#endregion
327
602
  //#region src/ws/orchestrator-client.ts
328
- const logger$13 = createLogger({ prefix: "orchestrator-client" });
603
+ const logger$14 = createLogger({ prefix: "orchestrator-client" });
329
604
  /**
330
605
  * How long to wait for `artifacts.upload.complete.ack` before failing the step.
331
606
  *
@@ -381,6 +656,8 @@ var OrchestratorClient = class OrchestratorClient {
381
656
  pendingEventEmitRequests = /* @__PURE__ */ new Map();
382
657
  /** Pending agent.api.request calls awaiting orchestrator response. */
383
658
  pendingApiRequests = /* @__PURE__ */ new Map();
659
+ /** Pending scaler.claim-credentials requests awaiting orchestrator response. */
660
+ pendingClaimCredentialsRequests = /* @__PURE__ */ new Map();
384
661
  /** Pending user-cache restore/save requests awaiting orchestrator response. */
385
662
  pendingUserCacheRequests = /* @__PURE__ */ new Map();
386
663
  /** Pending user-artifact upload/download requests awaiting orchestrator response. */
@@ -419,6 +696,7 @@ var OrchestratorClient = class OrchestratorClient {
419
696
  onJobDispatch;
420
697
  onJobCancel;
421
698
  token;
699
+ scalerClaimCode;
422
700
  heartbeatIntervalMs;
423
701
  maxReconnectDelayMs;
424
702
  getInFlightJobs;
@@ -437,8 +715,23 @@ var OrchestratorClient = class OrchestratorClient {
437
715
  * `pendingDispatch` is set by the orchestrator when a queued job has been
438
716
  * pre-bound to this agent and the dispatch.job message is in flight. The
439
717
  * agent must defer arming the short scaler-idle timer in this case.
718
+ *
719
+ * `warmPool` is set when this agent was pre-spawned to wait for work rather
720
+ * than for a specific queued job. The agent must arm no idle timer at all:
721
+ * the orchestrator's warm-pool reaper owns its lifetime.
440
722
  */
441
723
  onRegistered = null;
724
+ /**
725
+ * Callback invoked when a self-bootstrap claim exchange fails permanently
726
+ * (invalid / expired / already-consumed claim code, or the exchange itself
727
+ * errored or timed out). Only ever fires in self-bootstrap mode
728
+ * (`scalerClaimCode` set, no static token) — a statically-tokened agent never
729
+ * runs the claim exchange. A one-shot self-bootstrap agent cannot make any
730
+ * further progress after this, so server.ts wires it to a non-zero graceful
731
+ * shutdown; without it the health HTTP server keeps the process alive and a
732
+ * GitHub Actions run hangs until the job timeout.
733
+ */
734
+ onClaimFailedPermanently = null;
442
735
  constructor(options) {
443
736
  this.url = options.url;
444
737
  this.agentId = options.agentId;
@@ -447,6 +740,7 @@ var OrchestratorClient = class OrchestratorClient {
447
740
  this.onJobDispatch = options.onJobDispatch;
448
741
  this.onJobCancel = options.onJobCancel;
449
742
  this.token = options.token;
743
+ this.scalerClaimCode = options.scalerClaimCode;
450
744
  this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? 3e4;
451
745
  this.maxReconnectDelayMs = options.maxReconnectDelayMs ?? 6e4;
452
746
  this.getInFlightJobs = options.getInFlightJobs;
@@ -485,7 +779,7 @@ var OrchestratorClient = class OrchestratorClient {
485
779
  */
486
780
  connect() {
487
781
  if (this._state !== "disconnected") {
488
- logger$13.warn("connect() called while not disconnected", { state: this._state });
782
+ logger$14.warn("connect() called while not disconnected", { state: this._state });
489
783
  return;
490
784
  }
491
785
  this.intentionalDisconnect = false;
@@ -666,6 +960,39 @@ var OrchestratorClient = class OrchestratorClient {
666
960
  });
667
961
  }
668
962
  /**
963
+ * Send a scaler.claim-credentials WS message and await the response.
964
+ *
965
+ * Used by a provisioning workflow (via `ctx.kici.scaler.claimAgentCredentials`)
966
+ * to exchange a single-use claim code — delivered on a `kici.scaler.scale-up`
967
+ * event — for freshly minted ephemeral agent credentials. The token rides the
968
+ * response only; it is never logged. Times out after 15 seconds (matching the
969
+ * generic agent-API request timeout).
970
+ */
971
+ async sendClaimCredentials(claimCode) {
972
+ const requestId = randomUUID();
973
+ return new Promise((resolve, reject) => {
974
+ const timer = setTimeout(() => {
975
+ this.pendingClaimCredentialsRequests.delete(requestId);
976
+ reject(/* @__PURE__ */ new Error("scaler.claim-credentials timed out"));
977
+ }, 15e3);
978
+ this.pendingClaimCredentialsRequests.set(requestId, {
979
+ resolve: (response) => {
980
+ clearTimeout(timer);
981
+ resolve(response);
982
+ },
983
+ reject: (err) => {
984
+ clearTimeout(timer);
985
+ reject(err);
986
+ }
987
+ });
988
+ this.sendDirect({
989
+ type: "scaler.claim-credentials",
990
+ requestId,
991
+ claimCode
992
+ });
993
+ });
994
+ }
995
+ /**
669
996
  * Build this agent's fleet mini-bundle and stream it back to the orchestrator
670
997
  * as ordered fleet.bundle.chunk frames (the WS frame cap forbids one frame).
671
998
  * On failure, sends a single fleet.bundle.error. Public for unit testing.
@@ -1049,42 +1376,24 @@ var OrchestratorClient = class OrchestratorClient {
1049
1376
  }
1050
1377
  });
1051
1378
  } catch (err) {
1052
- logger$13.error("Failed to create WebSocket", { error: toErrorMessage(err) });
1379
+ logger$14.error("Failed to create WebSocket", { error: toErrorMessage(err) });
1053
1380
  this._state = "disconnected";
1054
1381
  this.scheduleReconnect();
1055
1382
  return;
1056
1383
  }
1057
1384
  this.ws.on("open", () => {
1058
- if (this.token) {
1059
- this._state = "authenticating";
1060
- logger$13.info("Connected to orchestrator, sending auth.request", {
1061
- url: this.url,
1062
- agentId: this.agentId
1063
- });
1064
- this.ws.send(JSON.stringify({
1065
- type: "auth.request",
1066
- token: this.token,
1067
- protocolVersion: PROTOCOL_VERSION
1068
- }));
1069
- } else {
1070
- this._state = "registering";
1071
- logger$13.info("Connected to orchestrator, sending agent.register (no token)", {
1072
- url: this.url,
1073
- agentId: this.agentId
1074
- });
1075
- this.sendAgentRegister();
1076
- }
1385
+ this.onWsOpen();
1077
1386
  });
1078
1387
  this.ws.on("message", (data) => {
1079
1388
  this.handleMessage(data);
1080
1389
  });
1081
1390
  this.ws.on("close", (code, reason) => {
1082
- logger$13.info("Orchestrator connection closed", {
1391
+ logger$14.info("Orchestrator connection closed", {
1083
1392
  code,
1084
1393
  reason: reason.toString()
1085
1394
  });
1086
1395
  if (code === WS_CLOSE_AGENT_AUTH_FAILED) {
1087
- logger$13.error("Orchestrator closed with auth-failed code -- token is invalid or revoked. NOT retrying.", {
1396
+ logger$14.error("Orchestrator closed with auth-failed code -- token is invalid or revoked. NOT retrying.", {
1088
1397
  code,
1089
1398
  reason: reason.toString()
1090
1399
  });
@@ -1101,6 +1410,8 @@ var OrchestratorClient = class OrchestratorClient {
1101
1410
  this.pendingEventEmitRequests.clear();
1102
1411
  for (const [_id, pending] of this.pendingApiRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
1103
1412
  this.pendingApiRequests.clear();
1413
+ for (const [_id, pending] of this.pendingClaimCredentialsRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
1414
+ this.pendingClaimCredentialsRequests.clear();
1104
1415
  for (const [_id, pending] of this.pendingUserCacheRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
1105
1416
  this.pendingUserCacheRequests.clear();
1106
1417
  for (const [id, pending] of this.pendingUserArtifactRequests) {
@@ -1121,11 +1432,102 @@ var OrchestratorClient = class OrchestratorClient {
1121
1432
  if (!this.intentionalDisconnect) this.scheduleReconnect();
1122
1433
  });
1123
1434
  this.ws.on("error", (err) => {
1124
- logger$13.error(`Orchestrator WebSocket error: ${err.message}`);
1435
+ logger$14.error(`Orchestrator WebSocket error: ${err.message}`);
1125
1436
  if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
1126
1437
  });
1127
1438
  }
1128
1439
  /**
1440
+ * WS-open handshake driver.
1441
+ *
1442
+ * Self-bootstrap runs first: when a `scalerClaimCode` is set and no static
1443
+ * token is present, the client exchanges the single-use code for its own
1444
+ * ephemeral credentials over `scaler.claim-credentials` — before any auth or
1445
+ * register — and adopts the minted token, agentId, and labels. A rejected or
1446
+ * expired code fails permanently (no reconnect), so a one-shot provisioning
1447
+ * run exits visibly rather than looping.
1448
+ *
1449
+ * Then the normal handshake: with a token (static or just-minted) send
1450
+ * `auth.request` and register after `auth.success`; without one, register
1451
+ * directly (unauthenticated mode).
1452
+ */
1453
+ async onWsOpen() {
1454
+ if (this.scalerClaimCode && !this.token) {
1455
+ if (!await this.performClaimExchange(this.scalerClaimCode)) return;
1456
+ }
1457
+ if (this.token) {
1458
+ this._state = "authenticating";
1459
+ logger$14.info("Connected to orchestrator, sending auth.request", {
1460
+ url: this.url,
1461
+ agentId: this.agentId
1462
+ });
1463
+ this.ws.send(JSON.stringify({
1464
+ type: "auth.request",
1465
+ token: this.token,
1466
+ protocolVersion: PROTOCOL_VERSION
1467
+ }));
1468
+ } else {
1469
+ this._state = "registering";
1470
+ logger$14.info("Connected to orchestrator, sending agent.register (no token)", {
1471
+ url: this.url,
1472
+ agentId: this.agentId
1473
+ });
1474
+ this.sendAgentRegister();
1475
+ }
1476
+ }
1477
+ /**
1478
+ * Exchange the single-use claim code for ephemeral credentials, adopting the
1479
+ * minted token / agentId / labels on success. Returns true when credentials
1480
+ * were adopted; false when the claim failed permanently (the connection has
1481
+ * already been torn down with no reconnect).
1482
+ */
1483
+ async performClaimExchange(claimCode) {
1484
+ logger$14.info("Self-bootstrap: exchanging claim code for ephemeral credentials", {
1485
+ url: this.url,
1486
+ agentId: this.agentId
1487
+ });
1488
+ let response;
1489
+ try {
1490
+ response = await this.sendClaimCredentials(claimCode);
1491
+ } catch (err) {
1492
+ this.failClaimPermanently(`claim exchange failed: ${toErrorMessage(err)}`);
1493
+ return false;
1494
+ }
1495
+ if (response.error || !response.credentials) {
1496
+ this.failClaimPermanently(response.error ?? "orchestrator returned no credentials");
1497
+ return false;
1498
+ }
1499
+ const { agentToken, agentId, labels } = response.credentials;
1500
+ this.token = agentToken;
1501
+ this.agentId = agentId;
1502
+ this.labels = this.mergeClaimedLabels(labels);
1503
+ logger$14.info("Self-bootstrap: adopted ephemeral credentials", { agentId });
1504
+ return true;
1505
+ }
1506
+ /**
1507
+ * Merge the claim-minted labels into the agent's own label set, deduped. The
1508
+ * self-reported `kici:os:` / `kici:arch:` / `kici:host:` facts are derived at
1509
+ * register time (`sendAgentRegister`), so they survive regardless.
1510
+ */
1511
+ mergeClaimedLabels(claimed) {
1512
+ return [.../* @__PURE__ */ new Set([...this.labels, ...claimed])];
1513
+ }
1514
+ /**
1515
+ * Fail a self-bootstrap claim permanently, using the same no-reconnect
1516
+ * mechanism as `auth.failure`: mark the auth context dead, suppress reconnect,
1517
+ * and close the socket. A one-shot provisioning run then exits visibly.
1518
+ */
1519
+ failClaimPermanently(reason) {
1520
+ logger$14.error("Self-bootstrap claim exchange failed -- claim code is invalid or expired. NOT retrying.", { reason });
1521
+ this.authFailed = true;
1522
+ this.intentionalDisconnect = true;
1523
+ if (this.ws) {
1524
+ this.ws.close(1e3, "Claim failed");
1525
+ this.ws = null;
1526
+ }
1527
+ this._state = "disconnected";
1528
+ this.onClaimFailedPermanently?.(reason);
1529
+ }
1530
+ /**
1129
1531
  * Resolve a pending user-artifact request from an `artifacts.upload.response`
1130
1532
  * / `artifacts.download.response` WS message, mapping the wire fields onto the
1131
1533
  * IPC response shape. No-op when no request is pending for the requestId.
@@ -1229,7 +1631,7 @@ var OrchestratorClient = class OrchestratorClient {
1229
1631
  this.pendingUserArtifactRequests.set(record.messageId, record.pending);
1230
1632
  this.resendableCompletes.set(record.messageId, record);
1231
1633
  this.sendDirect(record.frame);
1232
- logger$13.info("Re-sent artifact upload-complete after reconnect", {
1634
+ logger$14.info("Re-sent artifact upload-complete after reconnect", {
1233
1635
  messageId: record.messageId,
1234
1636
  attempt: record.attempts
1235
1637
  });
@@ -1251,12 +1653,55 @@ var OrchestratorClient = class OrchestratorClient {
1251
1653
  }
1252
1654
  return false;
1253
1655
  }
1656
+ /**
1657
+ * Resolve a pending scaler credential-claim request from its response
1658
+ * message. Returns true when it handled the message. Extracted from
1659
+ * handleMessage so that dispatcher stays under the function-length limit.
1660
+ */
1661
+ handleClaimCredentialsResponse(rawMsg) {
1662
+ if (rawMsg.type !== "scaler.claim-credentials.response") return false;
1663
+ const pending = this.pendingClaimCredentialsRequests.get(rawMsg.requestId);
1664
+ if (pending) {
1665
+ this.pendingClaimCredentialsRequests.delete(rawMsg.requestId);
1666
+ pending.resolve({
1667
+ credentials: rawMsg.credentials,
1668
+ error: rawMsg.error
1669
+ });
1670
+ }
1671
+ return true;
1672
+ }
1673
+ /**
1674
+ * Complete registration: adopt the orchestrator's negotiated capabilities,
1675
+ * enter the `registered` state, flush everything parked while disconnected,
1676
+ * and hand the ack's lifetime flags to `onRegistered`.
1677
+ */
1678
+ handleRegisterAck(msg) {
1679
+ logger$14.info("Registration acknowledged by orchestrator", {
1680
+ agentId: msg.agentId,
1681
+ labels: msg.labels,
1682
+ scalerManaged: msg.scalerManaged,
1683
+ pendingDispatch: msg.pendingDispatch ?? false,
1684
+ warmPool: msg.warmPool ?? false
1685
+ });
1686
+ this.orchCapabilities = msg.capabilities;
1687
+ this._state = "registered";
1688
+ this.reconnectAttempts = 0;
1689
+ this.startHeartbeat();
1690
+ this.flushBuffer();
1691
+ this.resendHeldCompletes();
1692
+ this.onRegistered?.({
1693
+ pendingDispatch: msg.pendingDispatch ?? false,
1694
+ warmPool: msg.warmPool ?? false
1695
+ });
1696
+ if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
1697
+ this.sendConfigAck(msg.agentId);
1698
+ }
1254
1699
  handleMessage(data) {
1255
1700
  let raw;
1256
1701
  try {
1257
1702
  raw = JSON.parse(data.toString());
1258
1703
  } catch {
1259
- logger$13.warn("Malformed JSON received from orchestrator");
1704
+ logger$14.warn("Malformed JSON received from orchestrator");
1260
1705
  return;
1261
1706
  }
1262
1707
  const rawMsg = raw;
@@ -1281,6 +1726,7 @@ var OrchestratorClient = class OrchestratorClient {
1281
1726
  }
1282
1727
  return;
1283
1728
  }
1729
+ if (this.handleClaimCredentialsResponse(rawMsg)) return;
1284
1730
  if (rawMsg.type === "agent.api.response") {
1285
1731
  const pending = this.pendingApiRequests.get(rawMsg.requestId);
1286
1732
  if (pending) {
@@ -1315,13 +1761,13 @@ var OrchestratorClient = class OrchestratorClient {
1315
1761
  switch (msg.type) {
1316
1762
  case "auth.success":
1317
1763
  if (this._state === "authenticating") {
1318
- logger$13.info("Authentication successful, sending agent.register", { connectionId: msg.connectionId });
1764
+ logger$14.info("Authentication successful, sending agent.register", { connectionId: msg.connectionId });
1319
1765
  this._state = "registering";
1320
1766
  this.sendAgentRegister();
1321
1767
  }
1322
1768
  break;
1323
1769
  case "auth.failure":
1324
- logger$13.error("Authentication FAILED -- token is invalid or expired. NOT retrying.", { reason: msg.reason });
1770
+ logger$14.error("Authentication FAILED -- token is invalid or expired. NOT retrying.", { reason: msg.reason });
1325
1771
  this.authFailed = true;
1326
1772
  this.intentionalDisconnect = true;
1327
1773
  if (this.ws) {
@@ -1331,31 +1777,17 @@ var OrchestratorClient = class OrchestratorClient {
1331
1777
  this._state = "disconnected";
1332
1778
  break;
1333
1779
  case "register.ack":
1334
- logger$13.info("Registration acknowledged by orchestrator", {
1335
- agentId: msg.agentId,
1336
- labels: msg.labels,
1337
- scalerManaged: msg.scalerManaged,
1338
- pendingDispatch: msg.pendingDispatch ?? false
1339
- });
1340
- this.orchCapabilities = msg.capabilities;
1341
- this._state = "registered";
1342
- this.reconnectAttempts = 0;
1343
- this.startHeartbeat();
1344
- this.flushBuffer();
1345
- this.resendHeldCompletes();
1346
- this.onRegistered?.({ pendingDispatch: msg.pendingDispatch ?? false });
1347
- if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
1348
- this.sendConfigAck(msg.agentId);
1780
+ this.handleRegisterAck(msg);
1349
1781
  break;
1350
1782
  case "job.dispatch":
1351
- logger$13.info("Job dispatch received", {
1783
+ logger$14.info("Job dispatch received", {
1352
1784
  runId: msg.runId,
1353
1785
  jobId: msg.jobId
1354
1786
  });
1355
1787
  this.onJobDispatch(msg);
1356
1788
  break;
1357
1789
  case "job.cancel":
1358
- logger$13.info("Job cancel received", {
1790
+ logger$14.info("Job cancel received", {
1359
1791
  runId: msg.runId,
1360
1792
  jobId: msg.jobId,
1361
1793
  reason: msg.reason
@@ -1363,7 +1795,7 @@ var OrchestratorClient = class OrchestratorClient {
1363
1795
  this.onJobCancel(msg);
1364
1796
  break;
1365
1797
  case "job.concurrency.ack": {
1366
- logger$13.info("Concurrency ack received", {
1798
+ logger$14.info("Concurrency ack received", {
1367
1799
  requestId: msg.requestId,
1368
1800
  action: msg.action
1369
1801
  });
@@ -1378,7 +1810,7 @@ var OrchestratorClient = class OrchestratorClient {
1378
1810
  break;
1379
1811
  }
1380
1812
  case "step.approval-resolved": {
1381
- logger$13.info("Step approval resolved", {
1813
+ logger$14.info("Step approval resolved", {
1382
1814
  requestId: msg.requestId,
1383
1815
  runId: msg.runId,
1384
1816
  jobId: msg.jobId,
@@ -1398,17 +1830,16 @@ var OrchestratorClient = class OrchestratorClient {
1398
1830
  break;
1399
1831
  }
1400
1832
  case "fleet.logs.request":
1401
- logger$13.info("Fleet log collection requested", {
1833
+ logger$14.info("Fleet log collection requested", {
1402
1834
  requestId: msg.requestId,
1403
1835
  logWindowHours: msg.logWindowHours
1404
1836
  });
1405
1837
  this.streamFleetBundle(msg);
1406
- break;
1407
1838
  }
1408
1839
  return;
1409
1840
  }
1410
1841
  if (heartbeatSchema.safeParse(raw).success) return;
1411
- logger$13.warn("Invalid message from orchestrator", { errors: parsed.error.issues });
1842
+ logger$14.warn("Invalid message from orchestrator", { errors: parsed.error.issues });
1412
1843
  }
1413
1844
  flushBuffer() {
1414
1845
  const events = this.eventBuffer.flush();
@@ -1422,11 +1853,11 @@ var OrchestratorClient = class OrchestratorClient {
1422
1853
  }
1423
1854
  this.disconnectedAt = null;
1424
1855
  if (events.length > 0) {
1425
- logger$13.info("Flushing event buffer", { count: events.length });
1856
+ logger$14.info("Flushing event buffer", { count: events.length });
1426
1857
  for (const msg of events) this.sendDirect(msg);
1427
1858
  }
1428
1859
  if (logLines.length > 0) {
1429
- logger$13.info("Flushing log buffer", { count: logLines.length });
1860
+ logger$14.info("Flushing log buffer", { count: logLines.length });
1430
1861
  for (let i = 0; i < logLines.length; i += OrchestratorClient.LOG_BATCH_SIZE) {
1431
1862
  const batch = logLines.slice(i, i + OrchestratorClient.LOG_BATCH_SIZE);
1432
1863
  this.sendAgentLogMessage(batch);
@@ -1479,14 +1910,14 @@ var OrchestratorClient = class OrchestratorClient {
1479
1910
  */
1480
1911
  blockMmdsAccess() {
1481
1912
  if (process.getuid?.() !== 0) {
1482
- logger$13.info("MMDS iptables block skipped (non-root) — network isolation handled by orchestrator");
1913
+ logger$14.info("MMDS iptables block skipped (non-root) — network isolation handled by orchestrator");
1483
1914
  return;
1484
1915
  }
1485
1916
  try {
1486
1917
  execSync("iptables -A OUTPUT -d 169.254.169.254 -j DROP", { timeout: 5e3 });
1487
- logger$13.info("MMDS access blocked via iptables");
1918
+ logger$14.info("MMDS access blocked via iptables");
1488
1919
  } catch (err) {
1489
- logger$13.warn("Failed to block MMDS access via iptables", { error: toErrorMessage(err) });
1920
+ logger$14.warn("Failed to block MMDS access via iptables", { error: toErrorMessage(err) });
1490
1921
  }
1491
1922
  }
1492
1923
  /**
@@ -1500,7 +1931,7 @@ var OrchestratorClient = class OrchestratorClient {
1500
1931
  messageId: `config-ack-${agentId}-${Date.now()}`,
1501
1932
  agentId
1502
1933
  }));
1503
- logger$13.info("Config ACK sent to orchestrator", { agentId });
1934
+ logger$14.info("Config ACK sent to orchestrator", { agentId });
1504
1935
  }
1505
1936
  }
1506
1937
  startHeartbeat() {
@@ -1527,6 +1958,7 @@ var OrchestratorClient = class OrchestratorClient {
1527
1958
  const autoLabels = [
1528
1959
  ...deriveOsArchLabels(os.platform(), os.arch()),
1529
1960
  hostLabel(os.hostname()),
1961
+ ...runtimeFactLabels(),
1530
1962
  ...resolveRoleLabels(this.roles)
1531
1963
  ];
1532
1964
  const allLabels = mergeAutoLabels(this.labels, autoLabels);
@@ -1540,7 +1972,7 @@ var OrchestratorClient = class OrchestratorClient {
1540
1972
  hostname: os.hostname(),
1541
1973
  osRelease: os.release(),
1542
1974
  osVersion: os.version(),
1543
- totalMemoryMb: Math.round(os.totalmem() / (1024 * 1024)),
1975
+ totalMemoryMb: Math.round(os.totalmem() / 1048576),
1544
1976
  cpuCount: os.cpus().length,
1545
1977
  nodeVersion: process.versions.node,
1546
1978
  ...(() => {
@@ -1566,12 +1998,12 @@ var OrchestratorClient = class OrchestratorClient {
1566
1998
  scheduleReconnect() {
1567
1999
  this.cancelReconnect();
1568
2000
  if (this.authFailed) {
1569
- logger$13.error("Not reconnecting: authentication permanently failed");
2001
+ logger$14.error("Not reconnecting: authentication permanently failed");
1570
2002
  return;
1571
2003
  }
1572
2004
  const delay = this.getReconnectDelay();
1573
2005
  this.reconnectAttempts++;
1574
- logger$13.info("Scheduling reconnect", {
2006
+ logger$14.info("Scheduling reconnect", {
1575
2007
  attempt: this.reconnectAttempts,
1576
2008
  delayMs: Math.round(delay)
1577
2009
  });
@@ -1656,14 +2088,14 @@ var init_console_capture = __esmMin((() => {
1656
2088
  init_console_capture();
1657
2089
  function safe(name, fallback = "unknown") {
1658
2090
  switch (name) {
1659
- case "version": return "0.5.0";
1660
- case "buildCommit": return "cb51c7d1e";
1661
- case "sdkVersion": return "0.5.0";
1662
- case "sdkBundleHash": return "5b85e7cffa4a08e39329ff80448a2744af8e5b5f9840ec3801fad45ed11a3de9";
1663
- case "sharedVersion": return "0.5.0";
1664
- case "sharedBundleHash": return "b6b6d1818d1fe648150422daf073dafc18b037404cacf82e0ec03b4a2863e4c4";
1665
- case "engineVersion": return "0.5.0";
1666
- case "engineBundleHash": return "35d174ce6f4748abffcad194b335b508cade5d1c8af16d9cd3bdcb744f38717e";
2091
+ case "version": return "0.6.0";
2092
+ case "buildCommit": return "e4a936029";
2093
+ case "sdkVersion": return "0.6.0";
2094
+ case "sdkBundleHash": return "22faf0da45de7c243ce87f80fd33ee51b1df52809fde457b16bb821678f65eb3";
2095
+ case "sharedVersion": return "0.6.0";
2096
+ case "sharedBundleHash": return "a79be949815735b9e36eecc716f7eb07aa36c7bffba62e798e7e19f31a474eff";
2097
+ case "engineVersion": return "0.6.0";
2098
+ case "engineBundleHash": return "e49533b19f122155b3a78d95d105f97da259c2a45dee579578324b0702c91708";
1667
2099
  default: return fallback;
1668
2100
  }
1669
2101
  }
@@ -1892,7 +2324,7 @@ init_npm_resolver();
1892
2324
  * even structurally matches the allocator pattern (`ledger` is a 6-char label
1893
2325
  * suffix), which a regex alone cannot distinguish.
1894
2326
  */
1895
- const AGENT_TMP_GC_MAX_AGE_MS = 1440 * 60 * 1e3;
2327
+ const AGENT_TMP_GC_MAX_AGE_MS = 864e5;
1896
2328
  /**
1897
2329
  * Bare `kici-<6 chars>` workdirs and labeled `kici-<label>-<6 chars>`
1898
2330
  * allocator dirs. The optional label group makes both families eligible.
@@ -1948,7 +2380,7 @@ async function gcStaleAgentTmpDirs(base = kiciTmpBase()) {
1948
2380
  * spawn fails and the caller clears the orchestrator's reboot-pending flag and
1949
2381
  * surfaces the error — the deadline sweep is the backstop.
1950
2382
  */
1951
- const logger$12 = createLogger({ prefix: "reboot" });
2383
+ const logger$13 = createLogger({ prefix: "reboot" });
1952
2384
  /** The OS reboot primitive for a Node platform string. */
1953
2385
  function rebootCommandFor(platform) {
1954
2386
  switch (platform) {
@@ -1986,7 +2418,7 @@ function issueReboot(platform = process.platform) {
1986
2418
  stdio: "ignore"
1987
2419
  });
1988
2420
  child.on("error", (err) => {
1989
- logger$12.error("Reboot command failed to spawn", {
2421
+ logger$13.error("Reboot command failed to spawn", {
1990
2422
  cmd,
1991
2423
  args,
1992
2424
  error: String(err)
@@ -1994,14 +2426,14 @@ function issueReboot(platform = process.platform) {
1994
2426
  reject(err);
1995
2427
  });
1996
2428
  child.on("exit", (code) => {
1997
- if (code !== 0 && code !== null) logger$12.warn("Reboot command exited non-zero (privilege denied?)", {
2429
+ if (code !== 0 && code !== null) logger$13.warn("Reboot command exited non-zero (privilege denied?)", {
1998
2430
  cmd,
1999
2431
  args,
2000
2432
  code
2001
2433
  });
2002
2434
  });
2003
2435
  child.unref();
2004
- logger$12.info("Issued host reboot", {
2436
+ logger$13.info("Issued host reboot", {
2005
2437
  cmd,
2006
2438
  args
2007
2439
  });
@@ -2012,8 +2444,18 @@ function issueReboot(platform = process.platform) {
2012
2444
  });
2013
2445
  }
2014
2446
  //#endregion
2447
+ //#region src/idle-shutdown.ts
2448
+ function decideIdleShutdown(input) {
2449
+ if (!input.scalerManaged || input.activeJobs > 0) return "none";
2450
+ if (input.warmPool) return "warm";
2451
+ if (input.pendingDispatch) return "pending-dispatch";
2452
+ return "idle";
2453
+ }
2454
+ //#endregion
2015
2455
  //#region src/metrics/prometheus.ts
2016
2456
  var prometheus_exports = /* @__PURE__ */ __exportAll({
2457
+ betweenJobsResetDurationSeconds: () => betweenJobsResetDurationSeconds,
2458
+ betweenJobsResetTotal: () => betweenJobsResetTotal,
2017
2459
  cloneDurationSeconds: () => cloneDurationSeconds,
2018
2460
  connectionStatus: () => connectionStatus$1,
2019
2461
  jobsActive: () => jobsActive$1,
@@ -2022,10 +2464,12 @@ var prometheus_exports = /* @__PURE__ */ __exportAll({
2022
2464
  logBackpressureEventsTotal: () => logBackpressureEventsTotal,
2023
2465
  logBytesTotal: () => logBytesTotal,
2024
2466
  logLinesDroppedTotal: () => logLinesDroppedTotal,
2467
+ orphanCleanupTotal: () => orphanCleanupTotal,
2468
+ orphansReapedTotal: () => orphansReapedTotal,
2025
2469
  stepDurationSeconds: () => stepDurationSeconds,
2026
2470
  stepsTotal: () => stepsTotal
2027
2471
  });
2028
- var meter, jobsTotal$1, jobsActive$1, stepsTotal, stepDurationSeconds, cloneDurationSeconds, logBytesTotal, logBackpressureEventsTotal, logLinesDroppedTotal, logBackpressureActive, connectionStatus$1;
2472
+ var meter, jobsTotal$1, jobsActive$1, stepsTotal, stepDurationSeconds, cloneDurationSeconds, logBytesTotal, logBackpressureEventsTotal, logLinesDroppedTotal, logBackpressureActive, connectionStatus$1, betweenJobsResetTotal, betweenJobsResetDurationSeconds, orphansReapedTotal, orphanCleanupTotal;
2029
2473
  var init_prometheus = __esmMin((() => {
2030
2474
  meter = createMeter("kici-agent");
2031
2475
  jobsTotal$1 = meter.createCounter("kici_agent_jobs_total", { description: "Total number of completed jobs" });
@@ -2060,6 +2504,20 @@ var init_prometheus = __esmMin((() => {
2060
2504
  logLinesDroppedTotal = meter.createCounter("kici_agent_log_lines_dropped_total", { description: "Total log lines dropped due to backpressure" });
2061
2505
  logBackpressureActive = meter.createUpDownCounter("kici_agent_log_backpressure_active", { description: "Current log-streamer backpressure state (0=normal, 1=active)" });
2062
2506
  connectionStatus$1 = meter.createUpDownCounter("kici_agent_connection_status", { description: "Orchestrator WebSocket connection status (0=disconnected, 1=connected)" });
2507
+ betweenJobsResetTotal = meter.createCounter("kici_agent_between_jobs_reset_total", { description: "Total operator between-jobs reset command invocations" });
2508
+ betweenJobsResetDurationSeconds = meter.createHistogram("kici_agent_between_jobs_reset_duration_seconds", {
2509
+ description: "Between-jobs reset command duration in seconds",
2510
+ advice: { explicitBucketBoundaries: [
2511
+ .1,
2512
+ 1,
2513
+ 5,
2514
+ 30,
2515
+ 60,
2516
+ 300
2517
+ ] }
2518
+ });
2519
+ orphansReapedTotal = meter.createCounter("kici_agent_orphans_reaped_total", { description: "Total processes killed by the between-jobs process-group reap" });
2520
+ orphanCleanupTotal = meter.createCounter("kici_agent_orphan_cleanup_total", { description: "Total out-of-band declared-cleanup re-runs" });
2063
2521
  }));
2064
2522
  //#endregion
2065
2523
  //#region src/checkout/ssh-auth.ts
@@ -2083,9 +2541,11 @@ async function setupSshAuth(opts) {
2083
2541
  if (opts.hostKeyPolicy === "pinned" && !opts.knownHosts) throw new Error("pinned hostKeyPolicy requires knownHosts content");
2084
2542
  const { path: tempDir, cleanup } = await makeTempDir("ssh");
2085
2543
  const keyPath = join(tempDir, "id");
2086
- await writeFile(keyPath, opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`, { mode: 384 });
2544
+ const pem = opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`;
2545
+ await writeFile(keyPath, pem, { mode: 384 });
2087
2546
  const knownHostsPath = join(tempDir, "known_hosts");
2088
- await writeFile(knownHostsPath, opts.hostKeyPolicy === "pinned" ? opts.knownHosts : "", { mode: 384 });
2547
+ const knownHostsBody = opts.hostKeyPolicy === "pinned" ? opts.knownHosts : "";
2548
+ await writeFile(knownHostsPath, knownHostsBody, { mode: 384 });
2089
2549
  const parts = [
2090
2550
  "ssh",
2091
2551
  "-i",
@@ -2117,6 +2577,37 @@ var init_ssh_auth = __esmMin((() => {}));
2117
2577
  //#endregion
2118
2578
  //#region src/checkout/git-clone.ts
2119
2579
  /**
2580
+ * Point a clone at the agent's credential helper.
2581
+ *
2582
+ * Only the helper PATH is written — never a secret — which is what makes it
2583
+ * safe to persist in `.git/config`. `useHttpPath` makes git include
2584
+ * `path=owner/repo.git` in every credential query, without which the helper
2585
+ * could not tell one repository from another and a write grant could not be
2586
+ * confined to its own repo.
2587
+ */
2588
+ function configureCredentialHelper(workDir, helperPath) {
2589
+ execFileSync("git", [
2590
+ "-C",
2591
+ workDir,
2592
+ "config",
2593
+ "credential.helper",
2594
+ helperPath
2595
+ ], {
2596
+ stdio: "pipe",
2597
+ timeout: 1e4
2598
+ });
2599
+ execFileSync("git", [
2600
+ "-C",
2601
+ workDir,
2602
+ "config",
2603
+ "credential.useHttpPath",
2604
+ "true"
2605
+ ], {
2606
+ stdio: "pipe",
2607
+ timeout: 1e4
2608
+ });
2609
+ }
2610
+ /**
2120
2611
  * Strip auth credentials from git error messages to prevent token leakage.
2121
2612
  * Node's execFileSync includes the full command line (including -c http.extraHeader
2122
2613
  * and GIT_SSH_COMMAND flags) in error messages, which would expose Base64-encoded
@@ -2143,7 +2634,7 @@ function redactSensitive(input) {
2143
2634
  * @throws Error if clone fails or SHA does not match
2144
2635
  */
2145
2636
  async function gitClone(options) {
2146
- const { repoUrl, ref, sha, workDir, token, gitAuth, depth = 1 } = options;
2637
+ const { repoUrl, ref, sha, workDir, token, gitAuth, depth = 1, credentialHelperPath, sshCleanupRegistry } = options;
2147
2638
  const auth = gitAuth ? gitAuth : token ? {
2148
2639
  kind: "basic",
2149
2640
  user: "x-access-token",
@@ -2197,6 +2688,7 @@ async function gitClone(options) {
2197
2688
  } catch (err) {
2198
2689
  throw sanitizeGitError(err);
2199
2690
  }
2691
+ if (credentialHelperPath) configureCredentialHelper(workDir, credentialHelperPath);
2200
2692
  if (!sha || sha === "HEAD") return;
2201
2693
  const envOpts = env ? { env } : {};
2202
2694
  if (!execFileSync("git", [
@@ -2252,7 +2744,12 @@ async function gitClone(options) {
2252
2744
  if (!recheckedSha.startsWith(sha)) throw new Error(`SHA mismatch: expected ${sha}, got ${recheckedSha}`);
2253
2745
  }
2254
2746
  } finally {
2255
- if (sshSetup) await sshSetup.cleanup().catch(() => {});
2747
+ if (sshSetup) {
2748
+ if (sshCleanupRegistry) {
2749
+ const setup = sshSetup;
2750
+ sshCleanupRegistry.defer(() => setup.cleanup().catch(() => {}));
2751
+ } else await sshSetup.cleanup().catch(() => {});
2752
+ }
2256
2753
  if (safeDirCleanup) await safeDirCleanup();
2257
2754
  }
2258
2755
  }
@@ -2260,6 +2757,408 @@ var init_git_clone = __esmMin((() => {
2260
2757
  init_ssh_auth();
2261
2758
  }));
2262
2759
  //#endregion
2760
+ //#region src/checkout/clone-job-repos.ts
2761
+ /**
2762
+ * Clone a job's repositories — from the host or from inside the sandbox.
2763
+ *
2764
+ * This used to live only inside the workflow runner, which meant the clone
2765
+ * always happened wherever the runner ran: for a container job, inside the
2766
+ * customer's image, which therefore had to ship git. Extracting it lets the
2767
+ * AGENT clone on the host and copy the tree in, so the image needs no git —
2768
+ * and it puts clone-time credentials on the host, where the credential helper
2769
+ * already works, rather than needing a route into a container hardened with
2770
+ * `CapDrop: ALL`.
2771
+ *
2772
+ * Both callers run the SAME code: the runner keeps calling it for bare-metal
2773
+ * and for the legacy container path, and the agent calls it for a host-side
2774
+ * checkout. A second implementation would be two subtly different clones.
2775
+ */
2776
+ /**
2777
+ * Clone whatever this job needs, or nothing.
2778
+ *
2779
+ * Three modes, unchanged from where this logic used to live: full-repo overlay
2780
+ * (no clone), global dual-clone (workflow repo + source repo), and the ordinary
2781
+ * single-repo clone.
2782
+ */
2783
+ async function cloneJobRepos(request, dirs, deps) {
2784
+ if (request.checkout === false) return;
2785
+ const helper = request.credentialHelperPath ? { credentialHelperPath: request.credentialHelperPath } : {};
2786
+ if (request.fullRepo) {
2787
+ await mkdir(dirs.workDir, { recursive: true });
2788
+ deps.log("Full-repo mode: skipping git clone (workspace from overlay tarball)");
2789
+ return;
2790
+ }
2791
+ if (deps.isGlobal) {
2792
+ await mkdir(dirs.workflowDir, { recursive: true });
2793
+ await mkdir(dirs.sourceDir, { recursive: true });
2794
+ const workflowAuth = request.workflowAuth ?? request.sourceAuth;
2795
+ const sourceAuth = request.sourceAuth ?? request.workflowAuth;
2796
+ deps.log(`Global workflow: cloning workflow repo ${request.workflowRepoUrl} ref=${request.workflowRef} into ${dirs.workflowDir}`);
2797
+ await gitClone({
2798
+ repoUrl: request.workflowRepoUrl,
2799
+ ref: request.workflowRef ?? "",
2800
+ sha: request.workflowSha ?? "",
2801
+ workDir: dirs.workflowDir,
2802
+ gitAuth: workflowAuth,
2803
+ token: workflowAuth ? void 0 : request.token,
2804
+ ...helper
2805
+ });
2806
+ await deps.excludeScratchFromGit(dirs.workflowDir);
2807
+ deps.log(`Global workflow: cloning source repo ${request.repoUrl} ref=${request.ref} into ${dirs.sourceDir}`);
2808
+ await gitClone({
2809
+ repoUrl: request.repoUrl,
2810
+ ref: request.ref,
2811
+ sha: request.sha,
2812
+ workDir: dirs.sourceDir,
2813
+ gitAuth: sourceAuth,
2814
+ token: sourceAuth ? void 0 : request.token,
2815
+ ...helper
2816
+ });
2817
+ deps.log("Dual-clone complete");
2818
+ return;
2819
+ }
2820
+ deps.log(`Cloning ${request.repoUrl} ref=${request.ref} into ${dirs.workDir}`);
2821
+ await gitClone({
2822
+ repoUrl: request.repoUrl,
2823
+ ref: request.ref,
2824
+ sha: request.sha,
2825
+ workDir: dirs.workDir,
2826
+ gitAuth: request.sourceAuth,
2827
+ token: request.sourceAuth ? void 0 : request.token,
2828
+ ...helper
2829
+ });
2830
+ await deps.excludeScratchFromGit(dirs.workDir);
2831
+ deps.log("Clone complete");
2832
+ }
2833
+ var init_clone_job_repos = __esmMin((() => {
2834
+ init_git_clone();
2835
+ }));
2836
+ //#endregion
2837
+ //#region src/checkout/grant-table.ts
2838
+ /**
2839
+ * Live write grants for this job.
2840
+ *
2841
+ * `withWrite` adds a grant before running its callback and revokes it in a
2842
+ * `finally`; the TTL is a backstop so a step that crashes cannot leave one
2843
+ * standing. The credential helper consults this table to decide whether to ask
2844
+ * the broker for a read-only or an elevated credential.
2845
+ *
2846
+ * Scope note, stated because the name could mislead: a grant is scoped to a
2847
+ * REPOSITORY and a TIME WINDOW — not to a step. The agent forks one process per
2848
+ * job and `parallel()` runs its children inside that process, so a concurrent
2849
+ * sibling step can push to the same repository while a grant is live. It cannot
2850
+ * reach a different repository.
2851
+ */
2852
+ /** Git presents paths as `/owner/repo.git`; callers use `owner/repo`. Compare one form. */
2853
+ function normalise(repoPath) {
2854
+ return repoPath.replace(/^\/+/, "").replace(/\.git$/, "").toLowerCase();
2855
+ }
2856
+ var GrantTable;
2857
+ var init_grant_table = __esmMin((() => {
2858
+ GrantTable = class {
2859
+ grants = /* @__PURE__ */ new Map();
2860
+ add(grant) {
2861
+ const id = randomUUID();
2862
+ this.grants.set(id, {
2863
+ ...grant,
2864
+ repoPath: normalise(grant.repoPath)
2865
+ });
2866
+ return id;
2867
+ }
2868
+ revoke(grantId) {
2869
+ this.grants.delete(grantId);
2870
+ }
2871
+ lookup(repoPath, now = Date.now()) {
2872
+ const wanted = normalise(repoPath);
2873
+ for (const [id, grant] of this.grants) {
2874
+ if (grant.expiresAt <= now) {
2875
+ this.grants.delete(id);
2876
+ continue;
2877
+ }
2878
+ if (grant.repoPath === wanted) return grant;
2879
+ }
2880
+ return null;
2881
+ }
2882
+ /** Live grant count, after reaping expired entries. Test and diagnostics use. */
2883
+ size(now = Date.now()) {
2884
+ for (const [id, grant] of this.grants) if (grant.expiresAt <= now) this.grants.delete(id);
2885
+ return this.grants.size;
2886
+ }
2887
+ };
2888
+ }));
2889
+ //#endregion
2890
+ //#region src/checkout/write-elevation.ts
2891
+ /** Requested entries the grant does not satisfy, as `key=value` strings. */
2892
+ function missingPermissions(requested, granted) {
2893
+ return Object.entries(requested).filter(([key, value]) => granted[key] !== value).map(([key, value]) => `${key}=${value}`);
2894
+ }
2895
+ async function elevateForWrite(args) {
2896
+ const { grant, expiresAt } = await args.request({
2897
+ repository: args.repository,
2898
+ permissions: args.permissions
2899
+ });
2900
+ if (grant.scoped) {
2901
+ const missing = missingPermissions(args.permissions, grant.permissions);
2902
+ if (missing.length > 0) throw new Error(`Cannot elevate '${args.repository}' for write: the app did not grant ${missing.join(", ")}. Grant the permission on the app installation, or request only what it holds.`);
2903
+ }
2904
+ const parsed = expiresAt ? Date.parse(expiresAt) : NaN;
2905
+ const expiry = Number.isFinite(parsed) ? parsed : Date.now() + DEFAULT_GRANT_TTL_MS;
2906
+ return {
2907
+ grantId: args.grants.add({
2908
+ repoPath: args.repository,
2909
+ permissions: args.permissions,
2910
+ expiresAt: expiry
2911
+ }),
2912
+ granted: grant
2913
+ };
2914
+ }
2915
+ var DEFAULT_GRANT_TTL_MS;
2916
+ var init_write_elevation = __esmMin((() => {
2917
+ DEFAULT_GRANT_TTL_MS = 36e5;
2918
+ }));
2919
+ //#endregion
2920
+ //#region src/checkout/credential-helper-host.ts
2921
+ /**
2922
+ * Host side of the git credential helper.
2923
+ *
2924
+ * Git spawns the helper as its own process, so it cannot call into the agent
2925
+ * directly. This module gives it a route: a per-job unix socket the agent
2926
+ * listens on, plus a tiny executable shim that connects to it and speaks git's
2927
+ * credential protocol on stdin/stdout.
2928
+ *
2929
+ * The shim is deliberately self-contained — no imports beyond `node:net` — so it
2930
+ * needs no module resolution, no bundle, and no dependency on where the agent
2931
+ * was installed. Its only job is to move bytes.
2932
+ *
2933
+ * Scope: this is the HOST path, which covers bare-metal jobs. A container job
2934
+ * clones and executes inside the container today and has no route to this
2935
+ * socket; that is fixed by the dual-mode container work, which moves the clone
2936
+ * to the host and injects `/opt/kici` read-only.
2937
+ */
2938
+ /** The shim source. Kept inline so the helper has no build step of its own. */
2939
+ function shimSource(socketPath) {
2940
+ return `#!/usr/bin/env node
2941
+ 'use strict';
2942
+ const net = require('node:net');
2943
+ const op = process.argv[2];
2944
+ // We persist nothing, so 'store' and 'erase' have nothing to do. Exiting
2945
+ // non-zero here would break pushes that are otherwise working.
2946
+ if (op !== 'get') process.exit(0);
2947
+ let input = '';
2948
+ process.stdin.setEncoding('utf8');
2949
+ process.stdin.on('data', (c) => { input += c; });
2950
+ process.stdin.on('end', () => {
2951
+ const query = {};
2952
+ for (const line of input.split('\\n')) {
2953
+ const eq = line.indexOf('=');
2954
+ if (eq <= 0) continue;
2955
+ const k = line.slice(0, eq);
2956
+ if (k === 'protocol' || k === 'host' || k === 'path') query[k] = line.slice(eq + 1);
2957
+ }
2958
+ const sock = net.createConnection(${JSON.stringify(socketPath)});
2959
+ let reply = '';
2960
+ const bail = () => { try { sock.destroy(); } catch {} process.exit(0); };
2961
+ sock.on('error', bail);
2962
+ sock.on('connect', () => sock.end(JSON.stringify(query) + '\\n'));
2963
+ sock.on('data', (d) => { reply += d.toString('utf8'); });
2964
+ sock.on('close', () => {
2965
+ let out = '';
2966
+ try {
2967
+ const parsed = JSON.parse(reply || 'null');
2968
+ if (parsed && parsed.username) {
2969
+ out = 'username=' + parsed.username + '\\npassword=' + parsed.password + '\\n';
2970
+ }
2971
+ } catch {
2972
+ // An unparseable reply is a miss, not a crash: git falls through.
2973
+ }
2974
+ // Exit only once stdout has flushed. process.exit() truncates a pending
2975
+ // pipe write, which silently produced an empty credential reply.
2976
+ if (out === '') process.exit(0);
2977
+ process.stdout.write(out, () => process.exit(0));
2978
+ });
2979
+ });
2980
+ `;
2981
+ }
2982
+ /**
2983
+ * Start the per-job helper socket and materialize the shim.
2984
+ *
2985
+ * `answer` is the agent's resolver: it consults the grant table and asks the
2986
+ * orchestrator broker. Returning null means "no credential", which the shim
2987
+ * turns into an empty reply so git falls through to its own mechanisms.
2988
+ */
2989
+ async function startCredentialHelperHost(args) {
2990
+ await mkdir(args.dir, {
2991
+ recursive: true,
2992
+ mode: 448
2993
+ });
2994
+ const socketPath = join(args.dir, "git-credential.sock");
2995
+ const helperPath = join(args.dir, "git-credential-kici.cjs");
2996
+ await writeFile(helperPath, shimSource(socketPath), { mode: 448 });
2997
+ await chmod(helperPath, 448);
2998
+ const server = createServer({ allowHalfOpen: true }, (socket) => handleConnection(socket, args.answer));
2999
+ await new Promise((resolve, reject) => {
3000
+ server.once("error", reject);
3001
+ server.listen(socketPath, () => {
3002
+ server.removeListener("error", reject);
3003
+ resolve();
3004
+ });
3005
+ });
3006
+ return {
3007
+ helperPath,
3008
+ close: () => closeServer(server)
3009
+ };
3010
+ }
3011
+ /** Read one newline-terminated query, answer it, close. */
3012
+ function handleConnection(socket, answer) {
3013
+ let buf = "";
3014
+ socket.setEncoding("utf8");
3015
+ socket.on("error", () => {});
3016
+ socket.on("data", (chunk) => {
3017
+ buf += chunk;
3018
+ });
3019
+ socket.on("end", () => {
3020
+ let query;
3021
+ try {
3022
+ query = JSON.parse(buf || "{}");
3023
+ } catch {
3024
+ socket.end("null\n");
3025
+ return;
3026
+ }
3027
+ answer(query).then((result) => socket.end(JSON.stringify(result) + "\n"), () => socket.end("null\n"));
3028
+ });
3029
+ }
3030
+ function closeServer(server) {
3031
+ return new Promise((resolve) => server.close(() => resolve()));
3032
+ }
3033
+ var init_credential_helper_host = __esmMin((() => {}));
3034
+ //#endregion
3035
+ //#region src/checkout/job-git-credentials.ts
3036
+ /**
3037
+ * Per-job git credential plumbing.
3038
+ *
3039
+ * Owns the three things a job needs to authenticate git: the grant table, the
3040
+ * helper socket git talks to, and the relay to the orchestrator broker. Kept in
3041
+ * its own module so the job runner wires one object rather than three, and so
3042
+ * this logic is testable without standing up a job.
3043
+ *
3044
+ * Scope: this is the HOST path, which covers bare-metal jobs. A container job
3045
+ * clones and executes inside the container today and has no route to the
3046
+ * socket; that is fixed by the dual-mode container work, which moves the clone
3047
+ * to the host and injects `/opt/kici` read-only.
3048
+ */
3049
+ /** `owner/repo.git` (or `/owner/repo`) as git spells it -> `owner/repo`. */
3050
+ function repositoryFromPath(path) {
3051
+ return (path ?? "").replace(/^\/+/, "").replace(/\.git$/, "");
3052
+ }
3053
+ /**
3054
+ * Stand up git credentials for one job.
3055
+ *
3056
+ * `sendApiRequest` is the agent's existing orchestrator relay — the same one
3057
+ * `ctx.kici` uses — so this adds no new transport.
3058
+ */
3059
+ async function startJobGitCredentials(args) {
3060
+ const grants = new GrantTable();
3061
+ /**
3062
+ * Pick the credential a request should use.
3063
+ *
3064
+ * A per-call name wins; otherwise `default` from the job's declared map;
3065
+ * otherwise undefined, which means the source credential — all a read needs.
3066
+ * An unknown name throws rather than silently falling back, because using a
3067
+ * different credential than the author named is the confusion the map exists
3068
+ * to remove.
3069
+ */
3070
+ const refFor = (name) => {
3071
+ if (name === void 0) return args.credentials?.default;
3072
+ const found = args.credentials?.[name];
3073
+ if (!found) throw new Error(`Unknown git credential '${name}'. Declare it in the job's gitCredentials map. Known: ${Object.keys(args.credentials ?? {}).join(", ") || "(none)"}`);
3074
+ return found;
3075
+ };
3076
+ const ask = async (repository, permissions, credentialName) => {
3077
+ const ref = refFor(credentialName);
3078
+ return await args.sendApiRequest(GIT_CREDENTIAL_REQUEST_METHOD, {
3079
+ jobId: args.jobId,
3080
+ repositories: [repository],
3081
+ permissions,
3082
+ ...ref ? { ref } : {}
3083
+ });
3084
+ };
3085
+ /** Answer one credential query from git, defaulting to read-only. */
3086
+ const answer = async (query) => {
3087
+ const repository = repositoryFromPath(query.path);
3088
+ if (repository === "") return null;
3089
+ const grant = grants.lookup(repository);
3090
+ const permissions = grant ? grant.permissions : { contents: "read" };
3091
+ const credential = await ask(repository, permissions);
3092
+ if (!credential || credential.kind !== "basic") return null;
3093
+ return {
3094
+ username: credential.user ?? "x-access-token",
3095
+ password: credential.secret
3096
+ };
3097
+ };
3098
+ let host = await startCredentialHelperHost({
3099
+ dir: args.dir,
3100
+ answer
3101
+ });
3102
+ const onGitGrantRequest = async (request) => {
3103
+ try {
3104
+ if (request.op === "revoke") {
3105
+ if (request.grantId) grants.revoke(request.grantId);
3106
+ return {
3107
+ type: "git.grant.response",
3108
+ requestId: request.requestId
3109
+ };
3110
+ }
3111
+ if (!request.repository) throw new Error("git.grant.request: elevate requires a repository");
3112
+ const { grantId, granted } = await elevateForWrite({
3113
+ repository: request.repository,
3114
+ permissions: request.permissions ?? { contents: "write" },
3115
+ grants,
3116
+ request: async ({ repository, permissions }) => {
3117
+ const result = await ask(repository, permissions, request.credentialName);
3118
+ return {
3119
+ grant: result.grant,
3120
+ expiresAt: result.expiresAt
3121
+ };
3122
+ }
3123
+ });
3124
+ return {
3125
+ type: "git.grant.response",
3126
+ requestId: request.requestId,
3127
+ grantId,
3128
+ granted
3129
+ };
3130
+ } catch (err) {
3131
+ return {
3132
+ type: "git.grant.response",
3133
+ requestId: request.requestId,
3134
+ error: err instanceof Error ? err.message : String(err)
3135
+ };
3136
+ }
3137
+ };
3138
+ return {
3139
+ helperPath: host.helperPath,
3140
+ onGitGrantRequest,
3141
+ withRef: (params) => {
3142
+ const { credential, ...rest } = params;
3143
+ const ref = refFor(typeof credential === "string" ? credential : void 0);
3144
+ return ref ? {
3145
+ ...rest,
3146
+ ref
3147
+ } : rest;
3148
+ },
3149
+ close: async () => {
3150
+ const open = host;
3151
+ host = void 0;
3152
+ if (open) await open.close();
3153
+ }
3154
+ };
3155
+ }
3156
+ var init_job_git_credentials = __esmMin((() => {
3157
+ init_grant_table();
3158
+ init_write_elevation();
3159
+ init_credential_helper_host();
3160
+ }));
3161
+ //#endregion
2263
3162
  //#region src/checkout/changed-files.ts
2264
3163
  /** Build the auth context for the fetches, mirroring git-clone.ts's auth. */
2265
3164
  async function buildAuthCtx(auth) {
@@ -2521,7 +3420,8 @@ var workflow_loader_exports = /* @__PURE__ */ __exportAll({
2521
3420
  */
2522
3421
  async function resolveWorkflowSdkSetters(workflowFilePath) {
2523
3422
  try {
2524
- const sdk = await import(pathToFileURL(createRequire(workflowFilePath).resolve("@kici-dev/sdk")).href);
3423
+ const sdkEntry = createRequire(workflowFilePath).resolve("@kici-dev/sdk");
3424
+ const sdk = await import(pathToFileURL(sdkEntry).href);
2525
3425
  if (typeof sdk.setStepOutputsMap === "function" && typeof sdk.setStepRefMap === "function" && typeof sdk.setJobOutputsMap === "function") return {
2526
3426
  setStepOutputsMap: sdk.setStepOutputsMap,
2527
3427
  setStepRefMap: sdk.setStepRefMap,
@@ -2694,8 +3594,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
2694
3594
  var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
2695
3595
  var init_workflow_loader = __esmMin((() => {
2696
3596
  init_generator_context();
2697
- AGENT_SDK_VERSION = "0.5.0";
2698
- AGENT_SDK_BUNDLE_HASH = "5b85e7cffa4a08e39329ff80448a2744af8e5b5f9840ec3801fad45ed11a3de9";
3597
+ AGENT_SDK_VERSION = "0.6.0";
3598
+ AGENT_SDK_BUNDLE_HASH = "22faf0da45de7c243ce87f80fd33ee51b1df52809fde457b16bb821678f65eb3";
2699
3599
  hookRegistered = false;
2700
3600
  }));
2701
3601
  //#endregion
@@ -2716,7 +3616,7 @@ var init_workflow_loader = __esmMin((() => {
2716
3616
  async function packKiciSource(workDir) {
2717
3617
  const kiciDir = join(workDir, ".kici");
2718
3618
  if (!existsSync(kiciDir)) throw new Error(`.kici/ not found at ${kiciDir}`);
2719
- logger$11.info("Packing .kici/ source tarball", { dir: workDir });
3619
+ logger$12.info("Packing .kici/ source tarball", { dir: workDir });
2720
3620
  const startTime = Date.now();
2721
3621
  const stream = c({
2722
3622
  gzip: true,
@@ -2730,7 +3630,7 @@ async function packKiciSource(workDir) {
2730
3630
  const hash = sha256(tarball);
2731
3631
  const sizeKB = (tarball.length / 1024).toFixed(2);
2732
3632
  const durationMs = Date.now() - startTime;
2733
- logger$11.info(".kici/ source packed", {
3633
+ logger$12.info(".kici/ source packed", {
2734
3634
  sizeKB,
2735
3635
  hash: hash.slice(0, 12),
2736
3636
  durationMs
@@ -2740,9 +3640,9 @@ async function packKiciSource(workDir) {
2740
3640
  hash
2741
3641
  };
2742
3642
  }
2743
- var logger$11;
3643
+ var logger$12;
2744
3644
  var init_source_packer = __esmMin((() => {
2745
- logger$11 = createLogger({ prefix: "source-packer" });
3645
+ logger$12 = createLogger({ prefix: "source-packer" });
2746
3646
  }));
2747
3647
  //#endregion
2748
3648
  //#region src/execution/dep-restore.ts
@@ -2868,7 +3768,7 @@ async function excludeScratchFromGit(repoWorkDir) {
2868
3768
  const suffix = existing.length === 0 || existing.endsWith("\n") ? "" : "\n";
2869
3769
  await fsPromises.appendFile(excludePath, `${suffix}# kici: hide dep-restore scratch dirs from customer git status\n${SCRATCH_DIR_GIT_EXCLUDE_GLOB}\n`);
2870
3770
  } catch (err) {
2871
- logger$10.warn("Failed to register scratch dir glob in .git/info/exclude", {
3771
+ logger$11.warn("Failed to register scratch dir glob in .git/info/exclude", {
2872
3772
  excludePath,
2873
3773
  error: err instanceof Error ? err.message : String(err)
2874
3774
  });
@@ -2929,7 +3829,7 @@ async function cleanupScratch(scratchDir) {
2929
3829
  force: true
2930
3830
  });
2931
3831
  } catch (cleanupErr) {
2932
- logger$10.warn("Scratch dir cleanup failed (orphan left behind)", {
3832
+ logger$11.warn("Scratch dir cleanup failed (orphan left behind)", {
2933
3833
  scratchDir,
2934
3834
  error: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
2935
3835
  });
@@ -2954,7 +3854,7 @@ async function cleanupScratch(scratchDir) {
2954
3854
  */
2955
3855
  async function restoreDeps(workDir, depsUrl, depsHash) {
2956
3856
  depsUrl = resolveOrchestratorUrl(depsUrl);
2957
- logger$10.info("Downloading dependency tarball", { url: depsUrl });
3857
+ logger$11.info("Downloading dependency tarball", { url: depsUrl });
2958
3858
  const kiciDir = join(workDir, ".kici");
2959
3859
  if (depsUrl.startsWith("file://")) {
2960
3860
  const localPath = fileURLToPath(depsUrl);
@@ -2967,8 +3867,8 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
2967
3867
  await extractTarball(data, scratchDir);
2968
3868
  await moveScratchIntoRepo(scratchDir, workDir);
2969
3869
  await cleanupScratch(scratchDir);
2970
- const sizeMB = (data.length / (1024 * 1024)).toFixed(2);
2971
- logger$10.info("Dependencies restored from cache (file)", {
3870
+ const sizeMB = (data.length / 1048576).toFixed(2);
3871
+ logger$11.info("Dependencies restored from cache (file)", {
2972
3872
  sizeMB,
2973
3873
  targetDir: workDir
2974
3874
  });
@@ -2977,7 +3877,7 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
2977
3877
  if (!depsUrl.startsWith("http://") && !depsUrl.startsWith("https://")) throw new Error(`Unsupported deps URL scheme: ${depsUrl}`);
2978
3878
  let lastError;
2979
3879
  for (let attempt = 0; attempt <= 2; attempt++) {
2980
- if (attempt > 0) logger$10.warn("Retrying dep tarball download", {
3880
+ if (attempt > 0) logger$11.warn("Retrying dep tarball download", {
2981
3881
  attempt,
2982
3882
  url: depsUrl
2983
3883
  });
@@ -2986,11 +3886,11 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
2986
3886
  if (depsHash && hash !== depsHash) throw new Error(`Dep tarball hash mismatch: expected ${depsHash}, got ${hash}`);
2987
3887
  await moveScratchIntoRepo(scratchDir, workDir);
2988
3888
  await cleanupScratch(scratchDir);
2989
- logger$10.info("Dependencies restored from cache (stream)", { targetDir: workDir });
3889
+ logger$11.info("Dependencies restored from cache (stream)", { targetDir: workDir });
2990
3890
  return;
2991
3891
  } catch (err) {
2992
3892
  lastError = err instanceof Error ? err : new Error(String(err));
2993
- logger$10.warn("Dep tarball download failed", {
3893
+ logger$11.warn("Dep tarball download failed", {
2994
3894
  attempt,
2995
3895
  error: lastError.message
2996
3896
  });
@@ -2998,10 +3898,10 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
2998
3898
  }
2999
3899
  throw new Error(`Dep tarball download failed after 3 attempts: ${lastError?.message}`);
3000
3900
  }
3001
- var logger$10, DOWNLOAD_TIMEOUT_MS$1, SCRATCH_DIR_BASENAME_PREFIX, SCRATCH_DIR_GIT_EXCLUDE_GLOB;
3901
+ var logger$11, DOWNLOAD_TIMEOUT_MS$1, SCRATCH_DIR_BASENAME_PREFIX, SCRATCH_DIR_GIT_EXCLUDE_GLOB;
3002
3902
  var init_dep_restore = __esmMin((() => {
3003
- logger$10 = createLogger({ prefix: "dep-restore" });
3004
- DOWNLOAD_TIMEOUT_MS$1 = 300 * 1e3;
3903
+ logger$11 = createLogger({ prefix: "dep-restore" });
3904
+ DOWNLOAD_TIMEOUT_MS$1 = 3e5;
3005
3905
  SCRATCH_DIR_BASENAME_PREFIX = ".dep-restore-scratch-";
3006
3906
  SCRATCH_DIR_GIT_EXCLUDE_GLOB = `.kici/${SCRATCH_DIR_BASENAME_PREFIX}*`;
3007
3907
  }));
@@ -3113,7 +4013,7 @@ async function uploadToPresignedUrl(url, data, opts) {
3113
4013
  for (let attempt = 0; attempt <= 2; attempt++) {
3114
4014
  if (attempt > 0) {
3115
4015
  const delayMs = baseDelayMs * 2 ** (attempt - 1);
3116
- logger$9.warn("Retrying pre-signed upload", {
4016
+ logger$10.warn("Retrying pre-signed upload", {
3117
4017
  attempt,
3118
4018
  delayMs,
3119
4019
  error: lastError?.message
@@ -3130,12 +4030,12 @@ async function uploadToPresignedUrl(url, data, opts) {
3130
4030
  }
3131
4031
  throw new Error(`Pre-signed upload failed after 3 attempts: ${lastError?.message}`);
3132
4032
  }
3133
- var logger$9, DOWNLOAD_TIMEOUT_MS, UPLOAD_TIMEOUT_MS, UPLOAD_RETRY_BASE_DELAY_MS, PresignedUploadHttpError;
4033
+ var logger$10, DOWNLOAD_TIMEOUT_MS, UPLOAD_TIMEOUT_MS, UPLOAD_RETRY_BASE_DELAY_MS, PresignedUploadHttpError;
3134
4034
  var init_download = __esmMin((() => {
3135
4035
  init_dep_restore();
3136
- logger$9 = createLogger({ prefix: "agent:download" });
3137
- DOWNLOAD_TIMEOUT_MS = 300 * 1e3;
3138
- UPLOAD_TIMEOUT_MS = 300 * 1e3;
4036
+ logger$10 = createLogger({ prefix: "agent:download" });
4037
+ DOWNLOAD_TIMEOUT_MS = 3e5;
4038
+ UPLOAD_TIMEOUT_MS = 3e5;
3139
4039
  UPLOAD_RETRY_BASE_DELAY_MS = 500;
3140
4040
  PresignedUploadHttpError = class extends Error {
3141
4041
  statusCode;
@@ -3181,7 +4081,7 @@ async function extractSourceTarball(data, targetDir) {
3181
4081
  }
3182
4082
  async function restoreSource(workDir, sourceTarUrl) {
3183
4083
  sourceTarUrl = resolveOrchestratorUrl(sourceTarUrl);
3184
- logger$8.info("Restoring .kici/ source from tarball", { sourceTarUrl });
4084
+ logger$9.info("Restoring .kici/ source from tarball", { sourceTarUrl });
3185
4085
  const startTime = Date.now();
3186
4086
  let data;
3187
4087
  if (sourceTarUrl.startsWith("file://")) {
@@ -3191,16 +4091,16 @@ async function restoreSource(workDir, sourceTarUrl) {
3191
4091
  else throw new Error(`Unsupported source tarball URL scheme: ${sourceTarUrl}`);
3192
4092
  await extractSourceTarball(data, workDir);
3193
4093
  const durationMs = Date.now() - startTime;
3194
- logger$8.info(".kici/ source restored", {
4094
+ logger$9.info(".kici/ source restored", {
3195
4095
  sizeKB: (data.length / 1024).toFixed(2),
3196
4096
  durationMs
3197
4097
  });
3198
4098
  }
3199
- var logger$8;
4099
+ var logger$9;
3200
4100
  var init_source_restore = __esmMin((() => {
3201
4101
  init_download();
3202
4102
  init_dep_restore();
3203
- logger$8 = createLogger({ prefix: "source-restore" });
4103
+ logger$9 = createLogger({ prefix: "source-restore" });
3204
4104
  }));
3205
4105
  //#endregion
3206
4106
  //#region src/execution/timeout-util.ts
@@ -3692,9 +4592,7 @@ async function stageAgentPayload(reach, privateKey, opts, deps) {
3692
4592
  case "ssh-push":
3693
4593
  await stageSshPush(reach, privateKey, opts, extractDir, deps);
3694
4594
  break;
3695
- case "s3-direct":
3696
- await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
3697
- break;
4595
+ case "s3-direct": await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
3698
4596
  }
3699
4597
  return { launcherPath: path.posix.join(extractDir, "kici-agent") };
3700
4598
  }
@@ -4193,23 +5091,31 @@ var init_streaming_zx_log = __esmMin((() => {}));
4193
5091
  *
4194
5092
  * @param jobs - Jobs returned by a DynamicJobFn
4195
5093
  * @param ctx - Eval-time context used to resolve dynamic fields on generated jobs
5094
+ * @param seenNames - Optional accumulator of job names already emitted earlier
5095
+ * in the same eval round; when supplied, a name already present throws and
5096
+ * every generated name is added so later generators in the round see it
4196
5097
  * @returns Serialized LockJob array ready for orchestrator dispatch
4197
5098
  * @throws Error if validation fails (duplicates, limit exceeded) or if a user-supplied
4198
5099
  * dynamic function throws / times out / returns an unsupported value
4199
5100
  */
4200
- async function serializeJobsToLock(jobs, ctx, staticNames, allowedGroups) {
5101
+ async function serializeJobsToLock(jobs, ctx, staticNames, allowedGroups, seenNames) {
4201
5102
  if (jobs.length > 100) throw new Error(`DynamicJobFn generated ${jobs.length} jobs, exceeding the limit of 100`);
4202
5103
  const generatedNames = /* @__PURE__ */ new Set();
5104
+ const roundNames = seenNames ?? /* @__PURE__ */ new Set();
4203
5105
  for (const job of jobs) {
4204
- if (generatedNames.has(job.name)) throw new Error(`Duplicate job name '${job.name}' in DynamicJobFn output`);
5106
+ if (roundNames.has(job.name)) throw new Error(`Duplicate job name '${job.name}' in dynamic job output`);
4205
5107
  generatedNames.add(job.name);
5108
+ roundNames.add(job.name);
4206
5109
  }
4207
5110
  const result = [];
4208
5111
  for (const job of jobs) result.push(await serializeJob(job, generatedNames, ctx, staticNames ?? /* @__PURE__ */ new Set(), allowedGroups ?? /* @__PURE__ */ new Set()));
4209
5112
  return result;
4210
5113
  }
4211
5114
  async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups) {
4212
- const { include: runsOn, exclude: excludeLabels } = normalizeRunsOnToMatchers(job.runsOn, `generated job '${job.name}' runsOn`);
5115
+ const { include: runsOn, exclude: excludeLabels } = job.invoke && job.runsOn === void 0 ? {
5116
+ include: [],
5117
+ exclude: []
5118
+ } : normalizeRunsOnToMatchers(job.runsOn, `generated job '${job.name}' runsOn`);
4213
5119
  const envRefs = job.contexts ?? (job.context !== void 0 ? [job.context] : void 0);
4214
5120
  let resolvedContexts;
4215
5121
  if (envRefs !== void 0 && envRefs.length > 0) {
@@ -4242,7 +5148,7 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
4242
5148
  return {
4243
5149
  _type: "static",
4244
5150
  name: job.name,
4245
- runsOn,
5151
+ ...job.invoke && runsOn.length === 0 ? {} : { runsOn },
4246
5152
  ...excludeLabels.length > 0 ? { excludeLabels } : {},
4247
5153
  needs: resolvedNeeds,
4248
5154
  ...dependsOnGroups.length > 0 ? { dependsOnGroups } : {},
@@ -4253,7 +5159,13 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
4253
5159
  ...job.description ? { description: job.description } : {},
4254
5160
  ...resolvedContexts !== void 0 ? { contexts: resolvedContexts } : {},
4255
5161
  ...resolvedEnv !== void 0 ? { env: resolvedEnv } : {},
4256
- ...resolvedConcurrencyGroup !== void 0 ? { concurrencyGroup: resolvedConcurrencyGroup } : {}
5162
+ ...resolvedConcurrencyGroup !== void 0 ? { concurrencyGroup: resolvedConcurrencyGroup } : {},
5163
+ ...job.invoke ? { invoke: {
5164
+ event: job.invoke.event,
5165
+ scope: job.invoke.scope,
5166
+ ...job.invoke.payload !== void 0 ? { payload: job.invoke.payload } : {},
5167
+ ...job.invoke.optional === true ? { optional: true } : {}
5168
+ } } : {}
4257
5169
  };
4258
5170
  }
4259
5171
  /**
@@ -4501,6 +5413,7 @@ async function generateDynamicJobs(workflow, shared) {
4501
5413
  workflowName: workflow.name
4502
5414
  };
4503
5415
  const jobs = [];
5416
+ const seenNames = /* @__PURE__ */ new Set();
4504
5417
  for (const generator of generators) {
4505
5418
  const generated = await generator(buildGeneratorContext({
4506
5419
  workflowName: workflow.name,
@@ -4511,7 +5424,7 @@ async function generateDynamicJobs(workflow, shared) {
4511
5424
  log: shared.log,
4512
5425
  kici: shared.kici
4513
5426
  }));
4514
- jobs.push(...await serializeJobsToLock(generated, serializerCtx));
5427
+ jobs.push(...await serializeJobsToLock(generated, serializerCtx, void 0, void 0, seenNames));
4515
5428
  }
4516
5429
  return jobs;
4517
5430
  }
@@ -4591,10 +5504,10 @@ async function evaluateAllCandidates(shared, into, deadline) {
4591
5504
  */
4592
5505
  async function runGlobalEvalRound(args) {
4593
5506
  const restoreEnv = applyGlobalWorkflowEnv(args.repos);
4594
- const shared = buildRoundState(args);
4595
5507
  const settled = [];
4596
5508
  let stopReason;
4597
5509
  try {
5510
+ const shared = buildRoundState(args);
4598
5511
  await withTimeout(() => evaluateAllCandidates(shared, settled, Date.now() + args.roundTimeoutMs), args.roundTimeoutMs, `global eval round (${args.candidates.length} candidate(s))`);
4599
5512
  } catch (error) {
4600
5513
  stopReason = error instanceof Error ? error.message : String(error);
@@ -4627,7 +5540,7 @@ var init_global_eval_runner = __esmMin((() => {
4627
5540
  })), DEFAULT_MAX_LOG_SIZE_BYTES, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_FLUSH_LINE_THRESHOLD, PAUSE_SAFETY_TIMEOUT_MS, LogStreamer;
4628
5541
  var init_log_streamer = __esmMin((() => {
4629
5542
  init_prometheus();
4630
- DEFAULT_MAX_LOG_SIZE_BYTES = 10 * 1024 * 1024;
5543
+ DEFAULT_MAX_LOG_SIZE_BYTES = 10485760;
4631
5544
  DEFAULT_FLUSH_INTERVAL_MS = 100;
4632
5545
  DEFAULT_FLUSH_LINE_THRESHOLD = 50;
4633
5546
  PAUSE_SAFETY_TIMEOUT_MS = 3e4;
@@ -4863,6 +5776,136 @@ var init_log_streamer = __esmMin((() => {
4863
5776
  };
4864
5777
  }));
4865
5778
  //#endregion
5779
+ //#region src/execution/image-build/resolve-build-spec.ts
5780
+ /**
5781
+ * Turn a job's `container.dockerfile` into an absolute, checked build spec.
5782
+ *
5783
+ * Pure on purpose. Every rule that decides what gets built — the anchoring, the
5784
+ * escape refusal, the tag shape — is decided here and unit-tested without a
5785
+ * container runtime anywhere near it. The half that needs a host lives in
5786
+ * `build-engine.ts`.
5787
+ */
5788
+ /**
5789
+ * Anchor a repo-relative path under `workDir`, refusing anything that leaves it.
5790
+ *
5791
+ * The SDK already refused these at workflow-definition time. Refusing them AGAIN
5792
+ * here is the point rather than a duplication: a lock file is repo content, so a
5793
+ * hand-edited lock could carry `../../etc/shadow` and reach a build context that
5794
+ * the author never wrote.
5795
+ */
5796
+ function anchor(workDir, p, field) {
5797
+ if (isAbsolute(p)) throw new Error(`container.${field} must stay inside the repository (got: ${p})`);
5798
+ const abs = resolve(workDir, p);
5799
+ const rel = relative(workDir, abs);
5800
+ if (rel.startsWith("..") || isAbsolute(rel)) throw new Error(`container.${field} must stay inside the repository (got: ${p})`);
5801
+ return abs;
5802
+ }
5803
+ /**
5804
+ * Reduce a value to a container-runtime tag component.
5805
+ *
5806
+ * A job name is author-supplied and may carry spaces, slashes or parentheses,
5807
+ * none of which a tag accepts.
5808
+ */
5809
+ function tagSafe(value, maxLength) {
5810
+ const slug = value.replace(/[^A-Za-z0-9_.-]/g, "-").replace(/^[-.]+|[-.]+$/g, "");
5811
+ return slug.length > 0 ? slug.slice(0, maxLength) : "job";
5812
+ }
5813
+ /**
5814
+ * Tag for a job's built image: a readable name, made unique by the job id.
5815
+ *
5816
+ * The job id is what guarantees uniqueness, and it is not decoration. The name
5817
+ * alone is truncated to fit a tag, and MATRIX LEGS of one job share a long
5818
+ * prefix and differ only in the suffix — `build (os=linux)` and
5819
+ * `build (os=darwin)` collide the moment the shared part exceeds the limit.
5820
+ * Two legs of one run on one host would then race for one tag, and a leg could
5821
+ * run its sibling's image.
5822
+ */
5823
+ function buildTagFor(jobName, jobId) {
5824
+ return `kici-build:${tagSafe(jobName, 48)}-${tagSafe(jobId, 12)}`;
5825
+ }
5826
+ /**
5827
+ * Resolve the build spec for a job, or `undefined` when the job builds nothing.
5828
+ *
5829
+ * `undefined` is the common case and is not a failure: a job with no container,
5830
+ * or one that names a finalized image, has nothing to build.
5831
+ */
5832
+ function resolveJobImageBuildSpec(args) {
5833
+ const { container, workDir, jobId, jobName } = args;
5834
+ const fileExists = args.fileExists ?? existsSync;
5835
+ if (!container || typeof container === "string") return void 0;
5836
+ const { dockerfile } = container;
5837
+ if (typeof dockerfile !== "string" || dockerfile.length === 0) return void 0;
5838
+ const dockerfilePath = anchor(workDir, dockerfile, "dockerfile");
5839
+ const contextDir = container.context === void 0 ? workDir : anchor(workDir, container.context, "context");
5840
+ if (!fileExists(dockerfilePath)) throw new Error(`container.dockerfile '${dockerfile}' does not exist in the repository (looked at ${dockerfilePath})`);
5841
+ return {
5842
+ dockerfilePath,
5843
+ contextDir,
5844
+ ...container.target !== void 0 ? { target: container.target } : {},
5845
+ args: { ...container.args ?? {} },
5846
+ tag: buildTagFor(jobName, jobId),
5847
+ labels: {
5848
+ "kici-managed": "true",
5849
+ "kici-job-id": jobId
5850
+ }
5851
+ };
5852
+ }
5853
+ var init_resolve_build_spec = __esmMin((() => {}));
5854
+ //#endregion
5855
+ //#region src/execution/image-build/build-step.ts
5856
+ /**
5857
+ * Run a job's image build and render it as a step in the run.
5858
+ *
5859
+ * The build happens on the AGENT, before the sandbox exists, so it cannot use
5860
+ * the runner's IPC pseudo-step channel that the cache phase uses — that channel
5861
+ * belongs to a process which has not started yet. It uses the agent-side seam
5862
+ * the synthetic `build` / `init` / `global-eval` jobs already use: a step-status
5863
+ * pair plus a log streamer bound to a step index.
5864
+ */
5865
+ /**
5866
+ * Build the job's image when it declared a Dockerfile, and return the tag the
5867
+ * sandbox should run.
5868
+ *
5869
+ * Returns `undefined` when there is nothing to build — a job with no container,
5870
+ * or one that names a finalized image. That is the common case and emits no
5871
+ * step at all: a run timeline should not grow an empty entry for work that did
5872
+ * not happen.
5873
+ */
5874
+ async function runJobImageBuild(args) {
5875
+ const { container, workDir, jobId, jobName, build, onLog, sendStepStatus } = args;
5876
+ const spec = resolveJobImageBuildSpec({
5877
+ container,
5878
+ workDir,
5879
+ jobId,
5880
+ jobName,
5881
+ ...args.fileExists ? { fileExists: args.fileExists } : {}
5882
+ });
5883
+ if (!spec) return void 0;
5884
+ const setup = { stepType: SetupStepType.enum["container:build"] };
5885
+ sendStepStatus(CONTAINER_BUILD_STEP_NAME, ExecutionStepStatus.enum.running, setup);
5886
+ onLog(`Building ${spec.tag} from ${spec.dockerfilePath}`);
5887
+ try {
5888
+ await build(spec, onLog);
5889
+ } catch (err) {
5890
+ const error = toErrorMessage(err);
5891
+ onLog(`Build failed: ${error}`);
5892
+ sendStepStatus(CONTAINER_BUILD_STEP_NAME, ExecutionStepStatus.enum.failed, {
5893
+ ...setup,
5894
+ error
5895
+ });
5896
+ throw err;
5897
+ }
5898
+ onLog(`Built ${spec.tag}`);
5899
+ sendStepStatus(CONTAINER_BUILD_STEP_NAME, ExecutionStepStatus.enum.success, setup);
5900
+ return spec.tag;
5901
+ }
5902
+ var CONTAINER_BUILD_STEP_NAME, CONTAINER_BUILD_STEP_INDEX;
5903
+ var init_build_step = __esmMin((() => {
5904
+ init_resolve_build_spec();
5905
+ CONTAINER_BUILD_STEP_NAME = "container:build";
5906
+ CONTAINER_BUILD_STEP_INDEX = 1e6;
5907
+ }));
5908
+ //#endregion
4866
5909
  //#region src/execution/overlay-applier.ts
4867
5910
  /**
4868
5911
  * Agent-side overlay application.
@@ -4910,7 +5953,7 @@ async function applyOverlay(config) {
4910
5953
  const { tarballUrl, cliPublicKey, orchestratorPrivateKey, repoDir } = config;
4911
5954
  const { path: tmpDir, cleanup } = await makeTempDir("overlay");
4912
5955
  try {
4913
- logger$7.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
5956
+ logger$8.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
4914
5957
  let encryptedData;
4915
5958
  try {
4916
5959
  encryptedData = await downloadUrl(tarballUrl);
@@ -4918,9 +5961,10 @@ async function applyOverlay(config) {
4918
5961
  throw new Error(`Overlay download failed from ${tarballUrl.replace(/\?.*$/, "?[redacted]")}: ${toErrorMessage(err)}`);
4919
5962
  }
4920
5963
  const cliPubKeyBuf = Buffer.from(cliPublicKey, "base64");
4921
- const aesKey = deriveSharedSecret(Buffer.from(orchestratorPrivateKey, "base64"), cliPubKeyBuf);
5964
+ const orchPrivKeyBuf = Buffer.from(orchestratorPrivateKey, "base64");
5965
+ const aesKey = deriveSharedSecret(orchPrivKeyBuf, cliPubKeyBuf);
4922
5966
  const decryptedData = decryptBuffer(encryptedData, aesKey);
4923
- logger$7.info("Extracting overlay tarball", { size: decryptedData.length });
5967
+ logger$8.info("Extracting overlay tarball", { size: decryptedData.length });
4924
5968
  const extractDir = path.join(tmpDir, "extracted");
4925
5969
  await fsPromises.mkdir(extractDir, { recursive: true });
4926
5970
  try {
@@ -4969,10 +6013,10 @@ async function applyOverlay(config) {
4969
6013
  await fsPromises.unlink(targetPath);
4970
6014
  filesDeleted++;
4971
6015
  } catch {
4972
- logger$7.debug("Deletion target not found, skipping", { file });
6016
+ logger$8.debug("Deletion target not found, skipping", { file });
4973
6017
  }
4974
6018
  }
4975
- logger$7.info("Overlay applied successfully", {
6019
+ logger$8.info("Overlay applied successfully", {
4976
6020
  filesApplied,
4977
6021
  filesDeleted
4978
6022
  });
@@ -4985,10 +6029,10 @@ async function applyOverlay(config) {
4985
6029
  await cleanup().catch(() => {});
4986
6030
  }
4987
6031
  }
4988
- var logger$7, IV_LENGTH$1, AUTH_TAG_LENGTH;
6032
+ var logger$8, IV_LENGTH$1, AUTH_TAG_LENGTH;
4989
6033
  var init_overlay_applier = __esmMin((() => {
4990
6034
  init_download();
4991
- logger$7 = createLogger({ prefix: "overlay-applier" });
6035
+ logger$8 = createLogger({ prefix: "overlay-applier" });
4992
6036
  IV_LENGTH$1 = 12;
4993
6037
  AUTH_TAG_LENGTH = 16;
4994
6038
  }));
@@ -5561,7 +6605,7 @@ async function installDeps(kiciDir, opts = {}) {
5561
6605
  const repoRoot = opts.repoRoot ?? dirname(kiciDir);
5562
6606
  const packageManager = await detectKiciPackageManager(repoRoot, kiciDir);
5563
6607
  const yarnFlavor = packageManager === PackageManager.Yarn ? await detectKiciYarnFlavor(repoRoot, kiciDir) : YarnFlavor.Classic;
5564
- logger$6.info("Installing deps inline", {
6608
+ logger$7.info("Installing deps inline", {
5565
6609
  packageManager,
5566
6610
  yarnFlavor,
5567
6611
  dir: kiciDir
@@ -5619,7 +6663,7 @@ async function installDeps(kiciDir, opts = {}) {
5619
6663
  if (packageManager === PackageManager.Yarn) await buildYarnWorkspaceClosure(repoRoot, kiciDir, yarnFlavor);
5620
6664
  const durationMs = Date.now() - startTime;
5621
6665
  process.stderr.write(`[dep-installer:trace] install complete: ${durationMs}ms\n`);
5622
- logger$6.info("Deps installed inline", {
6666
+ logger$7.info("Deps installed inline", {
5623
6667
  packageManager,
5624
6668
  durationMs
5625
6669
  });
@@ -5876,16 +6920,16 @@ function logSubprocessStreams(e, tokens) {
5876
6920
  if (e && typeof e === "object" && "stdout" in e) process.stderr.write(`[dep-installer:trace] stdout: ${redactNpmOutput(String(e.stdout), tokens).slice(0, 500)}\n`);
5877
6921
  if (e && typeof e === "object" && "stderr" in e) process.stderr.write(`[dep-installer:trace] stderr: ${redactNpmOutput(String(e.stderr), tokens).slice(0, 500)}\n`);
5878
6922
  }
5879
- var logger$6, execFileAsync, INSTALL_TIMEOUT_MS, INSTALL_MAX_BUFFER;
6923
+ var logger$7, execFileAsync, INSTALL_TIMEOUT_MS, INSTALL_MAX_BUFFER;
5880
6924
  var init_dep_installer = __esmMin((() => {
5881
6925
  init_npm_registry_config();
5882
6926
  init_yarnrc_berry_config();
5883
6927
  init_validate_kici_deps();
5884
6928
  init_workspace_siblings();
5885
- logger$6 = createLogger({ prefix: "dep-installer" });
6929
+ logger$7 = createLogger({ prefix: "dep-installer" });
5886
6930
  execFileAsync = promisify(execFile);
5887
6931
  INSTALL_TIMEOUT_MS = 6e5;
5888
- INSTALL_MAX_BUFFER = 128 * 1024 * 1024;
6932
+ INSTALL_MAX_BUFFER = 134217728;
5889
6933
  }));
5890
6934
  //#endregion
5891
6935
  //#region src/execution/dep-packer.ts
@@ -5930,7 +6974,7 @@ async function packNodeModules(kiciDir) {
5930
6974
  const nmRoot = packageManager === PackageManager.Yarn ? resolveYarnNodeModulesRoot(workDir, kiciDir) : join(kiciDir, "node_modules");
5931
6975
  if (!existsSync(nmRoot)) throw new Error(`node_modules not found at ${nmRoot}`);
5932
6976
  const entries = await closureEntries(workDir, kiciDir, packageManager);
5933
- logger$5.info("Packing dependency closure into tarball", {
6977
+ logger$6.info("Packing dependency closure into tarball", {
5934
6978
  dir: workDir,
5935
6979
  packageManager,
5936
6980
  entries
@@ -5945,8 +6989,8 @@ async function packNodeModules(kiciDir) {
5945
6989
  for await (const chunk of stream) chunks.push(Buffer.from(chunk));
5946
6990
  const tarball = Buffer.concat(chunks);
5947
6991
  const hash = sha256(tarball);
5948
- const sizeMB = (tarball.length / (1024 * 1024)).toFixed(2);
5949
- logger$5.info("Dependency closure packed", {
6992
+ const sizeMB = (tarball.length / 1048576).toFixed(2);
6993
+ logger$6.info("Dependency closure packed", {
5950
6994
  sizeMB,
5951
6995
  hash: hash.slice(0, 12),
5952
6996
  durationMs: Date.now() - startTime
@@ -5980,10 +7024,10 @@ async function closureEntries(workDir, kiciDir, packageManager) {
5980
7024
  }
5981
7025
  return [relative(workDir, join(kiciDir, "node_modules"))];
5982
7026
  }
5983
- var logger$5;
7027
+ var logger$6;
5984
7028
  var init_dep_packer = __esmMin((() => {
5985
7029
  init_workspace_siblings();
5986
- logger$5 = createLogger({ prefix: "dep-packer" });
7030
+ logger$6 = createLogger({ prefix: "dep-packer" });
5987
7031
  }));
5988
7032
  //#endregion
5989
7033
  //#region src/execution/sandbox/env-sanitizer.ts
@@ -6086,12 +7130,14 @@ function encryptSecretOutputs(outputs, runPublicKeyBase64) {
6086
7130
  format: "der"
6087
7131
  });
6088
7132
  const agentPublicKeyBase64 = Buffer.from(agentPublicKeyDer).toString("base64");
7133
+ const runPublicKeyDer = Buffer.from(runPublicKeyBase64, "base64");
7134
+ const runPublicKey = createPublicKey({
7135
+ key: runPublicKeyDer,
7136
+ format: "der",
7137
+ type: "spki"
7138
+ });
6089
7139
  const sharedSecret = diffieHellman({
6090
- publicKey: createPublicKey({
6091
- key: Buffer.from(runPublicKeyBase64, "base64"),
6092
- format: "der",
6093
- type: "spki"
6094
- }),
7140
+ publicKey: runPublicKey,
6095
7141
  privateKey: agentPriv
6096
7142
  });
6097
7143
  const aesKey = Buffer.from(hkdfSync("sha256", sharedSecret, Buffer.alloc(0), HKDF_INFO, 32));
@@ -6128,13 +7174,43 @@ var init_secret_encryption = __esmMin((() => {
6128
7174
  * logic to avoid duplication.
6129
7175
  */
6130
7176
  /**
7177
+ * Best-effort SIGKILL/SIGTERM of an entire process group led by `pid`.
7178
+ * `process.kill(-pid, signal)` targets the group whose leader is `pid` (the
7179
+ * child was spawned `detached`, so its pid is its group id). Returns 1 when the
7180
+ * group was signalled, 0 when it was already gone (ESRCH) or the pid is invalid.
7181
+ * We cannot cheaply count members, so the caller treats a non-zero return as
7182
+ * "reap attempted".
7183
+ */
7184
+ function killProcessGroup(pid, signal) {
7185
+ if (pid === void 0 || !Number.isInteger(pid) || pid <= 1) return 0;
7186
+ try {
7187
+ process.kill(-pid, signal);
7188
+ return 1;
7189
+ } catch {
7190
+ return 0;
7191
+ }
7192
+ }
7193
+ /**
7194
+ * SIGTERM the job's process group, wait a short grace, then SIGKILL. No-op
7195
+ * (returns 0) when the child was not spawned detached or has no pid. Returns the
7196
+ * count of reap attempts that signalled a live group (0, 1, or 2).
7197
+ */
7198
+ async function reapGroup(pid, detached, killFn = killProcessGroup, sleepMs = 2e3) {
7199
+ if (!detached || pid === void 0) return 0;
7200
+ const termed = killFn(pid, "SIGTERM");
7201
+ if (termed === 0) return 0;
7202
+ await new Promise((r) => setTimeout(r, sleepMs));
7203
+ return termed + killFn(pid, "SIGKILL");
7204
+ }
7205
+ /**
6131
7206
  * Build a JobExecutionRequest from a JobDispatch.
6132
7207
  *
6133
7208
  * Maps orchestrator dispatch fields to the subset needed by the workflow runner.
6134
7209
  */
6135
- function buildRequest(dispatch, workDir) {
7210
+ function buildRequest(dispatch, workDir, extra) {
6136
7211
  const jobConfig = dispatch.jobConfig;
6137
7212
  return {
7213
+ ...extra?.cleanupOnly ? { cleanupOnly: true } : {},
6138
7214
  runId: dispatch.runId,
6139
7215
  jobId: dispatch.jobId,
6140
7216
  workDir,
@@ -6144,6 +7220,7 @@ function buildRequest(dispatch, workDir) {
6144
7220
  token: dispatch.token,
6145
7221
  sourceAuth: dispatch.sourceAuth,
6146
7222
  workflowAuth: dispatch.workflowAuth,
7223
+ ...extra?.credentialHelperPath ? { credentialHelperPath: extra.credentialHelperPath } : {},
6147
7224
  sourceTarUrl: dispatch.sourceTarUrl,
6148
7225
  sourceTarHash: dispatch.sourceTarHash,
6149
7226
  depsUrl: dispatch.depsUrl,
@@ -6190,6 +7267,7 @@ function buildRequest(dispatch, workDir) {
6190
7267
  upstreamJobOutputs: dispatch.upstreamJobOutputs,
6191
7268
  upstreamJobStatuses: dispatch.upstreamJobStatuses,
6192
7269
  jobNeeds: jobConfig.needs,
7270
+ upstreamInvokeResults: dispatch.upstreamInvokeResults,
6193
7271
  npmRegistries: dispatch.npmRegistries,
6194
7272
  installEnvSecrets: dispatch.installEnvSecrets,
6195
7273
  jobIdShort: dispatch.jobId.slice(0, 8),
@@ -6328,20 +7406,22 @@ function fileCloneSourceBinds(repoUrl) {
6328
7406
  function spawnRunnerChild(options, sanitizedEnv, effectiveWorkDir) {
6329
7407
  const { runnerPath, useBwrap = false, networkIsolation = false, extraReadOnlyBinds = [] } = options;
6330
7408
  let child;
6331
- if (useBwrap) child = spawn("bwrap", [
6332
- ...buildBwrapArgs(effectiveWorkDir, process.execPath, networkIsolation, runnerPath, extraReadOnlyBinds),
6333
- process.execPath,
6334
- runnerPath
6335
- ], {
6336
- env: sanitizedEnv,
6337
- stdio: [
6338
- "pipe",
6339
- "pipe",
6340
- "pipe",
6341
- "ipc"
6342
- ]
6343
- });
6344
- else child = fork(runnerPath, [], {
7409
+ if (useBwrap) {
7410
+ const bwrapArgs = buildBwrapArgs(effectiveWorkDir, process.execPath, networkIsolation, runnerPath, extraReadOnlyBinds);
7411
+ child = spawn("bwrap", [
7412
+ ...bwrapArgs,
7413
+ process.execPath,
7414
+ runnerPath
7415
+ ], {
7416
+ env: sanitizedEnv,
7417
+ stdio: [
7418
+ "pipe",
7419
+ "pipe",
7420
+ "pipe",
7421
+ "ipc"
7422
+ ]
7423
+ });
7424
+ } else child = fork(runnerPath, [], {
6345
7425
  env: sanitizedEnv,
6346
7426
  stdio: [
6347
7427
  "pipe",
@@ -6349,7 +7429,8 @@ function spawnRunnerChild(options, sanitizedEnv, effectiveWorkDir) {
6349
7429
  "pipe",
6350
7430
  "ipc"
6351
7431
  ],
6352
- cwd: effectiveWorkDir || void 0
7432
+ cwd: effectiveWorkDir || void 0,
7433
+ detached: options.detachProcessGroup === true
6353
7434
  });
6354
7435
  return {
6355
7436
  child,
@@ -6450,6 +7531,24 @@ function relayCacheRequest$1(msg, ctx) {
6450
7531
  error: toErrorMessage(err)
6451
7532
  }));
6452
7533
  }
7534
+ /** Relay `git.grant.request` to the agent's grant table and pipe the response
7535
+ * (or an error response, or a "not configured" response when the callback
7536
+ * isn't wired) back into the sandbox runner. */
7537
+ function relayGitGrantRequest$1(msg, ctx) {
7538
+ if (!ctx.execOptions.onGitGrantRequest) {
7539
+ safeSendToChild(ctx.child, {
7540
+ type: "git.grant.response",
7541
+ requestId: msg.requestId,
7542
+ error: "Git credentials are not available in this agent configuration"
7543
+ });
7544
+ return;
7545
+ }
7546
+ ctx.execOptions.onGitGrantRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
7547
+ type: "git.grant.response",
7548
+ requestId: msg.requestId,
7549
+ error: toErrorMessage(err)
7550
+ }));
7551
+ }
6453
7552
  /** Relay `provenance.request` and pipe the orchestrator response (or an error
6454
7553
  * response, or a "not configured" response when the callback isn't wired) back
6455
7554
  * into the sandbox runner. */
@@ -6536,7 +7635,10 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
6536
7635
  case "ready":
6537
7636
  safeSendToChild(ctx.child, {
6538
7637
  type: "execute",
6539
- request: buildRequest(dispatch, ctx.effectiveWorkDir)
7638
+ request: buildRequest(dispatch, ctx.effectiveWorkDir, {
7639
+ cleanupOnly: ctx.cleanupOnly,
7640
+ credentialHelperPath: ctx.execOptions.credentialHelperPath
7641
+ })
6540
7642
  });
6541
7643
  return;
6542
7644
  case "log.line":
@@ -6585,6 +7687,9 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
6585
7687
  case "agent.api.request":
6586
7688
  relayAgentApiRequest(msg, ctx);
6587
7689
  return;
7690
+ case "git.grant.request":
7691
+ relayGitGrantRequest$1(msg, ctx);
7692
+ return;
6588
7693
  case "cache.request":
6589
7694
  relayCacheRequest$1(msg, ctx);
6590
7695
  return;
@@ -6597,6 +7702,12 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
6597
7702
  case "approval.request":
6598
7703
  relayApprovalRequest$1(msg, ctx);
6599
7704
  return;
7705
+ case "hooks-declared":
7706
+ ctx.state.declaresCleanup = msg.declaresCleanup;
7707
+ return;
7708
+ case "completion-hooks-done":
7709
+ ctx.state.completionHooksRan = true;
7710
+ return;
6600
7711
  case "job.complete":
6601
7712
  handleJobComplete(msg, dispatch, ctx);
6602
7713
  return;
@@ -6624,7 +7735,7 @@ function handleChildExitWithoutCompletion(code, signal, ctx) {
6624
7735
  /** Build the cancel function (state machine: running → cancelling →
6625
7736
  * force_killing). The caller is responsible for surfacing it to the
6626
7737
  * ForkRunnerHandle via the closure. */
6627
- function buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs) {
7738
+ function buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs, detached) {
6628
7739
  const doForceCancel = () => {
6629
7740
  ctx.state.forkState = "force_killing";
6630
7741
  safeSendToChild(child, {
@@ -6634,6 +7745,7 @@ function buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs)
6634
7745
  try {
6635
7746
  child.kill("SIGKILL");
6636
7747
  } catch {}
7748
+ if (detached) killProcessGroup(child.pid, "SIGKILL");
6637
7749
  clearAllCancelTimers(ctx);
6638
7750
  };
6639
7751
  return (force, gracePeriodMs) => {
@@ -6686,16 +7798,23 @@ function createForkRunner(options, execOptions) {
6686
7798
  }),
6687
7799
  abort: async () => {},
6688
7800
  kill: noopFn,
6689
- cancel: noopFn
7801
+ cancel: noopFn,
7802
+ detached: false,
7803
+ completionHooksRan: false,
7804
+ declaresCleanup: false,
7805
+ reap: async () => 0
6690
7806
  };
6691
7807
  }
6692
7808
  const stderrLines = setupChildStdioCapture(child);
6693
7809
  const defaultGracePeriodMs = 3e4;
6694
7810
  const agentMaxGracePeriodMs = options.maxGracePeriodMs ?? defaultGracePeriodMs;
6695
7811
  const cancelTimers = [];
7812
+ const detached = options.detachProcessGroup === true;
6696
7813
  const sharedState = {
6697
7814
  forkState: "running",
6698
- jobCompleted: false
7815
+ jobCompleted: false,
7816
+ completionHooksRan: false,
7817
+ declaresCleanup: false
6699
7818
  };
6700
7819
  let cancelFn = () => {};
6701
7820
  const result = new Promise((resolve) => {
@@ -6708,10 +7827,11 @@ function createForkRunner(options, execOptions) {
6708
7827
  resolve,
6709
7828
  cancelTimers,
6710
7829
  state: sharedState,
6711
- stderrLines
7830
+ stderrLines,
7831
+ cleanupOnly: options.cleanupOnly === true
6712
7832
  };
6713
7833
  process.stderr.write(`[fork-runner] Child process spawned: pid=${child.pid}, runner=${options.runnerPath}, cwd=${effectiveWorkDir}\n`);
6714
- cancelFn = buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs);
7834
+ cancelFn = buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs, detached);
6715
7835
  child.on("message", (msg) => relayChildIpcMessage(msg, dispatch, ctx));
6716
7836
  child.on("exit", (code, signal) => handleChildExitWithoutCompletion(code, signal, ctx));
6717
7837
  child.on("error", (err) => {
@@ -6742,7 +7862,15 @@ function createForkRunner(options, execOptions) {
6742
7862
  for (const timer of cancelTimers) clearTimeout(timer);
6743
7863
  cancelTimers.length = 0;
6744
7864
  },
6745
- cancel: (force, gracePeriodMs) => cancelFn(force, gracePeriodMs)
7865
+ cancel: (force, gracePeriodMs) => cancelFn(force, gracePeriodMs),
7866
+ detached,
7867
+ get completionHooksRan() {
7868
+ return sharedState.completionHooksRan;
7869
+ },
7870
+ get declaresCleanup() {
7871
+ return sharedState.declaresCleanup;
7872
+ },
7873
+ reap: () => reapGroup(child.pid, detached)
6746
7874
  };
6747
7875
  }
6748
7876
  var init_fork_runner = __esmMin((() => {
@@ -6767,22 +7895,25 @@ var init_fork_runner = __esmMin((() => {
6767
7895
  * network access. This mode provides credential isolation only and should
6768
7896
  * be used in trusted environments.
6769
7897
  */
6770
- var logger$4, BareMetalSandbox;
7898
+ var logger$5, BareMetalSandbox;
6771
7899
  var init_bare_metal_sandbox = __esmMin((() => {
6772
7900
  init_fork_runner();
6773
- logger$4 = createLogger({ prefix: "bare-metal-sandbox" });
7901
+ logger$5 = createLogger({ prefix: "bare-metal-sandbox" });
6774
7902
  BareMetalSandbox = class {
6775
7903
  runnerPath;
6776
7904
  useBwrap;
6777
7905
  sandboxNetwork;
6778
7906
  env;
7907
+ orphanCleanup;
6779
7908
  runner = null;
6780
7909
  workDir;
7910
+ lastOptions = null;
6781
7911
  constructor(options) {
6782
7912
  this.runnerPath = options.runnerPath;
6783
7913
  this.useBwrap = options.sandbox;
6784
7914
  this.sandboxNetwork = options.sandboxNetwork ?? "isolated";
6785
7915
  this.env = options.env;
7916
+ this.orphanCleanup = options.orphanCleanup ?? true;
6786
7917
  }
6787
7918
  /**
6788
7919
  * Validate that the runner path exists and bwrap is available (if needed).
@@ -6797,25 +7928,27 @@ var init_bare_metal_sandbox = __esmMin((() => {
6797
7928
  if (this.useBwrap) try {
6798
7929
  const { execSync } = await import("node:child_process");
6799
7930
  execSync("which bwrap", { stdio: "ignore" });
6800
- if (this.sandboxNetwork === "isolated") logger$4.info("Bubblewrap (bwrap) sandbox enabled with network isolation (--unshare-net)");
6801
- else logger$4.info("Bubblewrap (bwrap) sandbox enabled with host network (KICI_SANDBOX_NETWORK=host)");
7931
+ if (this.sandboxNetwork === "isolated") logger$5.info("Bubblewrap (bwrap) sandbox enabled with network isolation (--unshare-net)");
7932
+ else logger$5.info("Bubblewrap (bwrap) sandbox enabled with host network (KICI_SANDBOX_NETWORK=host)");
6802
7933
  } catch {
6803
7934
  throw new Error("Bubblewrap (bwrap) not found. Install bubblewrap or set sandbox=false. On Debian/Ubuntu: apt install bubblewrap");
6804
7935
  }
6805
- else logger$4.warn("Bare-metal without sandbox provides limited isolation. Only environment sanitization is active. Enable sandbox=true with bubblewrap for PID/IPC/filesystem namespace isolation.");
7936
+ else logger$5.warn("Bare-metal without sandbox provides limited isolation. Only environment sanitization is active. Enable sandbox=true with bubblewrap for PID/IPC/filesystem namespace isolation.");
6806
7937
  }
6807
7938
  /**
6808
7939
  * Execute a job by forking the workflow runner with sanitized environment.
6809
7940
  */
6810
7941
  async executeJob(options) {
6811
7942
  const extraReadOnlyBinds = this.useBwrap ? fileCloneSourceBinds(options.dispatch.repoUrl) : [];
7943
+ this.lastOptions = options;
6812
7944
  this.runner = createForkRunner({
6813
7945
  runnerPath: this.runnerPath,
6814
7946
  env: this.env,
6815
7947
  useBwrap: this.useBwrap,
6816
7948
  workDir: this.workDir,
6817
7949
  networkIsolation: this.useBwrap && this.sandboxNetwork === "isolated",
6818
- extraReadOnlyBinds
7950
+ extraReadOnlyBinds,
7951
+ detachProcessGroup: this.orphanCleanup && !this.useBwrap
6819
7952
  }, options);
6820
7953
  return this.runner.result;
6821
7954
  }
@@ -6828,13 +7961,53 @@ var init_bare_metal_sandbox = __esmMin((() => {
6828
7961
  if (this.runner) await this.runner.abort();
6829
7962
  }
6830
7963
  /**
6831
- * Clean up the child process if still running.
7964
+ * Clean up the child process if still running. The handle reference is kept
7965
+ * (not nulled) so the between-jobs phase can still read `completionHooksRan` /
7966
+ * `declaresCleanup` and reap the process group after teardown — the group
7967
+ * survives the single child's death. The next `executeJob` overwrites it.
6832
7968
  */
6833
7969
  async teardown() {
6834
- if (this.runner) {
6835
- this.runner.kill();
6836
- this.runner = null;
6837
- }
7970
+ if (this.runner) this.runner.kill();
7971
+ }
7972
+ /** Whether the runner signalled its completion hooks ran. */
7973
+ get completionHooksRan() {
7974
+ return this.runner?.completionHooksRan ?? true;
7975
+ }
7976
+ /** Whether the job declared an onFailure / cleanup hook. */
7977
+ get declaresCleanup() {
7978
+ return this.runner?.declaresCleanup ?? false;
7979
+ }
7980
+ /** Reap the finished job's process group. */
7981
+ async reap() {
7982
+ return this.runner?.reap() ?? 0;
7983
+ }
7984
+ /**
7985
+ * Re-run the finished job's declared cleanup / onFailure hooks against the
7986
+ * preserved workdir, in a fresh cleanup-only child. Reuses the last job's
7987
+ * dispatch + orchestrator relays but suppresses step/log callbacks (the
7988
+ * original job already reported and its log streamers are gone). Rejects when
7989
+ * the cleanup-only child fails so the caller can time it out.
7990
+ */
7991
+ async runCleanupOnly(workDir, signal) {
7992
+ const opts = this.lastOptions;
7993
+ if (!opts) return;
7994
+ const handle = createForkRunner({
7995
+ runnerPath: this.runnerPath,
7996
+ env: this.env,
7997
+ useBwrap: this.useBwrap,
7998
+ workDir,
7999
+ networkIsolation: this.useBwrap && this.sandboxNetwork === "isolated",
8000
+ cleanupOnly: true,
8001
+ maxGracePeriodMs: 5e3
8002
+ }, {
8003
+ ...opts,
8004
+ signal,
8005
+ onStepStatus: () => {},
8006
+ onLogLine: () => {}
8007
+ });
8008
+ const result = await handle.result;
8009
+ handle.kill();
8010
+ if (result.status !== ExecutionJobStatus.enum.success) throw new Error(result.error ?? `cleanup-only re-run did not succeed (${result.status})`);
6838
8011
  }
6839
8012
  };
6840
8013
  }));
@@ -6895,6 +8068,77 @@ var init_firecracker_sandbox = __esmMin((() => {
6895
8068
  };
6896
8069
  }));
6897
8070
  //#endregion
8071
+ //#region src/execution/sandbox/kici-runtime.ts
8072
+ /**
8073
+ * Launch the runner with the injected node, never the image's own.
8074
+ *
8075
+ * The customer image is not required to ship Node, so a bare `node` would
8076
+ * resolve to nothing (or, worse, to an unrelated build).
8077
+ */
8078
+ function runnerLaunchArgv(runnerMountPath) {
8079
+ return [KICI_RUNTIME_NODE, runnerMountPath];
8080
+ }
8081
+ var KICI_RUNTIME_MOUNT, KICI_RUNTIME_NODE_DIR, KICI_RUNTIME_NODE;
8082
+ var init_kici_runtime = __esmMin((() => {
8083
+ KICI_RUNTIME_MOUNT = "/opt/kici";
8084
+ KICI_RUNTIME_NODE_DIR = `${KICI_RUNTIME_MOUNT}/node`;
8085
+ KICI_RUNTIME_NODE = `${KICI_RUNTIME_NODE_DIR}/bin/node`;
8086
+ }));
8087
+ //#endregion
8088
+ //#region src/execution/sandbox/image-preflight.ts
8089
+ /**
8090
+ * Classify an image from the subset of {@link PROBE_PATHS} that exist in it.
8091
+ *
8092
+ * Pure, so the decision table is testable without a container runtime.
8093
+ */
8094
+ function classifyImageLibc(presentPaths) {
8095
+ const present = new Set(presentPaths);
8096
+ const hasMusl = MUSL_LOADERS.some((p) => present.has(p));
8097
+ const hasGlibc = GLIBC_LOADERS.some((p) => present.has(p));
8098
+ if (hasMusl) return "musl";
8099
+ if (!hasGlibc) return "static";
8100
+ return present.has(SHELL) ? "glibc" : "no-shell";
8101
+ }
8102
+ function rejection(image, libc) {
8103
+ switch (libc) {
8104
+ case "musl": return `image '${image}' uses musl libc; the musl runtime variant is not enabled (glibc images only in this version). Use a glibc image — for example the '-slim' rather than the '-alpine' tag.`;
8105
+ case "static": return `image '${image}' has no dynamic loader; container jobs require a glibc image with ${SHELL}.`;
8106
+ case "no-shell": return `image '${image}' has no ${SHELL}; container jobs require a shell for step commands.`;
8107
+ }
8108
+ }
8109
+ /**
8110
+ * Throw unless `image` can host the injected runtime.
8111
+ *
8112
+ * Stats the probe paths through a created-but-never-started container, so a
8113
+ * shell-less or musl image is diagnosed without executing anything in it —
8114
+ * running a probe command would fail for the very reason we are testing for.
8115
+ */
8116
+ async function assertImageRunnable(docker, image) {
8117
+ const container = await docker.createContainer({ Image: image });
8118
+ try {
8119
+ const present = [];
8120
+ for (const path of PROBE_PATHS) try {
8121
+ await container.infoArchive({ path });
8122
+ present.push(path);
8123
+ } catch {}
8124
+ const libc = classifyImageLibc(present);
8125
+ if (libc !== "glibc") throw new Error(rejection(image, libc));
8126
+ } finally {
8127
+ await container.remove({ force: true }).catch(() => void 0);
8128
+ }
8129
+ }
8130
+ var MUSL_LOADERS, GLIBC_LOADERS, SHELL, PROBE_PATHS;
8131
+ var init_image_preflight = __esmMin((() => {
8132
+ MUSL_LOADERS = ["/lib/ld-musl-x86_64.so.1", "/lib/ld-musl-aarch64.so.1"];
8133
+ GLIBC_LOADERS = ["/lib64/ld-linux-x86-64.so.2", "/lib/ld-linux-aarch64.so.1"];
8134
+ SHELL = "/bin/sh";
8135
+ PROBE_PATHS = [
8136
+ ...GLIBC_LOADERS,
8137
+ ...MUSL_LOADERS,
8138
+ SHELL
8139
+ ];
8140
+ }));
8141
+ //#endregion
6898
8142
  //#region src/execution/sandbox/container-hardening.ts
6899
8143
  /**
6900
8144
  * Build the hardened HostConfig fragment for a job sandbox container.
@@ -6947,8 +8191,11 @@ var init_container_hardening = __esmMin((() => {
6947
8191
  * - Agent-internal credentials (KICI_*, KICI_DATABASE_URL, etc.) NEVER enter the container
6948
8192
  * - IPC uses demuxed Docker stream with JSON-line parsing on stdout
6949
8193
  *
6950
- * The container image MUST have Node.js installed (a kici/runner base image
6951
- * is deferred -- for now this is a documented requirement).
8194
+ * The container image does NOT need Node.js or git. KiCI provisions its own
8195
+ * runtime a pinned, official glibc-2.17 Node plus the runner bundle, mounted
8196
+ * read-only at /opt/kici — and launches the runner with THAT node. The image
8197
+ * needs only a glibc and a shell, which the preflight asserts before the
8198
+ * container is created.
6952
8199
  */
6953
8200
  /**
6954
8201
  * Relay event.emit from the container runner to the orchestrator via
@@ -7048,6 +8295,30 @@ function relayCacheRequest(stream, options, cacheMsg) {
7048
8295
  }));
7049
8296
  }
7050
8297
  /**
8298
+ * Relay git.grant.request from the container runner to the agent's grant table
8299
+ * via options.onGitGrantRequest, then write the response back through `stream`.
8300
+ */
8301
+ function relayGitGrantRequest(stream, options, grantMsg) {
8302
+ const writeResponse = (response) => {
8303
+ try {
8304
+ stream.write(JSON.stringify(response) + "\n");
8305
+ } catch {}
8306
+ };
8307
+ if (!options.onGitGrantRequest) {
8308
+ writeResponse({
8309
+ type: "git.grant.response",
8310
+ requestId: grantMsg.requestId,
8311
+ error: "Git credentials are not available in this agent configuration"
8312
+ });
8313
+ return;
8314
+ }
8315
+ options.onGitGrantRequest(grantMsg).then((response) => writeResponse(response), (err) => writeResponse({
8316
+ type: "git.grant.response",
8317
+ requestId: grantMsg.requestId,
8318
+ error: toErrorMessage(err)
8319
+ }));
8320
+ }
8321
+ /**
7051
8322
  * Relay provenance.request from the container runner to the orchestrator via
7052
8323
  * options.onProvenanceRequest, then write the response back through `stream`.
7053
8324
  * If the agent doesn't expose a provenance relay, write a structured error so
@@ -7140,15 +8411,17 @@ function applyJobComplete(msg, stepResults, state, options) {
7140
8411
  if (msg.secretOutputs && options.dispatch.runPublicKey) try {
7141
8412
  state.encryptedSecretOutputs = encryptSecretOutputs(msg.secretOutputs, options.dispatch.runPublicKey);
7142
8413
  } catch (err) {
7143
- logger$3.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
8414
+ logger$4.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
7144
8415
  }
7145
8416
  }
7146
- var logger$3, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, HOOK_MOUNT_PATH, INTERNAL_ENV, ContainerSandbox;
8417
+ var logger$4, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, HOOK_MOUNT_PATH, INTERNAL_ENV, ContainerSandbox;
7147
8418
  var init_container_sandbox = __esmMin((() => {
7148
8419
  init_fork_runner();
8420
+ init_kici_runtime();
8421
+ init_image_preflight();
7149
8422
  init_secret_encryption();
7150
8423
  init_container_hardening();
7151
- logger$3 = createLogger({ prefix: "container-sandbox" });
8424
+ logger$4 = createLogger({ prefix: "container-sandbox" });
7152
8425
  MAX_STDERR_LINES = 20;
7153
8426
  ABORT_GRACE_MS = 1e4;
7154
8427
  CONTAINER_STOP_TIMEOUT = 10;
@@ -7158,6 +8431,17 @@ var init_container_sandbox = __esmMin((() => {
7158
8431
  docker;
7159
8432
  image;
7160
8433
  runnerPath;
8434
+ runtimeNodePath;
8435
+ runtimeImage;
8436
+ buildTag;
8437
+ /**
8438
+ * The runtime actually injected into this job's container — the configured
8439
+ * path, or the volume materialized during setup. Resolved once in setup()
8440
+ * because materialization needs the container runtime, and read by both the
8441
+ * bind list and the runner launch.
8442
+ */
8443
+ resolvedRuntimeNode;
8444
+ registryAuth;
7161
8445
  runnerMountPath;
7162
8446
  /** Host path to the pure-JS container loader-hook bundle (bind-mounted :ro). */
7163
8447
  hookHostPath;
@@ -7179,6 +8463,10 @@ var init_container_sandbox = __esmMin((() => {
7179
8463
  this.docker = options.docker;
7180
8464
  this.image = options.image;
7181
8465
  this.runnerPath = options.runnerPath;
8466
+ this.runtimeNodePath = options.runtimeNodePath;
8467
+ this.runtimeImage = options.runtimeImage;
8468
+ this.buildTag = options.buildTag;
8469
+ this.registryAuth = options.registryAuth;
7182
8470
  this.runnerMountPath = options.runnerMountPath ?? "/opt/kici/workflow-runner.js";
7183
8471
  this.hookHostPath = options.hookPath ?? join(dirname(options.runnerPath), "container-ts-loader-hook.js");
7184
8472
  this.env = options.env;
@@ -7202,17 +8490,44 @@ var init_container_sandbox = __esmMin((() => {
7202
8490
  await this.docker.getImage(this.image).inspect();
7203
8491
  return;
7204
8492
  } catch {}
7205
- logger$3.info("Pulling sandbox image (not present locally)", { image: this.image });
7206
- const stream = await this.docker.pull(this.image);
8493
+ logger$4.info("Pulling sandbox image (not present locally)", {
8494
+ image: this.image,
8495
+ authenticated: this.registryAuth !== void 0
8496
+ });
8497
+ const stream = await this.docker.pull(this.image, this.registryAuth ? { authconfig: this.registryAuth } : {});
7207
8498
  await new Promise((resolve, reject) => {
7208
8499
  this.docker.modem.followProgress(stream, (err) => err ? reject(err) : resolve());
7209
8500
  });
7210
- logger$3.info("Sandbox image pulled", { image: this.image });
8501
+ logger$4.info("Sandbox image pulled", { image: this.image });
8502
+ }
8503
+ /**
8504
+ * Resolve the Node tree to inject into the job container, materializing it
8505
+ * when only an image was configured.
8506
+ *
8507
+ * A configured path wins: a caller that already provisioned the tree should
8508
+ * not pay a copy to arrive at the same one. Returning `undefined` is a real
8509
+ * outcome, not a failure — an agent with no runtime source runs the job on
8510
+ * the image's own `node`, which is what a `node:*` image was always doing.
8511
+ *
8512
+ * A materialization that FAILS is not softened into that outcome. Continuing
8513
+ * would start the job against an image the operator never claimed ships Node,
8514
+ * and the resulting "node: not found" says nothing about the runtime that was
8515
+ * supposed to be there.
8516
+ */
8517
+ async resolveRuntimeNode() {
8518
+ if (this.runtimeNodePath) return this.runtimeNodePath;
8519
+ if (!this.runtimeImage) return void 0;
8520
+ return await ensureRuntimeVolume({
8521
+ docker: this.docker,
8522
+ agentImage: this.runtimeImage,
8523
+ subtree: RuntimeSubtree.enum.node,
8524
+ onProgress: (message) => logger$4.info(message, { jobId: this.jobId })
8525
+ });
7211
8526
  }
7212
8527
  async setup(options) {
7213
8528
  this.containerName = `kici-sandbox-${this.jobId}-${Date.now()}`;
7214
8529
  const envArray = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
7215
- logger$3.info("Creating sandbox container", {
8530
+ logger$4.info("Creating sandbox container", {
7216
8531
  name: this.containerName,
7217
8532
  image: this.image,
7218
8533
  workDir: options.workDir
@@ -7222,8 +8537,10 @@ var init_container_sandbox = __esmMin((() => {
7222
8537
  user: void 0
7223
8538
  };
7224
8539
  this.resolvedUser = hardened.user;
8540
+ this.resolvedRuntimeNode = await this.resolveRuntimeNode();
7225
8541
  const binds = this.buildBinds(options, hardened.hostConfig);
7226
8542
  await this.ensureImagePresent();
8543
+ if (this.resolvedRuntimeNode) await assertImageRunnable(this.docker, this.image);
7227
8544
  this.container = await this.docker.createContainer({
7228
8545
  Image: this.image,
7229
8546
  name: this.containerName,
@@ -7242,10 +8559,42 @@ var init_container_sandbox = __esmMin((() => {
7242
8559
  }
7243
8560
  });
7244
8561
  await this.container.start();
7245
- logger$3.info("Sandbox container started", {
8562
+ logger$4.info("Sandbox container started", {
7246
8563
  name: this.containerName,
7247
8564
  containerId: this.container.id.slice(0, 12)
7248
8565
  });
8566
+ if (options.workspaceFromHost) await this.copyWorkspaceIn(options.workDir);
8567
+ }
8568
+ /**
8569
+ * Populate the container's `/workspace` volume from the host working tree.
8570
+ *
8571
+ * `/workspace` is a container-owned anonymous volume rather than a host bind,
8572
+ * which is what dissolves the host-uid vs container-uid conflict once
8573
+ * `CapDrop: ['ALL']` removes CAP_DAC_OVERRIDE — so the tree is streamed in as
8574
+ * a tar rather than mounted.
8575
+ *
8576
+ * A failure here is fatal on purpose. Swallowing it would start the job
8577
+ * against an EMPTY workspace, which surfaces as a baffling "file not found"
8578
+ * in whichever step happens to touch the repo first.
8579
+ */
8580
+ async copyWorkspaceIn(workDir) {
8581
+ const started = Date.now();
8582
+ try {
8583
+ const stream = c({
8584
+ cwd: workDir,
8585
+ portable: true
8586
+ }, ["."]);
8587
+ const streamFailed = new Promise((_, reject) => {
8588
+ stream.on("error", reject);
8589
+ });
8590
+ await Promise.race([this.container.putArchive(stream, { path: "/workspace" }), streamFailed]);
8591
+ } catch (err) {
8592
+ throw new Error(`Failed to copy the host workspace into the sandbox container: ${err instanceof Error ? err.message : String(err)}`);
8593
+ }
8594
+ logger$4.info("Copied host workspace into sandbox", {
8595
+ name: this.containerName,
8596
+ durationMs: Date.now() - started
8597
+ });
7249
8598
  }
7250
8599
  /**
7251
8600
  * Build the container's read-only bind list.
@@ -7271,6 +8620,7 @@ var init_container_sandbox = __esmMin((() => {
7271
8620
  */
7272
8621
  buildBinds(options, hostConfig) {
7273
8622
  const binds = [`${this.runnerPath}:${this.runnerMountPath}:ro`, `${this.hookHostPath}:${HOOK_MOUNT_PATH}:ro`];
8623
+ if (this.resolvedRuntimeNode) binds.push(`${this.resolvedRuntimeNode}:${KICI_RUNTIME_NODE_DIR}:ro`);
7274
8624
  for (const dir of options.extraReadOnlyBinds ?? []) if (dir) binds.push(`${dir}:${dir}:ro`);
7275
8625
  if (hostConfig.NetworkMode === "host") {
7276
8626
  for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) binds.push(`${nssFile}:${nssFile}:ro`);
@@ -7285,7 +8635,7 @@ var init_container_sandbox = __esmMin((() => {
7285
8635
  try {
7286
8636
  outcome = await this.awaitJobCompletion(streamCtx, options);
7287
8637
  } catch (err) {
7288
- logger$3.error("Job execution error", {
8638
+ logger$4.error("Job execution error", {
7289
8639
  error: toErrorMessage(err),
7290
8640
  stderrTail: streamCtx.stderrLines.slice(-5).join("\n")
7291
8641
  });
@@ -7310,7 +8660,7 @@ var init_container_sandbox = __esmMin((() => {
7310
8660
  async attachExecStream(options) {
7311
8661
  const execEnv = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
7312
8662
  const stream = await (await this.container.exec({
7313
- Cmd: ["node", this.runnerMountPath],
8663
+ Cmd: this.resolvedRuntimeNode ? runnerLaunchArgv(this.runnerMountPath) : ["node", this.runnerMountPath],
7314
8664
  AttachStdin: true,
7315
8665
  AttachStdout: true,
7316
8666
  AttachStderr: true,
@@ -7336,7 +8686,7 @@ var init_container_sandbox = __esmMin((() => {
7336
8686
  });
7337
8687
  const abortHandler = () => {
7338
8688
  this.handleAbort().catch((err) => {
7339
- logger$3.warn("Error during abort", { error: toErrorMessage(err) });
8689
+ logger$4.warn("Error during abort", { error: toErrorMessage(err) });
7340
8690
  });
7341
8691
  };
7342
8692
  options.signal.addEventListener("abort", abortHandler, { once: true });
@@ -7373,7 +8723,7 @@ var init_container_sandbox = __esmMin((() => {
7373
8723
  try {
7374
8724
  msg = JSON.parse(line);
7375
8725
  } catch {
7376
- logger$3.warn("Non-JSON output from runner", { line: line.slice(0, 200) });
8726
+ logger$4.warn("Non-JSON output from runner", { line: line.slice(0, 200) });
7377
8727
  return;
7378
8728
  }
7379
8729
  if (this.dispatchRunnerMessage(msg, stream, options, stepNames, stepResults, state)) resolve({
@@ -7467,6 +8817,9 @@ var init_container_sandbox = __esmMin((() => {
7467
8817
  case "agent.api.request":
7468
8818
  relayApiRequest(stream, options, msg);
7469
8819
  return false;
8820
+ case "git.grant.request":
8821
+ relayGitGrantRequest(stream, options, msg);
8822
+ return false;
7470
8823
  case "cache.request":
7471
8824
  relayCacheRequest(stream, options, msg);
7472
8825
  return false;
@@ -7482,8 +8835,10 @@ var init_container_sandbox = __esmMin((() => {
7482
8835
  case "job.complete":
7483
8836
  applyJobComplete(msg, stepResults, state, options);
7484
8837
  return true;
8838
+ case "hooks-declared":
8839
+ case "completion-hooks-done": return false;
7485
8840
  default:
7486
- logger$3.warn("Unrecognized IPC message from container runner", { type: msg.type });
8841
+ logger$4.warn("Unrecognized IPC message from container runner", { type: msg.type });
7487
8842
  return false;
7488
8843
  }
7489
8844
  }
@@ -7507,17 +8862,33 @@ var init_container_sandbox = __esmMin((() => {
7507
8862
  async abort() {
7508
8863
  await this.handleAbort();
7509
8864
  }
8865
+ /**
8866
+ * Drop the tag a `container.dockerfile` build produced.
8867
+ *
8868
+ * Best-effort: teardown must not fail a job that already finished. The LAYER
8869
+ * cache — the thing that makes the next build fast — is not a tag and is
8870
+ * untouched by this.
8871
+ */
8872
+ async reclaimBuiltImage() {
8873
+ if (!this.buildTag) return;
8874
+ try {
8875
+ await this.docker.getImage(this.buildTag).remove({ force: true });
8876
+ } catch {}
8877
+ }
7510
8878
  async teardown() {
7511
- if (!this.container) return;
8879
+ if (!this.container) {
8880
+ await this.reclaimBuiltImage();
8881
+ return;
8882
+ }
7512
8883
  if (this.keepFailed && this.jobFailed) {
7513
- logger$3.info("Keeping failed container for debugging", {
8884
+ logger$4.info("Keeping failed container for debugging", {
7514
8885
  name: this.containerName,
7515
8886
  containerId: this.container.id.slice(0, 12)
7516
8887
  });
7517
8888
  this.container = null;
7518
8889
  return;
7519
8890
  }
7520
- logger$3.info("Tearing down sandbox container", { name: this.containerName });
8891
+ logger$4.info("Tearing down sandbox container", { name: this.containerName });
7521
8892
  try {
7522
8893
  await this.container.stop({ t: CONTAINER_STOP_TIMEOUT });
7523
8894
  } catch {}
@@ -7527,6 +8898,7 @@ var init_container_sandbox = __esmMin((() => {
7527
8898
  v: true
7528
8899
  });
7529
8900
  } catch {}
8901
+ await this.reclaimBuiltImage();
7530
8902
  this.container = null;
7531
8903
  }
7532
8904
  /**
@@ -7551,7 +8923,7 @@ var init_container_sandbox = __esmMin((() => {
7551
8923
  */
7552
8924
  async handleAbort() {
7553
8925
  if (!this.execStream && !this.container) return;
7554
- logger$3.info("Aborting sandbox execution", { name: this.containerName });
8926
+ logger$4.info("Aborting sandbox execution", { name: this.containerName });
7555
8927
  if (this.execStream) try {
7556
8928
  this.execStream.write(JSON.stringify({ type: "abort" }) + "\n");
7557
8929
  } catch {}
@@ -7823,6 +9195,7 @@ function resolveRunnerBundlePath(runnerPath) {
7823
9195
  * 4. Default -> 'bare-metal'
7824
9196
  */
7825
9197
  function determineExecutionMode(jobConfig, agentConfig) {
9198
+ if (agentConfig.jobImageAgent) return "bare-metal";
7826
9199
  if (jobConfig.container) return "container";
7827
9200
  if (agentConfig.executionMode) return agentConfig.executionMode;
7828
9201
  if (agentConfig.scalerManaged) return "firecracker";
@@ -7863,9 +9236,11 @@ async function resolveJobWorkDir(inPlace, repoUrl) {
7863
9236
  inPlace: false
7864
9237
  };
7865
9238
  }
7866
- var logger$2, DEFAULT_GLOBAL_EVAL_ROUND_TIMEOUT_MS, DEFAULT_GLOBAL_EVAL_CANDIDATE_TIMEOUT_MS, FILTER_SOURCE_DIRNAME, JobRunner$1;
9239
+ var logger$3, DEFAULT_GLOBAL_EVAL_ROUND_TIMEOUT_MS, DEFAULT_GLOBAL_EVAL_CANDIDATE_TIMEOUT_MS, FILTER_SOURCE_DIRNAME, JobRunner$1;
7867
9240
  var init_job_runner = __esmMin((() => {
7868
9241
  init_git_clone();
9242
+ init_clone_job_repos();
9243
+ init_job_git_credentials();
7869
9244
  init_changed_files();
7870
9245
  init_workflow_loader();
7871
9246
  init_source_packer();
@@ -7882,6 +9257,8 @@ var init_job_runner = __esmMin((() => {
7882
9257
  init_global_workflow_env();
7883
9258
  init_global_eval_runner();
7884
9259
  init_log_streamer();
9260
+ init_build_step();
9261
+ init_build_engine();
7885
9262
  init_overlay_applier();
7886
9263
  init_dep_installer();
7887
9264
  init_dep_restore();
@@ -7889,7 +9266,7 @@ var init_job_runner = __esmMin((() => {
7889
9266
  init_download();
7890
9267
  init_sandbox();
7891
9268
  init_prometheus();
7892
- logger$2 = createLogger({ prefix: "job-runner" });
9269
+ logger$3 = createLogger({ prefix: "job-runner" });
7893
9270
  DEFAULT_GLOBAL_EVAL_ROUND_TIMEOUT_MS = 12e4;
7894
9271
  DEFAULT_GLOBAL_EVAL_CANDIDATE_TIMEOUT_MS = 2e4;
7895
9272
  FILTER_SOURCE_DIRNAME = "__kici_filter_source__";
@@ -7905,6 +9282,8 @@ var init_job_runner = __esmMin((() => {
7905
9282
  _sendRunEvent;
7906
9283
  _sendConcurrencyReport;
7907
9284
  _sendApiRequest;
9285
+ /** Live git credentials per job, so the sandbox can reach the grant table. */
9286
+ jobGitCredentials = /* @__PURE__ */ new Map();
7908
9287
  _requestUserCache;
7909
9288
  _relayProvenance;
7910
9289
  _requestUserArtifact;
@@ -7913,9 +9292,19 @@ var init_job_runner = __esmMin((() => {
7913
9292
  activeJobs = /* @__PURE__ */ new Map();
7914
9293
  /** Active sandbox for the current job (used for abort). */
7915
9294
  activeSandbox = null;
9295
+ /** Supervisor-owned between-jobs phase (reap / cleanup re-run / reset). */
9296
+ betweenJobsController;
9297
+ /**
9298
+ * Facts about the just-finished standard job, captured before sandbox
9299
+ * teardown so the between-jobs controller (run in `execute`'s `.finally`) can
9300
+ * reach them. Null for special job types (init / build / dynamic), which run
9301
+ * no sandbox — the controller then only deletes the workdir and resets.
9302
+ */
9303
+ betweenJobsFacts = null;
7916
9304
  constructor(deps) {
7917
9305
  this.send = deps.send;
7918
9306
  this.config = deps.config;
9307
+ this.betweenJobsController = deps.betweenJobsController;
7919
9308
  this.requestUploadUrl = deps.requestUploadUrl;
7920
9309
  this.sendUploadComplete = deps.sendUploadComplete;
7921
9310
  this.sendEventEmit = deps.sendEventEmit;
@@ -7941,10 +9330,16 @@ var init_job_runner = __esmMin((() => {
7941
9330
  const abortController = new AbortController();
7942
9331
  const { workDir, cleanup, inPlace } = await resolveJobWorkDir(this.config.inPlace, dispatch.repoUrl);
7943
9332
  if (inPlace) dispatch.jobConfig.checkout = false;
9333
+ this.betweenJobsFacts = null;
9334
+ const gitCredentials = await this.startGitCredentials(jobId, dispatch.jobConfig);
7944
9335
  const completionPromise = this.runJob(dispatch, workDir, abortController).finally(async () => {
9336
+ const facts = this.betweenJobsFacts;
7945
9337
  this.activeJobs.delete(jobId);
7946
9338
  this.activeSandbox = null;
7947
- await cleanup();
9339
+ this.betweenJobsFacts = null;
9340
+ this.jobGitCredentials.delete(jobId);
9341
+ await gitCredentials?.close().catch(() => {});
9342
+ await this.runBetweenJobsPhase(facts, workDir, cleanup);
7948
9343
  });
7949
9344
  this.activeJobs.set(jobId, {
7950
9345
  abortController,
@@ -7954,6 +9349,63 @@ var init_job_runner = __esmMin((() => {
7954
9349
  return completionPromise;
7955
9350
  }
7956
9351
  /**
9352
+ * Stand up per-job git credentials, or `undefined` when the agent has no
9353
+ * orchestrator relay (unit harnesses, offline local runs).
9354
+ *
9355
+ * A failure here must not fail the job: without a helper, git falls back to
9356
+ * its own mechanisms exactly as it did before this existed. The job simply
9357
+ * cannot push.
9358
+ */
9359
+ async startGitCredentials(jobId, jobConfig) {
9360
+ if (!this._sendApiRequest) return void 0;
9361
+ try {
9362
+ const { path: dir } = await makeTempDir(`gitcred-${jobId}`);
9363
+ const credentials = await startJobGitCredentials({
9364
+ jobId,
9365
+ dir,
9366
+ sendApiRequest: this._sendApiRequest,
9367
+ credentials: jobConfig.gitCredentials
9368
+ });
9369
+ this.jobGitCredentials.set(jobId, credentials);
9370
+ return credentials;
9371
+ } catch (err) {
9372
+ logger$3.warn("Could not start git credentials for job; git push will not work", {
9373
+ jobId,
9374
+ error: toErrorMessage(err)
9375
+ });
9376
+ return;
9377
+ }
9378
+ }
9379
+ /**
9380
+ * Run the supervisor-owned between-jobs phase after a job finishes. Delegates
9381
+ * to the injected `BetweenJobsController` (out-of-band cleanup → reap →
9382
+ * workdir delete → operator reset) with facts captured before sandbox
9383
+ * teardown. When no controller is wired (unit harnesses) it just deletes the
9384
+ * workdir, preserving the historical behavior. Never throws.
9385
+ */
9386
+ async runBetweenJobsPhase(facts, workDir, deleteWorkdir) {
9387
+ if (!this.betweenJobsController) {
9388
+ await deleteWorkdir().catch((err) => logger$3.warn("Work directory cleanup error", { error: toErrorMessage(err) }));
9389
+ return;
9390
+ }
9391
+ const ctx = {
9392
+ completionHooksRan: facts?.completionHooksRan ?? true,
9393
+ jobFailed: facts?.jobFailed ?? false,
9394
+ backend: facts?.backend ?? "bare-metal",
9395
+ declaresCleanup: facts?.declaresCleanup ?? false,
9396
+ workDir,
9397
+ reap: facts?.reap ?? (() => Promise.resolve(0)),
9398
+ deleteWorkdir,
9399
+ ...facts?.cleanupSpawn ? { cleanupSpawn: facts.cleanupSpawn } : {}
9400
+ };
9401
+ try {
9402
+ await this.betweenJobsController.afterJob(ctx);
9403
+ } catch (err) {
9404
+ logger$3.error("Between-jobs phase error", { error: toErrorMessage(err) });
9405
+ await deleteWorkdir().catch(() => {});
9406
+ }
9407
+ }
9408
+ /**
7957
9409
  * Cancel a running job by signaling its abort controller
7958
9410
  * and aborting the active sandbox.
7959
9411
  *
@@ -8002,7 +9454,7 @@ var init_job_runner = __esmMin((() => {
8002
9454
  }
8003
9455
  if (jobConfig.buildOnly === true) {
8004
9456
  if (jobConfig.fullRepo) {
8005
- logger$2.warn("Build job received for fullRepo run -- skipping (should not happen)", {
9457
+ logger$3.warn("Build job received for fullRepo run -- skipping (should not happen)", {
8006
9458
  jobId,
8007
9459
  runId
8008
9460
  });
@@ -8024,7 +9476,7 @@ var init_job_runner = __esmMin((() => {
8024
9476
  async executeStandardJob(dispatch, workDir, abortController) {
8025
9477
  const { runId, jobId } = dispatch;
8026
9478
  const ctx = getRequestContext();
8027
- logger$2.info(`Run: ${ctx.runId ?? runId} | Trace: ${ctx.requestId ?? "N/A"}`);
9479
+ logger$3.info(`Run: ${ctx.runId ?? runId} | Trace: ${ctx.requestId ?? "N/A"}`);
8028
9480
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
8029
9481
  const heartbeatTimer = setInterval(() => {
8030
9482
  this.send({
@@ -8035,12 +9487,16 @@ var init_job_runner = __esmMin((() => {
8035
9487
  });
8036
9488
  }, this.config.jobHeartbeatIntervalMs);
8037
9489
  let sandbox;
9490
+ let backend = "bare-metal";
9491
+ let jobFailed = true;
8038
9492
  const logStreamers = /* @__PURE__ */ new Map();
8039
9493
  try {
8040
9494
  const setupResult = await this.setupSandboxForExecution(dispatch, workDir, abortController);
8041
9495
  if (!setupResult) return;
8042
9496
  sandbox = setupResult.sandbox;
9497
+ backend = setupResult.executionMode;
8043
9498
  const result = await this.runSandboxExecution(dispatch, sandbox, abortController, logStreamers);
9499
+ jobFailed = result.status !== ExecutionJobStatus.enum.success;
8044
9500
  this.reportExecutionResult(dispatch, result, logStreamers);
8045
9501
  } catch (error) {
8046
9502
  for (const streamer of logStreamers.values()) streamer.destroy();
@@ -8049,9 +9505,18 @@ var init_job_runner = __esmMin((() => {
8049
9505
  } finally {
8050
9506
  clearInterval(heartbeatTimer);
8051
9507
  if (sandbox) {
9508
+ const jobSandbox = sandbox;
9509
+ this.betweenJobsFacts = {
9510
+ completionHooksRan: jobSandbox.completionHooksRan ?? true,
9511
+ declaresCleanup: jobSandbox.declaresCleanup ?? false,
9512
+ backend,
9513
+ jobFailed,
9514
+ reap: () => jobSandbox.reap?.() ?? Promise.resolve(0),
9515
+ ...jobSandbox.runCleanupOnly ? { cleanupSpawn: (wd, sig) => jobSandbox.runCleanupOnly(wd, sig) } : {}
9516
+ };
8052
9517
  this.emitRunEvent(runId, "agent.teardown", { jobId });
8053
9518
  await sandbox.teardown().catch((err) => {
8054
- logger$2.warn("Sandbox teardown error", { error: toErrorMessage(err) });
9519
+ logger$3.warn("Sandbox teardown error", { error: toErrorMessage(err) });
8055
9520
  });
8056
9521
  }
8057
9522
  }
@@ -8071,7 +9536,8 @@ var init_job_runner = __esmMin((() => {
8071
9536
  }
8072
9537
  const executionMode = determineExecutionMode(jobConfig, {
8073
9538
  executionMode: this.config.executionMode,
8074
- scalerManaged: this.config.scalerManaged
9539
+ scalerManaged: this.config.scalerManaged,
9540
+ jobImageAgent: this.config.jobImageAgent
8075
9541
  });
8076
9542
  const runnerPath = resolveRunnerPath();
8077
9543
  const typedConfig = jobConfig;
@@ -8080,7 +9546,22 @@ var init_job_runner = __esmMin((() => {
8080
9546
  jobEnv: typedConfig.jobEnv ?? void 0,
8081
9547
  trustedEnv: this.config.trustedEnv
8082
9548
  });
8083
- logger$2.info("Creating execution sandbox", {
9549
+ const hostCheckout = executionMode === "container" && jobConfig.checkout !== false;
9550
+ if (hostCheckout) {
9551
+ const isGlobal = jobConfig.isGlobalWorkflow === true;
9552
+ await cloneJobRepos(dispatch, {
9553
+ workDir,
9554
+ workflowDir: isGlobal ? join(workDir, "workflow") : workDir,
9555
+ sourceDir: isGlobal ? join(workDir, "source") : workDir
9556
+ }, {
9557
+ isGlobal,
9558
+ log: (line) => logger$3.info(`[host-checkout] ${line}`, { jobId }),
9559
+ excludeScratchFromGit
9560
+ });
9561
+ jobConfig.checkout = false;
9562
+ }
9563
+ const builtImage = await this.buildJobImageIfDeclared(dispatch, jobConfig, workDir, abortController);
9564
+ logger$3.info("Creating execution sandbox", {
8084
9565
  executionMode,
8085
9566
  jobId,
8086
9567
  runnerPath
@@ -8089,13 +9570,16 @@ var init_job_runner = __esmMin((() => {
8089
9570
  runnerPath,
8090
9571
  env: sanitizedEnv,
8091
9572
  jobId,
8092
- jobConfig
9573
+ jobConfig,
9574
+ registryAuth: dispatch.containerRegistryAuth,
9575
+ ...builtImage ? { builtImage } : {}
8093
9576
  });
8094
9577
  this.activeSandbox = sandbox;
8095
9578
  await sandbox.setup({
8096
9579
  workDir,
8097
9580
  env: sanitizedEnv,
8098
- extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl)
9581
+ extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl),
9582
+ ...hostCheckout ? { workspaceFromHost: true } : {}
8099
9583
  });
8100
9584
  if (abortController.signal.aborted) {
8101
9585
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.cancelled);
@@ -8114,7 +9598,8 @@ var init_job_runner = __esmMin((() => {
8114
9598
  });
8115
9599
  return {
8116
9600
  sandbox,
8117
- sanitizedEnv
9601
+ sanitizedEnv,
9602
+ executionMode
8118
9603
  };
8119
9604
  }
8120
9605
  /**
@@ -8176,7 +9661,12 @@ var init_job_runner = __esmMin((() => {
8176
9661
  reason: ack.reason
8177
9662
  };
8178
9663
  },
8179
- onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => this._sendApiRequest(method, params)) : void 0,
9664
+ onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => {
9665
+ const patched = method === GIT_CREDENTIAL_REQUEST_METHOD ? this.jobGitCredentials.get(jobId)?.withRef(params ?? {}) ?? params : params;
9666
+ return this._sendApiRequest(method, patched);
9667
+ }) : void 0,
9668
+ onGitGrantRequest: this.jobGitCredentials.get(jobId)?.onGitGrantRequest,
9669
+ credentialHelperPath: this.jobGitCredentials.get(jobId)?.helperPath,
8180
9670
  onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
8181
9671
  onProvenanceRequest: this._relayProvenance ? async (request) => this._relayProvenance(jobId, request) : void 0,
8182
9672
  onArtifactRequest: this._requestUserArtifact ? async (request) => this._requestUserArtifact(jobId, request) : void 0,
@@ -8213,7 +9703,7 @@ var init_job_runner = __esmMin((() => {
8213
9703
  }
8214
9704
  if (result.status === ExecutionJobStatus.enum.failed) {
8215
9705
  const stepErrors = result.stepResults.filter((r) => r.error).map((r) => `${r.name}: ${r.error.message}`).join(" | ");
8216
- logger$2.error("Sandbox returned failed result", {
9706
+ logger$3.error("Sandbox returned failed result", {
8217
9707
  durationMs: result.durationMs,
8218
9708
  stepCount: result.stepResults.length,
8219
9709
  steps: result.stepResults.map((r) => `${r.name}:${r.status}`).join(","),
@@ -8249,7 +9739,7 @@ var init_job_runner = __esmMin((() => {
8249
9739
  async handleBringupJob(dispatch) {
8250
9740
  const { runId, jobId, jobConfig } = dispatch;
8251
9741
  const targetAgentId = String(jobConfig.bringupTarget ?? "");
8252
- logger$2.info("Starting bring-up job", {
9742
+ logger$3.info("Starting bring-up job", {
8253
9743
  jobId,
8254
9744
  runId,
8255
9745
  targetAgentId
@@ -8286,7 +9776,7 @@ var init_job_runner = __esmMin((() => {
8286
9776
  this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.failed, void 0, streamer.getTotalBytes());
8287
9777
  streamer.destroy();
8288
9778
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, { error: message });
8289
- logger$2.warn("Bring-up job failed", {
9779
+ logger$3.warn("Bring-up job failed", {
8290
9780
  jobId,
8291
9781
  runId,
8292
9782
  targetAgentId,
@@ -8304,7 +9794,7 @@ var init_job_runner = __esmMin((() => {
8304
9794
  async handleBuildJob(dispatch, workDir, abortController) {
8305
9795
  const { runId, jobId, jobConfig } = dispatch;
8306
9796
  const buildCtx = getRequestContext();
8307
- logger$2.info(`Run: ${buildCtx.runId ?? runId} | Trace: ${buildCtx.requestId ?? "N/A"}`);
9797
+ logger$3.info(`Run: ${buildCtx.runId ?? runId} | Trace: ${buildCtx.requestId ?? "N/A"}`);
8308
9798
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
8309
9799
  const buildStreamer = this.createStepStreamer(dispatch, 0);
8310
9800
  const buildLog = (msg) => buildStreamer.addLine(msg);
@@ -8393,14 +9883,14 @@ var init_job_runner = __esmMin((() => {
8393
9883
  const cliPublicKey = jobConfig.cliPublicKey;
8394
9884
  const orchestratorPrivateKey = jobConfig.orchestratorPrivateKey;
8395
9885
  if (tarballUrl && cliPublicKey && orchestratorPrivateKey) {
8396
- logger$2.info("Applying overlay tarball for test run", { jobId });
9886
+ logger$3.info("Applying overlay tarball for test run", { jobId });
8397
9887
  const overlayResult = await applyOverlay({
8398
9888
  tarballUrl,
8399
9889
  cliPublicKey,
8400
9890
  orchestratorPrivateKey,
8401
9891
  repoDir: workDir
8402
9892
  });
8403
- logger$2.info("Overlay applied", {
9893
+ logger$3.info("Overlay applied", {
8404
9894
  filesApplied: overlayResult.filesApplied,
8405
9895
  filesDeleted: overlayResult.filesDeleted
8406
9896
  });
@@ -8446,7 +9936,7 @@ var init_job_runner = __esmMin((() => {
8446
9936
  orchestratorPrivateKey: initJobConfig.orchestratorPrivateKey,
8447
9937
  repoDir: workDir
8448
9938
  });
8449
- logger$2.info("Init job: overlay applied", {
9939
+ logger$3.info("Init job: overlay applied", {
8450
9940
  jobId: dispatch.jobId,
8451
9941
  filesApplied: overlayResult.filesApplied,
8452
9942
  filesDeleted: overlayResult.filesDeleted
@@ -8475,9 +9965,12 @@ var init_job_runner = __esmMin((() => {
8475
9965
  platform: os.platform(),
8476
9966
  arch: os.arch()
8477
9967
  };
8478
- logger$2.info("Requesting dep upload URL from orchestrator", { lockfileHash: buildConfig.lockfileHash });
8479
- const depUploadUrl = await this.requestUploadUrl(dispatch.jobId, "deps", depKey);
8480
- logger$2.info("Uploading dep tarball to S3", {
9968
+ logger$3.info("Requesting dep upload URL from orchestrator", { lockfileHash: buildConfig.lockfileHash });
9969
+ const depUploadUrl = await this.requestUploadUrl(dispatch.jobId, "deps", {
9970
+ ...depKey,
9971
+ depsHash: hash
9972
+ });
9973
+ logger$3.info("Uploading dep tarball to S3", {
8481
9974
  size: tarball.length,
8482
9975
  hash: hash.slice(0, 12)
8483
9976
  });
@@ -8486,7 +9979,7 @@ var init_job_runner = __esmMin((() => {
8486
9979
  ...depKey,
8487
9980
  depsHash: hash
8488
9981
  });
8489
- logger$2.info("Dep tarball upload complete", { lockfileHash: buildConfig.lockfileHash });
9982
+ logger$3.info("Dep tarball upload complete", { lockfileHash: buildConfig.lockfileHash });
8490
9983
  buildLog(`Deps tarball uploaded (${tarball.length} bytes)`);
8491
9984
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running, {
8492
9985
  buildEvent: "deps_packed",
@@ -8512,15 +10005,15 @@ var init_job_runner = __esmMin((() => {
8512
10005
  platform: os.platform(),
8513
10006
  arch: os.arch()
8514
10007
  };
8515
- logger$2.info("Requesting source tarball upload URL from orchestrator", { contentHash: buildConfig.contentHash });
10008
+ logger$3.info("Requesting source tarball upload URL from orchestrator", { contentHash: buildConfig.contentHash });
8516
10009
  const sourceUploadUrl = await this.requestUploadUrl(dispatch.jobId, "source", sourceKey);
8517
- logger$2.info("Uploading source tarball to S3", {
10010
+ logger$3.info("Uploading source tarball to S3", {
8518
10011
  size: tarball.length,
8519
10012
  contentHash: buildConfig.contentHash
8520
10013
  });
8521
10014
  await uploadToPresignedUrl(sourceUploadUrl, tarball);
8522
10015
  this.sendUploadComplete(dispatch.jobId, "source", sourceKey);
8523
- logger$2.info("Source tarball upload complete", { contentHash: buildConfig.contentHash });
10016
+ logger$3.info("Source tarball upload complete", { contentHash: buildConfig.contentHash });
8524
10017
  buildLog(`Source tarball packed and uploaded (${tarball.length} bytes, hash: ${buildConfig.contentHash.slice(0, 12)})`);
8525
10018
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running, {
8526
10019
  buildEvent: "source_packed",
@@ -8599,9 +10092,9 @@ var init_job_runner = __esmMin((() => {
8599
10092
  const config = jobConfig;
8600
10093
  const workflowDir = join(workDir, "workflow");
8601
10094
  const sourceDir = join(workDir, "source");
8602
- logger$2.info("Starting global eval round", {
10095
+ logger$3.info("Starting global eval round", {
8603
10096
  jobId,
8604
- candidateCount: config.candidates.length,
10097
+ candidateCount: Array.isArray(config.candidates) ? config.candidates.length : 0,
8605
10098
  workflowRepoIdentifier: config.workflowRepoIdentifier
8606
10099
  });
8607
10100
  this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
@@ -8629,6 +10122,7 @@ var init_job_runner = __esmMin((() => {
8629
10122
  });
8630
10123
  }, this.config.jobHeartbeatIntervalMs);
8631
10124
  try {
10125
+ if (!Array.isArray(config.candidates)) throw new Error("Global eval round dispatch is malformed: `candidates` must be an array");
8632
10126
  if (abortController.signal.aborted) {
8633
10127
  await closeStreamer();
8634
10128
  this.sendStepStatus(dispatch, 0, "global-eval", ExecutionStepStatus.enum.skipped, void 0, evalStreamer.getTotalBytes());
@@ -8660,7 +10154,7 @@ var init_job_runner = __esmMin((() => {
8660
10154
  }));
8661
10155
  const running = roundResult.candidates.filter((c) => c.run).length;
8662
10156
  const indeterminate = roundResult.candidates.filter((c) => c.indeterminate).length;
8663
- logger$2.info("Global eval round completed", {
10157
+ logger$3.info("Global eval round completed", {
8664
10158
  jobId,
8665
10159
  candidateCount: roundResult.candidates.length,
8666
10160
  running,
@@ -8676,7 +10170,7 @@ var init_job_runner = __esmMin((() => {
8676
10170
  });
8677
10171
  } catch (err) {
8678
10172
  const errorMsg = toErrorMessage(err);
8679
- logger$2.error("Global eval round failed", {
10173
+ logger$3.error("Global eval round failed", {
8680
10174
  jobId,
8681
10175
  error: errorMsg
8682
10176
  });
@@ -8706,7 +10200,7 @@ var init_job_runner = __esmMin((() => {
8706
10200
  async handleInitJob(dispatch, workDir, abortController) {
8707
10201
  const { runId, jobId, jobConfig } = dispatch;
8708
10202
  const config = jobConfig;
8709
- logger$2.info("Starting init job", {
10203
+ logger$3.info("Starting init job", {
8710
10204
  jobId,
8711
10205
  targetJobName: config.targetJobName,
8712
10206
  workflowName: config.workflowName
@@ -8750,7 +10244,7 @@ var init_job_runner = __esmMin((() => {
8750
10244
  await this.materializeInitJobSource(dispatch, workDir, initLog);
8751
10245
  const kiciDir = join(workDir, ".kici");
8752
10246
  const hasPackage = await fileExists(join(kiciDir, "package.json"));
8753
- logger$2.info("Init job: checking deps", {
10247
+ logger$3.info("Init job: checking deps", {
8754
10248
  kiciDir,
8755
10249
  hasPackageJson: hasPackage,
8756
10250
  source: config.source
@@ -8776,7 +10270,7 @@ var init_job_runner = __esmMin((() => {
8776
10270
  hasFilter: config.hasFilter ?? false
8777
10271
  }, config.timeoutMs, filterInput);
8778
10272
  });
8779
- logger$2.info("Init job completed successfully", {
10273
+ logger$3.info("Init job completed successfully", {
8780
10274
  jobId,
8781
10275
  hasContext: initResult.contextNames !== void 0,
8782
10276
  hasEnv: initResult.env !== void 0,
@@ -8793,7 +10287,7 @@ var init_job_runner = __esmMin((() => {
8793
10287
  });
8794
10288
  } catch (err) {
8795
10289
  const errorMsg = toErrorMessage(err);
8796
- logger$2.error("Init job failed", {
10290
+ logger$3.error("Init job failed", {
8797
10291
  jobId,
8798
10292
  error: errorMsg
8799
10293
  });
@@ -8819,7 +10313,7 @@ var init_job_runner = __esmMin((() => {
8819
10313
  const { runId, jobId, jobConfig } = dispatch;
8820
10314
  const config = jobConfig;
8821
10315
  const timeoutMs = config.timeoutMs ?? 12e4;
8822
- logger$2.info("Starting DynamicJobFn evaluation", {
10316
+ logger$3.info("Starting DynamicJobFn evaluation", {
8823
10317
  jobId,
8824
10318
  workflowName: config.workflowName,
8825
10319
  sourceIndex: config.source.index
@@ -8843,14 +10337,7 @@ var init_job_runner = __esmMin((() => {
8843
10337
  return;
8844
10338
  }
8845
10339
  await materializeEvalWorkspace(dispatch, workDir, evalLog);
8846
- const { $: zx$ } = await import("zx");
8847
- const scopedDollar = zx$({
8848
- cwd: workDir,
8849
- env: { ...process.env },
8850
- verbose: true,
8851
- quiet: false,
8852
- log: makeStreamingZxLog((line, stream) => evalStreamer.addLine(line, stream))
8853
- });
10340
+ const scopedDollar = await buildEvalShell(workDir, (line, stream) => evalStreamer.addLine(line, stream));
8854
10341
  const evalSink = { addLine: (line) => evalStreamer.addLine(line) };
8855
10342
  const filterInput = config.hasFilter ? await buildInitFilterInput(dispatch, config.event, workDir, (line, stream) => evalStreamer.addLine(line, stream)) : void 0;
8856
10343
  const dynamicJobLogger = {
@@ -8891,7 +10378,7 @@ var init_job_runner = __esmMin((() => {
8891
10378
  workflowName: config.workflowName
8892
10379
  });
8893
10380
  });
8894
- logger$2.info("DynamicJobFn evaluation completed", {
10381
+ logger$3.info("DynamicJobFn evaluation completed", {
8895
10382
  jobId,
8896
10383
  generatedJobCount: lockJobs.length,
8897
10384
  jobNames: lockJobs.map((j) => j.name)
@@ -8906,7 +10393,7 @@ var init_job_runner = __esmMin((() => {
8906
10393
  });
8907
10394
  } catch (err) {
8908
10395
  const errorMsg = toErrorMessage(err);
8909
- logger$2.error("DynamicJobFn evaluation failed", {
10396
+ logger$3.error("DynamicJobFn evaluation failed", {
8910
10397
  jobId,
8911
10398
  error: errorMsg
8912
10399
  });
@@ -8930,13 +10417,50 @@ var init_job_runner = __esmMin((() => {
8930
10417
  }
8931
10418
  }
8932
10419
  /**
10420
+ * Build the job's container image when it declared a Dockerfile, and return
10421
+ * the tag the sandbox must run.
10422
+ *
10423
+ * Returns `undefined` for every other job — one with no container, or one
10424
+ * naming a finalized image — which is the common case and costs nothing.
10425
+ *
10426
+ * Extracted from `setupSandboxForExecution` so that function stays inside the
10427
+ * 200-line ceiling, and so the build's streamer lifecycle is visibly bounded
10428
+ * by one `finally`.
10429
+ */
10430
+ async buildJobImageIfDeclared(dispatch, jobConfig, workDir, abortController) {
10431
+ const container = jobConfig.container;
10432
+ if (!container || typeof container === "string" || !container.dockerfile) return void 0;
10433
+ const streamer = this.createStepStreamer(dispatch, CONTAINER_BUILD_STEP_INDEX);
10434
+ try {
10435
+ return await runJobImageBuild({
10436
+ container,
10437
+ workDir,
10438
+ jobId: dispatch.jobId,
10439
+ jobName: jobConfig.name ?? jobConfig.baseJobName ?? "job",
10440
+ onLog: (line) => streamer.addLine(line),
10441
+ build: async (spec, onLog) => buildJobImage({
10442
+ spec,
10443
+ cli: resolveBuildCli({ configured: this.config.containerBuildCli }),
10444
+ socketPath: sandboxSocketPath(),
10445
+ ...dispatch.containerRegistryAuth ? { authconfig: dispatch.containerRegistryAuth } : {},
10446
+ onLog,
10447
+ signal: abortController.signal
10448
+ }),
10449
+ sendStepStatus: (name, state, data) => this.sendStepStatus(dispatch, CONTAINER_BUILD_STEP_INDEX, name, state, data, streamer.getTotalBytes())
10450
+ });
10451
+ } finally {
10452
+ await streamer.flush();
10453
+ streamer.destroy();
10454
+ }
10455
+ }
10456
+ /**
8933
10457
  * Create the appropriate sandbox backend based on execution mode.
8934
10458
  */
8935
10459
  createSandbox(mode, opts) {
8936
10460
  switch (mode) {
8937
10461
  case "container": {
8938
10462
  const containerConfig = opts.jobConfig.container;
8939
- const image = typeof containerConfig === "string" ? containerConfig : containerConfig?.image ?? "node:20-alpine";
10463
+ const image = opts.builtImage ?? (typeof containerConfig === "string" ? containerConfig : containerConfig?.image ?? "node:20-alpine");
8940
10464
  return new ContainerSandbox({
8941
10465
  docker: new Docker(),
8942
10466
  image,
@@ -8953,7 +10477,11 @@ var init_job_runner = __esmMin((() => {
8953
10477
  nanoCpus: this.config.sandboxNanoCpus,
8954
10478
  networkMode: this.config.sandboxNetwork === "host" ? "host" : "default",
8955
10479
  grant: opts.jobConfig.sandboxGrant
8956
- }
10480
+ },
10481
+ ...opts.registryAuth ? { registryAuth: opts.registryAuth } : {},
10482
+ ...opts.builtImage ? { buildTag: opts.builtImage } : {},
10483
+ ...this.config.runtimeNodeSource ? { runtimeNodePath: this.config.runtimeNodeSource } : {},
10484
+ ...this.config.runtimeImage ? { runtimeImage: this.config.runtimeImage } : {}
8957
10485
  });
8958
10486
  }
8959
10487
  case "firecracker": return new FirecrackerSandbox({
@@ -8964,7 +10492,8 @@ var init_job_runner = __esmMin((() => {
8964
10492
  runnerPath: opts.runnerPath,
8965
10493
  env: opts.env,
8966
10494
  sandbox: this.config.sandbox,
8967
- sandboxNetwork: this.config.sandboxNetwork
10495
+ sandboxNetwork: this.config.sandboxNetwork,
10496
+ orphanCleanup: this.config.orphanCleanup
8968
10497
  });
8969
10498
  }
8970
10499
  }
@@ -9087,6 +10616,168 @@ var init_job_runner = __esmMin((() => {
9087
10616
  };
9088
10617
  }));
9089
10618
  //#endregion
10619
+ //#region src/execution/between-jobs-reset.ts
10620
+ /**
10621
+ * Run the operator between-jobs reset command. Fail-open: never throws, always
10622
+ * resolves to a status. Skips when unconfigured, or when runOn=on-failure and
10623
+ * the job succeeded.
10624
+ */
10625
+ async function runBetweenJobsReset(input) {
10626
+ const started = performance.now();
10627
+ const done = (status) => ({
10628
+ status,
10629
+ durationMs: Math.round(performance.now() - started)
10630
+ });
10631
+ if (!input.command) return done("skipped");
10632
+ if (input.runOn === "on-failure" && !input.jobFailed) return done("skipped");
10633
+ const exec = input.exec ?? defaultExec;
10634
+ try {
10635
+ const r = await exec(input.command, input.timeoutMs);
10636
+ if (r.timedOut) return done("timeout");
10637
+ return done(r.code === 0 ? "success" : "failed");
10638
+ } catch {
10639
+ return done("failed");
10640
+ }
10641
+ }
10642
+ var defaultExec;
10643
+ var init_between_jobs_reset = __esmMin((() => {
10644
+ defaultExec = (command, timeoutMs) => new Promise((resolve) => {
10645
+ const child = spawn("/bin/sh", ["-c", command], { stdio: "ignore" });
10646
+ let timedOut = false;
10647
+ const timer = setTimeout(() => {
10648
+ timedOut = true;
10649
+ child.kill("SIGKILL");
10650
+ }, timeoutMs);
10651
+ child.on("exit", (code) => {
10652
+ clearTimeout(timer);
10653
+ resolve({
10654
+ code,
10655
+ timedOut
10656
+ });
10657
+ });
10658
+ child.on("error", () => {
10659
+ clearTimeout(timer);
10660
+ resolve({
10661
+ code: 1,
10662
+ timedOut
10663
+ });
10664
+ });
10665
+ });
10666
+ }));
10667
+ //#endregion
10668
+ //#region src/execution/cleanup-rerun.ts
10669
+ /**
10670
+ * Re-run a hard-killed job's declared cleanup/onFailure hooks out-of-band,
10671
+ * against its preserved workdir, in a fresh bounded child. Returns 'skipped'
10672
+ * when there is nothing to do (no declared cleanup, wrong backend, or a
10673
+ * non-positive timeout). Never throws — a failure is reported as a status.
10674
+ */
10675
+ async function runDeclaredCleanupOutOfBand(input) {
10676
+ const started = performance.now();
10677
+ const done = (status) => ({
10678
+ status,
10679
+ durationMs: Math.round(performance.now() - started)
10680
+ });
10681
+ if (!input.declaresCleanup) return done("skipped");
10682
+ if (!HOST_SHARING_BACKENDS.has(input.backend)) return done("skipped");
10683
+ if (!Number.isFinite(input.timeoutMs) || input.timeoutMs <= 0) return done("skipped");
10684
+ const controller = new AbortController();
10685
+ const timer = setTimeout(() => controller.abort(), input.timeoutMs);
10686
+ try {
10687
+ await input.spawn(input.workDir, controller.signal);
10688
+ return done("success");
10689
+ } catch {
10690
+ return done(controller.signal.aborted ? "timeout" : "failed");
10691
+ } finally {
10692
+ clearTimeout(timer);
10693
+ }
10694
+ }
10695
+ var HOST_SHARING_BACKENDS;
10696
+ var init_cleanup_rerun = __esmMin((() => {
10697
+ HOST_SHARING_BACKENDS = /* @__PURE__ */ new Set(["bare-metal"]);
10698
+ }));
10699
+ //#endregion
10700
+ //#region src/execution/between-jobs-controller.ts
10701
+ var between_jobs_controller_exports = /* @__PURE__ */ __exportAll({ BetweenJobsController: () => BetweenJobsController$1 });
10702
+ var logger$2, BetweenJobsController$1;
10703
+ var init_between_jobs_controller = __esmMin((() => {
10704
+ init_between_jobs_reset();
10705
+ init_cleanup_rerun();
10706
+ init_prometheus();
10707
+ logger$2 = createLogger({ prefix: "between-jobs" });
10708
+ BetweenJobsController$1 = class {
10709
+ deps;
10710
+ _consecutiveResetFailures = 0;
10711
+ /** Consecutive between-jobs reset failures, for the supervisor's drain gate. */
10712
+ get consecutiveResetFailures() {
10713
+ return this._consecutiveResetFailures;
10714
+ }
10715
+ constructor(deps) {
10716
+ this.deps = deps;
10717
+ }
10718
+ async runRerunPhase(ctx) {
10719
+ if (ctx.completionHooksRan || !ctx.cleanupSpawn) return "skipped";
10720
+ const r = await (this.deps.rerun ?? runDeclaredCleanupOutOfBand)({
10721
+ workDir: ctx.workDir,
10722
+ backend: ctx.backend,
10723
+ declaresCleanup: ctx.declaresCleanup,
10724
+ timeoutMs: this.deps.config.betweenJobsResetTimeoutMs,
10725
+ spawn: ctx.cleanupSpawn
10726
+ });
10727
+ orphanCleanupTotal.add(1, { status: r.status });
10728
+ if (r.status === "failed" || r.status === "timeout") logger$2.warn("between-jobs out-of-band cleanup did not complete", { status: r.status });
10729
+ return r.status;
10730
+ }
10731
+ async runResetPhase(ctx) {
10732
+ const reset = await (this.deps.reset ?? runBetweenJobsReset)({
10733
+ command: this.deps.config.betweenJobsResetCommand,
10734
+ timeoutMs: this.deps.config.betweenJobsResetTimeoutMs,
10735
+ runOn: this.deps.config.betweenJobsResetRunOn,
10736
+ jobFailed: ctx.jobFailed
10737
+ });
10738
+ if (reset.status !== "skipped") {
10739
+ betweenJobsResetTotal.add(1, { status: reset.status });
10740
+ betweenJobsResetDurationSeconds.record(reset.durationMs / 1e3);
10741
+ }
10742
+ if (reset.status === "failed" || reset.status === "timeout") {
10743
+ this._consecutiveResetFailures += 1;
10744
+ logger$2.warn("between-jobs reset failed", {
10745
+ status: reset.status,
10746
+ consecutive: this._consecutiveResetFailures
10747
+ });
10748
+ } else if (reset.status === "success") this._consecutiveResetFailures = 0;
10749
+ return reset.status;
10750
+ }
10751
+ async afterJob(ctx) {
10752
+ let rerun = "skipped";
10753
+ try {
10754
+ rerun = await this.runRerunPhase(ctx);
10755
+ } catch (err) {
10756
+ logger$2.warn("between-jobs out-of-band cleanup threw", { error: toErrorMessage(err) });
10757
+ }
10758
+ let reaped = 0;
10759
+ if (this.deps.config.orphanCleanup) try {
10760
+ reaped = await ctx.reap();
10761
+ if (reaped > 0) orphansReapedTotal.add(reaped);
10762
+ } catch (err) {
10763
+ logger$2.warn("between-jobs reap threw", { error: toErrorMessage(err) });
10764
+ }
10765
+ try {
10766
+ await ctx.deleteWorkdir();
10767
+ } catch (err) {
10768
+ logger$2.warn("between-jobs workdir delete threw", { error: toErrorMessage(err) });
10769
+ }
10770
+ const reset = await this.runResetPhase(ctx);
10771
+ return {
10772
+ rerun,
10773
+ reaped,
10774
+ reset,
10775
+ consecutiveResetFailures: this._consecutiveResetFailures
10776
+ };
10777
+ }
10778
+ };
10779
+ }));
10780
+ //#endregion
9090
10781
  //#region src/server.ts
9091
10782
  /**
9092
10783
  * Agent entry point.
@@ -9108,23 +10799,30 @@ var init_job_runner = __esmMin((() => {
9108
10799
  */
9109
10800
  init_console_capture();
9110
10801
  init_npm_resolver();
9111
- const AGENT_VERSION = "0.5.0";
9112
- const BUILD_COMMIT = "cb51c7d1e";
9113
- const SDK_VERSION = "0.5.0";
9114
- const SDK_BUNDLE_HASH = "5b85e7cffa4a08e39329ff80448a2744af8e5b5f9840ec3801fad45ed11a3de9";
9115
- const SHARED_VERSION = "0.5.0";
9116
- const SHARED_BUNDLE_HASH = "b6b6d1818d1fe648150422daf073dafc18b037404cacf82e0ec03b4a2863e4c4";
9117
- const ENGINE_VERSION = "0.5.0";
9118
- const ENGINE_BUNDLE_HASH = "35d174ce6f4748abffcad194b335b508cade5d1c8af16d9cd3bdcb744f38717e";
10802
+ const AGENT_VERSION = "0.6.0";
10803
+ const BUILD_COMMIT = "e4a936029";
10804
+ const SDK_VERSION = "0.6.0";
10805
+ const SDK_BUNDLE_HASH = "22faf0da45de7c243ce87f80fd33ee51b1df52809fde457b16bb821678f65eb3";
10806
+ const SHARED_VERSION = "0.6.0";
10807
+ const SHARED_BUNDLE_HASH = "a79be949815735b9e36eecc716f7eb07aa36c7bffba62e798e7e19f31a474eff";
10808
+ const ENGINE_VERSION = "0.6.0";
10809
+ const ENGINE_BUNDLE_HASH = "e49533b19f122155b3a78d95d105f97da259c2a45dee579578324b0702c91708";
9119
10810
  initTelemetry({
9120
10811
  serviceName: "kici-agent",
9121
10812
  otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
9122
10813
  });
9123
10814
  const { connectionStatus, jobsActive, jobsTotal } = await Promise.resolve().then(() => (init_prometheus(), prometheus_exports));
9124
10815
  const { JobRunner } = await Promise.resolve().then(() => (init_job_runner(), job_runner_exports));
10816
+ const { BetweenJobsController } = await Promise.resolve().then(() => (init_between_jobs_controller(), between_jobs_controller_exports));
9125
10817
  setServiceName("agent");
9126
10818
  const logger$1 = createLogger({ prefix: "agent" });
9127
10819
  /**
10820
+ * Consecutive between-jobs reset failures after which a `drainOnResetFailure`
10821
+ * agent drains — a persistently dirty host stops taking work rather than
10822
+ * running every subsequent job against unclean state.
10823
+ */
10824
+ const RESET_FAILURE_DRAIN_THRESHOLD = 3;
10825
+ /**
9128
10826
  * Serialize the agent's current Prometheus metrics to text. The OTel
9129
10827
  * PrometheusExporter exposes no direct serialize method, so the metrics are
9130
10828
  * piped through its request handler with a mock ServerResponse. Returns an
@@ -9174,7 +10872,10 @@ await guardStartup(logger$1, async () => {
9174
10872
  name: "bash",
9175
10873
  reason: "required for step execution"
9176
10874
  }]);
9177
- if (toolErrors.length > 0) throw new Error("Agent required-tools validation failed:\n" + toolErrors.map((e) => ` - ${e}`).join("\n"));
10875
+ if (toolErrors.length > 0) {
10876
+ const jobImageHint = config.jobImageAgent ? "\nThis agent runs inside the job's own container image, so that image must provide these tools. Either add them to the image, or run the job on a pool whose agent stays outside it." : "";
10877
+ throw new Error("Agent required-tools validation failed:\n" + toolErrors.map((e) => ` - ${e}`).join("\n") + jobImageHint);
10878
+ }
9178
10879
  if (config.roles === void 0 || config.roles.includes("builder")) {
9179
10880
  const npmVersion = verifyNpmAvailable();
9180
10881
  logger$1.info("Builder role: npm verified", { npmVersion });
@@ -9183,9 +10884,17 @@ await guardStartup(logger$1, async () => {
9183
10884
  let idleShutdownTimer;
9184
10885
  let rebootIntent = false;
9185
10886
  let client;
10887
+ const betweenJobsController = new BetweenJobsController({ config: {
10888
+ betweenJobsResetCommand: config.betweenJobsResetCommand,
10889
+ betweenJobsResetTimeoutMs: config.betweenJobsResetTimeoutMs,
10890
+ betweenJobsResetRunOn: config.betweenJobsResetRunOn,
10891
+ orphanCleanup: config.orphanCleanup,
10892
+ drainOnResetFailure: config.drainOnResetFailure
10893
+ } });
9186
10894
  const jobRunner = new JobRunner({
9187
10895
  send: (msg) => client.send(msg),
9188
10896
  config,
10897
+ betweenJobsController,
9189
10898
  requestUploadUrl: (jobId, cacheType, key) => client.requestUploadUrl(jobId, cacheType, key),
9190
10899
  sendUploadComplete: (jobId, cacheType, key) => client.sendUploadComplete(jobId, cacheType, key),
9191
10900
  sendEventEmit: (jobId, requestId, eventName, payload, target) => client.sendEventEmit(jobId, requestId, eventName, payload, target),
@@ -9201,6 +10910,12 @@ await guardStartup(logger$1, async () => {
9201
10910
  rebootIntent = true;
9202
10911
  return result;
9203
10912
  }
10913
+ if (method === "scaler.claim-credentials") {
10914
+ const claimCode = String((params ?? {}).claimCode ?? "");
10915
+ const r = await client.sendClaimCredentials(claimCode);
10916
+ if (r.error) throw new Error(r.error);
10917
+ return r.credentials;
10918
+ }
9204
10919
  return client.sendApiRequest(method, params ?? {});
9205
10920
  },
9206
10921
  requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
@@ -9215,13 +10930,14 @@ await guardStartup(logger$1, async () => {
9215
10930
  messageId: randomUUID(),
9216
10931
  agentId: config.agentId,
9217
10932
  activeJobs: jobRunner.activeJobs.size,
9218
- memoryUsedMb: Math.round((os.totalmem() - os.freemem()) / (1024 * 1024)),
9219
- memoryAvailableMb: Math.round(os.freemem() / (1024 * 1024)),
10933
+ memoryUsedMb: Math.round((os.totalmem() - os.freemem()) / 1048576),
10934
+ memoryAvailableMb: Math.round(os.freemem() / 1048576),
9220
10935
  uptimeSeconds: Math.round(os.uptime())
9221
10936
  });
9222
10937
  }
9223
10938
  client = new OrchestratorClient({
9224
10939
  ...agentClientConnectionOptions(config),
10940
+ scalerClaimCode: config.scalerClaimCode,
9225
10941
  getFleetBundleInputs: async () => ({
9226
10942
  config,
9227
10943
  logDir: process.env.KICI_LOG_DIR,
@@ -9291,6 +11007,10 @@ await guardStartup(logger$1, async () => {
9291
11007
  }).finally(() => {
9292
11008
  jobsActive.add(-1);
9293
11009
  metricsReporter.collectAndSend().catch(() => {});
11010
+ if (config.drainOnResetFailure && !isDraining && betweenJobsController.consecutiveResetFailures >= RESET_FAILURE_DRAIN_THRESHOLD) {
11011
+ logger$1.warn("Draining agent after repeated between-jobs reset failures", { consecutive: betweenJobsController.consecutiveResetFailures });
11012
+ isDraining = true;
11013
+ }
9294
11014
  sendAgentStatus();
9295
11015
  if (rebootIntent) {
9296
11016
  rebootIntent = false;
@@ -9340,9 +11060,23 @@ await guardStartup(logger$1, async () => {
9340
11060
  }, idleMs);
9341
11061
  }
9342
11062
  }
9343
- client.onRegistered = ({ pendingDispatch }) => {
9344
- if (!config.scalerManaged || jobRunner.activeJobs.size > 0) return;
9345
- if (pendingDispatch) {
11063
+ client.onRegistered = ({ pendingDispatch, warmPool }) => {
11064
+ const decision = decideIdleShutdown({
11065
+ scalerManaged: config.scalerManaged,
11066
+ activeJobs: jobRunner.activeJobs.size,
11067
+ pendingDispatch,
11068
+ warmPool
11069
+ });
11070
+ if (decision === "none") return;
11071
+ if (decision === "warm") {
11072
+ logger$1.info("Warm-pool agent registered and ready, awaiting work");
11073
+ if (idleShutdownTimer) {
11074
+ clearTimeout(idleShutdownTimer);
11075
+ idleShutdownTimer = void 0;
11076
+ }
11077
+ return;
11078
+ }
11079
+ if (decision === "pending-dispatch") {
9346
11080
  const safetyMs = config.scalerPendingDispatchTimeoutMs;
9347
11081
  logger$1.info(`Scaler-managed agent registered with pending bound dispatch, deferring idle shutdown for ${safetyMs}ms`);
9348
11082
  if (idleShutdownTimer) clearTimeout(idleShutdownTimer);
@@ -9453,6 +11187,10 @@ await guardStartup(logger$1, async () => {
9453
11187
  }
9454
11188
  ]
9455
11189
  });
11190
+ client.onClaimFailedPermanently = (reason) => {
11191
+ logger$1.error("Self-bootstrap claim failed permanently, shutting down", { reason });
11192
+ gracefulShutdown("scaler-claim-failed", 1);
11193
+ };
9456
11194
  process.on("SIGUSR1", () => {
9457
11195
  logger$1.info("Received SIGUSR1, entering drain mode");
9458
11196
  isDraining = true;