@kici-dev/agent 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checkout/clone-job-repos.d.ts +53 -0
- package/dist/checkout/credential-helper-bin.d.ts +28 -0
- package/dist/checkout/credential-helper-host.d.ts +48 -0
- package/dist/checkout/credential-helper.d.ts +44 -0
- package/dist/checkout/git-clone.d.ts +26 -0
- package/dist/checkout/grant-table.d.ts +30 -0
- package/dist/checkout/job-git-credentials.d.ts +49 -0
- package/dist/checkout/write-elevation.d.ts +40 -0
- package/dist/config.d.ts +38 -0
- package/dist/container-ts-loader-hook.js +2047 -1814
- package/dist/execution/between-jobs-controller.d.ts +50 -0
- package/dist/execution/between-jobs-reset.d.ts +25 -0
- package/dist/execution/cleanup-rerun.d.ts +21 -0
- package/dist/execution/dynamic-job-serializer.d.ts +6 -2
- package/dist/execution/image-build/build-engine.d.ts +75 -0
- package/dist/execution/image-build/build-step.d.ts +57 -0
- package/dist/execution/image-build/resolve-build-spec.d.ts +41 -0
- package/dist/execution/image-build/runtime-facts.d.ts +31 -0
- package/dist/execution/job-runner.d.ts +48 -0
- package/dist/execution/sandbox/bare-metal-sandbox.d.ts +25 -1
- package/dist/execution/sandbox/container-sandbox.d.ts +97 -2
- package/dist/execution/sandbox/fork-runner.d.ts +55 -1
- package/dist/execution/sandbox/image-preflight.d.ts +38 -0
- package/dist/execution/sandbox/ipc-protocol.d.ts +96 -2
- package/dist/execution/sandbox/kici-runtime.d.ts +48 -0
- package/dist/execution/sandbox/step-loop.d.ts +7 -0
- package/dist/execution/sandbox/types.d.ts +55 -1
- package/dist/execution/sandbox/workflow-runner.d.ts +23 -3
- package/dist/idle-shutdown.d.ts +24 -0
- package/dist/index.js +133 -62
- package/dist/metrics/prometheus.d.ts +26 -0
- package/dist/server.js +2050 -312
- package/dist/workflow-runner-bundle.js +41961 -41319
- package/dist/workflow-runner.js +332 -136
- package/dist/ws/orchestrator-client.d.ts +95 -3
- package/package.json +10 -10
- package/sbom.spdx.json +460 -455
package/dist/server.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
|
|
2
2
|
import { dirname as __cjs_dirname } from "node:path";
|
|
3
|
-
|
|
3
|
+
const __filename = __cjs_fileURLToPath(import.meta.url);
|
|
4
|
+
__cjs_dirname(__filename);
|
|
4
5
|
import { createRequire, register } from "node:module";
|
|
5
6
|
import crypto$1, { createCipheriv, createHash, createPublicKey, diffieHellman, generateKeyPairSync, hkdfSync, randomBytes, randomUUID } from "node:crypto";
|
|
6
7
|
import * as os$1 from "node:os";
|
|
@@ -12,21 +13,23 @@ import winston from "winston";
|
|
|
12
13
|
import { AgentDeliveryMode, AgentPlatform, RingBuffer, addLogsToArchive, chunkBuffer, createHealthRoutes, createLogger, createMeter, createMetricsRoutes, deriveSharedSecret, getPrometheusExporter, getReconnectDelay, getRequestContext, guardStartup, initTelemetry, kiciTmpBase, logger, normalizeLineEndings, redactConfig, requestContext, setServiceName, setupGracefulShutdown, sha256, sha256File, toErrorMessage, validateRequiredTools } from "@kici-dev/shared";
|
|
13
14
|
import { z } from "zod";
|
|
14
15
|
import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
|
|
15
|
-
import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, SANDBOX_DEFAULT_VARS, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, validateNoReservedLabels } from "@kici-dev/engine";
|
|
16
|
+
import { ALLOWED_SYSTEM_VARS, ArtifactCompleteAckOutcome, CacheRunEventType, CacheStepType, ExecutionJobStatus, ExecutionStepStatus, InitFailureCategory, KNOWN_ROLES, LogStream, MAX_MATRIX_MATERIALIZATION, MatrixShapeError, PROTOCOL_VERSION, RuntimeFact, SANDBOX_DEFAULT_VARS, SetupStepType, WS_CLOSE_AGENT_AUTH_FAILED, WS_MAX_PAYLOAD_BYTES, applyIncludeExclude, buildTrustedPassthroughEnv, deriveOsArchLabels, expandMatrix, hasOrchAgentCapability, heartbeatSchema, hostLabel, matrixCombinationCount, mergeAutoLabels, orchestratorToAgentMessageSchema, parseHostPropertyAssignments, resolveRoleLabels, resolveWhenToRunOn, runtimeLabel, validateNoReservedLabels } from "@kici-dev/engine";
|
|
16
17
|
import { execFile, execFileSync, execSync, fork, spawn } from "node:child_process";
|
|
17
|
-
import
|
|
18
|
+
import fsPromises, { access, chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rm, unlink, writeFile } from "node:fs/promises";
|
|
18
19
|
import * as fs$1 from "node:fs";
|
|
19
20
|
import fs, { createReadStream, createWriteStream, existsSync } from "node:fs";
|
|
21
|
+
import path, { delimiter, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
22
|
+
import WebSocket from "ws";
|
|
20
23
|
import { ZipArchive } from "archiver";
|
|
21
|
-
import path, { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
22
24
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
23
25
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
24
26
|
import { format, promisify } from "node:util";
|
|
25
27
|
import { OTEL_DATA_POINT_TYPE, mapDataPointTypeToWireKind } from "@kici-dev/engine/metrics/metric-kind-compat";
|
|
26
28
|
import { gcStaleTmpDirs } from "@kici-dev/core/tmp-gc";
|
|
27
|
-
import fsPromises, { access, lstat, mkdir, readFile, readdir, realpath, unlink, writeFile } from "node:fs/promises";
|
|
28
29
|
import { makeTempDir } from "@kici-dev/core/tmp";
|
|
29
30
|
import Docker from "dockerode";
|
|
31
|
+
import { GIT_CREDENTIAL_REQUEST_METHOD } from "@kici-dev/engine/protocol/messages/git-credential-relay";
|
|
32
|
+
import { createServer } from "node:net";
|
|
30
33
|
import { buildKiciApi, buildNeedsContext, createFilterContext, isDynamicGroupRef, isDynamicJobFn, isParallelGroup, isStaticArray, isStaticObject, setJobOutputsMap, setStepOutputsMap, setStepRefMap } from "@kici-dev/sdk";
|
|
31
34
|
import { $ } from "zx";
|
|
32
35
|
import { c, x } from "tar";
|
|
@@ -38,6 +41,7 @@ import { normalizeRunsOnToMatchers } from "@kici-dev/engine/labels/compile";
|
|
|
38
41
|
import { PNPM_IGNORE_BUILD_GATE_ARG, PackageManager, YarnFlavor, detectPackageManagerFromManifests, detectYarnFlavor } from "@kici-dev/shared/package-manager";
|
|
39
42
|
import { parse, stringify } from "yaml";
|
|
40
43
|
import { createInterface } from "node:readline";
|
|
44
|
+
import { RuntimeSubtree, ensureRuntimeVolume } from "@kici-dev/shared/container-runtime";
|
|
41
45
|
var __defProp = Object.defineProperty;
|
|
42
46
|
var __esmMin = (fn, res, err) => () => {
|
|
43
47
|
if (err) throw err[0];
|
|
@@ -57,13 +61,258 @@ var __exportAll = (all, no_symbols) => {
|
|
|
57
61
|
return target;
|
|
58
62
|
};
|
|
59
63
|
//#endregion
|
|
64
|
+
//#region src/execution/image-build/build-engine.ts
|
|
65
|
+
/**
|
|
66
|
+
* Build a job's container image with the host's build CLI.
|
|
67
|
+
*
|
|
68
|
+
* The CLI is REQUIRED — there is deliberately no socket-API fallback. One build
|
|
69
|
+
* path means one set of Dockerfile semantics: `.dockerignore`, BuildKit and
|
|
70
|
+
* every directive behave as they do on the author's own machine, instead of
|
|
71
|
+
* depending on which agent happened to pick the job up. A host without a CLI
|
|
72
|
+
* cannot run a Dockerfile job, and says so in as many words.
|
|
73
|
+
*/
|
|
74
|
+
/** Is `bin` executable somewhere on `PATH`? */
|
|
75
|
+
function binaryOnPath(bin) {
|
|
76
|
+
return (process.env.PATH ?? "").split(delimiter).filter(Boolean).some((dir) => existsSync(join(dir, bin)));
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* The container socket the SANDBOX will use.
|
|
80
|
+
*
|
|
81
|
+
* Mirrors what dockerode's `new Docker()` resolves, and exists so build and run
|
|
82
|
+
* provably agree. A host with both runtimes whose sandbox socket points at
|
|
83
|
+
* podman would otherwise build with docker and then start the job container on
|
|
84
|
+
* a daemon that has never heard of that image — a failure that surfaces as
|
|
85
|
+
* "no such image" and names nothing.
|
|
86
|
+
*/
|
|
87
|
+
function sandboxSocketPath() {
|
|
88
|
+
return process.env.DOCKER_HOST ?? "/var/run/docker.sock";
|
|
89
|
+
}
|
|
90
|
+
function resolveBuildCli(args) {
|
|
91
|
+
const { configured } = args;
|
|
92
|
+
const onPath = args.onPath ?? binaryOnPath;
|
|
93
|
+
if (configured) {
|
|
94
|
+
if (!onPath(configured)) throw new Error(`KICI_CONTAINER_BUILD_CLI is set to '${configured}', but '${configured}' is not on PATH on this agent host.`);
|
|
95
|
+
return configured;
|
|
96
|
+
}
|
|
97
|
+
const found = CLI_PREFERENCE.find((bin) => onPath(bin));
|
|
98
|
+
if (!found) throw new Error("This job builds its container image from a Dockerfile, which needs a build CLI on the agent host. Neither 'docker' nor 'podman' is on PATH. Install one, or route the job to a pool whose hosts have one.");
|
|
99
|
+
return found;
|
|
100
|
+
}
|
|
101
|
+
/** Point the CLI at a specific daemon, in that CLI's own spelling. */
|
|
102
|
+
function socketFlags(cli, socketPath) {
|
|
103
|
+
const url = socketPath.includes("://") ? socketPath : `unix://${socketPath}`;
|
|
104
|
+
return cli === ContainerBuildCli.enum.docker ? ["-H", url] : ["--url", url];
|
|
105
|
+
}
|
|
106
|
+
function buildArgv(args) {
|
|
107
|
+
const { cli, spec, socketPath } = args;
|
|
108
|
+
const argv = [];
|
|
109
|
+
if (socketPath) argv.push(...socketFlags(cli, socketPath));
|
|
110
|
+
argv.push("build", "-f", spec.dockerfilePath);
|
|
111
|
+
if (spec.target !== void 0) argv.push("--target", spec.target);
|
|
112
|
+
for (const [k, v] of Object.entries(spec.args)) argv.push("--build-arg", `${k}=${v}`);
|
|
113
|
+
for (const [k, v] of Object.entries(spec.labels)) argv.push("--label", `${k}=${v}`);
|
|
114
|
+
argv.push("-t", spec.tag);
|
|
115
|
+
argv.push(spec.contextDir);
|
|
116
|
+
return argv;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Write `authconfig` into a throwaway config directory and return the env that
|
|
120
|
+
* points the CLI at it.
|
|
121
|
+
*
|
|
122
|
+
* A throwaway directory rather than the host's own credential file: the build
|
|
123
|
+
* runs on a shared agent host, and a job's registry token has no business being
|
|
124
|
+
* written where the next job (or the operator) can read it.
|
|
125
|
+
*/
|
|
126
|
+
async function withTempAuth(cli, authconfig) {
|
|
127
|
+
const dir = await mkdtemp(join(tmpdir(), "kici-build-auth-"));
|
|
128
|
+
const encoded = Buffer.from(`${authconfig.username}:${authconfig.password}`).toString("base64");
|
|
129
|
+
const body = JSON.stringify({ auths: { [authconfig.serveraddress]: { auth: encoded } } });
|
|
130
|
+
if (cli === ContainerBuildCli.enum.docker) {
|
|
131
|
+
await writeFile(join(dir, "config.json"), body, { mode: 384 });
|
|
132
|
+
return {
|
|
133
|
+
env: { DOCKER_CONFIG: dir },
|
|
134
|
+
dispose: () => rm(dir, {
|
|
135
|
+
recursive: true,
|
|
136
|
+
force: true
|
|
137
|
+
})
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
const file = join(dir, "auth.json");
|
|
141
|
+
await writeFile(file, body, { mode: 384 });
|
|
142
|
+
return {
|
|
143
|
+
env: { REGISTRY_AUTH_FILE: file },
|
|
144
|
+
dispose: () => rm(dir, {
|
|
145
|
+
recursive: true,
|
|
146
|
+
force: true
|
|
147
|
+
})
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Split a chunk stream into whole lines, carrying the remainder between chunks.
|
|
152
|
+
*
|
|
153
|
+
* `flush` matters: a builder whose last line has no trailing newline — which is
|
|
154
|
+
* exactly the shape of an error written just before exit — would otherwise leave
|
|
155
|
+
* that line in the carry, and the one line the author most needs would be the
|
|
156
|
+
* one that never reaches the run log.
|
|
157
|
+
*/
|
|
158
|
+
function makeLineSplitter(onLine) {
|
|
159
|
+
let carry = "";
|
|
160
|
+
return {
|
|
161
|
+
write: (chunk) => {
|
|
162
|
+
carry += chunk.toString("utf-8");
|
|
163
|
+
const lines = carry.split("\n");
|
|
164
|
+
carry = lines.pop() ?? "";
|
|
165
|
+
for (const line of lines) onLine(line);
|
|
166
|
+
},
|
|
167
|
+
flush: () => {
|
|
168
|
+
if (carry.length === 0) return;
|
|
169
|
+
onLine(carry);
|
|
170
|
+
carry = "";
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Run the build. Resolves when the image is tagged; rejects with the builder's
|
|
176
|
+
* own last words otherwise.
|
|
177
|
+
*
|
|
178
|
+
* The rejection carries real output rather than an exit code because the author
|
|
179
|
+
* is the one who has to act on it, and "exit 1" tells them nothing about which
|
|
180
|
+
* `RUN` failed.
|
|
181
|
+
*/
|
|
182
|
+
async function buildJobImage(args) {
|
|
183
|
+
const { spec, cli, socketPath, authconfig, onLog, signal } = args;
|
|
184
|
+
if (signal?.aborted) throw signal.reason ?? /* @__PURE__ */ new Error("job image build aborted");
|
|
185
|
+
const auth = authconfig ? await withTempAuth(cli, authconfig) : void 0;
|
|
186
|
+
const argv = buildArgv({
|
|
187
|
+
cli,
|
|
188
|
+
spec,
|
|
189
|
+
...socketPath ? { socketPath } : {}
|
|
190
|
+
});
|
|
191
|
+
try {
|
|
192
|
+
await new Promise((resolvePromise, reject) => {
|
|
193
|
+
const child = spawn(cli, argv, {
|
|
194
|
+
env: {
|
|
195
|
+
...process.env,
|
|
196
|
+
...auth?.env ?? {}
|
|
197
|
+
},
|
|
198
|
+
stdio: [
|
|
199
|
+
"ignore",
|
|
200
|
+
"pipe",
|
|
201
|
+
"pipe"
|
|
202
|
+
]
|
|
203
|
+
});
|
|
204
|
+
const tail = [];
|
|
205
|
+
const record = (line) => {
|
|
206
|
+
onLog(line);
|
|
207
|
+
tail.push(line);
|
|
208
|
+
if (tail.length > FAILURE_TAIL_LINES) tail.shift();
|
|
209
|
+
};
|
|
210
|
+
const outSplitter = makeLineSplitter(record);
|
|
211
|
+
const errSplitter = makeLineSplitter(record);
|
|
212
|
+
child.stdout.on("data", outSplitter.write);
|
|
213
|
+
child.stderr.on("data", errSplitter.write);
|
|
214
|
+
const onAbort = () => child.kill("SIGKILL");
|
|
215
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
216
|
+
child.on("error", (err) => {
|
|
217
|
+
signal?.removeEventListener("abort", onAbort);
|
|
218
|
+
reject(/* @__PURE__ */ new Error(`failed to start '${cli} build': ${err.message}`));
|
|
219
|
+
});
|
|
220
|
+
child.on("close", (code) => {
|
|
221
|
+
signal?.removeEventListener("abort", onAbort);
|
|
222
|
+
outSplitter.flush();
|
|
223
|
+
errSplitter.flush();
|
|
224
|
+
if (code === 0) {
|
|
225
|
+
resolvePromise();
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const detail = tail.length > 0 ? `\n${tail.join("\n")}` : "";
|
|
229
|
+
reject(/* @__PURE__ */ new Error(`'${cli} build' failed for ${spec.dockerfilePath} (exit ${code ?? "signal"})${detail}`));
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
} finally {
|
|
233
|
+
await auth?.dispose();
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
var ContainerBuildCli, CLI_PREFERENCE, FAILURE_TAIL_LINES;
|
|
237
|
+
var init_build_engine = __esmMin((() => {
|
|
238
|
+
ContainerBuildCli = z.enum(["docker", "podman"]);
|
|
239
|
+
CLI_PREFERENCE = [ContainerBuildCli.enum.docker, ContainerBuildCli.enum.podman];
|
|
240
|
+
FAILURE_TAIL_LINES = 20;
|
|
241
|
+
}));
|
|
242
|
+
//#endregion
|
|
60
243
|
//#region src/config.ts
|
|
244
|
+
init_build_engine();
|
|
61
245
|
/** Execution mode for the agent's sandbox backend. Mirrors the runtime enum. */
|
|
62
246
|
const ExecutionMode = z.enum([
|
|
63
247
|
"container",
|
|
64
248
|
"bare-metal",
|
|
65
249
|
"firecracker"
|
|
66
250
|
]);
|
|
251
|
+
/** When the operator between-jobs reset command runs. */
|
|
252
|
+
const BetweenJobsRunOn = z.enum(["always", "on-failure"]);
|
|
253
|
+
const configSchema = z.object({
|
|
254
|
+
orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
|
|
255
|
+
agentId: z.string().optional(),
|
|
256
|
+
labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
|
|
257
|
+
properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
|
|
258
|
+
roles: z.string().optional().transform((s) => {
|
|
259
|
+
if (s === void 0) return void 0;
|
|
260
|
+
if (s === "") return [];
|
|
261
|
+
return s.split(",").filter(Boolean);
|
|
262
|
+
}).refine((roles) => {
|
|
263
|
+
if (roles === void 0) return true;
|
|
264
|
+
const validValues = [...KNOWN_ROLES, "all"];
|
|
265
|
+
return roles.every((r) => validValues.includes(r));
|
|
266
|
+
}, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
|
|
267
|
+
if (roles === void 0) return void 0;
|
|
268
|
+
if (roles.includes("all")) return void 0;
|
|
269
|
+
if (roles.length === 0) return [];
|
|
270
|
+
return roles.filter((r) => r !== "all");
|
|
271
|
+
}),
|
|
272
|
+
port: z.coerce.number().default(8080),
|
|
273
|
+
logLevel: z.enum([
|
|
274
|
+
"debug",
|
|
275
|
+
"info",
|
|
276
|
+
"warn",
|
|
277
|
+
"error"
|
|
278
|
+
]).default("info"),
|
|
279
|
+
agentToken: z.string().optional(),
|
|
280
|
+
githubToken: z.string().optional(),
|
|
281
|
+
maxLogSizeBytes: z.coerce.number().default(10485760),
|
|
282
|
+
defaultStepTimeoutMs: z.coerce.number().default(18e5),
|
|
283
|
+
dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
|
|
284
|
+
jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
|
|
285
|
+
backpressureMode: z.enum(["pause", "drop"]).default("pause"),
|
|
286
|
+
agentPayloadDir: z.string().optional(),
|
|
287
|
+
agentCommand: z.string().optional(),
|
|
288
|
+
sandbox: z.string().default("false").transform((s) => s === "true"),
|
|
289
|
+
trustedEnv: z.string().default("false").transform((s) => s === "true"),
|
|
290
|
+
inPlace: z.string().default("false").transform((s) => s === "true"),
|
|
291
|
+
sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
|
|
292
|
+
sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
|
|
293
|
+
sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
|
|
294
|
+
sandboxUser: z.string().optional(),
|
|
295
|
+
sandboxPidsLimit: z.coerce.number().int().positive().default(512),
|
|
296
|
+
sandboxMemoryBytes: z.coerce.number().int().positive().default(2147483648),
|
|
297
|
+
sandboxNanoCpus: z.coerce.number().int().positive().default(2e9),
|
|
298
|
+
scalerManaged: z.string().optional().transform((s) => s === "1"),
|
|
299
|
+
jobImageAgent: z.string().optional().transform((s) => s === "1"),
|
|
300
|
+
runtimeImage: z.string().optional(),
|
|
301
|
+
runtimeNodeSource: z.string().optional(),
|
|
302
|
+
containerBuildCli: ContainerBuildCli.optional(),
|
|
303
|
+
scalerClaimCode: z.string().optional(),
|
|
304
|
+
scalerIdleTimeoutMs: z.coerce.number().default(5e3),
|
|
305
|
+
scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
|
|
306
|
+
executionMode: ExecutionMode.optional(),
|
|
307
|
+
otelExporterOtlpEndpoint: z.string().optional(),
|
|
308
|
+
concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
|
|
309
|
+
isOrchestratorHost: z.string().optional().transform((v) => v === "true"),
|
|
310
|
+
betweenJobsResetCommand: z.string().optional(),
|
|
311
|
+
betweenJobsResetTimeoutMs: z.coerce.number().int().positive().default(6e4),
|
|
312
|
+
betweenJobsResetRunOn: BetweenJobsRunOn.default("always"),
|
|
313
|
+
orphanCleanup: z.string().default("true").transform((s) => s !== "false"),
|
|
314
|
+
drainOnResetFailure: z.string().default("false").transform((s) => s === "true")
|
|
315
|
+
});
|
|
67
316
|
/**
|
|
68
317
|
* Env-var definition for the agent. Exported so the docs generator and the
|
|
69
318
|
* deploy-stg pre-validator can inspect / re-parse without round-tripping
|
|
@@ -71,59 +320,7 @@ const ExecutionMode = z.enum([
|
|
|
71
320
|
*/
|
|
72
321
|
const envDef = defineEnv({
|
|
73
322
|
service: "agent",
|
|
74
|
-
schema:
|
|
75
|
-
orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
|
|
76
|
-
agentId: z.string().optional(),
|
|
77
|
-
labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
|
|
78
|
-
properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
|
|
79
|
-
roles: z.string().optional().transform((s) => {
|
|
80
|
-
if (s === void 0) return void 0;
|
|
81
|
-
if (s === "") return [];
|
|
82
|
-
return s.split(",").filter(Boolean);
|
|
83
|
-
}).refine((roles) => {
|
|
84
|
-
if (roles === void 0) return true;
|
|
85
|
-
const validValues = [...KNOWN_ROLES, "all"];
|
|
86
|
-
return roles.every((r) => validValues.includes(r));
|
|
87
|
-
}, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
|
|
88
|
-
if (roles === void 0) return void 0;
|
|
89
|
-
if (roles.includes("all")) return void 0;
|
|
90
|
-
if (roles.length === 0) return [];
|
|
91
|
-
return roles.filter((r) => r !== "all");
|
|
92
|
-
}),
|
|
93
|
-
port: z.coerce.number().default(8080),
|
|
94
|
-
logLevel: z.enum([
|
|
95
|
-
"debug",
|
|
96
|
-
"info",
|
|
97
|
-
"warn",
|
|
98
|
-
"error"
|
|
99
|
-
]).default("info"),
|
|
100
|
-
agentToken: z.string().optional(),
|
|
101
|
-
githubToken: z.string().optional(),
|
|
102
|
-
maxLogSizeBytes: z.coerce.number().default(10 * 1024 * 1024),
|
|
103
|
-
defaultStepTimeoutMs: z.coerce.number().default(1800 * 1e3),
|
|
104
|
-
dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
|
|
105
|
-
jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
|
|
106
|
-
backpressureMode: z.enum(["pause", "drop"]).default("pause"),
|
|
107
|
-
agentPayloadDir: z.string().optional(),
|
|
108
|
-
agentCommand: z.string().optional(),
|
|
109
|
-
sandbox: z.string().default("false").transform((s) => s === "true"),
|
|
110
|
-
trustedEnv: z.string().default("false").transform((s) => s === "true"),
|
|
111
|
-
inPlace: z.string().default("false").transform((s) => s === "true"),
|
|
112
|
-
sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
|
|
113
|
-
sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
|
|
114
|
-
sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
|
|
115
|
-
sandboxUser: z.string().optional(),
|
|
116
|
-
sandboxPidsLimit: z.coerce.number().int().positive().default(512),
|
|
117
|
-
sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
|
|
118
|
-
sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
|
|
119
|
-
scalerManaged: z.string().optional().transform((s) => s === "1"),
|
|
120
|
-
scalerIdleTimeoutMs: z.coerce.number().default(5e3),
|
|
121
|
-
scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
|
|
122
|
-
executionMode: ExecutionMode.optional(),
|
|
123
|
-
otelExporterOtlpEndpoint: z.string().optional(),
|
|
124
|
-
concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
|
|
125
|
-
isOrchestratorHost: z.string().optional().transform((v) => v === "true")
|
|
126
|
-
}),
|
|
323
|
+
schema: configSchema,
|
|
127
324
|
envMap: {
|
|
128
325
|
orchestratorUrl: "KICI_ORCHESTRATOR_URL",
|
|
129
326
|
agentId: "KICI_AGENT_ID",
|
|
@@ -153,11 +350,21 @@ const envDef = defineEnv({
|
|
|
153
350
|
sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
|
|
154
351
|
sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
|
|
155
352
|
scalerManaged: "KICI_SCALER_MANAGED",
|
|
353
|
+
jobImageAgent: "KICI_JOB_IMAGE_AGENT",
|
|
354
|
+
runtimeImage: "KICI_RUNTIME_IMAGE",
|
|
355
|
+
runtimeNodeSource: "KICI_RUNTIME_NODE_SOURCE",
|
|
356
|
+
containerBuildCli: "KICI_CONTAINER_BUILD_CLI",
|
|
357
|
+
scalerClaimCode: "KICI_SCALER_CLAIM_CODE",
|
|
156
358
|
scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
|
|
157
359
|
scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
|
|
158
360
|
executionMode: "KICI_EXECUTION_MODE",
|
|
159
361
|
otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
|
|
160
|
-
concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS"
|
|
362
|
+
concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS",
|
|
363
|
+
betweenJobsResetCommand: "KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND",
|
|
364
|
+
betweenJobsResetTimeoutMs: "KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS",
|
|
365
|
+
betweenJobsResetRunOn: "KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON",
|
|
366
|
+
orphanCleanup: "KICI_AGENT_ORPHAN_CLEANUP",
|
|
367
|
+
drainOnResetFailure: "KICI_AGENT_DRAIN_ON_RESET_FAILURE"
|
|
161
368
|
}
|
|
162
369
|
});
|
|
163
370
|
/**
|
|
@@ -189,10 +396,16 @@ const envDef = defineEnv({
|
|
|
189
396
|
* - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
|
|
190
397
|
* - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
|
|
191
398
|
* - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
|
|
399
|
+
* - KICI_SCALER_CLAIM_CODE (optional single-use claim code the agent exchanges for its own ephemeral credentials before registering; ignored when KICI_AGENT_TOKEN is set)
|
|
192
400
|
* - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
|
|
193
401
|
* - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
|
|
194
402
|
* - KICI_EXECUTION_MODE (optional, options: container | bare-metal | firecracker) — override the runner's mode-pick logic
|
|
195
403
|
* - KICI_CONCURRENCY_WAIT_TIMEOUT_MS (default: 3_600_000) — workflow-runner timeout when long-polling for a slot-release follow-up `concurrency.ack`
|
|
404
|
+
* - KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND (optional) — host-reset command run between jobs on a reused agent (fail-open)
|
|
405
|
+
* - KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS (default: 60000) — reset command timeout
|
|
406
|
+
* - KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON (default: always, options: always | on-failure) — when the reset command runs
|
|
407
|
+
* - KICI_AGENT_ORPHAN_CLEANUP (default: true) — reap a finished job's leaked process tree (bare-metal); false = only signal the runner child
|
|
408
|
+
* - KICI_AGENT_DRAIN_ON_RESET_FAILURE (default: false) — drain the agent after repeated consecutive reset failures
|
|
196
409
|
*/
|
|
197
410
|
function loadConfig() {
|
|
198
411
|
const data = envDef.parse();
|
|
@@ -225,6 +438,68 @@ function agentClientConnectionOptions(config) {
|
|
|
225
438
|
};
|
|
226
439
|
}
|
|
227
440
|
//#endregion
|
|
441
|
+
//#region src/execution/image-build/runtime-facts.ts
|
|
442
|
+
/**
|
|
443
|
+
* What this agent's own host can do with containers, discovered at startup.
|
|
444
|
+
*
|
|
445
|
+
* The orchestrator cannot answer this. An agent runs on its own machine, and
|
|
446
|
+
* whether that machine has a container runtime is the agent's fact — probing
|
|
447
|
+
* the orchestrator's filesystem answers a different question, and doing so once
|
|
448
|
+
* stranded container jobs that had been running fine, because the probe and the
|
|
449
|
+
* job ran in different places.
|
|
450
|
+
*
|
|
451
|
+
* Reported as `kici:runtime:*` labels, which the register-time scope gate
|
|
452
|
+
* accepts unchallenged as self-reported facts. Deliberately NOT
|
|
453
|
+
* `kici:capability:*` — that prefix grants a privilege and stays token-bound.
|
|
454
|
+
*/
|
|
455
|
+
/** Sockets a container runtime answers on, in the order the agent would use them. */
|
|
456
|
+
function candidateSockets() {
|
|
457
|
+
const uid = typeof process.getuid === "function" ? process.getuid() : void 0;
|
|
458
|
+
const dockerHost = process.env.DOCKER_HOST?.replace(/^unix:\/\//, "");
|
|
459
|
+
return [
|
|
460
|
+
...dockerHost ? [{
|
|
461
|
+
fact: RuntimeFact.enum.docker,
|
|
462
|
+
path: dockerHost
|
|
463
|
+
}] : [],
|
|
464
|
+
{
|
|
465
|
+
fact: RuntimeFact.enum.docker,
|
|
466
|
+
path: "/var/run/docker.sock"
|
|
467
|
+
},
|
|
468
|
+
{
|
|
469
|
+
fact: RuntimeFact.enum.podman,
|
|
470
|
+
path: "/run/podman/podman.sock"
|
|
471
|
+
},
|
|
472
|
+
...uid !== void 0 ? [{
|
|
473
|
+
fact: RuntimeFact.enum.podman,
|
|
474
|
+
path: `/run/user/${uid}/podman/podman.sock`
|
|
475
|
+
}] : []
|
|
476
|
+
];
|
|
477
|
+
}
|
|
478
|
+
/** Is `bin` executable somewhere on PATH? */
|
|
479
|
+
function onPath(bin) {
|
|
480
|
+
return (process.env.PATH ?? "").split(delimiter).filter(Boolean).some((dir) => existsSync(join(dir, bin)));
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Discover this host's runtime facts.
|
|
484
|
+
*
|
|
485
|
+
* Presence of the socket FILE, not a handshake: registration must not block on
|
|
486
|
+
* a daemon that is slow or wedged, and a job that reaches a broken runtime
|
|
487
|
+
* still fails with the runtime's own error. The label answers "is there a
|
|
488
|
+
* runtime here at all", which is the routing question.
|
|
489
|
+
*/
|
|
490
|
+
function detectRuntimeFacts(deps = {}) {
|
|
491
|
+
const pathExists = deps.pathExists ?? existsSync;
|
|
492
|
+
const hasBinary = deps.binaryOnPath ?? onPath;
|
|
493
|
+
const facts = /* @__PURE__ */ new Set();
|
|
494
|
+
for (const { fact, path } of candidateSockets()) if (pathExists(path)) facts.add(fact);
|
|
495
|
+
if (hasBinary("docker") || hasBinary("podman")) facts.add(RuntimeFact.enum["container-build"]);
|
|
496
|
+
return [...facts];
|
|
497
|
+
}
|
|
498
|
+
/** The `kici:runtime:*` labels this host should register with. */
|
|
499
|
+
function runtimeFactLabels(deps = {}) {
|
|
500
|
+
return detectRuntimeFacts(deps).map(runtimeLabel);
|
|
501
|
+
}
|
|
502
|
+
//#endregion
|
|
228
503
|
//#region src/diagnostics/mini-bundle.ts
|
|
229
504
|
/**
|
|
230
505
|
* Agent fleet mini-bundle assembler.
|
|
@@ -325,7 +600,7 @@ var LogBuffer = class extends RingBuffer {
|
|
|
325
600
|
};
|
|
326
601
|
//#endregion
|
|
327
602
|
//#region src/ws/orchestrator-client.ts
|
|
328
|
-
const logger$
|
|
603
|
+
const logger$14 = createLogger({ prefix: "orchestrator-client" });
|
|
329
604
|
/**
|
|
330
605
|
* How long to wait for `artifacts.upload.complete.ack` before failing the step.
|
|
331
606
|
*
|
|
@@ -381,6 +656,8 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
381
656
|
pendingEventEmitRequests = /* @__PURE__ */ new Map();
|
|
382
657
|
/** Pending agent.api.request calls awaiting orchestrator response. */
|
|
383
658
|
pendingApiRequests = /* @__PURE__ */ new Map();
|
|
659
|
+
/** Pending scaler.claim-credentials requests awaiting orchestrator response. */
|
|
660
|
+
pendingClaimCredentialsRequests = /* @__PURE__ */ new Map();
|
|
384
661
|
/** Pending user-cache restore/save requests awaiting orchestrator response. */
|
|
385
662
|
pendingUserCacheRequests = /* @__PURE__ */ new Map();
|
|
386
663
|
/** Pending user-artifact upload/download requests awaiting orchestrator response. */
|
|
@@ -419,6 +696,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
419
696
|
onJobDispatch;
|
|
420
697
|
onJobCancel;
|
|
421
698
|
token;
|
|
699
|
+
scalerClaimCode;
|
|
422
700
|
heartbeatIntervalMs;
|
|
423
701
|
maxReconnectDelayMs;
|
|
424
702
|
getInFlightJobs;
|
|
@@ -437,8 +715,23 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
437
715
|
* `pendingDispatch` is set by the orchestrator when a queued job has been
|
|
438
716
|
* pre-bound to this agent and the dispatch.job message is in flight. The
|
|
439
717
|
* agent must defer arming the short scaler-idle timer in this case.
|
|
718
|
+
*
|
|
719
|
+
* `warmPool` is set when this agent was pre-spawned to wait for work rather
|
|
720
|
+
* than for a specific queued job. The agent must arm no idle timer at all:
|
|
721
|
+
* the orchestrator's warm-pool reaper owns its lifetime.
|
|
440
722
|
*/
|
|
441
723
|
onRegistered = null;
|
|
724
|
+
/**
|
|
725
|
+
* Callback invoked when a self-bootstrap claim exchange fails permanently
|
|
726
|
+
* (invalid / expired / already-consumed claim code, or the exchange itself
|
|
727
|
+
* errored or timed out). Only ever fires in self-bootstrap mode
|
|
728
|
+
* (`scalerClaimCode` set, no static token) — a statically-tokened agent never
|
|
729
|
+
* runs the claim exchange. A one-shot self-bootstrap agent cannot make any
|
|
730
|
+
* further progress after this, so server.ts wires it to a non-zero graceful
|
|
731
|
+
* shutdown; without it the health HTTP server keeps the process alive and a
|
|
732
|
+
* GitHub Actions run hangs until the job timeout.
|
|
733
|
+
*/
|
|
734
|
+
onClaimFailedPermanently = null;
|
|
442
735
|
constructor(options) {
|
|
443
736
|
this.url = options.url;
|
|
444
737
|
this.agentId = options.agentId;
|
|
@@ -447,6 +740,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
447
740
|
this.onJobDispatch = options.onJobDispatch;
|
|
448
741
|
this.onJobCancel = options.onJobCancel;
|
|
449
742
|
this.token = options.token;
|
|
743
|
+
this.scalerClaimCode = options.scalerClaimCode;
|
|
450
744
|
this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? 3e4;
|
|
451
745
|
this.maxReconnectDelayMs = options.maxReconnectDelayMs ?? 6e4;
|
|
452
746
|
this.getInFlightJobs = options.getInFlightJobs;
|
|
@@ -485,7 +779,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
485
779
|
*/
|
|
486
780
|
connect() {
|
|
487
781
|
if (this._state !== "disconnected") {
|
|
488
|
-
logger$
|
|
782
|
+
logger$14.warn("connect() called while not disconnected", { state: this._state });
|
|
489
783
|
return;
|
|
490
784
|
}
|
|
491
785
|
this.intentionalDisconnect = false;
|
|
@@ -666,6 +960,39 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
666
960
|
});
|
|
667
961
|
}
|
|
668
962
|
/**
|
|
963
|
+
* Send a scaler.claim-credentials WS message and await the response.
|
|
964
|
+
*
|
|
965
|
+
* Used by a provisioning workflow (via `ctx.kici.scaler.claimAgentCredentials`)
|
|
966
|
+
* to exchange a single-use claim code — delivered on a `kici.scaler.scale-up`
|
|
967
|
+
* event — for freshly minted ephemeral agent credentials. The token rides the
|
|
968
|
+
* response only; it is never logged. Times out after 15 seconds (matching the
|
|
969
|
+
* generic agent-API request timeout).
|
|
970
|
+
*/
|
|
971
|
+
async sendClaimCredentials(claimCode) {
|
|
972
|
+
const requestId = randomUUID();
|
|
973
|
+
return new Promise((resolve, reject) => {
|
|
974
|
+
const timer = setTimeout(() => {
|
|
975
|
+
this.pendingClaimCredentialsRequests.delete(requestId);
|
|
976
|
+
reject(/* @__PURE__ */ new Error("scaler.claim-credentials timed out"));
|
|
977
|
+
}, 15e3);
|
|
978
|
+
this.pendingClaimCredentialsRequests.set(requestId, {
|
|
979
|
+
resolve: (response) => {
|
|
980
|
+
clearTimeout(timer);
|
|
981
|
+
resolve(response);
|
|
982
|
+
},
|
|
983
|
+
reject: (err) => {
|
|
984
|
+
clearTimeout(timer);
|
|
985
|
+
reject(err);
|
|
986
|
+
}
|
|
987
|
+
});
|
|
988
|
+
this.sendDirect({
|
|
989
|
+
type: "scaler.claim-credentials",
|
|
990
|
+
requestId,
|
|
991
|
+
claimCode
|
|
992
|
+
});
|
|
993
|
+
});
|
|
994
|
+
}
|
|
995
|
+
/**
|
|
669
996
|
* Build this agent's fleet mini-bundle and stream it back to the orchestrator
|
|
670
997
|
* as ordered fleet.bundle.chunk frames (the WS frame cap forbids one frame).
|
|
671
998
|
* On failure, sends a single fleet.bundle.error. Public for unit testing.
|
|
@@ -1049,42 +1376,24 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1049
1376
|
}
|
|
1050
1377
|
});
|
|
1051
1378
|
} catch (err) {
|
|
1052
|
-
logger$
|
|
1379
|
+
logger$14.error("Failed to create WebSocket", { error: toErrorMessage(err) });
|
|
1053
1380
|
this._state = "disconnected";
|
|
1054
1381
|
this.scheduleReconnect();
|
|
1055
1382
|
return;
|
|
1056
1383
|
}
|
|
1057
1384
|
this.ws.on("open", () => {
|
|
1058
|
-
|
|
1059
|
-
this._state = "authenticating";
|
|
1060
|
-
logger$13.info("Connected to orchestrator, sending auth.request", {
|
|
1061
|
-
url: this.url,
|
|
1062
|
-
agentId: this.agentId
|
|
1063
|
-
});
|
|
1064
|
-
this.ws.send(JSON.stringify({
|
|
1065
|
-
type: "auth.request",
|
|
1066
|
-
token: this.token,
|
|
1067
|
-
protocolVersion: PROTOCOL_VERSION
|
|
1068
|
-
}));
|
|
1069
|
-
} else {
|
|
1070
|
-
this._state = "registering";
|
|
1071
|
-
logger$13.info("Connected to orchestrator, sending agent.register (no token)", {
|
|
1072
|
-
url: this.url,
|
|
1073
|
-
agentId: this.agentId
|
|
1074
|
-
});
|
|
1075
|
-
this.sendAgentRegister();
|
|
1076
|
-
}
|
|
1385
|
+
this.onWsOpen();
|
|
1077
1386
|
});
|
|
1078
1387
|
this.ws.on("message", (data) => {
|
|
1079
1388
|
this.handleMessage(data);
|
|
1080
1389
|
});
|
|
1081
1390
|
this.ws.on("close", (code, reason) => {
|
|
1082
|
-
logger$
|
|
1391
|
+
logger$14.info("Orchestrator connection closed", {
|
|
1083
1392
|
code,
|
|
1084
1393
|
reason: reason.toString()
|
|
1085
1394
|
});
|
|
1086
1395
|
if (code === WS_CLOSE_AGENT_AUTH_FAILED) {
|
|
1087
|
-
logger$
|
|
1396
|
+
logger$14.error("Orchestrator closed with auth-failed code -- token is invalid or revoked. NOT retrying.", {
|
|
1088
1397
|
code,
|
|
1089
1398
|
reason: reason.toString()
|
|
1090
1399
|
});
|
|
@@ -1101,6 +1410,8 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1101
1410
|
this.pendingEventEmitRequests.clear();
|
|
1102
1411
|
for (const [_id, pending] of this.pendingApiRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
1103
1412
|
this.pendingApiRequests.clear();
|
|
1413
|
+
for (const [_id, pending] of this.pendingClaimCredentialsRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
1414
|
+
this.pendingClaimCredentialsRequests.clear();
|
|
1104
1415
|
for (const [_id, pending] of this.pendingUserCacheRequests) pending.reject(/* @__PURE__ */ new Error("WebSocket disconnected"));
|
|
1105
1416
|
this.pendingUserCacheRequests.clear();
|
|
1106
1417
|
for (const [id, pending] of this.pendingUserArtifactRequests) {
|
|
@@ -1121,11 +1432,102 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1121
1432
|
if (!this.intentionalDisconnect) this.scheduleReconnect();
|
|
1122
1433
|
});
|
|
1123
1434
|
this.ws.on("error", (err) => {
|
|
1124
|
-
logger$
|
|
1435
|
+
logger$14.error(`Orchestrator WebSocket error: ${err.message}`);
|
|
1125
1436
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.close();
|
|
1126
1437
|
});
|
|
1127
1438
|
}
|
|
1128
1439
|
/**
|
|
1440
|
+
* WS-open handshake driver.
|
|
1441
|
+
*
|
|
1442
|
+
* Self-bootstrap runs first: when a `scalerClaimCode` is set and no static
|
|
1443
|
+
* token is present, the client exchanges the single-use code for its own
|
|
1444
|
+
* ephemeral credentials over `scaler.claim-credentials` — before any auth or
|
|
1445
|
+
* register — and adopts the minted token, agentId, and labels. A rejected or
|
|
1446
|
+
* expired code fails permanently (no reconnect), so a one-shot provisioning
|
|
1447
|
+
* run exits visibly rather than looping.
|
|
1448
|
+
*
|
|
1449
|
+
* Then the normal handshake: with a token (static or just-minted) send
|
|
1450
|
+
* `auth.request` and register after `auth.success`; without one, register
|
|
1451
|
+
* directly (unauthenticated mode).
|
|
1452
|
+
*/
|
|
1453
|
+
async onWsOpen() {
|
|
1454
|
+
if (this.scalerClaimCode && !this.token) {
|
|
1455
|
+
if (!await this.performClaimExchange(this.scalerClaimCode)) return;
|
|
1456
|
+
}
|
|
1457
|
+
if (this.token) {
|
|
1458
|
+
this._state = "authenticating";
|
|
1459
|
+
logger$14.info("Connected to orchestrator, sending auth.request", {
|
|
1460
|
+
url: this.url,
|
|
1461
|
+
agentId: this.agentId
|
|
1462
|
+
});
|
|
1463
|
+
this.ws.send(JSON.stringify({
|
|
1464
|
+
type: "auth.request",
|
|
1465
|
+
token: this.token,
|
|
1466
|
+
protocolVersion: PROTOCOL_VERSION
|
|
1467
|
+
}));
|
|
1468
|
+
} else {
|
|
1469
|
+
this._state = "registering";
|
|
1470
|
+
logger$14.info("Connected to orchestrator, sending agent.register (no token)", {
|
|
1471
|
+
url: this.url,
|
|
1472
|
+
agentId: this.agentId
|
|
1473
|
+
});
|
|
1474
|
+
this.sendAgentRegister();
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
/**
|
|
1478
|
+
* Exchange the single-use claim code for ephemeral credentials, adopting the
|
|
1479
|
+
* minted token / agentId / labels on success. Returns true when credentials
|
|
1480
|
+
* were adopted; false when the claim failed permanently (the connection has
|
|
1481
|
+
* already been torn down with no reconnect).
|
|
1482
|
+
*/
|
|
1483
|
+
async performClaimExchange(claimCode) {
|
|
1484
|
+
logger$14.info("Self-bootstrap: exchanging claim code for ephemeral credentials", {
|
|
1485
|
+
url: this.url,
|
|
1486
|
+
agentId: this.agentId
|
|
1487
|
+
});
|
|
1488
|
+
let response;
|
|
1489
|
+
try {
|
|
1490
|
+
response = await this.sendClaimCredentials(claimCode);
|
|
1491
|
+
} catch (err) {
|
|
1492
|
+
this.failClaimPermanently(`claim exchange failed: ${toErrorMessage(err)}`);
|
|
1493
|
+
return false;
|
|
1494
|
+
}
|
|
1495
|
+
if (response.error || !response.credentials) {
|
|
1496
|
+
this.failClaimPermanently(response.error ?? "orchestrator returned no credentials");
|
|
1497
|
+
return false;
|
|
1498
|
+
}
|
|
1499
|
+
const { agentToken, agentId, labels } = response.credentials;
|
|
1500
|
+
this.token = agentToken;
|
|
1501
|
+
this.agentId = agentId;
|
|
1502
|
+
this.labels = this.mergeClaimedLabels(labels);
|
|
1503
|
+
logger$14.info("Self-bootstrap: adopted ephemeral credentials", { agentId });
|
|
1504
|
+
return true;
|
|
1505
|
+
}
|
|
1506
|
+
/**
|
|
1507
|
+
* Merge the claim-minted labels into the agent's own label set, deduped. The
|
|
1508
|
+
* self-reported `kici:os:` / `kici:arch:` / `kici:host:` facts are derived at
|
|
1509
|
+
* register time (`sendAgentRegister`), so they survive regardless.
|
|
1510
|
+
*/
|
|
1511
|
+
mergeClaimedLabels(claimed) {
|
|
1512
|
+
return [.../* @__PURE__ */ new Set([...this.labels, ...claimed])];
|
|
1513
|
+
}
|
|
1514
|
+
/**
|
|
1515
|
+
* Fail a self-bootstrap claim permanently, using the same no-reconnect
|
|
1516
|
+
* mechanism as `auth.failure`: mark the auth context dead, suppress reconnect,
|
|
1517
|
+
* and close the socket. A one-shot provisioning run then exits visibly.
|
|
1518
|
+
*/
|
|
1519
|
+
failClaimPermanently(reason) {
|
|
1520
|
+
logger$14.error("Self-bootstrap claim exchange failed -- claim code is invalid or expired. NOT retrying.", { reason });
|
|
1521
|
+
this.authFailed = true;
|
|
1522
|
+
this.intentionalDisconnect = true;
|
|
1523
|
+
if (this.ws) {
|
|
1524
|
+
this.ws.close(1e3, "Claim failed");
|
|
1525
|
+
this.ws = null;
|
|
1526
|
+
}
|
|
1527
|
+
this._state = "disconnected";
|
|
1528
|
+
this.onClaimFailedPermanently?.(reason);
|
|
1529
|
+
}
|
|
1530
|
+
/**
|
|
1129
1531
|
* Resolve a pending user-artifact request from an `artifacts.upload.response`
|
|
1130
1532
|
* / `artifacts.download.response` WS message, mapping the wire fields onto the
|
|
1131
1533
|
* IPC response shape. No-op when no request is pending for the requestId.
|
|
@@ -1229,7 +1631,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1229
1631
|
this.pendingUserArtifactRequests.set(record.messageId, record.pending);
|
|
1230
1632
|
this.resendableCompletes.set(record.messageId, record);
|
|
1231
1633
|
this.sendDirect(record.frame);
|
|
1232
|
-
logger$
|
|
1634
|
+
logger$14.info("Re-sent artifact upload-complete after reconnect", {
|
|
1233
1635
|
messageId: record.messageId,
|
|
1234
1636
|
attempt: record.attempts
|
|
1235
1637
|
});
|
|
@@ -1251,12 +1653,55 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1251
1653
|
}
|
|
1252
1654
|
return false;
|
|
1253
1655
|
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Resolve a pending scaler credential-claim request from its response
|
|
1658
|
+
* message. Returns true when it handled the message. Extracted from
|
|
1659
|
+
* handleMessage so that dispatcher stays under the function-length limit.
|
|
1660
|
+
*/
|
|
1661
|
+
handleClaimCredentialsResponse(rawMsg) {
|
|
1662
|
+
if (rawMsg.type !== "scaler.claim-credentials.response") return false;
|
|
1663
|
+
const pending = this.pendingClaimCredentialsRequests.get(rawMsg.requestId);
|
|
1664
|
+
if (pending) {
|
|
1665
|
+
this.pendingClaimCredentialsRequests.delete(rawMsg.requestId);
|
|
1666
|
+
pending.resolve({
|
|
1667
|
+
credentials: rawMsg.credentials,
|
|
1668
|
+
error: rawMsg.error
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
return true;
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Complete registration: adopt the orchestrator's negotiated capabilities,
|
|
1675
|
+
* enter the `registered` state, flush everything parked while disconnected,
|
|
1676
|
+
* and hand the ack's lifetime flags to `onRegistered`.
|
|
1677
|
+
*/
|
|
1678
|
+
handleRegisterAck(msg) {
|
|
1679
|
+
logger$14.info("Registration acknowledged by orchestrator", {
|
|
1680
|
+
agentId: msg.agentId,
|
|
1681
|
+
labels: msg.labels,
|
|
1682
|
+
scalerManaged: msg.scalerManaged,
|
|
1683
|
+
pendingDispatch: msg.pendingDispatch ?? false,
|
|
1684
|
+
warmPool: msg.warmPool ?? false
|
|
1685
|
+
});
|
|
1686
|
+
this.orchCapabilities = msg.capabilities;
|
|
1687
|
+
this._state = "registered";
|
|
1688
|
+
this.reconnectAttempts = 0;
|
|
1689
|
+
this.startHeartbeat();
|
|
1690
|
+
this.flushBuffer();
|
|
1691
|
+
this.resendHeldCompletes();
|
|
1692
|
+
this.onRegistered?.({
|
|
1693
|
+
pendingDispatch: msg.pendingDispatch ?? false,
|
|
1694
|
+
warmPool: msg.warmPool ?? false
|
|
1695
|
+
});
|
|
1696
|
+
if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
|
|
1697
|
+
this.sendConfigAck(msg.agentId);
|
|
1698
|
+
}
|
|
1254
1699
|
handleMessage(data) {
|
|
1255
1700
|
let raw;
|
|
1256
1701
|
try {
|
|
1257
1702
|
raw = JSON.parse(data.toString());
|
|
1258
1703
|
} catch {
|
|
1259
|
-
logger$
|
|
1704
|
+
logger$14.warn("Malformed JSON received from orchestrator");
|
|
1260
1705
|
return;
|
|
1261
1706
|
}
|
|
1262
1707
|
const rawMsg = raw;
|
|
@@ -1281,6 +1726,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1281
1726
|
}
|
|
1282
1727
|
return;
|
|
1283
1728
|
}
|
|
1729
|
+
if (this.handleClaimCredentialsResponse(rawMsg)) return;
|
|
1284
1730
|
if (rawMsg.type === "agent.api.response") {
|
|
1285
1731
|
const pending = this.pendingApiRequests.get(rawMsg.requestId);
|
|
1286
1732
|
if (pending) {
|
|
@@ -1315,13 +1761,13 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1315
1761
|
switch (msg.type) {
|
|
1316
1762
|
case "auth.success":
|
|
1317
1763
|
if (this._state === "authenticating") {
|
|
1318
|
-
logger$
|
|
1764
|
+
logger$14.info("Authentication successful, sending agent.register", { connectionId: msg.connectionId });
|
|
1319
1765
|
this._state = "registering";
|
|
1320
1766
|
this.sendAgentRegister();
|
|
1321
1767
|
}
|
|
1322
1768
|
break;
|
|
1323
1769
|
case "auth.failure":
|
|
1324
|
-
logger$
|
|
1770
|
+
logger$14.error("Authentication FAILED -- token is invalid or expired. NOT retrying.", { reason: msg.reason });
|
|
1325
1771
|
this.authFailed = true;
|
|
1326
1772
|
this.intentionalDisconnect = true;
|
|
1327
1773
|
if (this.ws) {
|
|
@@ -1331,31 +1777,17 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1331
1777
|
this._state = "disconnected";
|
|
1332
1778
|
break;
|
|
1333
1779
|
case "register.ack":
|
|
1334
|
-
|
|
1335
|
-
agentId: msg.agentId,
|
|
1336
|
-
labels: msg.labels,
|
|
1337
|
-
scalerManaged: msg.scalerManaged,
|
|
1338
|
-
pendingDispatch: msg.pendingDispatch ?? false
|
|
1339
|
-
});
|
|
1340
|
-
this.orchCapabilities = msg.capabilities;
|
|
1341
|
-
this._state = "registered";
|
|
1342
|
-
this.reconnectAttempts = 0;
|
|
1343
|
-
this.startHeartbeat();
|
|
1344
|
-
this.flushBuffer();
|
|
1345
|
-
this.resendHeldCompletes();
|
|
1346
|
-
this.onRegistered?.({ pendingDispatch: msg.pendingDispatch ?? false });
|
|
1347
|
-
if (msg.scalerManaged || this.scalerManaged) this.blockMmdsAccess();
|
|
1348
|
-
this.sendConfigAck(msg.agentId);
|
|
1780
|
+
this.handleRegisterAck(msg);
|
|
1349
1781
|
break;
|
|
1350
1782
|
case "job.dispatch":
|
|
1351
|
-
logger$
|
|
1783
|
+
logger$14.info("Job dispatch received", {
|
|
1352
1784
|
runId: msg.runId,
|
|
1353
1785
|
jobId: msg.jobId
|
|
1354
1786
|
});
|
|
1355
1787
|
this.onJobDispatch(msg);
|
|
1356
1788
|
break;
|
|
1357
1789
|
case "job.cancel":
|
|
1358
|
-
logger$
|
|
1790
|
+
logger$14.info("Job cancel received", {
|
|
1359
1791
|
runId: msg.runId,
|
|
1360
1792
|
jobId: msg.jobId,
|
|
1361
1793
|
reason: msg.reason
|
|
@@ -1363,7 +1795,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1363
1795
|
this.onJobCancel(msg);
|
|
1364
1796
|
break;
|
|
1365
1797
|
case "job.concurrency.ack": {
|
|
1366
|
-
logger$
|
|
1798
|
+
logger$14.info("Concurrency ack received", {
|
|
1367
1799
|
requestId: msg.requestId,
|
|
1368
1800
|
action: msg.action
|
|
1369
1801
|
});
|
|
@@ -1378,7 +1810,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1378
1810
|
break;
|
|
1379
1811
|
}
|
|
1380
1812
|
case "step.approval-resolved": {
|
|
1381
|
-
logger$
|
|
1813
|
+
logger$14.info("Step approval resolved", {
|
|
1382
1814
|
requestId: msg.requestId,
|
|
1383
1815
|
runId: msg.runId,
|
|
1384
1816
|
jobId: msg.jobId,
|
|
@@ -1398,17 +1830,16 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1398
1830
|
break;
|
|
1399
1831
|
}
|
|
1400
1832
|
case "fleet.logs.request":
|
|
1401
|
-
logger$
|
|
1833
|
+
logger$14.info("Fleet log collection requested", {
|
|
1402
1834
|
requestId: msg.requestId,
|
|
1403
1835
|
logWindowHours: msg.logWindowHours
|
|
1404
1836
|
});
|
|
1405
1837
|
this.streamFleetBundle(msg);
|
|
1406
|
-
break;
|
|
1407
1838
|
}
|
|
1408
1839
|
return;
|
|
1409
1840
|
}
|
|
1410
1841
|
if (heartbeatSchema.safeParse(raw).success) return;
|
|
1411
|
-
logger$
|
|
1842
|
+
logger$14.warn("Invalid message from orchestrator", { errors: parsed.error.issues });
|
|
1412
1843
|
}
|
|
1413
1844
|
flushBuffer() {
|
|
1414
1845
|
const events = this.eventBuffer.flush();
|
|
@@ -1422,11 +1853,11 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1422
1853
|
}
|
|
1423
1854
|
this.disconnectedAt = null;
|
|
1424
1855
|
if (events.length > 0) {
|
|
1425
|
-
logger$
|
|
1856
|
+
logger$14.info("Flushing event buffer", { count: events.length });
|
|
1426
1857
|
for (const msg of events) this.sendDirect(msg);
|
|
1427
1858
|
}
|
|
1428
1859
|
if (logLines.length > 0) {
|
|
1429
|
-
logger$
|
|
1860
|
+
logger$14.info("Flushing log buffer", { count: logLines.length });
|
|
1430
1861
|
for (let i = 0; i < logLines.length; i += OrchestratorClient.LOG_BATCH_SIZE) {
|
|
1431
1862
|
const batch = logLines.slice(i, i + OrchestratorClient.LOG_BATCH_SIZE);
|
|
1432
1863
|
this.sendAgentLogMessage(batch);
|
|
@@ -1479,14 +1910,14 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1479
1910
|
*/
|
|
1480
1911
|
blockMmdsAccess() {
|
|
1481
1912
|
if (process.getuid?.() !== 0) {
|
|
1482
|
-
logger$
|
|
1913
|
+
logger$14.info("MMDS iptables block skipped (non-root) — network isolation handled by orchestrator");
|
|
1483
1914
|
return;
|
|
1484
1915
|
}
|
|
1485
1916
|
try {
|
|
1486
1917
|
execSync("iptables -A OUTPUT -d 169.254.169.254 -j DROP", { timeout: 5e3 });
|
|
1487
|
-
logger$
|
|
1918
|
+
logger$14.info("MMDS access blocked via iptables");
|
|
1488
1919
|
} catch (err) {
|
|
1489
|
-
logger$
|
|
1920
|
+
logger$14.warn("Failed to block MMDS access via iptables", { error: toErrorMessage(err) });
|
|
1490
1921
|
}
|
|
1491
1922
|
}
|
|
1492
1923
|
/**
|
|
@@ -1500,7 +1931,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1500
1931
|
messageId: `config-ack-${agentId}-${Date.now()}`,
|
|
1501
1932
|
agentId
|
|
1502
1933
|
}));
|
|
1503
|
-
logger$
|
|
1934
|
+
logger$14.info("Config ACK sent to orchestrator", { agentId });
|
|
1504
1935
|
}
|
|
1505
1936
|
}
|
|
1506
1937
|
startHeartbeat() {
|
|
@@ -1527,6 +1958,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1527
1958
|
const autoLabels = [
|
|
1528
1959
|
...deriveOsArchLabels(os.platform(), os.arch()),
|
|
1529
1960
|
hostLabel(os.hostname()),
|
|
1961
|
+
...runtimeFactLabels(),
|
|
1530
1962
|
...resolveRoleLabels(this.roles)
|
|
1531
1963
|
];
|
|
1532
1964
|
const allLabels = mergeAutoLabels(this.labels, autoLabels);
|
|
@@ -1540,7 +1972,7 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1540
1972
|
hostname: os.hostname(),
|
|
1541
1973
|
osRelease: os.release(),
|
|
1542
1974
|
osVersion: os.version(),
|
|
1543
|
-
totalMemoryMb: Math.round(os.totalmem() /
|
|
1975
|
+
totalMemoryMb: Math.round(os.totalmem() / 1048576),
|
|
1544
1976
|
cpuCount: os.cpus().length,
|
|
1545
1977
|
nodeVersion: process.versions.node,
|
|
1546
1978
|
...(() => {
|
|
@@ -1566,12 +1998,12 @@ var OrchestratorClient = class OrchestratorClient {
|
|
|
1566
1998
|
scheduleReconnect() {
|
|
1567
1999
|
this.cancelReconnect();
|
|
1568
2000
|
if (this.authFailed) {
|
|
1569
|
-
logger$
|
|
2001
|
+
logger$14.error("Not reconnecting: authentication permanently failed");
|
|
1570
2002
|
return;
|
|
1571
2003
|
}
|
|
1572
2004
|
const delay = this.getReconnectDelay();
|
|
1573
2005
|
this.reconnectAttempts++;
|
|
1574
|
-
logger$
|
|
2006
|
+
logger$14.info("Scheduling reconnect", {
|
|
1575
2007
|
attempt: this.reconnectAttempts,
|
|
1576
2008
|
delayMs: Math.round(delay)
|
|
1577
2009
|
});
|
|
@@ -1656,14 +2088,14 @@ var init_console_capture = __esmMin((() => {
|
|
|
1656
2088
|
init_console_capture();
|
|
1657
2089
|
function safe(name, fallback = "unknown") {
|
|
1658
2090
|
switch (name) {
|
|
1659
|
-
case "version": return "0.
|
|
1660
|
-
case "buildCommit": return "
|
|
1661
|
-
case "sdkVersion": return "0.
|
|
1662
|
-
case "sdkBundleHash": return "
|
|
1663
|
-
case "sharedVersion": return "0.
|
|
1664
|
-
case "sharedBundleHash": return "
|
|
1665
|
-
case "engineVersion": return "0.
|
|
1666
|
-
case "engineBundleHash": return "
|
|
2091
|
+
case "version": return "0.6.0";
|
|
2092
|
+
case "buildCommit": return "e4a936029";
|
|
2093
|
+
case "sdkVersion": return "0.6.0";
|
|
2094
|
+
case "sdkBundleHash": return "22faf0da45de7c243ce87f80fd33ee51b1df52809fde457b16bb821678f65eb3";
|
|
2095
|
+
case "sharedVersion": return "0.6.0";
|
|
2096
|
+
case "sharedBundleHash": return "a79be949815735b9e36eecc716f7eb07aa36c7bffba62e798e7e19f31a474eff";
|
|
2097
|
+
case "engineVersion": return "0.6.0";
|
|
2098
|
+
case "engineBundleHash": return "e49533b19f122155b3a78d95d105f97da259c2a45dee579578324b0702c91708";
|
|
1667
2099
|
default: return fallback;
|
|
1668
2100
|
}
|
|
1669
2101
|
}
|
|
@@ -1892,7 +2324,7 @@ init_npm_resolver();
|
|
|
1892
2324
|
* even structurally matches the allocator pattern (`ledger` is a 6-char label
|
|
1893
2325
|
* suffix), which a regex alone cannot distinguish.
|
|
1894
2326
|
*/
|
|
1895
|
-
const AGENT_TMP_GC_MAX_AGE_MS =
|
|
2327
|
+
const AGENT_TMP_GC_MAX_AGE_MS = 864e5;
|
|
1896
2328
|
/**
|
|
1897
2329
|
* Bare `kici-<6 chars>` workdirs and labeled `kici-<label>-<6 chars>`
|
|
1898
2330
|
* allocator dirs. The optional label group makes both families eligible.
|
|
@@ -1948,7 +2380,7 @@ async function gcStaleAgentTmpDirs(base = kiciTmpBase()) {
|
|
|
1948
2380
|
* spawn fails and the caller clears the orchestrator's reboot-pending flag and
|
|
1949
2381
|
* surfaces the error — the deadline sweep is the backstop.
|
|
1950
2382
|
*/
|
|
1951
|
-
const logger$
|
|
2383
|
+
const logger$13 = createLogger({ prefix: "reboot" });
|
|
1952
2384
|
/** The OS reboot primitive for a Node platform string. */
|
|
1953
2385
|
function rebootCommandFor(platform) {
|
|
1954
2386
|
switch (platform) {
|
|
@@ -1986,7 +2418,7 @@ function issueReboot(platform = process.platform) {
|
|
|
1986
2418
|
stdio: "ignore"
|
|
1987
2419
|
});
|
|
1988
2420
|
child.on("error", (err) => {
|
|
1989
|
-
logger$
|
|
2421
|
+
logger$13.error("Reboot command failed to spawn", {
|
|
1990
2422
|
cmd,
|
|
1991
2423
|
args,
|
|
1992
2424
|
error: String(err)
|
|
@@ -1994,14 +2426,14 @@ function issueReboot(platform = process.platform) {
|
|
|
1994
2426
|
reject(err);
|
|
1995
2427
|
});
|
|
1996
2428
|
child.on("exit", (code) => {
|
|
1997
|
-
if (code !== 0 && code !== null) logger$
|
|
2429
|
+
if (code !== 0 && code !== null) logger$13.warn("Reboot command exited non-zero (privilege denied?)", {
|
|
1998
2430
|
cmd,
|
|
1999
2431
|
args,
|
|
2000
2432
|
code
|
|
2001
2433
|
});
|
|
2002
2434
|
});
|
|
2003
2435
|
child.unref();
|
|
2004
|
-
logger$
|
|
2436
|
+
logger$13.info("Issued host reboot", {
|
|
2005
2437
|
cmd,
|
|
2006
2438
|
args
|
|
2007
2439
|
});
|
|
@@ -2012,8 +2444,18 @@ function issueReboot(platform = process.platform) {
|
|
|
2012
2444
|
});
|
|
2013
2445
|
}
|
|
2014
2446
|
//#endregion
|
|
2447
|
+
//#region src/idle-shutdown.ts
|
|
2448
|
+
function decideIdleShutdown(input) {
|
|
2449
|
+
if (!input.scalerManaged || input.activeJobs > 0) return "none";
|
|
2450
|
+
if (input.warmPool) return "warm";
|
|
2451
|
+
if (input.pendingDispatch) return "pending-dispatch";
|
|
2452
|
+
return "idle";
|
|
2453
|
+
}
|
|
2454
|
+
//#endregion
|
|
2015
2455
|
//#region src/metrics/prometheus.ts
|
|
2016
2456
|
var prometheus_exports = /* @__PURE__ */ __exportAll({
|
|
2457
|
+
betweenJobsResetDurationSeconds: () => betweenJobsResetDurationSeconds,
|
|
2458
|
+
betweenJobsResetTotal: () => betweenJobsResetTotal,
|
|
2017
2459
|
cloneDurationSeconds: () => cloneDurationSeconds,
|
|
2018
2460
|
connectionStatus: () => connectionStatus$1,
|
|
2019
2461
|
jobsActive: () => jobsActive$1,
|
|
@@ -2022,10 +2464,12 @@ var prometheus_exports = /* @__PURE__ */ __exportAll({
|
|
|
2022
2464
|
logBackpressureEventsTotal: () => logBackpressureEventsTotal,
|
|
2023
2465
|
logBytesTotal: () => logBytesTotal,
|
|
2024
2466
|
logLinesDroppedTotal: () => logLinesDroppedTotal,
|
|
2467
|
+
orphanCleanupTotal: () => orphanCleanupTotal,
|
|
2468
|
+
orphansReapedTotal: () => orphansReapedTotal,
|
|
2025
2469
|
stepDurationSeconds: () => stepDurationSeconds,
|
|
2026
2470
|
stepsTotal: () => stepsTotal
|
|
2027
2471
|
});
|
|
2028
|
-
var meter, jobsTotal$1, jobsActive$1, stepsTotal, stepDurationSeconds, cloneDurationSeconds, logBytesTotal, logBackpressureEventsTotal, logLinesDroppedTotal, logBackpressureActive, connectionStatus$1;
|
|
2472
|
+
var meter, jobsTotal$1, jobsActive$1, stepsTotal, stepDurationSeconds, cloneDurationSeconds, logBytesTotal, logBackpressureEventsTotal, logLinesDroppedTotal, logBackpressureActive, connectionStatus$1, betweenJobsResetTotal, betweenJobsResetDurationSeconds, orphansReapedTotal, orphanCleanupTotal;
|
|
2029
2473
|
var init_prometheus = __esmMin((() => {
|
|
2030
2474
|
meter = createMeter("kici-agent");
|
|
2031
2475
|
jobsTotal$1 = meter.createCounter("kici_agent_jobs_total", { description: "Total number of completed jobs" });
|
|
@@ -2060,6 +2504,20 @@ var init_prometheus = __esmMin((() => {
|
|
|
2060
2504
|
logLinesDroppedTotal = meter.createCounter("kici_agent_log_lines_dropped_total", { description: "Total log lines dropped due to backpressure" });
|
|
2061
2505
|
logBackpressureActive = meter.createUpDownCounter("kici_agent_log_backpressure_active", { description: "Current log-streamer backpressure state (0=normal, 1=active)" });
|
|
2062
2506
|
connectionStatus$1 = meter.createUpDownCounter("kici_agent_connection_status", { description: "Orchestrator WebSocket connection status (0=disconnected, 1=connected)" });
|
|
2507
|
+
betweenJobsResetTotal = meter.createCounter("kici_agent_between_jobs_reset_total", { description: "Total operator between-jobs reset command invocations" });
|
|
2508
|
+
betweenJobsResetDurationSeconds = meter.createHistogram("kici_agent_between_jobs_reset_duration_seconds", {
|
|
2509
|
+
description: "Between-jobs reset command duration in seconds",
|
|
2510
|
+
advice: { explicitBucketBoundaries: [
|
|
2511
|
+
.1,
|
|
2512
|
+
1,
|
|
2513
|
+
5,
|
|
2514
|
+
30,
|
|
2515
|
+
60,
|
|
2516
|
+
300
|
|
2517
|
+
] }
|
|
2518
|
+
});
|
|
2519
|
+
orphansReapedTotal = meter.createCounter("kici_agent_orphans_reaped_total", { description: "Total processes killed by the between-jobs process-group reap" });
|
|
2520
|
+
orphanCleanupTotal = meter.createCounter("kici_agent_orphan_cleanup_total", { description: "Total out-of-band declared-cleanup re-runs" });
|
|
2063
2521
|
}));
|
|
2064
2522
|
//#endregion
|
|
2065
2523
|
//#region src/checkout/ssh-auth.ts
|
|
@@ -2083,9 +2541,11 @@ async function setupSshAuth(opts) {
|
|
|
2083
2541
|
if (opts.hostKeyPolicy === "pinned" && !opts.knownHosts) throw new Error("pinned hostKeyPolicy requires knownHosts content");
|
|
2084
2542
|
const { path: tempDir, cleanup } = await makeTempDir("ssh");
|
|
2085
2543
|
const keyPath = join(tempDir, "id");
|
|
2086
|
-
|
|
2544
|
+
const pem = opts.privateKey.endsWith("\n") ? opts.privateKey : `${opts.privateKey}\n`;
|
|
2545
|
+
await writeFile(keyPath, pem, { mode: 384 });
|
|
2087
2546
|
const knownHostsPath = join(tempDir, "known_hosts");
|
|
2088
|
-
|
|
2547
|
+
const knownHostsBody = opts.hostKeyPolicy === "pinned" ? opts.knownHosts : "";
|
|
2548
|
+
await writeFile(knownHostsPath, knownHostsBody, { mode: 384 });
|
|
2089
2549
|
const parts = [
|
|
2090
2550
|
"ssh",
|
|
2091
2551
|
"-i",
|
|
@@ -2117,6 +2577,37 @@ var init_ssh_auth = __esmMin((() => {}));
|
|
|
2117
2577
|
//#endregion
|
|
2118
2578
|
//#region src/checkout/git-clone.ts
|
|
2119
2579
|
/**
|
|
2580
|
+
* Point a clone at the agent's credential helper.
|
|
2581
|
+
*
|
|
2582
|
+
* Only the helper PATH is written — never a secret — which is what makes it
|
|
2583
|
+
* safe to persist in `.git/config`. `useHttpPath` makes git include
|
|
2584
|
+
* `path=owner/repo.git` in every credential query, without which the helper
|
|
2585
|
+
* could not tell one repository from another and a write grant could not be
|
|
2586
|
+
* confined to its own repo.
|
|
2587
|
+
*/
|
|
2588
|
+
function configureCredentialHelper(workDir, helperPath) {
|
|
2589
|
+
execFileSync("git", [
|
|
2590
|
+
"-C",
|
|
2591
|
+
workDir,
|
|
2592
|
+
"config",
|
|
2593
|
+
"credential.helper",
|
|
2594
|
+
helperPath
|
|
2595
|
+
], {
|
|
2596
|
+
stdio: "pipe",
|
|
2597
|
+
timeout: 1e4
|
|
2598
|
+
});
|
|
2599
|
+
execFileSync("git", [
|
|
2600
|
+
"-C",
|
|
2601
|
+
workDir,
|
|
2602
|
+
"config",
|
|
2603
|
+
"credential.useHttpPath",
|
|
2604
|
+
"true"
|
|
2605
|
+
], {
|
|
2606
|
+
stdio: "pipe",
|
|
2607
|
+
timeout: 1e4
|
|
2608
|
+
});
|
|
2609
|
+
}
|
|
2610
|
+
/**
|
|
2120
2611
|
* Strip auth credentials from git error messages to prevent token leakage.
|
|
2121
2612
|
* Node's execFileSync includes the full command line (including -c http.extraHeader
|
|
2122
2613
|
* and GIT_SSH_COMMAND flags) in error messages, which would expose Base64-encoded
|
|
@@ -2143,7 +2634,7 @@ function redactSensitive(input) {
|
|
|
2143
2634
|
* @throws Error if clone fails or SHA does not match
|
|
2144
2635
|
*/
|
|
2145
2636
|
async function gitClone(options) {
|
|
2146
|
-
const { repoUrl, ref, sha, workDir, token, gitAuth, depth = 1 } = options;
|
|
2637
|
+
const { repoUrl, ref, sha, workDir, token, gitAuth, depth = 1, credentialHelperPath, sshCleanupRegistry } = options;
|
|
2147
2638
|
const auth = gitAuth ? gitAuth : token ? {
|
|
2148
2639
|
kind: "basic",
|
|
2149
2640
|
user: "x-access-token",
|
|
@@ -2197,6 +2688,7 @@ async function gitClone(options) {
|
|
|
2197
2688
|
} catch (err) {
|
|
2198
2689
|
throw sanitizeGitError(err);
|
|
2199
2690
|
}
|
|
2691
|
+
if (credentialHelperPath) configureCredentialHelper(workDir, credentialHelperPath);
|
|
2200
2692
|
if (!sha || sha === "HEAD") return;
|
|
2201
2693
|
const envOpts = env ? { env } : {};
|
|
2202
2694
|
if (!execFileSync("git", [
|
|
@@ -2252,7 +2744,12 @@ async function gitClone(options) {
|
|
|
2252
2744
|
if (!recheckedSha.startsWith(sha)) throw new Error(`SHA mismatch: expected ${sha}, got ${recheckedSha}`);
|
|
2253
2745
|
}
|
|
2254
2746
|
} finally {
|
|
2255
|
-
if (sshSetup)
|
|
2747
|
+
if (sshSetup) {
|
|
2748
|
+
if (sshCleanupRegistry) {
|
|
2749
|
+
const setup = sshSetup;
|
|
2750
|
+
sshCleanupRegistry.defer(() => setup.cleanup().catch(() => {}));
|
|
2751
|
+
} else await sshSetup.cleanup().catch(() => {});
|
|
2752
|
+
}
|
|
2256
2753
|
if (safeDirCleanup) await safeDirCleanup();
|
|
2257
2754
|
}
|
|
2258
2755
|
}
|
|
@@ -2260,6 +2757,408 @@ var init_git_clone = __esmMin((() => {
|
|
|
2260
2757
|
init_ssh_auth();
|
|
2261
2758
|
}));
|
|
2262
2759
|
//#endregion
|
|
2760
|
+
//#region src/checkout/clone-job-repos.ts
|
|
2761
|
+
/**
|
|
2762
|
+
* Clone a job's repositories — from the host or from inside the sandbox.
|
|
2763
|
+
*
|
|
2764
|
+
* This used to live only inside the workflow runner, which meant the clone
|
|
2765
|
+
* always happened wherever the runner ran: for a container job, inside the
|
|
2766
|
+
* customer's image, which therefore had to ship git. Extracting it lets the
|
|
2767
|
+
* AGENT clone on the host and copy the tree in, so the image needs no git —
|
|
2768
|
+
* and it puts clone-time credentials on the host, where the credential helper
|
|
2769
|
+
* already works, rather than needing a route into a container hardened with
|
|
2770
|
+
* `CapDrop: ALL`.
|
|
2771
|
+
*
|
|
2772
|
+
* Both callers run the SAME code: the runner keeps calling it for bare-metal
|
|
2773
|
+
* and for the legacy container path, and the agent calls it for a host-side
|
|
2774
|
+
* checkout. A second implementation would be two subtly different clones.
|
|
2775
|
+
*/
|
|
2776
|
+
/**
|
|
2777
|
+
* Clone whatever this job needs, or nothing.
|
|
2778
|
+
*
|
|
2779
|
+
* Three modes, unchanged from where this logic used to live: full-repo overlay
|
|
2780
|
+
* (no clone), global dual-clone (workflow repo + source repo), and the ordinary
|
|
2781
|
+
* single-repo clone.
|
|
2782
|
+
*/
|
|
2783
|
+
async function cloneJobRepos(request, dirs, deps) {
|
|
2784
|
+
if (request.checkout === false) return;
|
|
2785
|
+
const helper = request.credentialHelperPath ? { credentialHelperPath: request.credentialHelperPath } : {};
|
|
2786
|
+
if (request.fullRepo) {
|
|
2787
|
+
await mkdir(dirs.workDir, { recursive: true });
|
|
2788
|
+
deps.log("Full-repo mode: skipping git clone (workspace from overlay tarball)");
|
|
2789
|
+
return;
|
|
2790
|
+
}
|
|
2791
|
+
if (deps.isGlobal) {
|
|
2792
|
+
await mkdir(dirs.workflowDir, { recursive: true });
|
|
2793
|
+
await mkdir(dirs.sourceDir, { recursive: true });
|
|
2794
|
+
const workflowAuth = request.workflowAuth ?? request.sourceAuth;
|
|
2795
|
+
const sourceAuth = request.sourceAuth ?? request.workflowAuth;
|
|
2796
|
+
deps.log(`Global workflow: cloning workflow repo ${request.workflowRepoUrl} ref=${request.workflowRef} into ${dirs.workflowDir}`);
|
|
2797
|
+
await gitClone({
|
|
2798
|
+
repoUrl: request.workflowRepoUrl,
|
|
2799
|
+
ref: request.workflowRef ?? "",
|
|
2800
|
+
sha: request.workflowSha ?? "",
|
|
2801
|
+
workDir: dirs.workflowDir,
|
|
2802
|
+
gitAuth: workflowAuth,
|
|
2803
|
+
token: workflowAuth ? void 0 : request.token,
|
|
2804
|
+
...helper
|
|
2805
|
+
});
|
|
2806
|
+
await deps.excludeScratchFromGit(dirs.workflowDir);
|
|
2807
|
+
deps.log(`Global workflow: cloning source repo ${request.repoUrl} ref=${request.ref} into ${dirs.sourceDir}`);
|
|
2808
|
+
await gitClone({
|
|
2809
|
+
repoUrl: request.repoUrl,
|
|
2810
|
+
ref: request.ref,
|
|
2811
|
+
sha: request.sha,
|
|
2812
|
+
workDir: dirs.sourceDir,
|
|
2813
|
+
gitAuth: sourceAuth,
|
|
2814
|
+
token: sourceAuth ? void 0 : request.token,
|
|
2815
|
+
...helper
|
|
2816
|
+
});
|
|
2817
|
+
deps.log("Dual-clone complete");
|
|
2818
|
+
return;
|
|
2819
|
+
}
|
|
2820
|
+
deps.log(`Cloning ${request.repoUrl} ref=${request.ref} into ${dirs.workDir}`);
|
|
2821
|
+
await gitClone({
|
|
2822
|
+
repoUrl: request.repoUrl,
|
|
2823
|
+
ref: request.ref,
|
|
2824
|
+
sha: request.sha,
|
|
2825
|
+
workDir: dirs.workDir,
|
|
2826
|
+
gitAuth: request.sourceAuth,
|
|
2827
|
+
token: request.sourceAuth ? void 0 : request.token,
|
|
2828
|
+
...helper
|
|
2829
|
+
});
|
|
2830
|
+
await deps.excludeScratchFromGit(dirs.workDir);
|
|
2831
|
+
deps.log("Clone complete");
|
|
2832
|
+
}
|
|
2833
|
+
var init_clone_job_repos = __esmMin((() => {
|
|
2834
|
+
init_git_clone();
|
|
2835
|
+
}));
|
|
2836
|
+
//#endregion
|
|
2837
|
+
//#region src/checkout/grant-table.ts
|
|
2838
|
+
/**
|
|
2839
|
+
* Live write grants for this job.
|
|
2840
|
+
*
|
|
2841
|
+
* `withWrite` adds a grant before running its callback and revokes it in a
|
|
2842
|
+
* `finally`; the TTL is a backstop so a step that crashes cannot leave one
|
|
2843
|
+
* standing. The credential helper consults this table to decide whether to ask
|
|
2844
|
+
* the broker for a read-only or an elevated credential.
|
|
2845
|
+
*
|
|
2846
|
+
* Scope note, stated because the name could mislead: a grant is scoped to a
|
|
2847
|
+
* REPOSITORY and a TIME WINDOW — not to a step. The agent forks one process per
|
|
2848
|
+
* job and `parallel()` runs its children inside that process, so a concurrent
|
|
2849
|
+
* sibling step can push to the same repository while a grant is live. It cannot
|
|
2850
|
+
* reach a different repository.
|
|
2851
|
+
*/
|
|
2852
|
+
/** Git presents paths as `/owner/repo.git`; callers use `owner/repo`. Compare one form. */
|
|
2853
|
+
function normalise(repoPath) {
|
|
2854
|
+
return repoPath.replace(/^\/+/, "").replace(/\.git$/, "").toLowerCase();
|
|
2855
|
+
}
|
|
2856
|
+
var GrantTable;
|
|
2857
|
+
var init_grant_table = __esmMin((() => {
|
|
2858
|
+
GrantTable = class {
|
|
2859
|
+
grants = /* @__PURE__ */ new Map();
|
|
2860
|
+
add(grant) {
|
|
2861
|
+
const id = randomUUID();
|
|
2862
|
+
this.grants.set(id, {
|
|
2863
|
+
...grant,
|
|
2864
|
+
repoPath: normalise(grant.repoPath)
|
|
2865
|
+
});
|
|
2866
|
+
return id;
|
|
2867
|
+
}
|
|
2868
|
+
revoke(grantId) {
|
|
2869
|
+
this.grants.delete(grantId);
|
|
2870
|
+
}
|
|
2871
|
+
lookup(repoPath, now = Date.now()) {
|
|
2872
|
+
const wanted = normalise(repoPath);
|
|
2873
|
+
for (const [id, grant] of this.grants) {
|
|
2874
|
+
if (grant.expiresAt <= now) {
|
|
2875
|
+
this.grants.delete(id);
|
|
2876
|
+
continue;
|
|
2877
|
+
}
|
|
2878
|
+
if (grant.repoPath === wanted) return grant;
|
|
2879
|
+
}
|
|
2880
|
+
return null;
|
|
2881
|
+
}
|
|
2882
|
+
/** Live grant count, after reaping expired entries. Test and diagnostics use. */
|
|
2883
|
+
size(now = Date.now()) {
|
|
2884
|
+
for (const [id, grant] of this.grants) if (grant.expiresAt <= now) this.grants.delete(id);
|
|
2885
|
+
return this.grants.size;
|
|
2886
|
+
}
|
|
2887
|
+
};
|
|
2888
|
+
}));
|
|
2889
|
+
//#endregion
|
|
2890
|
+
//#region src/checkout/write-elevation.ts
|
|
2891
|
+
/** Requested entries the grant does not satisfy, as `key=value` strings. */
|
|
2892
|
+
function missingPermissions(requested, granted) {
|
|
2893
|
+
return Object.entries(requested).filter(([key, value]) => granted[key] !== value).map(([key, value]) => `${key}=${value}`);
|
|
2894
|
+
}
|
|
2895
|
+
async function elevateForWrite(args) {
|
|
2896
|
+
const { grant, expiresAt } = await args.request({
|
|
2897
|
+
repository: args.repository,
|
|
2898
|
+
permissions: args.permissions
|
|
2899
|
+
});
|
|
2900
|
+
if (grant.scoped) {
|
|
2901
|
+
const missing = missingPermissions(args.permissions, grant.permissions);
|
|
2902
|
+
if (missing.length > 0) throw new Error(`Cannot elevate '${args.repository}' for write: the app did not grant ${missing.join(", ")}. Grant the permission on the app installation, or request only what it holds.`);
|
|
2903
|
+
}
|
|
2904
|
+
const parsed = expiresAt ? Date.parse(expiresAt) : NaN;
|
|
2905
|
+
const expiry = Number.isFinite(parsed) ? parsed : Date.now() + DEFAULT_GRANT_TTL_MS;
|
|
2906
|
+
return {
|
|
2907
|
+
grantId: args.grants.add({
|
|
2908
|
+
repoPath: args.repository,
|
|
2909
|
+
permissions: args.permissions,
|
|
2910
|
+
expiresAt: expiry
|
|
2911
|
+
}),
|
|
2912
|
+
granted: grant
|
|
2913
|
+
};
|
|
2914
|
+
}
|
|
2915
|
+
var DEFAULT_GRANT_TTL_MS;
|
|
2916
|
+
var init_write_elevation = __esmMin((() => {
|
|
2917
|
+
DEFAULT_GRANT_TTL_MS = 36e5;
|
|
2918
|
+
}));
|
|
2919
|
+
//#endregion
|
|
2920
|
+
//#region src/checkout/credential-helper-host.ts
|
|
2921
|
+
/**
|
|
2922
|
+
* Host side of the git credential helper.
|
|
2923
|
+
*
|
|
2924
|
+
* Git spawns the helper as its own process, so it cannot call into the agent
|
|
2925
|
+
* directly. This module gives it a route: a per-job unix socket the agent
|
|
2926
|
+
* listens on, plus a tiny executable shim that connects to it and speaks git's
|
|
2927
|
+
* credential protocol on stdin/stdout.
|
|
2928
|
+
*
|
|
2929
|
+
* The shim is deliberately self-contained — no imports beyond `node:net` — so it
|
|
2930
|
+
* needs no module resolution, no bundle, and no dependency on where the agent
|
|
2931
|
+
* was installed. Its only job is to move bytes.
|
|
2932
|
+
*
|
|
2933
|
+
* Scope: this is the HOST path, which covers bare-metal jobs. A container job
|
|
2934
|
+
* clones and executes inside the container today and has no route to this
|
|
2935
|
+
* socket; that is fixed by the dual-mode container work, which moves the clone
|
|
2936
|
+
* to the host and injects `/opt/kici` read-only.
|
|
2937
|
+
*/
|
|
2938
|
+
/** The shim source. Kept inline so the helper has no build step of its own. */
|
|
2939
|
+
function shimSource(socketPath) {
|
|
2940
|
+
return `#!/usr/bin/env node
|
|
2941
|
+
'use strict';
|
|
2942
|
+
const net = require('node:net');
|
|
2943
|
+
const op = process.argv[2];
|
|
2944
|
+
// We persist nothing, so 'store' and 'erase' have nothing to do. Exiting
|
|
2945
|
+
// non-zero here would break pushes that are otherwise working.
|
|
2946
|
+
if (op !== 'get') process.exit(0);
|
|
2947
|
+
let input = '';
|
|
2948
|
+
process.stdin.setEncoding('utf8');
|
|
2949
|
+
process.stdin.on('data', (c) => { input += c; });
|
|
2950
|
+
process.stdin.on('end', () => {
|
|
2951
|
+
const query = {};
|
|
2952
|
+
for (const line of input.split('\\n')) {
|
|
2953
|
+
const eq = line.indexOf('=');
|
|
2954
|
+
if (eq <= 0) continue;
|
|
2955
|
+
const k = line.slice(0, eq);
|
|
2956
|
+
if (k === 'protocol' || k === 'host' || k === 'path') query[k] = line.slice(eq + 1);
|
|
2957
|
+
}
|
|
2958
|
+
const sock = net.createConnection(${JSON.stringify(socketPath)});
|
|
2959
|
+
let reply = '';
|
|
2960
|
+
const bail = () => { try { sock.destroy(); } catch {} process.exit(0); };
|
|
2961
|
+
sock.on('error', bail);
|
|
2962
|
+
sock.on('connect', () => sock.end(JSON.stringify(query) + '\\n'));
|
|
2963
|
+
sock.on('data', (d) => { reply += d.toString('utf8'); });
|
|
2964
|
+
sock.on('close', () => {
|
|
2965
|
+
let out = '';
|
|
2966
|
+
try {
|
|
2967
|
+
const parsed = JSON.parse(reply || 'null');
|
|
2968
|
+
if (parsed && parsed.username) {
|
|
2969
|
+
out = 'username=' + parsed.username + '\\npassword=' + parsed.password + '\\n';
|
|
2970
|
+
}
|
|
2971
|
+
} catch {
|
|
2972
|
+
// An unparseable reply is a miss, not a crash: git falls through.
|
|
2973
|
+
}
|
|
2974
|
+
// Exit only once stdout has flushed. process.exit() truncates a pending
|
|
2975
|
+
// pipe write, which silently produced an empty credential reply.
|
|
2976
|
+
if (out === '') process.exit(0);
|
|
2977
|
+
process.stdout.write(out, () => process.exit(0));
|
|
2978
|
+
});
|
|
2979
|
+
});
|
|
2980
|
+
`;
|
|
2981
|
+
}
|
|
2982
|
+
/**
|
|
2983
|
+
* Start the per-job helper socket and materialize the shim.
|
|
2984
|
+
*
|
|
2985
|
+
* `answer` is the agent's resolver: it consults the grant table and asks the
|
|
2986
|
+
* orchestrator broker. Returning null means "no credential", which the shim
|
|
2987
|
+
* turns into an empty reply so git falls through to its own mechanisms.
|
|
2988
|
+
*/
|
|
2989
|
+
async function startCredentialHelperHost(args) {
|
|
2990
|
+
await mkdir(args.dir, {
|
|
2991
|
+
recursive: true,
|
|
2992
|
+
mode: 448
|
|
2993
|
+
});
|
|
2994
|
+
const socketPath = join(args.dir, "git-credential.sock");
|
|
2995
|
+
const helperPath = join(args.dir, "git-credential-kici.cjs");
|
|
2996
|
+
await writeFile(helperPath, shimSource(socketPath), { mode: 448 });
|
|
2997
|
+
await chmod(helperPath, 448);
|
|
2998
|
+
const server = createServer({ allowHalfOpen: true }, (socket) => handleConnection(socket, args.answer));
|
|
2999
|
+
await new Promise((resolve, reject) => {
|
|
3000
|
+
server.once("error", reject);
|
|
3001
|
+
server.listen(socketPath, () => {
|
|
3002
|
+
server.removeListener("error", reject);
|
|
3003
|
+
resolve();
|
|
3004
|
+
});
|
|
3005
|
+
});
|
|
3006
|
+
return {
|
|
3007
|
+
helperPath,
|
|
3008
|
+
close: () => closeServer(server)
|
|
3009
|
+
};
|
|
3010
|
+
}
|
|
3011
|
+
/** Read one newline-terminated query, answer it, close. */
|
|
3012
|
+
function handleConnection(socket, answer) {
|
|
3013
|
+
let buf = "";
|
|
3014
|
+
socket.setEncoding("utf8");
|
|
3015
|
+
socket.on("error", () => {});
|
|
3016
|
+
socket.on("data", (chunk) => {
|
|
3017
|
+
buf += chunk;
|
|
3018
|
+
});
|
|
3019
|
+
socket.on("end", () => {
|
|
3020
|
+
let query;
|
|
3021
|
+
try {
|
|
3022
|
+
query = JSON.parse(buf || "{}");
|
|
3023
|
+
} catch {
|
|
3024
|
+
socket.end("null\n");
|
|
3025
|
+
return;
|
|
3026
|
+
}
|
|
3027
|
+
answer(query).then((result) => socket.end(JSON.stringify(result) + "\n"), () => socket.end("null\n"));
|
|
3028
|
+
});
|
|
3029
|
+
}
|
|
3030
|
+
function closeServer(server) {
|
|
3031
|
+
return new Promise((resolve) => server.close(() => resolve()));
|
|
3032
|
+
}
|
|
3033
|
+
var init_credential_helper_host = __esmMin((() => {}));
|
|
3034
|
+
//#endregion
|
|
3035
|
+
//#region src/checkout/job-git-credentials.ts
|
|
3036
|
+
/**
|
|
3037
|
+
* Per-job git credential plumbing.
|
|
3038
|
+
*
|
|
3039
|
+
* Owns the three things a job needs to authenticate git: the grant table, the
|
|
3040
|
+
* helper socket git talks to, and the relay to the orchestrator broker. Kept in
|
|
3041
|
+
* its own module so the job runner wires one object rather than three, and so
|
|
3042
|
+
* this logic is testable without standing up a job.
|
|
3043
|
+
*
|
|
3044
|
+
* Scope: this is the HOST path, which covers bare-metal jobs. A container job
|
|
3045
|
+
* clones and executes inside the container today and has no route to the
|
|
3046
|
+
* socket; that is fixed by the dual-mode container work, which moves the clone
|
|
3047
|
+
* to the host and injects `/opt/kici` read-only.
|
|
3048
|
+
*/
|
|
3049
|
+
/** `owner/repo.git` (or `/owner/repo`) as git spells it -> `owner/repo`. */
|
|
3050
|
+
function repositoryFromPath(path) {
|
|
3051
|
+
return (path ?? "").replace(/^\/+/, "").replace(/\.git$/, "");
|
|
3052
|
+
}
|
|
3053
|
+
/**
|
|
3054
|
+
* Stand up git credentials for one job.
|
|
3055
|
+
*
|
|
3056
|
+
* `sendApiRequest` is the agent's existing orchestrator relay — the same one
|
|
3057
|
+
* `ctx.kici` uses — so this adds no new transport.
|
|
3058
|
+
*/
|
|
3059
|
+
async function startJobGitCredentials(args) {
|
|
3060
|
+
const grants = new GrantTable();
|
|
3061
|
+
/**
|
|
3062
|
+
* Pick the credential a request should use.
|
|
3063
|
+
*
|
|
3064
|
+
* A per-call name wins; otherwise `default` from the job's declared map;
|
|
3065
|
+
* otherwise undefined, which means the source credential — all a read needs.
|
|
3066
|
+
* An unknown name throws rather than silently falling back, because using a
|
|
3067
|
+
* different credential than the author named is the confusion the map exists
|
|
3068
|
+
* to remove.
|
|
3069
|
+
*/
|
|
3070
|
+
const refFor = (name) => {
|
|
3071
|
+
if (name === void 0) return args.credentials?.default;
|
|
3072
|
+
const found = args.credentials?.[name];
|
|
3073
|
+
if (!found) throw new Error(`Unknown git credential '${name}'. Declare it in the job's gitCredentials map. Known: ${Object.keys(args.credentials ?? {}).join(", ") || "(none)"}`);
|
|
3074
|
+
return found;
|
|
3075
|
+
};
|
|
3076
|
+
const ask = async (repository, permissions, credentialName) => {
|
|
3077
|
+
const ref = refFor(credentialName);
|
|
3078
|
+
return await args.sendApiRequest(GIT_CREDENTIAL_REQUEST_METHOD, {
|
|
3079
|
+
jobId: args.jobId,
|
|
3080
|
+
repositories: [repository],
|
|
3081
|
+
permissions,
|
|
3082
|
+
...ref ? { ref } : {}
|
|
3083
|
+
});
|
|
3084
|
+
};
|
|
3085
|
+
/** Answer one credential query from git, defaulting to read-only. */
|
|
3086
|
+
const answer = async (query) => {
|
|
3087
|
+
const repository = repositoryFromPath(query.path);
|
|
3088
|
+
if (repository === "") return null;
|
|
3089
|
+
const grant = grants.lookup(repository);
|
|
3090
|
+
const permissions = grant ? grant.permissions : { contents: "read" };
|
|
3091
|
+
const credential = await ask(repository, permissions);
|
|
3092
|
+
if (!credential || credential.kind !== "basic") return null;
|
|
3093
|
+
return {
|
|
3094
|
+
username: credential.user ?? "x-access-token",
|
|
3095
|
+
password: credential.secret
|
|
3096
|
+
};
|
|
3097
|
+
};
|
|
3098
|
+
let host = await startCredentialHelperHost({
|
|
3099
|
+
dir: args.dir,
|
|
3100
|
+
answer
|
|
3101
|
+
});
|
|
3102
|
+
const onGitGrantRequest = async (request) => {
|
|
3103
|
+
try {
|
|
3104
|
+
if (request.op === "revoke") {
|
|
3105
|
+
if (request.grantId) grants.revoke(request.grantId);
|
|
3106
|
+
return {
|
|
3107
|
+
type: "git.grant.response",
|
|
3108
|
+
requestId: request.requestId
|
|
3109
|
+
};
|
|
3110
|
+
}
|
|
3111
|
+
if (!request.repository) throw new Error("git.grant.request: elevate requires a repository");
|
|
3112
|
+
const { grantId, granted } = await elevateForWrite({
|
|
3113
|
+
repository: request.repository,
|
|
3114
|
+
permissions: request.permissions ?? { contents: "write" },
|
|
3115
|
+
grants,
|
|
3116
|
+
request: async ({ repository, permissions }) => {
|
|
3117
|
+
const result = await ask(repository, permissions, request.credentialName);
|
|
3118
|
+
return {
|
|
3119
|
+
grant: result.grant,
|
|
3120
|
+
expiresAt: result.expiresAt
|
|
3121
|
+
};
|
|
3122
|
+
}
|
|
3123
|
+
});
|
|
3124
|
+
return {
|
|
3125
|
+
type: "git.grant.response",
|
|
3126
|
+
requestId: request.requestId,
|
|
3127
|
+
grantId,
|
|
3128
|
+
granted
|
|
3129
|
+
};
|
|
3130
|
+
} catch (err) {
|
|
3131
|
+
return {
|
|
3132
|
+
type: "git.grant.response",
|
|
3133
|
+
requestId: request.requestId,
|
|
3134
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3135
|
+
};
|
|
3136
|
+
}
|
|
3137
|
+
};
|
|
3138
|
+
return {
|
|
3139
|
+
helperPath: host.helperPath,
|
|
3140
|
+
onGitGrantRequest,
|
|
3141
|
+
withRef: (params) => {
|
|
3142
|
+
const { credential, ...rest } = params;
|
|
3143
|
+
const ref = refFor(typeof credential === "string" ? credential : void 0);
|
|
3144
|
+
return ref ? {
|
|
3145
|
+
...rest,
|
|
3146
|
+
ref
|
|
3147
|
+
} : rest;
|
|
3148
|
+
},
|
|
3149
|
+
close: async () => {
|
|
3150
|
+
const open = host;
|
|
3151
|
+
host = void 0;
|
|
3152
|
+
if (open) await open.close();
|
|
3153
|
+
}
|
|
3154
|
+
};
|
|
3155
|
+
}
|
|
3156
|
+
var init_job_git_credentials = __esmMin((() => {
|
|
3157
|
+
init_grant_table();
|
|
3158
|
+
init_write_elevation();
|
|
3159
|
+
init_credential_helper_host();
|
|
3160
|
+
}));
|
|
3161
|
+
//#endregion
|
|
2263
3162
|
//#region src/checkout/changed-files.ts
|
|
2264
3163
|
/** Build the auth context for the fetches, mirroring git-clone.ts's auth. */
|
|
2265
3164
|
async function buildAuthCtx(auth) {
|
|
@@ -2521,7 +3420,8 @@ var workflow_loader_exports = /* @__PURE__ */ __exportAll({
|
|
|
2521
3420
|
*/
|
|
2522
3421
|
async function resolveWorkflowSdkSetters(workflowFilePath) {
|
|
2523
3422
|
try {
|
|
2524
|
-
const
|
|
3423
|
+
const sdkEntry = createRequire(workflowFilePath).resolve("@kici-dev/sdk");
|
|
3424
|
+
const sdk = await import(pathToFileURL(sdkEntry).href);
|
|
2525
3425
|
if (typeof sdk.setStepOutputsMap === "function" && typeof sdk.setStepRefMap === "function" && typeof sdk.setJobOutputsMap === "function") return {
|
|
2526
3426
|
setStepOutputsMap: sdk.setStepOutputsMap,
|
|
2527
3427
|
setStepRefMap: sdk.setStepRefMap,
|
|
@@ -2694,8 +3594,8 @@ async function extractStepsFromDynamicJob(workflow, dynamicIndex, jobName, event
|
|
|
2694
3594
|
var AGENT_SDK_VERSION, AGENT_SDK_BUNDLE_HASH, hookRegistered;
|
|
2695
3595
|
var init_workflow_loader = __esmMin((() => {
|
|
2696
3596
|
init_generator_context();
|
|
2697
|
-
AGENT_SDK_VERSION = "0.
|
|
2698
|
-
AGENT_SDK_BUNDLE_HASH = "
|
|
3597
|
+
AGENT_SDK_VERSION = "0.6.0";
|
|
3598
|
+
AGENT_SDK_BUNDLE_HASH = "22faf0da45de7c243ce87f80fd33ee51b1df52809fde457b16bb821678f65eb3";
|
|
2699
3599
|
hookRegistered = false;
|
|
2700
3600
|
}));
|
|
2701
3601
|
//#endregion
|
|
@@ -2716,7 +3616,7 @@ var init_workflow_loader = __esmMin((() => {
|
|
|
2716
3616
|
async function packKiciSource(workDir) {
|
|
2717
3617
|
const kiciDir = join(workDir, ".kici");
|
|
2718
3618
|
if (!existsSync(kiciDir)) throw new Error(`.kici/ not found at ${kiciDir}`);
|
|
2719
|
-
logger$
|
|
3619
|
+
logger$12.info("Packing .kici/ source tarball", { dir: workDir });
|
|
2720
3620
|
const startTime = Date.now();
|
|
2721
3621
|
const stream = c({
|
|
2722
3622
|
gzip: true,
|
|
@@ -2730,7 +3630,7 @@ async function packKiciSource(workDir) {
|
|
|
2730
3630
|
const hash = sha256(tarball);
|
|
2731
3631
|
const sizeKB = (tarball.length / 1024).toFixed(2);
|
|
2732
3632
|
const durationMs = Date.now() - startTime;
|
|
2733
|
-
logger$
|
|
3633
|
+
logger$12.info(".kici/ source packed", {
|
|
2734
3634
|
sizeKB,
|
|
2735
3635
|
hash: hash.slice(0, 12),
|
|
2736
3636
|
durationMs
|
|
@@ -2740,9 +3640,9 @@ async function packKiciSource(workDir) {
|
|
|
2740
3640
|
hash
|
|
2741
3641
|
};
|
|
2742
3642
|
}
|
|
2743
|
-
var logger$
|
|
3643
|
+
var logger$12;
|
|
2744
3644
|
var init_source_packer = __esmMin((() => {
|
|
2745
|
-
logger$
|
|
3645
|
+
logger$12 = createLogger({ prefix: "source-packer" });
|
|
2746
3646
|
}));
|
|
2747
3647
|
//#endregion
|
|
2748
3648
|
//#region src/execution/dep-restore.ts
|
|
@@ -2868,7 +3768,7 @@ async function excludeScratchFromGit(repoWorkDir) {
|
|
|
2868
3768
|
const suffix = existing.length === 0 || existing.endsWith("\n") ? "" : "\n";
|
|
2869
3769
|
await fsPromises.appendFile(excludePath, `${suffix}# kici: hide dep-restore scratch dirs from customer git status\n${SCRATCH_DIR_GIT_EXCLUDE_GLOB}\n`);
|
|
2870
3770
|
} catch (err) {
|
|
2871
|
-
logger$
|
|
3771
|
+
logger$11.warn("Failed to register scratch dir glob in .git/info/exclude", {
|
|
2872
3772
|
excludePath,
|
|
2873
3773
|
error: err instanceof Error ? err.message : String(err)
|
|
2874
3774
|
});
|
|
@@ -2929,7 +3829,7 @@ async function cleanupScratch(scratchDir) {
|
|
|
2929
3829
|
force: true
|
|
2930
3830
|
});
|
|
2931
3831
|
} catch (cleanupErr) {
|
|
2932
|
-
logger$
|
|
3832
|
+
logger$11.warn("Scratch dir cleanup failed (orphan left behind)", {
|
|
2933
3833
|
scratchDir,
|
|
2934
3834
|
error: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
|
|
2935
3835
|
});
|
|
@@ -2954,7 +3854,7 @@ async function cleanupScratch(scratchDir) {
|
|
|
2954
3854
|
*/
|
|
2955
3855
|
async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
2956
3856
|
depsUrl = resolveOrchestratorUrl(depsUrl);
|
|
2957
|
-
logger$
|
|
3857
|
+
logger$11.info("Downloading dependency tarball", { url: depsUrl });
|
|
2958
3858
|
const kiciDir = join(workDir, ".kici");
|
|
2959
3859
|
if (depsUrl.startsWith("file://")) {
|
|
2960
3860
|
const localPath = fileURLToPath(depsUrl);
|
|
@@ -2967,8 +3867,8 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
2967
3867
|
await extractTarball(data, scratchDir);
|
|
2968
3868
|
await moveScratchIntoRepo(scratchDir, workDir);
|
|
2969
3869
|
await cleanupScratch(scratchDir);
|
|
2970
|
-
const sizeMB = (data.length /
|
|
2971
|
-
logger$
|
|
3870
|
+
const sizeMB = (data.length / 1048576).toFixed(2);
|
|
3871
|
+
logger$11.info("Dependencies restored from cache (file)", {
|
|
2972
3872
|
sizeMB,
|
|
2973
3873
|
targetDir: workDir
|
|
2974
3874
|
});
|
|
@@ -2977,7 +3877,7 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
2977
3877
|
if (!depsUrl.startsWith("http://") && !depsUrl.startsWith("https://")) throw new Error(`Unsupported deps URL scheme: ${depsUrl}`);
|
|
2978
3878
|
let lastError;
|
|
2979
3879
|
for (let attempt = 0; attempt <= 2; attempt++) {
|
|
2980
|
-
if (attempt > 0) logger$
|
|
3880
|
+
if (attempt > 0) logger$11.warn("Retrying dep tarball download", {
|
|
2981
3881
|
attempt,
|
|
2982
3882
|
url: depsUrl
|
|
2983
3883
|
});
|
|
@@ -2986,11 +3886,11 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
2986
3886
|
if (depsHash && hash !== depsHash) throw new Error(`Dep tarball hash mismatch: expected ${depsHash}, got ${hash}`);
|
|
2987
3887
|
await moveScratchIntoRepo(scratchDir, workDir);
|
|
2988
3888
|
await cleanupScratch(scratchDir);
|
|
2989
|
-
logger$
|
|
3889
|
+
logger$11.info("Dependencies restored from cache (stream)", { targetDir: workDir });
|
|
2990
3890
|
return;
|
|
2991
3891
|
} catch (err) {
|
|
2992
3892
|
lastError = err instanceof Error ? err : new Error(String(err));
|
|
2993
|
-
logger$
|
|
3893
|
+
logger$11.warn("Dep tarball download failed", {
|
|
2994
3894
|
attempt,
|
|
2995
3895
|
error: lastError.message
|
|
2996
3896
|
});
|
|
@@ -2998,10 +3898,10 @@ async function restoreDeps(workDir, depsUrl, depsHash) {
|
|
|
2998
3898
|
}
|
|
2999
3899
|
throw new Error(`Dep tarball download failed after 3 attempts: ${lastError?.message}`);
|
|
3000
3900
|
}
|
|
3001
|
-
var logger$
|
|
3901
|
+
var logger$11, DOWNLOAD_TIMEOUT_MS$1, SCRATCH_DIR_BASENAME_PREFIX, SCRATCH_DIR_GIT_EXCLUDE_GLOB;
|
|
3002
3902
|
var init_dep_restore = __esmMin((() => {
|
|
3003
|
-
logger$
|
|
3004
|
-
DOWNLOAD_TIMEOUT_MS$1 =
|
|
3903
|
+
logger$11 = createLogger({ prefix: "dep-restore" });
|
|
3904
|
+
DOWNLOAD_TIMEOUT_MS$1 = 3e5;
|
|
3005
3905
|
SCRATCH_DIR_BASENAME_PREFIX = ".dep-restore-scratch-";
|
|
3006
3906
|
SCRATCH_DIR_GIT_EXCLUDE_GLOB = `.kici/${SCRATCH_DIR_BASENAME_PREFIX}*`;
|
|
3007
3907
|
}));
|
|
@@ -3113,7 +4013,7 @@ async function uploadToPresignedUrl(url, data, opts) {
|
|
|
3113
4013
|
for (let attempt = 0; attempt <= 2; attempt++) {
|
|
3114
4014
|
if (attempt > 0) {
|
|
3115
4015
|
const delayMs = baseDelayMs * 2 ** (attempt - 1);
|
|
3116
|
-
logger$
|
|
4016
|
+
logger$10.warn("Retrying pre-signed upload", {
|
|
3117
4017
|
attempt,
|
|
3118
4018
|
delayMs,
|
|
3119
4019
|
error: lastError?.message
|
|
@@ -3130,12 +4030,12 @@ async function uploadToPresignedUrl(url, data, opts) {
|
|
|
3130
4030
|
}
|
|
3131
4031
|
throw new Error(`Pre-signed upload failed after 3 attempts: ${lastError?.message}`);
|
|
3132
4032
|
}
|
|
3133
|
-
var logger$
|
|
4033
|
+
var logger$10, DOWNLOAD_TIMEOUT_MS, UPLOAD_TIMEOUT_MS, UPLOAD_RETRY_BASE_DELAY_MS, PresignedUploadHttpError;
|
|
3134
4034
|
var init_download = __esmMin((() => {
|
|
3135
4035
|
init_dep_restore();
|
|
3136
|
-
logger$
|
|
3137
|
-
DOWNLOAD_TIMEOUT_MS =
|
|
3138
|
-
UPLOAD_TIMEOUT_MS =
|
|
4036
|
+
logger$10 = createLogger({ prefix: "agent:download" });
|
|
4037
|
+
DOWNLOAD_TIMEOUT_MS = 3e5;
|
|
4038
|
+
UPLOAD_TIMEOUT_MS = 3e5;
|
|
3139
4039
|
UPLOAD_RETRY_BASE_DELAY_MS = 500;
|
|
3140
4040
|
PresignedUploadHttpError = class extends Error {
|
|
3141
4041
|
statusCode;
|
|
@@ -3181,7 +4081,7 @@ async function extractSourceTarball(data, targetDir) {
|
|
|
3181
4081
|
}
|
|
3182
4082
|
async function restoreSource(workDir, sourceTarUrl) {
|
|
3183
4083
|
sourceTarUrl = resolveOrchestratorUrl(sourceTarUrl);
|
|
3184
|
-
logger$
|
|
4084
|
+
logger$9.info("Restoring .kici/ source from tarball", { sourceTarUrl });
|
|
3185
4085
|
const startTime = Date.now();
|
|
3186
4086
|
let data;
|
|
3187
4087
|
if (sourceTarUrl.startsWith("file://")) {
|
|
@@ -3191,16 +4091,16 @@ async function restoreSource(workDir, sourceTarUrl) {
|
|
|
3191
4091
|
else throw new Error(`Unsupported source tarball URL scheme: ${sourceTarUrl}`);
|
|
3192
4092
|
await extractSourceTarball(data, workDir);
|
|
3193
4093
|
const durationMs = Date.now() - startTime;
|
|
3194
|
-
logger$
|
|
4094
|
+
logger$9.info(".kici/ source restored", {
|
|
3195
4095
|
sizeKB: (data.length / 1024).toFixed(2),
|
|
3196
4096
|
durationMs
|
|
3197
4097
|
});
|
|
3198
4098
|
}
|
|
3199
|
-
var logger$
|
|
4099
|
+
var logger$9;
|
|
3200
4100
|
var init_source_restore = __esmMin((() => {
|
|
3201
4101
|
init_download();
|
|
3202
4102
|
init_dep_restore();
|
|
3203
|
-
logger$
|
|
4103
|
+
logger$9 = createLogger({ prefix: "source-restore" });
|
|
3204
4104
|
}));
|
|
3205
4105
|
//#endregion
|
|
3206
4106
|
//#region src/execution/timeout-util.ts
|
|
@@ -3692,9 +4592,7 @@ async function stageAgentPayload(reach, privateKey, opts, deps) {
|
|
|
3692
4592
|
case "ssh-push":
|
|
3693
4593
|
await stageSshPush(reach, privateKey, opts, extractDir, deps);
|
|
3694
4594
|
break;
|
|
3695
|
-
case "s3-direct":
|
|
3696
|
-
await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
|
|
3697
|
-
break;
|
|
4595
|
+
case "s3-direct": await stageS3Direct(reach, privateKey, opts.delivery.presignedUrl, opts.delivery.sha256, extractDir, deps);
|
|
3698
4596
|
}
|
|
3699
4597
|
return { launcherPath: path.posix.join(extractDir, "kici-agent") };
|
|
3700
4598
|
}
|
|
@@ -4193,23 +5091,31 @@ var init_streaming_zx_log = __esmMin((() => {}));
|
|
|
4193
5091
|
*
|
|
4194
5092
|
* @param jobs - Jobs returned by a DynamicJobFn
|
|
4195
5093
|
* @param ctx - Eval-time context used to resolve dynamic fields on generated jobs
|
|
5094
|
+
* @param seenNames - Optional accumulator of job names already emitted earlier
|
|
5095
|
+
* in the same eval round; when supplied, a name already present throws and
|
|
5096
|
+
* every generated name is added so later generators in the round see it
|
|
4196
5097
|
* @returns Serialized LockJob array ready for orchestrator dispatch
|
|
4197
5098
|
* @throws Error if validation fails (duplicates, limit exceeded) or if a user-supplied
|
|
4198
5099
|
* dynamic function throws / times out / returns an unsupported value
|
|
4199
5100
|
*/
|
|
4200
|
-
async function serializeJobsToLock(jobs, ctx, staticNames, allowedGroups) {
|
|
5101
|
+
async function serializeJobsToLock(jobs, ctx, staticNames, allowedGroups, seenNames) {
|
|
4201
5102
|
if (jobs.length > 100) throw new Error(`DynamicJobFn generated ${jobs.length} jobs, exceeding the limit of 100`);
|
|
4202
5103
|
const generatedNames = /* @__PURE__ */ new Set();
|
|
5104
|
+
const roundNames = seenNames ?? /* @__PURE__ */ new Set();
|
|
4203
5105
|
for (const job of jobs) {
|
|
4204
|
-
if (
|
|
5106
|
+
if (roundNames.has(job.name)) throw new Error(`Duplicate job name '${job.name}' in dynamic job output`);
|
|
4205
5107
|
generatedNames.add(job.name);
|
|
5108
|
+
roundNames.add(job.name);
|
|
4206
5109
|
}
|
|
4207
5110
|
const result = [];
|
|
4208
5111
|
for (const job of jobs) result.push(await serializeJob(job, generatedNames, ctx, staticNames ?? /* @__PURE__ */ new Set(), allowedGroups ?? /* @__PURE__ */ new Set()));
|
|
4209
5112
|
return result;
|
|
4210
5113
|
}
|
|
4211
5114
|
async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups) {
|
|
4212
|
-
const { include: runsOn, exclude: excludeLabels } =
|
|
5115
|
+
const { include: runsOn, exclude: excludeLabels } = job.invoke && job.runsOn === void 0 ? {
|
|
5116
|
+
include: [],
|
|
5117
|
+
exclude: []
|
|
5118
|
+
} : normalizeRunsOnToMatchers(job.runsOn, `generated job '${job.name}' runsOn`);
|
|
4213
5119
|
const envRefs = job.contexts ?? (job.context !== void 0 ? [job.context] : void 0);
|
|
4214
5120
|
let resolvedContexts;
|
|
4215
5121
|
if (envRefs !== void 0 && envRefs.length > 0) {
|
|
@@ -4242,7 +5148,7 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
|
|
|
4242
5148
|
return {
|
|
4243
5149
|
_type: "static",
|
|
4244
5150
|
name: job.name,
|
|
4245
|
-
runsOn,
|
|
5151
|
+
...job.invoke && runsOn.length === 0 ? {} : { runsOn },
|
|
4246
5152
|
...excludeLabels.length > 0 ? { excludeLabels } : {},
|
|
4247
5153
|
needs: resolvedNeeds,
|
|
4248
5154
|
...dependsOnGroups.length > 0 ? { dependsOnGroups } : {},
|
|
@@ -4253,7 +5159,13 @@ async function serializeJob(job, generatedNames, ctx, staticNames, allowedGroups
|
|
|
4253
5159
|
...job.description ? { description: job.description } : {},
|
|
4254
5160
|
...resolvedContexts !== void 0 ? { contexts: resolvedContexts } : {},
|
|
4255
5161
|
...resolvedEnv !== void 0 ? { env: resolvedEnv } : {},
|
|
4256
|
-
...resolvedConcurrencyGroup !== void 0 ? { concurrencyGroup: resolvedConcurrencyGroup } : {}
|
|
5162
|
+
...resolvedConcurrencyGroup !== void 0 ? { concurrencyGroup: resolvedConcurrencyGroup } : {},
|
|
5163
|
+
...job.invoke ? { invoke: {
|
|
5164
|
+
event: job.invoke.event,
|
|
5165
|
+
scope: job.invoke.scope,
|
|
5166
|
+
...job.invoke.payload !== void 0 ? { payload: job.invoke.payload } : {},
|
|
5167
|
+
...job.invoke.optional === true ? { optional: true } : {}
|
|
5168
|
+
} } : {}
|
|
4257
5169
|
};
|
|
4258
5170
|
}
|
|
4259
5171
|
/**
|
|
@@ -4501,6 +5413,7 @@ async function generateDynamicJobs(workflow, shared) {
|
|
|
4501
5413
|
workflowName: workflow.name
|
|
4502
5414
|
};
|
|
4503
5415
|
const jobs = [];
|
|
5416
|
+
const seenNames = /* @__PURE__ */ new Set();
|
|
4504
5417
|
for (const generator of generators) {
|
|
4505
5418
|
const generated = await generator(buildGeneratorContext({
|
|
4506
5419
|
workflowName: workflow.name,
|
|
@@ -4511,7 +5424,7 @@ async function generateDynamicJobs(workflow, shared) {
|
|
|
4511
5424
|
log: shared.log,
|
|
4512
5425
|
kici: shared.kici
|
|
4513
5426
|
}));
|
|
4514
|
-
jobs.push(...await serializeJobsToLock(generated, serializerCtx));
|
|
5427
|
+
jobs.push(...await serializeJobsToLock(generated, serializerCtx, void 0, void 0, seenNames));
|
|
4515
5428
|
}
|
|
4516
5429
|
return jobs;
|
|
4517
5430
|
}
|
|
@@ -4591,10 +5504,10 @@ async function evaluateAllCandidates(shared, into, deadline) {
|
|
|
4591
5504
|
*/
|
|
4592
5505
|
async function runGlobalEvalRound(args) {
|
|
4593
5506
|
const restoreEnv = applyGlobalWorkflowEnv(args.repos);
|
|
4594
|
-
const shared = buildRoundState(args);
|
|
4595
5507
|
const settled = [];
|
|
4596
5508
|
let stopReason;
|
|
4597
5509
|
try {
|
|
5510
|
+
const shared = buildRoundState(args);
|
|
4598
5511
|
await withTimeout(() => evaluateAllCandidates(shared, settled, Date.now() + args.roundTimeoutMs), args.roundTimeoutMs, `global eval round (${args.candidates.length} candidate(s))`);
|
|
4599
5512
|
} catch (error) {
|
|
4600
5513
|
stopReason = error instanceof Error ? error.message : String(error);
|
|
@@ -4627,7 +5540,7 @@ var init_global_eval_runner = __esmMin((() => {
|
|
|
4627
5540
|
})), DEFAULT_MAX_LOG_SIZE_BYTES, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_FLUSH_LINE_THRESHOLD, PAUSE_SAFETY_TIMEOUT_MS, LogStreamer;
|
|
4628
5541
|
var init_log_streamer = __esmMin((() => {
|
|
4629
5542
|
init_prometheus();
|
|
4630
|
-
DEFAULT_MAX_LOG_SIZE_BYTES =
|
|
5543
|
+
DEFAULT_MAX_LOG_SIZE_BYTES = 10485760;
|
|
4631
5544
|
DEFAULT_FLUSH_INTERVAL_MS = 100;
|
|
4632
5545
|
DEFAULT_FLUSH_LINE_THRESHOLD = 50;
|
|
4633
5546
|
PAUSE_SAFETY_TIMEOUT_MS = 3e4;
|
|
@@ -4863,6 +5776,136 @@ var init_log_streamer = __esmMin((() => {
|
|
|
4863
5776
|
};
|
|
4864
5777
|
}));
|
|
4865
5778
|
//#endregion
|
|
5779
|
+
//#region src/execution/image-build/resolve-build-spec.ts
|
|
5780
|
+
/**
|
|
5781
|
+
* Turn a job's `container.dockerfile` into an absolute, checked build spec.
|
|
5782
|
+
*
|
|
5783
|
+
* Pure on purpose. Every rule that decides what gets built — the anchoring, the
|
|
5784
|
+
* escape refusal, the tag shape — is decided here and unit-tested without a
|
|
5785
|
+
* container runtime anywhere near it. The half that needs a host lives in
|
|
5786
|
+
* `build-engine.ts`.
|
|
5787
|
+
*/
|
|
5788
|
+
/**
|
|
5789
|
+
* Anchor a repo-relative path under `workDir`, refusing anything that leaves it.
|
|
5790
|
+
*
|
|
5791
|
+
* The SDK already refused these at workflow-definition time. Refusing them AGAIN
|
|
5792
|
+
* here is the point rather than a duplication: a lock file is repo content, so a
|
|
5793
|
+
* hand-edited lock could carry `../../etc/shadow` and reach a build context that
|
|
5794
|
+
* the author never wrote.
|
|
5795
|
+
*/
|
|
5796
|
+
function anchor(workDir, p, field) {
|
|
5797
|
+
if (isAbsolute(p)) throw new Error(`container.${field} must stay inside the repository (got: ${p})`);
|
|
5798
|
+
const abs = resolve(workDir, p);
|
|
5799
|
+
const rel = relative(workDir, abs);
|
|
5800
|
+
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error(`container.${field} must stay inside the repository (got: ${p})`);
|
|
5801
|
+
return abs;
|
|
5802
|
+
}
|
|
5803
|
+
/**
|
|
5804
|
+
* Reduce a value to a container-runtime tag component.
|
|
5805
|
+
*
|
|
5806
|
+
* A job name is author-supplied and may carry spaces, slashes or parentheses,
|
|
5807
|
+
* none of which a tag accepts.
|
|
5808
|
+
*/
|
|
5809
|
+
function tagSafe(value, maxLength) {
|
|
5810
|
+
const slug = value.replace(/[^A-Za-z0-9_.-]/g, "-").replace(/^[-.]+|[-.]+$/g, "");
|
|
5811
|
+
return slug.length > 0 ? slug.slice(0, maxLength) : "job";
|
|
5812
|
+
}
|
|
5813
|
+
/**
|
|
5814
|
+
* Tag for a job's built image: a readable name, made unique by the job id.
|
|
5815
|
+
*
|
|
5816
|
+
* The job id is what guarantees uniqueness, and it is not decoration. The name
|
|
5817
|
+
* alone is truncated to fit a tag, and MATRIX LEGS of one job share a long
|
|
5818
|
+
* prefix and differ only in the suffix — `build (os=linux)` and
|
|
5819
|
+
* `build (os=darwin)` collide the moment the shared part exceeds the limit.
|
|
5820
|
+
* Two legs of one run on one host would then race for one tag, and a leg could
|
|
5821
|
+
* run its sibling's image.
|
|
5822
|
+
*/
|
|
5823
|
+
function buildTagFor(jobName, jobId) {
|
|
5824
|
+
return `kici-build:${tagSafe(jobName, 48)}-${tagSafe(jobId, 12)}`;
|
|
5825
|
+
}
|
|
5826
|
+
/**
|
|
5827
|
+
* Resolve the build spec for a job, or `undefined` when the job builds nothing.
|
|
5828
|
+
*
|
|
5829
|
+
* `undefined` is the common case and is not a failure: a job with no container,
|
|
5830
|
+
* or one that names a finalized image, has nothing to build.
|
|
5831
|
+
*/
|
|
5832
|
+
function resolveJobImageBuildSpec(args) {
|
|
5833
|
+
const { container, workDir, jobId, jobName } = args;
|
|
5834
|
+
const fileExists = args.fileExists ?? existsSync;
|
|
5835
|
+
if (!container || typeof container === "string") return void 0;
|
|
5836
|
+
const { dockerfile } = container;
|
|
5837
|
+
if (typeof dockerfile !== "string" || dockerfile.length === 0) return void 0;
|
|
5838
|
+
const dockerfilePath = anchor(workDir, dockerfile, "dockerfile");
|
|
5839
|
+
const contextDir = container.context === void 0 ? workDir : anchor(workDir, container.context, "context");
|
|
5840
|
+
if (!fileExists(dockerfilePath)) throw new Error(`container.dockerfile '${dockerfile}' does not exist in the repository (looked at ${dockerfilePath})`);
|
|
5841
|
+
return {
|
|
5842
|
+
dockerfilePath,
|
|
5843
|
+
contextDir,
|
|
5844
|
+
...container.target !== void 0 ? { target: container.target } : {},
|
|
5845
|
+
args: { ...container.args ?? {} },
|
|
5846
|
+
tag: buildTagFor(jobName, jobId),
|
|
5847
|
+
labels: {
|
|
5848
|
+
"kici-managed": "true",
|
|
5849
|
+
"kici-job-id": jobId
|
|
5850
|
+
}
|
|
5851
|
+
};
|
|
5852
|
+
}
|
|
5853
|
+
var init_resolve_build_spec = __esmMin((() => {}));
|
|
5854
|
+
//#endregion
|
|
5855
|
+
//#region src/execution/image-build/build-step.ts
|
|
5856
|
+
/**
|
|
5857
|
+
* Run a job's image build and render it as a step in the run.
|
|
5858
|
+
*
|
|
5859
|
+
* The build happens on the AGENT, before the sandbox exists, so it cannot use
|
|
5860
|
+
* the runner's IPC pseudo-step channel that the cache phase uses — that channel
|
|
5861
|
+
* belongs to a process which has not started yet. It uses the agent-side seam
|
|
5862
|
+
* the synthetic `build` / `init` / `global-eval` jobs already use: a step-status
|
|
5863
|
+
* pair plus a log streamer bound to a step index.
|
|
5864
|
+
*/
|
|
5865
|
+
/**
|
|
5866
|
+
* Build the job's image when it declared a Dockerfile, and return the tag the
|
|
5867
|
+
* sandbox should run.
|
|
5868
|
+
*
|
|
5869
|
+
* Returns `undefined` when there is nothing to build — a job with no container,
|
|
5870
|
+
* or one that names a finalized image. That is the common case and emits no
|
|
5871
|
+
* step at all: a run timeline should not grow an empty entry for work that did
|
|
5872
|
+
* not happen.
|
|
5873
|
+
*/
|
|
5874
|
+
async function runJobImageBuild(args) {
|
|
5875
|
+
const { container, workDir, jobId, jobName, build, onLog, sendStepStatus } = args;
|
|
5876
|
+
const spec = resolveJobImageBuildSpec({
|
|
5877
|
+
container,
|
|
5878
|
+
workDir,
|
|
5879
|
+
jobId,
|
|
5880
|
+
jobName,
|
|
5881
|
+
...args.fileExists ? { fileExists: args.fileExists } : {}
|
|
5882
|
+
});
|
|
5883
|
+
if (!spec) return void 0;
|
|
5884
|
+
const setup = { stepType: SetupStepType.enum["container:build"] };
|
|
5885
|
+
sendStepStatus(CONTAINER_BUILD_STEP_NAME, ExecutionStepStatus.enum.running, setup);
|
|
5886
|
+
onLog(`Building ${spec.tag} from ${spec.dockerfilePath}`);
|
|
5887
|
+
try {
|
|
5888
|
+
await build(spec, onLog);
|
|
5889
|
+
} catch (err) {
|
|
5890
|
+
const error = toErrorMessage(err);
|
|
5891
|
+
onLog(`Build failed: ${error}`);
|
|
5892
|
+
sendStepStatus(CONTAINER_BUILD_STEP_NAME, ExecutionStepStatus.enum.failed, {
|
|
5893
|
+
...setup,
|
|
5894
|
+
error
|
|
5895
|
+
});
|
|
5896
|
+
throw err;
|
|
5897
|
+
}
|
|
5898
|
+
onLog(`Built ${spec.tag}`);
|
|
5899
|
+
sendStepStatus(CONTAINER_BUILD_STEP_NAME, ExecutionStepStatus.enum.success, setup);
|
|
5900
|
+
return spec.tag;
|
|
5901
|
+
}
|
|
5902
|
+
var CONTAINER_BUILD_STEP_NAME, CONTAINER_BUILD_STEP_INDEX;
|
|
5903
|
+
var init_build_step = __esmMin((() => {
|
|
5904
|
+
init_resolve_build_spec();
|
|
5905
|
+
CONTAINER_BUILD_STEP_NAME = "container:build";
|
|
5906
|
+
CONTAINER_BUILD_STEP_INDEX = 1e6;
|
|
5907
|
+
}));
|
|
5908
|
+
//#endregion
|
|
4866
5909
|
//#region src/execution/overlay-applier.ts
|
|
4867
5910
|
/**
|
|
4868
5911
|
* Agent-side overlay application.
|
|
@@ -4910,7 +5953,7 @@ async function applyOverlay(config) {
|
|
|
4910
5953
|
const { tarballUrl, cliPublicKey, orchestratorPrivateKey, repoDir } = config;
|
|
4911
5954
|
const { path: tmpDir, cleanup } = await makeTempDir("overlay");
|
|
4912
5955
|
try {
|
|
4913
|
-
logger$
|
|
5956
|
+
logger$8.info("Downloading overlay tarball", { url: tarballUrl.replace(/\?.*$/, "?[redacted]") });
|
|
4914
5957
|
let encryptedData;
|
|
4915
5958
|
try {
|
|
4916
5959
|
encryptedData = await downloadUrl(tarballUrl);
|
|
@@ -4918,9 +5961,10 @@ async function applyOverlay(config) {
|
|
|
4918
5961
|
throw new Error(`Overlay download failed from ${tarballUrl.replace(/\?.*$/, "?[redacted]")}: ${toErrorMessage(err)}`);
|
|
4919
5962
|
}
|
|
4920
5963
|
const cliPubKeyBuf = Buffer.from(cliPublicKey, "base64");
|
|
4921
|
-
const
|
|
5964
|
+
const orchPrivKeyBuf = Buffer.from(orchestratorPrivateKey, "base64");
|
|
5965
|
+
const aesKey = deriveSharedSecret(orchPrivKeyBuf, cliPubKeyBuf);
|
|
4922
5966
|
const decryptedData = decryptBuffer(encryptedData, aesKey);
|
|
4923
|
-
logger$
|
|
5967
|
+
logger$8.info("Extracting overlay tarball", { size: decryptedData.length });
|
|
4924
5968
|
const extractDir = path.join(tmpDir, "extracted");
|
|
4925
5969
|
await fsPromises.mkdir(extractDir, { recursive: true });
|
|
4926
5970
|
try {
|
|
@@ -4969,10 +6013,10 @@ async function applyOverlay(config) {
|
|
|
4969
6013
|
await fsPromises.unlink(targetPath);
|
|
4970
6014
|
filesDeleted++;
|
|
4971
6015
|
} catch {
|
|
4972
|
-
logger$
|
|
6016
|
+
logger$8.debug("Deletion target not found, skipping", { file });
|
|
4973
6017
|
}
|
|
4974
6018
|
}
|
|
4975
|
-
logger$
|
|
6019
|
+
logger$8.info("Overlay applied successfully", {
|
|
4976
6020
|
filesApplied,
|
|
4977
6021
|
filesDeleted
|
|
4978
6022
|
});
|
|
@@ -4985,10 +6029,10 @@ async function applyOverlay(config) {
|
|
|
4985
6029
|
await cleanup().catch(() => {});
|
|
4986
6030
|
}
|
|
4987
6031
|
}
|
|
4988
|
-
var logger$
|
|
6032
|
+
var logger$8, IV_LENGTH$1, AUTH_TAG_LENGTH;
|
|
4989
6033
|
var init_overlay_applier = __esmMin((() => {
|
|
4990
6034
|
init_download();
|
|
4991
|
-
logger$
|
|
6035
|
+
logger$8 = createLogger({ prefix: "overlay-applier" });
|
|
4992
6036
|
IV_LENGTH$1 = 12;
|
|
4993
6037
|
AUTH_TAG_LENGTH = 16;
|
|
4994
6038
|
}));
|
|
@@ -5561,7 +6605,7 @@ async function installDeps(kiciDir, opts = {}) {
|
|
|
5561
6605
|
const repoRoot = opts.repoRoot ?? dirname(kiciDir);
|
|
5562
6606
|
const packageManager = await detectKiciPackageManager(repoRoot, kiciDir);
|
|
5563
6607
|
const yarnFlavor = packageManager === PackageManager.Yarn ? await detectKiciYarnFlavor(repoRoot, kiciDir) : YarnFlavor.Classic;
|
|
5564
|
-
logger$
|
|
6608
|
+
logger$7.info("Installing deps inline", {
|
|
5565
6609
|
packageManager,
|
|
5566
6610
|
yarnFlavor,
|
|
5567
6611
|
dir: kiciDir
|
|
@@ -5619,7 +6663,7 @@ async function installDeps(kiciDir, opts = {}) {
|
|
|
5619
6663
|
if (packageManager === PackageManager.Yarn) await buildYarnWorkspaceClosure(repoRoot, kiciDir, yarnFlavor);
|
|
5620
6664
|
const durationMs = Date.now() - startTime;
|
|
5621
6665
|
process.stderr.write(`[dep-installer:trace] install complete: ${durationMs}ms\n`);
|
|
5622
|
-
logger$
|
|
6666
|
+
logger$7.info("Deps installed inline", {
|
|
5623
6667
|
packageManager,
|
|
5624
6668
|
durationMs
|
|
5625
6669
|
});
|
|
@@ -5876,16 +6920,16 @@ function logSubprocessStreams(e, tokens) {
|
|
|
5876
6920
|
if (e && typeof e === "object" && "stdout" in e) process.stderr.write(`[dep-installer:trace] stdout: ${redactNpmOutput(String(e.stdout), tokens).slice(0, 500)}\n`);
|
|
5877
6921
|
if (e && typeof e === "object" && "stderr" in e) process.stderr.write(`[dep-installer:trace] stderr: ${redactNpmOutput(String(e.stderr), tokens).slice(0, 500)}\n`);
|
|
5878
6922
|
}
|
|
5879
|
-
var logger$
|
|
6923
|
+
var logger$7, execFileAsync, INSTALL_TIMEOUT_MS, INSTALL_MAX_BUFFER;
|
|
5880
6924
|
var init_dep_installer = __esmMin((() => {
|
|
5881
6925
|
init_npm_registry_config();
|
|
5882
6926
|
init_yarnrc_berry_config();
|
|
5883
6927
|
init_validate_kici_deps();
|
|
5884
6928
|
init_workspace_siblings();
|
|
5885
|
-
logger$
|
|
6929
|
+
logger$7 = createLogger({ prefix: "dep-installer" });
|
|
5886
6930
|
execFileAsync = promisify(execFile);
|
|
5887
6931
|
INSTALL_TIMEOUT_MS = 6e5;
|
|
5888
|
-
INSTALL_MAX_BUFFER =
|
|
6932
|
+
INSTALL_MAX_BUFFER = 134217728;
|
|
5889
6933
|
}));
|
|
5890
6934
|
//#endregion
|
|
5891
6935
|
//#region src/execution/dep-packer.ts
|
|
@@ -5930,7 +6974,7 @@ async function packNodeModules(kiciDir) {
|
|
|
5930
6974
|
const nmRoot = packageManager === PackageManager.Yarn ? resolveYarnNodeModulesRoot(workDir, kiciDir) : join(kiciDir, "node_modules");
|
|
5931
6975
|
if (!existsSync(nmRoot)) throw new Error(`node_modules not found at ${nmRoot}`);
|
|
5932
6976
|
const entries = await closureEntries(workDir, kiciDir, packageManager);
|
|
5933
|
-
logger$
|
|
6977
|
+
logger$6.info("Packing dependency closure into tarball", {
|
|
5934
6978
|
dir: workDir,
|
|
5935
6979
|
packageManager,
|
|
5936
6980
|
entries
|
|
@@ -5945,8 +6989,8 @@ async function packNodeModules(kiciDir) {
|
|
|
5945
6989
|
for await (const chunk of stream) chunks.push(Buffer.from(chunk));
|
|
5946
6990
|
const tarball = Buffer.concat(chunks);
|
|
5947
6991
|
const hash = sha256(tarball);
|
|
5948
|
-
const sizeMB = (tarball.length /
|
|
5949
|
-
logger$
|
|
6992
|
+
const sizeMB = (tarball.length / 1048576).toFixed(2);
|
|
6993
|
+
logger$6.info("Dependency closure packed", {
|
|
5950
6994
|
sizeMB,
|
|
5951
6995
|
hash: hash.slice(0, 12),
|
|
5952
6996
|
durationMs: Date.now() - startTime
|
|
@@ -5980,10 +7024,10 @@ async function closureEntries(workDir, kiciDir, packageManager) {
|
|
|
5980
7024
|
}
|
|
5981
7025
|
return [relative(workDir, join(kiciDir, "node_modules"))];
|
|
5982
7026
|
}
|
|
5983
|
-
var logger$
|
|
7027
|
+
var logger$6;
|
|
5984
7028
|
var init_dep_packer = __esmMin((() => {
|
|
5985
7029
|
init_workspace_siblings();
|
|
5986
|
-
logger$
|
|
7030
|
+
logger$6 = createLogger({ prefix: "dep-packer" });
|
|
5987
7031
|
}));
|
|
5988
7032
|
//#endregion
|
|
5989
7033
|
//#region src/execution/sandbox/env-sanitizer.ts
|
|
@@ -6086,12 +7130,14 @@ function encryptSecretOutputs(outputs, runPublicKeyBase64) {
|
|
|
6086
7130
|
format: "der"
|
|
6087
7131
|
});
|
|
6088
7132
|
const agentPublicKeyBase64 = Buffer.from(agentPublicKeyDer).toString("base64");
|
|
7133
|
+
const runPublicKeyDer = Buffer.from(runPublicKeyBase64, "base64");
|
|
7134
|
+
const runPublicKey = createPublicKey({
|
|
7135
|
+
key: runPublicKeyDer,
|
|
7136
|
+
format: "der",
|
|
7137
|
+
type: "spki"
|
|
7138
|
+
});
|
|
6089
7139
|
const sharedSecret = diffieHellman({
|
|
6090
|
-
publicKey:
|
|
6091
|
-
key: Buffer.from(runPublicKeyBase64, "base64"),
|
|
6092
|
-
format: "der",
|
|
6093
|
-
type: "spki"
|
|
6094
|
-
}),
|
|
7140
|
+
publicKey: runPublicKey,
|
|
6095
7141
|
privateKey: agentPriv
|
|
6096
7142
|
});
|
|
6097
7143
|
const aesKey = Buffer.from(hkdfSync("sha256", sharedSecret, Buffer.alloc(0), HKDF_INFO, 32));
|
|
@@ -6128,13 +7174,43 @@ var init_secret_encryption = __esmMin((() => {
|
|
|
6128
7174
|
* logic to avoid duplication.
|
|
6129
7175
|
*/
|
|
6130
7176
|
/**
|
|
7177
|
+
* Best-effort SIGKILL/SIGTERM of an entire process group led by `pid`.
|
|
7178
|
+
* `process.kill(-pid, signal)` targets the group whose leader is `pid` (the
|
|
7179
|
+
* child was spawned `detached`, so its pid is its group id). Returns 1 when the
|
|
7180
|
+
* group was signalled, 0 when it was already gone (ESRCH) or the pid is invalid.
|
|
7181
|
+
* We cannot cheaply count members, so the caller treats a non-zero return as
|
|
7182
|
+
* "reap attempted".
|
|
7183
|
+
*/
|
|
7184
|
+
function killProcessGroup(pid, signal) {
|
|
7185
|
+
if (pid === void 0 || !Number.isInteger(pid) || pid <= 1) return 0;
|
|
7186
|
+
try {
|
|
7187
|
+
process.kill(-pid, signal);
|
|
7188
|
+
return 1;
|
|
7189
|
+
} catch {
|
|
7190
|
+
return 0;
|
|
7191
|
+
}
|
|
7192
|
+
}
|
|
7193
|
+
/**
|
|
7194
|
+
* SIGTERM the job's process group, wait a short grace, then SIGKILL. No-op
|
|
7195
|
+
* (returns 0) when the child was not spawned detached or has no pid. Returns the
|
|
7196
|
+
* count of reap attempts that signalled a live group (0, 1, or 2).
|
|
7197
|
+
*/
|
|
7198
|
+
async function reapGroup(pid, detached, killFn = killProcessGroup, sleepMs = 2e3) {
|
|
7199
|
+
if (!detached || pid === void 0) return 0;
|
|
7200
|
+
const termed = killFn(pid, "SIGTERM");
|
|
7201
|
+
if (termed === 0) return 0;
|
|
7202
|
+
await new Promise((r) => setTimeout(r, sleepMs));
|
|
7203
|
+
return termed + killFn(pid, "SIGKILL");
|
|
7204
|
+
}
|
|
7205
|
+
/**
|
|
6131
7206
|
* Build a JobExecutionRequest from a JobDispatch.
|
|
6132
7207
|
*
|
|
6133
7208
|
* Maps orchestrator dispatch fields to the subset needed by the workflow runner.
|
|
6134
7209
|
*/
|
|
6135
|
-
function buildRequest(dispatch, workDir) {
|
|
7210
|
+
function buildRequest(dispatch, workDir, extra) {
|
|
6136
7211
|
const jobConfig = dispatch.jobConfig;
|
|
6137
7212
|
return {
|
|
7213
|
+
...extra?.cleanupOnly ? { cleanupOnly: true } : {},
|
|
6138
7214
|
runId: dispatch.runId,
|
|
6139
7215
|
jobId: dispatch.jobId,
|
|
6140
7216
|
workDir,
|
|
@@ -6144,6 +7220,7 @@ function buildRequest(dispatch, workDir) {
|
|
|
6144
7220
|
token: dispatch.token,
|
|
6145
7221
|
sourceAuth: dispatch.sourceAuth,
|
|
6146
7222
|
workflowAuth: dispatch.workflowAuth,
|
|
7223
|
+
...extra?.credentialHelperPath ? { credentialHelperPath: extra.credentialHelperPath } : {},
|
|
6147
7224
|
sourceTarUrl: dispatch.sourceTarUrl,
|
|
6148
7225
|
sourceTarHash: dispatch.sourceTarHash,
|
|
6149
7226
|
depsUrl: dispatch.depsUrl,
|
|
@@ -6190,6 +7267,7 @@ function buildRequest(dispatch, workDir) {
|
|
|
6190
7267
|
upstreamJobOutputs: dispatch.upstreamJobOutputs,
|
|
6191
7268
|
upstreamJobStatuses: dispatch.upstreamJobStatuses,
|
|
6192
7269
|
jobNeeds: jobConfig.needs,
|
|
7270
|
+
upstreamInvokeResults: dispatch.upstreamInvokeResults,
|
|
6193
7271
|
npmRegistries: dispatch.npmRegistries,
|
|
6194
7272
|
installEnvSecrets: dispatch.installEnvSecrets,
|
|
6195
7273
|
jobIdShort: dispatch.jobId.slice(0, 8),
|
|
@@ -6328,20 +7406,22 @@ function fileCloneSourceBinds(repoUrl) {
|
|
|
6328
7406
|
function spawnRunnerChild(options, sanitizedEnv, effectiveWorkDir) {
|
|
6329
7407
|
const { runnerPath, useBwrap = false, networkIsolation = false, extraReadOnlyBinds = [] } = options;
|
|
6330
7408
|
let child;
|
|
6331
|
-
if (useBwrap)
|
|
6332
|
-
|
|
6333
|
-
|
|
6334
|
-
|
|
6335
|
-
|
|
6336
|
-
|
|
6337
|
-
|
|
6338
|
-
|
|
6339
|
-
|
|
6340
|
-
|
|
6341
|
-
|
|
6342
|
-
|
|
6343
|
-
|
|
6344
|
-
|
|
7409
|
+
if (useBwrap) {
|
|
7410
|
+
const bwrapArgs = buildBwrapArgs(effectiveWorkDir, process.execPath, networkIsolation, runnerPath, extraReadOnlyBinds);
|
|
7411
|
+
child = spawn("bwrap", [
|
|
7412
|
+
...bwrapArgs,
|
|
7413
|
+
process.execPath,
|
|
7414
|
+
runnerPath
|
|
7415
|
+
], {
|
|
7416
|
+
env: sanitizedEnv,
|
|
7417
|
+
stdio: [
|
|
7418
|
+
"pipe",
|
|
7419
|
+
"pipe",
|
|
7420
|
+
"pipe",
|
|
7421
|
+
"ipc"
|
|
7422
|
+
]
|
|
7423
|
+
});
|
|
7424
|
+
} else child = fork(runnerPath, [], {
|
|
6345
7425
|
env: sanitizedEnv,
|
|
6346
7426
|
stdio: [
|
|
6347
7427
|
"pipe",
|
|
@@ -6349,7 +7429,8 @@ function spawnRunnerChild(options, sanitizedEnv, effectiveWorkDir) {
|
|
|
6349
7429
|
"pipe",
|
|
6350
7430
|
"ipc"
|
|
6351
7431
|
],
|
|
6352
|
-
cwd: effectiveWorkDir || void 0
|
|
7432
|
+
cwd: effectiveWorkDir || void 0,
|
|
7433
|
+
detached: options.detachProcessGroup === true
|
|
6353
7434
|
});
|
|
6354
7435
|
return {
|
|
6355
7436
|
child,
|
|
@@ -6450,6 +7531,24 @@ function relayCacheRequest$1(msg, ctx) {
|
|
|
6450
7531
|
error: toErrorMessage(err)
|
|
6451
7532
|
}));
|
|
6452
7533
|
}
|
|
7534
|
+
/** Relay `git.grant.request` to the agent's grant table and pipe the response
|
|
7535
|
+
* (or an error response, or a "not configured" response when the callback
|
|
7536
|
+
* isn't wired) back into the sandbox runner. */
|
|
7537
|
+
function relayGitGrantRequest$1(msg, ctx) {
|
|
7538
|
+
if (!ctx.execOptions.onGitGrantRequest) {
|
|
7539
|
+
safeSendToChild(ctx.child, {
|
|
7540
|
+
type: "git.grant.response",
|
|
7541
|
+
requestId: msg.requestId,
|
|
7542
|
+
error: "Git credentials are not available in this agent configuration"
|
|
7543
|
+
});
|
|
7544
|
+
return;
|
|
7545
|
+
}
|
|
7546
|
+
ctx.execOptions.onGitGrantRequest(msg).then((response) => safeSendToChild(ctx.child, response), (err) => safeSendToChild(ctx.child, {
|
|
7547
|
+
type: "git.grant.response",
|
|
7548
|
+
requestId: msg.requestId,
|
|
7549
|
+
error: toErrorMessage(err)
|
|
7550
|
+
}));
|
|
7551
|
+
}
|
|
6453
7552
|
/** Relay `provenance.request` and pipe the orchestrator response (or an error
|
|
6454
7553
|
* response, or a "not configured" response when the callback isn't wired) back
|
|
6455
7554
|
* into the sandbox runner. */
|
|
@@ -6536,7 +7635,10 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
6536
7635
|
case "ready":
|
|
6537
7636
|
safeSendToChild(ctx.child, {
|
|
6538
7637
|
type: "execute",
|
|
6539
|
-
request: buildRequest(dispatch, ctx.effectiveWorkDir
|
|
7638
|
+
request: buildRequest(dispatch, ctx.effectiveWorkDir, {
|
|
7639
|
+
cleanupOnly: ctx.cleanupOnly,
|
|
7640
|
+
credentialHelperPath: ctx.execOptions.credentialHelperPath
|
|
7641
|
+
})
|
|
6540
7642
|
});
|
|
6541
7643
|
return;
|
|
6542
7644
|
case "log.line":
|
|
@@ -6585,6 +7687,9 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
6585
7687
|
case "agent.api.request":
|
|
6586
7688
|
relayAgentApiRequest(msg, ctx);
|
|
6587
7689
|
return;
|
|
7690
|
+
case "git.grant.request":
|
|
7691
|
+
relayGitGrantRequest$1(msg, ctx);
|
|
7692
|
+
return;
|
|
6588
7693
|
case "cache.request":
|
|
6589
7694
|
relayCacheRequest$1(msg, ctx);
|
|
6590
7695
|
return;
|
|
@@ -6597,6 +7702,12 @@ function relayChildIpcMessage(msg, dispatch, ctx) {
|
|
|
6597
7702
|
case "approval.request":
|
|
6598
7703
|
relayApprovalRequest$1(msg, ctx);
|
|
6599
7704
|
return;
|
|
7705
|
+
case "hooks-declared":
|
|
7706
|
+
ctx.state.declaresCleanup = msg.declaresCleanup;
|
|
7707
|
+
return;
|
|
7708
|
+
case "completion-hooks-done":
|
|
7709
|
+
ctx.state.completionHooksRan = true;
|
|
7710
|
+
return;
|
|
6600
7711
|
case "job.complete":
|
|
6601
7712
|
handleJobComplete(msg, dispatch, ctx);
|
|
6602
7713
|
return;
|
|
@@ -6624,7 +7735,7 @@ function handleChildExitWithoutCompletion(code, signal, ctx) {
|
|
|
6624
7735
|
/** Build the cancel function (state machine: running → cancelling →
|
|
6625
7736
|
* force_killing). The caller is responsible for surfacing it to the
|
|
6626
7737
|
* ForkRunnerHandle via the closure. */
|
|
6627
|
-
function buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs) {
|
|
7738
|
+
function buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs, detached) {
|
|
6628
7739
|
const doForceCancel = () => {
|
|
6629
7740
|
ctx.state.forkState = "force_killing";
|
|
6630
7741
|
safeSendToChild(child, {
|
|
@@ -6634,6 +7745,7 @@ function buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs)
|
|
|
6634
7745
|
try {
|
|
6635
7746
|
child.kill("SIGKILL");
|
|
6636
7747
|
} catch {}
|
|
7748
|
+
if (detached) killProcessGroup(child.pid, "SIGKILL");
|
|
6637
7749
|
clearAllCancelTimers(ctx);
|
|
6638
7750
|
};
|
|
6639
7751
|
return (force, gracePeriodMs) => {
|
|
@@ -6686,16 +7798,23 @@ function createForkRunner(options, execOptions) {
|
|
|
6686
7798
|
}),
|
|
6687
7799
|
abort: async () => {},
|
|
6688
7800
|
kill: noopFn,
|
|
6689
|
-
cancel: noopFn
|
|
7801
|
+
cancel: noopFn,
|
|
7802
|
+
detached: false,
|
|
7803
|
+
completionHooksRan: false,
|
|
7804
|
+
declaresCleanup: false,
|
|
7805
|
+
reap: async () => 0
|
|
6690
7806
|
};
|
|
6691
7807
|
}
|
|
6692
7808
|
const stderrLines = setupChildStdioCapture(child);
|
|
6693
7809
|
const defaultGracePeriodMs = 3e4;
|
|
6694
7810
|
const agentMaxGracePeriodMs = options.maxGracePeriodMs ?? defaultGracePeriodMs;
|
|
6695
7811
|
const cancelTimers = [];
|
|
7812
|
+
const detached = options.detachProcessGroup === true;
|
|
6696
7813
|
const sharedState = {
|
|
6697
7814
|
forkState: "running",
|
|
6698
|
-
jobCompleted: false
|
|
7815
|
+
jobCompleted: false,
|
|
7816
|
+
completionHooksRan: false,
|
|
7817
|
+
declaresCleanup: false
|
|
6699
7818
|
};
|
|
6700
7819
|
let cancelFn = () => {};
|
|
6701
7820
|
const result = new Promise((resolve) => {
|
|
@@ -6708,10 +7827,11 @@ function createForkRunner(options, execOptions) {
|
|
|
6708
7827
|
resolve,
|
|
6709
7828
|
cancelTimers,
|
|
6710
7829
|
state: sharedState,
|
|
6711
|
-
stderrLines
|
|
7830
|
+
stderrLines,
|
|
7831
|
+
cleanupOnly: options.cleanupOnly === true
|
|
6712
7832
|
};
|
|
6713
7833
|
process.stderr.write(`[fork-runner] Child process spawned: pid=${child.pid}, runner=${options.runnerPath}, cwd=${effectiveWorkDir}\n`);
|
|
6714
|
-
cancelFn = buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs);
|
|
7834
|
+
cancelFn = buildCancelFn(child, ctx, defaultGracePeriodMs, agentMaxGracePeriodMs, detached);
|
|
6715
7835
|
child.on("message", (msg) => relayChildIpcMessage(msg, dispatch, ctx));
|
|
6716
7836
|
child.on("exit", (code, signal) => handleChildExitWithoutCompletion(code, signal, ctx));
|
|
6717
7837
|
child.on("error", (err) => {
|
|
@@ -6742,7 +7862,15 @@ function createForkRunner(options, execOptions) {
|
|
|
6742
7862
|
for (const timer of cancelTimers) clearTimeout(timer);
|
|
6743
7863
|
cancelTimers.length = 0;
|
|
6744
7864
|
},
|
|
6745
|
-
cancel: (force, gracePeriodMs) => cancelFn(force, gracePeriodMs)
|
|
7865
|
+
cancel: (force, gracePeriodMs) => cancelFn(force, gracePeriodMs),
|
|
7866
|
+
detached,
|
|
7867
|
+
get completionHooksRan() {
|
|
7868
|
+
return sharedState.completionHooksRan;
|
|
7869
|
+
},
|
|
7870
|
+
get declaresCleanup() {
|
|
7871
|
+
return sharedState.declaresCleanup;
|
|
7872
|
+
},
|
|
7873
|
+
reap: () => reapGroup(child.pid, detached)
|
|
6746
7874
|
};
|
|
6747
7875
|
}
|
|
6748
7876
|
var init_fork_runner = __esmMin((() => {
|
|
@@ -6767,22 +7895,25 @@ var init_fork_runner = __esmMin((() => {
|
|
|
6767
7895
|
* network access. This mode provides credential isolation only and should
|
|
6768
7896
|
* be used in trusted environments.
|
|
6769
7897
|
*/
|
|
6770
|
-
var logger$
|
|
7898
|
+
var logger$5, BareMetalSandbox;
|
|
6771
7899
|
var init_bare_metal_sandbox = __esmMin((() => {
|
|
6772
7900
|
init_fork_runner();
|
|
6773
|
-
logger$
|
|
7901
|
+
logger$5 = createLogger({ prefix: "bare-metal-sandbox" });
|
|
6774
7902
|
BareMetalSandbox = class {
|
|
6775
7903
|
runnerPath;
|
|
6776
7904
|
useBwrap;
|
|
6777
7905
|
sandboxNetwork;
|
|
6778
7906
|
env;
|
|
7907
|
+
orphanCleanup;
|
|
6779
7908
|
runner = null;
|
|
6780
7909
|
workDir;
|
|
7910
|
+
lastOptions = null;
|
|
6781
7911
|
constructor(options) {
|
|
6782
7912
|
this.runnerPath = options.runnerPath;
|
|
6783
7913
|
this.useBwrap = options.sandbox;
|
|
6784
7914
|
this.sandboxNetwork = options.sandboxNetwork ?? "isolated";
|
|
6785
7915
|
this.env = options.env;
|
|
7916
|
+
this.orphanCleanup = options.orphanCleanup ?? true;
|
|
6786
7917
|
}
|
|
6787
7918
|
/**
|
|
6788
7919
|
* Validate that the runner path exists and bwrap is available (if needed).
|
|
@@ -6797,25 +7928,27 @@ var init_bare_metal_sandbox = __esmMin((() => {
|
|
|
6797
7928
|
if (this.useBwrap) try {
|
|
6798
7929
|
const { execSync } = await import("node:child_process");
|
|
6799
7930
|
execSync("which bwrap", { stdio: "ignore" });
|
|
6800
|
-
if (this.sandboxNetwork === "isolated") logger$
|
|
6801
|
-
else logger$
|
|
7931
|
+
if (this.sandboxNetwork === "isolated") logger$5.info("Bubblewrap (bwrap) sandbox enabled with network isolation (--unshare-net)");
|
|
7932
|
+
else logger$5.info("Bubblewrap (bwrap) sandbox enabled with host network (KICI_SANDBOX_NETWORK=host)");
|
|
6802
7933
|
} catch {
|
|
6803
7934
|
throw new Error("Bubblewrap (bwrap) not found. Install bubblewrap or set sandbox=false. On Debian/Ubuntu: apt install bubblewrap");
|
|
6804
7935
|
}
|
|
6805
|
-
else logger$
|
|
7936
|
+
else logger$5.warn("Bare-metal without sandbox provides limited isolation. Only environment sanitization is active. Enable sandbox=true with bubblewrap for PID/IPC/filesystem namespace isolation.");
|
|
6806
7937
|
}
|
|
6807
7938
|
/**
|
|
6808
7939
|
* Execute a job by forking the workflow runner with sanitized environment.
|
|
6809
7940
|
*/
|
|
6810
7941
|
async executeJob(options) {
|
|
6811
7942
|
const extraReadOnlyBinds = this.useBwrap ? fileCloneSourceBinds(options.dispatch.repoUrl) : [];
|
|
7943
|
+
this.lastOptions = options;
|
|
6812
7944
|
this.runner = createForkRunner({
|
|
6813
7945
|
runnerPath: this.runnerPath,
|
|
6814
7946
|
env: this.env,
|
|
6815
7947
|
useBwrap: this.useBwrap,
|
|
6816
7948
|
workDir: this.workDir,
|
|
6817
7949
|
networkIsolation: this.useBwrap && this.sandboxNetwork === "isolated",
|
|
6818
|
-
extraReadOnlyBinds
|
|
7950
|
+
extraReadOnlyBinds,
|
|
7951
|
+
detachProcessGroup: this.orphanCleanup && !this.useBwrap
|
|
6819
7952
|
}, options);
|
|
6820
7953
|
return this.runner.result;
|
|
6821
7954
|
}
|
|
@@ -6828,13 +7961,53 @@ var init_bare_metal_sandbox = __esmMin((() => {
|
|
|
6828
7961
|
if (this.runner) await this.runner.abort();
|
|
6829
7962
|
}
|
|
6830
7963
|
/**
|
|
6831
|
-
* Clean up the child process if still running.
|
|
7964
|
+
* Clean up the child process if still running. The handle reference is kept
|
|
7965
|
+
* (not nulled) so the between-jobs phase can still read `completionHooksRan` /
|
|
7966
|
+
* `declaresCleanup` and reap the process group after teardown — the group
|
|
7967
|
+
* survives the single child's death. The next `executeJob` overwrites it.
|
|
6832
7968
|
*/
|
|
6833
7969
|
async teardown() {
|
|
6834
|
-
if (this.runner)
|
|
6835
|
-
|
|
6836
|
-
|
|
6837
|
-
|
|
7970
|
+
if (this.runner) this.runner.kill();
|
|
7971
|
+
}
|
|
7972
|
+
/** Whether the runner signalled its completion hooks ran. */
|
|
7973
|
+
get completionHooksRan() {
|
|
7974
|
+
return this.runner?.completionHooksRan ?? true;
|
|
7975
|
+
}
|
|
7976
|
+
/** Whether the job declared an onFailure / cleanup hook. */
|
|
7977
|
+
get declaresCleanup() {
|
|
7978
|
+
return this.runner?.declaresCleanup ?? false;
|
|
7979
|
+
}
|
|
7980
|
+
/** Reap the finished job's process group. */
|
|
7981
|
+
async reap() {
|
|
7982
|
+
return this.runner?.reap() ?? 0;
|
|
7983
|
+
}
|
|
7984
|
+
/**
|
|
7985
|
+
* Re-run the finished job's declared cleanup / onFailure hooks against the
|
|
7986
|
+
* preserved workdir, in a fresh cleanup-only child. Reuses the last job's
|
|
7987
|
+
* dispatch + orchestrator relays but suppresses step/log callbacks (the
|
|
7988
|
+
* original job already reported and its log streamers are gone). Rejects when
|
|
7989
|
+
* the cleanup-only child fails so the caller can time it out.
|
|
7990
|
+
*/
|
|
7991
|
+
async runCleanupOnly(workDir, signal) {
|
|
7992
|
+
const opts = this.lastOptions;
|
|
7993
|
+
if (!opts) return;
|
|
7994
|
+
const handle = createForkRunner({
|
|
7995
|
+
runnerPath: this.runnerPath,
|
|
7996
|
+
env: this.env,
|
|
7997
|
+
useBwrap: this.useBwrap,
|
|
7998
|
+
workDir,
|
|
7999
|
+
networkIsolation: this.useBwrap && this.sandboxNetwork === "isolated",
|
|
8000
|
+
cleanupOnly: true,
|
|
8001
|
+
maxGracePeriodMs: 5e3
|
|
8002
|
+
}, {
|
|
8003
|
+
...opts,
|
|
8004
|
+
signal,
|
|
8005
|
+
onStepStatus: () => {},
|
|
8006
|
+
onLogLine: () => {}
|
|
8007
|
+
});
|
|
8008
|
+
const result = await handle.result;
|
|
8009
|
+
handle.kill();
|
|
8010
|
+
if (result.status !== ExecutionJobStatus.enum.success) throw new Error(result.error ?? `cleanup-only re-run did not succeed (${result.status})`);
|
|
6838
8011
|
}
|
|
6839
8012
|
};
|
|
6840
8013
|
}));
|
|
@@ -6895,6 +8068,77 @@ var init_firecracker_sandbox = __esmMin((() => {
|
|
|
6895
8068
|
};
|
|
6896
8069
|
}));
|
|
6897
8070
|
//#endregion
|
|
8071
|
+
//#region src/execution/sandbox/kici-runtime.ts
|
|
8072
|
+
/**
|
|
8073
|
+
* Launch the runner with the injected node, never the image's own.
|
|
8074
|
+
*
|
|
8075
|
+
* The customer image is not required to ship Node, so a bare `node` would
|
|
8076
|
+
* resolve to nothing (or, worse, to an unrelated build).
|
|
8077
|
+
*/
|
|
8078
|
+
function runnerLaunchArgv(runnerMountPath) {
|
|
8079
|
+
return [KICI_RUNTIME_NODE, runnerMountPath];
|
|
8080
|
+
}
|
|
8081
|
+
var KICI_RUNTIME_MOUNT, KICI_RUNTIME_NODE_DIR, KICI_RUNTIME_NODE;
|
|
8082
|
+
var init_kici_runtime = __esmMin((() => {
|
|
8083
|
+
KICI_RUNTIME_MOUNT = "/opt/kici";
|
|
8084
|
+
KICI_RUNTIME_NODE_DIR = `${KICI_RUNTIME_MOUNT}/node`;
|
|
8085
|
+
KICI_RUNTIME_NODE = `${KICI_RUNTIME_NODE_DIR}/bin/node`;
|
|
8086
|
+
}));
|
|
8087
|
+
//#endregion
|
|
8088
|
+
//#region src/execution/sandbox/image-preflight.ts
|
|
8089
|
+
/**
|
|
8090
|
+
* Classify an image from the subset of {@link PROBE_PATHS} that exist in it.
|
|
8091
|
+
*
|
|
8092
|
+
* Pure, so the decision table is testable without a container runtime.
|
|
8093
|
+
*/
|
|
8094
|
+
function classifyImageLibc(presentPaths) {
|
|
8095
|
+
const present = new Set(presentPaths);
|
|
8096
|
+
const hasMusl = MUSL_LOADERS.some((p) => present.has(p));
|
|
8097
|
+
const hasGlibc = GLIBC_LOADERS.some((p) => present.has(p));
|
|
8098
|
+
if (hasMusl) return "musl";
|
|
8099
|
+
if (!hasGlibc) return "static";
|
|
8100
|
+
return present.has(SHELL) ? "glibc" : "no-shell";
|
|
8101
|
+
}
|
|
8102
|
+
function rejection(image, libc) {
|
|
8103
|
+
switch (libc) {
|
|
8104
|
+
case "musl": return `image '${image}' uses musl libc; the musl runtime variant is not enabled (glibc images only in this version). Use a glibc image — for example the '-slim' rather than the '-alpine' tag.`;
|
|
8105
|
+
case "static": return `image '${image}' has no dynamic loader; container jobs require a glibc image with ${SHELL}.`;
|
|
8106
|
+
case "no-shell": return `image '${image}' has no ${SHELL}; container jobs require a shell for step commands.`;
|
|
8107
|
+
}
|
|
8108
|
+
}
|
|
8109
|
+
/**
|
|
8110
|
+
* Throw unless `image` can host the injected runtime.
|
|
8111
|
+
*
|
|
8112
|
+
* Stats the probe paths through a created-but-never-started container, so a
|
|
8113
|
+
* shell-less or musl image is diagnosed without executing anything in it —
|
|
8114
|
+
* running a probe command would fail for the very reason we are testing for.
|
|
8115
|
+
*/
|
|
8116
|
+
async function assertImageRunnable(docker, image) {
|
|
8117
|
+
const container = await docker.createContainer({ Image: image });
|
|
8118
|
+
try {
|
|
8119
|
+
const present = [];
|
|
8120
|
+
for (const path of PROBE_PATHS) try {
|
|
8121
|
+
await container.infoArchive({ path });
|
|
8122
|
+
present.push(path);
|
|
8123
|
+
} catch {}
|
|
8124
|
+
const libc = classifyImageLibc(present);
|
|
8125
|
+
if (libc !== "glibc") throw new Error(rejection(image, libc));
|
|
8126
|
+
} finally {
|
|
8127
|
+
await container.remove({ force: true }).catch(() => void 0);
|
|
8128
|
+
}
|
|
8129
|
+
}
|
|
8130
|
+
var MUSL_LOADERS, GLIBC_LOADERS, SHELL, PROBE_PATHS;
|
|
8131
|
+
var init_image_preflight = __esmMin((() => {
|
|
8132
|
+
MUSL_LOADERS = ["/lib/ld-musl-x86_64.so.1", "/lib/ld-musl-aarch64.so.1"];
|
|
8133
|
+
GLIBC_LOADERS = ["/lib64/ld-linux-x86-64.so.2", "/lib/ld-linux-aarch64.so.1"];
|
|
8134
|
+
SHELL = "/bin/sh";
|
|
8135
|
+
PROBE_PATHS = [
|
|
8136
|
+
...GLIBC_LOADERS,
|
|
8137
|
+
...MUSL_LOADERS,
|
|
8138
|
+
SHELL
|
|
8139
|
+
];
|
|
8140
|
+
}));
|
|
8141
|
+
//#endregion
|
|
6898
8142
|
//#region src/execution/sandbox/container-hardening.ts
|
|
6899
8143
|
/**
|
|
6900
8144
|
* Build the hardened HostConfig fragment for a job sandbox container.
|
|
@@ -6947,8 +8191,11 @@ var init_container_hardening = __esmMin((() => {
|
|
|
6947
8191
|
* - Agent-internal credentials (KICI_*, KICI_DATABASE_URL, etc.) NEVER enter the container
|
|
6948
8192
|
* - IPC uses demuxed Docker stream with JSON-line parsing on stdout
|
|
6949
8193
|
*
|
|
6950
|
-
* The container image
|
|
6951
|
-
*
|
|
8194
|
+
* The container image does NOT need Node.js or git. KiCI provisions its own
|
|
8195
|
+
* runtime — a pinned, official glibc-2.17 Node plus the runner bundle, mounted
|
|
8196
|
+
* read-only at /opt/kici — and launches the runner with THAT node. The image
|
|
8197
|
+
* needs only a glibc and a shell, which the preflight asserts before the
|
|
8198
|
+
* container is created.
|
|
6952
8199
|
*/
|
|
6953
8200
|
/**
|
|
6954
8201
|
* Relay event.emit from the container runner to the orchestrator via
|
|
@@ -7048,6 +8295,30 @@ function relayCacheRequest(stream, options, cacheMsg) {
|
|
|
7048
8295
|
}));
|
|
7049
8296
|
}
|
|
7050
8297
|
/**
|
|
8298
|
+
* Relay git.grant.request from the container runner to the agent's grant table
|
|
8299
|
+
* via options.onGitGrantRequest, then write the response back through `stream`.
|
|
8300
|
+
*/
|
|
8301
|
+
function relayGitGrantRequest(stream, options, grantMsg) {
|
|
8302
|
+
const writeResponse = (response) => {
|
|
8303
|
+
try {
|
|
8304
|
+
stream.write(JSON.stringify(response) + "\n");
|
|
8305
|
+
} catch {}
|
|
8306
|
+
};
|
|
8307
|
+
if (!options.onGitGrantRequest) {
|
|
8308
|
+
writeResponse({
|
|
8309
|
+
type: "git.grant.response",
|
|
8310
|
+
requestId: grantMsg.requestId,
|
|
8311
|
+
error: "Git credentials are not available in this agent configuration"
|
|
8312
|
+
});
|
|
8313
|
+
return;
|
|
8314
|
+
}
|
|
8315
|
+
options.onGitGrantRequest(grantMsg).then((response) => writeResponse(response), (err) => writeResponse({
|
|
8316
|
+
type: "git.grant.response",
|
|
8317
|
+
requestId: grantMsg.requestId,
|
|
8318
|
+
error: toErrorMessage(err)
|
|
8319
|
+
}));
|
|
8320
|
+
}
|
|
8321
|
+
/**
|
|
7051
8322
|
* Relay provenance.request from the container runner to the orchestrator via
|
|
7052
8323
|
* options.onProvenanceRequest, then write the response back through `stream`.
|
|
7053
8324
|
* If the agent doesn't expose a provenance relay, write a structured error so
|
|
@@ -7140,15 +8411,17 @@ function applyJobComplete(msg, stepResults, state, options) {
|
|
|
7140
8411
|
if (msg.secretOutputs && options.dispatch.runPublicKey) try {
|
|
7141
8412
|
state.encryptedSecretOutputs = encryptSecretOutputs(msg.secretOutputs, options.dispatch.runPublicKey);
|
|
7142
8413
|
} catch (err) {
|
|
7143
|
-
logger$
|
|
8414
|
+
logger$4.warn("Failed to encrypt secret outputs", { error: toErrorMessage(err) });
|
|
7144
8415
|
}
|
|
7145
8416
|
}
|
|
7146
|
-
var logger$
|
|
8417
|
+
var logger$4, MAX_STDERR_LINES, ABORT_GRACE_MS, CONTAINER_STOP_TIMEOUT, HOOK_MOUNT_PATH, INTERNAL_ENV, ContainerSandbox;
|
|
7147
8418
|
var init_container_sandbox = __esmMin((() => {
|
|
7148
8419
|
init_fork_runner();
|
|
8420
|
+
init_kici_runtime();
|
|
8421
|
+
init_image_preflight();
|
|
7149
8422
|
init_secret_encryption();
|
|
7150
8423
|
init_container_hardening();
|
|
7151
|
-
logger$
|
|
8424
|
+
logger$4 = createLogger({ prefix: "container-sandbox" });
|
|
7152
8425
|
MAX_STDERR_LINES = 20;
|
|
7153
8426
|
ABORT_GRACE_MS = 1e4;
|
|
7154
8427
|
CONTAINER_STOP_TIMEOUT = 10;
|
|
@@ -7158,6 +8431,17 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7158
8431
|
docker;
|
|
7159
8432
|
image;
|
|
7160
8433
|
runnerPath;
|
|
8434
|
+
runtimeNodePath;
|
|
8435
|
+
runtimeImage;
|
|
8436
|
+
buildTag;
|
|
8437
|
+
/**
|
|
8438
|
+
* The runtime actually injected into this job's container — the configured
|
|
8439
|
+
* path, or the volume materialized during setup. Resolved once in setup()
|
|
8440
|
+
* because materialization needs the container runtime, and read by both the
|
|
8441
|
+
* bind list and the runner launch.
|
|
8442
|
+
*/
|
|
8443
|
+
resolvedRuntimeNode;
|
|
8444
|
+
registryAuth;
|
|
7161
8445
|
runnerMountPath;
|
|
7162
8446
|
/** Host path to the pure-JS container loader-hook bundle (bind-mounted :ro). */
|
|
7163
8447
|
hookHostPath;
|
|
@@ -7179,6 +8463,10 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7179
8463
|
this.docker = options.docker;
|
|
7180
8464
|
this.image = options.image;
|
|
7181
8465
|
this.runnerPath = options.runnerPath;
|
|
8466
|
+
this.runtimeNodePath = options.runtimeNodePath;
|
|
8467
|
+
this.runtimeImage = options.runtimeImage;
|
|
8468
|
+
this.buildTag = options.buildTag;
|
|
8469
|
+
this.registryAuth = options.registryAuth;
|
|
7182
8470
|
this.runnerMountPath = options.runnerMountPath ?? "/opt/kici/workflow-runner.js";
|
|
7183
8471
|
this.hookHostPath = options.hookPath ?? join(dirname(options.runnerPath), "container-ts-loader-hook.js");
|
|
7184
8472
|
this.env = options.env;
|
|
@@ -7202,17 +8490,44 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7202
8490
|
await this.docker.getImage(this.image).inspect();
|
|
7203
8491
|
return;
|
|
7204
8492
|
} catch {}
|
|
7205
|
-
logger$
|
|
7206
|
-
|
|
8493
|
+
logger$4.info("Pulling sandbox image (not present locally)", {
|
|
8494
|
+
image: this.image,
|
|
8495
|
+
authenticated: this.registryAuth !== void 0
|
|
8496
|
+
});
|
|
8497
|
+
const stream = await this.docker.pull(this.image, this.registryAuth ? { authconfig: this.registryAuth } : {});
|
|
7207
8498
|
await new Promise((resolve, reject) => {
|
|
7208
8499
|
this.docker.modem.followProgress(stream, (err) => err ? reject(err) : resolve());
|
|
7209
8500
|
});
|
|
7210
|
-
logger$
|
|
8501
|
+
logger$4.info("Sandbox image pulled", { image: this.image });
|
|
8502
|
+
}
|
|
8503
|
+
/**
|
|
8504
|
+
* Resolve the Node tree to inject into the job container, materializing it
|
|
8505
|
+
* when only an image was configured.
|
|
8506
|
+
*
|
|
8507
|
+
* A configured path wins: a caller that already provisioned the tree should
|
|
8508
|
+
* not pay a copy to arrive at the same one. Returning `undefined` is a real
|
|
8509
|
+
* outcome, not a failure — an agent with no runtime source runs the job on
|
|
8510
|
+
* the image's own `node`, which is what a `node:*` image was always doing.
|
|
8511
|
+
*
|
|
8512
|
+
* A materialization that FAILS is not softened into that outcome. Continuing
|
|
8513
|
+
* would start the job against an image the operator never claimed ships Node,
|
|
8514
|
+
* and the resulting "node: not found" says nothing about the runtime that was
|
|
8515
|
+
* supposed to be there.
|
|
8516
|
+
*/
|
|
8517
|
+
async resolveRuntimeNode() {
|
|
8518
|
+
if (this.runtimeNodePath) return this.runtimeNodePath;
|
|
8519
|
+
if (!this.runtimeImage) return void 0;
|
|
8520
|
+
return await ensureRuntimeVolume({
|
|
8521
|
+
docker: this.docker,
|
|
8522
|
+
agentImage: this.runtimeImage,
|
|
8523
|
+
subtree: RuntimeSubtree.enum.node,
|
|
8524
|
+
onProgress: (message) => logger$4.info(message, { jobId: this.jobId })
|
|
8525
|
+
});
|
|
7211
8526
|
}
|
|
7212
8527
|
async setup(options) {
|
|
7213
8528
|
this.containerName = `kici-sandbox-${this.jobId}-${Date.now()}`;
|
|
7214
8529
|
const envArray = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
|
|
7215
|
-
logger$
|
|
8530
|
+
logger$4.info("Creating sandbox container", {
|
|
7216
8531
|
name: this.containerName,
|
|
7217
8532
|
image: this.image,
|
|
7218
8533
|
workDir: options.workDir
|
|
@@ -7222,8 +8537,10 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7222
8537
|
user: void 0
|
|
7223
8538
|
};
|
|
7224
8539
|
this.resolvedUser = hardened.user;
|
|
8540
|
+
this.resolvedRuntimeNode = await this.resolveRuntimeNode();
|
|
7225
8541
|
const binds = this.buildBinds(options, hardened.hostConfig);
|
|
7226
8542
|
await this.ensureImagePresent();
|
|
8543
|
+
if (this.resolvedRuntimeNode) await assertImageRunnable(this.docker, this.image);
|
|
7227
8544
|
this.container = await this.docker.createContainer({
|
|
7228
8545
|
Image: this.image,
|
|
7229
8546
|
name: this.containerName,
|
|
@@ -7242,10 +8559,42 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7242
8559
|
}
|
|
7243
8560
|
});
|
|
7244
8561
|
await this.container.start();
|
|
7245
|
-
logger$
|
|
8562
|
+
logger$4.info("Sandbox container started", {
|
|
7246
8563
|
name: this.containerName,
|
|
7247
8564
|
containerId: this.container.id.slice(0, 12)
|
|
7248
8565
|
});
|
|
8566
|
+
if (options.workspaceFromHost) await this.copyWorkspaceIn(options.workDir);
|
|
8567
|
+
}
|
|
8568
|
+
/**
|
|
8569
|
+
* Populate the container's `/workspace` volume from the host working tree.
|
|
8570
|
+
*
|
|
8571
|
+
* `/workspace` is a container-owned anonymous volume rather than a host bind,
|
|
8572
|
+
* which is what dissolves the host-uid vs container-uid conflict once
|
|
8573
|
+
* `CapDrop: ['ALL']` removes CAP_DAC_OVERRIDE — so the tree is streamed in as
|
|
8574
|
+
* a tar rather than mounted.
|
|
8575
|
+
*
|
|
8576
|
+
* A failure here is fatal on purpose. Swallowing it would start the job
|
|
8577
|
+
* against an EMPTY workspace, which surfaces as a baffling "file not found"
|
|
8578
|
+
* in whichever step happens to touch the repo first.
|
|
8579
|
+
*/
|
|
8580
|
+
async copyWorkspaceIn(workDir) {
|
|
8581
|
+
const started = Date.now();
|
|
8582
|
+
try {
|
|
8583
|
+
const stream = c({
|
|
8584
|
+
cwd: workDir,
|
|
8585
|
+
portable: true
|
|
8586
|
+
}, ["."]);
|
|
8587
|
+
const streamFailed = new Promise((_, reject) => {
|
|
8588
|
+
stream.on("error", reject);
|
|
8589
|
+
});
|
|
8590
|
+
await Promise.race([this.container.putArchive(stream, { path: "/workspace" }), streamFailed]);
|
|
8591
|
+
} catch (err) {
|
|
8592
|
+
throw new Error(`Failed to copy the host workspace into the sandbox container: ${err instanceof Error ? err.message : String(err)}`);
|
|
8593
|
+
}
|
|
8594
|
+
logger$4.info("Copied host workspace into sandbox", {
|
|
8595
|
+
name: this.containerName,
|
|
8596
|
+
durationMs: Date.now() - started
|
|
8597
|
+
});
|
|
7249
8598
|
}
|
|
7250
8599
|
/**
|
|
7251
8600
|
* Build the container's read-only bind list.
|
|
@@ -7271,6 +8620,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7271
8620
|
*/
|
|
7272
8621
|
buildBinds(options, hostConfig) {
|
|
7273
8622
|
const binds = [`${this.runnerPath}:${this.runnerMountPath}:ro`, `${this.hookHostPath}:${HOOK_MOUNT_PATH}:ro`];
|
|
8623
|
+
if (this.resolvedRuntimeNode) binds.push(`${this.resolvedRuntimeNode}:${KICI_RUNTIME_NODE_DIR}:ro`);
|
|
7274
8624
|
for (const dir of options.extraReadOnlyBinds ?? []) if (dir) binds.push(`${dir}:${dir}:ro`);
|
|
7275
8625
|
if (hostConfig.NetworkMode === "host") {
|
|
7276
8626
|
for (const nssFile of ["/etc/hosts", "/etc/nsswitch.conf"]) if (existsSync(nssFile)) binds.push(`${nssFile}:${nssFile}:ro`);
|
|
@@ -7285,7 +8635,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7285
8635
|
try {
|
|
7286
8636
|
outcome = await this.awaitJobCompletion(streamCtx, options);
|
|
7287
8637
|
} catch (err) {
|
|
7288
|
-
logger$
|
|
8638
|
+
logger$4.error("Job execution error", {
|
|
7289
8639
|
error: toErrorMessage(err),
|
|
7290
8640
|
stderrTail: streamCtx.stderrLines.slice(-5).join("\n")
|
|
7291
8641
|
});
|
|
@@ -7310,7 +8660,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7310
8660
|
async attachExecStream(options) {
|
|
7311
8661
|
const execEnv = [...Object.entries(this.env).map(([k, v]) => `${k}=${v}`), ...INTERNAL_ENV];
|
|
7312
8662
|
const stream = await (await this.container.exec({
|
|
7313
|
-
Cmd: ["node", this.runnerMountPath],
|
|
8663
|
+
Cmd: this.resolvedRuntimeNode ? runnerLaunchArgv(this.runnerMountPath) : ["node", this.runnerMountPath],
|
|
7314
8664
|
AttachStdin: true,
|
|
7315
8665
|
AttachStdout: true,
|
|
7316
8666
|
AttachStderr: true,
|
|
@@ -7336,7 +8686,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7336
8686
|
});
|
|
7337
8687
|
const abortHandler = () => {
|
|
7338
8688
|
this.handleAbort().catch((err) => {
|
|
7339
|
-
logger$
|
|
8689
|
+
logger$4.warn("Error during abort", { error: toErrorMessage(err) });
|
|
7340
8690
|
});
|
|
7341
8691
|
};
|
|
7342
8692
|
options.signal.addEventListener("abort", abortHandler, { once: true });
|
|
@@ -7373,7 +8723,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7373
8723
|
try {
|
|
7374
8724
|
msg = JSON.parse(line);
|
|
7375
8725
|
} catch {
|
|
7376
|
-
logger$
|
|
8726
|
+
logger$4.warn("Non-JSON output from runner", { line: line.slice(0, 200) });
|
|
7377
8727
|
return;
|
|
7378
8728
|
}
|
|
7379
8729
|
if (this.dispatchRunnerMessage(msg, stream, options, stepNames, stepResults, state)) resolve({
|
|
@@ -7467,6 +8817,9 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7467
8817
|
case "agent.api.request":
|
|
7468
8818
|
relayApiRequest(stream, options, msg);
|
|
7469
8819
|
return false;
|
|
8820
|
+
case "git.grant.request":
|
|
8821
|
+
relayGitGrantRequest(stream, options, msg);
|
|
8822
|
+
return false;
|
|
7470
8823
|
case "cache.request":
|
|
7471
8824
|
relayCacheRequest(stream, options, msg);
|
|
7472
8825
|
return false;
|
|
@@ -7482,8 +8835,10 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7482
8835
|
case "job.complete":
|
|
7483
8836
|
applyJobComplete(msg, stepResults, state, options);
|
|
7484
8837
|
return true;
|
|
8838
|
+
case "hooks-declared":
|
|
8839
|
+
case "completion-hooks-done": return false;
|
|
7485
8840
|
default:
|
|
7486
|
-
logger$
|
|
8841
|
+
logger$4.warn("Unrecognized IPC message from container runner", { type: msg.type });
|
|
7487
8842
|
return false;
|
|
7488
8843
|
}
|
|
7489
8844
|
}
|
|
@@ -7507,17 +8862,33 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7507
8862
|
async abort() {
|
|
7508
8863
|
await this.handleAbort();
|
|
7509
8864
|
}
|
|
8865
|
+
/**
|
|
8866
|
+
* Drop the tag a `container.dockerfile` build produced.
|
|
8867
|
+
*
|
|
8868
|
+
* Best-effort: teardown must not fail a job that already finished. The LAYER
|
|
8869
|
+
* cache — the thing that makes the next build fast — is not a tag and is
|
|
8870
|
+
* untouched by this.
|
|
8871
|
+
*/
|
|
8872
|
+
async reclaimBuiltImage() {
|
|
8873
|
+
if (!this.buildTag) return;
|
|
8874
|
+
try {
|
|
8875
|
+
await this.docker.getImage(this.buildTag).remove({ force: true });
|
|
8876
|
+
} catch {}
|
|
8877
|
+
}
|
|
7510
8878
|
async teardown() {
|
|
7511
|
-
if (!this.container)
|
|
8879
|
+
if (!this.container) {
|
|
8880
|
+
await this.reclaimBuiltImage();
|
|
8881
|
+
return;
|
|
8882
|
+
}
|
|
7512
8883
|
if (this.keepFailed && this.jobFailed) {
|
|
7513
|
-
logger$
|
|
8884
|
+
logger$4.info("Keeping failed container for debugging", {
|
|
7514
8885
|
name: this.containerName,
|
|
7515
8886
|
containerId: this.container.id.slice(0, 12)
|
|
7516
8887
|
});
|
|
7517
8888
|
this.container = null;
|
|
7518
8889
|
return;
|
|
7519
8890
|
}
|
|
7520
|
-
logger$
|
|
8891
|
+
logger$4.info("Tearing down sandbox container", { name: this.containerName });
|
|
7521
8892
|
try {
|
|
7522
8893
|
await this.container.stop({ t: CONTAINER_STOP_TIMEOUT });
|
|
7523
8894
|
} catch {}
|
|
@@ -7527,6 +8898,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7527
8898
|
v: true
|
|
7528
8899
|
});
|
|
7529
8900
|
} catch {}
|
|
8901
|
+
await this.reclaimBuiltImage();
|
|
7530
8902
|
this.container = null;
|
|
7531
8903
|
}
|
|
7532
8904
|
/**
|
|
@@ -7551,7 +8923,7 @@ var init_container_sandbox = __esmMin((() => {
|
|
|
7551
8923
|
*/
|
|
7552
8924
|
async handleAbort() {
|
|
7553
8925
|
if (!this.execStream && !this.container) return;
|
|
7554
|
-
logger$
|
|
8926
|
+
logger$4.info("Aborting sandbox execution", { name: this.containerName });
|
|
7555
8927
|
if (this.execStream) try {
|
|
7556
8928
|
this.execStream.write(JSON.stringify({ type: "abort" }) + "\n");
|
|
7557
8929
|
} catch {}
|
|
@@ -7823,6 +9195,7 @@ function resolveRunnerBundlePath(runnerPath) {
|
|
|
7823
9195
|
* 4. Default -> 'bare-metal'
|
|
7824
9196
|
*/
|
|
7825
9197
|
function determineExecutionMode(jobConfig, agentConfig) {
|
|
9198
|
+
if (agentConfig.jobImageAgent) return "bare-metal";
|
|
7826
9199
|
if (jobConfig.container) return "container";
|
|
7827
9200
|
if (agentConfig.executionMode) return agentConfig.executionMode;
|
|
7828
9201
|
if (agentConfig.scalerManaged) return "firecracker";
|
|
@@ -7863,9 +9236,11 @@ async function resolveJobWorkDir(inPlace, repoUrl) {
|
|
|
7863
9236
|
inPlace: false
|
|
7864
9237
|
};
|
|
7865
9238
|
}
|
|
7866
|
-
var logger$
|
|
9239
|
+
var logger$3, DEFAULT_GLOBAL_EVAL_ROUND_TIMEOUT_MS, DEFAULT_GLOBAL_EVAL_CANDIDATE_TIMEOUT_MS, FILTER_SOURCE_DIRNAME, JobRunner$1;
|
|
7867
9240
|
var init_job_runner = __esmMin((() => {
|
|
7868
9241
|
init_git_clone();
|
|
9242
|
+
init_clone_job_repos();
|
|
9243
|
+
init_job_git_credentials();
|
|
7869
9244
|
init_changed_files();
|
|
7870
9245
|
init_workflow_loader();
|
|
7871
9246
|
init_source_packer();
|
|
@@ -7882,6 +9257,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
7882
9257
|
init_global_workflow_env();
|
|
7883
9258
|
init_global_eval_runner();
|
|
7884
9259
|
init_log_streamer();
|
|
9260
|
+
init_build_step();
|
|
9261
|
+
init_build_engine();
|
|
7885
9262
|
init_overlay_applier();
|
|
7886
9263
|
init_dep_installer();
|
|
7887
9264
|
init_dep_restore();
|
|
@@ -7889,7 +9266,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
7889
9266
|
init_download();
|
|
7890
9267
|
init_sandbox();
|
|
7891
9268
|
init_prometheus();
|
|
7892
|
-
logger$
|
|
9269
|
+
logger$3 = createLogger({ prefix: "job-runner" });
|
|
7893
9270
|
DEFAULT_GLOBAL_EVAL_ROUND_TIMEOUT_MS = 12e4;
|
|
7894
9271
|
DEFAULT_GLOBAL_EVAL_CANDIDATE_TIMEOUT_MS = 2e4;
|
|
7895
9272
|
FILTER_SOURCE_DIRNAME = "__kici_filter_source__";
|
|
@@ -7905,6 +9282,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
7905
9282
|
_sendRunEvent;
|
|
7906
9283
|
_sendConcurrencyReport;
|
|
7907
9284
|
_sendApiRequest;
|
|
9285
|
+
/** Live git credentials per job, so the sandbox can reach the grant table. */
|
|
9286
|
+
jobGitCredentials = /* @__PURE__ */ new Map();
|
|
7908
9287
|
_requestUserCache;
|
|
7909
9288
|
_relayProvenance;
|
|
7910
9289
|
_requestUserArtifact;
|
|
@@ -7913,9 +9292,19 @@ var init_job_runner = __esmMin((() => {
|
|
|
7913
9292
|
activeJobs = /* @__PURE__ */ new Map();
|
|
7914
9293
|
/** Active sandbox for the current job (used for abort). */
|
|
7915
9294
|
activeSandbox = null;
|
|
9295
|
+
/** Supervisor-owned between-jobs phase (reap / cleanup re-run / reset). */
|
|
9296
|
+
betweenJobsController;
|
|
9297
|
+
/**
|
|
9298
|
+
* Facts about the just-finished standard job, captured before sandbox
|
|
9299
|
+
* teardown so the between-jobs controller (run in `execute`'s `.finally`) can
|
|
9300
|
+
* reach them. Null for special job types (init / build / dynamic), which run
|
|
9301
|
+
* no sandbox — the controller then only deletes the workdir and resets.
|
|
9302
|
+
*/
|
|
9303
|
+
betweenJobsFacts = null;
|
|
7916
9304
|
constructor(deps) {
|
|
7917
9305
|
this.send = deps.send;
|
|
7918
9306
|
this.config = deps.config;
|
|
9307
|
+
this.betweenJobsController = deps.betweenJobsController;
|
|
7919
9308
|
this.requestUploadUrl = deps.requestUploadUrl;
|
|
7920
9309
|
this.sendUploadComplete = deps.sendUploadComplete;
|
|
7921
9310
|
this.sendEventEmit = deps.sendEventEmit;
|
|
@@ -7941,10 +9330,16 @@ var init_job_runner = __esmMin((() => {
|
|
|
7941
9330
|
const abortController = new AbortController();
|
|
7942
9331
|
const { workDir, cleanup, inPlace } = await resolveJobWorkDir(this.config.inPlace, dispatch.repoUrl);
|
|
7943
9332
|
if (inPlace) dispatch.jobConfig.checkout = false;
|
|
9333
|
+
this.betweenJobsFacts = null;
|
|
9334
|
+
const gitCredentials = await this.startGitCredentials(jobId, dispatch.jobConfig);
|
|
7944
9335
|
const completionPromise = this.runJob(dispatch, workDir, abortController).finally(async () => {
|
|
9336
|
+
const facts = this.betweenJobsFacts;
|
|
7945
9337
|
this.activeJobs.delete(jobId);
|
|
7946
9338
|
this.activeSandbox = null;
|
|
7947
|
-
|
|
9339
|
+
this.betweenJobsFacts = null;
|
|
9340
|
+
this.jobGitCredentials.delete(jobId);
|
|
9341
|
+
await gitCredentials?.close().catch(() => {});
|
|
9342
|
+
await this.runBetweenJobsPhase(facts, workDir, cleanup);
|
|
7948
9343
|
});
|
|
7949
9344
|
this.activeJobs.set(jobId, {
|
|
7950
9345
|
abortController,
|
|
@@ -7954,6 +9349,63 @@ var init_job_runner = __esmMin((() => {
|
|
|
7954
9349
|
return completionPromise;
|
|
7955
9350
|
}
|
|
7956
9351
|
/**
|
|
9352
|
+
* Stand up per-job git credentials, or `undefined` when the agent has no
|
|
9353
|
+
* orchestrator relay (unit harnesses, offline local runs).
|
|
9354
|
+
*
|
|
9355
|
+
* A failure here must not fail the job: without a helper, git falls back to
|
|
9356
|
+
* its own mechanisms exactly as it did before this existed. The job simply
|
|
9357
|
+
* cannot push.
|
|
9358
|
+
*/
|
|
9359
|
+
async startGitCredentials(jobId, jobConfig) {
|
|
9360
|
+
if (!this._sendApiRequest) return void 0;
|
|
9361
|
+
try {
|
|
9362
|
+
const { path: dir } = await makeTempDir(`gitcred-${jobId}`);
|
|
9363
|
+
const credentials = await startJobGitCredentials({
|
|
9364
|
+
jobId,
|
|
9365
|
+
dir,
|
|
9366
|
+
sendApiRequest: this._sendApiRequest,
|
|
9367
|
+
credentials: jobConfig.gitCredentials
|
|
9368
|
+
});
|
|
9369
|
+
this.jobGitCredentials.set(jobId, credentials);
|
|
9370
|
+
return credentials;
|
|
9371
|
+
} catch (err) {
|
|
9372
|
+
logger$3.warn("Could not start git credentials for job; git push will not work", {
|
|
9373
|
+
jobId,
|
|
9374
|
+
error: toErrorMessage(err)
|
|
9375
|
+
});
|
|
9376
|
+
return;
|
|
9377
|
+
}
|
|
9378
|
+
}
|
|
9379
|
+
/**
|
|
9380
|
+
* Run the supervisor-owned between-jobs phase after a job finishes. Delegates
|
|
9381
|
+
* to the injected `BetweenJobsController` (out-of-band cleanup → reap →
|
|
9382
|
+
* workdir delete → operator reset) with facts captured before sandbox
|
|
9383
|
+
* teardown. When no controller is wired (unit harnesses) it just deletes the
|
|
9384
|
+
* workdir, preserving the historical behavior. Never throws.
|
|
9385
|
+
*/
|
|
9386
|
+
async runBetweenJobsPhase(facts, workDir, deleteWorkdir) {
|
|
9387
|
+
if (!this.betweenJobsController) {
|
|
9388
|
+
await deleteWorkdir().catch((err) => logger$3.warn("Work directory cleanup error", { error: toErrorMessage(err) }));
|
|
9389
|
+
return;
|
|
9390
|
+
}
|
|
9391
|
+
const ctx = {
|
|
9392
|
+
completionHooksRan: facts?.completionHooksRan ?? true,
|
|
9393
|
+
jobFailed: facts?.jobFailed ?? false,
|
|
9394
|
+
backend: facts?.backend ?? "bare-metal",
|
|
9395
|
+
declaresCleanup: facts?.declaresCleanup ?? false,
|
|
9396
|
+
workDir,
|
|
9397
|
+
reap: facts?.reap ?? (() => Promise.resolve(0)),
|
|
9398
|
+
deleteWorkdir,
|
|
9399
|
+
...facts?.cleanupSpawn ? { cleanupSpawn: facts.cleanupSpawn } : {}
|
|
9400
|
+
};
|
|
9401
|
+
try {
|
|
9402
|
+
await this.betweenJobsController.afterJob(ctx);
|
|
9403
|
+
} catch (err) {
|
|
9404
|
+
logger$3.error("Between-jobs phase error", { error: toErrorMessage(err) });
|
|
9405
|
+
await deleteWorkdir().catch(() => {});
|
|
9406
|
+
}
|
|
9407
|
+
}
|
|
9408
|
+
/**
|
|
7957
9409
|
* Cancel a running job by signaling its abort controller
|
|
7958
9410
|
* and aborting the active sandbox.
|
|
7959
9411
|
*
|
|
@@ -8002,7 +9454,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8002
9454
|
}
|
|
8003
9455
|
if (jobConfig.buildOnly === true) {
|
|
8004
9456
|
if (jobConfig.fullRepo) {
|
|
8005
|
-
logger$
|
|
9457
|
+
logger$3.warn("Build job received for fullRepo run -- skipping (should not happen)", {
|
|
8006
9458
|
jobId,
|
|
8007
9459
|
runId
|
|
8008
9460
|
});
|
|
@@ -8024,7 +9476,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8024
9476
|
async executeStandardJob(dispatch, workDir, abortController) {
|
|
8025
9477
|
const { runId, jobId } = dispatch;
|
|
8026
9478
|
const ctx = getRequestContext();
|
|
8027
|
-
logger$
|
|
9479
|
+
logger$3.info(`Run: ${ctx.runId ?? runId} | Trace: ${ctx.requestId ?? "N/A"}`);
|
|
8028
9480
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
|
|
8029
9481
|
const heartbeatTimer = setInterval(() => {
|
|
8030
9482
|
this.send({
|
|
@@ -8035,12 +9487,16 @@ var init_job_runner = __esmMin((() => {
|
|
|
8035
9487
|
});
|
|
8036
9488
|
}, this.config.jobHeartbeatIntervalMs);
|
|
8037
9489
|
let sandbox;
|
|
9490
|
+
let backend = "bare-metal";
|
|
9491
|
+
let jobFailed = true;
|
|
8038
9492
|
const logStreamers = /* @__PURE__ */ new Map();
|
|
8039
9493
|
try {
|
|
8040
9494
|
const setupResult = await this.setupSandboxForExecution(dispatch, workDir, abortController);
|
|
8041
9495
|
if (!setupResult) return;
|
|
8042
9496
|
sandbox = setupResult.sandbox;
|
|
9497
|
+
backend = setupResult.executionMode;
|
|
8043
9498
|
const result = await this.runSandboxExecution(dispatch, sandbox, abortController, logStreamers);
|
|
9499
|
+
jobFailed = result.status !== ExecutionJobStatus.enum.success;
|
|
8044
9500
|
this.reportExecutionResult(dispatch, result, logStreamers);
|
|
8045
9501
|
} catch (error) {
|
|
8046
9502
|
for (const streamer of logStreamers.values()) streamer.destroy();
|
|
@@ -8049,9 +9505,18 @@ var init_job_runner = __esmMin((() => {
|
|
|
8049
9505
|
} finally {
|
|
8050
9506
|
clearInterval(heartbeatTimer);
|
|
8051
9507
|
if (sandbox) {
|
|
9508
|
+
const jobSandbox = sandbox;
|
|
9509
|
+
this.betweenJobsFacts = {
|
|
9510
|
+
completionHooksRan: jobSandbox.completionHooksRan ?? true,
|
|
9511
|
+
declaresCleanup: jobSandbox.declaresCleanup ?? false,
|
|
9512
|
+
backend,
|
|
9513
|
+
jobFailed,
|
|
9514
|
+
reap: () => jobSandbox.reap?.() ?? Promise.resolve(0),
|
|
9515
|
+
...jobSandbox.runCleanupOnly ? { cleanupSpawn: (wd, sig) => jobSandbox.runCleanupOnly(wd, sig) } : {}
|
|
9516
|
+
};
|
|
8052
9517
|
this.emitRunEvent(runId, "agent.teardown", { jobId });
|
|
8053
9518
|
await sandbox.teardown().catch((err) => {
|
|
8054
|
-
logger$
|
|
9519
|
+
logger$3.warn("Sandbox teardown error", { error: toErrorMessage(err) });
|
|
8055
9520
|
});
|
|
8056
9521
|
}
|
|
8057
9522
|
}
|
|
@@ -8071,7 +9536,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
8071
9536
|
}
|
|
8072
9537
|
const executionMode = determineExecutionMode(jobConfig, {
|
|
8073
9538
|
executionMode: this.config.executionMode,
|
|
8074
|
-
scalerManaged: this.config.scalerManaged
|
|
9539
|
+
scalerManaged: this.config.scalerManaged,
|
|
9540
|
+
jobImageAgent: this.config.jobImageAgent
|
|
8075
9541
|
});
|
|
8076
9542
|
const runnerPath = resolveRunnerPath();
|
|
8077
9543
|
const typedConfig = jobConfig;
|
|
@@ -8080,7 +9546,22 @@ var init_job_runner = __esmMin((() => {
|
|
|
8080
9546
|
jobEnv: typedConfig.jobEnv ?? void 0,
|
|
8081
9547
|
trustedEnv: this.config.trustedEnv
|
|
8082
9548
|
});
|
|
8083
|
-
|
|
9549
|
+
const hostCheckout = executionMode === "container" && jobConfig.checkout !== false;
|
|
9550
|
+
if (hostCheckout) {
|
|
9551
|
+
const isGlobal = jobConfig.isGlobalWorkflow === true;
|
|
9552
|
+
await cloneJobRepos(dispatch, {
|
|
9553
|
+
workDir,
|
|
9554
|
+
workflowDir: isGlobal ? join(workDir, "workflow") : workDir,
|
|
9555
|
+
sourceDir: isGlobal ? join(workDir, "source") : workDir
|
|
9556
|
+
}, {
|
|
9557
|
+
isGlobal,
|
|
9558
|
+
log: (line) => logger$3.info(`[host-checkout] ${line}`, { jobId }),
|
|
9559
|
+
excludeScratchFromGit
|
|
9560
|
+
});
|
|
9561
|
+
jobConfig.checkout = false;
|
|
9562
|
+
}
|
|
9563
|
+
const builtImage = await this.buildJobImageIfDeclared(dispatch, jobConfig, workDir, abortController);
|
|
9564
|
+
logger$3.info("Creating execution sandbox", {
|
|
8084
9565
|
executionMode,
|
|
8085
9566
|
jobId,
|
|
8086
9567
|
runnerPath
|
|
@@ -8089,13 +9570,16 @@ var init_job_runner = __esmMin((() => {
|
|
|
8089
9570
|
runnerPath,
|
|
8090
9571
|
env: sanitizedEnv,
|
|
8091
9572
|
jobId,
|
|
8092
|
-
jobConfig
|
|
9573
|
+
jobConfig,
|
|
9574
|
+
registryAuth: dispatch.containerRegistryAuth,
|
|
9575
|
+
...builtImage ? { builtImage } : {}
|
|
8093
9576
|
});
|
|
8094
9577
|
this.activeSandbox = sandbox;
|
|
8095
9578
|
await sandbox.setup({
|
|
8096
9579
|
workDir,
|
|
8097
9580
|
env: sanitizedEnv,
|
|
8098
|
-
extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl)
|
|
9581
|
+
extraReadOnlyBinds: fileCloneSourceBinds(dispatch.repoUrl),
|
|
9582
|
+
...hostCheckout ? { workspaceFromHost: true } : {}
|
|
8099
9583
|
});
|
|
8100
9584
|
if (abortController.signal.aborted) {
|
|
8101
9585
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.cancelled);
|
|
@@ -8114,7 +9598,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
8114
9598
|
});
|
|
8115
9599
|
return {
|
|
8116
9600
|
sandbox,
|
|
8117
|
-
sanitizedEnv
|
|
9601
|
+
sanitizedEnv,
|
|
9602
|
+
executionMode
|
|
8118
9603
|
};
|
|
8119
9604
|
}
|
|
8120
9605
|
/**
|
|
@@ -8176,7 +9661,12 @@ var init_job_runner = __esmMin((() => {
|
|
|
8176
9661
|
reason: ack.reason
|
|
8177
9662
|
};
|
|
8178
9663
|
},
|
|
8179
|
-
onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) =>
|
|
9664
|
+
onApiRequest: this._sendApiRequest ? withBootstrapInterception(async (method, params) => {
|
|
9665
|
+
const patched = method === GIT_CREDENTIAL_REQUEST_METHOD ? this.jobGitCredentials.get(jobId)?.withRef(params ?? {}) ?? params : params;
|
|
9666
|
+
return this._sendApiRequest(method, patched);
|
|
9667
|
+
}) : void 0,
|
|
9668
|
+
onGitGrantRequest: this.jobGitCredentials.get(jobId)?.onGitGrantRequest,
|
|
9669
|
+
credentialHelperPath: this.jobGitCredentials.get(jobId)?.helperPath,
|
|
8180
9670
|
onCacheRequest: this._requestUserCache ? async (request) => this._requestUserCache(jobId, request) : void 0,
|
|
8181
9671
|
onProvenanceRequest: this._relayProvenance ? async (request) => this._relayProvenance(jobId, request) : void 0,
|
|
8182
9672
|
onArtifactRequest: this._requestUserArtifact ? async (request) => this._requestUserArtifact(jobId, request) : void 0,
|
|
@@ -8213,7 +9703,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8213
9703
|
}
|
|
8214
9704
|
if (result.status === ExecutionJobStatus.enum.failed) {
|
|
8215
9705
|
const stepErrors = result.stepResults.filter((r) => r.error).map((r) => `${r.name}: ${r.error.message}`).join(" | ");
|
|
8216
|
-
logger$
|
|
9706
|
+
logger$3.error("Sandbox returned failed result", {
|
|
8217
9707
|
durationMs: result.durationMs,
|
|
8218
9708
|
stepCount: result.stepResults.length,
|
|
8219
9709
|
steps: result.stepResults.map((r) => `${r.name}:${r.status}`).join(","),
|
|
@@ -8249,7 +9739,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8249
9739
|
async handleBringupJob(dispatch) {
|
|
8250
9740
|
const { runId, jobId, jobConfig } = dispatch;
|
|
8251
9741
|
const targetAgentId = String(jobConfig.bringupTarget ?? "");
|
|
8252
|
-
logger$
|
|
9742
|
+
logger$3.info("Starting bring-up job", {
|
|
8253
9743
|
jobId,
|
|
8254
9744
|
runId,
|
|
8255
9745
|
targetAgentId
|
|
@@ -8286,7 +9776,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8286
9776
|
this.sendStepStatus(dispatch, 0, "bring-up", ExecutionStepStatus.enum.failed, void 0, streamer.getTotalBytes());
|
|
8287
9777
|
streamer.destroy();
|
|
8288
9778
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.failed, { error: message });
|
|
8289
|
-
logger$
|
|
9779
|
+
logger$3.warn("Bring-up job failed", {
|
|
8290
9780
|
jobId,
|
|
8291
9781
|
runId,
|
|
8292
9782
|
targetAgentId,
|
|
@@ -8304,7 +9794,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8304
9794
|
async handleBuildJob(dispatch, workDir, abortController) {
|
|
8305
9795
|
const { runId, jobId, jobConfig } = dispatch;
|
|
8306
9796
|
const buildCtx = getRequestContext();
|
|
8307
|
-
logger$
|
|
9797
|
+
logger$3.info(`Run: ${buildCtx.runId ?? runId} | Trace: ${buildCtx.requestId ?? "N/A"}`);
|
|
8308
9798
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
|
|
8309
9799
|
const buildStreamer = this.createStepStreamer(dispatch, 0);
|
|
8310
9800
|
const buildLog = (msg) => buildStreamer.addLine(msg);
|
|
@@ -8393,14 +9883,14 @@ var init_job_runner = __esmMin((() => {
|
|
|
8393
9883
|
const cliPublicKey = jobConfig.cliPublicKey;
|
|
8394
9884
|
const orchestratorPrivateKey = jobConfig.orchestratorPrivateKey;
|
|
8395
9885
|
if (tarballUrl && cliPublicKey && orchestratorPrivateKey) {
|
|
8396
|
-
logger$
|
|
9886
|
+
logger$3.info("Applying overlay tarball for test run", { jobId });
|
|
8397
9887
|
const overlayResult = await applyOverlay({
|
|
8398
9888
|
tarballUrl,
|
|
8399
9889
|
cliPublicKey,
|
|
8400
9890
|
orchestratorPrivateKey,
|
|
8401
9891
|
repoDir: workDir
|
|
8402
9892
|
});
|
|
8403
|
-
logger$
|
|
9893
|
+
logger$3.info("Overlay applied", {
|
|
8404
9894
|
filesApplied: overlayResult.filesApplied,
|
|
8405
9895
|
filesDeleted: overlayResult.filesDeleted
|
|
8406
9896
|
});
|
|
@@ -8446,7 +9936,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8446
9936
|
orchestratorPrivateKey: initJobConfig.orchestratorPrivateKey,
|
|
8447
9937
|
repoDir: workDir
|
|
8448
9938
|
});
|
|
8449
|
-
logger$
|
|
9939
|
+
logger$3.info("Init job: overlay applied", {
|
|
8450
9940
|
jobId: dispatch.jobId,
|
|
8451
9941
|
filesApplied: overlayResult.filesApplied,
|
|
8452
9942
|
filesDeleted: overlayResult.filesDeleted
|
|
@@ -8475,9 +9965,12 @@ var init_job_runner = __esmMin((() => {
|
|
|
8475
9965
|
platform: os.platform(),
|
|
8476
9966
|
arch: os.arch()
|
|
8477
9967
|
};
|
|
8478
|
-
logger$
|
|
8479
|
-
const depUploadUrl = await this.requestUploadUrl(dispatch.jobId, "deps",
|
|
8480
|
-
|
|
9968
|
+
logger$3.info("Requesting dep upload URL from orchestrator", { lockfileHash: buildConfig.lockfileHash });
|
|
9969
|
+
const depUploadUrl = await this.requestUploadUrl(dispatch.jobId, "deps", {
|
|
9970
|
+
...depKey,
|
|
9971
|
+
depsHash: hash
|
|
9972
|
+
});
|
|
9973
|
+
logger$3.info("Uploading dep tarball to S3", {
|
|
8481
9974
|
size: tarball.length,
|
|
8482
9975
|
hash: hash.slice(0, 12)
|
|
8483
9976
|
});
|
|
@@ -8486,7 +9979,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8486
9979
|
...depKey,
|
|
8487
9980
|
depsHash: hash
|
|
8488
9981
|
});
|
|
8489
|
-
logger$
|
|
9982
|
+
logger$3.info("Dep tarball upload complete", { lockfileHash: buildConfig.lockfileHash });
|
|
8490
9983
|
buildLog(`Deps tarball uploaded (${tarball.length} bytes)`);
|
|
8491
9984
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running, {
|
|
8492
9985
|
buildEvent: "deps_packed",
|
|
@@ -8512,15 +10005,15 @@ var init_job_runner = __esmMin((() => {
|
|
|
8512
10005
|
platform: os.platform(),
|
|
8513
10006
|
arch: os.arch()
|
|
8514
10007
|
};
|
|
8515
|
-
logger$
|
|
10008
|
+
logger$3.info("Requesting source tarball upload URL from orchestrator", { contentHash: buildConfig.contentHash });
|
|
8516
10009
|
const sourceUploadUrl = await this.requestUploadUrl(dispatch.jobId, "source", sourceKey);
|
|
8517
|
-
logger$
|
|
10010
|
+
logger$3.info("Uploading source tarball to S3", {
|
|
8518
10011
|
size: tarball.length,
|
|
8519
10012
|
contentHash: buildConfig.contentHash
|
|
8520
10013
|
});
|
|
8521
10014
|
await uploadToPresignedUrl(sourceUploadUrl, tarball);
|
|
8522
10015
|
this.sendUploadComplete(dispatch.jobId, "source", sourceKey);
|
|
8523
|
-
logger$
|
|
10016
|
+
logger$3.info("Source tarball upload complete", { contentHash: buildConfig.contentHash });
|
|
8524
10017
|
buildLog(`Source tarball packed and uploaded (${tarball.length} bytes, hash: ${buildConfig.contentHash.slice(0, 12)})`);
|
|
8525
10018
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running, {
|
|
8526
10019
|
buildEvent: "source_packed",
|
|
@@ -8599,9 +10092,9 @@ var init_job_runner = __esmMin((() => {
|
|
|
8599
10092
|
const config = jobConfig;
|
|
8600
10093
|
const workflowDir = join(workDir, "workflow");
|
|
8601
10094
|
const sourceDir = join(workDir, "source");
|
|
8602
|
-
logger$
|
|
10095
|
+
logger$3.info("Starting global eval round", {
|
|
8603
10096
|
jobId,
|
|
8604
|
-
candidateCount: config.candidates.length,
|
|
10097
|
+
candidateCount: Array.isArray(config.candidates) ? config.candidates.length : 0,
|
|
8605
10098
|
workflowRepoIdentifier: config.workflowRepoIdentifier
|
|
8606
10099
|
});
|
|
8607
10100
|
this.sendJobStatus(dispatch, ExecutionJobStatus.enum.running);
|
|
@@ -8629,6 +10122,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8629
10122
|
});
|
|
8630
10123
|
}, this.config.jobHeartbeatIntervalMs);
|
|
8631
10124
|
try {
|
|
10125
|
+
if (!Array.isArray(config.candidates)) throw new Error("Global eval round dispatch is malformed: `candidates` must be an array");
|
|
8632
10126
|
if (abortController.signal.aborted) {
|
|
8633
10127
|
await closeStreamer();
|
|
8634
10128
|
this.sendStepStatus(dispatch, 0, "global-eval", ExecutionStepStatus.enum.skipped, void 0, evalStreamer.getTotalBytes());
|
|
@@ -8660,7 +10154,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8660
10154
|
}));
|
|
8661
10155
|
const running = roundResult.candidates.filter((c) => c.run).length;
|
|
8662
10156
|
const indeterminate = roundResult.candidates.filter((c) => c.indeterminate).length;
|
|
8663
|
-
logger$
|
|
10157
|
+
logger$3.info("Global eval round completed", {
|
|
8664
10158
|
jobId,
|
|
8665
10159
|
candidateCount: roundResult.candidates.length,
|
|
8666
10160
|
running,
|
|
@@ -8676,7 +10170,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8676
10170
|
});
|
|
8677
10171
|
} catch (err) {
|
|
8678
10172
|
const errorMsg = toErrorMessage(err);
|
|
8679
|
-
logger$
|
|
10173
|
+
logger$3.error("Global eval round failed", {
|
|
8680
10174
|
jobId,
|
|
8681
10175
|
error: errorMsg
|
|
8682
10176
|
});
|
|
@@ -8706,7 +10200,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8706
10200
|
async handleInitJob(dispatch, workDir, abortController) {
|
|
8707
10201
|
const { runId, jobId, jobConfig } = dispatch;
|
|
8708
10202
|
const config = jobConfig;
|
|
8709
|
-
logger$
|
|
10203
|
+
logger$3.info("Starting init job", {
|
|
8710
10204
|
jobId,
|
|
8711
10205
|
targetJobName: config.targetJobName,
|
|
8712
10206
|
workflowName: config.workflowName
|
|
@@ -8750,7 +10244,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8750
10244
|
await this.materializeInitJobSource(dispatch, workDir, initLog);
|
|
8751
10245
|
const kiciDir = join(workDir, ".kici");
|
|
8752
10246
|
const hasPackage = await fileExists(join(kiciDir, "package.json"));
|
|
8753
|
-
logger$
|
|
10247
|
+
logger$3.info("Init job: checking deps", {
|
|
8754
10248
|
kiciDir,
|
|
8755
10249
|
hasPackageJson: hasPackage,
|
|
8756
10250
|
source: config.source
|
|
@@ -8776,7 +10270,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8776
10270
|
hasFilter: config.hasFilter ?? false
|
|
8777
10271
|
}, config.timeoutMs, filterInput);
|
|
8778
10272
|
});
|
|
8779
|
-
logger$
|
|
10273
|
+
logger$3.info("Init job completed successfully", {
|
|
8780
10274
|
jobId,
|
|
8781
10275
|
hasContext: initResult.contextNames !== void 0,
|
|
8782
10276
|
hasEnv: initResult.env !== void 0,
|
|
@@ -8793,7 +10287,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8793
10287
|
});
|
|
8794
10288
|
} catch (err) {
|
|
8795
10289
|
const errorMsg = toErrorMessage(err);
|
|
8796
|
-
logger$
|
|
10290
|
+
logger$3.error("Init job failed", {
|
|
8797
10291
|
jobId,
|
|
8798
10292
|
error: errorMsg
|
|
8799
10293
|
});
|
|
@@ -8819,7 +10313,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8819
10313
|
const { runId, jobId, jobConfig } = dispatch;
|
|
8820
10314
|
const config = jobConfig;
|
|
8821
10315
|
const timeoutMs = config.timeoutMs ?? 12e4;
|
|
8822
|
-
logger$
|
|
10316
|
+
logger$3.info("Starting DynamicJobFn evaluation", {
|
|
8823
10317
|
jobId,
|
|
8824
10318
|
workflowName: config.workflowName,
|
|
8825
10319
|
sourceIndex: config.source.index
|
|
@@ -8843,14 +10337,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8843
10337
|
return;
|
|
8844
10338
|
}
|
|
8845
10339
|
await materializeEvalWorkspace(dispatch, workDir, evalLog);
|
|
8846
|
-
const
|
|
8847
|
-
const scopedDollar = zx$({
|
|
8848
|
-
cwd: workDir,
|
|
8849
|
-
env: { ...process.env },
|
|
8850
|
-
verbose: true,
|
|
8851
|
-
quiet: false,
|
|
8852
|
-
log: makeStreamingZxLog((line, stream) => evalStreamer.addLine(line, stream))
|
|
8853
|
-
});
|
|
10340
|
+
const scopedDollar = await buildEvalShell(workDir, (line, stream) => evalStreamer.addLine(line, stream));
|
|
8854
10341
|
const evalSink = { addLine: (line) => evalStreamer.addLine(line) };
|
|
8855
10342
|
const filterInput = config.hasFilter ? await buildInitFilterInput(dispatch, config.event, workDir, (line, stream) => evalStreamer.addLine(line, stream)) : void 0;
|
|
8856
10343
|
const dynamicJobLogger = {
|
|
@@ -8891,7 +10378,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8891
10378
|
workflowName: config.workflowName
|
|
8892
10379
|
});
|
|
8893
10380
|
});
|
|
8894
|
-
logger$
|
|
10381
|
+
logger$3.info("DynamicJobFn evaluation completed", {
|
|
8895
10382
|
jobId,
|
|
8896
10383
|
generatedJobCount: lockJobs.length,
|
|
8897
10384
|
jobNames: lockJobs.map((j) => j.name)
|
|
@@ -8906,7 +10393,7 @@ var init_job_runner = __esmMin((() => {
|
|
|
8906
10393
|
});
|
|
8907
10394
|
} catch (err) {
|
|
8908
10395
|
const errorMsg = toErrorMessage(err);
|
|
8909
|
-
logger$
|
|
10396
|
+
logger$3.error("DynamicJobFn evaluation failed", {
|
|
8910
10397
|
jobId,
|
|
8911
10398
|
error: errorMsg
|
|
8912
10399
|
});
|
|
@@ -8930,13 +10417,50 @@ var init_job_runner = __esmMin((() => {
|
|
|
8930
10417
|
}
|
|
8931
10418
|
}
|
|
8932
10419
|
/**
|
|
10420
|
+
* Build the job's container image when it declared a Dockerfile, and return
|
|
10421
|
+
* the tag the sandbox must run.
|
|
10422
|
+
*
|
|
10423
|
+
* Returns `undefined` for every other job — one with no container, or one
|
|
10424
|
+
* naming a finalized image — which is the common case and costs nothing.
|
|
10425
|
+
*
|
|
10426
|
+
* Extracted from `setupSandboxForExecution` so that function stays inside the
|
|
10427
|
+
* 200-line ceiling, and so the build's streamer lifecycle is visibly bounded
|
|
10428
|
+
* by one `finally`.
|
|
10429
|
+
*/
|
|
10430
|
+
async buildJobImageIfDeclared(dispatch, jobConfig, workDir, abortController) {
|
|
10431
|
+
const container = jobConfig.container;
|
|
10432
|
+
if (!container || typeof container === "string" || !container.dockerfile) return void 0;
|
|
10433
|
+
const streamer = this.createStepStreamer(dispatch, CONTAINER_BUILD_STEP_INDEX);
|
|
10434
|
+
try {
|
|
10435
|
+
return await runJobImageBuild({
|
|
10436
|
+
container,
|
|
10437
|
+
workDir,
|
|
10438
|
+
jobId: dispatch.jobId,
|
|
10439
|
+
jobName: jobConfig.name ?? jobConfig.baseJobName ?? "job",
|
|
10440
|
+
onLog: (line) => streamer.addLine(line),
|
|
10441
|
+
build: async (spec, onLog) => buildJobImage({
|
|
10442
|
+
spec,
|
|
10443
|
+
cli: resolveBuildCli({ configured: this.config.containerBuildCli }),
|
|
10444
|
+
socketPath: sandboxSocketPath(),
|
|
10445
|
+
...dispatch.containerRegistryAuth ? { authconfig: dispatch.containerRegistryAuth } : {},
|
|
10446
|
+
onLog,
|
|
10447
|
+
signal: abortController.signal
|
|
10448
|
+
}),
|
|
10449
|
+
sendStepStatus: (name, state, data) => this.sendStepStatus(dispatch, CONTAINER_BUILD_STEP_INDEX, name, state, data, streamer.getTotalBytes())
|
|
10450
|
+
});
|
|
10451
|
+
} finally {
|
|
10452
|
+
await streamer.flush();
|
|
10453
|
+
streamer.destroy();
|
|
10454
|
+
}
|
|
10455
|
+
}
|
|
10456
|
+
/**
|
|
8933
10457
|
* Create the appropriate sandbox backend based on execution mode.
|
|
8934
10458
|
*/
|
|
8935
10459
|
createSandbox(mode, opts) {
|
|
8936
10460
|
switch (mode) {
|
|
8937
10461
|
case "container": {
|
|
8938
10462
|
const containerConfig = opts.jobConfig.container;
|
|
8939
|
-
const image = typeof containerConfig === "string" ? containerConfig : containerConfig?.image ?? "node:20-alpine";
|
|
10463
|
+
const image = opts.builtImage ?? (typeof containerConfig === "string" ? containerConfig : containerConfig?.image ?? "node:20-alpine");
|
|
8940
10464
|
return new ContainerSandbox({
|
|
8941
10465
|
docker: new Docker(),
|
|
8942
10466
|
image,
|
|
@@ -8953,7 +10477,11 @@ var init_job_runner = __esmMin((() => {
|
|
|
8953
10477
|
nanoCpus: this.config.sandboxNanoCpus,
|
|
8954
10478
|
networkMode: this.config.sandboxNetwork === "host" ? "host" : "default",
|
|
8955
10479
|
grant: opts.jobConfig.sandboxGrant
|
|
8956
|
-
}
|
|
10480
|
+
},
|
|
10481
|
+
...opts.registryAuth ? { registryAuth: opts.registryAuth } : {},
|
|
10482
|
+
...opts.builtImage ? { buildTag: opts.builtImage } : {},
|
|
10483
|
+
...this.config.runtimeNodeSource ? { runtimeNodePath: this.config.runtimeNodeSource } : {},
|
|
10484
|
+
...this.config.runtimeImage ? { runtimeImage: this.config.runtimeImage } : {}
|
|
8957
10485
|
});
|
|
8958
10486
|
}
|
|
8959
10487
|
case "firecracker": return new FirecrackerSandbox({
|
|
@@ -8964,7 +10492,8 @@ var init_job_runner = __esmMin((() => {
|
|
|
8964
10492
|
runnerPath: opts.runnerPath,
|
|
8965
10493
|
env: opts.env,
|
|
8966
10494
|
sandbox: this.config.sandbox,
|
|
8967
|
-
sandboxNetwork: this.config.sandboxNetwork
|
|
10495
|
+
sandboxNetwork: this.config.sandboxNetwork,
|
|
10496
|
+
orphanCleanup: this.config.orphanCleanup
|
|
8968
10497
|
});
|
|
8969
10498
|
}
|
|
8970
10499
|
}
|
|
@@ -9087,6 +10616,168 @@ var init_job_runner = __esmMin((() => {
|
|
|
9087
10616
|
};
|
|
9088
10617
|
}));
|
|
9089
10618
|
//#endregion
|
|
10619
|
+
//#region src/execution/between-jobs-reset.ts
|
|
10620
|
+
/**
|
|
10621
|
+
* Run the operator between-jobs reset command. Fail-open: never throws, always
|
|
10622
|
+
* resolves to a status. Skips when unconfigured, or when runOn=on-failure and
|
|
10623
|
+
* the job succeeded.
|
|
10624
|
+
*/
|
|
10625
|
+
async function runBetweenJobsReset(input) {
|
|
10626
|
+
const started = performance.now();
|
|
10627
|
+
const done = (status) => ({
|
|
10628
|
+
status,
|
|
10629
|
+
durationMs: Math.round(performance.now() - started)
|
|
10630
|
+
});
|
|
10631
|
+
if (!input.command) return done("skipped");
|
|
10632
|
+
if (input.runOn === "on-failure" && !input.jobFailed) return done("skipped");
|
|
10633
|
+
const exec = input.exec ?? defaultExec;
|
|
10634
|
+
try {
|
|
10635
|
+
const r = await exec(input.command, input.timeoutMs);
|
|
10636
|
+
if (r.timedOut) return done("timeout");
|
|
10637
|
+
return done(r.code === 0 ? "success" : "failed");
|
|
10638
|
+
} catch {
|
|
10639
|
+
return done("failed");
|
|
10640
|
+
}
|
|
10641
|
+
}
|
|
10642
|
+
var defaultExec;
|
|
10643
|
+
var init_between_jobs_reset = __esmMin((() => {
|
|
10644
|
+
defaultExec = (command, timeoutMs) => new Promise((resolve) => {
|
|
10645
|
+
const child = spawn("/bin/sh", ["-c", command], { stdio: "ignore" });
|
|
10646
|
+
let timedOut = false;
|
|
10647
|
+
const timer = setTimeout(() => {
|
|
10648
|
+
timedOut = true;
|
|
10649
|
+
child.kill("SIGKILL");
|
|
10650
|
+
}, timeoutMs);
|
|
10651
|
+
child.on("exit", (code) => {
|
|
10652
|
+
clearTimeout(timer);
|
|
10653
|
+
resolve({
|
|
10654
|
+
code,
|
|
10655
|
+
timedOut
|
|
10656
|
+
});
|
|
10657
|
+
});
|
|
10658
|
+
child.on("error", () => {
|
|
10659
|
+
clearTimeout(timer);
|
|
10660
|
+
resolve({
|
|
10661
|
+
code: 1,
|
|
10662
|
+
timedOut
|
|
10663
|
+
});
|
|
10664
|
+
});
|
|
10665
|
+
});
|
|
10666
|
+
}));
|
|
10667
|
+
//#endregion
|
|
10668
|
+
//#region src/execution/cleanup-rerun.ts
|
|
10669
|
+
/**
|
|
10670
|
+
* Re-run a hard-killed job's declared cleanup/onFailure hooks out-of-band,
|
|
10671
|
+
* against its preserved workdir, in a fresh bounded child. Returns 'skipped'
|
|
10672
|
+
* when there is nothing to do (no declared cleanup, wrong backend, or a
|
|
10673
|
+
* non-positive timeout). Never throws — a failure is reported as a status.
|
|
10674
|
+
*/
|
|
10675
|
+
async function runDeclaredCleanupOutOfBand(input) {
|
|
10676
|
+
const started = performance.now();
|
|
10677
|
+
const done = (status) => ({
|
|
10678
|
+
status,
|
|
10679
|
+
durationMs: Math.round(performance.now() - started)
|
|
10680
|
+
});
|
|
10681
|
+
if (!input.declaresCleanup) return done("skipped");
|
|
10682
|
+
if (!HOST_SHARING_BACKENDS.has(input.backend)) return done("skipped");
|
|
10683
|
+
if (!Number.isFinite(input.timeoutMs) || input.timeoutMs <= 0) return done("skipped");
|
|
10684
|
+
const controller = new AbortController();
|
|
10685
|
+
const timer = setTimeout(() => controller.abort(), input.timeoutMs);
|
|
10686
|
+
try {
|
|
10687
|
+
await input.spawn(input.workDir, controller.signal);
|
|
10688
|
+
return done("success");
|
|
10689
|
+
} catch {
|
|
10690
|
+
return done(controller.signal.aborted ? "timeout" : "failed");
|
|
10691
|
+
} finally {
|
|
10692
|
+
clearTimeout(timer);
|
|
10693
|
+
}
|
|
10694
|
+
}
|
|
10695
|
+
var HOST_SHARING_BACKENDS;
|
|
10696
|
+
var init_cleanup_rerun = __esmMin((() => {
|
|
10697
|
+
HOST_SHARING_BACKENDS = /* @__PURE__ */ new Set(["bare-metal"]);
|
|
10698
|
+
}));
|
|
10699
|
+
//#endregion
|
|
10700
|
+
//#region src/execution/between-jobs-controller.ts
|
|
10701
|
+
var between_jobs_controller_exports = /* @__PURE__ */ __exportAll({ BetweenJobsController: () => BetweenJobsController$1 });
|
|
10702
|
+
var logger$2, BetweenJobsController$1;
|
|
10703
|
+
var init_between_jobs_controller = __esmMin((() => {
|
|
10704
|
+
init_between_jobs_reset();
|
|
10705
|
+
init_cleanup_rerun();
|
|
10706
|
+
init_prometheus();
|
|
10707
|
+
logger$2 = createLogger({ prefix: "between-jobs" });
|
|
10708
|
+
BetweenJobsController$1 = class {
|
|
10709
|
+
deps;
|
|
10710
|
+
_consecutiveResetFailures = 0;
|
|
10711
|
+
/** Consecutive between-jobs reset failures, for the supervisor's drain gate. */
|
|
10712
|
+
get consecutiveResetFailures() {
|
|
10713
|
+
return this._consecutiveResetFailures;
|
|
10714
|
+
}
|
|
10715
|
+
constructor(deps) {
|
|
10716
|
+
this.deps = deps;
|
|
10717
|
+
}
|
|
10718
|
+
async runRerunPhase(ctx) {
|
|
10719
|
+
if (ctx.completionHooksRan || !ctx.cleanupSpawn) return "skipped";
|
|
10720
|
+
const r = await (this.deps.rerun ?? runDeclaredCleanupOutOfBand)({
|
|
10721
|
+
workDir: ctx.workDir,
|
|
10722
|
+
backend: ctx.backend,
|
|
10723
|
+
declaresCleanup: ctx.declaresCleanup,
|
|
10724
|
+
timeoutMs: this.deps.config.betweenJobsResetTimeoutMs,
|
|
10725
|
+
spawn: ctx.cleanupSpawn
|
|
10726
|
+
});
|
|
10727
|
+
orphanCleanupTotal.add(1, { status: r.status });
|
|
10728
|
+
if (r.status === "failed" || r.status === "timeout") logger$2.warn("between-jobs out-of-band cleanup did not complete", { status: r.status });
|
|
10729
|
+
return r.status;
|
|
10730
|
+
}
|
|
10731
|
+
async runResetPhase(ctx) {
|
|
10732
|
+
const reset = await (this.deps.reset ?? runBetweenJobsReset)({
|
|
10733
|
+
command: this.deps.config.betweenJobsResetCommand,
|
|
10734
|
+
timeoutMs: this.deps.config.betweenJobsResetTimeoutMs,
|
|
10735
|
+
runOn: this.deps.config.betweenJobsResetRunOn,
|
|
10736
|
+
jobFailed: ctx.jobFailed
|
|
10737
|
+
});
|
|
10738
|
+
if (reset.status !== "skipped") {
|
|
10739
|
+
betweenJobsResetTotal.add(1, { status: reset.status });
|
|
10740
|
+
betweenJobsResetDurationSeconds.record(reset.durationMs / 1e3);
|
|
10741
|
+
}
|
|
10742
|
+
if (reset.status === "failed" || reset.status === "timeout") {
|
|
10743
|
+
this._consecutiveResetFailures += 1;
|
|
10744
|
+
logger$2.warn("between-jobs reset failed", {
|
|
10745
|
+
status: reset.status,
|
|
10746
|
+
consecutive: this._consecutiveResetFailures
|
|
10747
|
+
});
|
|
10748
|
+
} else if (reset.status === "success") this._consecutiveResetFailures = 0;
|
|
10749
|
+
return reset.status;
|
|
10750
|
+
}
|
|
10751
|
+
async afterJob(ctx) {
|
|
10752
|
+
let rerun = "skipped";
|
|
10753
|
+
try {
|
|
10754
|
+
rerun = await this.runRerunPhase(ctx);
|
|
10755
|
+
} catch (err) {
|
|
10756
|
+
logger$2.warn("between-jobs out-of-band cleanup threw", { error: toErrorMessage(err) });
|
|
10757
|
+
}
|
|
10758
|
+
let reaped = 0;
|
|
10759
|
+
if (this.deps.config.orphanCleanup) try {
|
|
10760
|
+
reaped = await ctx.reap();
|
|
10761
|
+
if (reaped > 0) orphansReapedTotal.add(reaped);
|
|
10762
|
+
} catch (err) {
|
|
10763
|
+
logger$2.warn("between-jobs reap threw", { error: toErrorMessage(err) });
|
|
10764
|
+
}
|
|
10765
|
+
try {
|
|
10766
|
+
await ctx.deleteWorkdir();
|
|
10767
|
+
} catch (err) {
|
|
10768
|
+
logger$2.warn("between-jobs workdir delete threw", { error: toErrorMessage(err) });
|
|
10769
|
+
}
|
|
10770
|
+
const reset = await this.runResetPhase(ctx);
|
|
10771
|
+
return {
|
|
10772
|
+
rerun,
|
|
10773
|
+
reaped,
|
|
10774
|
+
reset,
|
|
10775
|
+
consecutiveResetFailures: this._consecutiveResetFailures
|
|
10776
|
+
};
|
|
10777
|
+
}
|
|
10778
|
+
};
|
|
10779
|
+
}));
|
|
10780
|
+
//#endregion
|
|
9090
10781
|
//#region src/server.ts
|
|
9091
10782
|
/**
|
|
9092
10783
|
* Agent entry point.
|
|
@@ -9108,23 +10799,30 @@ var init_job_runner = __esmMin((() => {
|
|
|
9108
10799
|
*/
|
|
9109
10800
|
init_console_capture();
|
|
9110
10801
|
init_npm_resolver();
|
|
9111
|
-
const AGENT_VERSION = "0.
|
|
9112
|
-
const BUILD_COMMIT = "
|
|
9113
|
-
const SDK_VERSION = "0.
|
|
9114
|
-
const SDK_BUNDLE_HASH = "
|
|
9115
|
-
const SHARED_VERSION = "0.
|
|
9116
|
-
const SHARED_BUNDLE_HASH = "
|
|
9117
|
-
const ENGINE_VERSION = "0.
|
|
9118
|
-
const ENGINE_BUNDLE_HASH = "
|
|
10802
|
+
const AGENT_VERSION = "0.6.0";
|
|
10803
|
+
const BUILD_COMMIT = "e4a936029";
|
|
10804
|
+
const SDK_VERSION = "0.6.0";
|
|
10805
|
+
const SDK_BUNDLE_HASH = "22faf0da45de7c243ce87f80fd33ee51b1df52809fde457b16bb821678f65eb3";
|
|
10806
|
+
const SHARED_VERSION = "0.6.0";
|
|
10807
|
+
const SHARED_BUNDLE_HASH = "a79be949815735b9e36eecc716f7eb07aa36c7bffba62e798e7e19f31a474eff";
|
|
10808
|
+
const ENGINE_VERSION = "0.6.0";
|
|
10809
|
+
const ENGINE_BUNDLE_HASH = "e49533b19f122155b3a78d95d105f97da259c2a45dee579578324b0702c91708";
|
|
9119
10810
|
initTelemetry({
|
|
9120
10811
|
serviceName: "kici-agent",
|
|
9121
10812
|
otlpEndpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
|
|
9122
10813
|
});
|
|
9123
10814
|
const { connectionStatus, jobsActive, jobsTotal } = await Promise.resolve().then(() => (init_prometheus(), prometheus_exports));
|
|
9124
10815
|
const { JobRunner } = await Promise.resolve().then(() => (init_job_runner(), job_runner_exports));
|
|
10816
|
+
const { BetweenJobsController } = await Promise.resolve().then(() => (init_between_jobs_controller(), between_jobs_controller_exports));
|
|
9125
10817
|
setServiceName("agent");
|
|
9126
10818
|
const logger$1 = createLogger({ prefix: "agent" });
|
|
9127
10819
|
/**
|
|
10820
|
+
* Consecutive between-jobs reset failures after which a `drainOnResetFailure`
|
|
10821
|
+
* agent drains — a persistently dirty host stops taking work rather than
|
|
10822
|
+
* running every subsequent job against unclean state.
|
|
10823
|
+
*/
|
|
10824
|
+
const RESET_FAILURE_DRAIN_THRESHOLD = 3;
|
|
10825
|
+
/**
|
|
9128
10826
|
* Serialize the agent's current Prometheus metrics to text. The OTel
|
|
9129
10827
|
* PrometheusExporter exposes no direct serialize method, so the metrics are
|
|
9130
10828
|
* piped through its request handler with a mock ServerResponse. Returns an
|
|
@@ -9174,7 +10872,10 @@ await guardStartup(logger$1, async () => {
|
|
|
9174
10872
|
name: "bash",
|
|
9175
10873
|
reason: "required for step execution"
|
|
9176
10874
|
}]);
|
|
9177
|
-
if (toolErrors.length > 0)
|
|
10875
|
+
if (toolErrors.length > 0) {
|
|
10876
|
+
const jobImageHint = config.jobImageAgent ? "\nThis agent runs inside the job's own container image, so that image must provide these tools. Either add them to the image, or run the job on a pool whose agent stays outside it." : "";
|
|
10877
|
+
throw new Error("Agent required-tools validation failed:\n" + toolErrors.map((e) => ` - ${e}`).join("\n") + jobImageHint);
|
|
10878
|
+
}
|
|
9178
10879
|
if (config.roles === void 0 || config.roles.includes("builder")) {
|
|
9179
10880
|
const npmVersion = verifyNpmAvailable();
|
|
9180
10881
|
logger$1.info("Builder role: npm verified", { npmVersion });
|
|
@@ -9183,9 +10884,17 @@ await guardStartup(logger$1, async () => {
|
|
|
9183
10884
|
let idleShutdownTimer;
|
|
9184
10885
|
let rebootIntent = false;
|
|
9185
10886
|
let client;
|
|
10887
|
+
const betweenJobsController = new BetweenJobsController({ config: {
|
|
10888
|
+
betweenJobsResetCommand: config.betweenJobsResetCommand,
|
|
10889
|
+
betweenJobsResetTimeoutMs: config.betweenJobsResetTimeoutMs,
|
|
10890
|
+
betweenJobsResetRunOn: config.betweenJobsResetRunOn,
|
|
10891
|
+
orphanCleanup: config.orphanCleanup,
|
|
10892
|
+
drainOnResetFailure: config.drainOnResetFailure
|
|
10893
|
+
} });
|
|
9186
10894
|
const jobRunner = new JobRunner({
|
|
9187
10895
|
send: (msg) => client.send(msg),
|
|
9188
10896
|
config,
|
|
10897
|
+
betweenJobsController,
|
|
9189
10898
|
requestUploadUrl: (jobId, cacheType, key) => client.requestUploadUrl(jobId, cacheType, key),
|
|
9190
10899
|
sendUploadComplete: (jobId, cacheType, key) => client.sendUploadComplete(jobId, cacheType, key),
|
|
9191
10900
|
sendEventEmit: (jobId, requestId, eventName, payload, target) => client.sendEventEmit(jobId, requestId, eventName, payload, target),
|
|
@@ -9201,6 +10910,12 @@ await guardStartup(logger$1, async () => {
|
|
|
9201
10910
|
rebootIntent = true;
|
|
9202
10911
|
return result;
|
|
9203
10912
|
}
|
|
10913
|
+
if (method === "scaler.claim-credentials") {
|
|
10914
|
+
const claimCode = String((params ?? {}).claimCode ?? "");
|
|
10915
|
+
const r = await client.sendClaimCredentials(claimCode);
|
|
10916
|
+
if (r.error) throw new Error(r.error);
|
|
10917
|
+
return r.credentials;
|
|
10918
|
+
}
|
|
9204
10919
|
return client.sendApiRequest(method, params ?? {});
|
|
9205
10920
|
},
|
|
9206
10921
|
requestUserCache: (jobId, request) => client.requestUserCache(jobId, request),
|
|
@@ -9215,13 +10930,14 @@ await guardStartup(logger$1, async () => {
|
|
|
9215
10930
|
messageId: randomUUID(),
|
|
9216
10931
|
agentId: config.agentId,
|
|
9217
10932
|
activeJobs: jobRunner.activeJobs.size,
|
|
9218
|
-
memoryUsedMb: Math.round((os.totalmem() - os.freemem()) /
|
|
9219
|
-
memoryAvailableMb: Math.round(os.freemem() /
|
|
10933
|
+
memoryUsedMb: Math.round((os.totalmem() - os.freemem()) / 1048576),
|
|
10934
|
+
memoryAvailableMb: Math.round(os.freemem() / 1048576),
|
|
9220
10935
|
uptimeSeconds: Math.round(os.uptime())
|
|
9221
10936
|
});
|
|
9222
10937
|
}
|
|
9223
10938
|
client = new OrchestratorClient({
|
|
9224
10939
|
...agentClientConnectionOptions(config),
|
|
10940
|
+
scalerClaimCode: config.scalerClaimCode,
|
|
9225
10941
|
getFleetBundleInputs: async () => ({
|
|
9226
10942
|
config,
|
|
9227
10943
|
logDir: process.env.KICI_LOG_DIR,
|
|
@@ -9291,6 +11007,10 @@ await guardStartup(logger$1, async () => {
|
|
|
9291
11007
|
}).finally(() => {
|
|
9292
11008
|
jobsActive.add(-1);
|
|
9293
11009
|
metricsReporter.collectAndSend().catch(() => {});
|
|
11010
|
+
if (config.drainOnResetFailure && !isDraining && betweenJobsController.consecutiveResetFailures >= RESET_FAILURE_DRAIN_THRESHOLD) {
|
|
11011
|
+
logger$1.warn("Draining agent after repeated between-jobs reset failures", { consecutive: betweenJobsController.consecutiveResetFailures });
|
|
11012
|
+
isDraining = true;
|
|
11013
|
+
}
|
|
9294
11014
|
sendAgentStatus();
|
|
9295
11015
|
if (rebootIntent) {
|
|
9296
11016
|
rebootIntent = false;
|
|
@@ -9340,9 +11060,23 @@ await guardStartup(logger$1, async () => {
|
|
|
9340
11060
|
}, idleMs);
|
|
9341
11061
|
}
|
|
9342
11062
|
}
|
|
9343
|
-
client.onRegistered = ({ pendingDispatch }) => {
|
|
9344
|
-
|
|
9345
|
-
|
|
11063
|
+
client.onRegistered = ({ pendingDispatch, warmPool }) => {
|
|
11064
|
+
const decision = decideIdleShutdown({
|
|
11065
|
+
scalerManaged: config.scalerManaged,
|
|
11066
|
+
activeJobs: jobRunner.activeJobs.size,
|
|
11067
|
+
pendingDispatch,
|
|
11068
|
+
warmPool
|
|
11069
|
+
});
|
|
11070
|
+
if (decision === "none") return;
|
|
11071
|
+
if (decision === "warm") {
|
|
11072
|
+
logger$1.info("Warm-pool agent registered and ready, awaiting work");
|
|
11073
|
+
if (idleShutdownTimer) {
|
|
11074
|
+
clearTimeout(idleShutdownTimer);
|
|
11075
|
+
idleShutdownTimer = void 0;
|
|
11076
|
+
}
|
|
11077
|
+
return;
|
|
11078
|
+
}
|
|
11079
|
+
if (decision === "pending-dispatch") {
|
|
9346
11080
|
const safetyMs = config.scalerPendingDispatchTimeoutMs;
|
|
9347
11081
|
logger$1.info(`Scaler-managed agent registered with pending bound dispatch, deferring idle shutdown for ${safetyMs}ms`);
|
|
9348
11082
|
if (idleShutdownTimer) clearTimeout(idleShutdownTimer);
|
|
@@ -9453,6 +11187,10 @@ await guardStartup(logger$1, async () => {
|
|
|
9453
11187
|
}
|
|
9454
11188
|
]
|
|
9455
11189
|
});
|
|
11190
|
+
client.onClaimFailedPermanently = (reason) => {
|
|
11191
|
+
logger$1.error("Self-bootstrap claim failed permanently, shutting down", { reason });
|
|
11192
|
+
gracefulShutdown("scaler-claim-failed", 1);
|
|
11193
|
+
};
|
|
9456
11194
|
process.on("SIGUSR1", () => {
|
|
9457
11195
|
logger$1.info("Received SIGUSR1, entering drain mode");
|
|
9458
11196
|
isDraining = true;
|