@kici-dev/agent 0.5.0 → 0.6.0

package/dist/index.js

@@ -6,12 +6,12 @@ import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/s
 import { KNOWN_ROLES, parseHostPropertyAssignments, validateNoReservedLabels } from "@kici-dev/engine";
 import { execFile } from "node:child_process";
 import { access, cp, lstat, mkdir, readFile, readdir, realpath, rename, rm, unlink, writeFile } from "node:fs/promises";
-import { makeTempDir } from "@kici-dev/core/tmp";
+import { existsSync } from "node:fs";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { makeTempDir } from "@kici-dev/core/tmp";
 import { promisify } from "node:util";
 import { createLogger, sha256, toErrorMessage } from "@kici-dev/shared";
 import { PNPM_IGNORE_BUILD_GATE_ARG, PackageManager, YarnFlavor, detectPackageManagerFromManifests, detectYarnFlavor } from "@kici-dev/shared/package-manager";
-import { existsSync } from "node:fs";
 import { parse, stringify } from "yaml";
 import { Readable, Transform } from "node:stream";
 import { pipeline } from "node:stream/promises";
@@ -40,6 +40,20 @@ var __exportAll = (all, no_symbols) => {
 	return target;
 };
 //#endregion
+//#region src/execution/image-build/build-engine.ts
+/**
+* Build a job's container image with the host's build CLI.
+*
+* The CLI is REQUIRED — there is deliberately no socket-API fallback. One build
+* path means one set of Dockerfile semantics: `.dockerignore`, BuildKit and
+* every directive behave as they do on the author's own machine, instead of
+* depending on which agent happened to pick the job up. A host without a CLI
+* cannot run a Dockerfile job, and says so in as many words.
+*/
+/** The build CLIs an agent host may provide. */
+const ContainerBuildCli = z.enum(["docker", "podman"]);
+ContainerBuildCli.enum.docker, ContainerBuildCli.enum.podman;
+//#endregion
 //#region src/config.ts
 /** Execution mode for the agent's sandbox backend. Mirrors the runtime enum. */
 const ExecutionMode = z.enum([
@@ -47,6 +61,71 @@ const ExecutionMode = z.enum([
 	"bare-metal",
 	"firecracker"
 ]);
+/** When the operator between-jobs reset command runs. */
+const BetweenJobsRunOn = z.enum(["always", "on-failure"]);
+const configSchema = z.object({
+	orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
+	agentId: z.string().optional(),
+	labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
+	properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
+	roles: z.string().optional().transform((s) => {
+		if (s === void 0) return void 0;
+		if (s === "") return [];
+		return s.split(",").filter(Boolean);
+	}).refine((roles) => {
+		if (roles === void 0) return true;
+		const validValues = [...KNOWN_ROLES, "all"];
+		return roles.every((r) => validValues.includes(r));
+	}, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
+		if (roles === void 0) return void 0;
+		if (roles.includes("all")) return void 0;
+		if (roles.length === 0) return [];
+		return roles.filter((r) => r !== "all");
+	}),
+	port: z.coerce.number().default(8080),
+	logLevel: z.enum([
+		"debug",
+		"info",
+		"warn",
+		"error"
+	]).default("info"),
+	agentToken: z.string().optional(),
+	githubToken: z.string().optional(),
+	maxLogSizeBytes: z.coerce.number().default(10485760),
+	defaultStepTimeoutMs: z.coerce.number().default(18e5),
+	dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
+	jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
+	backpressureMode: z.enum(["pause", "drop"]).default("pause"),
+	agentPayloadDir: z.string().optional(),
+	agentCommand: z.string().optional(),
+	sandbox: z.string().default("false").transform((s) => s === "true"),
+	trustedEnv: z.string().default("false").transform((s) => s === "true"),
+	inPlace: z.string().default("false").transform((s) => s === "true"),
+	sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
+	sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
+	sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
+	sandboxUser: z.string().optional(),
+	sandboxPidsLimit: z.coerce.number().int().positive().default(512),
+	sandboxMemoryBytes: z.coerce.number().int().positive().default(2147483648),
+	sandboxNanoCpus: z.coerce.number().int().positive().default(2e9),
+	scalerManaged: z.string().optional().transform((s) => s === "1"),
+	jobImageAgent: z.string().optional().transform((s) => s === "1"),
+	runtimeImage: z.string().optional(),
+	runtimeNodeSource: z.string().optional(),
+	containerBuildCli: ContainerBuildCli.optional(),
+	scalerClaimCode: z.string().optional(),
+	scalerIdleTimeoutMs: z.coerce.number().default(5e3),
+	scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
+	executionMode: ExecutionMode.optional(),
+	otelExporterOtlpEndpoint: z.string().optional(),
+	concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
+	isOrchestratorHost: z.string().optional().transform((v) => v === "true"),
+	betweenJobsResetCommand: z.string().optional(),
+	betweenJobsResetTimeoutMs: z.coerce.number().int().positive().default(6e4),
+	betweenJobsResetRunOn: BetweenJobsRunOn.default("always"),
+	orphanCleanup: z.string().default("true").transform((s) => s !== "false"),
+	drainOnResetFailure: z.string().default("false").transform((s) => s === "true")
+});
 /**
 * Env-var definition for the agent. Exported so the docs generator and the
 * deploy-stg pre-validator can inspect / re-parse without round-tripping
@@ -54,59 +133,7 @@ const ExecutionMode = z.enum([
 */
 const envDef = defineEnv({
 	service: "agent",
-	schema: z.object({
-		orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
-		agentId: z.string().optional(),
-		labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
-		properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
-		roles: z.string().optional().transform((s) => {
-			if (s === void 0) return void 0;
-			if (s === "") return [];
-			return s.split(",").filter(Boolean);
-		}).refine((roles) => {
-			if (roles === void 0) return true;
-			const validValues = [...KNOWN_ROLES, "all"];
-			return roles.every((r) => validValues.includes(r));
-		}, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
-			if (roles === void 0) return void 0;
-			if (roles.includes("all")) return void 0;
-			if (roles.length === 0) return [];
-			return roles.filter((r) => r !== "all");
-		}),
-		port: z.coerce.number().default(8080),
-		logLevel: z.enum([
-			"debug",
-			"info",
-			"warn",
-			"error"
-		]).default("info"),
-		agentToken: z.string().optional(),
-		githubToken: z.string().optional(),
-		maxLogSizeBytes: z.coerce.number().default(10 * 1024 * 1024),
-		defaultStepTimeoutMs: z.coerce.number().default(1800 * 1e3),
-		dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
-		jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
-		backpressureMode: z.enum(["pause", "drop"]).default("pause"),
-		agentPayloadDir: z.string().optional(),
-		agentCommand: z.string().optional(),
-		sandbox: z.string().default("false").transform((s) => s === "true"),
-		trustedEnv: z.string().default("false").transform((s) => s === "true"),
-		inPlace: z.string().default("false").transform((s) => s === "true"),
-		sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
-		sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
-		sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
-		sandboxUser: z.string().optional(),
-		sandboxPidsLimit: z.coerce.number().int().positive().default(512),
-		sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
-		sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
-		scalerManaged: z.string().optional().transform((s) => s === "1"),
-		scalerIdleTimeoutMs: z.coerce.number().default(5e3),
-		scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
-		executionMode: ExecutionMode.optional(),
-		otelExporterOtlpEndpoint: z.string().optional(),
-		concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
-		isOrchestratorHost: z.string().optional().transform((v) => v === "true")
-	}),
+	schema: configSchema,
 	envMap: {
 		orchestratorUrl: "KICI_ORCHESTRATOR_URL",
 		agentId: "KICI_AGENT_ID",
@@ -136,11 +163,21 @@ const envDef = defineEnv({
 		sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
 		sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
 		scalerManaged: "KICI_SCALER_MANAGED",
+		jobImageAgent: "KICI_JOB_IMAGE_AGENT",
+		runtimeImage: "KICI_RUNTIME_IMAGE",
+		runtimeNodeSource: "KICI_RUNTIME_NODE_SOURCE",
+		containerBuildCli: "KICI_CONTAINER_BUILD_CLI",
+		scalerClaimCode: "KICI_SCALER_CLAIM_CODE",
 		scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
 		scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
 		executionMode: "KICI_EXECUTION_MODE",
 		otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
-		concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS"
+		concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS",
+		betweenJobsResetCommand: "KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND",
+		betweenJobsResetTimeoutMs: "KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS",
+		betweenJobsResetRunOn: "KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON",
+		orphanCleanup: "KICI_AGENT_ORPHAN_CLEANUP",
+		drainOnResetFailure: "KICI_AGENT_DRAIN_ON_RESET_FAILURE"
 	}
 });
 /**
@@ -172,10 +209,16 @@ const envDef = defineEnv({
 * - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
 * - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
 * - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
+* - KICI_SCALER_CLAIM_CODE (optional single-use claim code the agent exchanges for its own ephemeral credentials before registering; ignored when KICI_AGENT_TOKEN is set)
 * - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
 * - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
 * - KICI_EXECUTION_MODE (optional, options: container | bare-metal | firecracker) — override the runner's mode-pick logic
 * - KICI_CONCURRENCY_WAIT_TIMEOUT_MS (default: 3_600_000) — workflow-runner timeout when long-polling for a slot-release follow-up `concurrency.ack`
+* - KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND (optional) — host-reset command run between jobs on a reused agent (fail-open)
+* - KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS (default: 60000) — reset command timeout
+* - KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON (default: always, options: always | on-failure) — when the reset command runs
+* - KICI_AGENT_ORPHAN_CLEANUP (default: true) — reap a finished job's leaked process tree (bare-metal); false = only signal the runner child
+* - KICI_AGENT_DRAIN_ON_RESET_FAILURE (default: false) — drain the agent after repeated consecutive reset failures
 */
 function loadConfig() {
 	const data = envDef.parse();
@@ -752,7 +795,7 @@ const logger$3 = createLogger({ prefix: "dep-installer" });
 const execFileAsync = promisify(execFile);
 /** Install subprocess timeout (10 min) and stdout/stderr buffer (128 MiB). */
 const INSTALL_TIMEOUT_MS = 6e5;
-const INSTALL_MAX_BUFFER = 128 * 1024 * 1024;
+const INSTALL_MAX_BUFFER = 134217728;
 /**
 * Detect the package manager for the cloned repo from its committed manifests.
 * A pnpm workspace's `packageManager` field + `pnpm-lock.yaml` live at the repo
@@ -1268,8 +1311,8 @@ var logger$1, DOWNLOAD_TIMEOUT_MS$1, UPLOAD_TIMEOUT_MS, UPLOAD_RETRY_BASE_DELAY_
 var init_download = __esmMin((() => {
 	init_dep_restore();
 	logger$1 = createLogger({ prefix: "agent:download" });
-	DOWNLOAD_TIMEOUT_MS$1 = 300 * 1e3;
-	UPLOAD_TIMEOUT_MS = 300 * 1e3;
+	DOWNLOAD_TIMEOUT_MS$1 = 3e5;
+	UPLOAD_TIMEOUT_MS = 3e5;
 	UPLOAD_RETRY_BASE_DELAY_MS = 500;
 	PresignedUploadHttpError = class extends Error {
 		statusCode;
@@ -1301,11 +1344,13 @@ var init_download = __esmMin((() => {
 * right destination (repo entries under `workDir`, home entries under the
 * homedir). Extraction lands in a scratch dir first, then moves each group
 * into place so a partial restore never leaves half-written paths in the live
-* tree (mirrors dep-restore).
+* tree (mirrors dep-restore). The scratch dir honors `KICI_TMPDIR`, which may
+* sit on a different filesystem from the workspace, so the move falls back to
+* copy-then-remove on `EXDEV` rather than assuming a same-filesystem rename.
 */
 const logger = createLogger({ prefix: "cache-engine" });
 /** Download timeout for a presigned cache GET: 5 minutes. */
-const DOWNLOAD_TIMEOUT_MS = 300 * 1e3;
+const DOWNLOAD_TIMEOUT_MS = 3e5;
 /**
 * Resolve a cache path. `~`-prefixed -> home root; otherwise repo-root-relative.
 * Rejects absolute paths and `..` escapes so a workflow cannot read or clobber
@@ -1380,6 +1425,32 @@ async function packCachePaths(workDir, paths, roots) {
 		await cleanup();
 	}
 }
+/**
+* Move a tree from `src` to `dest`. Attempts a rename (same-filesystem, cheap)
+* and falls back to copy-then-remove only on `EXDEV` — the errno `rename(2)`
+* reports when the scratch dir (which honors `KICI_TMPDIR`) is on a different
+* filesystem from the destination workspace. `verbatimSymlinks` keeps a cached
+* symlink graph intact (mirroring packCachePaths), and `preserveTimestamps`
+* keeps mtimes stable so the fallback is byte-for-byte equivalent to the
+* rename. Any non-`EXDEV` error propagates so a real permission or corruption
+* failure is never silently turned into a copy.
+*/
+async function moveOrCopy(src, dest) {
+	try {
+		await rename(src, dest);
+	} catch (err) {
+		if (err.code !== "EXDEV") throw err;
+		await cp(src, dest, {
+			recursive: true,
+			verbatimSymlinks: true,
+			preserveTimestamps: true
+		});
+		await rm(src, {
+			recursive: true,
+			force: true
+		});
+	}
+}
 /** Move the extracted `__repo__` / `__home__` groups from a scratch dir into place. */
 async function moveAnchoredGroups(scratchDir, workDir, home) {
 	for (const anchor of await readdir(scratchDir)) {
@@ -1393,7 +1464,7 @@ async function moveAnchoredGroups(scratchDir, workDir, home) {
 				recursive: true,
 				force: true
 			});
-			await rename(join(anchorDir, child), dest);
+			await moveOrCopy(join(anchorDir, child), dest);
 		}
 	}
 }

package/dist/metrics/prometheus.d.ts

@@ -92,4 +92,30 @@ export declare const logBackpressureActive: import("@opentelemetry/api").UpDownC
  * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const connectionStatus: import("@opentelemetry/api").UpDownCounter<import("@opentelemetry/api").Attributes>;
+/**
+ * Total operator between-jobs reset command invocations.
+ * Labels:
+ * - status: success | failed | timeout
+ * - scaler: injected orch-side; `stateful` for static agents
+ */
+export declare const betweenJobsResetTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/**
+ * Between-jobs reset command duration in seconds.
+ * Labels:
+ * - scaler: injected orch-side
+ */
+export declare const betweenJobsResetDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
+/**
+ * Processes killed by the between-jobs process-group reap.
+ * Labels:
+ * - scaler: injected orch-side
+ */
+export declare const orphansReapedTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/**
+ * Out-of-band declared-cleanup re-runs after a hard-killed runner.
+ * Labels:
+ * - status: success | failed | timeout | skipped
+ * - scaler: injected orch-side
+ */
+export declare const orphanCleanupTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 //# sourceMappingURL=prometheus.d.ts.map