@kici-dev/agent 0.4.0 → 0.6.0

package/dist/index.js

@@ -6,12 +6,12 @@ import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/s
 import { KNOWN_ROLES, parseHostPropertyAssignments, validateNoReservedLabels } from "@kici-dev/engine";
 import { execFile } from "node:child_process";
 import { access, cp, lstat, mkdir, readFile, readdir, realpath, rename, rm, unlink, writeFile } from "node:fs/promises";
-import { makeTempDir } from "@kici-dev/core/tmp";
+import { existsSync } from "node:fs";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { makeTempDir } from "@kici-dev/core/tmp";
 import { promisify } from "node:util";
 import { createLogger, sha256, toErrorMessage } from "@kici-dev/shared";
 import { PNPM_IGNORE_BUILD_GATE_ARG, PackageManager, YarnFlavor, detectPackageManagerFromManifests, detectYarnFlavor } from "@kici-dev/shared/package-manager";
-import { existsSync } from "node:fs";
 import { parse, stringify } from "yaml";
 import { Readable, Transform } from "node:stream";
 import { pipeline } from "node:stream/promises";
@@ -40,6 +40,20 @@ var __exportAll = (all, no_symbols) => {
 	return target;
 };
 //#endregion
+//#region src/execution/image-build/build-engine.ts
+/**
+* Build a job's container image with the host's build CLI.
+*
+* The CLI is REQUIRED — there is deliberately no socket-API fallback. One build
+* path means one set of Dockerfile semantics: `.dockerignore`, BuildKit and
+* every directive behave as they do on the author's own machine, instead of
+* depending on which agent happened to pick the job up. A host without a CLI
+* cannot run a Dockerfile job, and says so in as many words.
+*/
+/** The build CLIs an agent host may provide. */
+const ContainerBuildCli = z.enum(["docker", "podman"]);
+ContainerBuildCli.enum.docker, ContainerBuildCli.enum.podman;
+//#endregion
 //#region src/config.ts
 /** Execution mode for the agent's sandbox backend. Mirrors the runtime enum. */
 const ExecutionMode = z.enum([
@@ -47,6 +61,71 @@ const ExecutionMode = z.enum([
 	"bare-metal",
 	"firecracker"
 ]);
+/** When the operator between-jobs reset command runs. */
+const BetweenJobsRunOn = z.enum(["always", "on-failure"]);
+const configSchema = z.object({
+	orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
+	agentId: z.string().optional(),
+	labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
+	properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
+	roles: z.string().optional().transform((s) => {
+		if (s === void 0) return void 0;
+		if (s === "") return [];
+		return s.split(",").filter(Boolean);
+	}).refine((roles) => {
+		if (roles === void 0) return true;
+		const validValues = [...KNOWN_ROLES, "all"];
+		return roles.every((r) => validValues.includes(r));
+	}, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
+		if (roles === void 0) return void 0;
+		if (roles.includes("all")) return void 0;
+		if (roles.length === 0) return [];
+		return roles.filter((r) => r !== "all");
+	}),
+	port: z.coerce.number().default(8080),
+	logLevel: z.enum([
+		"debug",
+		"info",
+		"warn",
+		"error"
+	]).default("info"),
+	agentToken: z.string().optional(),
+	githubToken: z.string().optional(),
+	maxLogSizeBytes: z.coerce.number().default(10485760),
+	defaultStepTimeoutMs: z.coerce.number().default(18e5),
+	dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
+	jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
+	backpressureMode: z.enum(["pause", "drop"]).default("pause"),
+	agentPayloadDir: z.string().optional(),
+	agentCommand: z.string().optional(),
+	sandbox: z.string().default("false").transform((s) => s === "true"),
+	trustedEnv: z.string().default("false").transform((s) => s === "true"),
+	inPlace: z.string().default("false").transform((s) => s === "true"),
+	sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
+	sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
+	sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
+	sandboxUser: z.string().optional(),
+	sandboxPidsLimit: z.coerce.number().int().positive().default(512),
+	sandboxMemoryBytes: z.coerce.number().int().positive().default(2147483648),
+	sandboxNanoCpus: z.coerce.number().int().positive().default(2e9),
+	scalerManaged: z.string().optional().transform((s) => s === "1"),
+	jobImageAgent: z.string().optional().transform((s) => s === "1"),
+	runtimeImage: z.string().optional(),
+	runtimeNodeSource: z.string().optional(),
+	containerBuildCli: ContainerBuildCli.optional(),
+	scalerClaimCode: z.string().optional(),
+	scalerIdleTimeoutMs: z.coerce.number().default(5e3),
+	scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
+	executionMode: ExecutionMode.optional(),
+	otelExporterOtlpEndpoint: z.string().optional(),
+	concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
+	isOrchestratorHost: z.string().optional().transform((v) => v === "true"),
+	betweenJobsResetCommand: z.string().optional(),
+	betweenJobsResetTimeoutMs: z.coerce.number().int().positive().default(6e4),
+	betweenJobsResetRunOn: BetweenJobsRunOn.default("always"),
+	orphanCleanup: z.string().default("true").transform((s) => s !== "false"),
+	drainOnResetFailure: z.string().default("false").transform((s) => s === "true")
+});
 /**
 * Env-var definition for the agent. Exported so the docs generator and the
 * deploy-stg pre-validator can inspect / re-parse without round-tripping
@@ -54,59 +133,7 @@ const ExecutionMode = z.enum([
 */
 const envDef = defineEnv({
 	service: "agent",
-	schema: z.object({
-		orchestratorUrl: z.string().url().min(1, "KICI_ORCHESTRATOR_URL is required"),
-		agentId: z.string().optional(),
-		labels: z.string().default("").transform((s) => s.split(",").filter(Boolean)),
-		properties: z.string().default("").transform((s) => parseHostPropertyAssignments(s.split(",").filter(Boolean))),
-		roles: z.string().optional().transform((s) => {
-			if (s === void 0) return void 0;
-			if (s === "") return [];
-			return s.split(",").filter(Boolean);
-		}).refine((roles) => {
-			if (roles === void 0) return true;
-			const validValues = [...KNOWN_ROLES, "all"];
-			return roles.every((r) => validValues.includes(r));
-		}, { message: `KICI_ROLES must contain only: ${[...KNOWN_ROLES, "all"].join(", ")}` }).transform((roles) => {
-			if (roles === void 0) return void 0;
-			if (roles.includes("all")) return void 0;
-			if (roles.length === 0) return [];
-			return roles.filter((r) => r !== "all");
-		}),
-		port: z.coerce.number().default(8080),
-		logLevel: z.enum([
-			"debug",
-			"info",
-			"warn",
-			"error"
-		]).default("info"),
-		agentToken: z.string().optional(),
-		githubToken: z.string().optional(),
-		maxLogSizeBytes: z.coerce.number().default(10 * 1024 * 1024),
-		defaultStepTimeoutMs: z.coerce.number().default(1800 * 1e3),
-		dockerKeepFailed: z.string().default("false").transform((s) => s === "true"),
-		jobHeartbeatIntervalMs: z.coerce.number().default(6e4),
-		backpressureMode: z.enum(["pause", "drop"]).default("pause"),
-		agentPayloadDir: z.string().optional(),
-		agentCommand: z.string().optional(),
-		sandbox: z.string().default("false").transform((s) => s === "true"),
-		trustedEnv: z.string().default("false").transform((s) => s === "true"),
-		inPlace: z.string().default("false").transform((s) => s === "true"),
-		sandboxNetwork: z.enum(["isolated", "host"]).default("isolated"),
-		sandboxHardened: z.string().default("true").transform((s) => s !== "false"),
-		sandboxReadonlyRootfs: z.string().default("false").transform((s) => s === "true"),
-		sandboxUser: z.string().optional(),
-		sandboxPidsLimit: z.coerce.number().int().positive().default(512),
-		sandboxMemoryBytes: z.coerce.number().int().positive().default(2 * 1024 * 1024 * 1024),
-		sandboxNanoCpus: z.coerce.number().int().positive().default(2 * 1e9),
-		scalerManaged: z.string().optional().transform((s) => s === "1"),
-		scalerIdleTimeoutMs: z.coerce.number().default(5e3),
-		scalerPendingDispatchTimeoutMs: z.coerce.number().default(6e4),
-		executionMode: ExecutionMode.optional(),
-		otelExporterOtlpEndpoint: z.string().optional(),
-		concurrencyWaitTimeoutMs: z.coerce.number().int().min(1e3).default(36e5),
-		isOrchestratorHost: z.string().optional().transform((v) => v === "true")
-	}),
+	schema: configSchema,
 	envMap: {
 		orchestratorUrl: "KICI_ORCHESTRATOR_URL",
 		agentId: "KICI_AGENT_ID",
@@ -136,11 +163,21 @@ const envDef = defineEnv({
 		sandboxMemoryBytes: "KICI_SANDBOX_MEMORY_BYTES",
 		sandboxNanoCpus: "KICI_SANDBOX_NANO_CPUS",
 		scalerManaged: "KICI_SCALER_MANAGED",
+		jobImageAgent: "KICI_JOB_IMAGE_AGENT",
+		runtimeImage: "KICI_RUNTIME_IMAGE",
+		runtimeNodeSource: "KICI_RUNTIME_NODE_SOURCE",
+		containerBuildCli: "KICI_CONTAINER_BUILD_CLI",
+		scalerClaimCode: "KICI_SCALER_CLAIM_CODE",
 		scalerIdleTimeoutMs: "KICI_SCALER_IDLE_TIMEOUT",
 		scalerPendingDispatchTimeoutMs: "KICI_SCALER_PENDING_DISPATCH_TIMEOUT",
 		executionMode: "KICI_EXECUTION_MODE",
 		otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
-		concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS"
+		concurrencyWaitTimeoutMs: "KICI_CONCURRENCY_WAIT_TIMEOUT_MS",
+		betweenJobsResetCommand: "KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND",
+		betweenJobsResetTimeoutMs: "KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS",
+		betweenJobsResetRunOn: "KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON",
+		orphanCleanup: "KICI_AGENT_ORPHAN_CLEANUP",
+		drainOnResetFailure: "KICI_AGENT_DRAIN_ON_RESET_FAILURE"
 	}
 });
 /**
@@ -172,10 +209,16 @@ const envDef = defineEnv({
 * - KICI_SANDBOX_MEMORY_BYTES (default: 2 GiB) — memory cap in bytes for the job container cgroup
 * - KICI_SANDBOX_NANO_CPUS (default: 2 CPUs) — CPU cap in nano-CPUs for the job container cgroup
 * - KICI_SCALER_MANAGED (set to "1" by the orchestrator's auto-scaler — agent self-shuts down on idle)
+* - KICI_SCALER_CLAIM_CODE (optional single-use claim code the agent exchanges for its own ephemeral credentials before registering; ignored when KICI_AGENT_TOKEN is set)
 * - KICI_SCALER_IDLE_TIMEOUT (ms, default 5000) — how long a scaler-managed agent waits before shutdown after going idle
 * - KICI_SCALER_PENDING_DISPATCH_TIMEOUT (ms, default 60000) — extended idle window when register.ack signals a queued bound job
 * - KICI_EXECUTION_MODE (optional, options: container | bare-metal | firecracker) — override the runner's mode-pick logic
 * - KICI_CONCURRENCY_WAIT_TIMEOUT_MS (default: 3_600_000) — workflow-runner timeout when long-polling for a slot-release follow-up `concurrency.ack`
+* - KICI_AGENT_BETWEEN_JOBS_RESET_COMMAND (optional) — host-reset command run between jobs on a reused agent (fail-open)
+* - KICI_AGENT_BETWEEN_JOBS_RESET_TIMEOUT_MS (default: 60000) — reset command timeout
+* - KICI_AGENT_BETWEEN_JOBS_RESET_RUN_ON (default: always, options: always | on-failure) — when the reset command runs
+* - KICI_AGENT_ORPHAN_CLEANUP (default: true) — reap a finished job's leaked process tree (bare-metal); false = only signal the runner child
+* - KICI_AGENT_DRAIN_ON_RESET_FAILURE (default: false) — drain the agent after repeated consecutive reset failures
 */
 function loadConfig() {
 	const data = envDef.parse();
@@ -748,11 +791,11 @@ function isAbsoluteRel(rel) {
 * env vars — the install runs with `--ignore-scripts` whenever a private
 * registry is configured.
 */
-const logger$2 = createLogger({ prefix: "dep-installer" });
+const logger$3 = createLogger({ prefix: "dep-installer" });
 const execFileAsync = promisify(execFile);
 /** Install subprocess timeout (10 min) and stdout/stderr buffer (128 MiB). */
 const INSTALL_TIMEOUT_MS = 6e5;
-const INSTALL_MAX_BUFFER = 128 * 1024 * 1024;
+const INSTALL_MAX_BUFFER = 134217728;
 /**
 * Detect the package manager for the cloned repo from its committed manifests.
 * A pnpm workspace's `packageManager` field + `pnpm-lock.yaml` live at the repo
@@ -793,7 +836,7 @@ async function installDeps(kiciDir, opts = {}) {
 	const repoRoot = opts.repoRoot ?? dirname(kiciDir);
 	const packageManager = await detectKiciPackageManager(repoRoot, kiciDir);
 	const yarnFlavor = packageManager === PackageManager.Yarn ? await detectKiciYarnFlavor(repoRoot, kiciDir) : YarnFlavor.Classic;
-	logger$2.info("Installing deps inline", {
+	logger$3.info("Installing deps inline", {
 		packageManager,
 		yarnFlavor,
 		dir: kiciDir
@@ -851,7 +894,7 @@ async function installDeps(kiciDir, opts = {}) {
 	if (packageManager === PackageManager.Yarn) await buildYarnWorkspaceClosure(repoRoot, kiciDir, yarnFlavor);
 	const durationMs = Date.now() - startTime;
 	process.stderr.write(`[dep-installer:trace] install complete: ${durationMs}ms\n`);
-	logger$2.info("Deps installed inline", {
+	logger$3.info("Deps installed inline", {
 		packageManager,
 		durationMs
 	});
@@ -1143,10 +1186,25 @@ var init_dep_restore = __esmMin((() => {
 * dep-restore.ts and workflow-loader.ts.
 */
 var download_exports = /* @__PURE__ */ __exportAll({
+	UPLOAD_MAX_RETRIES: () => 2,
 	downloadUrl: () => downloadUrl,
 	uploadToPresignedUrl: () => uploadToPresignedUrl
 });
 /**
+* Whether a failed upload attempt is worth repeating.
+*
+* A transport failure (connection refused, reset, DNS) never reached a
+* responder, and 5xx / 429 are the object-storage overload signals AWS
+* documents as retry-with-backoff (S3 answers `SlowDown` with 503). Every other
+* status is a decision the server will repeat: a 403 from an expired or
+* malformed signature, a 400 from a malformed request. Retrying those burns the
+* ceiling without a chance of success and delays the real error.
+*/
+function isRetryableUploadFailure(err) {
+	if (!(err instanceof PresignedUploadHttpError)) return true;
+	return err.statusCode >= 500 || err.statusCode === 429;
+}
+/**
 * Download content from an HTTP/HTTPS URL.
 *
 * Includes a 5-minute timeout to prevent the agent from hanging indefinitely
@@ -1170,21 +1228,10 @@ function downloadUrl(url) {
 		}).on("error", reject);
 	});
 }
-/**
-* Upload a buffer to a pre-signed S3 URL via HTTP PUT.
-*
-* Used for direct-to-S3 uploads of bundles and dep tarballs. Localhost /
-* 127.0.0.1 URLs are rewritten via `resolveOrchestratorUrl` so the
-* filesystem cache backend's signed URLs work from container agents that
-* can't reach the orchestrator's host loopback directly.
-*
-* @param url - The pre-signed URL to upload to
-* @param data - The buffer to upload
-*/
-function uploadToPresignedUrl(url, data) {
+/** One PUT of the whole buffer. Rejects with {@link PresignedUploadHttpError} on a non-2xx. */
+function putOnce(resolvedUrl, data, timeoutMs) {
 	return new Promise((resolve, reject) => {
-		const resolved = resolveOrchestratorUrl(url);
-		const parsed = new URL(resolved);
+		const parsed = new URL(resolvedUrl);
 		const req = (parsed.protocol === "https:" ? https : http).request({
 			hostname: parsed.hostname,
 			port: parsed.port,
@@ -1193,7 +1240,7 @@ function uploadToPresignedUrl(url, data) {
 			headers: { "Content-Length": data.length }
 		}, (res) => {
 			if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
-				reject(/* @__PURE__ */ new Error(`HTTP ${res.statusCode} uploading to pre-signed URL`));
+				reject(new PresignedUploadHttpError(res.statusCode));
 				res.resume();
 				return;
 			}
@@ -1201,14 +1248,80 @@ function uploadToPresignedUrl(url, data) {
 			res.on("end", () => resolve());
 			res.on("error", reject);
 		});
+		req.setTimeout(timeoutMs, () => {
+			req.destroy(/* @__PURE__ */ new Error(`Pre-signed upload timed out after ${timeoutMs}ms`));
+		});
 		req.on("error", reject);
 		req.end(data);
 	});
 }
-var DOWNLOAD_TIMEOUT_MS$1;
+/**
+* Upload a buffer to a pre-signed S3 URL via HTTP PUT, retrying a transient
+* failure.
+*
+* Used for direct-to-S3 uploads of bundles and dep tarballs. Localhost /
+* 127.0.0.1 URLs are rewritten via `resolveOrchestratorUrl` so the
+* filesystem cache backend's signed URLs work from container agents that
+* can't reach the orchestrator's host loopback directly.
+*
+* **Why retrying is safe here.** A pre-signed PUT writes one whole object at a
+* single key: there is no multipart session, no append, and no
+* server-generated identity, so a repeat attempt writes the same bytes to the
+* same key and the last write wins. S3 also only makes an object visible once
+* the body has been received in full, so an attempt that died mid-body left
+* nothing behind. A retry therefore cannot double-write or produce a torn
+* object — which is why every AWS SDK retries PUTs by default.
+*
+* Only a failure that can plausibly differ next time is repeated — see
+* {@link isRetryableUploadFailure}.
+*
+* @param url - The pre-signed URL to upload to
+* @param data - The buffer to upload
+* @param opts.baseDelayMs - Backoff before the first retry (doubles thereafter)
+* @param opts.timeoutMs - Per-attempt socket-inactivity timeout (see
+*   {@link UPLOAD_TIMEOUT_MS}); an override exists so a test can drive the
+*   stall path without waiting out the production budget.
+*/
+async function uploadToPresignedUrl(url, data, opts) {
+	const resolved = resolveOrchestratorUrl(url);
+	const baseDelayMs = opts?.baseDelayMs ?? UPLOAD_RETRY_BASE_DELAY_MS;
+	const timeoutMs = opts?.timeoutMs ?? UPLOAD_TIMEOUT_MS;
+	let lastError;
+	for (let attempt = 0; attempt <= 2; attempt++) {
+		if (attempt > 0) {
+			const delayMs = baseDelayMs * 2 ** (attempt - 1);
+			logger$1.warn("Retrying pre-signed upload", {
+				attempt,
+				delayMs,
+				error: lastError?.message
+			});
+			await new Promise((r) => setTimeout(r, delayMs));
+		}
+		try {
+			await putOnce(resolved, data, timeoutMs);
+			return;
+		} catch (err) {
+			lastError = err instanceof Error ? err : new Error(String(err));
+			if (!isRetryableUploadFailure(lastError)) throw lastError;
+		}
+	}
+	throw new Error(`Pre-signed upload failed after 3 attempts: ${lastError?.message}`);
+}
+var logger$1, DOWNLOAD_TIMEOUT_MS$1, UPLOAD_TIMEOUT_MS, UPLOAD_RETRY_BASE_DELAY_MS, PresignedUploadHttpError;
 var init_download = __esmMin((() => {
 	init_dep_restore();
-	DOWNLOAD_TIMEOUT_MS$1 = 300 * 1e3;
+	logger$1 = createLogger({ prefix: "agent:download" });
+	DOWNLOAD_TIMEOUT_MS$1 = 3e5;
+	UPLOAD_TIMEOUT_MS = 3e5;
+	UPLOAD_RETRY_BASE_DELAY_MS = 500;
+	PresignedUploadHttpError = class extends Error {
+		statusCode;
+		constructor(statusCode) {
+			super(`HTTP ${statusCode} uploading to pre-signed URL`);
+			this.statusCode = statusCode;
+			this.name = "PresignedUploadHttpError";
+		}
+	};
 }));
 //#endregion
 //#region src/execution/cache/cache-engine.ts
@@ -1231,11 +1344,13 @@ var init_download = __esmMin((() => {
 * right destination (repo entries under `workDir`, home entries under the
 * homedir). Extraction lands in a scratch dir first, then moves each group
 * into place so a partial restore never leaves half-written paths in the live
-* tree (mirrors dep-restore).
+* tree (mirrors dep-restore). The scratch dir honors `KICI_TMPDIR`, which may
+* sit on a different filesystem from the workspace, so the move falls back to
+* copy-then-remove on `EXDEV` rather than assuming a same-filesystem rename.
 */
 const logger = createLogger({ prefix: "cache-engine" });
 /** Download timeout for a presigned cache GET: 5 minutes. */
-const DOWNLOAD_TIMEOUT_MS = 300 * 1e3;
+const DOWNLOAD_TIMEOUT_MS = 3e5;
 /**
 * Resolve a cache path. `~`-prefixed -> home root; otherwise repo-root-relative.
 * Rejects absolute paths and `..` escapes so a workflow cannot read or clobber
@@ -1310,6 +1425,32 @@ async function packCachePaths(workDir, paths, roots) {
 		await cleanup();
 	}
 }
+/**
+* Move a tree from `src` to `dest`. Attempts a rename (same-filesystem, cheap)
+* and falls back to copy-then-remove only on `EXDEV` — the errno `rename(2)`
+* reports when the scratch dir (which honors `KICI_TMPDIR`) is on a different
+* filesystem from the destination workspace. `verbatimSymlinks` keeps a cached
+* symlink graph intact (mirroring packCachePaths), and `preserveTimestamps`
+* keeps mtimes stable so the fallback is byte-for-byte equivalent to the
+* rename. Any non-`EXDEV` error propagates so a real permission or corruption
+* failure is never silently turned into a copy.
+*/
+async function moveOrCopy(src, dest) {
+	try {
+		await rename(src, dest);
+	} catch (err) {
+		if (err.code !== "EXDEV") throw err;
+		await cp(src, dest, {
+			recursive: true,
+			verbatimSymlinks: true,
+			preserveTimestamps: true
+		});
+		await rm(src, {
+			recursive: true,
+			force: true
+		});
+	}
+}
 /** Move the extracted `__repo__` / `__home__` groups from a scratch dir into place. */
 async function moveAnchoredGroups(scratchDir, workDir, home) {
 	for (const anchor of await readdir(scratchDir)) {
@@ -1323,7 +1464,7 @@ async function moveAnchoredGroups(scratchDir, workDir, home) {
 				recursive: true,
 				force: true
 			});
-			await rename(join(anchorDir, child), dest);
+			await moveOrCopy(join(anchorDir, child), dest);
 		}
 	}
 }

package/dist/metrics/prometheus.d.ts

@@ -2,40 +2,40 @@
  * Total completed jobs.
  * Labels:
  * - status: success | failed | cancelled
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const jobsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
  * Currently running jobs.
  * Labels:
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const jobsActive: import("@opentelemetry/api").UpDownCounter<import("@opentelemetry/api").Attributes>;
 /**
  * Total completed steps.
  * Labels:
  * - status: success | failed | skipped
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const stepsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
  * Step execution duration in seconds.
  * Advisory boundaries cover sub-second steps through 30-minute long-running steps.
  * Labels:
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const stepDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
 /**
  * Git clone duration in seconds.
  * Advisory boundaries cover fast shallow clones through large repo clones.
  * Labels:
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const cloneDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
 /**
  * Total log bytes streamed back to orchestrator.
  * Labels:
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const logBytesTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
@@ -51,7 +51,7 @@ export declare const logBytesTotal: import("@opentelemetry/api").Counter<import(
  * Labels:
  * - mode: `pause` (producer paused, no data loss) | `drop` (lines
  *   discarded with a `[N lines dropped due to backpressure]` marker)
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const logBackpressureEventsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
@@ -64,7 +64,7 @@ export declare const logBackpressureEventsTotal: import("@opentelemetry/api").Co
  * producer.
  *
  * Labels:
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const logLinesDroppedTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
@@ -81,7 +81,7 @@ export declare const logLinesDroppedTotal: import("@opentelemetry/api").Counter<
  *
  * Labels:
  * - mode: `pause` | `drop`
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const logBackpressureActive: import("@opentelemetry/api").UpDownCounter<import("@opentelemetry/api").Attributes>;
 /**
@@ -89,7 +89,33 @@ export declare const logBackpressureActive: import("@opentelemetry/api").UpDownC
  * Use add(1) for connected, add(-1) for disconnected.
  *
  * Labels:
- * - scaler: injected by orch-side AgentMetricsAggregator (Phase 5b); `stateful` for static agents, backend name (`container` / `firecracker` / `bare-metal`) for scaler-managed
+ * - scaler: injected by orch-side AgentMetricsAggregator; `stateful` for static agents, backend type (`container` / `firecracker` / `bare-metal`) for scaler-managed
  */
 export declare const connectionStatus: import("@opentelemetry/api").UpDownCounter<import("@opentelemetry/api").Attributes>;
+/**
+ * Total operator between-jobs reset command invocations.
+ * Labels:
+ * - status: success | failed | timeout
+ * - scaler: injected orch-side; `stateful` for static agents
+ */
+export declare const betweenJobsResetTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/**
+ * Between-jobs reset command duration in seconds.
+ * Labels:
+ * - scaler: injected orch-side
+ */
+export declare const betweenJobsResetDurationSeconds: import("@opentelemetry/api").Histogram<import("@opentelemetry/api").Attributes>;
+/**
+ * Processes killed by the between-jobs process-group reap.
+ * Labels:
+ * - scaler: injected orch-side
+ */
+export declare const orphansReapedTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/**
+ * Out-of-band declared-cleanup re-runs after a hard-killed runner.
+ * Labels:
+ * - status: success | failed | timeout | skipped
+ * - scaler: injected orch-side
+ */
+export declare const orphanCleanupTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 //# sourceMappingURL=prometheus.d.ts.map