@kici-dev/agent 0.1.14 → 0.1.16

package/dist/execution/sandbox/workflow-runner.d.ts

@@ -13,5 +13,13 @@
  * This file is compiled alongside the agent by rolldown (existing build), but
  * runs as a SEPARATE process spawned by the sandbox backend.
  */
-export {};
+import type { Job, GenericInitConfig } from '@kici-dev/sdk';
+/** Raw provider webhook body for ctx.rawPayload — nested in the envelope. */
+export declare function rawPayloadFromEvent(event: Record<string, unknown> | undefined): Record<string, unknown> | undefined;
+/**
+ * Normalize `Job.init` (config | config[] | false | undefined) to an ordered
+ * array of init specs. `false` is an explicit opt-out and `undefined` (no
+ * config) both resolve to an empty list — the init phase is then a no-op.
+ */
+export declare function resolveInitSpecs(job: Job | undefined): GenericInitConfig[];
 //# sourceMappingURL=workflow-runner.d.ts.map

package/dist/execution/tmp-gc.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Startup garbage collection for this agent's own temp-directory families.
+ *
+ * Job workdirs (`kici-<6 random chars>`, see job-runner.ts) and isolated
+ * pnpm stores (`kici-pnpm-store-*`, see dep-installer.ts) clean themselves
+ * up in `finally` blocks — but a hard process death (SIGKILL, OOM kill)
+ * skips those, and on a long-lived bare-metal agent the leftovers then
+ * accumulate forever. Collecting anything older than a day at startup is
+ * safe on shared hosts: no job lives remotely that long (job timeouts are
+ * minutes), so a concurrent agent's in-flight dirs are never eligible.
+ */
+/**
+ * Collect this agent's stale temp dirs. `base` is overridable for tests;
+ * production callers use the default temp root. Never throws.
+ */
+export declare function gcStaleAgentTmpDirs(base?: string): Promise<string[]>;
+//# sourceMappingURL=tmp-gc.d.ts.map

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export { loadConfig, type AppConfig } from './config.js';
 export { installDeps, type InstallDepsOptions } from './execution/dep-installer.js';
 export { findLocalProtocolDeps, assertResolvableDeps, formatUnresolvableDepError, kiciHasLocalProtocolDeps, LocalDepProtocol, type LocalProtocolDep, } from './execution/validate-kici-deps.js';
+export { createCacheApi, packCachePaths, extractCacheTarball, downloadAndExtractCache, resolveCachePath, type CacheTransport, type CacheRoots, } from './execution/cache/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,19 +1,34 @@
-import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
-import { dirname as __cjs_dirname } from "node:path";
-__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
 import "node:module";
-import { hostname, tmpdir } from "node:os";
-import { randomUUID } from "node:crypto";
+import { homedir, hostname, tmpdir } from "node:os";
+import { createHash, randomUUID } from "node:crypto";
 import { z } from "zod";
 import { LOGGER_ENV_VARS, defineEnv, validateUnknownKiciVars } from "@kici-dev/shared/env";
 import { KNOWN_ROLES, validateNoReservedLabels } from "@kici-dev/engine";
 import { execFile } from "node:child_process";
-import { access, mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises";
-import { dirname, isAbsolute, join, relative, resolve } from "node:path";
+import { access, cp, mkdir, mkdtemp, readFile, readdir, rename, rm, unlink, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { promisify } from "node:util";
-import { createLogger, toErrorMessage } from "@kici-dev/shared";
+import { createLogger, sha256, toErrorMessage } from "@kici-dev/shared";
 import { PNPM_IGNORE_BUILD_GATE_ARG, PackageManager, detectPackageManagerFromManifests } from "@kici-dev/shared/package-manager";
 import { existsSync } from "node:fs";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { createGunzip } from "node:zlib";
+import { c, x } from "tar";
+import https from "node:https";
+import http from "node:http";
+import "node:url";
+var __defProp = Object.defineProperty;
+var __esmMin = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __exportAll = (all, no_symbols) => {
+	let target = {};
+	for (var name in all) __defProp(target, name, {
+		get: all[name],
+		enumerable: true
+	});
+	if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
+	return target;
+};
 import.meta.url;
 //#endregion
 //#region src/config.ts
@@ -444,7 +459,7 @@ async function assertResolvableDeps(args) {
 * env vars — the install runs with `--ignore-scripts` whenever a private
 * registry is configured.
 */
-const logger = createLogger({ prefix: "dep-installer" });
+const logger$2 = createLogger({ prefix: "dep-installer" });
 const execFileAsync = promisify(execFile);
 /** Install subprocess timeout (10 min) and stdout/stderr buffer (128 MiB). */
 const INSTALL_TIMEOUT_MS = 6e5;
@@ -479,7 +494,7 @@ async function detectKiciPackageManager(repoRoot, kiciDir) {
 async function installDeps(kiciDir, opts = {}) {
 	const repoRoot = opts.repoRoot ?? dirname(kiciDir);
 	const packageManager = await detectKiciPackageManager(repoRoot, kiciDir);
-	logger.info("Installing deps inline", {
+	logger$2.info("Installing deps inline", {
 		packageManager,
 		dir: kiciDir
 	});
@@ -520,7 +535,7 @@ async function installDeps(kiciDir, opts = {}) {
 	if (packageManager === PackageManager.Pnpm && await kiciHasLocalProtocolDeps(kiciDir)) await buildWorkspaceClosure(repoRoot);
 	const durationMs = Date.now() - startTime;
 	process.stderr.write(`[dep-installer:trace] install complete: ${durationMs}ms\n`);
-	logger.info("Deps installed inline", {
+	logger$2.info("Deps installed inline", {
 		packageManager,
 		durationMs
 	});
@@ -658,7 +673,322 @@ function logSubprocessStreams(e, tokens) {
 	if (e && typeof e === "object" && "stdout" in e) process.stderr.write(`[dep-installer:trace] stdout: ${redactNpmOutput(String(e.stdout), tokens).slice(0, 500)}\n`);
 	if (e && typeof e === "object" && "stderr" in e) process.stderr.write(`[dep-installer:trace] stderr: ${redactNpmOutput(String(e.stderr), tokens).slice(0, 500)}\n`);
 }
+/**
+* Rewrite localhost URLs to use the orchestrator host.
+*
+* The orchestrator rewrites file:// cache URLs to http://localhost:PORT/...
+* but agent containers can't reach localhost. This utility replaces the
+* host with the orchestrator's host derived from KICI_ORCHESTRATOR_URL.
+*/
+function resolveOrchestratorUrl(url) {
+	if (!url.match(/^https?:\/\/(localhost|127\.0\.0\.1)[:/]/)) return url;
+	const orchestratorUrl = process.env.KICI_ORCHESTRATOR_URL;
+	if (!orchestratorUrl) return url;
+	try {
+		const orchestratorParsed = new URL(orchestratorUrl.replace(/^ws/, "http"));
+		const parsed = new URL(url);
+		parsed.hostname = orchestratorParsed.hostname;
+		return parsed.toString();
+	} catch {
+		return url;
+	}
+}
+var SCRATCH_DIR_BASENAME_PREFIX;
+var init_dep_restore = __esmMin((() => {
+	createLogger({ prefix: "dep-restore" });
+	SCRATCH_DIR_BASENAME_PREFIX = ".dep-restore-scratch-";
+	`${SCRATCH_DIR_BASENAME_PREFIX}`;
+}));
+//#endregion
+//#region src/execution/download.ts
+/**
+* Shared HTTP/HTTPS download utility.
+*
+* Extracted from workflow-loader.ts to avoid duplication across
+* dep-restore.ts and workflow-loader.ts.
+*/
+var download_exports = /* @__PURE__ */ __exportAll({
+	downloadUrl: () => downloadUrl,
+	uploadToPresignedUrl: () => uploadToPresignedUrl
+});
+/**
+* Download content from an HTTP/HTTPS URL.
+*
+* Includes a 5-minute timeout to prevent the agent from hanging indefinitely
+* on slow or unresponsive endpoints.
+*
+* @param url - The URL to download from
+* @returns The response body as a Buffer
+*/
+function downloadUrl(url) {
+	return new Promise((resolve, reject) => {
+		(url.startsWith("https:") ? https : http).get(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS$1) }, (res) => {
+			if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
+				reject(/* @__PURE__ */ new Error(`HTTP ${res.statusCode} downloading from ${url}`));
+				res.resume();
+				return;
+			}
+			const chunks = [];
+			res.on("data", (chunk) => chunks.push(chunk));
+			res.on("end", () => resolve(Buffer.concat(chunks)));
+			res.on("error", reject);
+		}).on("error", reject);
+	});
+}
+/**
+* Upload a buffer to a pre-signed S3 URL via HTTP PUT.
+*
+* Used for direct-to-S3 uploads of bundles and dep tarballs. Localhost /
+* 127.0.0.1 URLs are rewritten via `resolveOrchestratorUrl` so the
+* filesystem cache backend's signed URLs work from container agents that
+* can't reach the orchestrator's host loopback directly.
+*
+* @param url - The pre-signed URL to upload to
+* @param data - The buffer to upload
+*/
+function uploadToPresignedUrl(url, data) {
+	return new Promise((resolve, reject) => {
+		const resolved = resolveOrchestratorUrl(url);
+		const parsed = new URL(resolved);
+		const req = (parsed.protocol === "https:" ? https : http).request({
+			hostname: parsed.hostname,
+			port: parsed.port,
+			path: parsed.pathname + parsed.search,
+			method: "PUT",
+			headers: { "Content-Length": data.length }
+		}, (res) => {
+			if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
+				reject(/* @__PURE__ */ new Error(`HTTP ${res.statusCode} uploading to pre-signed URL`));
+				res.resume();
+				return;
+			}
+			res.resume();
+			res.on("end", () => resolve());
+			res.on("error", reject);
+		});
+		req.on("error", reject);
+		req.end(data);
+	});
+}
+var DOWNLOAD_TIMEOUT_MS$1;
+var init_download = __esmMin((() => {
+	init_dep_restore();
+	DOWNLOAD_TIMEOUT_MS$1 = 300 * 1e3;
+}));
+//#endregion
+//#region src/execution/cache/cache-engine.ts
+/**
+* User-facing cache engine (sandbox-side).
+*
+* Packs `CacheSpec.paths` into a gzip tarball (mirrors dep-packer's tar+sha256
+* approach) and restores a tarball with on-the-fly SHA-256 verification
+* (mirrors dep-restore's streaming pipeline). Drives the orchestrator over an
+* injected request-response transport (IPC -> agent WS -> orchestrator).
+*
+* Path safety: each path is either `~`-prefixed (home-relative) or
+* repo-root-relative; absolute paths and `..` escapes are rejected so a
+* workflow cannot exfiltrate or clobber files outside its tree/home.
+*
+* Multi-root layout: a spec may mix repo-relative and home-relative paths.
+* Each entry is staged under an anchor prefix — `__repo__/<rel>` for
+* repo-root-relative entries, `__home__/<rel>` for `~`-prefixed entries — so a
+* single tarball can carry both roots and extract restores each group to the
+* right destination (repo entries under `workDir`, home entries under the
+* homedir). Extraction lands in a scratch dir first, then moves each group
+* into place so a partial restore never leaves half-written paths in the live
+* tree (mirrors dep-restore).
+*/
+const logger = createLogger({ prefix: "cache-engine" });
+/** Download timeout for a presigned cache GET: 5 minutes. */
+const DOWNLOAD_TIMEOUT_MS = 300 * 1e3;
+/** Anchor prefix for repo-root-relative cache entries inside the tar. */
+const REPO_ANCHOR = "__repo__";
+/** Anchor prefix for home-relative (`~`) cache entries inside the tar. */
+const HOME_ANCHOR = "__home__";
+/**
+* Resolve a cache path. `~`-prefixed -> home root; otherwise repo-root-relative.
+* Rejects absolute paths and `..` escapes so a workflow cannot read or clobber
+* files outside its tree / home.
+*/
+function resolveCachePath(workDir, p, roots) {
+	const home = roots?.home ?? homedir();
+	if (p === "~" || p.startsWith("~/")) {
+		const rel = p === "~" ? "" : p.slice(2);
+		return rel ? join(home, rel) : home;
+	}
+	if (isAbsolute(p)) throw new Error(`cache path must be repo-relative or ~-prefixed: ${p}`);
+	const resolved = resolve(workDir, p);
+	const rel = relative(workDir, resolved);
+	if (rel === ".." || rel.startsWith(`..${sep}`)) throw new Error(`cache path escapes the repo root: ${p}`);
+	return resolved;
+}
+/** Resolve + anchor every spec path; rejects escapes via resolveCachePath. */
+function anchorEntries(workDir, paths, roots) {
+	const home = roots?.home ?? homedir();
+	return paths.map((p) => {
+		const abs = resolveCachePath(workDir, p, roots);
+		const isHome = p === "~" || p.startsWith("~/");
+		return {
+			abs,
+			anchor: isHome ? HOME_ANCHOR : REPO_ANCHOR,
+			rel: relative(isHome ? home : workDir, abs)
+		};
+	});
+}
+/**
+* Pack the spec's paths into a gzip tarball + its SHA-256.
+*
+* Each path is copied into a staging dir under its anchor prefix
+* (`__repo__/<rel>` or `__home__/<rel>`), the staging dir is tarred (portable
+* mode strips uid/gid/mtime), and the staging dir is removed. The resulting
+* tarball self-describes which root each entry restores to.
+*/
+async function packCachePaths(workDir, paths, roots) {
+	const entries = anchorEntries(workDir, paths, roots);
+	const staging = await mkdtemp(join(tmpdir(), "kici-cache-pack-"));
+	try {
+		const topLevel = /* @__PURE__ */ new Set();
+		for (const e of entries) {
+			const dest = join(staging, e.anchor, e.rel);
+			await mkdir(dirname(dest), { recursive: true });
+			await cp(e.abs, dest, {
+				recursive: true,
+				verbatimSymlinks: true
+			});
+			topLevel.add(e.anchor);
+		}
+		const stream = c({
+			gzip: true,
+			portable: true,
+			cwd: staging
+		}, [...topLevel]);
+		const chunks = [];
+		for await (const chunk of stream) chunks.push(Buffer.from(chunk));
+		const tarball = Buffer.concat(chunks);
+		const hash = sha256(tarball);
+		logger.info("packed user cache", {
+			sizeBytes: tarball.length,
+			hash: hash.slice(0, 12),
+			paths
+		});
+		return {
+			tarball,
+			hash
+		};
+	} finally {
+		await rm(staging, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/** Move the extracted `__repo__` / `__home__` groups from a scratch dir into place. */
+async function moveAnchoredGroups(scratchDir, workDir, home) {
+	for (const anchor of await readdir(scratchDir)) {
+		const anchorDir = join(scratchDir, anchor);
+		const destRoot = anchor === HOME_ANCHOR ? home : anchor === REPO_ANCHOR ? workDir : null;
+		if (!destRoot) continue;
+		for (const child of await readdir(anchorDir)) {
+			const dest = join(destRoot, child);
+			await mkdir(dirname(dest), { recursive: true });
+			await rm(dest, {
+				recursive: true,
+				force: true
+			});
+			await rename(join(anchorDir, child), dest);
+		}
+	}
+}
+/**
+* Extract a cache tarball buffer, verifying its SHA-256 first, then move each
+* anchored group into place (repo entries under `workDir`, home entries under
+* the home root). Extracts into a scratch dir so a partial restore never
+* leaves half-written paths in the live tree.
+*/
+async function extractCacheTarball(tarball, workDir, expectedHash, roots) {
+	const actual = sha256(tarball);
+	if (actual !== expectedHash) throw new Error(`Cache tarball checksum mismatch: expected ${expectedHash}, got ${actual}`);
+	const home = roots?.home ?? homedir();
+	await mkdir(workDir, { recursive: true });
+	const scratch = await mkdtemp(join(tmpdir(), "kici-cache-extract-"));
+	try {
+		await new Promise((res, rej) => {
+			Readable.from(tarball).pipe(x({
+				cwd: scratch,
+				gzip: true
+			})).on("finish", res).on("error", rej);
+		});
+		await moveAnchoredGroups(scratch, workDir, home);
+	} finally {
+		await rm(scratch, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/**
+* Stream-download a presigned URL, verify its SHA-256 on the fly (mirrors
+* dep-restore's response -> hash -> gunzip -> tar pipeline), then move the
+* anchored groups into place. Extracts into a scratch dir so a failed download
+* never half-writes the live tree.
+*/
+async function downloadAndExtractCache(url, workDir, expectedHash, roots) {
+	const home = roots?.home ?? homedir();
+	const response = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) });
+	if (!response.ok || !response.body) throw new Error(`cache download HTTP ${response.status}`);
+	const hash = createHash("sha256");
+	const hashTransform = new Transform({ transform(chunk, _enc, cb) {
+		hash.update(chunk);
+		cb(null, chunk);
+	} });
+	await mkdir(workDir, { recursive: true });
+	const scratch = await mkdtemp(join(tmpdir(), "kici-cache-extract-"));
+	try {
+		await pipeline(Readable.fromWeb(response.body), hashTransform, createGunzip(), x({ cwd: scratch }));
+		const digest = hash.digest("hex");
+		if (digest !== expectedHash) throw new Error(`Cache tarball checksum mismatch on download: expected ${expectedHash}, got ${digest}`);
+		await moveAnchoredGroups(scratch, workDir, home);
+	} finally {
+		await rm(scratch, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/** Build the imperative `ctx.cache` API bound to a workDir + transport. */
+function createCacheApi(workDir, transport, roots) {
+	return {
+		async restore(spec) {
+			const r = await transport.restore(spec.key, spec.restoreKeys);
+			if (!r.hit || !r.downloadUrl || !r.tarHash) return { hit: false };
+			await downloadAndExtractCache(r.downloadUrl, workDir, r.tarHash, roots);
+			logger.info("user cache restored", {
+				key: spec.key,
+				matchedKey: r.matchedKey
+			});
+			return {
+				hit: true,
+				matchedKey: r.matchedKey
+			};
+		},
+		async save(spec) {
+			const begin = await transport.beginSave(spec.key);
+			if (begin.skip || !begin.uploadUrl) {
+				logger.info("user cache save skipped (key exists)", { key: spec.key });
+				return;
+			}
+			const { tarball, hash } = await packCachePaths(workDir, spec.paths, roots);
+			const { uploadToPresignedUrl } = await Promise.resolve().then(() => (init_download(), download_exports));
+			await uploadToPresignedUrl(begin.uploadUrl, tarball);
+			await transport.completeSave(spec.key, hash, tarball.length);
+			logger.info("user cache saved", {
+				key: spec.key,
+				sizeBytes: tarball.length
+			});
+		}
+	};
+}
 //#endregion
-export { LocalDepProtocol, assertResolvableDeps, findLocalProtocolDeps, formatUnresolvableDepError, installDeps, kiciHasLocalProtocolDeps, loadConfig };
+export { LocalDepProtocol, assertResolvableDeps, createCacheApi, downloadAndExtractCache, extractCacheTarball, findLocalProtocolDeps, formatUnresolvableDepError, installDeps, kiciHasLocalProtocolDeps, loadConfig, packCachePaths, resolveCachePath };
 
 //# sourceMappingURL=index.js.map