@mastra/deployer-sandbox 0.2.2 → 0.3.0-alpha.1

package/dist/index.js

@@ -379,6 +379,18 @@ var SandboxDeployer = class extends Deployer {
 	}
 };
 //#endregion
+//#region src/types.ts
+/** Error thrown before deployment when requested resource limits cannot be enforced. */
+var SandboxWorkerCapabilityError = class extends Error {
+	capability;
+	code = "SANDBOX_WORKER_CAPABILITY_UNAVAILABLE";
+	constructor(capability, message, options) {
+		super(message ?? `Worker resource-limit capability "${capability}" is unavailable.`, options);
+		this.capability = capability;
+		this.name = "SandboxWorkerCapabilityError";
+	}
+};
+//#endregion
 //#region src/worker.ts
 const ARCHIVE = ".mastra-worker.tar.gz";
 const RUNTIME_DIR = ".mastra/executions";
@@ -388,13 +400,17 @@ const ARTIFACT_LOCK = ".mastra-artifact-lock";
 const EXECUTION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
 const DEFAULT_INPUT_LIMIT = 16 * 1024 * 1024;
 const DEFAULT_OUTPUT_READ_LIMIT = 1024 * 1024;
+const RESOURCE_CAPABILITY_PREFIX = "MASTRA_WORKER_CAPABILITY:";
 async function deployWorkerToSandbox(options) {
 	validateOptions(options);
-	const { sandbox, dir, executionId, command, mode = "worker", args = [], env = {}, workingDirectory = ".", installCommand = "npm install --omit=dev", startupTimeoutMs = 1e4, executionTimeoutMs, terminationGraceMs = 5e3, inputLimitBytes = DEFAULT_INPUT_LIMIT } = options;
+	const { sandbox, dir, executionId, command, mode = "worker", args = [], env = {}, workingDirectory = ".", installCommand = "npm install --omit=dev", startupTimeoutMs = 1e4, executionTimeoutMs, resourceLimits: requestedResourceLimits, terminationGraceMs = 5e3, inputLimitBytes = DEFAULT_INPUT_LIMIT } = options;
+	const resourceLimits = normalizeResourceLimits(requestedResourceLimits);
+	if (resourceLimits) await preflightResourceLimits(sandbox, resourceLimits);
 	const remoteDir = await resolveRemoteDir(sandbox, options.remoteDir);
 	const config = {
 		sandbox,
 		remoteDir,
+		resolveRemoteDir: async () => remoteDir,
 		command,
 		args,
 		env,
@@ -402,6 +418,7 @@ async function deployWorkerToSandbox(options) {
 		mode,
 		startupTimeoutMs,
 		executionTimeoutMs,
+		resourceLimits,
 		terminationGraceMs,
 		inputLimitBytes
 	};
@@ -431,6 +448,20 @@ async function deployWorkerToSandbox(options) {
 	}
 	return createExecution(config, executionId, options.input);
 }
+/** Reattach to a persisted worker execution without its original launch configuration. */
+async function attachWorkerDeployment(options) {
+	if (!options.sandbox.executeCommand) throw new Error(`Sandbox provider "${options.sandbox.provider}" does not support executeCommand, which is required for worker deploys.`);
+	validateExecutionId(options.executionId);
+	if (options.terminationGraceMs !== void 0 && (!Number.isFinite(options.terminationGraceMs) || options.terminationGraceMs <= 0)) throw new Error("terminationGraceMs must be greater than zero.");
+	let remoteDir;
+	const config = {
+		sandbox: options.sandbox,
+		resolveRemoteDir: async () => remoteDir ??= await resolveRemoteDir(options.sandbox, options.remoteDir),
+		terminationGraceMs: options.terminationGraceMs ?? 5e3
+	};
+	const info = await getInfoSafe(options.sandbox);
+	return execution(config, options.executionId, info?.id ?? options.sandbox.id, info?.timeoutAt);
+}
 function validateOptions(options) {
 	if (!options.sandbox.executeCommand) throw new Error(`Sandbox provider "${options.sandbox.provider}" does not support executeCommand, which is required for worker deploys.`);
 	validateExecutionId(options.executionId);
@@ -445,6 +476,22 @@ function validateOptions(options) {
 		["executionTimeoutMs", options.executionTimeoutMs],
 		["terminationGraceMs", options.terminationGraceMs]
 	]) if (value !== void 0 && (!Number.isFinite(value) || value <= 0)) throw new Error(`${name} must be greater than zero.`);
+	const resourceLimits = options.resourceLimits;
+	if (resourceLimits) {
+		const knownLimits = /* @__PURE__ */ new Set([
+			"cpuTimeSeconds",
+			"addressSpaceBytes",
+			"fileSizeBytes",
+			"openFiles"
+		]);
+		for (const name of Object.keys(resourceLimits)) if (!knownLimits.has(name)) throw new Error(`Unknown worker resource limit: ${name}.`);
+		for (const [name, value] of [
+			["cpuTimeSeconds", resourceLimits.cpuTimeSeconds],
+			["addressSpaceBytes", resourceLimits.addressSpaceBytes],
+			["fileSizeBytes", resourceLimits.fileSizeBytes],
+			["openFiles", resourceLimits.openFiles]
+		]) if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) throw new Error(`Worker resourceLimits.${name} must be a positive safe integer.`);
+	}
 }
 function validateRelativePath(value, label) {
 	if (!value || posix.isAbsolute(value) || posix.normalize(value).startsWith("..")) throw new Error(`Worker ${label} must stay within the deployed artifact root.`);
@@ -452,6 +499,60 @@ function validateRelativePath(value, label) {
 function validateInput(input) {
 	if (input?.type === "file") validateRelativePath(input.path, "input file path");
 }
+function normalizeResourceLimits(limits) {
+	if (!limits || Object.values(limits).every((value) => value === void 0)) return void 0;
+	return {
+		cpuTimeSeconds: limits.cpuTimeSeconds,
+		addressSpaceBytes: limits.addressSpaceBytes,
+		addressSpaceKilobytes: limits.addressSpaceBytes === void 0 ? void 0 : Math.floor(limits.addressSpaceBytes / 1024),
+		fileSizeBytes: limits.fileSizeBytes,
+		fileSizeBlocks: limits.fileSizeBytes === void 0 ? void 0 : Math.floor(limits.fileSizeBytes / 512),
+		openFiles: limits.openFiles
+	};
+}
+async function preflightResourceLimits(sandbox, resourceLimits) {
+	const checks = [];
+	if (resourceLimits.cpuTimeSeconds !== void 0) checks.push(`check_limit cpu_time -t ${resourceLimits.cpuTimeSeconds}`);
+	if (resourceLimits.addressSpaceKilobytes !== void 0) checks.push(`check_limit address_space -v ${resourceLimits.addressSpaceKilobytes}`);
+	if (resourceLimits.fileSizeBlocks !== void 0) checks.push(`check_limit file_size -f ${resourceLimits.fileSizeBlocks}`);
+	if (resourceLimits.openFiles !== void 0) checks.push(`check_limit open_files -n ${resourceLimits.openFiles}`);
+	if (resourceLimits.cpuTimeSeconds !== void 0) checks.push(`kill -l XCPU >/dev/null 2>&1 || fail cpu_signal`);
+	if (resourceLimits.fileSizeBytes !== void 0) checks.push(`kill -l XFSZ >/dev/null 2>&1 || fail file_size_signal`);
+	const script = `
+fail() {
+  printf '${RESOURCE_CAPABILITY_PREFIX}%s\\n' "$1" >&2
+  exit 1
+}
+check_limit() {
+  capability="$1"
+  flag="$2"
+  value="$3"
+  (
+    ulimit -S "$flag" "$value" >/dev/null 2>&1 || exit 1
+    ulimit -H "$flag" "$value" >/dev/null 2>&1 || exit 1
+    [ "$(ulimit -S "$flag")" = "$value" ] || exit 1
+    [ "$(ulimit -H "$flag")" = "$value" ] || exit 1
+    if ulimit -H "$flag" "$((value + 1))" >/dev/null 2>&1; then exit 1; fi
+  ) || fail "$capability"
+}
+[ "$(uname -s 2>/dev/null)" = Linux ] && [ -r /proc/self/stat ] || fail linux_proc
+command -v setsid >/dev/null 2>&1 || fail process_groups
+setsid sh -c 'kill -0 -$$ 2>/dev/null' || fail process_groups
+${checks.join("\n")}
+`;
+	let result;
+	try {
+		result = await runInSandbox(sandbox, script, {
+			allowFailure: true,
+			label: "preflight worker resource limits"
+		});
+	} catch (error) {
+		throw new SandboxWorkerCapabilityError("sandbox_command", void 0, { cause: error });
+	}
+	if (result.exitCode === 0) return;
+	const detail = `${result.stderr}\n${result.stdout}`;
+	throw new SandboxWorkerCapabilityError(detail.match(new RegExp(`${RESOURCE_CAPABILITY_PREFIX}([a-z_]+)`))?.[1] ?? "sandbox_command", void 0, { cause: new Error(detail.trim() || "Resource-limit preflight command failed.") });
+}
 async function acquireLock(sandbox, lock, timeout, label) {
 	const timeoutMs = timeout ?? 6e5;
 	const attempts = Math.max(1, Math.ceil(timeoutMs / 1e3));
@@ -503,25 +604,32 @@ async function createExecution(config, executionId, input) {
 		await writeFailedStatus(config.sandbox, paths, executionId, "launch", error);
 		throw workerPhaseError("launch", error);
 	}
-	const startup = await waitForStartup(config, executionId, paths);
-	if (startup.state === "timed_out") await cancelExecution(config, executionId, paths, "startup");
+	const resolvePaths = async () => paths;
+	const startup = await waitForStartup(config, executionId, resolvePaths);
+	if (startup.state === "timed_out") await cancelExecution(config, executionId, resolvePaths, "startup");
 	if (startup.state === "failed" || startup.state === "timed_out" || startup.state === "provider_unavailable") throw new Error(`Worker ${startup.state} during startup${"message" in startup && startup.message ? `: ${startup.message}` : ""}.`);
 	const info = await getInfoSafe(config.sandbox);
-	return deployment(config, executionId, paths, info?.id ?? config.sandbox.id ?? "unknown", info?.timeoutAt);
+	return deployment(config, executionId, info?.id ?? config.sandbox.id ?? "unknown", info?.timeoutAt);
 }
-function deployment(config, executionId, paths, sandboxId, expiresAt) {
+function execution(config, executionId, sandboxId, expiresAt) {
+	const resolvePaths = async () => executionPaths(await config.resolveRemoteDir(), executionId);
 	return {
 		sandboxId,
 		executionId,
 		expiresAt,
-		status: (options) => readWorkerStatus(config.sandbox, executionId, paths, options),
-		readOutput: (stream, options) => readOutput(config.sandbox, executionId, paths, stream, options),
-		cancel: () => cancelExecution(config, executionId, paths),
+		status: (options) => readWorkerStatus(config.sandbox, executionId, resolvePaths, options),
+		readOutput: (stream, options) => readOutput(config.sandbox, executionId, resolvePaths, stream, options),
+		cancel: () => cancelExecution(config, executionId, resolvePaths),
 		stop: async () => {
 			if (!config.sandbox.stop) throw new Error(`Sandbox provider "${config.sandbox.provider}" does not support stop.`);
 			await config.sandbox.stop();
 		},
-		destroy: (options) => destroyWithRetry(config.sandbox, options),
+		destroy: (options) => destroyWithRetry(config.sandbox, options)
+	};
+}
+function deployment(config, executionId, sandboxId, expiresAt) {
+	return {
+		...execution(config, executionId, sandboxId, expiresAt),
 		relaunch: async (options) => {
 			if (options.executionId === executionId) throw new Error("Relaunch requires a new executionId.");
 			validateInput(options.input);
@@ -543,8 +651,31 @@ function buildExecutionScript(config, paths, stdinPath) {
 	const envPrefix = Object.entries(config.env).map(([key, value]) => `${key}=${shellQuote(value)}`).join(" ");
 	const executable = [shellQuote(config.command), ...config.args.map(shellQuote)].join(" ");
 	const target = `${envPrefix ? `env ${envPrefix} ` : ""}${executable}`;
+	const limitCommands = [];
+	if (config.resourceLimits?.cpuTimeSeconds !== void 0) {
+		limitCommands.push(`ulimit -S -t ${config.resourceLimits.cpuTimeSeconds}`);
+		limitCommands.push(`ulimit -H -t ${config.resourceLimits.cpuTimeSeconds}`);
+	}
+	if (config.resourceLimits?.addressSpaceKilobytes !== void 0) {
+		limitCommands.push(`ulimit -S -v ${config.resourceLimits.addressSpaceKilobytes}`);
+		limitCommands.push(`ulimit -H -v ${config.resourceLimits.addressSpaceKilobytes}`);
+	}
+	if (config.resourceLimits?.fileSizeBlocks !== void 0) {
+		limitCommands.push(`ulimit -S -f ${config.resourceLimits.fileSizeBlocks}`);
+		limitCommands.push(`ulimit -H -f ${config.resourceLimits.fileSizeBlocks}`);
+	}
+	if (config.resourceLimits?.openFiles !== void 0) {
+		limitCommands.push(`ulimit -S -n ${config.resourceLimits.openFiles}`);
+		limitCommands.push(`ulimit -H -n ${config.resourceLimits.openFiles}`);
+	}
+	const workload = config.resourceLimits ? `${limitCommands.map((command) => `${command} || exit 125`).join("\n")}\nexec ${target}` : `exec ${target}`;
 	const graceAttempts = Math.max(1, Math.ceil(config.terminationGraceMs / 1e3));
 	const state = (value) => `tmp=${shellQuote(`${paths.status}.tmp.$$`)}; printf '%s\\n' ${shellQuote(value)} > "$tmp"; mv "$tmp" ${shellQuote(paths.status)};`;
+	const normalExitStatus = `signal=''; if [ "$code" -gt 128 ]; then signal="SIG$((code - 128))"; fi; tmp=${shellQuote(`${paths.status}.tmp.$$`)}; printf 'exited|%s|%s|%s\\n' "$execution_id" "$code" "$signal" > "$tmp"; mv "$tmp" ${shellQuote(paths.status)};`;
+	const resourceExitBranches = [];
+	if (config.resourceLimits?.cpuTimeSeconds !== void 0) resourceExitBranches.push(`if xcpu="$(kill -l XCPU 2>/dev/null)" && [ -n "$xcpu" ] && [ "$code" -eq $((128 + xcpu)) ]; then ${state(`resource_exhausted|${paths.executionId}|cpu|${config.resourceLimits.cpuTimeSeconds}|SIGXCPU`)}`);
+	if (config.resourceLimits?.fileSizeBytes !== void 0) resourceExitBranches.push(`${resourceExitBranches.length ? "elif" : "if"} xfsz="$(kill -l XFSZ 2>/dev/null)" && [ -n "$xfsz" ] && [ "$code" -eq $((128 + xfsz)) ]; then ${state(`resource_exhausted|${paths.executionId}|file_size|${config.resourceLimits.fileSizeBytes}|SIGXFSZ`)}`);
+	const resourceExitStatus = resourceExitBranches.length ? `${resourceExitBranches.join(" ")} else ${normalExitStatus} fi` : normalExitStatus;
 	return [
 		"#!/bin/sh",
 		`cd ${shellQuote(cwd)}`,
@@ -555,7 +686,7 @@ function buildExecutionScript(config, paths, stdinPath) {
 		`tokenfile=${shellQuote(paths.pidToken)}`,
 		`: > "$stdout"; : > "$stderr"`,
 		state(`starting|${paths.executionId}`),
-		`setsid sh -c ${shellQuote(`exec ${target}${stdinPath ? ` < ${shellQuote(stdinPath)}` : ""}`)} > "$stdout" 2> "$stderr" &`,
+		`setsid sh -c ${shellQuote(`${workload}${stdinPath ? ` < ${shellQuote(stdinPath)}` : ""}`)} > "$stdout" 2> "$stderr" &`,
 		"child=$!",
 		"printf %s \"$child\" > \"$pidfile\"",
 		`if [ -r "/proc/$child/stat" ]; then awk '{print $22}' "/proc/$child/stat" > "$tokenfile"; else : > "$tokenfile"; fi`,
@@ -567,7 +698,7 @@ function buildExecutionScript(config, paths, stdinPath) {
 		"code=$?",
 		...config.executionTimeoutMs ? ["kill \"$watchdog\" 2>/dev/null || true"] : [],
 		`current="$(cat ${shellQuote(paths.status)} 2>/dev/null || true)"`,
-		`case "$current" in timed_out*) ;; *) if [ "$cancelled" -eq 1 ]; then ${state(`cancelled|${paths.executionId}|TERM`)} else signal=''; if [ "$code" -gt 128 ]; then signal="SIG$((code - 128))"; fi; tmp=${shellQuote(`${paths.status}.tmp.$$`)}; printf 'exited|%s|%s|%s\\n' "$execution_id" "$code" "$signal" > "$tmp"; mv "$tmp" ${shellQuote(paths.status)}; fi ;; esac`,
+		`case "$current" in timed_out*) ;; *) if [ "$cancelled" -eq 1 ]; then ${state(`cancelled|${paths.executionId}|TERM`)} else ${resourceExitStatus} fi ;; esac`,
 		"rm -f \"$pidfile\" \"$tokenfile\"",
 		"exit \"$code\""
 	].join("\n");
@@ -575,10 +706,10 @@ function buildExecutionScript(config, paths, stdinPath) {
 async function launchExecution(sandbox, paths) {
 	await runInSandbox(sandbox, `setsid nohup sh ${shellQuote(paths.script)} >/dev/null 2>&1 & echo $!`, { label: "launch worker execution" });
 }
-async function waitForStartup(config, executionId, paths) {
+async function waitForStartup(config, executionId, resolvePaths) {
 	const deadline = Date.now() + config.startupTimeoutMs;
 	while (Date.now() < deadline) {
-		const status = await readWorkerStatus(config.sandbox, executionId, paths);
+		const status = await readWorkerStatus(config.sandbox, executionId, resolvePaths);
 		if (status.state !== "unknown" && status.state !== "starting") return status;
 		await new Promise((resolve) => setTimeout(resolve, 100));
 	}
@@ -588,7 +719,7 @@ async function waitForStartup(config, executionId, paths) {
 		phase: "startup"
 	};
 }
-async function readWorkerStatus(sandbox, executionId, paths, options) {
+async function readWorkerStatus(sandbox, executionId, resolvePaths, options) {
 	const providerState = sandbox.status;
 	if (providerState === "destroyed" || providerState === "destroying") return {
 		state: "provider_unavailable",
@@ -613,6 +744,7 @@ async function readWorkerStatus(sandbox, executionId, paths, options) {
 		}
 	}
 	try {
+		const paths = await resolvePaths();
 		const result = await runInSandbox(sandbox, [
 			`status="$(cat ${shellQuote(paths.status)} 2>/dev/null || true)"`,
 			`if [ -f ${shellQuote(paths.pid)} ]; then`,
@@ -644,7 +776,7 @@ async function readWorkerStatus(sandbox, executionId, paths, options) {
 	}
 }
 function parseStatus(executionId, value) {
-	const [state, recordedId, first, second] = value.split("|");
+	const [state, recordedId, first, second, third] = value.split("|");
 	if (recordedId !== executionId || state === "stale") return {
 		state: "unknown",
 		executionId
@@ -669,6 +801,27 @@ function parseStatus(executionId, value) {
 			executionId
 		};
 	}
+	if (state === "resource_exhausted") {
+		const limit = Number(second);
+		if (first === "cpu" && Number.isSafeInteger(limit) && limit > 0 && third === "SIGXCPU") return {
+			state,
+			executionId,
+			resource: first,
+			limit,
+			signal: third
+		};
+		if (first === "file_size" && Number.isSafeInteger(limit) && limit > 0 && third === "SIGXFSZ") return {
+			state,
+			executionId,
+			resource: first,
+			limit,
+			signal: third
+		};
+		return {
+			state: "unknown",
+			executionId
+		};
+	}
 	if (state === "cancelled") return {
 		state,
 		executionId,
@@ -690,9 +843,10 @@ function parseStatus(executionId, value) {
 		executionId
 	};
 }
-async function cancelExecution(config, executionId, paths, timeoutPhase) {
-	const current = await readWorkerStatus(config.sandbox, executionId, paths);
+async function cancelExecution(config, executionId, resolvePaths, timeoutPhase) {
+	const current = await readWorkerStatus(config.sandbox, executionId, resolvePaths);
 	if (current.state !== "running" && current.state !== "starting") return current;
+	const paths = await resolvePaths();
 	const attempts = Math.max(1, Math.ceil(config.terminationGraceMs / 1e3));
 	const terminal = timeoutPhase ? `timed_out|${executionId}|startup` : `cancelled|${executionId}|TERM`;
 	await runInSandbox(config.sandbox, [
@@ -713,11 +867,12 @@ async function cancelExecution(config, executionId, paths, timeoutPhase) {
 	});
 	return parseStatus(executionId, terminal);
 }
-async function readOutput(sandbox, executionId, paths, stream, options) {
+async function readOutput(sandbox, executionId, resolvePaths, stream, options) {
 	const offset = Math.max(0, Math.floor(options?.offset ?? 0));
 	const maxBytes = Math.max(1, Math.floor(options?.maxBytes ?? DEFAULT_OUTPUT_READ_LIMIT));
-	const path = stream === "stdout" ? paths.stdout : paths.stderr;
 	try {
+		const paths = await resolvePaths();
+		const path = stream === "stdout" ? paths.stdout : paths.stderr;
 		const result = await runInSandbox(sandbox, `size=$(wc -c < ${shellQuote(path)} 2>/dev/null || echo 0); printf '%s\\n' "$size"; tail -c +${offset + 1} ${shellQuote(path)} 2>/dev/null | head -c ${maxBytes} | base64`, {
 			allowFailure: true,
 			label: `read worker ${stream}`
@@ -728,9 +883,10 @@ async function readOutput(sandbox, executionId, paths, stream, options) {
 		const encoded = newline === -1 ? "" : result.stdout.slice(newline + 1).replace(/\s/g, "");
 		const data = Buffer.from(encoded, "base64");
 		const nextOffset = offset + data.byteLength;
-		const status = await readWorkerStatus(sandbox, executionId, paths);
+		const status = await readWorkerStatus(sandbox, executionId, resolvePaths);
 		const terminal = [
 			"exited",
+			"resource_exhausted",
 			"cancelled",
 			"timed_out",
 			"failed"
@@ -819,6 +975,6 @@ function sanitizeStatusValue(value) {
 	return value.replace(/[|\r\n]/g, " ").slice(0, 500);
 }
 //#endregion
-export { MANIFEST_FILENAME, SandboxDeployer, buildLaunchScript, deployToSandbox, deployWorkerToSandbox, readDeploymentManifest, updateEdgeConfigAlias, writeDeploymentManifest };
+export { MANIFEST_FILENAME, SandboxDeployer, SandboxWorkerCapabilityError, attachWorkerDeployment, buildLaunchScript, deployToSandbox, deployWorkerToSandbox, readDeploymentManifest, updateEdgeConfigAlias, writeDeploymentManifest };
 
 //# sourceMappingURL=index.js.map