@mastra/deployer-sandbox 0.2.3-alpha.0 → 0.3.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +13 -0
- package/dist/index.cjs +178 -20
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +177 -21
- package/dist/index.js.map +1 -1
- package/dist/studio/assets/{core-R8IQBlLo.js → core-Dx7mVglv.js} +1 -1
- package/dist/studio/assets/{index-DXHR3Phj.js → index-DanFGEb-.js} +2 -2
- package/dist/studio/assets/{main-jIvIzWG_.js → main-CAJWNDlQ.js} +284 -389
- package/dist/studio/assets/style-fSltWkSH.css +1 -0
- package/dist/studio/index.html +2 -2
- package/dist/types.d.ts +41 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/worker.d.ts +3 -1
- package/dist/worker.d.ts.map +1 -1
- package/package.json +5 -4
- package/dist/studio/assets/style-CUsRHh-q.css +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
# @mastra/deployer-sandbox
|
|
2
2
|
|
|
3
|
+
## 0.3.0-alpha.1
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- Added `attachWorkerDeployment()` so restarted supervisors can reconstruct worker handles from persisted sandbox and execution identities. ([#21271](https://github.com/mastra-ai/mastra/pull/21271))
|
|
8
|
+
|
|
9
|
+
```typescript
|
|
10
|
+
const worker = await attachWorkerDeployment({ sandbox, executionId });
|
|
11
|
+
const status = await worker.status();
|
|
12
|
+
const output = await worker.readOutput('stdout', { offset });
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
- Added fail-closed hard resource limits for sandbox workers. ([#21273](https://github.com/mastra-ai/mastra/pull/21273))
|
|
16
|
+
|
|
17
|
+
Workers can now opt into per-attempt CPU time, address-space, file-size, and open-file limits:
|
|
18
|
+
|
|
19
|
+
```ts
|
|
20
|
+
await deployWorkerToSandbox({
|
|
21
|
+
// ...
|
|
22
|
+
resourceLimits: {
|
|
23
|
+
cpuTimeSeconds: 30,
|
|
24
|
+
addressSpaceBytes: 536_870_912,
|
|
25
|
+
fileSizeBytes: 10_485_760,
|
|
26
|
+
openFiles: 256,
|
|
27
|
+
},
|
|
28
|
+
});
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Requested limits are capability-checked before deployment. CPU and file-size signal exhaustion is reported through the typed `resource_exhausted` status.
|
|
32
|
+
|
|
33
|
+
### Patch Changes
|
|
34
|
+
|
|
35
|
+
- Updated dependencies [[`dc4a25d`](https://github.com/mastra-ai/mastra/commit/dc4a25d41af4e2fe97a816070eaec6aa963ab53b), [`dc4a25d`](https://github.com/mastra-ai/mastra/commit/dc4a25d41af4e2fe97a816070eaec6aa963ab53b)]:
|
|
36
|
+
- @mastra/core@1.58.0-alpha.15
|
|
37
|
+
- @mastra/deployer@1.58.0-alpha.15
|
|
38
|
+
|
|
3
39
|
## 0.2.3-alpha.0
|
|
4
40
|
|
|
5
41
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -91,6 +91,19 @@ Each caller-provided execution ID gets isolated runtime state. Input is bounded
|
|
|
91
91
|
|
|
92
92
|
Dependency installs are serialized and cached using the artifact's `package.json`, supported lockfiles, and install command. `cancel()` sends TERM and then KILL to the process group when necessary. `stop()` snapshot-stops the provider sandbox without assuming process preservation. `relaunch({ executionId })` starts the recorded command under a new execution identity. `destroy()` retries permanent sandbox deletion and returns a typed result.
|
|
93
93
|
|
|
94
|
+
A restarted supervisor can reconstruct an operational handle from the persisted sandbox and execution identities. The attached handle supports status, offset-based output reads, cancellation, stop, and destroy, but not relaunch because the original launch configuration isn't persisted.
|
|
95
|
+
|
|
96
|
+
```typescript
|
|
97
|
+
import { attachWorkerDeployment } from '@mastra/deployer-sandbox';
|
|
98
|
+
import { VercelSandbox } from '@mastra/vercel';
|
|
99
|
+
|
|
100
|
+
const sandbox = new VercelSandbox({ sandboxName: persistedSandboxId });
|
|
101
|
+
const worker = await attachWorkerDeployment({ sandbox, executionId: persistedExecutionId });
|
|
102
|
+
|
|
103
|
+
const status = await worker.status();
|
|
104
|
+
const stdout = await worker.readOutput('stdout', { offset: persistedStdoutOffset });
|
|
105
|
+
```
|
|
106
|
+
|
|
94
107
|
Pass `wake: true` to resume a stopped server sandbox before returning — useful in a route handler that fronts the sandbox. If the server isn't healthy after the resume (some providers restore the filesystem but not processes), the wake relaunches it:
|
|
95
108
|
|
|
96
109
|
```typescript
|
package/dist/index.cjs
CHANGED
|
@@ -380,6 +380,18 @@ var SandboxDeployer = class extends _mastra_deployer.Deployer {
|
|
|
380
380
|
}
|
|
381
381
|
};
|
|
382
382
|
//#endregion
|
|
383
|
+
//#region src/types.ts
|
|
384
|
+
/** Error thrown before deployment when requested resource limits cannot be enforced. */
|
|
385
|
+
var SandboxWorkerCapabilityError = class extends Error {
|
|
386
|
+
capability;
|
|
387
|
+
code = "SANDBOX_WORKER_CAPABILITY_UNAVAILABLE";
|
|
388
|
+
constructor(capability, message, options) {
|
|
389
|
+
super(message ?? `Worker resource-limit capability "${capability}" is unavailable.`, options);
|
|
390
|
+
this.capability = capability;
|
|
391
|
+
this.name = "SandboxWorkerCapabilityError";
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
//#endregion
|
|
383
395
|
//#region src/worker.ts
|
|
384
396
|
const ARCHIVE = ".mastra-worker.tar.gz";
|
|
385
397
|
const RUNTIME_DIR = ".mastra/executions";
|
|
@@ -389,13 +401,17 @@ const ARTIFACT_LOCK = ".mastra-artifact-lock";
|
|
|
389
401
|
const EXECUTION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
390
402
|
const DEFAULT_INPUT_LIMIT = 16 * 1024 * 1024;
|
|
391
403
|
const DEFAULT_OUTPUT_READ_LIMIT = 1024 * 1024;
|
|
404
|
+
const RESOURCE_CAPABILITY_PREFIX = "MASTRA_WORKER_CAPABILITY:";
|
|
392
405
|
async function deployWorkerToSandbox(options) {
|
|
393
406
|
validateOptions(options);
|
|
394
|
-
const { sandbox, dir, executionId, command, mode = "worker", args = [], env = {}, workingDirectory = ".", installCommand = "npm install --omit=dev", startupTimeoutMs = 1e4, executionTimeoutMs, terminationGraceMs = 5e3, inputLimitBytes = DEFAULT_INPUT_LIMIT } = options;
|
|
407
|
+
const { sandbox, dir, executionId, command, mode = "worker", args = [], env = {}, workingDirectory = ".", installCommand = "npm install --omit=dev", startupTimeoutMs = 1e4, executionTimeoutMs, resourceLimits: requestedResourceLimits, terminationGraceMs = 5e3, inputLimitBytes = DEFAULT_INPUT_LIMIT } = options;
|
|
408
|
+
const resourceLimits = normalizeResourceLimits(requestedResourceLimits);
|
|
409
|
+
if (resourceLimits) await preflightResourceLimits(sandbox, resourceLimits);
|
|
395
410
|
const remoteDir = await require_shared.resolveRemoteDir(sandbox, options.remoteDir);
|
|
396
411
|
const config = {
|
|
397
412
|
sandbox,
|
|
398
413
|
remoteDir,
|
|
414
|
+
resolveRemoteDir: async () => remoteDir,
|
|
399
415
|
command,
|
|
400
416
|
args,
|
|
401
417
|
env,
|
|
@@ -403,6 +419,7 @@ async function deployWorkerToSandbox(options) {
|
|
|
403
419
|
mode,
|
|
404
420
|
startupTimeoutMs,
|
|
405
421
|
executionTimeoutMs,
|
|
422
|
+
resourceLimits,
|
|
406
423
|
terminationGraceMs,
|
|
407
424
|
inputLimitBytes
|
|
408
425
|
};
|
|
@@ -432,6 +449,20 @@ async function deployWorkerToSandbox(options) {
|
|
|
432
449
|
}
|
|
433
450
|
return createExecution(config, executionId, options.input);
|
|
434
451
|
}
|
|
452
|
+
/** Reattach to a persisted worker execution without its original launch configuration. */
|
|
453
|
+
async function attachWorkerDeployment(options) {
|
|
454
|
+
if (!options.sandbox.executeCommand) throw new Error(`Sandbox provider "${options.sandbox.provider}" does not support executeCommand, which is required for worker deploys.`);
|
|
455
|
+
validateExecutionId(options.executionId);
|
|
456
|
+
if (options.terminationGraceMs !== void 0 && (!Number.isFinite(options.terminationGraceMs) || options.terminationGraceMs <= 0)) throw new Error("terminationGraceMs must be greater than zero.");
|
|
457
|
+
let remoteDir;
|
|
458
|
+
const config = {
|
|
459
|
+
sandbox: options.sandbox,
|
|
460
|
+
resolveRemoteDir: async () => remoteDir ??= await require_shared.resolveRemoteDir(options.sandbox, options.remoteDir),
|
|
461
|
+
terminationGraceMs: options.terminationGraceMs ?? 5e3
|
|
462
|
+
};
|
|
463
|
+
const info = await require_shared.getInfoSafe(options.sandbox);
|
|
464
|
+
return execution(config, options.executionId, info?.id ?? options.sandbox.id, info?.timeoutAt);
|
|
465
|
+
}
|
|
435
466
|
function validateOptions(options) {
|
|
436
467
|
if (!options.sandbox.executeCommand) throw new Error(`Sandbox provider "${options.sandbox.provider}" does not support executeCommand, which is required for worker deploys.`);
|
|
437
468
|
validateExecutionId(options.executionId);
|
|
@@ -446,6 +477,22 @@ function validateOptions(options) {
|
|
|
446
477
|
["executionTimeoutMs", options.executionTimeoutMs],
|
|
447
478
|
["terminationGraceMs", options.terminationGraceMs]
|
|
448
479
|
]) if (value !== void 0 && (!Number.isFinite(value) || value <= 0)) throw new Error(`${name} must be greater than zero.`);
|
|
480
|
+
const resourceLimits = options.resourceLimits;
|
|
481
|
+
if (resourceLimits) {
|
|
482
|
+
const knownLimits = /* @__PURE__ */ new Set([
|
|
483
|
+
"cpuTimeSeconds",
|
|
484
|
+
"addressSpaceBytes",
|
|
485
|
+
"fileSizeBytes",
|
|
486
|
+
"openFiles"
|
|
487
|
+
]);
|
|
488
|
+
for (const name of Object.keys(resourceLimits)) if (!knownLimits.has(name)) throw new Error(`Unknown worker resource limit: ${name}.`);
|
|
489
|
+
for (const [name, value] of [
|
|
490
|
+
["cpuTimeSeconds", resourceLimits.cpuTimeSeconds],
|
|
491
|
+
["addressSpaceBytes", resourceLimits.addressSpaceBytes],
|
|
492
|
+
["fileSizeBytes", resourceLimits.fileSizeBytes],
|
|
493
|
+
["openFiles", resourceLimits.openFiles]
|
|
494
|
+
]) if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) throw new Error(`Worker resourceLimits.${name} must be a positive safe integer.`);
|
|
495
|
+
}
|
|
449
496
|
}
|
|
450
497
|
function validateRelativePath(value, label) {
|
|
451
498
|
if (!value || path.posix.isAbsolute(value) || path.posix.normalize(value).startsWith("..")) throw new Error(`Worker ${label} must stay within the deployed artifact root.`);
|
|
@@ -453,6 +500,60 @@ function validateRelativePath(value, label) {
|
|
|
453
500
|
function validateInput(input) {
|
|
454
501
|
if (input?.type === "file") validateRelativePath(input.path, "input file path");
|
|
455
502
|
}
|
|
503
|
+
function normalizeResourceLimits(limits) {
|
|
504
|
+
if (!limits || Object.values(limits).every((value) => value === void 0)) return void 0;
|
|
505
|
+
return {
|
|
506
|
+
cpuTimeSeconds: limits.cpuTimeSeconds,
|
|
507
|
+
addressSpaceBytes: limits.addressSpaceBytes,
|
|
508
|
+
addressSpaceKilobytes: limits.addressSpaceBytes === void 0 ? void 0 : Math.floor(limits.addressSpaceBytes / 1024),
|
|
509
|
+
fileSizeBytes: limits.fileSizeBytes,
|
|
510
|
+
fileSizeBlocks: limits.fileSizeBytes === void 0 ? void 0 : Math.floor(limits.fileSizeBytes / 512),
|
|
511
|
+
openFiles: limits.openFiles
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
async function preflightResourceLimits(sandbox, resourceLimits) {
|
|
515
|
+
const checks = [];
|
|
516
|
+
if (resourceLimits.cpuTimeSeconds !== void 0) checks.push(`check_limit cpu_time -t ${resourceLimits.cpuTimeSeconds}`);
|
|
517
|
+
if (resourceLimits.addressSpaceKilobytes !== void 0) checks.push(`check_limit address_space -v ${resourceLimits.addressSpaceKilobytes}`);
|
|
518
|
+
if (resourceLimits.fileSizeBlocks !== void 0) checks.push(`check_limit file_size -f ${resourceLimits.fileSizeBlocks}`);
|
|
519
|
+
if (resourceLimits.openFiles !== void 0) checks.push(`check_limit open_files -n ${resourceLimits.openFiles}`);
|
|
520
|
+
if (resourceLimits.cpuTimeSeconds !== void 0) checks.push(`kill -l XCPU >/dev/null 2>&1 || fail cpu_signal`);
|
|
521
|
+
if (resourceLimits.fileSizeBytes !== void 0) checks.push(`kill -l XFSZ >/dev/null 2>&1 || fail file_size_signal`);
|
|
522
|
+
const script = `
|
|
523
|
+
fail() {
|
|
524
|
+
printf '${RESOURCE_CAPABILITY_PREFIX}%s\\n' "$1" >&2
|
|
525
|
+
exit 1
|
|
526
|
+
}
|
|
527
|
+
check_limit() {
|
|
528
|
+
capability="$1"
|
|
529
|
+
flag="$2"
|
|
530
|
+
value="$3"
|
|
531
|
+
(
|
|
532
|
+
ulimit -S "$flag" "$value" >/dev/null 2>&1 || exit 1
|
|
533
|
+
ulimit -H "$flag" "$value" >/dev/null 2>&1 || exit 1
|
|
534
|
+
[ "$(ulimit -S "$flag")" = "$value" ] || exit 1
|
|
535
|
+
[ "$(ulimit -H "$flag")" = "$value" ] || exit 1
|
|
536
|
+
if ulimit -H "$flag" "$((value + 1))" >/dev/null 2>&1; then exit 1; fi
|
|
537
|
+
) || fail "$capability"
|
|
538
|
+
}
|
|
539
|
+
[ "$(uname -s 2>/dev/null)" = Linux ] && [ -r /proc/self/stat ] || fail linux_proc
|
|
540
|
+
command -v setsid >/dev/null 2>&1 || fail process_groups
|
|
541
|
+
setsid sh -c 'kill -0 -$$ 2>/dev/null' || fail process_groups
|
|
542
|
+
${checks.join("\n")}
|
|
543
|
+
`;
|
|
544
|
+
let result;
|
|
545
|
+
try {
|
|
546
|
+
result = await require_shared.runInSandbox(sandbox, script, {
|
|
547
|
+
allowFailure: true,
|
|
548
|
+
label: "preflight worker resource limits"
|
|
549
|
+
});
|
|
550
|
+
} catch (error) {
|
|
551
|
+
throw new SandboxWorkerCapabilityError("sandbox_command", void 0, { cause: error });
|
|
552
|
+
}
|
|
553
|
+
if (result.exitCode === 0) return;
|
|
554
|
+
const detail = `${result.stderr}\n${result.stdout}`;
|
|
555
|
+
throw new SandboxWorkerCapabilityError(detail.match(new RegExp(`${RESOURCE_CAPABILITY_PREFIX}([a-z_]+)`))?.[1] ?? "sandbox_command", void 0, { cause: new Error(detail.trim() || "Resource-limit preflight command failed.") });
|
|
556
|
+
}
|
|
456
557
|
async function acquireLock(sandbox, lock, timeout, label) {
|
|
457
558
|
const timeoutMs = timeout ?? 6e5;
|
|
458
559
|
const attempts = Math.max(1, Math.ceil(timeoutMs / 1e3));
|
|
@@ -504,25 +605,32 @@ async function createExecution(config, executionId, input) {
|
|
|
504
605
|
await writeFailedStatus(config.sandbox, paths, executionId, "launch", error);
|
|
505
606
|
throw workerPhaseError("launch", error);
|
|
506
607
|
}
|
|
507
|
-
const
|
|
508
|
-
|
|
608
|
+
const resolvePaths = async () => paths;
|
|
609
|
+
const startup = await waitForStartup(config, executionId, resolvePaths);
|
|
610
|
+
if (startup.state === "timed_out") await cancelExecution(config, executionId, resolvePaths, "startup");
|
|
509
611
|
if (startup.state === "failed" || startup.state === "timed_out" || startup.state === "provider_unavailable") throw new Error(`Worker ${startup.state} during startup${"message" in startup && startup.message ? `: ${startup.message}` : ""}.`);
|
|
510
612
|
const info = await require_shared.getInfoSafe(config.sandbox);
|
|
511
|
-
return deployment(config, executionId,
|
|
613
|
+
return deployment(config, executionId, info?.id ?? config.sandbox.id ?? "unknown", info?.timeoutAt);
|
|
512
614
|
}
|
|
513
|
-
function
|
|
615
|
+
function execution(config, executionId, sandboxId, expiresAt) {
|
|
616
|
+
const resolvePaths = async () => executionPaths(await config.resolveRemoteDir(), executionId);
|
|
514
617
|
return {
|
|
515
618
|
sandboxId,
|
|
516
619
|
executionId,
|
|
517
620
|
expiresAt,
|
|
518
|
-
status: (options) => readWorkerStatus(config.sandbox, executionId,
|
|
519
|
-
readOutput: (stream, options) => readOutput(config.sandbox, executionId,
|
|
520
|
-
cancel: () => cancelExecution(config, executionId,
|
|
621
|
+
status: (options) => readWorkerStatus(config.sandbox, executionId, resolvePaths, options),
|
|
622
|
+
readOutput: (stream, options) => readOutput(config.sandbox, executionId, resolvePaths, stream, options),
|
|
623
|
+
cancel: () => cancelExecution(config, executionId, resolvePaths),
|
|
521
624
|
stop: async () => {
|
|
522
625
|
if (!config.sandbox.stop) throw new Error(`Sandbox provider "${config.sandbox.provider}" does not support stop.`);
|
|
523
626
|
await config.sandbox.stop();
|
|
524
627
|
},
|
|
525
|
-
destroy: (options) => destroyWithRetry(config.sandbox, options)
|
|
628
|
+
destroy: (options) => destroyWithRetry(config.sandbox, options)
|
|
629
|
+
};
|
|
630
|
+
}
|
|
631
|
+
function deployment(config, executionId, sandboxId, expiresAt) {
|
|
632
|
+
return {
|
|
633
|
+
...execution(config, executionId, sandboxId, expiresAt),
|
|
526
634
|
relaunch: async (options) => {
|
|
527
635
|
if (options.executionId === executionId) throw new Error("Relaunch requires a new executionId.");
|
|
528
636
|
validateInput(options.input);
|
|
@@ -544,8 +652,31 @@ function buildExecutionScript(config, paths, stdinPath) {
|
|
|
544
652
|
const envPrefix = Object.entries(config.env).map(([key, value]) => `${key}=${require_shared.shellQuote(value)}`).join(" ");
|
|
545
653
|
const executable = [require_shared.shellQuote(config.command), ...config.args.map(require_shared.shellQuote)].join(" ");
|
|
546
654
|
const target = `${envPrefix ? `env ${envPrefix} ` : ""}${executable}`;
|
|
655
|
+
const limitCommands = [];
|
|
656
|
+
if (config.resourceLimits?.cpuTimeSeconds !== void 0) {
|
|
657
|
+
limitCommands.push(`ulimit -S -t ${config.resourceLimits.cpuTimeSeconds}`);
|
|
658
|
+
limitCommands.push(`ulimit -H -t ${config.resourceLimits.cpuTimeSeconds}`);
|
|
659
|
+
}
|
|
660
|
+
if (config.resourceLimits?.addressSpaceKilobytes !== void 0) {
|
|
661
|
+
limitCommands.push(`ulimit -S -v ${config.resourceLimits.addressSpaceKilobytes}`);
|
|
662
|
+
limitCommands.push(`ulimit -H -v ${config.resourceLimits.addressSpaceKilobytes}`);
|
|
663
|
+
}
|
|
664
|
+
if (config.resourceLimits?.fileSizeBlocks !== void 0) {
|
|
665
|
+
limitCommands.push(`ulimit -S -f ${config.resourceLimits.fileSizeBlocks}`);
|
|
666
|
+
limitCommands.push(`ulimit -H -f ${config.resourceLimits.fileSizeBlocks}`);
|
|
667
|
+
}
|
|
668
|
+
if (config.resourceLimits?.openFiles !== void 0) {
|
|
669
|
+
limitCommands.push(`ulimit -S -n ${config.resourceLimits.openFiles}`);
|
|
670
|
+
limitCommands.push(`ulimit -H -n ${config.resourceLimits.openFiles}`);
|
|
671
|
+
}
|
|
672
|
+
const workload = config.resourceLimits ? `${limitCommands.map((command) => `${command} || exit 125`).join("\n")}\nexec ${target}` : `exec ${target}`;
|
|
547
673
|
const graceAttempts = Math.max(1, Math.ceil(config.terminationGraceMs / 1e3));
|
|
548
674
|
const state = (value) => `tmp=${require_shared.shellQuote(`${paths.status}.tmp.$$`)}; printf '%s\\n' ${require_shared.shellQuote(value)} > "$tmp"; mv "$tmp" ${require_shared.shellQuote(paths.status)};`;
|
|
675
|
+
const normalExitStatus = `signal=''; if [ "$code" -gt 128 ]; then signal="SIG$((code - 128))"; fi; tmp=${require_shared.shellQuote(`${paths.status}.tmp.$$`)}; printf 'exited|%s|%s|%s\\n' "$execution_id" "$code" "$signal" > "$tmp"; mv "$tmp" ${require_shared.shellQuote(paths.status)};`;
|
|
676
|
+
const resourceExitBranches = [];
|
|
677
|
+
if (config.resourceLimits?.cpuTimeSeconds !== void 0) resourceExitBranches.push(`if xcpu="$(kill -l XCPU 2>/dev/null)" && [ -n "$xcpu" ] && [ "$code" -eq $((128 + xcpu)) ]; then ${state(`resource_exhausted|${paths.executionId}|cpu|${config.resourceLimits.cpuTimeSeconds}|SIGXCPU`)}`);
|
|
678
|
+
if (config.resourceLimits?.fileSizeBytes !== void 0) resourceExitBranches.push(`${resourceExitBranches.length ? "elif" : "if"} xfsz="$(kill -l XFSZ 2>/dev/null)" && [ -n "$xfsz" ] && [ "$code" -eq $((128 + xfsz)) ]; then ${state(`resource_exhausted|${paths.executionId}|file_size|${config.resourceLimits.fileSizeBytes}|SIGXFSZ`)}`);
|
|
679
|
+
const resourceExitStatus = resourceExitBranches.length ? `${resourceExitBranches.join(" ")} else ${normalExitStatus} fi` : normalExitStatus;
|
|
549
680
|
return [
|
|
550
681
|
"#!/bin/sh",
|
|
551
682
|
`cd ${require_shared.shellQuote(cwd)}`,
|
|
@@ -556,7 +687,7 @@ function buildExecutionScript(config, paths, stdinPath) {
|
|
|
556
687
|
`tokenfile=${require_shared.shellQuote(paths.pidToken)}`,
|
|
557
688
|
`: > "$stdout"; : > "$stderr"`,
|
|
558
689
|
state(`starting|${paths.executionId}`),
|
|
559
|
-
`setsid sh -c ${require_shared.shellQuote(
|
|
690
|
+
`setsid sh -c ${require_shared.shellQuote(`${workload}${stdinPath ? ` < ${require_shared.shellQuote(stdinPath)}` : ""}`)} > "$stdout" 2> "$stderr" &`,
|
|
560
691
|
"child=$!",
|
|
561
692
|
"printf %s \"$child\" > \"$pidfile\"",
|
|
562
693
|
`if [ -r "/proc/$child/stat" ]; then awk '{print $22}' "/proc/$child/stat" > "$tokenfile"; else : > "$tokenfile"; fi`,
|
|
@@ -568,7 +699,7 @@ function buildExecutionScript(config, paths, stdinPath) {
|
|
|
568
699
|
"code=$?",
|
|
569
700
|
...config.executionTimeoutMs ? ["kill \"$watchdog\" 2>/dev/null || true"] : [],
|
|
570
701
|
`current="$(cat ${require_shared.shellQuote(paths.status)} 2>/dev/null || true)"`,
|
|
571
|
-
`case "$current" in timed_out*) ;; *) if [ "$cancelled" -eq 1 ]; then ${state(`cancelled|${paths.executionId}|TERM`)} else
|
|
702
|
+
`case "$current" in timed_out*) ;; *) if [ "$cancelled" -eq 1 ]; then ${state(`cancelled|${paths.executionId}|TERM`)} else ${resourceExitStatus} fi ;; esac`,
|
|
572
703
|
"rm -f \"$pidfile\" \"$tokenfile\"",
|
|
573
704
|
"exit \"$code\""
|
|
574
705
|
].join("\n");
|
|
@@ -576,10 +707,10 @@ function buildExecutionScript(config, paths, stdinPath) {
|
|
|
576
707
|
async function launchExecution(sandbox, paths) {
|
|
577
708
|
await require_shared.runInSandbox(sandbox, `setsid nohup sh ${require_shared.shellQuote(paths.script)} >/dev/null 2>&1 & echo $!`, { label: "launch worker execution" });
|
|
578
709
|
}
|
|
579
|
-
async function waitForStartup(config, executionId,
|
|
710
|
+
async function waitForStartup(config, executionId, resolvePaths) {
|
|
580
711
|
const deadline = Date.now() + config.startupTimeoutMs;
|
|
581
712
|
while (Date.now() < deadline) {
|
|
582
|
-
const status = await readWorkerStatus(config.sandbox, executionId,
|
|
713
|
+
const status = await readWorkerStatus(config.sandbox, executionId, resolvePaths);
|
|
583
714
|
if (status.state !== "unknown" && status.state !== "starting") return status;
|
|
584
715
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
585
716
|
}
|
|
@@ -589,7 +720,7 @@ async function waitForStartup(config, executionId, paths) {
|
|
|
589
720
|
phase: "startup"
|
|
590
721
|
};
|
|
591
722
|
}
|
|
592
|
-
async function readWorkerStatus(sandbox, executionId,
|
|
723
|
+
async function readWorkerStatus(sandbox, executionId, resolvePaths, options) {
|
|
593
724
|
const providerState = sandbox.status;
|
|
594
725
|
if (providerState === "destroyed" || providerState === "destroying") return {
|
|
595
726
|
state: "provider_unavailable",
|
|
@@ -614,6 +745,7 @@ async function readWorkerStatus(sandbox, executionId, paths, options) {
|
|
|
614
745
|
}
|
|
615
746
|
}
|
|
616
747
|
try {
|
|
748
|
+
const paths = await resolvePaths();
|
|
617
749
|
const result = await require_shared.runInSandbox(sandbox, [
|
|
618
750
|
`status="$(cat ${require_shared.shellQuote(paths.status)} 2>/dev/null || true)"`,
|
|
619
751
|
`if [ -f ${require_shared.shellQuote(paths.pid)} ]; then`,
|
|
@@ -645,7 +777,7 @@ async function readWorkerStatus(sandbox, executionId, paths, options) {
|
|
|
645
777
|
}
|
|
646
778
|
}
|
|
647
779
|
function parseStatus(executionId, value) {
|
|
648
|
-
const [state, recordedId, first, second] = value.split("|");
|
|
780
|
+
const [state, recordedId, first, second, third] = value.split("|");
|
|
649
781
|
if (recordedId !== executionId || state === "stale") return {
|
|
650
782
|
state: "unknown",
|
|
651
783
|
executionId
|
|
@@ -670,6 +802,27 @@ function parseStatus(executionId, value) {
|
|
|
670
802
|
executionId
|
|
671
803
|
};
|
|
672
804
|
}
|
|
805
|
+
if (state === "resource_exhausted") {
|
|
806
|
+
const limit = Number(second);
|
|
807
|
+
if (first === "cpu" && Number.isSafeInteger(limit) && limit > 0 && third === "SIGXCPU") return {
|
|
808
|
+
state,
|
|
809
|
+
executionId,
|
|
810
|
+
resource: first,
|
|
811
|
+
limit,
|
|
812
|
+
signal: third
|
|
813
|
+
};
|
|
814
|
+
if (first === "file_size" && Number.isSafeInteger(limit) && limit > 0 && third === "SIGXFSZ") return {
|
|
815
|
+
state,
|
|
816
|
+
executionId,
|
|
817
|
+
resource: first,
|
|
818
|
+
limit,
|
|
819
|
+
signal: third
|
|
820
|
+
};
|
|
821
|
+
return {
|
|
822
|
+
state: "unknown",
|
|
823
|
+
executionId
|
|
824
|
+
};
|
|
825
|
+
}
|
|
673
826
|
if (state === "cancelled") return {
|
|
674
827
|
state,
|
|
675
828
|
executionId,
|
|
@@ -691,9 +844,10 @@ function parseStatus(executionId, value) {
|
|
|
691
844
|
executionId
|
|
692
845
|
};
|
|
693
846
|
}
|
|
694
|
-
async function cancelExecution(config, executionId,
|
|
695
|
-
const current = await readWorkerStatus(config.sandbox, executionId,
|
|
847
|
+
async function cancelExecution(config, executionId, resolvePaths, timeoutPhase) {
|
|
848
|
+
const current = await readWorkerStatus(config.sandbox, executionId, resolvePaths);
|
|
696
849
|
if (current.state !== "running" && current.state !== "starting") return current;
|
|
850
|
+
const paths = await resolvePaths();
|
|
697
851
|
const attempts = Math.max(1, Math.ceil(config.terminationGraceMs / 1e3));
|
|
698
852
|
const terminal = timeoutPhase ? `timed_out|${executionId}|startup` : `cancelled|${executionId}|TERM`;
|
|
699
853
|
await require_shared.runInSandbox(config.sandbox, [
|
|
@@ -714,11 +868,12 @@ async function cancelExecution(config, executionId, paths, timeoutPhase) {
|
|
|
714
868
|
});
|
|
715
869
|
return parseStatus(executionId, terminal);
|
|
716
870
|
}
|
|
717
|
-
async function readOutput(sandbox, executionId,
|
|
871
|
+
async function readOutput(sandbox, executionId, resolvePaths, stream, options) {
|
|
718
872
|
const offset = Math.max(0, Math.floor(options?.offset ?? 0));
|
|
719
873
|
const maxBytes = Math.max(1, Math.floor(options?.maxBytes ?? DEFAULT_OUTPUT_READ_LIMIT));
|
|
720
|
-
const path$2 = stream === "stdout" ? paths.stdout : paths.stderr;
|
|
721
874
|
try {
|
|
875
|
+
const paths = await resolvePaths();
|
|
876
|
+
const path$2 = stream === "stdout" ? paths.stdout : paths.stderr;
|
|
722
877
|
const result = await require_shared.runInSandbox(sandbox, `size=$(wc -c < ${require_shared.shellQuote(path$2)} 2>/dev/null || echo 0); printf '%s\\n' "$size"; tail -c +${offset + 1} ${require_shared.shellQuote(path$2)} 2>/dev/null | head -c ${maxBytes} | base64`, {
|
|
723
878
|
allowFailure: true,
|
|
724
879
|
label: `read worker ${stream}`
|
|
@@ -729,9 +884,10 @@ async function readOutput(sandbox, executionId, paths, stream, options) {
|
|
|
729
884
|
const encoded = newline === -1 ? "" : result.stdout.slice(newline + 1).replace(/\s/g, "");
|
|
730
885
|
const data = Buffer.from(encoded, "base64");
|
|
731
886
|
const nextOffset = offset + data.byteLength;
|
|
732
|
-
const status = await readWorkerStatus(sandbox, executionId,
|
|
887
|
+
const status = await readWorkerStatus(sandbox, executionId, resolvePaths);
|
|
733
888
|
const terminal = [
|
|
734
889
|
"exited",
|
|
890
|
+
"resource_exhausted",
|
|
735
891
|
"cancelled",
|
|
736
892
|
"timed_out",
|
|
737
893
|
"failed"
|
|
@@ -822,6 +978,8 @@ function sanitizeStatusValue(value) {
|
|
|
822
978
|
//#endregion
|
|
823
979
|
exports.MANIFEST_FILENAME = MANIFEST_FILENAME;
|
|
824
980
|
exports.SandboxDeployer = SandboxDeployer;
|
|
981
|
+
exports.SandboxWorkerCapabilityError = SandboxWorkerCapabilityError;
|
|
982
|
+
exports.attachWorkerDeployment = attachWorkerDeployment;
|
|
825
983
|
exports.buildLaunchScript = buildLaunchScript;
|
|
826
984
|
exports.deployToSandbox = deployToSandbox;
|
|
827
985
|
exports.deployWorkerToSandbox = deployWorkerToSandbox;
|