@multiplayer-app/sandbox 6.0.0-sandbox-docker-image.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -6
- package/dist/cli.js +1056 -152
- package/dist/cli.js.map +4 -4
- package/dist/config.d.ts +5 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/executor/detect.d.ts.map +1 -1
- package/dist/executor/docker-executor.d.ts +16 -2
- package/dist/executor/docker-executor.d.ts.map +1 -1
- package/dist/executor/docker-sandbox-executor.d.ts +13 -1
- package/dist/executor/docker-sandbox-executor.d.ts.map +1 -1
- package/dist/executor/env-encoding.d.ts +9 -0
- package/dist/executor/env-encoding.d.ts.map +1 -0
- package/dist/executor/fargate-executor.d.ts +13 -1
- package/dist/executor/fargate-executor.d.ts.map +1 -1
- package/dist/executor/firecracker-executor.d.ts +7 -1
- package/dist/executor/firecracker-executor.d.ts.map +1 -1
- package/dist/executor/index.d.ts +2 -1
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/lambda-microvm-executor.d.ts +9 -0
- package/dist/executor/lambda-microvm-executor.d.ts.map +1 -1
- package/dist/executor/mock-executor.d.ts +4 -1
- package/dist/executor/mock-executor.d.ts.map +1 -1
- package/dist/executor/qemu-executor.d.ts +7 -1
- package/dist/executor/qemu-executor.d.ts.map +1 -1
- package/dist/executor/remote-executor.d.ts +10 -1
- package/dist/executor/remote-executor.d.ts.map +1 -1
- package/dist/executor/types.d.ts +12 -1
- package/dist/executor/types.d.ts.map +1 -1
- package/dist/health-server.d.ts +10 -0
- package/dist/health-server.d.ts.map +1 -0
- package/dist/images.d.ts +2 -0
- package/dist/images.d.ts.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +991 -150
- package/dist/index.js.map +4 -4
- package/dist/prepare-images.d.ts +14 -0
- package/dist/prepare-images.d.ts.map +1 -0
- package/dist/services/serial-relay.d.ts.map +1 -1
- package/dist/services/vnc-relay.d.ts.map +1 -1
- package/dist/socket-client.d.ts.map +1 -1
- package/dist/vendor/builder-types/access.d.ts +1 -1
- package/dist/vendor/builder-types/deployment.d.ts +58 -0
- package/dist/vendor/builder-types/deployment.d.ts.map +1 -0
- package/dist/vendor/builder-types/events.d.ts +6 -1
- package/dist/vendor/builder-types/events.d.ts.map +1 -1
- package/dist/vendor/builder-types/index.d.ts +1 -0
- package/dist/vendor/builder-types/index.d.ts.map +1 -1
- package/dist/vendor/builder-types/release.d.ts +7 -3
- package/dist/vendor/builder-types/release.d.ts.map +1 -1
- package/dist/vendor/builder-types/vm.d.ts +28 -1
- package/dist/vendor/builder-types/vm.d.ts.map +1 -1
- package/package.json +6 -1
package/dist/cli.js
CHANGED
|
@@ -28,7 +28,7 @@ var import_config5 = require("dotenv/config");
|
|
|
28
28
|
var import_commander = require("commander");
|
|
29
29
|
|
|
30
30
|
// package.json
|
|
31
|
-
var version = "
|
|
31
|
+
var version = "7.0.0";
|
|
32
32
|
|
|
33
33
|
// src/config.ts
|
|
34
34
|
var import_os2 = __toESM(require("os"));
|
|
@@ -48,6 +48,12 @@ var VM_STACKS = ["generic", "node", "python", "static"];
|
|
|
48
48
|
var WorkerEvent = {
|
|
49
49
|
// worker → sandbox
|
|
50
50
|
HEARTBEAT: "worker:heartbeat",
|
|
51
|
+
// worker → sandbox, fire-and-forget like HEARTBEAT — sent once when a 'job' VM's
|
|
52
|
+
// entrypoint exits on its own (the worker has already torn the resource down by then).
|
|
53
|
+
VM_RUN_EXITED: "vm:run-exited",
|
|
54
|
+
// sandbox → worker, generated by the control plane and reused by this worker process
|
|
55
|
+
// on Socket.IO reconnects. This keeps duplicate configured WORKER_ID values isolated.
|
|
56
|
+
ASSIGNED_ID: "worker:assigned-id",
|
|
51
57
|
// sandbox → worker, once right after auth on every (re)connect — the VMs sandbox's
|
|
52
58
|
// records say this workerId owns, so the worker can try to reattach before its first
|
|
53
59
|
// heartbeat. Always sent, even with an empty array, as a handshake signal.
|
|
@@ -83,6 +89,7 @@ var VM_ACTION_EVENTS = {
|
|
|
83
89
|
|
|
84
90
|
// src/images.ts
|
|
85
91
|
var IMAGE_FILES = ["vmlinuz", "initramfs.img"];
|
|
92
|
+
var PUBLISHED_IMAGE_STACKS = VM_STACKS.filter((stack) => stack !== "generic");
|
|
86
93
|
function hostArch() {
|
|
87
94
|
switch (process.arch) {
|
|
88
95
|
case "arm64":
|
|
@@ -141,7 +148,7 @@ async function downloadFromHttp(opts, stack, file, dest) {
|
|
|
141
148
|
async function pullImages(opts, destDir) {
|
|
142
149
|
const isS3 = opts.location.startsWith("s3://");
|
|
143
150
|
const downloaded = [];
|
|
144
|
-
for (const stack of opts.stacks ??
|
|
151
|
+
for (const stack of opts.stacks ?? PUBLISHED_IMAGE_STACKS) {
|
|
145
152
|
const stackDir = import_path.default.join(destDir, stack);
|
|
146
153
|
import_fs.default.mkdirSync(stackDir, { recursive: true });
|
|
147
154
|
for (const file of IMAGE_FILES) {
|
|
@@ -200,18 +207,32 @@ function parseArnList(raw) {
|
|
|
200
207
|
const arns = raw.split(",").map((s) => s.trim()).filter(Boolean);
|
|
201
208
|
return arns.length ? arns : void 0;
|
|
202
209
|
}
|
|
210
|
+
function resolveSandboxEndpoint(rawUrl, explicitSocketPath) {
|
|
211
|
+
const url = new URL(rawUrl);
|
|
212
|
+
const basePath = url.pathname.replace(/\/+$/, "") || "/v0";
|
|
213
|
+
return {
|
|
214
|
+
// Socket.IO treats a URL pathname as a namespace, so connect to the origin
|
|
215
|
+
// and pass the derived HTTP transport path separately.
|
|
216
|
+
sandboxUrl: url.origin,
|
|
217
|
+
workerSocketPath: explicitSocketPath ?? `${basePath}/sapi/workers/ws`
|
|
218
|
+
};
|
|
219
|
+
}
|
|
203
220
|
function loadWorkerConfigFromEnv(env = process.env) {
|
|
204
221
|
const workerId = env.WORKER_ID?.trim() || import_os2.default.hostname();
|
|
205
|
-
const token = env.WORKER_TOKEN;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
222
|
+
const token = env.WORKER_TOKEN?.trim() || void 0;
|
|
223
|
+
const sandboxEndpoint = resolveSandboxEndpoint(
|
|
224
|
+
env.SANDBOX_URL ?? "https://api.sandbox.filingramp.com/v0",
|
|
225
|
+
env.WORKER_SOCKET_PATH
|
|
226
|
+
);
|
|
209
227
|
return {
|
|
210
228
|
workerId,
|
|
211
229
|
token,
|
|
212
|
-
sandboxUrl:
|
|
213
|
-
workerSocketPath:
|
|
230
|
+
sandboxUrl: sandboxEndpoint.sandboxUrl,
|
|
231
|
+
workerSocketPath: sandboxEndpoint.workerSocketPath,
|
|
214
232
|
maxVms: Number(env.MAX_VMS) || 10,
|
|
233
|
+
healthHost: env.SANDBOX_HEALTH_HOST ?? "0.0.0.0",
|
|
234
|
+
healthPort: Number(env.SANDBOX_HEALTH_PORT) || 3e3,
|
|
235
|
+
healthPathPrefix: (env.API_PREFIX ?? "/v0/api").replace(/\/+$/, ""),
|
|
215
236
|
region: env.WORKER_REGION,
|
|
216
237
|
remote: {
|
|
217
238
|
// No default: an unset AGENT_URL means the remote engine is not configured on this worker
|
|
@@ -269,8 +290,8 @@ function loadWorkerConfigFromEnv(env = process.env) {
|
|
|
269
290
|
bin: env.SBX_BIN ?? "sbx",
|
|
270
291
|
// Bridged from the same local images `docker` (imagePrefix-imageTag) already uses —
|
|
271
292
|
// `docker save` + `sbx template load`, see docker-sandbox-executor.ts.
|
|
272
|
-
imagePrefix: env.SBX_IMAGE_PREFIX ?? "multiplayer-builder-vm",
|
|
273
|
-
imageTag: env.SBX_IMAGE_TAG ?? "local",
|
|
293
|
+
imagePrefix: env.SBX_IMAGE_PREFIX ?? env.DOCKER_IMAGE_PREFIX ?? "multiplayer-builder-vm",
|
|
294
|
+
imageTag: env.SBX_IMAGE_TAG ?? env.DOCKER_IMAGE_TAG ?? "local",
|
|
274
295
|
stackTemplates: parseStackMap(env.SBX_STACK_TEMPLATES),
|
|
275
296
|
// `sbx create shell` doesn't run the image's own CMD, so the engine launches
|
|
276
297
|
// it explicitly — defaults match each Dockerfile's own WORKDIR /app + CMD.
|
|
@@ -316,7 +337,7 @@ var import_os3 = require("os");
|
|
|
316
337
|
// ../logger/src/config.ts
|
|
317
338
|
var NODE_ENV = process.env.NODE_ENV || "development";
|
|
318
339
|
var isProduction = NODE_ENV === "production";
|
|
319
|
-
var APP_NAME = process.env.npm_package_name?.split("/").pop() || "
|
|
340
|
+
var APP_NAME = process.env.APP_NAME || process.env.npm_package_name?.split("/").pop() || "unknown-app";
|
|
320
341
|
var LOG_LEVEL = process.env.LOG_LEVEL || (isProduction ? "info" : "debug");
|
|
321
342
|
|
|
322
343
|
// ../logger/src/logger.ts
|
|
@@ -358,6 +379,7 @@ var CONTENT_TYPES = {
|
|
|
358
379
|
".txt": "text/plain; charset=utf-8",
|
|
359
380
|
".md": "text/plain; charset=utf-8"
|
|
360
381
|
};
|
|
382
|
+
var MOCK_JOB_DURATION_MS = 250;
|
|
361
383
|
function resolveWorkspacePath(workspaceDir, relativePath) {
|
|
362
384
|
const normalized = import_path3.default.normalize(relativePath).replace(/^([/\\])+/, "");
|
|
363
385
|
const resolved = import_path3.default.resolve(workspaceDir, normalized);
|
|
@@ -374,12 +396,16 @@ function startStandInServer(spec, workspaceDir) {
|
|
|
374
396
|
try {
|
|
375
397
|
const filePath = resolveWorkspacePath(workspaceDir, relative);
|
|
376
398
|
const content = await import_promises2.default.readFile(filePath);
|
|
377
|
-
res.writeHead(200, {
|
|
399
|
+
res.writeHead(200, {
|
|
400
|
+
"content-type": CONTENT_TYPES[import_path3.default.extname(filePath).toLowerCase()] ?? "application/octet-stream"
|
|
401
|
+
});
|
|
378
402
|
res.end(content);
|
|
379
403
|
} catch {
|
|
380
404
|
res.writeHead(200, { "content-type": "text/plain" });
|
|
381
|
-
res.end(
|
|
382
|
-
`)
|
|
405
|
+
res.end(
|
|
406
|
+
`mock vm ${spec.vmId} (${spec.stack}) \u2014 session ${spec.sessionId} \u2014 ${req.method} ${req.url}
|
|
407
|
+
`
|
|
408
|
+
);
|
|
383
409
|
}
|
|
384
410
|
})();
|
|
385
411
|
});
|
|
@@ -393,6 +419,7 @@ function startStandInServer(spec, workspaceDir) {
|
|
|
393
419
|
var MockExecutor = class {
|
|
394
420
|
constructor() {
|
|
395
421
|
this.vms = /* @__PURE__ */ new Map();
|
|
422
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
396
423
|
}
|
|
397
424
|
async create(spec) {
|
|
398
425
|
const workspaceDir = await import_promises2.default.mkdtemp(import_path3.default.join(import_os5.default.tmpdir(), `mock-vm-${spec.vmId}-`));
|
|
@@ -402,12 +429,38 @@ var MockExecutor = class {
|
|
|
402
429
|
sessionId: spec.sessionId,
|
|
403
430
|
stack: spec.stack,
|
|
404
431
|
resources: spec.resources,
|
|
405
|
-
running: true
|
|
432
|
+
running: true,
|
|
433
|
+
// No real process to configure — just echoed back for introspection/tests.
|
|
434
|
+
...spec.env ? { metadata: { env: spec.env } } : {}
|
|
406
435
|
};
|
|
407
|
-
|
|
408
|
-
|
|
436
|
+
const runMode = spec.runMode ?? "service";
|
|
437
|
+
this.vms.set(spec.vmId, { handle, server, port, workspaceDir, runMode, startedAt: Date.now() });
|
|
438
|
+
src_default.info({ vmId: spec.vmId, port, workspaceDir, runMode }, "mock vm created");
|
|
409
439
|
return handle;
|
|
410
440
|
}
|
|
441
|
+
/** No real process to watch — just synthesize a successful run shortly after create(). */
|
|
442
|
+
onExit(vmId, handler) {
|
|
443
|
+
const startedAt = this.vms.get(vmId)?.startedAt ?? Date.now();
|
|
444
|
+
this.exitHandlers.set(vmId, handler);
|
|
445
|
+
setTimeout(() => {
|
|
446
|
+
void (async () => {
|
|
447
|
+
const registered = this.exitHandlers.get(vmId);
|
|
448
|
+
this.exitHandlers.delete(vmId);
|
|
449
|
+
if (!registered) return;
|
|
450
|
+
const result = {
|
|
451
|
+
exitCode: 0,
|
|
452
|
+
stdout: `mock job ${vmId} completed
|
|
453
|
+
`,
|
|
454
|
+
stderr: "",
|
|
455
|
+
truncated: false,
|
|
456
|
+
durationMs: Date.now() - startedAt,
|
|
457
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
458
|
+
};
|
|
459
|
+
await this.destroy(vmId);
|
|
460
|
+
registered(result);
|
|
461
|
+
})();
|
|
462
|
+
}, MOCK_JOB_DURATION_MS);
|
|
463
|
+
}
|
|
411
464
|
async pause(vmId) {
|
|
412
465
|
return this.setRunning(vmId, false);
|
|
413
466
|
}
|
|
@@ -452,7 +505,10 @@ var MockExecutor = class {
|
|
|
452
505
|
resolve({
|
|
453
506
|
statusCode: proxyRes.statusCode ?? 502,
|
|
454
507
|
headers: Object.fromEntries(
|
|
455
|
-
Object.entries(proxyRes.headers).map(([key, value]) => [
|
|
508
|
+
Object.entries(proxyRes.headers).map(([key, value]) => [
|
|
509
|
+
key,
|
|
510
|
+
Array.isArray(value) ? value.join(", ") : value ?? ""
|
|
511
|
+
])
|
|
456
512
|
),
|
|
457
513
|
body: Buffer.concat(chunks)
|
|
458
514
|
});
|
|
@@ -470,7 +526,9 @@ var MockExecutor = class {
|
|
|
470
526
|
for (const file of files) {
|
|
471
527
|
const buffer = Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8");
|
|
472
528
|
if (buffer.byteLength > MAX_FILE_BYTES) {
|
|
473
|
-
throw new Error(
|
|
529
|
+
throw new Error(
|
|
530
|
+
`File too large (${buffer.byteLength} bytes, max ${MAX_FILE_BYTES}): ${file.path}`
|
|
531
|
+
);
|
|
474
532
|
}
|
|
475
533
|
const filePath = resolveWorkspacePath(vm.workspaceDir, file.path);
|
|
476
534
|
await import_promises2.default.mkdir(import_path3.default.dirname(filePath), { recursive: true });
|
|
@@ -541,11 +599,22 @@ function withTimeout(promise, ms) {
|
|
|
541
599
|
]);
|
|
542
600
|
}
|
|
543
601
|
|
|
602
|
+
// src/executor/env-encoding.ts
|
|
603
|
+
function encodeEnvB64(env) {
|
|
604
|
+
if (!env || Object.keys(env).length === 0) return void 0;
|
|
605
|
+
const lines = Object.entries(env).map(([key, value]) => `${key}=${value}`).join("\n");
|
|
606
|
+
return Buffer.from(lines).toString("base64");
|
|
607
|
+
}
|
|
608
|
+
|
|
544
609
|
// src/executor/qemu-executor.ts
|
|
610
|
+
var MAX_OUTPUT_BYTES2 = 1024 * 1024;
|
|
611
|
+
var JOB_EXIT_SENTINEL = "__JOB_EXIT_CODE__:";
|
|
545
612
|
var QemuExecutor = class {
|
|
546
613
|
constructor(config) {
|
|
547
614
|
this.config = config;
|
|
548
615
|
this.vms = /* @__PURE__ */ new Map();
|
|
616
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
617
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
549
618
|
this.usedHttpPorts = /* @__PURE__ */ new Set();
|
|
550
619
|
this.usedDisplays = /* @__PURE__ */ new Set();
|
|
551
620
|
// VNC display indices
|
|
@@ -594,12 +663,15 @@ var QemuExecutor = class {
|
|
|
594
663
|
}
|
|
595
664
|
const resolved = this.config.imagesDir ? resolveStackImages(this.config.imagesDir, stack) ?? resolveStackImages(this.config.imagesDir, "generic") : void 0;
|
|
596
665
|
if (!resolved) {
|
|
597
|
-
throw new Error(
|
|
666
|
+
throw new Error(
|
|
667
|
+
`no base image for stack "${stack}" under ${this.config.imagesDir} \u2014 run "worker-cli images pull"`
|
|
668
|
+
);
|
|
598
669
|
}
|
|
599
670
|
return resolved;
|
|
600
671
|
}
|
|
601
672
|
async create(spec) {
|
|
602
673
|
const { kernelPath, initrdPath } = this.resolveImages(spec.stack);
|
|
674
|
+
const envB64 = encodeEnvB64(spec.env);
|
|
603
675
|
const hostPort = this.allocateHttpPort();
|
|
604
676
|
const vncDisplay = this.allocateVncDisplay();
|
|
605
677
|
const serialPort = this.allocateSerialPort();
|
|
@@ -629,7 +701,9 @@ var QemuExecutor = class {
|
|
|
629
701
|
[
|
|
630
702
|
`console=tty0 console=ttyAMA0 net.ifnames=0 init=/init VM_ID=${spec.vmId} STACK=${spec.stack}`,
|
|
631
703
|
spec.sessionId ? `SESSION_ID=${spec.sessionId}` : "",
|
|
632
|
-
spec.appUrl ? `APP_URL_B64=${Buffer.from(spec.appUrl).toString("base64")}` : ""
|
|
704
|
+
spec.appUrl ? `APP_URL_B64=${Buffer.from(spec.appUrl).toString("base64")}` : "",
|
|
705
|
+
spec.runMode === "job" ? "RUN_MODE=job" : "",
|
|
706
|
+
envB64 ? `ENV_B64=${envB64}` : ""
|
|
633
707
|
].filter(Boolean).join(" "),
|
|
634
708
|
// Networking
|
|
635
709
|
"-netdev",
|
|
@@ -654,15 +728,24 @@ var QemuExecutor = class {
|
|
|
654
728
|
src_default.info({ vmId: spec.vmId, bin: this.bin, args }, "spawning qemu");
|
|
655
729
|
const proc = (0, import_child_process2.spawn)(this.bin, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
656
730
|
proc.stdout?.on("data", (d) => src_default.info({ vmId: spec.vmId }, d.toString().trimEnd()));
|
|
657
|
-
proc.stderr?.on(
|
|
731
|
+
proc.stderr?.on(
|
|
732
|
+
"data",
|
|
733
|
+
(d) => src_default.error({ vmId: spec.vmId }, d.toString().trimEnd())
|
|
734
|
+
);
|
|
735
|
+
const runMode = spec.runMode ?? "service";
|
|
736
|
+
const startedAt = Date.now();
|
|
658
737
|
proc.on("exit", (code, signal) => {
|
|
659
738
|
const log = code === 0 || code === null ? src_default.info.bind(src_default) : src_default.error.bind(src_default);
|
|
660
739
|
log({ vmId: spec.vmId, code, signal }, "qemu process exited");
|
|
661
740
|
const vm2 = this.vms.get(spec.vmId);
|
|
662
|
-
if (vm2)
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
741
|
+
if (!vm2) return;
|
|
742
|
+
this.usedHttpPorts.delete(vm2.hostPort);
|
|
743
|
+
this.usedDisplays.delete(vm2.vncDisplay);
|
|
744
|
+
this.usedSerialPorts.delete(vm2.serialPort);
|
|
745
|
+
if (vm2.runMode === "job") {
|
|
746
|
+
vm2.handle = { ...vm2.handle, running: false };
|
|
747
|
+
this.reportJobResult(spec.vmId, vm2);
|
|
748
|
+
} else {
|
|
666
749
|
this.vms.delete(spec.vmId);
|
|
667
750
|
}
|
|
668
751
|
});
|
|
@@ -677,12 +760,61 @@ var QemuExecutor = class {
|
|
|
677
760
|
serialPort,
|
|
678
761
|
metadata
|
|
679
762
|
};
|
|
680
|
-
const vm = {
|
|
763
|
+
const vm = {
|
|
764
|
+
handle,
|
|
765
|
+
process: proc,
|
|
766
|
+
pid: proc.pid,
|
|
767
|
+
hostPort,
|
|
768
|
+
qmpSocketPath,
|
|
769
|
+
vncDisplay,
|
|
770
|
+
serialPort,
|
|
771
|
+
runMode,
|
|
772
|
+
startedAt
|
|
773
|
+
};
|
|
681
774
|
this.vms.set(spec.vmId, vm);
|
|
775
|
+
if (runMode === "job") vm.serialBuffer = connectSerialBuffer(serialPort);
|
|
682
776
|
await waitForQmp(qmpSocketPath);
|
|
683
|
-
src_default.info({ vmId: spec.vmId, hostPort, vncPort, serialPort }, "qemu vm ready");
|
|
777
|
+
src_default.info({ vmId: spec.vmId, hostPort, vncPort, serialPort, runMode }, "qemu vm ready");
|
|
684
778
|
return handle;
|
|
685
779
|
}
|
|
780
|
+
/** Scrapes JOB_EXIT_SENTINEL out of the buffered serial console and reports the result. */
|
|
781
|
+
reportJobResult(vmId, vm) {
|
|
782
|
+
vm.serialBuffer?.socket.destroy();
|
|
783
|
+
const output = Buffer.concat(vm.serialBuffer?.chunks ?? []).toString("utf8");
|
|
784
|
+
const sentinelIndex = output.lastIndexOf(JOB_EXIT_SENTINEL);
|
|
785
|
+
const exitCode = sentinelIndex === -1 ? 1 : Number.parseInt(output.slice(sentinelIndex + JOB_EXIT_SENTINEL.length), 10) || 0;
|
|
786
|
+
const stdout = sentinelIndex === -1 ? output : output.slice(0, sentinelIndex);
|
|
787
|
+
const result = {
|
|
788
|
+
exitCode,
|
|
789
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES2),
|
|
790
|
+
stderr: "",
|
|
791
|
+
truncated: (vm.serialBuffer?.bytes ?? 0) > MAX_OUTPUT_BYTES2,
|
|
792
|
+
durationMs: Date.now() - vm.startedAt,
|
|
793
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
794
|
+
};
|
|
795
|
+
src_default.info({ vmId, exitCode }, "qemu job vm exited");
|
|
796
|
+
this.deliverJobResult(vmId, result);
|
|
797
|
+
}
|
|
798
|
+
deliverJobResult(vmId, result) {
|
|
799
|
+
const handler = this.exitHandlers.get(vmId);
|
|
800
|
+
if (handler) {
|
|
801
|
+
this.exitHandlers.delete(vmId);
|
|
802
|
+
this.vms.delete(vmId);
|
|
803
|
+
handler(result);
|
|
804
|
+
} else {
|
|
805
|
+
this.pendingResults.set(vmId, result);
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
onExit(vmId, handler) {
|
|
809
|
+
const pending = this.pendingResults.get(vmId);
|
|
810
|
+
if (pending) {
|
|
811
|
+
this.pendingResults.delete(vmId);
|
|
812
|
+
this.vms.delete(vmId);
|
|
813
|
+
handler(pending);
|
|
814
|
+
return;
|
|
815
|
+
}
|
|
816
|
+
this.exitHandlers.set(vmId, handler);
|
|
817
|
+
}
|
|
686
818
|
async pause(vmId) {
|
|
687
819
|
await this.qmp(vmId, "stop");
|
|
688
820
|
return this.setRunning(vmId, false);
|
|
@@ -769,7 +901,9 @@ var QemuExecutor = class {
|
|
|
769
901
|
hostPort: meta.hostPort,
|
|
770
902
|
qmpSocketPath: meta.qmpSocketPath,
|
|
771
903
|
vncDisplay: meta.vncDisplay,
|
|
772
|
-
serialPort: meta.serialPort
|
|
904
|
+
serialPort: meta.serialPort,
|
|
905
|
+
runMode: record.runMode,
|
|
906
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
773
907
|
});
|
|
774
908
|
src_default.info({ vmId: record.vmId }, "qemu vm reattached");
|
|
775
909
|
return handle;
|
|
@@ -789,10 +923,16 @@ var QemuExecutor = class {
|
|
|
789
923
|
async proxyHttp(vmId, request) {
|
|
790
924
|
const vm = this.vms.get(vmId);
|
|
791
925
|
if (!vm?.handle.running) throw new Error(`vm ${vmId} is not running`);
|
|
792
|
-
const { default:
|
|
926
|
+
const { default: http8 } = await import("http");
|
|
793
927
|
return new Promise((resolve, reject) => {
|
|
794
|
-
const req =
|
|
795
|
-
{
|
|
928
|
+
const req = http8.request(
|
|
929
|
+
{
|
|
930
|
+
host: "127.0.0.1",
|
|
931
|
+
port: vm.hostPort,
|
|
932
|
+
method: request.method,
|
|
933
|
+
path: request.path,
|
|
934
|
+
headers: request.headers
|
|
935
|
+
},
|
|
796
936
|
(res) => {
|
|
797
937
|
const chunks = [];
|
|
798
938
|
res.on("data", (c) => chunks.push(c));
|
|
@@ -801,7 +941,10 @@ var QemuExecutor = class {
|
|
|
801
941
|
() => resolve({
|
|
802
942
|
statusCode: res.statusCode ?? 502,
|
|
803
943
|
headers: Object.fromEntries(
|
|
804
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
944
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
945
|
+
k,
|
|
946
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
947
|
+
])
|
|
805
948
|
),
|
|
806
949
|
body: Buffer.concat(chunks)
|
|
807
950
|
})
|
|
@@ -825,11 +968,27 @@ var QemuExecutor = class {
|
|
|
825
968
|
await sendQmpCommand(vm.qmpSocketPath, execute);
|
|
826
969
|
}
|
|
827
970
|
};
|
|
971
|
+
function connectSerialBuffer(port) {
|
|
972
|
+
const buffer = {
|
|
973
|
+
chunks: [],
|
|
974
|
+
bytes: 0,
|
|
975
|
+
socket: import_net.default.connect({ host: "127.0.0.1", port })
|
|
976
|
+
};
|
|
977
|
+
buffer.socket.on("data", (chunk) => {
|
|
978
|
+
if (buffer.bytes >= MAX_OUTPUT_BYTES2) return;
|
|
979
|
+
buffer.chunks.push(chunk);
|
|
980
|
+
buffer.bytes += chunk.byteLength;
|
|
981
|
+
});
|
|
982
|
+
buffer.socket.on("error", () => {
|
|
983
|
+
});
|
|
984
|
+
return buffer;
|
|
985
|
+
}
|
|
828
986
|
function waitForQmp(socketPath, timeoutMs = 3e4) {
|
|
829
987
|
return new Promise((resolve, reject) => {
|
|
830
988
|
const deadline = Date.now() + timeoutMs;
|
|
831
989
|
const attempt = () => {
|
|
832
|
-
if (Date.now() > deadline)
|
|
990
|
+
if (Date.now() > deadline)
|
|
991
|
+
return reject(new Error(`timed out waiting for QMP socket at ${socketPath}`));
|
|
833
992
|
const sock = import_net.default.createConnection(socketPath);
|
|
834
993
|
sock.once("connect", () => {
|
|
835
994
|
sock.destroy();
|
|
@@ -891,11 +1050,15 @@ var import_path5 = __toESM(require("path"));
|
|
|
891
1050
|
var import_promises4 = __toESM(require("fs/promises"));
|
|
892
1051
|
var import_os7 = __toESM(require("os"));
|
|
893
1052
|
var execAsync = (0, import_util.promisify)(import_child_process4.exec);
|
|
1053
|
+
var MAX_OUTPUT_BYTES3 = 1024 * 1024;
|
|
1054
|
+
var JOB_EXIT_SENTINEL2 = "__JOB_EXIT_CODE__:";
|
|
894
1055
|
var FirecrackerExecutor = class {
|
|
895
1056
|
constructor(config) {
|
|
896
1057
|
this.config = config;
|
|
897
1058
|
this.vms = /* @__PURE__ */ new Map();
|
|
898
1059
|
this.usedSubnets = /* @__PURE__ */ new Set();
|
|
1060
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
1061
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
899
1062
|
this.tapCounter = 0;
|
|
900
1063
|
this.networkReady = false;
|
|
901
1064
|
this.bin = config.bin ?? "firecracker";
|
|
@@ -918,7 +1081,9 @@ var FirecrackerExecutor = class {
|
|
|
918
1081
|
"iptables -t nat -C POSTROUTING -s 172.16.0.0/12 ! -d 172.16.0.0/12 -j MASQUERADE"
|
|
919
1082
|
).catch(() => null);
|
|
920
1083
|
if (!alreadySet) {
|
|
921
|
-
await execAsync(
|
|
1084
|
+
await execAsync(
|
|
1085
|
+
"iptables -t nat -A POSTROUTING -s 172.16.0.0/12 ! -d 172.16.0.0/12 -j MASQUERADE"
|
|
1086
|
+
);
|
|
922
1087
|
}
|
|
923
1088
|
this.networkReady = true;
|
|
924
1089
|
}
|
|
@@ -929,7 +1094,9 @@ var FirecrackerExecutor = class {
|
|
|
929
1094
|
}
|
|
930
1095
|
const resolved = this.config.imagesDir ? resolveStackImages(this.config.imagesDir, stack) ?? resolveStackImages(this.config.imagesDir, "generic") : void 0;
|
|
931
1096
|
if (!resolved) {
|
|
932
|
-
throw new Error(
|
|
1097
|
+
throw new Error(
|
|
1098
|
+
`no base image for stack "${stack}" under ${this.config.imagesDir} \u2014 run "worker-cli images pull"`
|
|
1099
|
+
);
|
|
933
1100
|
}
|
|
934
1101
|
return resolved;
|
|
935
1102
|
}
|
|
@@ -947,17 +1114,35 @@ var FirecrackerExecutor = class {
|
|
|
947
1114
|
await execAsync(`ip tuntap add dev ${tapName} mode tap`);
|
|
948
1115
|
await execAsync(`ip addr add ${hostIp}/30 dev ${tapName}`);
|
|
949
1116
|
await execAsync(`ip link set ${tapName} up`);
|
|
1117
|
+
const runMode = spec.runMode ?? "service";
|
|
1118
|
+
const envB64 = encodeEnvB64(spec.env);
|
|
1119
|
+
const startedAt = Date.now();
|
|
1120
|
+
const serialBuffer = runMode === "job" ? { chunks: [], bytes: 0 } : void 0;
|
|
950
1121
|
const proc = (0, import_child_process3.spawn)(this.bin, ["--api-sock", apiSocketPath], {
|
|
951
1122
|
stdio: ["pipe", "pipe", "pipe"]
|
|
952
1123
|
});
|
|
953
|
-
proc.stdout?.on("data", (d) =>
|
|
954
|
-
|
|
1124
|
+
proc.stdout?.on("data", (d) => {
|
|
1125
|
+
src_default.debug({ vmId: spec.vmId }, d.toString().trimEnd());
|
|
1126
|
+
if (serialBuffer && serialBuffer.bytes < MAX_OUTPUT_BYTES3) {
|
|
1127
|
+
serialBuffer.chunks.push(d);
|
|
1128
|
+
serialBuffer.bytes += d.byteLength;
|
|
1129
|
+
}
|
|
1130
|
+
});
|
|
1131
|
+
proc.stderr?.on(
|
|
1132
|
+
"data",
|
|
1133
|
+
(d) => src_default.debug({ vmId: spec.vmId }, d.toString().trimEnd())
|
|
1134
|
+
);
|
|
955
1135
|
proc.on("exit", (code, signal) => {
|
|
956
1136
|
src_default.info({ vmId: spec.vmId, code, signal }, "firecracker process exited");
|
|
957
1137
|
const vm = this.vms.get(spec.vmId);
|
|
958
1138
|
if (vm) {
|
|
959
1139
|
this.usedSubnets.delete(vm.subnet);
|
|
960
|
-
|
|
1140
|
+
if (vm.runMode === "job") {
|
|
1141
|
+
vm.handle = { ...vm.handle, running: false };
|
|
1142
|
+
this.reportJobResult(spec.vmId, vm);
|
|
1143
|
+
} else {
|
|
1144
|
+
this.vms.delete(spec.vmId);
|
|
1145
|
+
}
|
|
961
1146
|
}
|
|
962
1147
|
execAsync(`ip link del ${tapName}`).catch(() => {
|
|
963
1148
|
});
|
|
@@ -984,7 +1169,9 @@ var FirecrackerExecutor = class {
|
|
|
984
1169
|
// base64: kernel cmdline is space-separated, raw presigned URLs are not safe
|
|
985
1170
|
...spec.appUrl ? [`APP_URL_B64=${Buffer.from(spec.appUrl).toString("base64")}`] : [],
|
|
986
1171
|
`GUEST_IP=${guestIp}`,
|
|
987
|
-
`HOST_IP=${hostIp}
|
|
1172
|
+
`HOST_IP=${hostIp}`,
|
|
1173
|
+
...runMode === "job" ? ["RUN_MODE=job"] : [],
|
|
1174
|
+
...envB64 ? [`ENV_B64=${envB64}`] : []
|
|
988
1175
|
].join(" ")
|
|
989
1176
|
});
|
|
990
1177
|
await fcRequest(apiSocketPath, "PUT", "/network-interfaces/eth0", {
|
|
@@ -994,7 +1181,7 @@ var FirecrackerExecutor = class {
|
|
|
994
1181
|
host_dev_name: tapName
|
|
995
1182
|
});
|
|
996
1183
|
await fcRequest(apiSocketPath, "PUT", "/actions", { action_type: "InstanceStart" });
|
|
997
|
-
await waitForHttp(guestIp, 8080);
|
|
1184
|
+
if (runMode !== "job") await waitForHttp(guestIp, 8080);
|
|
998
1185
|
const metadata = { apiSocketPath, tapName, subnet, guestIp, pid: proc.pid };
|
|
999
1186
|
const handle = {
|
|
1000
1187
|
vmId: spec.vmId,
|
|
@@ -1004,10 +1191,58 @@ var FirecrackerExecutor = class {
|
|
|
1004
1191
|
running: true,
|
|
1005
1192
|
metadata
|
|
1006
1193
|
};
|
|
1007
|
-
this.vms.set(spec.vmId, {
|
|
1008
|
-
|
|
1194
|
+
this.vms.set(spec.vmId, {
|
|
1195
|
+
handle,
|
|
1196
|
+
process: proc,
|
|
1197
|
+
pid: proc.pid,
|
|
1198
|
+
apiSocketPath,
|
|
1199
|
+
tapName,
|
|
1200
|
+
subnet,
|
|
1201
|
+
guestIp,
|
|
1202
|
+
runMode,
|
|
1203
|
+
startedAt,
|
|
1204
|
+
serialBuffer
|
|
1205
|
+
});
|
|
1206
|
+
src_default.info({ vmId: spec.vmId, guestIp, tapName, runMode }, "firecracker vm ready");
|
|
1009
1207
|
return handle;
|
|
1010
1208
|
}
|
|
1209
|
+
/** Scrapes JOB_EXIT_SENTINEL out of the buffered serial stdout and reports the result. */
|
|
1210
|
+
reportJobResult(vmId, vm) {
|
|
1211
|
+
const output = Buffer.concat(vm.serialBuffer?.chunks ?? []).toString("utf8");
|
|
1212
|
+
const sentinelIndex = output.lastIndexOf(JOB_EXIT_SENTINEL2);
|
|
1213
|
+
const exitCode = sentinelIndex === -1 ? 1 : Number.parseInt(output.slice(sentinelIndex + JOB_EXIT_SENTINEL2.length), 10) || 0;
|
|
1214
|
+
const stdout = sentinelIndex === -1 ? output : output.slice(0, sentinelIndex);
|
|
1215
|
+
const result = {
|
|
1216
|
+
exitCode,
|
|
1217
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES3),
|
|
1218
|
+
stderr: "",
|
|
1219
|
+
truncated: (vm.serialBuffer?.bytes ?? 0) > MAX_OUTPUT_BYTES3,
|
|
1220
|
+
durationMs: Date.now() - vm.startedAt,
|
|
1221
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1222
|
+
};
|
|
1223
|
+
src_default.info({ vmId, exitCode }, "firecracker job vm exited");
|
|
1224
|
+
this.deliverJobResult(vmId, result);
|
|
1225
|
+
}
|
|
1226
|
+
deliverJobResult(vmId, result) {
|
|
1227
|
+
const handler = this.exitHandlers.get(vmId);
|
|
1228
|
+
if (handler) {
|
|
1229
|
+
this.exitHandlers.delete(vmId);
|
|
1230
|
+
this.vms.delete(vmId);
|
|
1231
|
+
handler(result);
|
|
1232
|
+
} else {
|
|
1233
|
+
this.pendingResults.set(vmId, result);
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
onExit(vmId, handler) {
|
|
1237
|
+
const pending = this.pendingResults.get(vmId);
|
|
1238
|
+
if (pending) {
|
|
1239
|
+
this.pendingResults.delete(vmId);
|
|
1240
|
+
this.vms.delete(vmId);
|
|
1241
|
+
handler(pending);
|
|
1242
|
+
return;
|
|
1243
|
+
}
|
|
1244
|
+
this.exitHandlers.set(vmId, handler);
|
|
1245
|
+
}
|
|
1011
1246
|
async pause(vmId) {
|
|
1012
1247
|
await this.patchVmState(vmId, "Paused");
|
|
1013
1248
|
return this.setRunning(vmId, false);
|
|
@@ -1095,9 +1330,14 @@ var FirecrackerExecutor = class {
|
|
|
1095
1330
|
apiSocketPath: meta.apiSocketPath,
|
|
1096
1331
|
tapName: meta.tapName,
|
|
1097
1332
|
subnet: meta.subnet,
|
|
1098
|
-
guestIp: meta.guestIp
|
|
1333
|
+
guestIp: meta.guestIp,
|
|
1334
|
+
runMode: record.runMode,
|
|
1335
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
1099
1336
|
});
|
|
1100
|
-
src_default.info(
|
|
1337
|
+
src_default.info(
|
|
1338
|
+
{ vmId: record.vmId, guestIp: meta.guestIp, tapName: meta.tapName },
|
|
1339
|
+
"firecracker vm reattached"
|
|
1340
|
+
);
|
|
1101
1341
|
return handle;
|
|
1102
1342
|
}
|
|
1103
1343
|
async proxyHttp(vmId, request) {
|
|
@@ -1120,7 +1360,10 @@ var FirecrackerExecutor = class {
|
|
|
1120
1360
|
() => resolve({
|
|
1121
1361
|
statusCode: res.statusCode ?? 502,
|
|
1122
1362
|
headers: Object.fromEntries(
|
|
1123
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
1363
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
1364
|
+
k,
|
|
1365
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
1366
|
+
])
|
|
1124
1367
|
),
|
|
1125
1368
|
body: Buffer.concat(chunks)
|
|
1126
1369
|
})
|
|
@@ -1184,7 +1427,8 @@ function waitForSocket(socketPath, timeoutMs = 1e4) {
|
|
|
1184
1427
|
return new Promise((resolve, reject) => {
|
|
1185
1428
|
const deadline = Date.now() + timeoutMs;
|
|
1186
1429
|
const attempt = () => {
|
|
1187
|
-
if (Date.now() > deadline)
|
|
1430
|
+
if (Date.now() > deadline)
|
|
1431
|
+
return reject(new Error(`timed out waiting for Firecracker socket ${socketPath}`));
|
|
1188
1432
|
const sock = import_net2.default.createConnection(socketPath);
|
|
1189
1433
|
sock.once("connect", () => {
|
|
1190
1434
|
sock.destroy();
|
|
@@ -1199,7 +1443,8 @@ function waitForHttp(host, port, timeoutMs = 6e4) {
|
|
|
1199
1443
|
return new Promise((resolve, reject) => {
|
|
1200
1444
|
const deadline = Date.now() + timeoutMs;
|
|
1201
1445
|
const attempt = () => {
|
|
1202
|
-
if (Date.now() > deadline)
|
|
1446
|
+
if (Date.now() > deadline)
|
|
1447
|
+
return reject(new Error(`timed out waiting for HTTP at ${host}:${port}`));
|
|
1203
1448
|
const req = import_http2.default.get(`http://${host}:${port}/health`, (res) => {
|
|
1204
1449
|
if (res.statusCode && res.statusCode < 500) return resolve();
|
|
1205
1450
|
res.resume();
|
|
@@ -1221,7 +1466,7 @@ var import_util2 = require("util");
|
|
|
1221
1466
|
var import_http3 = __toESM(require("http"));
|
|
1222
1467
|
var DEFAULT_EXEC_TIMEOUT_MS2 = 6e4;
|
|
1223
1468
|
var MAX_EXEC_TIMEOUT_MS2 = 3e5;
|
|
1224
|
-
var
|
|
1469
|
+
var MAX_OUTPUT_BYTES4 = 1024 * 1024;
|
|
1225
1470
|
var MAX_FILE_BYTES2 = 5 * 1024 * 1024;
|
|
1226
1471
|
var WORKSPACE_DIR = "/workspace";
|
|
1227
1472
|
var execFileAsync = (0, import_util2.promisify)(import_child_process5.execFile);
|
|
@@ -1237,6 +1482,9 @@ var DockerExecutor = class {
|
|
|
1237
1482
|
this.config = config;
|
|
1238
1483
|
this.vms = /* @__PURE__ */ new Map();
|
|
1239
1484
|
this.usedPorts = /* @__PURE__ */ new Set();
|
|
1485
|
+
/** Job VMs that exited before onExit() was called to claim the result (create() → onExit() isn't atomic). */
|
|
1486
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
1487
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
1240
1488
|
this.bin = config.bin;
|
|
1241
1489
|
}
|
|
1242
1490
|
allocatePort() {
|
|
@@ -1256,7 +1504,9 @@ var DockerExecutor = class {
|
|
|
1256
1504
|
if (await this.imageExists(stackImage)) return stackImage;
|
|
1257
1505
|
const genericImage = `${imagePrefix}-generic:${imageTag}`;
|
|
1258
1506
|
if (await this.imageExists(genericImage)) return genericImage;
|
|
1259
|
-
throw new Error(
|
|
1507
|
+
throw new Error(
|
|
1508
|
+
`no local docker image for stack "${stack}" (${stackImage}) \u2014 run scripts/setup-docker-images.sh`
|
|
1509
|
+
);
|
|
1260
1510
|
}
|
|
1261
1511
|
async imageExists(image) {
|
|
1262
1512
|
try {
|
|
@@ -1287,6 +1537,7 @@ var DockerExecutor = class {
|
|
|
1287
1537
|
`STACK=${spec.stack}`,
|
|
1288
1538
|
...spec.sessionId ? ["-e", `SESSION_ID=${spec.sessionId}`] : [],
|
|
1289
1539
|
...spec.appUrl ? ["-e", `APP_URL=${spec.appUrl}`] : [],
|
|
1540
|
+
...Object.entries(spec.env ?? {}).flatMap(([key, value]) => ["-e", `${key}=${value}`]),
|
|
1290
1541
|
...this.config.network ? ["--network", this.config.network] : [],
|
|
1291
1542
|
image
|
|
1292
1543
|
];
|
|
@@ -1307,22 +1558,91 @@ var DockerExecutor = class {
|
|
|
1307
1558
|
running: true,
|
|
1308
1559
|
metadata: { containerId, containerName, hostPort }
|
|
1309
1560
|
};
|
|
1310
|
-
|
|
1561
|
+
const runMode = spec.runMode ?? "service";
|
|
1562
|
+
this.vms.set(spec.vmId, {
|
|
1563
|
+
handle,
|
|
1564
|
+
containerId,
|
|
1565
|
+
containerName,
|
|
1566
|
+
hostPort,
|
|
1567
|
+
runMode,
|
|
1568
|
+
startedAt: Date.now()
|
|
1569
|
+
});
|
|
1311
1570
|
this.watchExit(spec.vmId, containerId);
|
|
1312
|
-
await this.waitForAgent(hostPort);
|
|
1313
|
-
src_default.info({ vmId: spec.vmId, containerId, hostPort }, "docker vm ready");
|
|
1571
|
+
if (runMode !== "job") await this.waitForAgent(hostPort);
|
|
1572
|
+
src_default.info({ vmId: spec.vmId, containerId, hostPort, runMode }, "docker vm ready");
|
|
1314
1573
|
return handle;
|
|
1315
1574
|
}
|
|
1316
|
-
/**
|
|
1575
|
+
/**
|
|
1576
|
+
* For 'service' VMs: frees the allocated port once the container exits on its own (crash,
|
|
1577
|
+
* OOM kill, `docker stop` from outside us) — same as before.
|
|
1578
|
+
* For 'job' VMs: captures the exit code + logs, tears the container down, and reports the
|
|
1579
|
+
* result to whoever calls onExit() (immediately if they already have, otherwise once they do).
|
|
1580
|
+
*/
|
|
1317
1581
|
watchExit(vmId, containerId) {
|
|
1318
|
-
execFileAsync(this.bin, ["wait", containerId]).catch(() => {
|
|
1319
|
-
}).then(() => {
|
|
1582
|
+
execFileAsync(this.bin, ["wait", containerId]).catch(() => ({ stdout: "" })).then(async ({ stdout }) => {
|
|
1320
1583
|
const vm = this.vms.get(vmId);
|
|
1321
|
-
if (vm
|
|
1584
|
+
if (!vm || vm.containerId !== containerId) return;
|
|
1585
|
+
if (vm.runMode !== "job") {
|
|
1322
1586
|
vm.handle = { ...vm.handle, running: false };
|
|
1587
|
+
return;
|
|
1323
1588
|
}
|
|
1589
|
+
const exitCode = Number.parseInt(stdout.trim(), 10) || 0;
|
|
1590
|
+
const logs = await this.captureLogs(containerId);
|
|
1591
|
+
await execFileAsync(this.bin, ["rm", "-f", containerId]).catch(
|
|
1592
|
+
(error) => src_default.warn({ vmId, err: error }, "docker rm after job exit failed")
|
|
1593
|
+
);
|
|
1594
|
+
this.usedPorts.delete(vm.hostPort);
|
|
1595
|
+
vm.handle = { ...vm.handle, running: false };
|
|
1596
|
+
const result = {
|
|
1597
|
+
exitCode,
|
|
1598
|
+
...logs,
|
|
1599
|
+
durationMs: Date.now() - vm.startedAt,
|
|
1600
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1601
|
+
};
|
|
1602
|
+
src_default.info({ vmId, exitCode }, "docker job vm exited");
|
|
1603
|
+
this.deliverJobResult(vmId, result);
|
|
1324
1604
|
});
|
|
1325
1605
|
}
|
|
1606
|
+
deliverJobResult(vmId, result) {
|
|
1607
|
+
const handler = this.exitHandlers.get(vmId);
|
|
1608
|
+
if (handler) {
|
|
1609
|
+
this.exitHandlers.delete(vmId);
|
|
1610
|
+
this.vms.delete(vmId);
|
|
1611
|
+
handler(result);
|
|
1612
|
+
} else {
|
|
1613
|
+
this.pendingResults.set(vmId, result);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
/** `docker logs` mirrors the container's stdout/stderr onto its own, so the CLI's streams stay separate. */
|
|
1617
|
+
async captureLogs(containerId) {
|
|
1618
|
+
try {
|
|
1619
|
+
const { stdout, stderr } = await execFileAsync(this.bin, ["logs", containerId], {
|
|
1620
|
+
maxBuffer: MAX_OUTPUT_BYTES4
|
|
1621
|
+
});
|
|
1622
|
+
return {
|
|
1623
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES4),
|
|
1624
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES4),
|
|
1625
|
+
truncated: stdout.length > MAX_OUTPUT_BYTES4 || stderr.length > MAX_OUTPUT_BYTES4
|
|
1626
|
+
};
|
|
1627
|
+
} catch (error) {
|
|
1628
|
+
const failed = error;
|
|
1629
|
+
return {
|
|
1630
|
+
stdout: (failed.stdout ?? "").slice(0, MAX_OUTPUT_BYTES4),
|
|
1631
|
+
stderr: (failed.stderr ?? "").slice(0, MAX_OUTPUT_BYTES4),
|
|
1632
|
+
truncated: true
|
|
1633
|
+
};
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
onExit(vmId, handler) {
|
|
1637
|
+
const pending = this.pendingResults.get(vmId);
|
|
1638
|
+
if (pending) {
|
|
1639
|
+
this.pendingResults.delete(vmId);
|
|
1640
|
+
this.vms.delete(vmId);
|
|
1641
|
+
handler(pending);
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
this.exitHandlers.set(vmId, handler);
|
|
1645
|
+
}
|
|
1326
1646
|
async pause(vmId) {
|
|
1327
1647
|
const vm = this.mustGet(vmId);
|
|
1328
1648
|
await execFileAsync(this.bin, ["pause", vm.containerId]);
|
|
@@ -1368,7 +1688,12 @@ var DockerExecutor = class {
|
|
|
1368
1688
|
if (!meta?.containerId || meta.hostPort === void 0) return void 0;
|
|
1369
1689
|
let running;
|
|
1370
1690
|
try {
|
|
1371
|
-
const { stdout } = await execFileAsync(this.bin, [
|
|
1691
|
+
const { stdout } = await execFileAsync(this.bin, [
|
|
1692
|
+
"inspect",
|
|
1693
|
+
"-f",
|
|
1694
|
+
"{{.State.Running}}",
|
|
1695
|
+
meta.containerId
|
|
1696
|
+
]);
|
|
1372
1697
|
running = stdout.trim() === "true";
|
|
1373
1698
|
} catch {
|
|
1374
1699
|
return void 0;
|
|
@@ -1386,18 +1711,58 @@ var DockerExecutor = class {
|
|
|
1386
1711
|
handle,
|
|
1387
1712
|
containerId: meta.containerId,
|
|
1388
1713
|
containerName: meta.containerName ?? `multiplayer-vm-${record.vmId}`,
|
|
1389
|
-
hostPort: meta.hostPort
|
|
1714
|
+
hostPort: meta.hostPort,
|
|
1715
|
+
runMode: record.runMode,
|
|
1716
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
1390
1717
|
});
|
|
1391
|
-
if (running)
|
|
1718
|
+
if (running) {
|
|
1719
|
+
this.watchExit(record.vmId, meta.containerId);
|
|
1720
|
+
} else if (record.runMode === "job") {
|
|
1721
|
+
void this.finishExitedJob(record.vmId, meta.containerId);
|
|
1722
|
+
}
|
|
1392
1723
|
src_default.info({ vmId: record.vmId, containerId: meta.containerId }, "docker vm reattached");
|
|
1393
1724
|
return handle;
|
|
1394
1725
|
}
|
|
1726
|
+
/** Same capture-and-report sequence as watchExit's job branch, for a container found already stopped. */
|
|
1727
|
+
async finishExitedJob(vmId, containerId) {
|
|
1728
|
+
const vm = this.vms.get(vmId);
|
|
1729
|
+
if (!vm) return;
|
|
1730
|
+
let exitCode = 0;
|
|
1731
|
+
try {
|
|
1732
|
+
const { stdout } = await execFileAsync(this.bin, [
|
|
1733
|
+
"inspect",
|
|
1734
|
+
"-f",
|
|
1735
|
+
"{{.State.ExitCode}}",
|
|
1736
|
+
containerId
|
|
1737
|
+
]);
|
|
1738
|
+
exitCode = Number.parseInt(stdout.trim(), 10) || 0;
|
|
1739
|
+
} catch {
|
|
1740
|
+
}
|
|
1741
|
+
const logs = await this.captureLogs(containerId);
|
|
1742
|
+
await execFileAsync(this.bin, ["rm", "-f", containerId]).catch(() => {
|
|
1743
|
+
});
|
|
1744
|
+
this.usedPorts.delete(vm.hostPort);
|
|
1745
|
+
vm.handle = { ...vm.handle, running: false };
|
|
1746
|
+
const result = {
|
|
1747
|
+
exitCode,
|
|
1748
|
+
...logs,
|
|
1749
|
+
durationMs: Date.now() - vm.startedAt,
|
|
1750
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1751
|
+
};
|
|
1752
|
+
this.deliverJobResult(vmId, result);
|
|
1753
|
+
}
|
|
1395
1754
|
async proxyHttp(vmId, request) {
|
|
1396
1755
|
const vm = this.mustGet(vmId);
|
|
1397
1756
|
if (!vm.handle.running) throw new Error(`vm ${vmId} is not running`);
|
|
1398
1757
|
return new Promise((resolve, reject) => {
|
|
1399
1758
|
const req = import_http3.default.request(
|
|
1400
|
-
{
|
|
1759
|
+
{
|
|
1760
|
+
host: "127.0.0.1",
|
|
1761
|
+
port: vm.hostPort,
|
|
1762
|
+
method: request.method,
|
|
1763
|
+
path: request.path,
|
|
1764
|
+
headers: request.headers
|
|
1765
|
+
},
|
|
1401
1766
|
(res) => {
|
|
1402
1767
|
const chunks = [];
|
|
1403
1768
|
res.on("data", (c) => chunks.push(c));
|
|
@@ -1406,7 +1771,10 @@ var DockerExecutor = class {
|
|
|
1406
1771
|
() => resolve({
|
|
1407
1772
|
statusCode: res.statusCode ?? 502,
|
|
1408
1773
|
headers: Object.fromEntries(
|
|
1409
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
1774
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
1775
|
+
k,
|
|
1776
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
1777
|
+
])
|
|
1410
1778
|
),
|
|
1411
1779
|
body: Buffer.concat(chunks)
|
|
1412
1780
|
})
|
|
@@ -1427,15 +1795,15 @@ var DockerExecutor = class {
|
|
|
1427
1795
|
(0, import_child_process5.execFile)(
|
|
1428
1796
|
this.bin,
|
|
1429
1797
|
["exec", "-w", cwd, vm.containerId, "/bin/sh", "-c", request.command],
|
|
1430
|
-
{ timeout: timeoutMs, maxBuffer:
|
|
1798
|
+
{ timeout: timeoutMs, maxBuffer: MAX_OUTPUT_BYTES4 },
|
|
1431
1799
|
(error, stdout, stderr) => {
|
|
1432
1800
|
const failed = error;
|
|
1433
1801
|
const timedOut = Boolean(failed?.killed);
|
|
1434
1802
|
const truncated = failed?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
|
|
1435
1803
|
resolve({
|
|
1436
1804
|
exitCode: typeof failed?.code === "number" ? failed.code : failed ? 1 : 0,
|
|
1437
|
-
stdout: stdout.slice(0,
|
|
1438
|
-
stderr: stderr.slice(0,
|
|
1805
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES4),
|
|
1806
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES4),
|
|
1439
1807
|
durationMs: Date.now() - startedAt,
|
|
1440
1808
|
truncated: Boolean(truncated),
|
|
1441
1809
|
timedOut
|
|
@@ -1451,11 +1819,19 @@ var DockerExecutor = class {
|
|
|
1451
1819
|
const relativePath = assertSafeWorkspacePath(file.path);
|
|
1452
1820
|
const buffer = Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8");
|
|
1453
1821
|
if (buffer.byteLength > MAX_FILE_BYTES2) {
|
|
1454
|
-
throw new Error(
|
|
1822
|
+
throw new Error(
|
|
1823
|
+
`File too large (${buffer.byteLength} bytes, max ${MAX_FILE_BYTES2}): ${file.path}`
|
|
1824
|
+
);
|
|
1455
1825
|
}
|
|
1456
1826
|
const dir = relativePath.includes("/") ? relativePath.slice(0, relativePath.lastIndexOf("/")) : "";
|
|
1457
1827
|
if (dir) {
|
|
1458
|
-
await execFileAsync(this.bin, [
|
|
1828
|
+
await execFileAsync(this.bin, [
|
|
1829
|
+
"exec",
|
|
1830
|
+
vm.containerId,
|
|
1831
|
+
"mkdir",
|
|
1832
|
+
"-p",
|
|
1833
|
+
`${WORKSPACE_DIR}/${dir}`
|
|
1834
|
+
]);
|
|
1459
1835
|
}
|
|
1460
1836
|
await this.copyIntoContainer(vm.containerId, `${WORKSPACE_DIR}/${relativePath}`, buffer);
|
|
1461
1837
|
written.push({ path: relativePath, bytes: buffer.byteLength });
|
|
@@ -1468,13 +1844,18 @@ var DockerExecutor = class {
|
|
|
1468
1844
|
const entryName = destPath.replace(/^\//, "");
|
|
1469
1845
|
const tarStream = buildSingleFileTar(entryName, content);
|
|
1470
1846
|
return new Promise((resolve, reject) => {
|
|
1471
|
-
const proc = (0, import_child_process5.spawn)(this.bin, ["cp", "-", `${containerId}:/`], {
|
|
1847
|
+
const proc = (0, import_child_process5.spawn)(this.bin, ["cp", "-", `${containerId}:/`], {
|
|
1848
|
+
stdio: ["pipe", "ignore", "pipe"]
|
|
1849
|
+
});
|
|
1472
1850
|
let stderr = "";
|
|
1473
1851
|
proc.stderr?.on("data", (d) => {
|
|
1474
1852
|
stderr += d.toString();
|
|
1475
1853
|
});
|
|
1476
1854
|
proc.on("error", reject);
|
|
1477
|
-
proc.on(
|
|
1855
|
+
proc.on(
|
|
1856
|
+
"exit",
|
|
1857
|
+
(code) => code === 0 ? resolve() : reject(new Error(`docker cp exited ${code}: ${stderr}`))
|
|
1858
|
+
);
|
|
1478
1859
|
proc.stdin.end(tarStream);
|
|
1479
1860
|
});
|
|
1480
1861
|
}
|
|
@@ -1503,7 +1884,8 @@ var DockerExecutor = class {
|
|
|
1503
1884
|
});
|
|
1504
1885
|
});
|
|
1505
1886
|
if (ok) return;
|
|
1506
|
-
if (Date.now() > deadline)
|
|
1887
|
+
if (Date.now() > deadline)
|
|
1888
|
+
throw new Error(`timed out waiting for docker vm agent on port ${port}`);
|
|
1507
1889
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
1508
1890
|
}
|
|
1509
1891
|
}
|
|
@@ -1515,7 +1897,11 @@ function buildSingleFileTar(name, content) {
|
|
|
1515
1897
|
header.write("0000000\0", 108, "utf8");
|
|
1516
1898
|
header.write("0000000\0", 116, "utf8");
|
|
1517
1899
|
header.write(content.byteLength.toString(8).padStart(11, "0") + "\0", 124, "utf8");
|
|
1518
|
-
header.write(
|
|
1900
|
+
header.write(
|
|
1901
|
+
Math.floor(Date.now() / 1e3).toString(8).padStart(11, "0") + "\0",
|
|
1902
|
+
136,
|
|
1903
|
+
"utf8"
|
|
1904
|
+
);
|
|
1519
1905
|
header.write(" ", 148, "utf8");
|
|
1520
1906
|
header.write("0", 156, "utf8");
|
|
1521
1907
|
header.write("ustar\0", 257, "utf8");
|
|
@@ -1538,7 +1924,7 @@ var import_path6 = __toESM(require("path"));
|
|
|
1538
1924
|
var import_http4 = __toESM(require("http"));
|
|
1539
1925
|
var DEFAULT_EXEC_TIMEOUT_MS3 = 6e4;
|
|
1540
1926
|
var MAX_EXEC_TIMEOUT_MS3 = 3e5;
|
|
1541
|
-
var
|
|
1927
|
+
var MAX_OUTPUT_BYTES5 = 1024 * 1024;
|
|
1542
1928
|
var MAX_FILE_BYTES3 = 5 * 1024 * 1024;
|
|
1543
1929
|
var WORKSPACE_DIR2 = "/workspace";
|
|
1544
1930
|
var AGENT_DIR = "/app";
|
|
@@ -1559,6 +1945,8 @@ var DockerSandboxExecutor = class {
|
|
|
1559
1945
|
this.vms = /* @__PURE__ */ new Map();
|
|
1560
1946
|
this.usedPorts = /* @__PURE__ */ new Set();
|
|
1561
1947
|
this.loadedTemplates = /* @__PURE__ */ new Set();
|
|
1948
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
1949
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
1562
1950
|
this.bin = config.bin;
|
|
1563
1951
|
}
|
|
1564
1952
|
allocatePort() {
|
|
@@ -1590,7 +1978,9 @@ var DockerSandboxExecutor = class {
|
|
|
1590
1978
|
await execFileAsync2("docker", ["save", localImage, "-o", tmpFile]);
|
|
1591
1979
|
await execFileAsync2(this.bin, ["template", "load", tmpFile]);
|
|
1592
1980
|
} catch (error) {
|
|
1593
|
-
throw new Error(
|
|
1981
|
+
throw new Error(
|
|
1982
|
+
`failed to bridge local image "${localImage}" into sbx \u2014 run scripts/setup-docker-images.sh: ${error}`
|
|
1983
|
+
);
|
|
1594
1984
|
} finally {
|
|
1595
1985
|
await import_promises5.default.rm(tmpFile, { force: true });
|
|
1596
1986
|
}
|
|
@@ -1624,12 +2014,14 @@ var DockerSandboxExecutor = class {
|
|
|
1624
2014
|
"-e",
|
|
1625
2015
|
`STACK=${spec.stack}`,
|
|
1626
2016
|
...spec.sessionId ? ["-e", `SESSION_ID=${spec.sessionId}`] : [],
|
|
1627
|
-
...spec.appUrl ? ["-e", `APP_URL=${spec.appUrl}`] : []
|
|
2017
|
+
...spec.appUrl ? ["-e", `APP_URL=${spec.appUrl}`] : [],
|
|
2018
|
+
...Object.entries(spec.env ?? {}).flatMap(([key, value]) => ["-e", `${key}=${value}`])
|
|
1628
2019
|
];
|
|
2020
|
+
const runMode = spec.runMode ?? "service";
|
|
1629
2021
|
src_default.info({ vmId: spec.vmId, bin: this.bin, args: createArgs }, "creating docker sandbox");
|
|
1630
2022
|
try {
|
|
1631
2023
|
await execFileAsync2(this.bin, createArgs);
|
|
1632
|
-
this.launchAgent(sandboxName, agentCommand);
|
|
2024
|
+
if (runMode !== "job") this.launchAgent(sandboxName, agentCommand);
|
|
1633
2025
|
} catch (error) {
|
|
1634
2026
|
this.usedPorts.delete(hostPort);
|
|
1635
2027
|
await import_promises5.default.rm(vmDir, { recursive: true, force: true }).catch(() => {
|
|
@@ -1644,11 +2036,84 @@ var DockerSandboxExecutor = class {
|
|
|
1644
2036
|
running: true,
|
|
1645
2037
|
metadata: { sandboxName, hostPort, vmDir, agentCommand }
|
|
1646
2038
|
};
|
|
1647
|
-
this.vms.set(spec.vmId, {
|
|
1648
|
-
|
|
1649
|
-
|
|
2039
|
+
this.vms.set(spec.vmId, {
|
|
2040
|
+
handle,
|
|
2041
|
+
sandboxName,
|
|
2042
|
+
vmDir,
|
|
2043
|
+
hostPort,
|
|
2044
|
+
agentCommand,
|
|
2045
|
+
runMode,
|
|
2046
|
+
startedAt: Date.now()
|
|
2047
|
+
});
|
|
2048
|
+
if (runMode === "job") {
|
|
2049
|
+
this.runJob(spec.vmId, sandboxName, agentCommand);
|
|
2050
|
+
} else {
|
|
2051
|
+
await this.waitForAgent(hostPort);
|
|
2052
|
+
}
|
|
2053
|
+
src_default.info({ vmId: spec.vmId, sandboxName, hostPort, runMode }, "docker sandbox ready");
|
|
1650
2054
|
return handle;
|
|
1651
2055
|
}
|
|
2056
|
+
/**
|
|
2057
|
+
* Job mode: unlike `launchAgent` (fire-and-forget `sbx exec -d`, used for persistent
|
|
2058
|
+
* services), run the entrypoint in the *foreground* — the CLI call itself blocks until the
|
|
2059
|
+
* command exits and hands back its exit code + captured stdout/stderr directly, so there's
|
|
2060
|
+
* no separate wait/logs step needed the way DockerExecutor needs for a backgrounded container.
|
|
2061
|
+
*/
|
|
2062
|
+
runJob(vmId, sandboxName, agentCommand) {
|
|
2063
|
+
const startedAt = this.vms.get(vmId)?.startedAt ?? Date.now();
|
|
2064
|
+
(0, import_child_process6.execFile)(
|
|
2065
|
+
this.bin,
|
|
2066
|
+
["exec", "-w", AGENT_DIR, sandboxName, "/bin/sh", "-c", agentCommand],
|
|
2067
|
+
{ maxBuffer: MAX_OUTPUT_BYTES5 },
|
|
2068
|
+
(error, stdout, stderr) => {
|
|
2069
|
+
const failed = error;
|
|
2070
|
+
const truncated = failed?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
|
|
2071
|
+
const result = {
|
|
2072
|
+
exitCode: typeof failed?.code === "number" ? failed.code : failed ? 1 : 0,
|
|
2073
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES5),
|
|
2074
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES5),
|
|
2075
|
+
truncated: Boolean(truncated),
|
|
2076
|
+
durationMs: Date.now() - startedAt,
|
|
2077
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2078
|
+
};
|
|
2079
|
+
void this.finishJob(vmId, sandboxName, result);
|
|
2080
|
+
}
|
|
2081
|
+
);
|
|
2082
|
+
}
|
|
2083
|
+
async finishJob(vmId, sandboxName, result) {
|
|
2084
|
+
src_default.info({ vmId, exitCode: result.exitCode }, "docker sandbox job exited");
|
|
2085
|
+
await execFileAsync2(this.bin, ["rm", "-f", sandboxName]).catch(
|
|
2086
|
+
(error) => src_default.warn({ vmId, err: error }, "sbx rm after job exit failed")
|
|
2087
|
+
);
|
|
2088
|
+
const vm = this.vms.get(vmId);
|
|
2089
|
+
if (vm) {
|
|
2090
|
+
await import_promises5.default.rm(vm.vmDir, { recursive: true, force: true }).catch(() => {
|
|
2091
|
+
});
|
|
2092
|
+
this.usedPorts.delete(vm.hostPort);
|
|
2093
|
+
vm.handle = { ...vm.handle, running: false };
|
|
2094
|
+
}
|
|
2095
|
+
this.deliverJobResult(vmId, result);
|
|
2096
|
+
}
|
|
2097
|
+
deliverJobResult(vmId, result) {
|
|
2098
|
+
const handler = this.exitHandlers.get(vmId);
|
|
2099
|
+
if (handler) {
|
|
2100
|
+
this.exitHandlers.delete(vmId);
|
|
2101
|
+
this.vms.delete(vmId);
|
|
2102
|
+
handler(result);
|
|
2103
|
+
} else {
|
|
2104
|
+
this.pendingResults.set(vmId, result);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
onExit(vmId, handler) {
|
|
2108
|
+
const pending = this.pendingResults.get(vmId);
|
|
2109
|
+
if (pending) {
|
|
2110
|
+
this.pendingResults.delete(vmId);
|
|
2111
|
+
this.vms.delete(vmId);
|
|
2112
|
+
handler(pending);
|
|
2113
|
+
return;
|
|
2114
|
+
}
|
|
2115
|
+
this.exitHandlers.set(vmId, handler);
|
|
2116
|
+
}
|
|
1652
2117
|
/**
|
|
1653
2118
|
* `sbx create shell` doesn't run the image's own CMD — launch the guest agent explicitly.
|
|
1654
2119
|
*
|
|
@@ -1659,10 +2124,17 @@ var DockerSandboxExecutor = class {
|
|
|
1659
2124
|
* process exiting, so it's spawned detached and never awaited.
|
|
1660
2125
|
*/
|
|
1661
2126
|
launchAgent(sandboxName, agentCommand) {
|
|
1662
|
-
const proc = (0, import_child_process6.spawn)(
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
2127
|
+
const proc = (0, import_child_process6.spawn)(
|
|
2128
|
+
this.bin,
|
|
2129
|
+
["exec", "-d", "-w", AGENT_DIR, sandboxName, "/bin/sh", "-c", agentCommand],
|
|
2130
|
+
{
|
|
2131
|
+
stdio: "ignore"
|
|
2132
|
+
}
|
|
2133
|
+
);
|
|
2134
|
+
proc.on(
|
|
2135
|
+
"error",
|
|
2136
|
+
(error) => src_default.warn({ sandboxName, err: error }, "sbx exec -d failed to spawn")
|
|
2137
|
+
);
|
|
1666
2138
|
proc.unref();
|
|
1667
2139
|
}
|
|
1668
2140
|
async pause(vmId) {
|
|
@@ -1708,7 +2180,8 @@ var DockerSandboxExecutor = class {
|
|
|
1708
2180
|
*/
|
|
1709
2181
|
async reattach(record) {
|
|
1710
2182
|
const meta = record.runtimeMetadata;
|
|
1711
|
-
if (!meta?.sandboxName || meta.hostPort === void 0 || !meta.vmDir || !meta.agentCommand)
|
|
2183
|
+
if (!meta?.sandboxName || meta.hostPort === void 0 || !meta.vmDir || !meta.agentCommand)
|
|
2184
|
+
return void 0;
|
|
1712
2185
|
let running;
|
|
1713
2186
|
try {
|
|
1714
2187
|
await execFileAsync2(this.bin, ["exec", meta.sandboxName, "true"]);
|
|
@@ -1735,8 +2208,20 @@ var DockerSandboxExecutor = class {
|
|
|
1735
2208
|
sandboxName: meta.sandboxName,
|
|
1736
2209
|
vmDir: meta.vmDir,
|
|
1737
2210
|
hostPort: meta.hostPort,
|
|
1738
|
-
agentCommand: meta.agentCommand
|
|
2211
|
+
agentCommand: meta.agentCommand,
|
|
2212
|
+
runMode: record.runMode,
|
|
2213
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
1739
2214
|
});
|
|
2215
|
+
if (record.runMode === "job") {
|
|
2216
|
+
void this.finishJob(record.vmId, meta.sandboxName, {
|
|
2217
|
+
exitCode: -1,
|
|
2218
|
+
stdout: "",
|
|
2219
|
+
stderr: "worker restarted while this job was still running",
|
|
2220
|
+
truncated: false,
|
|
2221
|
+
durationMs: Date.now() - (Date.parse(record.createdAt) || Date.now()),
|
|
2222
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2223
|
+
});
|
|
2224
|
+
}
|
|
1740
2225
|
src_default.info({ vmId: record.vmId, sandboxName: meta.sandboxName }, "docker sandbox reattached");
|
|
1741
2226
|
return handle;
|
|
1742
2227
|
}
|
|
@@ -1745,7 +2230,13 @@ var DockerSandboxExecutor = class {
|
|
|
1745
2230
|
if (!vm.handle.running) throw new Error(`vm ${vmId} is not running`);
|
|
1746
2231
|
return new Promise((resolve, reject) => {
|
|
1747
2232
|
const req = import_http4.default.request(
|
|
1748
|
-
{
|
|
2233
|
+
{
|
|
2234
|
+
host: "127.0.0.1",
|
|
2235
|
+
port: vm.hostPort,
|
|
2236
|
+
method: request.method,
|
|
2237
|
+
path: request.path,
|
|
2238
|
+
headers: request.headers
|
|
2239
|
+
},
|
|
1749
2240
|
(res) => {
|
|
1750
2241
|
const chunks = [];
|
|
1751
2242
|
res.on("data", (c) => chunks.push(c));
|
|
@@ -1754,7 +2245,10 @@ var DockerSandboxExecutor = class {
|
|
|
1754
2245
|
() => resolve({
|
|
1755
2246
|
statusCode: res.statusCode ?? 502,
|
|
1756
2247
|
headers: Object.fromEntries(
|
|
1757
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
2248
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
2249
|
+
k,
|
|
2250
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
2251
|
+
])
|
|
1758
2252
|
),
|
|
1759
2253
|
body: Buffer.concat(chunks)
|
|
1760
2254
|
})
|
|
@@ -1775,15 +2269,15 @@ var DockerSandboxExecutor = class {
|
|
|
1775
2269
|
(0, import_child_process6.execFile)(
|
|
1776
2270
|
this.bin,
|
|
1777
2271
|
["exec", "-w", cwd, vm.sandboxName, "/bin/sh", "-c", request.command],
|
|
1778
|
-
{ timeout: timeoutMs, maxBuffer:
|
|
2272
|
+
{ timeout: timeoutMs, maxBuffer: MAX_OUTPUT_BYTES5 },
|
|
1779
2273
|
(error, stdout, stderr) => {
|
|
1780
2274
|
const failed = error;
|
|
1781
2275
|
const timedOut = Boolean(failed?.killed);
|
|
1782
2276
|
const truncated = failed?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
|
|
1783
2277
|
resolve({
|
|
1784
2278
|
exitCode: typeof failed?.code === "number" ? failed.code : failed ? 1 : 0,
|
|
1785
|
-
stdout: stdout.slice(0,
|
|
1786
|
-
stderr: stderr.slice(0,
|
|
2279
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES5),
|
|
2280
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES5),
|
|
1787
2281
|
durationMs: Date.now() - startedAt,
|
|
1788
2282
|
truncated: Boolean(truncated),
|
|
1789
2283
|
timedOut
|
|
@@ -1799,16 +2293,31 @@ var DockerSandboxExecutor = class {
|
|
|
1799
2293
|
const relativePath = assertSafeWorkspacePath2(file.path);
|
|
1800
2294
|
const buffer = Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8");
|
|
1801
2295
|
if (buffer.byteLength > MAX_FILE_BYTES3) {
|
|
1802
|
-
throw new Error(
|
|
2296
|
+
throw new Error(
|
|
2297
|
+
`File too large (${buffer.byteLength} bytes, max ${MAX_FILE_BYTES3}): ${file.path}`
|
|
2298
|
+
);
|
|
1803
2299
|
}
|
|
1804
2300
|
const dir = relativePath.includes("/") ? relativePath.slice(0, relativePath.lastIndexOf("/")) : "";
|
|
1805
2301
|
if (dir) {
|
|
1806
|
-
await execFileAsync2(this.bin, [
|
|
2302
|
+
await execFileAsync2(this.bin, [
|
|
2303
|
+
"exec",
|
|
2304
|
+
vm.sandboxName,
|
|
2305
|
+
"mkdir",
|
|
2306
|
+
"-p",
|
|
2307
|
+
`${WORKSPACE_DIR2}/${dir}`
|
|
2308
|
+
]);
|
|
1807
2309
|
}
|
|
1808
|
-
const tmpFile = import_path6.default.join(
|
|
2310
|
+
const tmpFile = import_path6.default.join(
|
|
2311
|
+
import_os8.default.tmpdir(),
|
|
2312
|
+
`sbx-write-${vmId}-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
2313
|
+
);
|
|
1809
2314
|
await import_promises5.default.writeFile(tmpFile, buffer);
|
|
1810
2315
|
try {
|
|
1811
|
-
await execFileAsync2(this.bin, [
|
|
2316
|
+
await execFileAsync2(this.bin, [
|
|
2317
|
+
"cp",
|
|
2318
|
+
tmpFile,
|
|
2319
|
+
`${vm.sandboxName}:${WORKSPACE_DIR2}/${relativePath}`
|
|
2320
|
+
]);
|
|
1812
2321
|
} finally {
|
|
1813
2322
|
await import_promises5.default.rm(tmpFile, { force: true });
|
|
1814
2323
|
}
|
|
@@ -1844,7 +2353,8 @@ var DockerSandboxExecutor = class {
|
|
|
1844
2353
|
const deadline = Date.now() + timeoutMs;
|
|
1845
2354
|
for (; ; ) {
|
|
1846
2355
|
if (await this.probeAgent(port)) return;
|
|
1847
|
-
if (Date.now() > deadline)
|
|
2356
|
+
if (Date.now() > deadline)
|
|
2357
|
+
throw new Error(`timed out waiting for docker sandbox agent on port ${port}`);
|
|
1848
2358
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
1849
2359
|
}
|
|
1850
2360
|
}
|
|
@@ -1853,12 +2363,14 @@ var DockerSandboxExecutor = class {
|
|
|
1853
2363
|
// src/executor/remote-executor.ts
|
|
1854
2364
|
var import_http5 = __toESM(require("http"));
|
|
1855
2365
|
var import_https = __toESM(require("https"));
|
|
2366
|
+
var JOB_POLL_INTERVAL_MS = 3e3;
|
|
1856
2367
|
var RemoteExecutor = class {
|
|
1857
2368
|
constructor(config) {
|
|
1858
2369
|
// The remote agent is the actual source of truth, but MultiEngineExecutor.route() (used by
|
|
1859
2370
|
// pause/resume/stop/restore/destroy/proxyHttp/exec/writeFiles) needs get() to find the
|
|
1860
2371
|
// right executor for a vmId — cache the handle each call returns so it does.
|
|
1861
2372
|
this.vms = /* @__PURE__ */ new Map();
|
|
2373
|
+
this.jobStartedAt = /* @__PURE__ */ new Map();
|
|
1862
2374
|
this.base = config.agentUrl.replace(/\/$/, "");
|
|
1863
2375
|
this.headers = {
|
|
1864
2376
|
"Content-Type": "application/json",
|
|
@@ -1868,8 +2380,38 @@ var RemoteExecutor = class {
|
|
|
1868
2380
|
async create(spec) {
|
|
1869
2381
|
const handle = await this.request("POST", "/vms", spec);
|
|
1870
2382
|
this.vms.set(spec.vmId, handle);
|
|
2383
|
+
if (spec.runMode === "job") this.jobStartedAt.set(spec.vmId, Date.now());
|
|
1871
2384
|
return handle;
|
|
1872
2385
|
}
|
|
2386
|
+
/**
|
|
2387
|
+
* The remote agent's own wire protocol (VmHandle) has no exit-code/output field — extending
|
|
2388
|
+
* it is out of this repo's control, so this only detects completion (poll GET /vms/:vmId
|
|
2389
|
+
* until running flips false) without a real exit code or captured output. Good enough to
|
|
2390
|
+
* unblock the record from being stuck in 'running' forever; a real RemoteExecutor deployment
|
|
2391
|
+
* wanting full job support needs its agent protocol extended first.
|
|
2392
|
+
*/
|
|
2393
|
+
onExit(vmId, handler) {
|
|
2394
|
+
const startedAt = this.jobStartedAt.get(vmId) ?? Date.now();
|
|
2395
|
+
const poll = async () => {
|
|
2396
|
+
const handle = await this.request("GET", `/vms/${vmId}`).catch(() => void 0);
|
|
2397
|
+
if (handle) this.vms.set(vmId, handle);
|
|
2398
|
+
if (!handle || !handle.running) {
|
|
2399
|
+
this.jobStartedAt.delete(vmId);
|
|
2400
|
+
this.vms.delete(vmId);
|
|
2401
|
+
handler({
|
|
2402
|
+
exitCode: 0,
|
|
2403
|
+
stdout: "",
|
|
2404
|
+
stderr: "remote engine does not report job exit codes/output \u2014 completion detected via polling only",
|
|
2405
|
+
truncated: false,
|
|
2406
|
+
durationMs: Date.now() - startedAt,
|
|
2407
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2408
|
+
});
|
|
2409
|
+
return;
|
|
2410
|
+
}
|
|
2411
|
+
setTimeout(() => void poll(), JOB_POLL_INTERVAL_MS);
|
|
2412
|
+
};
|
|
2413
|
+
void poll();
|
|
2414
|
+
}
|
|
1873
2415
|
async pause(vmId) {
|
|
1874
2416
|
return this.updateCached(vmId, await this.request("POST", `/vms/${vmId}/pause`));
|
|
1875
2417
|
}
|
|
@@ -1997,7 +2539,13 @@ var LambdaMicrovmExecutor = class {
|
|
|
1997
2539
|
} : {},
|
|
1998
2540
|
// The guest's /run lifecycle hook receives this payload — lets the image
|
|
1999
2541
|
// identify which builder session/VM it serves.
|
|
2000
|
-
runHookPayload: JSON.stringify({
|
|
2542
|
+
runHookPayload: JSON.stringify({
|
|
2543
|
+
vmId: spec.vmId,
|
|
2544
|
+
sessionId: spec.sessionId,
|
|
2545
|
+
stack: spec.stack,
|
|
2546
|
+
appUrl: spec.appUrl,
|
|
2547
|
+
env: spec.env
|
|
2548
|
+
})
|
|
2001
2549
|
})
|
|
2002
2550
|
);
|
|
2003
2551
|
if (!run.microvmId || !run.endpoint) {
|
|
@@ -2085,7 +2633,10 @@ var LambdaMicrovmExecutor = class {
|
|
|
2085
2633
|
() => resolve({
|
|
2086
2634
|
statusCode: res.statusCode ?? 502,
|
|
2087
2635
|
headers: Object.fromEntries(
|
|
2088
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
2636
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
2637
|
+
k,
|
|
2638
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
2639
|
+
])
|
|
2089
2640
|
),
|
|
2090
2641
|
body: Buffer.concat(chunks)
|
|
2091
2642
|
})
|
|
@@ -2118,7 +2669,10 @@ var LambdaMicrovmExecutor = class {
|
|
|
2118
2669
|
metadata: meta
|
|
2119
2670
|
};
|
|
2120
2671
|
this.vms.set(record.vmId, { handle, microvmId: meta.microvmId, endpoint: meta.endpoint });
|
|
2121
|
-
src_default.info(
|
|
2672
|
+
src_default.info(
|
|
2673
|
+
{ vmId: record.vmId, microvmId: meta.microvmId, state: described.state },
|
|
2674
|
+
"[LambdaMicrovm] vm reattached"
|
|
2675
|
+
);
|
|
2122
2676
|
return handle;
|
|
2123
2677
|
}
|
|
2124
2678
|
mustGet(vmId) {
|
|
@@ -2158,7 +2712,9 @@ var LambdaMicrovmExecutor = class {
|
|
|
2158
2712
|
throw new Error(`microvm ${microvmId} entered ${resp.state} while waiting for ${target}`);
|
|
2159
2713
|
}
|
|
2160
2714
|
if (Date.now() > deadline) {
|
|
2161
|
-
throw new Error(
|
|
2715
|
+
throw new Error(
|
|
2716
|
+
`timed out waiting for microvm ${microvmId} to reach ${target} (still ${resp.state})`
|
|
2717
|
+
);
|
|
2162
2718
|
}
|
|
2163
2719
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
2164
2720
|
}
|
|
@@ -2168,6 +2724,7 @@ var LambdaMicrovmExecutor = class {
|
|
|
2168
2724
|
// src/executor/fargate-executor.ts
|
|
2169
2725
|
var import_http6 = __toESM(require("http"));
|
|
2170
2726
|
var import_client_ecs = require("@aws-sdk/client-ecs");
|
|
2727
|
+
var JOB_POLL_INTERVAL_MS2 = 5e3;
|
|
2171
2728
|
var FARGATE_MEM_RANGES = {
|
|
2172
2729
|
256: [512, 2048],
|
|
2173
2730
|
512: [1024, 4096],
|
|
@@ -2187,6 +2744,8 @@ var FargateExecutor = class {
|
|
|
2187
2744
|
this.config = config;
|
|
2188
2745
|
this.vms = /* @__PURE__ */ new Map();
|
|
2189
2746
|
this.containerNames = /* @__PURE__ */ new Map();
|
|
2747
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
2748
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
2190
2749
|
this.client = new import_client_ecs.ECSClient(config.region ? { region: config.region } : {});
|
|
2191
2750
|
}
|
|
2192
2751
|
/** First container of the task definition — the target for env overrides. Cached per definition. */
|
|
@@ -2213,11 +2772,16 @@ var FargateExecutor = class {
|
|
|
2213
2772
|
}
|
|
2214
2773
|
if (task.lastStatus === "RUNNING") {
|
|
2215
2774
|
const ip = this.config.assignPublicIp ? await this.resolvePublicIp(task.attachments) : task.containers?.[0]?.networkInterfaces?.[0]?.privateIpv4Address;
|
|
2216
|
-
if (!ip)
|
|
2775
|
+
if (!ip)
|
|
2776
|
+
throw new Error(
|
|
2777
|
+
`fargate task ${taskArn} is RUNNING but has no ${this.config.assignPublicIp ? "public" : "private"} IP`
|
|
2778
|
+
);
|
|
2217
2779
|
return ip;
|
|
2218
2780
|
}
|
|
2219
2781
|
if (Date.now() > deadline) {
|
|
2220
|
-
throw new Error(
|
|
2782
|
+
throw new Error(
|
|
2783
|
+
`fargate task did not reach RUNNING within ${timeoutMs}ms (last status: ${task.lastStatus})`
|
|
2784
|
+
);
|
|
2221
2785
|
}
|
|
2222
2786
|
await new Promise((resolve) => setTimeout(resolve, 3e3));
|
|
2223
2787
|
}
|
|
@@ -2227,7 +2791,9 @@ var FargateExecutor = class {
|
|
|
2227
2791
|
if (!eniId) return void 0;
|
|
2228
2792
|
const { EC2Client, DescribeNetworkInterfacesCommand } = await import("@aws-sdk/client-ec2");
|
|
2229
2793
|
const ec2 = new EC2Client(this.config.region ? { region: this.config.region } : {});
|
|
2230
|
-
const described = await ec2.send(
|
|
2794
|
+
const described = await ec2.send(
|
|
2795
|
+
new DescribeNetworkInterfacesCommand({ NetworkInterfaceIds: [eniId] })
|
|
2796
|
+
);
|
|
2231
2797
|
return described.NetworkInterfaces?.[0]?.Association?.PublicIp;
|
|
2232
2798
|
}
|
|
2233
2799
|
async create(spec) {
|
|
@@ -2261,7 +2827,8 @@ var FargateExecutor = class {
|
|
|
2261
2827
|
...spec.sessionId ? [{ name: "SESSION_ID", value: spec.sessionId }] : [],
|
|
2262
2828
|
{ name: "STACK", value: spec.stack },
|
|
2263
2829
|
// Release artifact: the container agent pulls, unzips, and starts it
|
|
2264
|
-
...spec.appUrl ? [{ name: "APP_URL", value: spec.appUrl }] : []
|
|
2830
|
+
...spec.appUrl ? [{ name: "APP_URL", value: spec.appUrl }] : [],
|
|
2831
|
+
...Object.entries(spec.env ?? {}).map(([name, value]) => ({ name, value }))
|
|
2265
2832
|
]
|
|
2266
2833
|
}
|
|
2267
2834
|
]
|
|
@@ -2277,6 +2844,7 @@ var FargateExecutor = class {
|
|
|
2277
2844
|
throw new Error(`RunTask failed: ${run.failures?.[0]?.reason ?? "no task returned"}`);
|
|
2278
2845
|
}
|
|
2279
2846
|
src_default.info({ vmId: spec.vmId, taskArn, taskDefinition, cpu, memory }, "fargate task started");
|
|
2847
|
+
const runMode = spec.runMode ?? "service";
|
|
2280
2848
|
try {
|
|
2281
2849
|
const ip = await this.waitForRunning(taskArn);
|
|
2282
2850
|
const handle = {
|
|
@@ -2287,11 +2855,18 @@ var FargateExecutor = class {
|
|
|
2287
2855
|
running: true,
|
|
2288
2856
|
metadata: { taskArn, ip }
|
|
2289
2857
|
};
|
|
2290
|
-
this.vms.set(spec.vmId, { handle, taskArn, ip });
|
|
2291
|
-
|
|
2858
|
+
this.vms.set(spec.vmId, { handle, taskArn, ip, runMode, startedAt: Date.now() });
|
|
2859
|
+
if (runMode === "job") void this.watchJob(spec.vmId, taskArn);
|
|
2860
|
+
src_default.info({ vmId: spec.vmId, ip, runMode }, "fargate task running");
|
|
2292
2861
|
return handle;
|
|
2293
2862
|
} catch (error) {
|
|
2294
|
-
await this.client.send(
|
|
2863
|
+
await this.client.send(
|
|
2864
|
+
new import_client_ecs.StopTaskCommand({
|
|
2865
|
+
cluster: this.config.cluster,
|
|
2866
|
+
task: taskArn,
|
|
2867
|
+
reason: "startup failed"
|
|
2868
|
+
})
|
|
2869
|
+
).catch(() => {
|
|
2295
2870
|
});
|
|
2296
2871
|
throw error;
|
|
2297
2872
|
}
|
|
@@ -2308,7 +2883,11 @@ var FargateExecutor = class {
|
|
|
2308
2883
|
async stop(vmId) {
|
|
2309
2884
|
const vm = this.mustGet(vmId);
|
|
2310
2885
|
await this.client.send(
|
|
2311
|
-
new import_client_ecs.StopTaskCommand({
|
|
2886
|
+
new import_client_ecs.StopTaskCommand({
|
|
2887
|
+
cluster: this.config.cluster,
|
|
2888
|
+
task: vm.taskArn,
|
|
2889
|
+
reason: "stopped via API"
|
|
2890
|
+
})
|
|
2312
2891
|
);
|
|
2313
2892
|
vm.handle.running = false;
|
|
2314
2893
|
return vm.handle;
|
|
@@ -2316,7 +2895,13 @@ var FargateExecutor = class {
|
|
|
2316
2895
|
async destroy(vmId) {
|
|
2317
2896
|
const vm = this.vms.get(vmId);
|
|
2318
2897
|
if (!vm) return;
|
|
2319
|
-
await this.client.send(
|
|
2898
|
+
await this.client.send(
|
|
2899
|
+
new import_client_ecs.StopTaskCommand({
|
|
2900
|
+
cluster: this.config.cluster,
|
|
2901
|
+
task: vm.taskArn,
|
|
2902
|
+
reason: "destroyed via API"
|
|
2903
|
+
})
|
|
2904
|
+
).catch((error) => src_default.warn({ vmId, err: error }, "fargate stop on destroy failed"));
|
|
2320
2905
|
this.vms.delete(vmId);
|
|
2321
2906
|
}
|
|
2322
2907
|
get(vmId) {
|
|
@@ -2345,7 +2930,10 @@ var FargateExecutor = class {
|
|
|
2345
2930
|
() => resolve({
|
|
2346
2931
|
statusCode: res.statusCode ?? 502,
|
|
2347
2932
|
headers: Object.fromEntries(
|
|
2348
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
2933
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
2934
|
+
k,
|
|
2935
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
2936
|
+
])
|
|
2349
2937
|
),
|
|
2350
2938
|
body: Buffer.concat(chunks)
|
|
2351
2939
|
})
|
|
@@ -2378,10 +2966,82 @@ var FargateExecutor = class {
|
|
|
2378
2966
|
running: true,
|
|
2379
2967
|
metadata: { taskArn: meta.taskArn, ip }
|
|
2380
2968
|
};
|
|
2381
|
-
this.vms.set(record.vmId, {
|
|
2969
|
+
this.vms.set(record.vmId, {
|
|
2970
|
+
handle,
|
|
2971
|
+
taskArn: meta.taskArn,
|
|
2972
|
+
ip,
|
|
2973
|
+
runMode: record.runMode,
|
|
2974
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
2975
|
+
});
|
|
2976
|
+
if (record.runMode === "job") void this.watchJob(record.vmId, meta.taskArn);
|
|
2382
2977
|
src_default.info({ vmId: record.vmId, taskArn: meta.taskArn, ip }, "fargate task reattached");
|
|
2383
2978
|
return handle;
|
|
2384
2979
|
}
|
|
2980
|
+
/**
|
|
2981
|
+
* No push notification exists for "task stopped" without EventBridge wiring (v1 doesn't set
|
|
2982
|
+
* that up), so poll DescribeTasks. Exit code comes straight from the container's own
|
|
2983
|
+
* `exitCode` field — no CloudWatch Logs integration yet, so stdout/stderr capture isn't
|
|
2984
|
+
* implemented for this engine (the caller can still find full output in CloudWatch directly,
|
|
2985
|
+
* via the task's own log configuration).
|
|
2986
|
+
*/
|
|
2987
|
+
async watchJob(vmId, taskArn) {
|
|
2988
|
+
const startedAt = this.vms.get(vmId)?.startedAt ?? Date.now();
|
|
2989
|
+
for (; ; ) {
|
|
2990
|
+
await new Promise((resolve) => setTimeout(resolve, JOB_POLL_INTERVAL_MS2));
|
|
2991
|
+
if (!this.vms.has(vmId)) return;
|
|
2992
|
+
const described = await this.client.send(new import_client_ecs.DescribeTasksCommand({ cluster: this.config.cluster, tasks: [taskArn] })).catch(() => void 0);
|
|
2993
|
+
const task = described?.tasks?.[0];
|
|
2994
|
+
if (!task) {
|
|
2995
|
+
this.finishJob(vmId, {
|
|
2996
|
+
exitCode: -1,
|
|
2997
|
+
stdout: "",
|
|
2998
|
+
stderr: "fargate task not found while polling for completion",
|
|
2999
|
+
truncated: false,
|
|
3000
|
+
durationMs: Date.now() - startedAt,
|
|
3001
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3002
|
+
});
|
|
3003
|
+
return;
|
|
3004
|
+
}
|
|
3005
|
+
if (task.lastStatus === "STOPPED") {
|
|
3006
|
+
const exitCode = task.containers?.[0]?.exitCode;
|
|
3007
|
+
this.finishJob(vmId, {
|
|
3008
|
+
exitCode: exitCode ?? 1,
|
|
3009
|
+
stdout: "",
|
|
3010
|
+
stderr: [
|
|
3011
|
+
task.stoppedReason ? `stoppedReason: ${task.stoppedReason}` : void 0,
|
|
3012
|
+
"output capture not implemented for the fargate engine \u2014 see CloudWatch Logs for this task"
|
|
3013
|
+
].filter(Boolean).join("; "),
|
|
3014
|
+
truncated: false,
|
|
3015
|
+
durationMs: Date.now() - startedAt,
|
|
3016
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3017
|
+
});
|
|
3018
|
+
return;
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
finishJob(vmId, result) {
|
|
3023
|
+
src_default.info({ vmId, exitCode: result.exitCode }, "fargate job task exited");
|
|
3024
|
+
const vm = this.vms.get(vmId);
|
|
3025
|
+
if (vm) vm.handle = { ...vm.handle, running: false };
|
|
3026
|
+
const handler = this.exitHandlers.get(vmId);
|
|
3027
|
+
if (handler) {
|
|
3028
|
+
this.exitHandlers.delete(vmId);
|
|
3029
|
+
this.vms.delete(vmId);
|
|
3030
|
+
handler(result);
|
|
3031
|
+
} else {
|
|
3032
|
+
this.pendingResults.set(vmId, result);
|
|
3033
|
+
}
|
|
3034
|
+
}
|
|
3035
|
+
onExit(vmId, handler) {
|
|
3036
|
+
const pending = this.pendingResults.get(vmId);
|
|
3037
|
+
if (pending) {
|
|
3038
|
+
this.pendingResults.delete(vmId);
|
|
3039
|
+
this.vms.delete(vmId);
|
|
3040
|
+
handler(pending);
|
|
3041
|
+
return;
|
|
3042
|
+
}
|
|
3043
|
+
this.exitHandlers.set(vmId, handler);
|
|
3044
|
+
}
|
|
2385
3045
|
mustGet(vmId) {
|
|
2386
3046
|
const vm = this.vms.get(vmId);
|
|
2387
3047
|
if (!vm) throw new Error(`vm ${vmId} not found`);
|
|
@@ -2392,7 +3052,16 @@ var FargateExecutor = class {
|
|
|
2392
3052
|
// src/executor/detect.ts
|
|
2393
3053
|
var import_fs2 = __toESM(require("fs"));
|
|
2394
3054
|
var import_child_process7 = require("child_process");
|
|
2395
|
-
var ENGINE_PRIORITY = [
|
|
3055
|
+
var ENGINE_PRIORITY = [
|
|
3056
|
+
"firecracker",
|
|
3057
|
+
"qemu",
|
|
3058
|
+
"docker-sandbox",
|
|
3059
|
+
"docker",
|
|
3060
|
+
"lambda-microvm",
|
|
3061
|
+
"fargate",
|
|
3062
|
+
"remote",
|
|
3063
|
+
"mock"
|
|
3064
|
+
];
|
|
2396
3065
|
function binaryOnPath(bin) {
|
|
2397
3066
|
if (bin.includes("/")) {
|
|
2398
3067
|
try {
|
|
@@ -2418,22 +3087,35 @@ function kvmUsable() {
|
|
|
2418
3087
|
}
|
|
2419
3088
|
}
|
|
2420
3089
|
function hasBaseImages(cfg) {
|
|
2421
|
-
if (cfg.kernelPath && cfg.initrdPath)
|
|
3090
|
+
if (cfg.kernelPath && cfg.initrdPath)
|
|
3091
|
+
return import_fs2.default.existsSync(cfg.kernelPath) && import_fs2.default.existsSync(cfg.initrdPath);
|
|
2422
3092
|
return availableStacks(cfg.imagesDir).length > 0;
|
|
2423
3093
|
}
|
|
2424
3094
|
function probeQemu(config) {
|
|
2425
3095
|
const { bin, accel } = config.qemu;
|
|
2426
3096
|
if (!hasBaseImages(config.qemu)) {
|
|
2427
|
-
return {
|
|
3097
|
+
return {
|
|
3098
|
+
engine: "qemu",
|
|
3099
|
+
available: false,
|
|
3100
|
+
reason: `no base images under ${config.qemu.imagesDir} \u2014 run "worker-cli images pull" or scripts/setup-vm-images.sh`
|
|
3101
|
+
};
|
|
2428
3102
|
}
|
|
2429
3103
|
if (!binaryOnPath(bin)) {
|
|
2430
3104
|
return { engine: "qemu", available: false, reason: `${bin} not found on PATH` };
|
|
2431
3105
|
}
|
|
2432
3106
|
if (accel === "hvf" && process.platform !== "darwin") {
|
|
2433
|
-
return {
|
|
3107
|
+
return {
|
|
3108
|
+
engine: "qemu",
|
|
3109
|
+
available: false,
|
|
3110
|
+
reason: "QEMU_ACCEL=hvf requires macOS (use kvm or tcg)"
|
|
3111
|
+
};
|
|
2434
3112
|
}
|
|
2435
3113
|
if (accel === "kvm" && !kvmUsable()) {
|
|
2436
|
-
return {
|
|
3114
|
+
return {
|
|
3115
|
+
engine: "qemu",
|
|
3116
|
+
available: false,
|
|
3117
|
+
reason: "QEMU_ACCEL=kvm but /dev/kvm is not accessible"
|
|
3118
|
+
};
|
|
2437
3119
|
}
|
|
2438
3120
|
return { engine: "qemu", available: true };
|
|
2439
3121
|
}
|
|
@@ -2446,17 +3128,22 @@ function probeFirecracker(config) {
|
|
|
2446
3128
|
return { engine: "firecracker", available: false, reason: "/dev/kvm is not accessible" };
|
|
2447
3129
|
}
|
|
2448
3130
|
if (!hasBaseImages(config.firecracker)) {
|
|
2449
|
-
return {
|
|
3131
|
+
return {
|
|
3132
|
+
engine: "firecracker",
|
|
3133
|
+
available: false,
|
|
3134
|
+
reason: `no base images under ${config.firecracker.imagesDir} \u2014 run "worker-cli images pull" or scripts/setup-vm-images.sh`
|
|
3135
|
+
};
|
|
2450
3136
|
}
|
|
2451
3137
|
if (!binaryOnPath(bin)) {
|
|
2452
3138
|
return { engine: "firecracker", available: false, reason: `${bin} not found on PATH` };
|
|
2453
3139
|
}
|
|
2454
3140
|
return { engine: "firecracker", available: true };
|
|
2455
3141
|
}
|
|
2456
|
-
function hasLocalDockerImage(imagePrefix, imageTag) {
|
|
3142
|
+
function hasLocalDockerImage(bin, imagePrefix, imageTag, stackImages = {}) {
|
|
2457
3143
|
return VM_STACKS.some((stack) => {
|
|
2458
3144
|
try {
|
|
2459
|
-
|
|
3145
|
+
const image = stackImages[stack] ?? `${imagePrefix}-${stack}:${imageTag}`;
|
|
3146
|
+
(0, import_child_process7.execFileSync)(bin, ["image", "inspect", image], { stdio: "ignore" });
|
|
2460
3147
|
return true;
|
|
2461
3148
|
} catch {
|
|
2462
3149
|
return false;
|
|
@@ -2464,7 +3151,7 @@ function hasLocalDockerImage(imagePrefix, imageTag) {
|
|
|
2464
3151
|
});
|
|
2465
3152
|
}
|
|
2466
3153
|
function probeDocker(config) {
|
|
2467
|
-
const { bin, imagePrefix, imageTag } = config.docker;
|
|
3154
|
+
const { bin, imagePrefix, imageTag, stackImages } = config.docker;
|
|
2468
3155
|
if (!binaryOnPath(bin)) {
|
|
2469
3156
|
return { engine: "docker", available: false, reason: `${bin} not found on PATH` };
|
|
2470
3157
|
}
|
|
@@ -2473,8 +3160,12 @@ function probeDocker(config) {
|
|
|
2473
3160
|
} catch {
|
|
2474
3161
|
return { engine: "docker", available: false, reason: "docker daemon not reachable" };
|
|
2475
3162
|
}
|
|
2476
|
-
if (!hasLocalDockerImage(imagePrefix, imageTag)) {
|
|
2477
|
-
return {
|
|
3163
|
+
if (!hasLocalDockerImage(bin, imagePrefix, imageTag, stackImages)) {
|
|
3164
|
+
return {
|
|
3165
|
+
engine: "docker",
|
|
3166
|
+
available: false,
|
|
3167
|
+
reason: "no local docker images \u2014 run scripts/setup-docker-images.sh"
|
|
3168
|
+
};
|
|
2478
3169
|
}
|
|
2479
3170
|
return { engine: "docker", available: true };
|
|
2480
3171
|
}
|
|
@@ -2486,10 +3177,18 @@ function probeDockerSandbox(config) {
|
|
|
2486
3177
|
try {
|
|
2487
3178
|
(0, import_child_process7.execFileSync)(bin, ["ls"], { stdio: "ignore" });
|
|
2488
3179
|
} catch {
|
|
2489
|
-
return {
|
|
3180
|
+
return {
|
|
3181
|
+
engine: "docker-sandbox",
|
|
3182
|
+
available: false,
|
|
3183
|
+
reason: 'sbx daemon not reachable \u2014 run "sbx daemon start" and "sbx login"'
|
|
3184
|
+
};
|
|
2490
3185
|
}
|
|
2491
|
-
if (!hasLocalDockerImage(imagePrefix, imageTag)) {
|
|
2492
|
-
return {
|
|
3186
|
+
if (!hasLocalDockerImage(config.docker.bin, imagePrefix, imageTag)) {
|
|
3187
|
+
return {
|
|
3188
|
+
engine: "docker-sandbox",
|
|
3189
|
+
available: false,
|
|
3190
|
+
reason: "no local docker images \u2014 run scripts/setup-docker-images.sh"
|
|
3191
|
+
};
|
|
2493
3192
|
}
|
|
2494
3193
|
return { engine: "docker-sandbox", available: true };
|
|
2495
3194
|
}
|
|
@@ -2498,7 +3197,11 @@ function probeRemote(config) {
|
|
|
2498
3197
|
}
|
|
2499
3198
|
function probeLambdaMicrovm(config) {
|
|
2500
3199
|
const { imageArn, stackImageArns } = config.lambdaMicrovm;
|
|
2501
|
-
return imageArn || Object.keys(stackImageArns).length ? { engine: "lambda-microvm", available: true } : {
|
|
3200
|
+
return imageArn || Object.keys(stackImageArns).length ? { engine: "lambda-microvm", available: true } : {
|
|
3201
|
+
engine: "lambda-microvm",
|
|
3202
|
+
available: false,
|
|
3203
|
+
reason: "MICROVM_IMAGE_ARN / MICROVM_STACK_IMAGE_ARNS not set"
|
|
3204
|
+
};
|
|
2502
3205
|
}
|
|
2503
3206
|
function probeFargate(config) {
|
|
2504
3207
|
const { cluster, taskDefinition, stackTaskDefinitions, subnets } = config.fargate;
|
|
@@ -2506,7 +3209,11 @@ function probeFargate(config) {
|
|
|
2506
3209
|
return { engine: "fargate", available: false, reason: "FARGATE_CLUSTER not set" };
|
|
2507
3210
|
}
|
|
2508
3211
|
if (!taskDefinition && !Object.keys(stackTaskDefinitions).length) {
|
|
2509
|
-
return {
|
|
3212
|
+
return {
|
|
3213
|
+
engine: "fargate",
|
|
3214
|
+
available: false,
|
|
3215
|
+
reason: "FARGATE_TASK_DEFINITION / FARGATE_STACK_TASK_DEFS not set"
|
|
3216
|
+
};
|
|
2510
3217
|
}
|
|
2511
3218
|
if (!subnets.length) {
|
|
2512
3219
|
return { engine: "fargate", available: false, reason: "FARGATE_SUBNETS not set" };
|
|
@@ -2577,7 +3284,9 @@ var MultiEngineExecutor = class {
|
|
|
2577
3284
|
const engine = spec.engine ?? this.defaultEngine;
|
|
2578
3285
|
const executor = this.executors.get(engine);
|
|
2579
3286
|
if (!executor) {
|
|
2580
|
-
throw new Error(
|
|
3287
|
+
throw new Error(
|
|
3288
|
+
`engine "${engine}" is not available on this worker (available: ${this.engines.join(", ")})`
|
|
3289
|
+
);
|
|
2581
3290
|
}
|
|
2582
3291
|
const handle = await executor.create(spec);
|
|
2583
3292
|
return { ...handle, engine };
|
|
@@ -2623,9 +3332,15 @@ var MultiEngineExecutor = class {
|
|
|
2623
3332
|
}
|
|
2624
3333
|
writeFiles(vmId, files) {
|
|
2625
3334
|
const executor = this.route(vmId);
|
|
2626
|
-
if (!executor.writeFiles)
|
|
3335
|
+
if (!executor.writeFiles)
|
|
3336
|
+
return Promise.reject(new Error("executor does not support guest file writes"));
|
|
2627
3337
|
return executor.writeFiles(vmId, files);
|
|
2628
3338
|
}
|
|
3339
|
+
onExit(vmId, handler) {
|
|
3340
|
+
const executor = this.route(vmId);
|
|
3341
|
+
if (!executor.onExit) throw new Error("executor does not support job run mode");
|
|
3342
|
+
executor.onExit(vmId, handler);
|
|
3343
|
+
}
|
|
2629
3344
|
/**
|
|
2630
3345
|
* Recover VMs sandbox says this workerId owns but that aren't currently tracked (a worker
|
|
2631
3346
|
* restart, most likely). Runs in parallel — probing many VMs serially would needlessly
|
|
@@ -2640,8 +3355,10 @@ var MultiEngineExecutor = class {
|
|
|
2640
3355
|
const results = await Promise.allSettled(
|
|
2641
3356
|
pending.map(async (record) => {
|
|
2642
3357
|
const executor = record.engine ? this.executors.get(record.engine) : void 0;
|
|
2643
|
-
if (!executor)
|
|
2644
|
-
|
|
3358
|
+
if (!executor)
|
|
3359
|
+
throw new Error(`engine "${record.engine ?? "unknown"}" not available on this worker`);
|
|
3360
|
+
if (!executor.reattach)
|
|
3361
|
+
throw new Error(`engine "${record.engine}" does not support reattach`);
|
|
2645
3362
|
const handle = await executor.reattach(record);
|
|
2646
3363
|
if (!handle) throw new Error("resource no longer exists");
|
|
2647
3364
|
})
|
|
@@ -2649,7 +3366,11 @@ var MultiEngineExecutor = class {
|
|
|
2649
3366
|
results.forEach((result, i) => {
|
|
2650
3367
|
const vmId = pending[i].vmId;
|
|
2651
3368
|
if (result.status === "fulfilled") src_default.info({ vmId }, "vm reattached");
|
|
2652
|
-
else
|
|
3369
|
+
else
|
|
3370
|
+
src_default.warn(
|
|
3371
|
+
{ vmId, err: result.reason },
|
|
3372
|
+
"vm reattach failed \u2014 staying unavailable until next touch"
|
|
3373
|
+
);
|
|
2653
3374
|
});
|
|
2654
3375
|
}
|
|
2655
3376
|
route(vmId) {
|
|
@@ -2707,16 +3428,52 @@ function wrap(handler) {
|
|
|
2707
3428
|
};
|
|
2708
3429
|
}
|
|
2709
3430
|
function registerCommandHandlers(socket, executor) {
|
|
2710
|
-
socket.on(
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
3431
|
+
socket.on(
|
|
3432
|
+
WorkerEvent.VM_CREATE,
|
|
3433
|
+
wrap(async (spec) => {
|
|
3434
|
+
const handle = await executor.create(spec);
|
|
3435
|
+
if (spec.runMode === "job") {
|
|
3436
|
+
try {
|
|
3437
|
+
executor.onExit(
|
|
3438
|
+
spec.vmId,
|
|
3439
|
+
(result) => socket.emit(WorkerEvent.VM_RUN_EXITED, { vmId: spec.vmId, ...result })
|
|
3440
|
+
);
|
|
3441
|
+
} catch (error) {
|
|
3442
|
+
await executor.destroy(spec.vmId).catch(() => {
|
|
3443
|
+
});
|
|
3444
|
+
throw error;
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
return handle;
|
|
3448
|
+
})
|
|
3449
|
+
);
|
|
3450
|
+
socket.on(
|
|
3451
|
+
WorkerEvent.VM_PAUSE,
|
|
3452
|
+
wrap(({ vmId }) => executor.pause(vmId))
|
|
3453
|
+
);
|
|
3454
|
+
socket.on(
|
|
3455
|
+
WorkerEvent.VM_RESUME,
|
|
3456
|
+
wrap(({ vmId }) => executor.resume(vmId))
|
|
3457
|
+
);
|
|
3458
|
+
socket.on(
|
|
3459
|
+
WorkerEvent.VM_STOP,
|
|
3460
|
+
wrap(({ vmId }) => executor.stop(vmId))
|
|
3461
|
+
);
|
|
3462
|
+
socket.on(
|
|
3463
|
+
WorkerEvent.VM_RESTORE,
|
|
3464
|
+
wrap(({ vmId }) => executor.restore(vmId))
|
|
3465
|
+
);
|
|
3466
|
+
socket.on(
|
|
3467
|
+
WorkerEvent.VM_DESTROY,
|
|
3468
|
+
wrap(({ vmId }) => executor.destroy(vmId))
|
|
3469
|
+
);
|
|
2716
3470
|
socket.on(
|
|
2717
3471
|
WorkerEvent.PROXY_HTTP,
|
|
2718
3472
|
wrap(
|
|
2719
|
-
({
|
|
3473
|
+
({
|
|
3474
|
+
vmId,
|
|
3475
|
+
...request
|
|
3476
|
+
}) => executor.proxyHttp(vmId, request)
|
|
2720
3477
|
)
|
|
2721
3478
|
);
|
|
2722
3479
|
socket.on(
|
|
@@ -2739,14 +3496,23 @@ function registerCommandHandlers(socket, executor) {
|
|
|
2739
3496
|
);
|
|
2740
3497
|
}
|
|
2741
3498
|
function connectToSandbox(config, executor) {
|
|
3499
|
+
let assignedWorkerId;
|
|
2742
3500
|
const socket = (0, import_socket.io)(config.sandboxUrl, {
|
|
2743
3501
|
path: config.workerSocketPath,
|
|
2744
|
-
auth:
|
|
3502
|
+
auth: (callback) => callback({
|
|
3503
|
+
workerId: config.workerId,
|
|
3504
|
+
...assignedWorkerId ? { assignedWorkerId } : {},
|
|
3505
|
+
...config.token ? { token: config.token } : {}
|
|
3506
|
+
}),
|
|
2745
3507
|
transports: ["websocket"],
|
|
2746
3508
|
reconnection: true
|
|
2747
3509
|
});
|
|
2748
3510
|
let heartbeatTimer;
|
|
2749
3511
|
let reattachFallbackTimer;
|
|
3512
|
+
socket.on(WorkerEvent.ASSIGNED_ID, ({ workerId }) => {
|
|
3513
|
+
assignedWorkerId = workerId;
|
|
3514
|
+
src_default.info({ workerId: config.workerId, assignedWorkerId }, "worker runtime ID assigned");
|
|
3515
|
+
});
|
|
2750
3516
|
const sendHeartbeat = () => socket.emit(WorkerEvent.HEARTBEAT, buildHeartbeat(config, executor));
|
|
2751
3517
|
function startHeartbeat() {
|
|
2752
3518
|
if (heartbeatTimer) return;
|
|
@@ -2754,15 +3520,37 @@ function connectToSandbox(config, executor) {
|
|
|
2754
3520
|
heartbeatTimer = setInterval(sendHeartbeat, HEARTBEAT_INTERVAL_MS);
|
|
2755
3521
|
}
|
|
2756
3522
|
socket.on("connect", () => {
|
|
2757
|
-
src_default.info(
|
|
3523
|
+
src_default.info(
|
|
3524
|
+
{ workerId: config.workerId, assignedWorkerId, url: config.sandboxUrl },
|
|
3525
|
+
"connected to sandbox"
|
|
3526
|
+
);
|
|
2758
3527
|
socket.off(WorkerEvent.REATTACH);
|
|
2759
3528
|
reattachFallbackTimer = setTimeout(() => {
|
|
2760
|
-
src_default.warn(
|
|
3529
|
+
src_default.warn(
|
|
3530
|
+
{ workerId: config.workerId },
|
|
3531
|
+
"no reattach handshake from sandbox \u2014 starting heartbeat anyway"
|
|
3532
|
+
);
|
|
2761
3533
|
startHeartbeat();
|
|
2762
3534
|
}, REATTACH_FALLBACK_MS);
|
|
2763
3535
|
socket.once(WorkerEvent.REATTACH, (records) => {
|
|
2764
3536
|
clearTimeout(reattachFallbackTimer);
|
|
2765
|
-
executor.reattachAll(records).catch((error) => src_default.error({ err: error }, "reattach batch failed")).finally(
|
|
3537
|
+
executor.reattachAll(records).catch((error) => src_default.error({ err: error }, "reattach batch failed")).finally(() => {
|
|
3538
|
+
for (const record of records) {
|
|
3539
|
+
if (record.runMode !== "job" || !executor.get(record.vmId)) continue;
|
|
3540
|
+
try {
|
|
3541
|
+
executor.onExit(
|
|
3542
|
+
record.vmId,
|
|
3543
|
+
(result) => socket.emit(WorkerEvent.VM_RUN_EXITED, { vmId: record.vmId, ...result })
|
|
3544
|
+
);
|
|
3545
|
+
} catch (error) {
|
|
3546
|
+
src_default.warn(
|
|
3547
|
+
{ vmId: record.vmId, err: error },
|
|
3548
|
+
"reattached job vm has no onExit support"
|
|
3549
|
+
);
|
|
3550
|
+
}
|
|
3551
|
+
}
|
|
3552
|
+
startHeartbeat();
|
|
3553
|
+
});
|
|
2766
3554
|
});
|
|
2767
3555
|
});
|
|
2768
3556
|
socket.on("disconnect", (reason) => {
|
|
@@ -2773,7 +3561,10 @@ function connectToSandbox(config, executor) {
|
|
|
2773
3561
|
socket.off(WorkerEvent.REATTACH);
|
|
2774
3562
|
});
|
|
2775
3563
|
socket.on("connect_error", (error) => {
|
|
2776
|
-
src_default.error({
|
|
3564
|
+
src_default.error({
|
|
3565
|
+
err: error,
|
|
3566
|
+
sandboxUrl: config.sandboxUrl
|
|
3567
|
+
}, "sandbox connection error");
|
|
2777
3568
|
});
|
|
2778
3569
|
registerCommandHandlers(socket, executor);
|
|
2779
3570
|
return socket;
|
|
@@ -2807,7 +3598,11 @@ var VncRelay = class {
|
|
|
2807
3598
|
const rfbc = this.rfbc;
|
|
2808
3599
|
rfbc.on("connect", () => {
|
|
2809
3600
|
rfbc.autoUpdate = true;
|
|
2810
|
-
this.socket.emit(WorkerEvent.VNC_INIT, {
|
|
3601
|
+
this.socket.emit(WorkerEvent.VNC_INIT, {
|
|
3602
|
+
vmId: this.vmId,
|
|
3603
|
+
width: rfbc.width,
|
|
3604
|
+
height: rfbc.height
|
|
3605
|
+
});
|
|
2811
3606
|
this.interval = setInterval(() => {
|
|
2812
3607
|
rfbc.requestUpdate(false, 0, 0, rfbc.width, rfbc.height);
|
|
2813
3608
|
}, 300);
|
|
@@ -2842,8 +3637,15 @@ var VncRelay = class {
|
|
|
2842
3637
|
}
|
|
2843
3638
|
});
|
|
2844
3639
|
rfbc.on("resize", (rect) => {
|
|
2845
|
-
src_default.info(
|
|
2846
|
-
|
|
3640
|
+
src_default.info(
|
|
3641
|
+
{ vmId: this.vmId, width: rect.width, height: rect.height },
|
|
3642
|
+
"[VNC] desktop resized"
|
|
3643
|
+
);
|
|
3644
|
+
this.socket.emit(WorkerEvent.VNC_INIT, {
|
|
3645
|
+
vmId: this.vmId,
|
|
3646
|
+
width: rect.width,
|
|
3647
|
+
height: rect.height
|
|
3648
|
+
});
|
|
2847
3649
|
rfbc.requestUpdate(false, 0, 0, rect.width, rect.height);
|
|
2848
3650
|
});
|
|
2849
3651
|
rfbc.on("error", (err) => src_default.error({ vmId: this.vmId, err }, "[VNC] RFB error"));
|
|
@@ -2904,7 +3706,10 @@ var SerialRelay = class {
|
|
|
2904
3706
|
sock.once("connect", () => {
|
|
2905
3707
|
this.conn = sock;
|
|
2906
3708
|
src_default.info({ vmId: this.vmId, port }, "[Serial] TCP connected");
|
|
2907
|
-
sock.on(
|
|
3709
|
+
sock.on(
|
|
3710
|
+
"data",
|
|
3711
|
+
(d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
|
|
3712
|
+
);
|
|
2908
3713
|
sock.on("end", () => src_default.info({ vmId: this.vmId }, "[Serial] TCP ended"));
|
|
2909
3714
|
sock.on("error", (err) => src_default.error({ vmId: this.vmId, err }, "[Serial] TCP error"));
|
|
2910
3715
|
});
|
|
@@ -2914,8 +3719,14 @@ var SerialRelay = class {
|
|
|
2914
3719
|
}
|
|
2915
3720
|
connectPipe() {
|
|
2916
3721
|
const { process: proc } = this.source;
|
|
2917
|
-
proc.stdout?.on(
|
|
2918
|
-
|
|
3722
|
+
proc.stdout?.on(
|
|
3723
|
+
"data",
|
|
3724
|
+
(d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
|
|
3725
|
+
);
|
|
3726
|
+
proc.stderr?.on(
|
|
3727
|
+
"data",
|
|
3728
|
+
(d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
|
|
3729
|
+
);
|
|
2919
3730
|
}
|
|
2920
3731
|
destroy() {
|
|
2921
3732
|
if (this.inputHandler) this.socket.off(WorkerEvent.TERMINAL_INPUT, this.inputHandler);
|
|
@@ -2959,10 +3770,93 @@ function attachConsoleRelays(executor, sandboxSocket) {
|
|
|
2959
3770
|
};
|
|
2960
3771
|
}
|
|
2961
3772
|
|
|
3773
|
+
// src/prepare-images.ts
|
|
3774
|
+
var import_child_process8 = require("child_process");
|
|
3775
|
+
var import_util4 = require("util");
|
|
3776
|
+
var execFileAsync3 = (0, import_util4.promisify)(import_child_process8.execFile);
|
|
3777
|
+
function artifactImageLocation(env = process.env) {
|
|
3778
|
+
const bucket = env.S3_BUCKET_ARTIFACTS?.trim();
|
|
3779
|
+
if (!bucket) return void 0;
|
|
3780
|
+
if (bucket.includes("/") || bucket.includes("://")) {
|
|
3781
|
+
throw new Error("S3_BUCKET_ARTIFACTS must contain only the bucket name (no scheme or path)");
|
|
3782
|
+
}
|
|
3783
|
+
const version2 = env.NODE_ENV === "development" || env.BUILDER_ENV === "local" ? "local" : version;
|
|
3784
|
+
return `s3://${bucket}/${version2}/qemu`;
|
|
3785
|
+
}
|
|
3786
|
+
function allStacksAvailable(imagesDir) {
|
|
3787
|
+
const available = new Set(availableStacks(imagesDir));
|
|
3788
|
+
return PUBLISHED_IMAGE_STACKS.every((stack) => available.has(stack));
|
|
3789
|
+
}
|
|
3790
|
+
async function dockerAvailable(bin) {
|
|
3791
|
+
try {
|
|
3792
|
+
await execFileAsync3(bin, ["info"]);
|
|
3793
|
+
return true;
|
|
3794
|
+
} catch {
|
|
3795
|
+
return false;
|
|
3796
|
+
}
|
|
3797
|
+
}
|
|
3798
|
+
async function dockerImageExists(bin, image) {
|
|
3799
|
+
try {
|
|
3800
|
+
await execFileAsync3(bin, ["image", "inspect", image]);
|
|
3801
|
+
return true;
|
|
3802
|
+
} catch {
|
|
3803
|
+
return false;
|
|
3804
|
+
}
|
|
3805
|
+
}
|
|
3806
|
+
async function prepareWorkerImages(config, env = process.env) {
|
|
3807
|
+
const vmImagesUrl = artifactImageLocation(env);
|
|
3808
|
+
const refreshVmImages = env.VM_IMAGES_REFRESH === "true";
|
|
3809
|
+
if (vmImagesUrl) {
|
|
3810
|
+
const destinations = /* @__PURE__ */ new Set([config.qemu.imagesDir, config.firecracker.imagesDir]);
|
|
3811
|
+
for (const imagesDir of destinations) {
|
|
3812
|
+
if (!refreshVmImages && allStacksAvailable(imagesDir)) continue;
|
|
3813
|
+
await pullImages({
|
|
3814
|
+
location: vmImagesUrl,
|
|
3815
|
+
arch: hostArch(),
|
|
3816
|
+
endpoint: env.S3_ARTIFACTS_ENDPOINT
|
|
3817
|
+
}, imagesDir);
|
|
3818
|
+
}
|
|
3819
|
+
}
|
|
3820
|
+
const shouldPullDocker = env.DOCKER_PULL_IMAGES === "true" || env.DOCKER_PULL_IMAGES !== "false" && config.docker.imagePrefix.includes("/");
|
|
3821
|
+
if (!shouldPullDocker || !await dockerAvailable(config.docker.bin)) return;
|
|
3822
|
+
for (const stack of PUBLISHED_IMAGE_STACKS) {
|
|
3823
|
+
const image = config.docker.stackImages[stack] ?? `${config.docker.imagePrefix}-${stack}:${config.docker.imageTag}`;
|
|
3824
|
+
if (!await dockerImageExists(config.docker.bin, image)) {
|
|
3825
|
+
await execFileAsync3(config.docker.bin, ["pull", image]);
|
|
3826
|
+
}
|
|
3827
|
+
}
|
|
3828
|
+
}
|
|
3829
|
+
|
|
3830
|
+
// src/health-server.ts
|
|
3831
|
+
var import_http7 = __toESM(require("http"));
|
|
3832
|
+
async function startSandboxHealthServer(options) {
|
|
3833
|
+
const server = import_http7.default.createServer((req, res) => {
|
|
3834
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
3835
|
+
if (pathname === `${options.pathPrefix}/healthz`) {
|
|
3836
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
3837
|
+
res.end(JSON.stringify({ ok: true, status: "alive" }));
|
|
3838
|
+
return;
|
|
3839
|
+
}
|
|
3840
|
+
if (pathname === `${options.pathPrefix}/health`) {
|
|
3841
|
+
const ready = options.isReady();
|
|
3842
|
+
res.writeHead(ready ? 200 : 503, { "content-type": "application/json" });
|
|
3843
|
+
res.end(JSON.stringify({ ok: ready, status: ready ? "ready" : "not-ready" }));
|
|
3844
|
+
return;
|
|
3845
|
+
}
|
|
3846
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
3847
|
+
res.end(JSON.stringify({ ok: false, error: "not found" }));
|
|
3848
|
+
});
|
|
3849
|
+
await new Promise((resolve, reject) => {
|
|
3850
|
+
server.listen(options.port, options.host, resolve);
|
|
3851
|
+
server.once("error", reject);
|
|
3852
|
+
});
|
|
3853
|
+
return server;
|
|
3854
|
+
}
|
|
3855
|
+
|
|
2962
3856
|
// src/cli.ts
|
|
2963
3857
|
var program = new import_commander.Command();
|
|
2964
3858
|
program.name("sandbox").description("Run and manage a Multiplayer sandbox worker").version(version);
|
|
2965
|
-
program.command("start").description("start a worker and connect it to the sandbox control plane").option("--token <token>", "worker authentication token (env: WORKER_TOKEN)").option("--worker-id <id>", "
|
|
3859
|
+
program.command("start").description("start a worker and connect it to the sandbox control plane").option("--token <token>", "worker authentication token (env: WORKER_TOKEN)").option("--worker-id <id>", "worker pool/authentication ID (env: WORKER_ID; default: hostname)").option("--sandbox-url <url>", "control-plane base URL (env: SANDBOX_URL; default: https://api.sandbox.filingramp.com/v0)").option("--max-vms <count>", "maximum concurrent VMs (env: MAX_VMS)").action(async (opts) => {
|
|
2966
3860
|
const config = loadWorkerConfigFromEnv({
|
|
2967
3861
|
...process.env,
|
|
2968
3862
|
...opts.token ? { WORKER_TOKEN: opts.token } : {},
|
|
@@ -2970,8 +3864,17 @@ program.command("start").description("start a worker and connect it to the sandb
|
|
|
2970
3864
|
...opts.sandboxUrl ? { SANDBOX_URL: opts.sandboxUrl } : {},
|
|
2971
3865
|
...opts.maxVms ? { MAX_VMS: opts.maxVms } : {}
|
|
2972
3866
|
});
|
|
3867
|
+
const socketRef = {};
|
|
3868
|
+
const healthServer = await startSandboxHealthServer({
|
|
3869
|
+
host: config.healthHost,
|
|
3870
|
+
port: config.healthPort,
|
|
3871
|
+
pathPrefix: config.healthPathPrefix,
|
|
3872
|
+
isReady: () => socketRef.current?.connected === true
|
|
3873
|
+
});
|
|
3874
|
+
await prepareWorkerImages(config);
|
|
2973
3875
|
const executor = createExecutor(config);
|
|
2974
3876
|
const socket = connectToSandbox(config, executor);
|
|
3877
|
+
socketRef.current = socket;
|
|
2975
3878
|
attachConsoleRelays(executor, socket);
|
|
2976
3879
|
let stopping = false;
|
|
2977
3880
|
const stop = async (signal) => {
|
|
@@ -2980,6 +3883,7 @@ program.command("start").description("start a worker and connect it to the sandb
|
|
|
2980
3883
|
console.log(`
|
|
2981
3884
|
${signal} received; shutting down sandbox worker...`);
|
|
2982
3885
|
socket.disconnect();
|
|
3886
|
+
healthServer.close();
|
|
2983
3887
|
const results = await Promise.allSettled(executor.list().map((vm) => executor.destroy(vm.vmId)));
|
|
2984
3888
|
const failed = results.filter((result) => result.status === "rejected");
|
|
2985
3889
|
if (failed.length) console.error(`Failed to destroy ${failed.length} VM(s) during shutdown`);
|
|
@@ -2991,7 +3895,7 @@ ${signal} received; shutting down sandbox worker...`);
|
|
|
2991
3895
|
process.once("SIGTERM", () => {
|
|
2992
3896
|
void stop("SIGTERM");
|
|
2993
3897
|
});
|
|
2994
|
-
|
|
3898
|
+
src_default.info(`Starting sandbox worker ${config.workerId} \u2192 ${config.sandboxUrl}`);
|
|
2995
3899
|
});
|
|
2996
3900
|
program.command("detect").description("probe this host and report which micro-VM engines it can run").action(() => {
|
|
2997
3901
|
const probes = probeEngines(loadWorkerConfigFromEnv({ ...process.env, WORKER_ID: "-", WORKER_TOKEN: "-" }));
|
|
@@ -3001,13 +3905,13 @@ program.command("detect").description("probe this host and report which micro-VM
|
|
|
3001
3905
|
}
|
|
3002
3906
|
});
|
|
3003
3907
|
var images = program.command("images").description("manage engine base images (kernel + initramfs for QEMU/Firecracker)");
|
|
3004
|
-
images.command("pull").description("download per-stack base images from S3 or an HTTPS mirror into the standard location").requiredOption("--from <location>", "s3://bucket[/prefix] or https:// base URL (
|
|
3908
|
+
images.command("pull").description("download per-stack base images from S3 or an HTTPS mirror into the standard location").requiredOption("--from <location>", "s3://bucket[/prefix] or https:// base URL (default derived from S3_BUCKET_ARTIFACTS)", artifactImageLocation()).option("--arch <arch>", "image architecture", hostArch()).option("--stack <stack...>", "stacks to pull (default: all)").option("--dest <dir>", "destination directory (default: ~/.multiplayer/vm-images/<arch>)").option("--endpoint <url>", "custom S3 endpoint, e.g. a MinIO URL (env: S3_ARTIFACTS_ENDPOINT)", process.env.S3_ARTIFACTS_ENDPOINT).action(async (opts) => {
|
|
3005
3909
|
const dest = opts.dest ?? defaultImagesDir(opts.arch);
|
|
3006
3910
|
const files = await pullImages({ location: opts.from, arch: opts.arch, stacks: opts.stack, endpoint: opts.endpoint }, dest);
|
|
3007
3911
|
for (const file of files) console.log(`pulled ${file}`);
|
|
3008
3912
|
console.log('\nbase images ready \u2014 "sandbox detect" shows which engines they enable');
|
|
3009
3913
|
});
|
|
3010
|
-
images.command("push").description("upload locally built per-stack base images (scripts/setup-vm-images.sh) to S3").requiredOption("--to <location>", "s3://bucket[/prefix] (
|
|
3914
|
+
images.command("push").description("upload locally built per-stack base images (scripts/setup-vm-images.sh) to S3").requiredOption("--to <location>", "s3://bucket[/prefix] (default derived from S3_BUCKET_ARTIFACTS)", artifactImageLocation()).option("--arch <arch>", "image architecture", hostArch()).option("--stack <stack...>", "stacks to push (default: all found locally)").option("--src <dir>", "source directory (default: ~/.multiplayer/vm-images/<arch>)").option("--endpoint <url>", "custom S3 endpoint, e.g. a MinIO URL (env: S3_ARTIFACTS_ENDPOINT)", process.env.S3_ARTIFACTS_ENDPOINT).action(async (opts) => {
|
|
3011
3915
|
const src = opts.src ?? defaultImagesDir(opts.arch);
|
|
3012
3916
|
const uploaded = await pushImages({ location: opts.to, arch: opts.arch, stacks: opts.stack, endpoint: opts.endpoint }, src);
|
|
3013
3917
|
for (const uri of uploaded) console.log(`pushed ${uri}`);
|