@multiplayer-app/sandbox 6.0.0-sandbox-docker-image.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -6
- package/dist/cli.js +1056 -152
- package/dist/cli.js.map +4 -4
- package/dist/config.d.ts +5 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/executor/detect.d.ts.map +1 -1
- package/dist/executor/docker-executor.d.ts +16 -2
- package/dist/executor/docker-executor.d.ts.map +1 -1
- package/dist/executor/docker-sandbox-executor.d.ts +13 -1
- package/dist/executor/docker-sandbox-executor.d.ts.map +1 -1
- package/dist/executor/env-encoding.d.ts +9 -0
- package/dist/executor/env-encoding.d.ts.map +1 -0
- package/dist/executor/fargate-executor.d.ts +13 -1
- package/dist/executor/fargate-executor.d.ts.map +1 -1
- package/dist/executor/firecracker-executor.d.ts +7 -1
- package/dist/executor/firecracker-executor.d.ts.map +1 -1
- package/dist/executor/index.d.ts +2 -1
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/lambda-microvm-executor.d.ts +9 -0
- package/dist/executor/lambda-microvm-executor.d.ts.map +1 -1
- package/dist/executor/mock-executor.d.ts +4 -1
- package/dist/executor/mock-executor.d.ts.map +1 -1
- package/dist/executor/qemu-executor.d.ts +7 -1
- package/dist/executor/qemu-executor.d.ts.map +1 -1
- package/dist/executor/remote-executor.d.ts +10 -1
- package/dist/executor/remote-executor.d.ts.map +1 -1
- package/dist/executor/types.d.ts +12 -1
- package/dist/executor/types.d.ts.map +1 -1
- package/dist/health-server.d.ts +10 -0
- package/dist/health-server.d.ts.map +1 -0
- package/dist/images.d.ts +2 -0
- package/dist/images.d.ts.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +991 -150
- package/dist/index.js.map +4 -4
- package/dist/prepare-images.d.ts +14 -0
- package/dist/prepare-images.d.ts.map +1 -0
- package/dist/services/serial-relay.d.ts.map +1 -1
- package/dist/services/vnc-relay.d.ts.map +1 -1
- package/dist/socket-client.d.ts.map +1 -1
- package/dist/vendor/builder-types/access.d.ts +1 -1
- package/dist/vendor/builder-types/deployment.d.ts +58 -0
- package/dist/vendor/builder-types/deployment.d.ts.map +1 -0
- package/dist/vendor/builder-types/events.d.ts +6 -1
- package/dist/vendor/builder-types/events.d.ts.map +1 -1
- package/dist/vendor/builder-types/index.d.ts +1 -0
- package/dist/vendor/builder-types/index.d.ts.map +1 -1
- package/dist/vendor/builder-types/release.d.ts +7 -3
- package/dist/vendor/builder-types/release.d.ts.map +1 -1
- package/dist/vendor/builder-types/vm.d.ts +28 -1
- package/dist/vendor/builder-types/vm.d.ts.map +1 -1
- package/package.json +6 -1
package/dist/index.js
CHANGED
|
@@ -32,6 +32,7 @@ var index_exports = {};
|
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
IMAGE_FILES: () => IMAGE_FILES,
|
|
34
34
|
MultiEngineExecutor: () => MultiEngineExecutor,
|
|
35
|
+
PUBLISHED_IMAGE_STACKS: () => PUBLISHED_IMAGE_STACKS,
|
|
35
36
|
SerialRelay: () => SerialRelay,
|
|
36
37
|
VncRelay: () => VncRelay,
|
|
37
38
|
WORKER_VERSION: () => version,
|
|
@@ -47,7 +48,8 @@ __export(index_exports, {
|
|
|
47
48
|
probeEngines: () => probeEngines,
|
|
48
49
|
pullImages: () => pullImages,
|
|
49
50
|
pushImages: () => pushImages,
|
|
50
|
-
resolveStackImages: () => resolveStackImages
|
|
51
|
+
resolveStackImages: () => resolveStackImages,
|
|
52
|
+
startSandboxHealthServer: () => startSandboxHealthServer
|
|
51
53
|
});
|
|
52
54
|
module.exports = __toCommonJS(index_exports);
|
|
53
55
|
|
|
@@ -56,7 +58,7 @@ var import_os2 = __toESM(require("os"));
|
|
|
56
58
|
var import_path2 = __toESM(require("path"));
|
|
57
59
|
|
|
58
60
|
// package.json
|
|
59
|
-
var version = "
|
|
61
|
+
var version = "7.0.0";
|
|
60
62
|
|
|
61
63
|
// src/images.ts
|
|
62
64
|
var import_fs = __toESM(require("fs"));
|
|
@@ -72,6 +74,12 @@ var VM_STACKS = ["generic", "node", "python", "static"];
|
|
|
72
74
|
var WorkerEvent = {
|
|
73
75
|
// worker → sandbox
|
|
74
76
|
HEARTBEAT: "worker:heartbeat",
|
|
77
|
+
// worker → sandbox, fire-and-forget like HEARTBEAT — sent once when a 'job' VM's
|
|
78
|
+
// entrypoint exits on its own (the worker has already torn the resource down by then).
|
|
79
|
+
VM_RUN_EXITED: "vm:run-exited",
|
|
80
|
+
// sandbox → worker, generated by the control plane and reused by this worker process
|
|
81
|
+
// on Socket.IO reconnects. This keeps duplicate configured WORKER_ID values isolated.
|
|
82
|
+
ASSIGNED_ID: "worker:assigned-id",
|
|
75
83
|
// sandbox → worker, once right after auth on every (re)connect — the VMs sandbox's
|
|
76
84
|
// records say this workerId owns, so the worker can try to reattach before its first
|
|
77
85
|
// heartbeat. Always sent, even with an empty array, as a handshake signal.
|
|
@@ -107,6 +115,7 @@ var VM_ACTION_EVENTS = {
|
|
|
107
115
|
|
|
108
116
|
// src/images.ts
|
|
109
117
|
var IMAGE_FILES = ["vmlinuz", "initramfs.img"];
|
|
118
|
+
var PUBLISHED_IMAGE_STACKS = VM_STACKS.filter((stack) => stack !== "generic");
|
|
110
119
|
function hostArch() {
|
|
111
120
|
switch (process.arch) {
|
|
112
121
|
case "arm64":
|
|
@@ -165,7 +174,7 @@ async function downloadFromHttp(opts, stack, file, dest) {
|
|
|
165
174
|
async function pullImages(opts, destDir) {
|
|
166
175
|
const isS3 = opts.location.startsWith("s3://");
|
|
167
176
|
const downloaded = [];
|
|
168
|
-
for (const stack of opts.stacks ??
|
|
177
|
+
for (const stack of opts.stacks ?? PUBLISHED_IMAGE_STACKS) {
|
|
169
178
|
const stackDir = import_path.default.join(destDir, stack);
|
|
170
179
|
import_fs.default.mkdirSync(stackDir, { recursive: true });
|
|
171
180
|
for (const file of IMAGE_FILES) {
|
|
@@ -224,18 +233,32 @@ function parseArnList(raw) {
|
|
|
224
233
|
const arns = raw.split(",").map((s) => s.trim()).filter(Boolean);
|
|
225
234
|
return arns.length ? arns : void 0;
|
|
226
235
|
}
|
|
236
|
+
function resolveSandboxEndpoint(rawUrl, explicitSocketPath) {
|
|
237
|
+
const url = new URL(rawUrl);
|
|
238
|
+
const basePath = url.pathname.replace(/\/+$/, "") || "/v0";
|
|
239
|
+
return {
|
|
240
|
+
// Socket.IO treats a URL pathname as a namespace, so connect to the origin
|
|
241
|
+
// and pass the derived HTTP transport path separately.
|
|
242
|
+
sandboxUrl: url.origin,
|
|
243
|
+
workerSocketPath: explicitSocketPath ?? `${basePath}/sapi/workers/ws`
|
|
244
|
+
};
|
|
245
|
+
}
|
|
227
246
|
function loadWorkerConfigFromEnv(env = process.env) {
|
|
228
247
|
const workerId = env.WORKER_ID?.trim() || import_os2.default.hostname();
|
|
229
|
-
const token = env.WORKER_TOKEN;
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
248
|
+
const token = env.WORKER_TOKEN?.trim() || void 0;
|
|
249
|
+
const sandboxEndpoint = resolveSandboxEndpoint(
|
|
250
|
+
env.SANDBOX_URL ?? "https://api.sandbox.filingramp.com/v0",
|
|
251
|
+
env.WORKER_SOCKET_PATH
|
|
252
|
+
);
|
|
233
253
|
return {
|
|
234
254
|
workerId,
|
|
235
255
|
token,
|
|
236
|
-
sandboxUrl:
|
|
237
|
-
workerSocketPath:
|
|
256
|
+
sandboxUrl: sandboxEndpoint.sandboxUrl,
|
|
257
|
+
workerSocketPath: sandboxEndpoint.workerSocketPath,
|
|
238
258
|
maxVms: Number(env.MAX_VMS) || 10,
|
|
259
|
+
healthHost: env.SANDBOX_HEALTH_HOST ?? "0.0.0.0",
|
|
260
|
+
healthPort: Number(env.SANDBOX_HEALTH_PORT) || 3e3,
|
|
261
|
+
healthPathPrefix: (env.API_PREFIX ?? "/v0/api").replace(/\/+$/, ""),
|
|
239
262
|
region: env.WORKER_REGION,
|
|
240
263
|
remote: {
|
|
241
264
|
// No default: an unset AGENT_URL means the remote engine is not configured on this worker
|
|
@@ -293,8 +316,8 @@ function loadWorkerConfigFromEnv(env = process.env) {
|
|
|
293
316
|
bin: env.SBX_BIN ?? "sbx",
|
|
294
317
|
// Bridged from the same local images `docker` (imagePrefix-imageTag) already uses —
|
|
295
318
|
// `docker save` + `sbx template load`, see docker-sandbox-executor.ts.
|
|
296
|
-
imagePrefix: env.SBX_IMAGE_PREFIX ?? "multiplayer-builder-vm",
|
|
297
|
-
imageTag: env.SBX_IMAGE_TAG ?? "local",
|
|
319
|
+
imagePrefix: env.SBX_IMAGE_PREFIX ?? env.DOCKER_IMAGE_PREFIX ?? "multiplayer-builder-vm",
|
|
320
|
+
imageTag: env.SBX_IMAGE_TAG ?? env.DOCKER_IMAGE_TAG ?? "local",
|
|
298
321
|
stackTemplates: parseStackMap(env.SBX_STACK_TEMPLATES),
|
|
299
322
|
// `sbx create shell` doesn't run the image's own CMD, so the engine launches
|
|
300
323
|
// it explicitly — defaults match each Dockerfile's own WORKDIR /app + CMD.
|
|
@@ -340,7 +363,7 @@ var import_os3 = require("os");
|
|
|
340
363
|
// ../logger/src/config.ts
|
|
341
364
|
var NODE_ENV = process.env.NODE_ENV || "development";
|
|
342
365
|
var isProduction = NODE_ENV === "production";
|
|
343
|
-
var APP_NAME = process.env.npm_package_name?.split("/").pop() || "
|
|
366
|
+
var APP_NAME = process.env.APP_NAME || process.env.npm_package_name?.split("/").pop() || "unknown-app";
|
|
344
367
|
var LOG_LEVEL = process.env.LOG_LEVEL || (isProduction ? "info" : "debug");
|
|
345
368
|
|
|
346
369
|
// ../logger/src/logger.ts
|
|
@@ -382,6 +405,7 @@ var CONTENT_TYPES = {
|
|
|
382
405
|
".txt": "text/plain; charset=utf-8",
|
|
383
406
|
".md": "text/plain; charset=utf-8"
|
|
384
407
|
};
|
|
408
|
+
var MOCK_JOB_DURATION_MS = 250;
|
|
385
409
|
function resolveWorkspacePath(workspaceDir, relativePath) {
|
|
386
410
|
const normalized = import_path3.default.normalize(relativePath).replace(/^([/\\])+/, "");
|
|
387
411
|
const resolved = import_path3.default.resolve(workspaceDir, normalized);
|
|
@@ -398,12 +422,16 @@ function startStandInServer(spec, workspaceDir) {
|
|
|
398
422
|
try {
|
|
399
423
|
const filePath = resolveWorkspacePath(workspaceDir, relative);
|
|
400
424
|
const content = await import_promises2.default.readFile(filePath);
|
|
401
|
-
res.writeHead(200, {
|
|
425
|
+
res.writeHead(200, {
|
|
426
|
+
"content-type": CONTENT_TYPES[import_path3.default.extname(filePath).toLowerCase()] ?? "application/octet-stream"
|
|
427
|
+
});
|
|
402
428
|
res.end(content);
|
|
403
429
|
} catch {
|
|
404
430
|
res.writeHead(200, { "content-type": "text/plain" });
|
|
405
|
-
res.end(
|
|
406
|
-
`)
|
|
431
|
+
res.end(
|
|
432
|
+
`mock vm ${spec.vmId} (${spec.stack}) \u2014 session ${spec.sessionId} \u2014 ${req.method} ${req.url}
|
|
433
|
+
`
|
|
434
|
+
);
|
|
407
435
|
}
|
|
408
436
|
})();
|
|
409
437
|
});
|
|
@@ -417,6 +445,7 @@ function startStandInServer(spec, workspaceDir) {
|
|
|
417
445
|
var MockExecutor = class {
|
|
418
446
|
constructor() {
|
|
419
447
|
this.vms = /* @__PURE__ */ new Map();
|
|
448
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
420
449
|
}
|
|
421
450
|
async create(spec) {
|
|
422
451
|
const workspaceDir = await import_promises2.default.mkdtemp(import_path3.default.join(import_os5.default.tmpdir(), `mock-vm-${spec.vmId}-`));
|
|
@@ -426,12 +455,38 @@ var MockExecutor = class {
|
|
|
426
455
|
sessionId: spec.sessionId,
|
|
427
456
|
stack: spec.stack,
|
|
428
457
|
resources: spec.resources,
|
|
429
|
-
running: true
|
|
458
|
+
running: true,
|
|
459
|
+
// No real process to configure — just echoed back for introspection/tests.
|
|
460
|
+
...spec.env ? { metadata: { env: spec.env } } : {}
|
|
430
461
|
};
|
|
431
|
-
|
|
432
|
-
|
|
462
|
+
const runMode = spec.runMode ?? "service";
|
|
463
|
+
this.vms.set(spec.vmId, { handle, server, port, workspaceDir, runMode, startedAt: Date.now() });
|
|
464
|
+
src_default.info({ vmId: spec.vmId, port, workspaceDir, runMode }, "mock vm created");
|
|
433
465
|
return handle;
|
|
434
466
|
}
|
|
467
|
+
/** No real process to watch — just synthesize a successful run shortly after create(). */
|
|
468
|
+
onExit(vmId, handler) {
|
|
469
|
+
const startedAt = this.vms.get(vmId)?.startedAt ?? Date.now();
|
|
470
|
+
this.exitHandlers.set(vmId, handler);
|
|
471
|
+
setTimeout(() => {
|
|
472
|
+
void (async () => {
|
|
473
|
+
const registered = this.exitHandlers.get(vmId);
|
|
474
|
+
this.exitHandlers.delete(vmId);
|
|
475
|
+
if (!registered) return;
|
|
476
|
+
const result = {
|
|
477
|
+
exitCode: 0,
|
|
478
|
+
stdout: `mock job ${vmId} completed
|
|
479
|
+
`,
|
|
480
|
+
stderr: "",
|
|
481
|
+
truncated: false,
|
|
482
|
+
durationMs: Date.now() - startedAt,
|
|
483
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
484
|
+
};
|
|
485
|
+
await this.destroy(vmId);
|
|
486
|
+
registered(result);
|
|
487
|
+
})();
|
|
488
|
+
}, MOCK_JOB_DURATION_MS);
|
|
489
|
+
}
|
|
435
490
|
async pause(vmId) {
|
|
436
491
|
return this.setRunning(vmId, false);
|
|
437
492
|
}
|
|
@@ -476,7 +531,10 @@ var MockExecutor = class {
|
|
|
476
531
|
resolve({
|
|
477
532
|
statusCode: proxyRes.statusCode ?? 502,
|
|
478
533
|
headers: Object.fromEntries(
|
|
479
|
-
Object.entries(proxyRes.headers).map(([key, value]) => [
|
|
534
|
+
Object.entries(proxyRes.headers).map(([key, value]) => [
|
|
535
|
+
key,
|
|
536
|
+
Array.isArray(value) ? value.join(", ") : value ?? ""
|
|
537
|
+
])
|
|
480
538
|
),
|
|
481
539
|
body: Buffer.concat(chunks)
|
|
482
540
|
});
|
|
@@ -494,7 +552,9 @@ var MockExecutor = class {
|
|
|
494
552
|
for (const file of files) {
|
|
495
553
|
const buffer = Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8");
|
|
496
554
|
if (buffer.byteLength > MAX_FILE_BYTES) {
|
|
497
|
-
throw new Error(
|
|
555
|
+
throw new Error(
|
|
556
|
+
`File too large (${buffer.byteLength} bytes, max ${MAX_FILE_BYTES}): ${file.path}`
|
|
557
|
+
);
|
|
498
558
|
}
|
|
499
559
|
const filePath = resolveWorkspacePath(vm.workspaceDir, file.path);
|
|
500
560
|
await import_promises2.default.mkdir(import_path3.default.dirname(filePath), { recursive: true });
|
|
@@ -565,11 +625,22 @@ function withTimeout(promise, ms) {
|
|
|
565
625
|
]);
|
|
566
626
|
}
|
|
567
627
|
|
|
628
|
+
// src/executor/env-encoding.ts
|
|
629
|
+
function encodeEnvB64(env) {
|
|
630
|
+
if (!env || Object.keys(env).length === 0) return void 0;
|
|
631
|
+
const lines = Object.entries(env).map(([key, value]) => `${key}=${value}`).join("\n");
|
|
632
|
+
return Buffer.from(lines).toString("base64");
|
|
633
|
+
}
|
|
634
|
+
|
|
568
635
|
// src/executor/qemu-executor.ts
|
|
636
|
+
var MAX_OUTPUT_BYTES2 = 1024 * 1024;
|
|
637
|
+
var JOB_EXIT_SENTINEL = "__JOB_EXIT_CODE__:";
|
|
569
638
|
var QemuExecutor = class {
|
|
570
639
|
constructor(config) {
|
|
571
640
|
this.config = config;
|
|
572
641
|
this.vms = /* @__PURE__ */ new Map();
|
|
642
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
643
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
573
644
|
this.usedHttpPorts = /* @__PURE__ */ new Set();
|
|
574
645
|
this.usedDisplays = /* @__PURE__ */ new Set();
|
|
575
646
|
// VNC display indices
|
|
@@ -618,12 +689,15 @@ var QemuExecutor = class {
|
|
|
618
689
|
}
|
|
619
690
|
const resolved = this.config.imagesDir ? resolveStackImages(this.config.imagesDir, stack) ?? resolveStackImages(this.config.imagesDir, "generic") : void 0;
|
|
620
691
|
if (!resolved) {
|
|
621
|
-
throw new Error(
|
|
692
|
+
throw new Error(
|
|
693
|
+
`no base image for stack "${stack}" under ${this.config.imagesDir} \u2014 run "worker-cli images pull"`
|
|
694
|
+
);
|
|
622
695
|
}
|
|
623
696
|
return resolved;
|
|
624
697
|
}
|
|
625
698
|
async create(spec) {
|
|
626
699
|
const { kernelPath, initrdPath } = this.resolveImages(spec.stack);
|
|
700
|
+
const envB64 = encodeEnvB64(spec.env);
|
|
627
701
|
const hostPort = this.allocateHttpPort();
|
|
628
702
|
const vncDisplay = this.allocateVncDisplay();
|
|
629
703
|
const serialPort = this.allocateSerialPort();
|
|
@@ -653,7 +727,9 @@ var QemuExecutor = class {
|
|
|
653
727
|
[
|
|
654
728
|
`console=tty0 console=ttyAMA0 net.ifnames=0 init=/init VM_ID=${spec.vmId} STACK=${spec.stack}`,
|
|
655
729
|
spec.sessionId ? `SESSION_ID=${spec.sessionId}` : "",
|
|
656
|
-
spec.appUrl ? `APP_URL_B64=${Buffer.from(spec.appUrl).toString("base64")}` : ""
|
|
730
|
+
spec.appUrl ? `APP_URL_B64=${Buffer.from(spec.appUrl).toString("base64")}` : "",
|
|
731
|
+
spec.runMode === "job" ? "RUN_MODE=job" : "",
|
|
732
|
+
envB64 ? `ENV_B64=${envB64}` : ""
|
|
657
733
|
].filter(Boolean).join(" "),
|
|
658
734
|
// Networking
|
|
659
735
|
"-netdev",
|
|
@@ -678,15 +754,24 @@ var QemuExecutor = class {
|
|
|
678
754
|
src_default.info({ vmId: spec.vmId, bin: this.bin, args }, "spawning qemu");
|
|
679
755
|
const proc = (0, import_child_process2.spawn)(this.bin, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
680
756
|
proc.stdout?.on("data", (d) => src_default.info({ vmId: spec.vmId }, d.toString().trimEnd()));
|
|
681
|
-
proc.stderr?.on(
|
|
757
|
+
proc.stderr?.on(
|
|
758
|
+
"data",
|
|
759
|
+
(d) => src_default.error({ vmId: spec.vmId }, d.toString().trimEnd())
|
|
760
|
+
);
|
|
761
|
+
const runMode = spec.runMode ?? "service";
|
|
762
|
+
const startedAt = Date.now();
|
|
682
763
|
proc.on("exit", (code, signal) => {
|
|
683
764
|
const log = code === 0 || code === null ? src_default.info.bind(src_default) : src_default.error.bind(src_default);
|
|
684
765
|
log({ vmId: spec.vmId, code, signal }, "qemu process exited");
|
|
685
766
|
const vm2 = this.vms.get(spec.vmId);
|
|
686
|
-
if (vm2)
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
767
|
+
if (!vm2) return;
|
|
768
|
+
this.usedHttpPorts.delete(vm2.hostPort);
|
|
769
|
+
this.usedDisplays.delete(vm2.vncDisplay);
|
|
770
|
+
this.usedSerialPorts.delete(vm2.serialPort);
|
|
771
|
+
if (vm2.runMode === "job") {
|
|
772
|
+
vm2.handle = { ...vm2.handle, running: false };
|
|
773
|
+
this.reportJobResult(spec.vmId, vm2);
|
|
774
|
+
} else {
|
|
690
775
|
this.vms.delete(spec.vmId);
|
|
691
776
|
}
|
|
692
777
|
});
|
|
@@ -701,12 +786,61 @@ var QemuExecutor = class {
|
|
|
701
786
|
serialPort,
|
|
702
787
|
metadata
|
|
703
788
|
};
|
|
704
|
-
const vm = {
|
|
789
|
+
const vm = {
|
|
790
|
+
handle,
|
|
791
|
+
process: proc,
|
|
792
|
+
pid: proc.pid,
|
|
793
|
+
hostPort,
|
|
794
|
+
qmpSocketPath,
|
|
795
|
+
vncDisplay,
|
|
796
|
+
serialPort,
|
|
797
|
+
runMode,
|
|
798
|
+
startedAt
|
|
799
|
+
};
|
|
705
800
|
this.vms.set(spec.vmId, vm);
|
|
801
|
+
if (runMode === "job") vm.serialBuffer = connectSerialBuffer(serialPort);
|
|
706
802
|
await waitForQmp(qmpSocketPath);
|
|
707
|
-
src_default.info({ vmId: spec.vmId, hostPort, vncPort, serialPort }, "qemu vm ready");
|
|
803
|
+
src_default.info({ vmId: spec.vmId, hostPort, vncPort, serialPort, runMode }, "qemu vm ready");
|
|
708
804
|
return handle;
|
|
709
805
|
}
|
|
806
|
+
/** Scrapes JOB_EXIT_SENTINEL out of the buffered serial console and reports the result. */
|
|
807
|
+
reportJobResult(vmId, vm) {
|
|
808
|
+
vm.serialBuffer?.socket.destroy();
|
|
809
|
+
const output = Buffer.concat(vm.serialBuffer?.chunks ?? []).toString("utf8");
|
|
810
|
+
const sentinelIndex = output.lastIndexOf(JOB_EXIT_SENTINEL);
|
|
811
|
+
const exitCode = sentinelIndex === -1 ? 1 : Number.parseInt(output.slice(sentinelIndex + JOB_EXIT_SENTINEL.length), 10) || 0;
|
|
812
|
+
const stdout = sentinelIndex === -1 ? output : output.slice(0, sentinelIndex);
|
|
813
|
+
const result = {
|
|
814
|
+
exitCode,
|
|
815
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES2),
|
|
816
|
+
stderr: "",
|
|
817
|
+
truncated: (vm.serialBuffer?.bytes ?? 0) > MAX_OUTPUT_BYTES2,
|
|
818
|
+
durationMs: Date.now() - vm.startedAt,
|
|
819
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
820
|
+
};
|
|
821
|
+
src_default.info({ vmId, exitCode }, "qemu job vm exited");
|
|
822
|
+
this.deliverJobResult(vmId, result);
|
|
823
|
+
}
|
|
824
|
+
deliverJobResult(vmId, result) {
|
|
825
|
+
const handler = this.exitHandlers.get(vmId);
|
|
826
|
+
if (handler) {
|
|
827
|
+
this.exitHandlers.delete(vmId);
|
|
828
|
+
this.vms.delete(vmId);
|
|
829
|
+
handler(result);
|
|
830
|
+
} else {
|
|
831
|
+
this.pendingResults.set(vmId, result);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
onExit(vmId, handler) {
|
|
835
|
+
const pending = this.pendingResults.get(vmId);
|
|
836
|
+
if (pending) {
|
|
837
|
+
this.pendingResults.delete(vmId);
|
|
838
|
+
this.vms.delete(vmId);
|
|
839
|
+
handler(pending);
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
this.exitHandlers.set(vmId, handler);
|
|
843
|
+
}
|
|
710
844
|
async pause(vmId) {
|
|
711
845
|
await this.qmp(vmId, "stop");
|
|
712
846
|
return this.setRunning(vmId, false);
|
|
@@ -793,7 +927,9 @@ var QemuExecutor = class {
|
|
|
793
927
|
hostPort: meta.hostPort,
|
|
794
928
|
qmpSocketPath: meta.qmpSocketPath,
|
|
795
929
|
vncDisplay: meta.vncDisplay,
|
|
796
|
-
serialPort: meta.serialPort
|
|
930
|
+
serialPort: meta.serialPort,
|
|
931
|
+
runMode: record.runMode,
|
|
932
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
797
933
|
});
|
|
798
934
|
src_default.info({ vmId: record.vmId }, "qemu vm reattached");
|
|
799
935
|
return handle;
|
|
@@ -813,10 +949,16 @@ var QemuExecutor = class {
|
|
|
813
949
|
async proxyHttp(vmId, request) {
|
|
814
950
|
const vm = this.vms.get(vmId);
|
|
815
951
|
if (!vm?.handle.running) throw new Error(`vm ${vmId} is not running`);
|
|
816
|
-
const { default:
|
|
952
|
+
const { default: http8 } = await import("http");
|
|
817
953
|
return new Promise((resolve, reject) => {
|
|
818
|
-
const req =
|
|
819
|
-
{
|
|
954
|
+
const req = http8.request(
|
|
955
|
+
{
|
|
956
|
+
host: "127.0.0.1",
|
|
957
|
+
port: vm.hostPort,
|
|
958
|
+
method: request.method,
|
|
959
|
+
path: request.path,
|
|
960
|
+
headers: request.headers
|
|
961
|
+
},
|
|
820
962
|
(res) => {
|
|
821
963
|
const chunks = [];
|
|
822
964
|
res.on("data", (c) => chunks.push(c));
|
|
@@ -825,7 +967,10 @@ var QemuExecutor = class {
|
|
|
825
967
|
() => resolve({
|
|
826
968
|
statusCode: res.statusCode ?? 502,
|
|
827
969
|
headers: Object.fromEntries(
|
|
828
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
970
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
971
|
+
k,
|
|
972
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
973
|
+
])
|
|
829
974
|
),
|
|
830
975
|
body: Buffer.concat(chunks)
|
|
831
976
|
})
|
|
@@ -849,11 +994,27 @@ var QemuExecutor = class {
|
|
|
849
994
|
await sendQmpCommand(vm.qmpSocketPath, execute);
|
|
850
995
|
}
|
|
851
996
|
};
|
|
997
|
+
function connectSerialBuffer(port) {
|
|
998
|
+
const buffer = {
|
|
999
|
+
chunks: [],
|
|
1000
|
+
bytes: 0,
|
|
1001
|
+
socket: import_net.default.connect({ host: "127.0.0.1", port })
|
|
1002
|
+
};
|
|
1003
|
+
buffer.socket.on("data", (chunk) => {
|
|
1004
|
+
if (buffer.bytes >= MAX_OUTPUT_BYTES2) return;
|
|
1005
|
+
buffer.chunks.push(chunk);
|
|
1006
|
+
buffer.bytes += chunk.byteLength;
|
|
1007
|
+
});
|
|
1008
|
+
buffer.socket.on("error", () => {
|
|
1009
|
+
});
|
|
1010
|
+
return buffer;
|
|
1011
|
+
}
|
|
852
1012
|
function waitForQmp(socketPath, timeoutMs = 3e4) {
|
|
853
1013
|
return new Promise((resolve, reject) => {
|
|
854
1014
|
const deadline = Date.now() + timeoutMs;
|
|
855
1015
|
const attempt = () => {
|
|
856
|
-
if (Date.now() > deadline)
|
|
1016
|
+
if (Date.now() > deadline)
|
|
1017
|
+
return reject(new Error(`timed out waiting for QMP socket at ${socketPath}`));
|
|
857
1018
|
const sock = import_net.default.createConnection(socketPath);
|
|
858
1019
|
sock.once("connect", () => {
|
|
859
1020
|
sock.destroy();
|
|
@@ -915,11 +1076,15 @@ var import_path5 = __toESM(require("path"));
|
|
|
915
1076
|
var import_promises4 = __toESM(require("fs/promises"));
|
|
916
1077
|
var import_os7 = __toESM(require("os"));
|
|
917
1078
|
var execAsync = (0, import_util.promisify)(import_child_process4.exec);
|
|
1079
|
+
var MAX_OUTPUT_BYTES3 = 1024 * 1024;
|
|
1080
|
+
var JOB_EXIT_SENTINEL2 = "__JOB_EXIT_CODE__:";
|
|
918
1081
|
var FirecrackerExecutor = class {
|
|
919
1082
|
constructor(config) {
|
|
920
1083
|
this.config = config;
|
|
921
1084
|
this.vms = /* @__PURE__ */ new Map();
|
|
922
1085
|
this.usedSubnets = /* @__PURE__ */ new Set();
|
|
1086
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
1087
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
923
1088
|
this.tapCounter = 0;
|
|
924
1089
|
this.networkReady = false;
|
|
925
1090
|
this.bin = config.bin ?? "firecracker";
|
|
@@ -942,7 +1107,9 @@ var FirecrackerExecutor = class {
|
|
|
942
1107
|
"iptables -t nat -C POSTROUTING -s 172.16.0.0/12 ! -d 172.16.0.0/12 -j MASQUERADE"
|
|
943
1108
|
).catch(() => null);
|
|
944
1109
|
if (!alreadySet) {
|
|
945
|
-
await execAsync(
|
|
1110
|
+
await execAsync(
|
|
1111
|
+
"iptables -t nat -A POSTROUTING -s 172.16.0.0/12 ! -d 172.16.0.0/12 -j MASQUERADE"
|
|
1112
|
+
);
|
|
946
1113
|
}
|
|
947
1114
|
this.networkReady = true;
|
|
948
1115
|
}
|
|
@@ -953,7 +1120,9 @@ var FirecrackerExecutor = class {
|
|
|
953
1120
|
}
|
|
954
1121
|
const resolved = this.config.imagesDir ? resolveStackImages(this.config.imagesDir, stack) ?? resolveStackImages(this.config.imagesDir, "generic") : void 0;
|
|
955
1122
|
if (!resolved) {
|
|
956
|
-
throw new Error(
|
|
1123
|
+
throw new Error(
|
|
1124
|
+
`no base image for stack "${stack}" under ${this.config.imagesDir} \u2014 run "worker-cli images pull"`
|
|
1125
|
+
);
|
|
957
1126
|
}
|
|
958
1127
|
return resolved;
|
|
959
1128
|
}
|
|
@@ -971,17 +1140,35 @@ var FirecrackerExecutor = class {
|
|
|
971
1140
|
await execAsync(`ip tuntap add dev ${tapName} mode tap`);
|
|
972
1141
|
await execAsync(`ip addr add ${hostIp}/30 dev ${tapName}`);
|
|
973
1142
|
await execAsync(`ip link set ${tapName} up`);
|
|
1143
|
+
const runMode = spec.runMode ?? "service";
|
|
1144
|
+
const envB64 = encodeEnvB64(spec.env);
|
|
1145
|
+
const startedAt = Date.now();
|
|
1146
|
+
const serialBuffer = runMode === "job" ? { chunks: [], bytes: 0 } : void 0;
|
|
974
1147
|
const proc = (0, import_child_process3.spawn)(this.bin, ["--api-sock", apiSocketPath], {
|
|
975
1148
|
stdio: ["pipe", "pipe", "pipe"]
|
|
976
1149
|
});
|
|
977
|
-
proc.stdout?.on("data", (d) =>
|
|
978
|
-
|
|
1150
|
+
proc.stdout?.on("data", (d) => {
|
|
1151
|
+
src_default.debug({ vmId: spec.vmId }, d.toString().trimEnd());
|
|
1152
|
+
if (serialBuffer && serialBuffer.bytes < MAX_OUTPUT_BYTES3) {
|
|
1153
|
+
serialBuffer.chunks.push(d);
|
|
1154
|
+
serialBuffer.bytes += d.byteLength;
|
|
1155
|
+
}
|
|
1156
|
+
});
|
|
1157
|
+
proc.stderr?.on(
|
|
1158
|
+
"data",
|
|
1159
|
+
(d) => src_default.debug({ vmId: spec.vmId }, d.toString().trimEnd())
|
|
1160
|
+
);
|
|
979
1161
|
proc.on("exit", (code, signal) => {
|
|
980
1162
|
src_default.info({ vmId: spec.vmId, code, signal }, "firecracker process exited");
|
|
981
1163
|
const vm = this.vms.get(spec.vmId);
|
|
982
1164
|
if (vm) {
|
|
983
1165
|
this.usedSubnets.delete(vm.subnet);
|
|
984
|
-
|
|
1166
|
+
if (vm.runMode === "job") {
|
|
1167
|
+
vm.handle = { ...vm.handle, running: false };
|
|
1168
|
+
this.reportJobResult(spec.vmId, vm);
|
|
1169
|
+
} else {
|
|
1170
|
+
this.vms.delete(spec.vmId);
|
|
1171
|
+
}
|
|
985
1172
|
}
|
|
986
1173
|
execAsync(`ip link del ${tapName}`).catch(() => {
|
|
987
1174
|
});
|
|
@@ -1008,7 +1195,9 @@ var FirecrackerExecutor = class {
|
|
|
1008
1195
|
// base64: kernel cmdline is space-separated, raw presigned URLs are not safe
|
|
1009
1196
|
...spec.appUrl ? [`APP_URL_B64=${Buffer.from(spec.appUrl).toString("base64")}`] : [],
|
|
1010
1197
|
`GUEST_IP=${guestIp}`,
|
|
1011
|
-
`HOST_IP=${hostIp}
|
|
1198
|
+
`HOST_IP=${hostIp}`,
|
|
1199
|
+
...runMode === "job" ? ["RUN_MODE=job"] : [],
|
|
1200
|
+
...envB64 ? [`ENV_B64=${envB64}`] : []
|
|
1012
1201
|
].join(" ")
|
|
1013
1202
|
});
|
|
1014
1203
|
await fcRequest(apiSocketPath, "PUT", "/network-interfaces/eth0", {
|
|
@@ -1018,7 +1207,7 @@ var FirecrackerExecutor = class {
|
|
|
1018
1207
|
host_dev_name: tapName
|
|
1019
1208
|
});
|
|
1020
1209
|
await fcRequest(apiSocketPath, "PUT", "/actions", { action_type: "InstanceStart" });
|
|
1021
|
-
await waitForHttp(guestIp, 8080);
|
|
1210
|
+
if (runMode !== "job") await waitForHttp(guestIp, 8080);
|
|
1022
1211
|
const metadata = { apiSocketPath, tapName, subnet, guestIp, pid: proc.pid };
|
|
1023
1212
|
const handle = {
|
|
1024
1213
|
vmId: spec.vmId,
|
|
@@ -1028,10 +1217,58 @@ var FirecrackerExecutor = class {
|
|
|
1028
1217
|
running: true,
|
|
1029
1218
|
metadata
|
|
1030
1219
|
};
|
|
1031
|
-
this.vms.set(spec.vmId, {
|
|
1032
|
-
|
|
1220
|
+
this.vms.set(spec.vmId, {
|
|
1221
|
+
handle,
|
|
1222
|
+
process: proc,
|
|
1223
|
+
pid: proc.pid,
|
|
1224
|
+
apiSocketPath,
|
|
1225
|
+
tapName,
|
|
1226
|
+
subnet,
|
|
1227
|
+
guestIp,
|
|
1228
|
+
runMode,
|
|
1229
|
+
startedAt,
|
|
1230
|
+
serialBuffer
|
|
1231
|
+
});
|
|
1232
|
+
src_default.info({ vmId: spec.vmId, guestIp, tapName, runMode }, "firecracker vm ready");
|
|
1033
1233
|
return handle;
|
|
1034
1234
|
}
|
|
1235
|
+
/** Scrapes JOB_EXIT_SENTINEL out of the buffered serial stdout and reports the result. */
|
|
1236
|
+
reportJobResult(vmId, vm) {
|
|
1237
|
+
const output = Buffer.concat(vm.serialBuffer?.chunks ?? []).toString("utf8");
|
|
1238
|
+
const sentinelIndex = output.lastIndexOf(JOB_EXIT_SENTINEL2);
|
|
1239
|
+
const exitCode = sentinelIndex === -1 ? 1 : Number.parseInt(output.slice(sentinelIndex + JOB_EXIT_SENTINEL2.length), 10) || 0;
|
|
1240
|
+
const stdout = sentinelIndex === -1 ? output : output.slice(0, sentinelIndex);
|
|
1241
|
+
const result = {
|
|
1242
|
+
exitCode,
|
|
1243
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES3),
|
|
1244
|
+
stderr: "",
|
|
1245
|
+
truncated: (vm.serialBuffer?.bytes ?? 0) > MAX_OUTPUT_BYTES3,
|
|
1246
|
+
durationMs: Date.now() - vm.startedAt,
|
|
1247
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1248
|
+
};
|
|
1249
|
+
src_default.info({ vmId, exitCode }, "firecracker job vm exited");
|
|
1250
|
+
this.deliverJobResult(vmId, result);
|
|
1251
|
+
}
|
|
1252
|
+
deliverJobResult(vmId, result) {
|
|
1253
|
+
const handler = this.exitHandlers.get(vmId);
|
|
1254
|
+
if (handler) {
|
|
1255
|
+
this.exitHandlers.delete(vmId);
|
|
1256
|
+
this.vms.delete(vmId);
|
|
1257
|
+
handler(result);
|
|
1258
|
+
} else {
|
|
1259
|
+
this.pendingResults.set(vmId, result);
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
onExit(vmId, handler) {
|
|
1263
|
+
const pending = this.pendingResults.get(vmId);
|
|
1264
|
+
if (pending) {
|
|
1265
|
+
this.pendingResults.delete(vmId);
|
|
1266
|
+
this.vms.delete(vmId);
|
|
1267
|
+
handler(pending);
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
this.exitHandlers.set(vmId, handler);
|
|
1271
|
+
}
|
|
1035
1272
|
async pause(vmId) {
|
|
1036
1273
|
await this.patchVmState(vmId, "Paused");
|
|
1037
1274
|
return this.setRunning(vmId, false);
|
|
@@ -1119,9 +1356,14 @@ var FirecrackerExecutor = class {
|
|
|
1119
1356
|
apiSocketPath: meta.apiSocketPath,
|
|
1120
1357
|
tapName: meta.tapName,
|
|
1121
1358
|
subnet: meta.subnet,
|
|
1122
|
-
guestIp: meta.guestIp
|
|
1359
|
+
guestIp: meta.guestIp,
|
|
1360
|
+
runMode: record.runMode,
|
|
1361
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
1123
1362
|
});
|
|
1124
|
-
src_default.info(
|
|
1363
|
+
src_default.info(
|
|
1364
|
+
{ vmId: record.vmId, guestIp: meta.guestIp, tapName: meta.tapName },
|
|
1365
|
+
"firecracker vm reattached"
|
|
1366
|
+
);
|
|
1125
1367
|
return handle;
|
|
1126
1368
|
}
|
|
1127
1369
|
async proxyHttp(vmId, request) {
|
|
@@ -1144,7 +1386,10 @@ var FirecrackerExecutor = class {
|
|
|
1144
1386
|
() => resolve({
|
|
1145
1387
|
statusCode: res.statusCode ?? 502,
|
|
1146
1388
|
headers: Object.fromEntries(
|
|
1147
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
1389
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
1390
|
+
k,
|
|
1391
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
1392
|
+
])
|
|
1148
1393
|
),
|
|
1149
1394
|
body: Buffer.concat(chunks)
|
|
1150
1395
|
})
|
|
@@ -1208,7 +1453,8 @@ function waitForSocket(socketPath, timeoutMs = 1e4) {
|
|
|
1208
1453
|
return new Promise((resolve, reject) => {
|
|
1209
1454
|
const deadline = Date.now() + timeoutMs;
|
|
1210
1455
|
const attempt = () => {
|
|
1211
|
-
if (Date.now() > deadline)
|
|
1456
|
+
if (Date.now() > deadline)
|
|
1457
|
+
return reject(new Error(`timed out waiting for Firecracker socket ${socketPath}`));
|
|
1212
1458
|
const sock = import_net2.default.createConnection(socketPath);
|
|
1213
1459
|
sock.once("connect", () => {
|
|
1214
1460
|
sock.destroy();
|
|
@@ -1223,7 +1469,8 @@ function waitForHttp(host, port, timeoutMs = 6e4) {
|
|
|
1223
1469
|
return new Promise((resolve, reject) => {
|
|
1224
1470
|
const deadline = Date.now() + timeoutMs;
|
|
1225
1471
|
const attempt = () => {
|
|
1226
|
-
if (Date.now() > deadline)
|
|
1472
|
+
if (Date.now() > deadline)
|
|
1473
|
+
return reject(new Error(`timed out waiting for HTTP at ${host}:${port}`));
|
|
1227
1474
|
const req = import_http2.default.get(`http://${host}:${port}/health`, (res) => {
|
|
1228
1475
|
if (res.statusCode && res.statusCode < 500) return resolve();
|
|
1229
1476
|
res.resume();
|
|
@@ -1245,7 +1492,7 @@ var import_util2 = require("util");
|
|
|
1245
1492
|
var import_http3 = __toESM(require("http"));
|
|
1246
1493
|
var DEFAULT_EXEC_TIMEOUT_MS2 = 6e4;
|
|
1247
1494
|
var MAX_EXEC_TIMEOUT_MS2 = 3e5;
|
|
1248
|
-
var
|
|
1495
|
+
var MAX_OUTPUT_BYTES4 = 1024 * 1024;
|
|
1249
1496
|
var MAX_FILE_BYTES2 = 5 * 1024 * 1024;
|
|
1250
1497
|
var WORKSPACE_DIR = "/workspace";
|
|
1251
1498
|
var execFileAsync = (0, import_util2.promisify)(import_child_process5.execFile);
|
|
@@ -1261,6 +1508,9 @@ var DockerExecutor = class {
|
|
|
1261
1508
|
this.config = config;
|
|
1262
1509
|
this.vms = /* @__PURE__ */ new Map();
|
|
1263
1510
|
this.usedPorts = /* @__PURE__ */ new Set();
|
|
1511
|
+
/** Job VMs that exited before onExit() was called to claim the result (create() → onExit() isn't atomic). */
|
|
1512
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
1513
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
1264
1514
|
this.bin = config.bin;
|
|
1265
1515
|
}
|
|
1266
1516
|
allocatePort() {
|
|
@@ -1280,7 +1530,9 @@ var DockerExecutor = class {
|
|
|
1280
1530
|
if (await this.imageExists(stackImage)) return stackImage;
|
|
1281
1531
|
const genericImage = `${imagePrefix}-generic:${imageTag}`;
|
|
1282
1532
|
if (await this.imageExists(genericImage)) return genericImage;
|
|
1283
|
-
throw new Error(
|
|
1533
|
+
throw new Error(
|
|
1534
|
+
`no local docker image for stack "${stack}" (${stackImage}) \u2014 run scripts/setup-docker-images.sh`
|
|
1535
|
+
);
|
|
1284
1536
|
}
|
|
1285
1537
|
async imageExists(image) {
|
|
1286
1538
|
try {
|
|
@@ -1311,6 +1563,7 @@ var DockerExecutor = class {
|
|
|
1311
1563
|
`STACK=${spec.stack}`,
|
|
1312
1564
|
...spec.sessionId ? ["-e", `SESSION_ID=${spec.sessionId}`] : [],
|
|
1313
1565
|
...spec.appUrl ? ["-e", `APP_URL=${spec.appUrl}`] : [],
|
|
1566
|
+
...Object.entries(spec.env ?? {}).flatMap(([key, value]) => ["-e", `${key}=${value}`]),
|
|
1314
1567
|
...this.config.network ? ["--network", this.config.network] : [],
|
|
1315
1568
|
image
|
|
1316
1569
|
];
|
|
@@ -1331,22 +1584,91 @@ var DockerExecutor = class {
|
|
|
1331
1584
|
running: true,
|
|
1332
1585
|
metadata: { containerId, containerName, hostPort }
|
|
1333
1586
|
};
|
|
1334
|
-
|
|
1587
|
+
const runMode = spec.runMode ?? "service";
|
|
1588
|
+
this.vms.set(spec.vmId, {
|
|
1589
|
+
handle,
|
|
1590
|
+
containerId,
|
|
1591
|
+
containerName,
|
|
1592
|
+
hostPort,
|
|
1593
|
+
runMode,
|
|
1594
|
+
startedAt: Date.now()
|
|
1595
|
+
});
|
|
1335
1596
|
this.watchExit(spec.vmId, containerId);
|
|
1336
|
-
await this.waitForAgent(hostPort);
|
|
1337
|
-
src_default.info({ vmId: spec.vmId, containerId, hostPort }, "docker vm ready");
|
|
1597
|
+
if (runMode !== "job") await this.waitForAgent(hostPort);
|
|
1598
|
+
src_default.info({ vmId: spec.vmId, containerId, hostPort, runMode }, "docker vm ready");
|
|
1338
1599
|
return handle;
|
|
1339
1600
|
}
|
|
1340
|
-
/**
|
|
1601
|
+
/**
|
|
1602
|
+
* For 'service' VMs: frees the allocated port once the container exits on its own (crash,
|
|
1603
|
+
* OOM kill, `docker stop` from outside us) — same as before.
|
|
1604
|
+
* For 'job' VMs: captures the exit code + logs, tears the container down, and reports the
|
|
1605
|
+
* result to whoever calls onExit() (immediately if they already have, otherwise once they do).
|
|
1606
|
+
*/
|
|
1341
1607
|
watchExit(vmId, containerId) {
|
|
1342
|
-
execFileAsync(this.bin, ["wait", containerId]).catch(() => {
|
|
1343
|
-
}).then(() => {
|
|
1608
|
+
execFileAsync(this.bin, ["wait", containerId]).catch(() => ({ stdout: "" })).then(async ({ stdout }) => {
|
|
1344
1609
|
const vm = this.vms.get(vmId);
|
|
1345
|
-
if (vm
|
|
1610
|
+
if (!vm || vm.containerId !== containerId) return;
|
|
1611
|
+
if (vm.runMode !== "job") {
|
|
1346
1612
|
vm.handle = { ...vm.handle, running: false };
|
|
1613
|
+
return;
|
|
1347
1614
|
}
|
|
1615
|
+
const exitCode = Number.parseInt(stdout.trim(), 10) || 0;
|
|
1616
|
+
const logs = await this.captureLogs(containerId);
|
|
1617
|
+
await execFileAsync(this.bin, ["rm", "-f", containerId]).catch(
|
|
1618
|
+
(error) => src_default.warn({ vmId, err: error }, "docker rm after job exit failed")
|
|
1619
|
+
);
|
|
1620
|
+
this.usedPorts.delete(vm.hostPort);
|
|
1621
|
+
vm.handle = { ...vm.handle, running: false };
|
|
1622
|
+
const result = {
|
|
1623
|
+
exitCode,
|
|
1624
|
+
...logs,
|
|
1625
|
+
durationMs: Date.now() - vm.startedAt,
|
|
1626
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1627
|
+
};
|
|
1628
|
+
src_default.info({ vmId, exitCode }, "docker job vm exited");
|
|
1629
|
+
this.deliverJobResult(vmId, result);
|
|
1348
1630
|
});
|
|
1349
1631
|
}
|
|
1632
|
+
deliverJobResult(vmId, result) {
|
|
1633
|
+
const handler = this.exitHandlers.get(vmId);
|
|
1634
|
+
if (handler) {
|
|
1635
|
+
this.exitHandlers.delete(vmId);
|
|
1636
|
+
this.vms.delete(vmId);
|
|
1637
|
+
handler(result);
|
|
1638
|
+
} else {
|
|
1639
|
+
this.pendingResults.set(vmId, result);
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
/** `docker logs` mirrors the container's stdout/stderr onto its own, so the CLI's streams stay separate. */
|
|
1643
|
+
async captureLogs(containerId) {
|
|
1644
|
+
try {
|
|
1645
|
+
const { stdout, stderr } = await execFileAsync(this.bin, ["logs", containerId], {
|
|
1646
|
+
maxBuffer: MAX_OUTPUT_BYTES4
|
|
1647
|
+
});
|
|
1648
|
+
return {
|
|
1649
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES4),
|
|
1650
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES4),
|
|
1651
|
+
truncated: stdout.length > MAX_OUTPUT_BYTES4 || stderr.length > MAX_OUTPUT_BYTES4
|
|
1652
|
+
};
|
|
1653
|
+
} catch (error) {
|
|
1654
|
+
const failed = error;
|
|
1655
|
+
return {
|
|
1656
|
+
stdout: (failed.stdout ?? "").slice(0, MAX_OUTPUT_BYTES4),
|
|
1657
|
+
stderr: (failed.stderr ?? "").slice(0, MAX_OUTPUT_BYTES4),
|
|
1658
|
+
truncated: true
|
|
1659
|
+
};
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
onExit(vmId, handler) {
|
|
1663
|
+
const pending = this.pendingResults.get(vmId);
|
|
1664
|
+
if (pending) {
|
|
1665
|
+
this.pendingResults.delete(vmId);
|
|
1666
|
+
this.vms.delete(vmId);
|
|
1667
|
+
handler(pending);
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
this.exitHandlers.set(vmId, handler);
|
|
1671
|
+
}
|
|
1350
1672
|
async pause(vmId) {
|
|
1351
1673
|
const vm = this.mustGet(vmId);
|
|
1352
1674
|
await execFileAsync(this.bin, ["pause", vm.containerId]);
|
|
@@ -1392,7 +1714,12 @@ var DockerExecutor = class {
|
|
|
1392
1714
|
if (!meta?.containerId || meta.hostPort === void 0) return void 0;
|
|
1393
1715
|
let running;
|
|
1394
1716
|
try {
|
|
1395
|
-
const { stdout } = await execFileAsync(this.bin, [
|
|
1717
|
+
const { stdout } = await execFileAsync(this.bin, [
|
|
1718
|
+
"inspect",
|
|
1719
|
+
"-f",
|
|
1720
|
+
"{{.State.Running}}",
|
|
1721
|
+
meta.containerId
|
|
1722
|
+
]);
|
|
1396
1723
|
running = stdout.trim() === "true";
|
|
1397
1724
|
} catch {
|
|
1398
1725
|
return void 0;
|
|
@@ -1410,18 +1737,58 @@ var DockerExecutor = class {
|
|
|
1410
1737
|
handle,
|
|
1411
1738
|
containerId: meta.containerId,
|
|
1412
1739
|
containerName: meta.containerName ?? `multiplayer-vm-${record.vmId}`,
|
|
1413
|
-
hostPort: meta.hostPort
|
|
1740
|
+
hostPort: meta.hostPort,
|
|
1741
|
+
runMode: record.runMode,
|
|
1742
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
1414
1743
|
});
|
|
1415
|
-
if (running)
|
|
1744
|
+
if (running) {
|
|
1745
|
+
this.watchExit(record.vmId, meta.containerId);
|
|
1746
|
+
} else if (record.runMode === "job") {
|
|
1747
|
+
void this.finishExitedJob(record.vmId, meta.containerId);
|
|
1748
|
+
}
|
|
1416
1749
|
src_default.info({ vmId: record.vmId, containerId: meta.containerId }, "docker vm reattached");
|
|
1417
1750
|
return handle;
|
|
1418
1751
|
}
|
|
1752
|
+
/** Same capture-and-report sequence as watchExit's job branch, for a container found already stopped. */
|
|
1753
|
+
async finishExitedJob(vmId, containerId) {
|
|
1754
|
+
const vm = this.vms.get(vmId);
|
|
1755
|
+
if (!vm) return;
|
|
1756
|
+
let exitCode = 0;
|
|
1757
|
+
try {
|
|
1758
|
+
const { stdout } = await execFileAsync(this.bin, [
|
|
1759
|
+
"inspect",
|
|
1760
|
+
"-f",
|
|
1761
|
+
"{{.State.ExitCode}}",
|
|
1762
|
+
containerId
|
|
1763
|
+
]);
|
|
1764
|
+
exitCode = Number.parseInt(stdout.trim(), 10) || 0;
|
|
1765
|
+
} catch {
|
|
1766
|
+
}
|
|
1767
|
+
const logs = await this.captureLogs(containerId);
|
|
1768
|
+
await execFileAsync(this.bin, ["rm", "-f", containerId]).catch(() => {
|
|
1769
|
+
});
|
|
1770
|
+
this.usedPorts.delete(vm.hostPort);
|
|
1771
|
+
vm.handle = { ...vm.handle, running: false };
|
|
1772
|
+
const result = {
|
|
1773
|
+
exitCode,
|
|
1774
|
+
...logs,
|
|
1775
|
+
durationMs: Date.now() - vm.startedAt,
|
|
1776
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1777
|
+
};
|
|
1778
|
+
this.deliverJobResult(vmId, result);
|
|
1779
|
+
}
|
|
1419
1780
|
async proxyHttp(vmId, request) {
|
|
1420
1781
|
const vm = this.mustGet(vmId);
|
|
1421
1782
|
if (!vm.handle.running) throw new Error(`vm ${vmId} is not running`);
|
|
1422
1783
|
return new Promise((resolve, reject) => {
|
|
1423
1784
|
const req = import_http3.default.request(
|
|
1424
|
-
{
|
|
1785
|
+
{
|
|
1786
|
+
host: "127.0.0.1",
|
|
1787
|
+
port: vm.hostPort,
|
|
1788
|
+
method: request.method,
|
|
1789
|
+
path: request.path,
|
|
1790
|
+
headers: request.headers
|
|
1791
|
+
},
|
|
1425
1792
|
(res) => {
|
|
1426
1793
|
const chunks = [];
|
|
1427
1794
|
res.on("data", (c) => chunks.push(c));
|
|
@@ -1430,7 +1797,10 @@ var DockerExecutor = class {
|
|
|
1430
1797
|
() => resolve({
|
|
1431
1798
|
statusCode: res.statusCode ?? 502,
|
|
1432
1799
|
headers: Object.fromEntries(
|
|
1433
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
1800
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
1801
|
+
k,
|
|
1802
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
1803
|
+
])
|
|
1434
1804
|
),
|
|
1435
1805
|
body: Buffer.concat(chunks)
|
|
1436
1806
|
})
|
|
@@ -1451,15 +1821,15 @@ var DockerExecutor = class {
|
|
|
1451
1821
|
(0, import_child_process5.execFile)(
|
|
1452
1822
|
this.bin,
|
|
1453
1823
|
["exec", "-w", cwd, vm.containerId, "/bin/sh", "-c", request.command],
|
|
1454
|
-
{ timeout: timeoutMs, maxBuffer:
|
|
1824
|
+
{ timeout: timeoutMs, maxBuffer: MAX_OUTPUT_BYTES4 },
|
|
1455
1825
|
(error, stdout, stderr) => {
|
|
1456
1826
|
const failed = error;
|
|
1457
1827
|
const timedOut = Boolean(failed?.killed);
|
|
1458
1828
|
const truncated = failed?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
|
|
1459
1829
|
resolve({
|
|
1460
1830
|
exitCode: typeof failed?.code === "number" ? failed.code : failed ? 1 : 0,
|
|
1461
|
-
stdout: stdout.slice(0,
|
|
1462
|
-
stderr: stderr.slice(0,
|
|
1831
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES4),
|
|
1832
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES4),
|
|
1463
1833
|
durationMs: Date.now() - startedAt,
|
|
1464
1834
|
truncated: Boolean(truncated),
|
|
1465
1835
|
timedOut
|
|
@@ -1475,11 +1845,19 @@ var DockerExecutor = class {
|
|
|
1475
1845
|
const relativePath = assertSafeWorkspacePath(file.path);
|
|
1476
1846
|
const buffer = Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8");
|
|
1477
1847
|
if (buffer.byteLength > MAX_FILE_BYTES2) {
|
|
1478
|
-
throw new Error(
|
|
1848
|
+
throw new Error(
|
|
1849
|
+
`File too large (${buffer.byteLength} bytes, max ${MAX_FILE_BYTES2}): ${file.path}`
|
|
1850
|
+
);
|
|
1479
1851
|
}
|
|
1480
1852
|
const dir = relativePath.includes("/") ? relativePath.slice(0, relativePath.lastIndexOf("/")) : "";
|
|
1481
1853
|
if (dir) {
|
|
1482
|
-
await execFileAsync(this.bin, [
|
|
1854
|
+
await execFileAsync(this.bin, [
|
|
1855
|
+
"exec",
|
|
1856
|
+
vm.containerId,
|
|
1857
|
+
"mkdir",
|
|
1858
|
+
"-p",
|
|
1859
|
+
`${WORKSPACE_DIR}/${dir}`
|
|
1860
|
+
]);
|
|
1483
1861
|
}
|
|
1484
1862
|
await this.copyIntoContainer(vm.containerId, `${WORKSPACE_DIR}/${relativePath}`, buffer);
|
|
1485
1863
|
written.push({ path: relativePath, bytes: buffer.byteLength });
|
|
@@ -1492,13 +1870,18 @@ var DockerExecutor = class {
|
|
|
1492
1870
|
const entryName = destPath.replace(/^\//, "");
|
|
1493
1871
|
const tarStream = buildSingleFileTar(entryName, content);
|
|
1494
1872
|
return new Promise((resolve, reject) => {
|
|
1495
|
-
const proc = (0, import_child_process5.spawn)(this.bin, ["cp", "-", `${containerId}:/`], {
|
|
1873
|
+
const proc = (0, import_child_process5.spawn)(this.bin, ["cp", "-", `${containerId}:/`], {
|
|
1874
|
+
stdio: ["pipe", "ignore", "pipe"]
|
|
1875
|
+
});
|
|
1496
1876
|
let stderr = "";
|
|
1497
1877
|
proc.stderr?.on("data", (d) => {
|
|
1498
1878
|
stderr += d.toString();
|
|
1499
1879
|
});
|
|
1500
1880
|
proc.on("error", reject);
|
|
1501
|
-
proc.on(
|
|
1881
|
+
proc.on(
|
|
1882
|
+
"exit",
|
|
1883
|
+
(code) => code === 0 ? resolve() : reject(new Error(`docker cp exited ${code}: ${stderr}`))
|
|
1884
|
+
);
|
|
1502
1885
|
proc.stdin.end(tarStream);
|
|
1503
1886
|
});
|
|
1504
1887
|
}
|
|
@@ -1527,7 +1910,8 @@ var DockerExecutor = class {
|
|
|
1527
1910
|
});
|
|
1528
1911
|
});
|
|
1529
1912
|
if (ok) return;
|
|
1530
|
-
if (Date.now() > deadline)
|
|
1913
|
+
if (Date.now() > deadline)
|
|
1914
|
+
throw new Error(`timed out waiting for docker vm agent on port ${port}`);
|
|
1531
1915
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
1532
1916
|
}
|
|
1533
1917
|
}
|
|
@@ -1539,7 +1923,11 @@ function buildSingleFileTar(name, content) {
|
|
|
1539
1923
|
header.write("0000000\0", 108, "utf8");
|
|
1540
1924
|
header.write("0000000\0", 116, "utf8");
|
|
1541
1925
|
header.write(content.byteLength.toString(8).padStart(11, "0") + "\0", 124, "utf8");
|
|
1542
|
-
header.write(
|
|
1926
|
+
header.write(
|
|
1927
|
+
Math.floor(Date.now() / 1e3).toString(8).padStart(11, "0") + "\0",
|
|
1928
|
+
136,
|
|
1929
|
+
"utf8"
|
|
1930
|
+
);
|
|
1543
1931
|
header.write(" ", 148, "utf8");
|
|
1544
1932
|
header.write("0", 156, "utf8");
|
|
1545
1933
|
header.write("ustar\0", 257, "utf8");
|
|
@@ -1562,7 +1950,7 @@ var import_path6 = __toESM(require("path"));
|
|
|
1562
1950
|
var import_http4 = __toESM(require("http"));
|
|
1563
1951
|
var DEFAULT_EXEC_TIMEOUT_MS3 = 6e4;
|
|
1564
1952
|
var MAX_EXEC_TIMEOUT_MS3 = 3e5;
|
|
1565
|
-
var
|
|
1953
|
+
var MAX_OUTPUT_BYTES5 = 1024 * 1024;
|
|
1566
1954
|
var MAX_FILE_BYTES3 = 5 * 1024 * 1024;
|
|
1567
1955
|
var WORKSPACE_DIR2 = "/workspace";
|
|
1568
1956
|
var AGENT_DIR = "/app";
|
|
@@ -1583,6 +1971,8 @@ var DockerSandboxExecutor = class {
|
|
|
1583
1971
|
this.vms = /* @__PURE__ */ new Map();
|
|
1584
1972
|
this.usedPorts = /* @__PURE__ */ new Set();
|
|
1585
1973
|
this.loadedTemplates = /* @__PURE__ */ new Set();
|
|
1974
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
1975
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
1586
1976
|
this.bin = config.bin;
|
|
1587
1977
|
}
|
|
1588
1978
|
allocatePort() {
|
|
@@ -1614,7 +2004,9 @@ var DockerSandboxExecutor = class {
|
|
|
1614
2004
|
await execFileAsync2("docker", ["save", localImage, "-o", tmpFile]);
|
|
1615
2005
|
await execFileAsync2(this.bin, ["template", "load", tmpFile]);
|
|
1616
2006
|
} catch (error) {
|
|
1617
|
-
throw new Error(
|
|
2007
|
+
throw new Error(
|
|
2008
|
+
`failed to bridge local image "${localImage}" into sbx \u2014 run scripts/setup-docker-images.sh: ${error}`
|
|
2009
|
+
);
|
|
1618
2010
|
} finally {
|
|
1619
2011
|
await import_promises5.default.rm(tmpFile, { force: true });
|
|
1620
2012
|
}
|
|
@@ -1648,12 +2040,14 @@ var DockerSandboxExecutor = class {
|
|
|
1648
2040
|
"-e",
|
|
1649
2041
|
`STACK=${spec.stack}`,
|
|
1650
2042
|
...spec.sessionId ? ["-e", `SESSION_ID=${spec.sessionId}`] : [],
|
|
1651
|
-
...spec.appUrl ? ["-e", `APP_URL=${spec.appUrl}`] : []
|
|
2043
|
+
...spec.appUrl ? ["-e", `APP_URL=${spec.appUrl}`] : [],
|
|
2044
|
+
...Object.entries(spec.env ?? {}).flatMap(([key, value]) => ["-e", `${key}=${value}`])
|
|
1652
2045
|
];
|
|
2046
|
+
const runMode = spec.runMode ?? "service";
|
|
1653
2047
|
src_default.info({ vmId: spec.vmId, bin: this.bin, args: createArgs }, "creating docker sandbox");
|
|
1654
2048
|
try {
|
|
1655
2049
|
await execFileAsync2(this.bin, createArgs);
|
|
1656
|
-
this.launchAgent(sandboxName, agentCommand);
|
|
2050
|
+
if (runMode !== "job") this.launchAgent(sandboxName, agentCommand);
|
|
1657
2051
|
} catch (error) {
|
|
1658
2052
|
this.usedPorts.delete(hostPort);
|
|
1659
2053
|
await import_promises5.default.rm(vmDir, { recursive: true, force: true }).catch(() => {
|
|
@@ -1668,11 +2062,84 @@ var DockerSandboxExecutor = class {
|
|
|
1668
2062
|
running: true,
|
|
1669
2063
|
metadata: { sandboxName, hostPort, vmDir, agentCommand }
|
|
1670
2064
|
};
|
|
1671
|
-
this.vms.set(spec.vmId, {
|
|
1672
|
-
|
|
1673
|
-
|
|
2065
|
+
this.vms.set(spec.vmId, {
|
|
2066
|
+
handle,
|
|
2067
|
+
sandboxName,
|
|
2068
|
+
vmDir,
|
|
2069
|
+
hostPort,
|
|
2070
|
+
agentCommand,
|
|
2071
|
+
runMode,
|
|
2072
|
+
startedAt: Date.now()
|
|
2073
|
+
});
|
|
2074
|
+
if (runMode === "job") {
|
|
2075
|
+
this.runJob(spec.vmId, sandboxName, agentCommand);
|
|
2076
|
+
} else {
|
|
2077
|
+
await this.waitForAgent(hostPort);
|
|
2078
|
+
}
|
|
2079
|
+
src_default.info({ vmId: spec.vmId, sandboxName, hostPort, runMode }, "docker sandbox ready");
|
|
1674
2080
|
return handle;
|
|
1675
2081
|
}
|
|
2082
|
+
/**
|
|
2083
|
+
* Job mode: unlike `launchAgent` (fire-and-forget `sbx exec -d`, used for persistent
|
|
2084
|
+
* services), run the entrypoint in the *foreground* — the CLI call itself blocks until the
|
|
2085
|
+
* command exits and hands back its exit code + captured stdout/stderr directly, so there's
|
|
2086
|
+
* no separate wait/logs step needed the way DockerExecutor needs for a backgrounded container.
|
|
2087
|
+
*/
|
|
2088
|
+
runJob(vmId, sandboxName, agentCommand) {
|
|
2089
|
+
const startedAt = this.vms.get(vmId)?.startedAt ?? Date.now();
|
|
2090
|
+
(0, import_child_process6.execFile)(
|
|
2091
|
+
this.bin,
|
|
2092
|
+
["exec", "-w", AGENT_DIR, sandboxName, "/bin/sh", "-c", agentCommand],
|
|
2093
|
+
{ maxBuffer: MAX_OUTPUT_BYTES5 },
|
|
2094
|
+
(error, stdout, stderr) => {
|
|
2095
|
+
const failed = error;
|
|
2096
|
+
const truncated = failed?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
|
|
2097
|
+
const result = {
|
|
2098
|
+
exitCode: typeof failed?.code === "number" ? failed.code : failed ? 1 : 0,
|
|
2099
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES5),
|
|
2100
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES5),
|
|
2101
|
+
truncated: Boolean(truncated),
|
|
2102
|
+
durationMs: Date.now() - startedAt,
|
|
2103
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2104
|
+
};
|
|
2105
|
+
void this.finishJob(vmId, sandboxName, result);
|
|
2106
|
+
}
|
|
2107
|
+
);
|
|
2108
|
+
}
|
|
2109
|
+
async finishJob(vmId, sandboxName, result) {
|
|
2110
|
+
src_default.info({ vmId, exitCode: result.exitCode }, "docker sandbox job exited");
|
|
2111
|
+
await execFileAsync2(this.bin, ["rm", "-f", sandboxName]).catch(
|
|
2112
|
+
(error) => src_default.warn({ vmId, err: error }, "sbx rm after job exit failed")
|
|
2113
|
+
);
|
|
2114
|
+
const vm = this.vms.get(vmId);
|
|
2115
|
+
if (vm) {
|
|
2116
|
+
await import_promises5.default.rm(vm.vmDir, { recursive: true, force: true }).catch(() => {
|
|
2117
|
+
});
|
|
2118
|
+
this.usedPorts.delete(vm.hostPort);
|
|
2119
|
+
vm.handle = { ...vm.handle, running: false };
|
|
2120
|
+
}
|
|
2121
|
+
this.deliverJobResult(vmId, result);
|
|
2122
|
+
}
|
|
2123
|
+
deliverJobResult(vmId, result) {
|
|
2124
|
+
const handler = this.exitHandlers.get(vmId);
|
|
2125
|
+
if (handler) {
|
|
2126
|
+
this.exitHandlers.delete(vmId);
|
|
2127
|
+
this.vms.delete(vmId);
|
|
2128
|
+
handler(result);
|
|
2129
|
+
} else {
|
|
2130
|
+
this.pendingResults.set(vmId, result);
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
onExit(vmId, handler) {
|
|
2134
|
+
const pending = this.pendingResults.get(vmId);
|
|
2135
|
+
if (pending) {
|
|
2136
|
+
this.pendingResults.delete(vmId);
|
|
2137
|
+
this.vms.delete(vmId);
|
|
2138
|
+
handler(pending);
|
|
2139
|
+
return;
|
|
2140
|
+
}
|
|
2141
|
+
this.exitHandlers.set(vmId, handler);
|
|
2142
|
+
}
|
|
1676
2143
|
/**
|
|
1677
2144
|
* `sbx create shell` doesn't run the image's own CMD — launch the guest agent explicitly.
|
|
1678
2145
|
*
|
|
@@ -1683,10 +2150,17 @@ var DockerSandboxExecutor = class {
|
|
|
1683
2150
|
* process exiting, so it's spawned detached and never awaited.
|
|
1684
2151
|
*/
|
|
1685
2152
|
launchAgent(sandboxName, agentCommand) {
|
|
1686
|
-
const proc = (0, import_child_process6.spawn)(
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
2153
|
+
const proc = (0, import_child_process6.spawn)(
|
|
2154
|
+
this.bin,
|
|
2155
|
+
["exec", "-d", "-w", AGENT_DIR, sandboxName, "/bin/sh", "-c", agentCommand],
|
|
2156
|
+
{
|
|
2157
|
+
stdio: "ignore"
|
|
2158
|
+
}
|
|
2159
|
+
);
|
|
2160
|
+
proc.on(
|
|
2161
|
+
"error",
|
|
2162
|
+
(error) => src_default.warn({ sandboxName, err: error }, "sbx exec -d failed to spawn")
|
|
2163
|
+
);
|
|
1690
2164
|
proc.unref();
|
|
1691
2165
|
}
|
|
1692
2166
|
async pause(vmId) {
|
|
@@ -1732,7 +2206,8 @@ var DockerSandboxExecutor = class {
|
|
|
1732
2206
|
*/
|
|
1733
2207
|
async reattach(record) {
|
|
1734
2208
|
const meta = record.runtimeMetadata;
|
|
1735
|
-
if (!meta?.sandboxName || meta.hostPort === void 0 || !meta.vmDir || !meta.agentCommand)
|
|
2209
|
+
if (!meta?.sandboxName || meta.hostPort === void 0 || !meta.vmDir || !meta.agentCommand)
|
|
2210
|
+
return void 0;
|
|
1736
2211
|
let running;
|
|
1737
2212
|
try {
|
|
1738
2213
|
await execFileAsync2(this.bin, ["exec", meta.sandboxName, "true"]);
|
|
@@ -1759,8 +2234,20 @@ var DockerSandboxExecutor = class {
|
|
|
1759
2234
|
sandboxName: meta.sandboxName,
|
|
1760
2235
|
vmDir: meta.vmDir,
|
|
1761
2236
|
hostPort: meta.hostPort,
|
|
1762
|
-
agentCommand: meta.agentCommand
|
|
2237
|
+
agentCommand: meta.agentCommand,
|
|
2238
|
+
runMode: record.runMode,
|
|
2239
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
1763
2240
|
});
|
|
2241
|
+
if (record.runMode === "job") {
|
|
2242
|
+
void this.finishJob(record.vmId, meta.sandboxName, {
|
|
2243
|
+
exitCode: -1,
|
|
2244
|
+
stdout: "",
|
|
2245
|
+
stderr: "worker restarted while this job was still running",
|
|
2246
|
+
truncated: false,
|
|
2247
|
+
durationMs: Date.now() - (Date.parse(record.createdAt) || Date.now()),
|
|
2248
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2249
|
+
});
|
|
2250
|
+
}
|
|
1764
2251
|
src_default.info({ vmId: record.vmId, sandboxName: meta.sandboxName }, "docker sandbox reattached");
|
|
1765
2252
|
return handle;
|
|
1766
2253
|
}
|
|
@@ -1769,7 +2256,13 @@ var DockerSandboxExecutor = class {
|
|
|
1769
2256
|
if (!vm.handle.running) throw new Error(`vm ${vmId} is not running`);
|
|
1770
2257
|
return new Promise((resolve, reject) => {
|
|
1771
2258
|
const req = import_http4.default.request(
|
|
1772
|
-
{
|
|
2259
|
+
{
|
|
2260
|
+
host: "127.0.0.1",
|
|
2261
|
+
port: vm.hostPort,
|
|
2262
|
+
method: request.method,
|
|
2263
|
+
path: request.path,
|
|
2264
|
+
headers: request.headers
|
|
2265
|
+
},
|
|
1773
2266
|
(res) => {
|
|
1774
2267
|
const chunks = [];
|
|
1775
2268
|
res.on("data", (c) => chunks.push(c));
|
|
@@ -1778,7 +2271,10 @@ var DockerSandboxExecutor = class {
|
|
|
1778
2271
|
() => resolve({
|
|
1779
2272
|
statusCode: res.statusCode ?? 502,
|
|
1780
2273
|
headers: Object.fromEntries(
|
|
1781
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
2274
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
2275
|
+
k,
|
|
2276
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
2277
|
+
])
|
|
1782
2278
|
),
|
|
1783
2279
|
body: Buffer.concat(chunks)
|
|
1784
2280
|
})
|
|
@@ -1799,15 +2295,15 @@ var DockerSandboxExecutor = class {
|
|
|
1799
2295
|
(0, import_child_process6.execFile)(
|
|
1800
2296
|
this.bin,
|
|
1801
2297
|
["exec", "-w", cwd, vm.sandboxName, "/bin/sh", "-c", request.command],
|
|
1802
|
-
{ timeout: timeoutMs, maxBuffer:
|
|
2298
|
+
{ timeout: timeoutMs, maxBuffer: MAX_OUTPUT_BYTES5 },
|
|
1803
2299
|
(error, stdout, stderr) => {
|
|
1804
2300
|
const failed = error;
|
|
1805
2301
|
const timedOut = Boolean(failed?.killed);
|
|
1806
2302
|
const truncated = failed?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
|
|
1807
2303
|
resolve({
|
|
1808
2304
|
exitCode: typeof failed?.code === "number" ? failed.code : failed ? 1 : 0,
|
|
1809
|
-
stdout: stdout.slice(0,
|
|
1810
|
-
stderr: stderr.slice(0,
|
|
2305
|
+
stdout: stdout.slice(0, MAX_OUTPUT_BYTES5),
|
|
2306
|
+
stderr: stderr.slice(0, MAX_OUTPUT_BYTES5),
|
|
1811
2307
|
durationMs: Date.now() - startedAt,
|
|
1812
2308
|
truncated: Boolean(truncated),
|
|
1813
2309
|
timedOut
|
|
@@ -1823,16 +2319,31 @@ var DockerSandboxExecutor = class {
|
|
|
1823
2319
|
const relativePath = assertSafeWorkspacePath2(file.path);
|
|
1824
2320
|
const buffer = Buffer.from(file.content, file.encoding === "base64" ? "base64" : "utf8");
|
|
1825
2321
|
if (buffer.byteLength > MAX_FILE_BYTES3) {
|
|
1826
|
-
throw new Error(
|
|
2322
|
+
throw new Error(
|
|
2323
|
+
`File too large (${buffer.byteLength} bytes, max ${MAX_FILE_BYTES3}): ${file.path}`
|
|
2324
|
+
);
|
|
1827
2325
|
}
|
|
1828
2326
|
const dir = relativePath.includes("/") ? relativePath.slice(0, relativePath.lastIndexOf("/")) : "";
|
|
1829
2327
|
if (dir) {
|
|
1830
|
-
await execFileAsync2(this.bin, [
|
|
2328
|
+
await execFileAsync2(this.bin, [
|
|
2329
|
+
"exec",
|
|
2330
|
+
vm.sandboxName,
|
|
2331
|
+
"mkdir",
|
|
2332
|
+
"-p",
|
|
2333
|
+
`${WORKSPACE_DIR2}/${dir}`
|
|
2334
|
+
]);
|
|
1831
2335
|
}
|
|
1832
|
-
const tmpFile = import_path6.default.join(
|
|
2336
|
+
const tmpFile = import_path6.default.join(
|
|
2337
|
+
import_os8.default.tmpdir(),
|
|
2338
|
+
`sbx-write-${vmId}-${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
2339
|
+
);
|
|
1833
2340
|
await import_promises5.default.writeFile(tmpFile, buffer);
|
|
1834
2341
|
try {
|
|
1835
|
-
await execFileAsync2(this.bin, [
|
|
2342
|
+
await execFileAsync2(this.bin, [
|
|
2343
|
+
"cp",
|
|
2344
|
+
tmpFile,
|
|
2345
|
+
`${vm.sandboxName}:${WORKSPACE_DIR2}/${relativePath}`
|
|
2346
|
+
]);
|
|
1836
2347
|
} finally {
|
|
1837
2348
|
await import_promises5.default.rm(tmpFile, { force: true });
|
|
1838
2349
|
}
|
|
@@ -1868,7 +2379,8 @@ var DockerSandboxExecutor = class {
|
|
|
1868
2379
|
const deadline = Date.now() + timeoutMs;
|
|
1869
2380
|
for (; ; ) {
|
|
1870
2381
|
if (await this.probeAgent(port)) return;
|
|
1871
|
-
if (Date.now() > deadline)
|
|
2382
|
+
if (Date.now() > deadline)
|
|
2383
|
+
throw new Error(`timed out waiting for docker sandbox agent on port ${port}`);
|
|
1872
2384
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
1873
2385
|
}
|
|
1874
2386
|
}
|
|
@@ -1877,12 +2389,14 @@ var DockerSandboxExecutor = class {
|
|
|
1877
2389
|
// src/executor/remote-executor.ts
|
|
1878
2390
|
var import_http5 = __toESM(require("http"));
|
|
1879
2391
|
var import_https = __toESM(require("https"));
|
|
2392
|
+
var JOB_POLL_INTERVAL_MS = 3e3;
|
|
1880
2393
|
var RemoteExecutor = class {
|
|
1881
2394
|
constructor(config) {
|
|
1882
2395
|
// The remote agent is the actual source of truth, but MultiEngineExecutor.route() (used by
|
|
1883
2396
|
// pause/resume/stop/restore/destroy/proxyHttp/exec/writeFiles) needs get() to find the
|
|
1884
2397
|
// right executor for a vmId — cache the handle each call returns so it does.
|
|
1885
2398
|
this.vms = /* @__PURE__ */ new Map();
|
|
2399
|
+
this.jobStartedAt = /* @__PURE__ */ new Map();
|
|
1886
2400
|
this.base = config.agentUrl.replace(/\/$/, "");
|
|
1887
2401
|
this.headers = {
|
|
1888
2402
|
"Content-Type": "application/json",
|
|
@@ -1892,8 +2406,38 @@ var RemoteExecutor = class {
|
|
|
1892
2406
|
async create(spec) {
|
|
1893
2407
|
const handle = await this.request("POST", "/vms", spec);
|
|
1894
2408
|
this.vms.set(spec.vmId, handle);
|
|
2409
|
+
if (spec.runMode === "job") this.jobStartedAt.set(spec.vmId, Date.now());
|
|
1895
2410
|
return handle;
|
|
1896
2411
|
}
|
|
2412
|
+
/**
|
|
2413
|
+
* The remote agent's own wire protocol (VmHandle) has no exit-code/output field — extending
|
|
2414
|
+
* it is out of this repo's control, so this only detects completion (poll GET /vms/:vmId
|
|
2415
|
+
* until running flips false) without a real exit code or captured output. Good enough to
|
|
2416
|
+
* unblock the record from being stuck in 'running' forever; a real RemoteExecutor deployment
|
|
2417
|
+
* wanting full job support needs its agent protocol extended first.
|
|
2418
|
+
*/
|
|
2419
|
+
onExit(vmId, handler) {
|
|
2420
|
+
const startedAt = this.jobStartedAt.get(vmId) ?? Date.now();
|
|
2421
|
+
const poll = async () => {
|
|
2422
|
+
const handle = await this.request("GET", `/vms/${vmId}`).catch(() => void 0);
|
|
2423
|
+
if (handle) this.vms.set(vmId, handle);
|
|
2424
|
+
if (!handle || !handle.running) {
|
|
2425
|
+
this.jobStartedAt.delete(vmId);
|
|
2426
|
+
this.vms.delete(vmId);
|
|
2427
|
+
handler({
|
|
2428
|
+
exitCode: 0,
|
|
2429
|
+
stdout: "",
|
|
2430
|
+
stderr: "remote engine does not report job exit codes/output \u2014 completion detected via polling only",
|
|
2431
|
+
truncated: false,
|
|
2432
|
+
durationMs: Date.now() - startedAt,
|
|
2433
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2434
|
+
});
|
|
2435
|
+
return;
|
|
2436
|
+
}
|
|
2437
|
+
setTimeout(() => void poll(), JOB_POLL_INTERVAL_MS);
|
|
2438
|
+
};
|
|
2439
|
+
void poll();
|
|
2440
|
+
}
|
|
1897
2441
|
async pause(vmId) {
|
|
1898
2442
|
return this.updateCached(vmId, await this.request("POST", `/vms/${vmId}/pause`));
|
|
1899
2443
|
}
|
|
@@ -2021,7 +2565,13 @@ var LambdaMicrovmExecutor = class {
|
|
|
2021
2565
|
} : {},
|
|
2022
2566
|
// The guest's /run lifecycle hook receives this payload — lets the image
|
|
2023
2567
|
// identify which builder session/VM it serves.
|
|
2024
|
-
runHookPayload: JSON.stringify({
|
|
2568
|
+
runHookPayload: JSON.stringify({
|
|
2569
|
+
vmId: spec.vmId,
|
|
2570
|
+
sessionId: spec.sessionId,
|
|
2571
|
+
stack: spec.stack,
|
|
2572
|
+
appUrl: spec.appUrl,
|
|
2573
|
+
env: spec.env
|
|
2574
|
+
})
|
|
2025
2575
|
})
|
|
2026
2576
|
);
|
|
2027
2577
|
if (!run.microvmId || !run.endpoint) {
|
|
@@ -2109,7 +2659,10 @@ var LambdaMicrovmExecutor = class {
|
|
|
2109
2659
|
() => resolve({
|
|
2110
2660
|
statusCode: res.statusCode ?? 502,
|
|
2111
2661
|
headers: Object.fromEntries(
|
|
2112
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
2662
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
2663
|
+
k,
|
|
2664
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
2665
|
+
])
|
|
2113
2666
|
),
|
|
2114
2667
|
body: Buffer.concat(chunks)
|
|
2115
2668
|
})
|
|
@@ -2142,7 +2695,10 @@ var LambdaMicrovmExecutor = class {
|
|
|
2142
2695
|
metadata: meta
|
|
2143
2696
|
};
|
|
2144
2697
|
this.vms.set(record.vmId, { handle, microvmId: meta.microvmId, endpoint: meta.endpoint });
|
|
2145
|
-
src_default.info(
|
|
2698
|
+
src_default.info(
|
|
2699
|
+
{ vmId: record.vmId, microvmId: meta.microvmId, state: described.state },
|
|
2700
|
+
"[LambdaMicrovm] vm reattached"
|
|
2701
|
+
);
|
|
2146
2702
|
return handle;
|
|
2147
2703
|
}
|
|
2148
2704
|
mustGet(vmId) {
|
|
@@ -2182,7 +2738,9 @@ var LambdaMicrovmExecutor = class {
|
|
|
2182
2738
|
throw new Error(`microvm ${microvmId} entered ${resp.state} while waiting for ${target}`);
|
|
2183
2739
|
}
|
|
2184
2740
|
if (Date.now() > deadline) {
|
|
2185
|
-
throw new Error(
|
|
2741
|
+
throw new Error(
|
|
2742
|
+
`timed out waiting for microvm ${microvmId} to reach ${target} (still ${resp.state})`
|
|
2743
|
+
);
|
|
2186
2744
|
}
|
|
2187
2745
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
2188
2746
|
}
|
|
@@ -2192,6 +2750,7 @@ var LambdaMicrovmExecutor = class {
|
|
|
2192
2750
|
// src/executor/fargate-executor.ts
|
|
2193
2751
|
var import_http6 = __toESM(require("http"));
|
|
2194
2752
|
var import_client_ecs = require("@aws-sdk/client-ecs");
|
|
2753
|
+
var JOB_POLL_INTERVAL_MS2 = 5e3;
|
|
2195
2754
|
var FARGATE_MEM_RANGES = {
|
|
2196
2755
|
256: [512, 2048],
|
|
2197
2756
|
512: [1024, 4096],
|
|
@@ -2211,6 +2770,8 @@ var FargateExecutor = class {
|
|
|
2211
2770
|
this.config = config;
|
|
2212
2771
|
this.vms = /* @__PURE__ */ new Map();
|
|
2213
2772
|
this.containerNames = /* @__PURE__ */ new Map();
|
|
2773
|
+
this.pendingResults = /* @__PURE__ */ new Map();
|
|
2774
|
+
this.exitHandlers = /* @__PURE__ */ new Map();
|
|
2214
2775
|
this.client = new import_client_ecs.ECSClient(config.region ? { region: config.region } : {});
|
|
2215
2776
|
}
|
|
2216
2777
|
/** First container of the task definition — the target for env overrides. Cached per definition. */
|
|
@@ -2237,11 +2798,16 @@ var FargateExecutor = class {
|
|
|
2237
2798
|
}
|
|
2238
2799
|
if (task.lastStatus === "RUNNING") {
|
|
2239
2800
|
const ip = this.config.assignPublicIp ? await this.resolvePublicIp(task.attachments) : task.containers?.[0]?.networkInterfaces?.[0]?.privateIpv4Address;
|
|
2240
|
-
if (!ip)
|
|
2801
|
+
if (!ip)
|
|
2802
|
+
throw new Error(
|
|
2803
|
+
`fargate task ${taskArn} is RUNNING but has no ${this.config.assignPublicIp ? "public" : "private"} IP`
|
|
2804
|
+
);
|
|
2241
2805
|
return ip;
|
|
2242
2806
|
}
|
|
2243
2807
|
if (Date.now() > deadline) {
|
|
2244
|
-
throw new Error(
|
|
2808
|
+
throw new Error(
|
|
2809
|
+
`fargate task did not reach RUNNING within ${timeoutMs}ms (last status: ${task.lastStatus})`
|
|
2810
|
+
);
|
|
2245
2811
|
}
|
|
2246
2812
|
await new Promise((resolve) => setTimeout(resolve, 3e3));
|
|
2247
2813
|
}
|
|
@@ -2251,7 +2817,9 @@ var FargateExecutor = class {
|
|
|
2251
2817
|
if (!eniId) return void 0;
|
|
2252
2818
|
const { EC2Client, DescribeNetworkInterfacesCommand } = await import("@aws-sdk/client-ec2");
|
|
2253
2819
|
const ec2 = new EC2Client(this.config.region ? { region: this.config.region } : {});
|
|
2254
|
-
const described = await ec2.send(
|
|
2820
|
+
const described = await ec2.send(
|
|
2821
|
+
new DescribeNetworkInterfacesCommand({ NetworkInterfaceIds: [eniId] })
|
|
2822
|
+
);
|
|
2255
2823
|
return described.NetworkInterfaces?.[0]?.Association?.PublicIp;
|
|
2256
2824
|
}
|
|
2257
2825
|
async create(spec) {
|
|
@@ -2285,7 +2853,8 @@ var FargateExecutor = class {
|
|
|
2285
2853
|
...spec.sessionId ? [{ name: "SESSION_ID", value: spec.sessionId }] : [],
|
|
2286
2854
|
{ name: "STACK", value: spec.stack },
|
|
2287
2855
|
// Release artifact: the container agent pulls, unzips, and starts it
|
|
2288
|
-
...spec.appUrl ? [{ name: "APP_URL", value: spec.appUrl }] : []
|
|
2856
|
+
...spec.appUrl ? [{ name: "APP_URL", value: spec.appUrl }] : [],
|
|
2857
|
+
...Object.entries(spec.env ?? {}).map(([name, value]) => ({ name, value }))
|
|
2289
2858
|
]
|
|
2290
2859
|
}
|
|
2291
2860
|
]
|
|
@@ -2301,6 +2870,7 @@ var FargateExecutor = class {
|
|
|
2301
2870
|
throw new Error(`RunTask failed: ${run.failures?.[0]?.reason ?? "no task returned"}`);
|
|
2302
2871
|
}
|
|
2303
2872
|
src_default.info({ vmId: spec.vmId, taskArn, taskDefinition, cpu, memory }, "fargate task started");
|
|
2873
|
+
const runMode = spec.runMode ?? "service";
|
|
2304
2874
|
try {
|
|
2305
2875
|
const ip = await this.waitForRunning(taskArn);
|
|
2306
2876
|
const handle = {
|
|
@@ -2311,11 +2881,18 @@ var FargateExecutor = class {
|
|
|
2311
2881
|
running: true,
|
|
2312
2882
|
metadata: { taskArn, ip }
|
|
2313
2883
|
};
|
|
2314
|
-
this.vms.set(spec.vmId, { handle, taskArn, ip });
|
|
2315
|
-
|
|
2884
|
+
this.vms.set(spec.vmId, { handle, taskArn, ip, runMode, startedAt: Date.now() });
|
|
2885
|
+
if (runMode === "job") void this.watchJob(spec.vmId, taskArn);
|
|
2886
|
+
src_default.info({ vmId: spec.vmId, ip, runMode }, "fargate task running");
|
|
2316
2887
|
return handle;
|
|
2317
2888
|
} catch (error) {
|
|
2318
|
-
await this.client.send(
|
|
2889
|
+
await this.client.send(
|
|
2890
|
+
new import_client_ecs.StopTaskCommand({
|
|
2891
|
+
cluster: this.config.cluster,
|
|
2892
|
+
task: taskArn,
|
|
2893
|
+
reason: "startup failed"
|
|
2894
|
+
})
|
|
2895
|
+
).catch(() => {
|
|
2319
2896
|
});
|
|
2320
2897
|
throw error;
|
|
2321
2898
|
}
|
|
@@ -2332,7 +2909,11 @@ var FargateExecutor = class {
|
|
|
2332
2909
|
async stop(vmId) {
|
|
2333
2910
|
const vm = this.mustGet(vmId);
|
|
2334
2911
|
await this.client.send(
|
|
2335
|
-
new import_client_ecs.StopTaskCommand({
|
|
2912
|
+
new import_client_ecs.StopTaskCommand({
|
|
2913
|
+
cluster: this.config.cluster,
|
|
2914
|
+
task: vm.taskArn,
|
|
2915
|
+
reason: "stopped via API"
|
|
2916
|
+
})
|
|
2336
2917
|
);
|
|
2337
2918
|
vm.handle.running = false;
|
|
2338
2919
|
return vm.handle;
|
|
@@ -2340,7 +2921,13 @@ var FargateExecutor = class {
|
|
|
2340
2921
|
async destroy(vmId) {
|
|
2341
2922
|
const vm = this.vms.get(vmId);
|
|
2342
2923
|
if (!vm) return;
|
|
2343
|
-
await this.client.send(
|
|
2924
|
+
await this.client.send(
|
|
2925
|
+
new import_client_ecs.StopTaskCommand({
|
|
2926
|
+
cluster: this.config.cluster,
|
|
2927
|
+
task: vm.taskArn,
|
|
2928
|
+
reason: "destroyed via API"
|
|
2929
|
+
})
|
|
2930
|
+
).catch((error) => src_default.warn({ vmId, err: error }, "fargate stop on destroy failed"));
|
|
2344
2931
|
this.vms.delete(vmId);
|
|
2345
2932
|
}
|
|
2346
2933
|
get(vmId) {
|
|
@@ -2369,7 +2956,10 @@ var FargateExecutor = class {
|
|
|
2369
2956
|
() => resolve({
|
|
2370
2957
|
statusCode: res.statusCode ?? 502,
|
|
2371
2958
|
headers: Object.fromEntries(
|
|
2372
|
-
Object.entries(res.headers).map(([k, v]) => [
|
|
2959
|
+
Object.entries(res.headers).map(([k, v]) => [
|
|
2960
|
+
k,
|
|
2961
|
+
Array.isArray(v) ? v.join(", ") : v ?? ""
|
|
2962
|
+
])
|
|
2373
2963
|
),
|
|
2374
2964
|
body: Buffer.concat(chunks)
|
|
2375
2965
|
})
|
|
@@ -2402,10 +2992,82 @@ var FargateExecutor = class {
|
|
|
2402
2992
|
running: true,
|
|
2403
2993
|
metadata: { taskArn: meta.taskArn, ip }
|
|
2404
2994
|
};
|
|
2405
|
-
this.vms.set(record.vmId, {
|
|
2995
|
+
this.vms.set(record.vmId, {
|
|
2996
|
+
handle,
|
|
2997
|
+
taskArn: meta.taskArn,
|
|
2998
|
+
ip,
|
|
2999
|
+
runMode: record.runMode,
|
|
3000
|
+
startedAt: Date.parse(record.createdAt) || Date.now()
|
|
3001
|
+
});
|
|
3002
|
+
if (record.runMode === "job") void this.watchJob(record.vmId, meta.taskArn);
|
|
2406
3003
|
src_default.info({ vmId: record.vmId, taskArn: meta.taskArn, ip }, "fargate task reattached");
|
|
2407
3004
|
return handle;
|
|
2408
3005
|
}
|
|
3006
|
+
/**
|
|
3007
|
+
* No push notification exists for "task stopped" without EventBridge wiring (v1 doesn't set
|
|
3008
|
+
* that up), so poll DescribeTasks. Exit code comes straight from the container's own
|
|
3009
|
+
* `exitCode` field — no CloudWatch Logs integration yet, so stdout/stderr capture isn't
|
|
3010
|
+
* implemented for this engine (the caller can still find full output in CloudWatch directly,
|
|
3011
|
+
* via the task's own log configuration).
|
|
3012
|
+
*/
|
|
3013
|
+
async watchJob(vmId, taskArn) {
|
|
3014
|
+
const startedAt = this.vms.get(vmId)?.startedAt ?? Date.now();
|
|
3015
|
+
for (; ; ) {
|
|
3016
|
+
await new Promise((resolve) => setTimeout(resolve, JOB_POLL_INTERVAL_MS2));
|
|
3017
|
+
if (!this.vms.has(vmId)) return;
|
|
3018
|
+
const described = await this.client.send(new import_client_ecs.DescribeTasksCommand({ cluster: this.config.cluster, tasks: [taskArn] })).catch(() => void 0);
|
|
3019
|
+
const task = described?.tasks?.[0];
|
|
3020
|
+
if (!task) {
|
|
3021
|
+
this.finishJob(vmId, {
|
|
3022
|
+
exitCode: -1,
|
|
3023
|
+
stdout: "",
|
|
3024
|
+
stderr: "fargate task not found while polling for completion",
|
|
3025
|
+
truncated: false,
|
|
3026
|
+
durationMs: Date.now() - startedAt,
|
|
3027
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3028
|
+
});
|
|
3029
|
+
return;
|
|
3030
|
+
}
|
|
3031
|
+
if (task.lastStatus === "STOPPED") {
|
|
3032
|
+
const exitCode = task.containers?.[0]?.exitCode;
|
|
3033
|
+
this.finishJob(vmId, {
|
|
3034
|
+
exitCode: exitCode ?? 1,
|
|
3035
|
+
stdout: "",
|
|
3036
|
+
stderr: [
|
|
3037
|
+
task.stoppedReason ? `stoppedReason: ${task.stoppedReason}` : void 0,
|
|
3038
|
+
"output capture not implemented for the fargate engine \u2014 see CloudWatch Logs for this task"
|
|
3039
|
+
].filter(Boolean).join("; "),
|
|
3040
|
+
truncated: false,
|
|
3041
|
+
durationMs: Date.now() - startedAt,
|
|
3042
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3043
|
+
});
|
|
3044
|
+
return;
|
|
3045
|
+
}
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
finishJob(vmId, result) {
|
|
3049
|
+
src_default.info({ vmId, exitCode: result.exitCode }, "fargate job task exited");
|
|
3050
|
+
const vm = this.vms.get(vmId);
|
|
3051
|
+
if (vm) vm.handle = { ...vm.handle, running: false };
|
|
3052
|
+
const handler = this.exitHandlers.get(vmId);
|
|
3053
|
+
if (handler) {
|
|
3054
|
+
this.exitHandlers.delete(vmId);
|
|
3055
|
+
this.vms.delete(vmId);
|
|
3056
|
+
handler(result);
|
|
3057
|
+
} else {
|
|
3058
|
+
this.pendingResults.set(vmId, result);
|
|
3059
|
+
}
|
|
3060
|
+
}
|
|
3061
|
+
onExit(vmId, handler) {
|
|
3062
|
+
const pending = this.pendingResults.get(vmId);
|
|
3063
|
+
if (pending) {
|
|
3064
|
+
this.pendingResults.delete(vmId);
|
|
3065
|
+
this.vms.delete(vmId);
|
|
3066
|
+
handler(pending);
|
|
3067
|
+
return;
|
|
3068
|
+
}
|
|
3069
|
+
this.exitHandlers.set(vmId, handler);
|
|
3070
|
+
}
|
|
2409
3071
|
mustGet(vmId) {
|
|
2410
3072
|
const vm = this.vms.get(vmId);
|
|
2411
3073
|
if (!vm) throw new Error(`vm ${vmId} not found`);
|
|
@@ -2416,7 +3078,16 @@ var FargateExecutor = class {
|
|
|
2416
3078
|
// src/executor/detect.ts
|
|
2417
3079
|
var import_fs2 = __toESM(require("fs"));
|
|
2418
3080
|
var import_child_process7 = require("child_process");
|
|
2419
|
-
var ENGINE_PRIORITY = [
|
|
3081
|
+
var ENGINE_PRIORITY = [
|
|
3082
|
+
"firecracker",
|
|
3083
|
+
"qemu",
|
|
3084
|
+
"docker-sandbox",
|
|
3085
|
+
"docker",
|
|
3086
|
+
"lambda-microvm",
|
|
3087
|
+
"fargate",
|
|
3088
|
+
"remote",
|
|
3089
|
+
"mock"
|
|
3090
|
+
];
|
|
2420
3091
|
function binaryOnPath(bin) {
|
|
2421
3092
|
if (bin.includes("/")) {
|
|
2422
3093
|
try {
|
|
@@ -2442,22 +3113,35 @@ function kvmUsable() {
|
|
|
2442
3113
|
}
|
|
2443
3114
|
}
|
|
2444
3115
|
function hasBaseImages(cfg) {
|
|
2445
|
-
if (cfg.kernelPath && cfg.initrdPath)
|
|
3116
|
+
if (cfg.kernelPath && cfg.initrdPath)
|
|
3117
|
+
return import_fs2.default.existsSync(cfg.kernelPath) && import_fs2.default.existsSync(cfg.initrdPath);
|
|
2446
3118
|
return availableStacks(cfg.imagesDir).length > 0;
|
|
2447
3119
|
}
|
|
2448
3120
|
function probeQemu(config) {
|
|
2449
3121
|
const { bin, accel } = config.qemu;
|
|
2450
3122
|
if (!hasBaseImages(config.qemu)) {
|
|
2451
|
-
return {
|
|
3123
|
+
return {
|
|
3124
|
+
engine: "qemu",
|
|
3125
|
+
available: false,
|
|
3126
|
+
reason: `no base images under ${config.qemu.imagesDir} \u2014 run "worker-cli images pull" or scripts/setup-vm-images.sh`
|
|
3127
|
+
};
|
|
2452
3128
|
}
|
|
2453
3129
|
if (!binaryOnPath(bin)) {
|
|
2454
3130
|
return { engine: "qemu", available: false, reason: `${bin} not found on PATH` };
|
|
2455
3131
|
}
|
|
2456
3132
|
if (accel === "hvf" && process.platform !== "darwin") {
|
|
2457
|
-
return {
|
|
3133
|
+
return {
|
|
3134
|
+
engine: "qemu",
|
|
3135
|
+
available: false,
|
|
3136
|
+
reason: "QEMU_ACCEL=hvf requires macOS (use kvm or tcg)"
|
|
3137
|
+
};
|
|
2458
3138
|
}
|
|
2459
3139
|
if (accel === "kvm" && !kvmUsable()) {
|
|
2460
|
-
return {
|
|
3140
|
+
return {
|
|
3141
|
+
engine: "qemu",
|
|
3142
|
+
available: false,
|
|
3143
|
+
reason: "QEMU_ACCEL=kvm but /dev/kvm is not accessible"
|
|
3144
|
+
};
|
|
2461
3145
|
}
|
|
2462
3146
|
return { engine: "qemu", available: true };
|
|
2463
3147
|
}
|
|
@@ -2470,17 +3154,22 @@ function probeFirecracker(config) {
|
|
|
2470
3154
|
return { engine: "firecracker", available: false, reason: "/dev/kvm is not accessible" };
|
|
2471
3155
|
}
|
|
2472
3156
|
if (!hasBaseImages(config.firecracker)) {
|
|
2473
|
-
return {
|
|
3157
|
+
return {
|
|
3158
|
+
engine: "firecracker",
|
|
3159
|
+
available: false,
|
|
3160
|
+
reason: `no base images under ${config.firecracker.imagesDir} \u2014 run "worker-cli images pull" or scripts/setup-vm-images.sh`
|
|
3161
|
+
};
|
|
2474
3162
|
}
|
|
2475
3163
|
if (!binaryOnPath(bin)) {
|
|
2476
3164
|
return { engine: "firecracker", available: false, reason: `${bin} not found on PATH` };
|
|
2477
3165
|
}
|
|
2478
3166
|
return { engine: "firecracker", available: true };
|
|
2479
3167
|
}
|
|
2480
|
-
function hasLocalDockerImage(imagePrefix, imageTag) {
|
|
3168
|
+
function hasLocalDockerImage(bin, imagePrefix, imageTag, stackImages = {}) {
|
|
2481
3169
|
return VM_STACKS.some((stack) => {
|
|
2482
3170
|
try {
|
|
2483
|
-
|
|
3171
|
+
const image = stackImages[stack] ?? `${imagePrefix}-${stack}:${imageTag}`;
|
|
3172
|
+
(0, import_child_process7.execFileSync)(bin, ["image", "inspect", image], { stdio: "ignore" });
|
|
2484
3173
|
return true;
|
|
2485
3174
|
} catch {
|
|
2486
3175
|
return false;
|
|
@@ -2488,7 +3177,7 @@ function hasLocalDockerImage(imagePrefix, imageTag) {
|
|
|
2488
3177
|
});
|
|
2489
3178
|
}
|
|
2490
3179
|
function probeDocker(config) {
|
|
2491
|
-
const { bin, imagePrefix, imageTag } = config.docker;
|
|
3180
|
+
const { bin, imagePrefix, imageTag, stackImages } = config.docker;
|
|
2492
3181
|
if (!binaryOnPath(bin)) {
|
|
2493
3182
|
return { engine: "docker", available: false, reason: `${bin} not found on PATH` };
|
|
2494
3183
|
}
|
|
@@ -2497,8 +3186,12 @@ function probeDocker(config) {
|
|
|
2497
3186
|
} catch {
|
|
2498
3187
|
return { engine: "docker", available: false, reason: "docker daemon not reachable" };
|
|
2499
3188
|
}
|
|
2500
|
-
if (!hasLocalDockerImage(imagePrefix, imageTag)) {
|
|
2501
|
-
return {
|
|
3189
|
+
if (!hasLocalDockerImage(bin, imagePrefix, imageTag, stackImages)) {
|
|
3190
|
+
return {
|
|
3191
|
+
engine: "docker",
|
|
3192
|
+
available: false,
|
|
3193
|
+
reason: "no local docker images \u2014 run scripts/setup-docker-images.sh"
|
|
3194
|
+
};
|
|
2502
3195
|
}
|
|
2503
3196
|
return { engine: "docker", available: true };
|
|
2504
3197
|
}
|
|
@@ -2510,10 +3203,18 @@ function probeDockerSandbox(config) {
|
|
|
2510
3203
|
try {
|
|
2511
3204
|
(0, import_child_process7.execFileSync)(bin, ["ls"], { stdio: "ignore" });
|
|
2512
3205
|
} catch {
|
|
2513
|
-
return {
|
|
3206
|
+
return {
|
|
3207
|
+
engine: "docker-sandbox",
|
|
3208
|
+
available: false,
|
|
3209
|
+
reason: 'sbx daemon not reachable \u2014 run "sbx daemon start" and "sbx login"'
|
|
3210
|
+
};
|
|
2514
3211
|
}
|
|
2515
|
-
if (!hasLocalDockerImage(imagePrefix, imageTag)) {
|
|
2516
|
-
return {
|
|
3212
|
+
if (!hasLocalDockerImage(config.docker.bin, imagePrefix, imageTag)) {
|
|
3213
|
+
return {
|
|
3214
|
+
engine: "docker-sandbox",
|
|
3215
|
+
available: false,
|
|
3216
|
+
reason: "no local docker images \u2014 run scripts/setup-docker-images.sh"
|
|
3217
|
+
};
|
|
2517
3218
|
}
|
|
2518
3219
|
return { engine: "docker-sandbox", available: true };
|
|
2519
3220
|
}
|
|
@@ -2522,7 +3223,11 @@ function probeRemote(config) {
|
|
|
2522
3223
|
}
|
|
2523
3224
|
function probeLambdaMicrovm(config) {
|
|
2524
3225
|
const { imageArn, stackImageArns } = config.lambdaMicrovm;
|
|
2525
|
-
return imageArn || Object.keys(stackImageArns).length ? { engine: "lambda-microvm", available: true } : {
|
|
3226
|
+
return imageArn || Object.keys(stackImageArns).length ? { engine: "lambda-microvm", available: true } : {
|
|
3227
|
+
engine: "lambda-microvm",
|
|
3228
|
+
available: false,
|
|
3229
|
+
reason: "MICROVM_IMAGE_ARN / MICROVM_STACK_IMAGE_ARNS not set"
|
|
3230
|
+
};
|
|
2526
3231
|
}
|
|
2527
3232
|
function probeFargate(config) {
|
|
2528
3233
|
const { cluster, taskDefinition, stackTaskDefinitions, subnets } = config.fargate;
|
|
@@ -2530,7 +3235,11 @@ function probeFargate(config) {
|
|
|
2530
3235
|
return { engine: "fargate", available: false, reason: "FARGATE_CLUSTER not set" };
|
|
2531
3236
|
}
|
|
2532
3237
|
if (!taskDefinition && !Object.keys(stackTaskDefinitions).length) {
|
|
2533
|
-
return {
|
|
3238
|
+
return {
|
|
3239
|
+
engine: "fargate",
|
|
3240
|
+
available: false,
|
|
3241
|
+
reason: "FARGATE_TASK_DEFINITION / FARGATE_STACK_TASK_DEFS not set"
|
|
3242
|
+
};
|
|
2534
3243
|
}
|
|
2535
3244
|
if (!subnets.length) {
|
|
2536
3245
|
return { engine: "fargate", available: false, reason: "FARGATE_SUBNETS not set" };
|
|
@@ -2601,7 +3310,9 @@ var MultiEngineExecutor = class {
|
|
|
2601
3310
|
const engine = spec.engine ?? this.defaultEngine;
|
|
2602
3311
|
const executor = this.executors.get(engine);
|
|
2603
3312
|
if (!executor) {
|
|
2604
|
-
throw new Error(
|
|
3313
|
+
throw new Error(
|
|
3314
|
+
`engine "${engine}" is not available on this worker (available: ${this.engines.join(", ")})`
|
|
3315
|
+
);
|
|
2605
3316
|
}
|
|
2606
3317
|
const handle = await executor.create(spec);
|
|
2607
3318
|
return { ...handle, engine };
|
|
@@ -2647,9 +3358,15 @@ var MultiEngineExecutor = class {
|
|
|
2647
3358
|
}
|
|
2648
3359
|
writeFiles(vmId, files) {
|
|
2649
3360
|
const executor = this.route(vmId);
|
|
2650
|
-
if (!executor.writeFiles)
|
|
3361
|
+
if (!executor.writeFiles)
|
|
3362
|
+
return Promise.reject(new Error("executor does not support guest file writes"));
|
|
2651
3363
|
return executor.writeFiles(vmId, files);
|
|
2652
3364
|
}
|
|
3365
|
+
onExit(vmId, handler) {
|
|
3366
|
+
const executor = this.route(vmId);
|
|
3367
|
+
if (!executor.onExit) throw new Error("executor does not support job run mode");
|
|
3368
|
+
executor.onExit(vmId, handler);
|
|
3369
|
+
}
|
|
2653
3370
|
/**
|
|
2654
3371
|
* Recover VMs sandbox says this workerId owns but that aren't currently tracked (a worker
|
|
2655
3372
|
* restart, most likely). Runs in parallel — probing many VMs serially would needlessly
|
|
@@ -2664,8 +3381,10 @@ var MultiEngineExecutor = class {
|
|
|
2664
3381
|
const results = await Promise.allSettled(
|
|
2665
3382
|
pending.map(async (record) => {
|
|
2666
3383
|
const executor = record.engine ? this.executors.get(record.engine) : void 0;
|
|
2667
|
-
if (!executor)
|
|
2668
|
-
|
|
3384
|
+
if (!executor)
|
|
3385
|
+
throw new Error(`engine "${record.engine ?? "unknown"}" not available on this worker`);
|
|
3386
|
+
if (!executor.reattach)
|
|
3387
|
+
throw new Error(`engine "${record.engine}" does not support reattach`);
|
|
2669
3388
|
const handle = await executor.reattach(record);
|
|
2670
3389
|
if (!handle) throw new Error("resource no longer exists");
|
|
2671
3390
|
})
|
|
@@ -2673,7 +3392,11 @@ var MultiEngineExecutor = class {
|
|
|
2673
3392
|
results.forEach((result, i) => {
|
|
2674
3393
|
const vmId = pending[i].vmId;
|
|
2675
3394
|
if (result.status === "fulfilled") src_default.info({ vmId }, "vm reattached");
|
|
2676
|
-
else
|
|
3395
|
+
else
|
|
3396
|
+
src_default.warn(
|
|
3397
|
+
{ vmId, err: result.reason },
|
|
3398
|
+
"vm reattach failed \u2014 staying unavailable until next touch"
|
|
3399
|
+
);
|
|
2677
3400
|
});
|
|
2678
3401
|
}
|
|
2679
3402
|
route(vmId) {
|
|
@@ -2731,16 +3454,52 @@ function wrap(handler) {
|
|
|
2731
3454
|
};
|
|
2732
3455
|
}
|
|
2733
3456
|
function registerCommandHandlers(socket, executor) {
|
|
2734
|
-
socket.on(
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
3457
|
+
socket.on(
|
|
3458
|
+
WorkerEvent.VM_CREATE,
|
|
3459
|
+
wrap(async (spec) => {
|
|
3460
|
+
const handle = await executor.create(spec);
|
|
3461
|
+
if (spec.runMode === "job") {
|
|
3462
|
+
try {
|
|
3463
|
+
executor.onExit(
|
|
3464
|
+
spec.vmId,
|
|
3465
|
+
(result) => socket.emit(WorkerEvent.VM_RUN_EXITED, { vmId: spec.vmId, ...result })
|
|
3466
|
+
);
|
|
3467
|
+
} catch (error) {
|
|
3468
|
+
await executor.destroy(spec.vmId).catch(() => {
|
|
3469
|
+
});
|
|
3470
|
+
throw error;
|
|
3471
|
+
}
|
|
3472
|
+
}
|
|
3473
|
+
return handle;
|
|
3474
|
+
})
|
|
3475
|
+
);
|
|
3476
|
+
socket.on(
|
|
3477
|
+
WorkerEvent.VM_PAUSE,
|
|
3478
|
+
wrap(({ vmId }) => executor.pause(vmId))
|
|
3479
|
+
);
|
|
3480
|
+
socket.on(
|
|
3481
|
+
WorkerEvent.VM_RESUME,
|
|
3482
|
+
wrap(({ vmId }) => executor.resume(vmId))
|
|
3483
|
+
);
|
|
3484
|
+
socket.on(
|
|
3485
|
+
WorkerEvent.VM_STOP,
|
|
3486
|
+
wrap(({ vmId }) => executor.stop(vmId))
|
|
3487
|
+
);
|
|
3488
|
+
socket.on(
|
|
3489
|
+
WorkerEvent.VM_RESTORE,
|
|
3490
|
+
wrap(({ vmId }) => executor.restore(vmId))
|
|
3491
|
+
);
|
|
3492
|
+
socket.on(
|
|
3493
|
+
WorkerEvent.VM_DESTROY,
|
|
3494
|
+
wrap(({ vmId }) => executor.destroy(vmId))
|
|
3495
|
+
);
|
|
2740
3496
|
socket.on(
|
|
2741
3497
|
WorkerEvent.PROXY_HTTP,
|
|
2742
3498
|
wrap(
|
|
2743
|
-
({
|
|
3499
|
+
({
|
|
3500
|
+
vmId,
|
|
3501
|
+
...request
|
|
3502
|
+
}) => executor.proxyHttp(vmId, request)
|
|
2744
3503
|
)
|
|
2745
3504
|
);
|
|
2746
3505
|
socket.on(
|
|
@@ -2763,14 +3522,23 @@ function registerCommandHandlers(socket, executor) {
|
|
|
2763
3522
|
);
|
|
2764
3523
|
}
|
|
2765
3524
|
function connectToSandbox(config, executor) {
|
|
3525
|
+
let assignedWorkerId;
|
|
2766
3526
|
const socket = (0, import_socket.io)(config.sandboxUrl, {
|
|
2767
3527
|
path: config.workerSocketPath,
|
|
2768
|
-
auth:
|
|
3528
|
+
auth: (callback) => callback({
|
|
3529
|
+
workerId: config.workerId,
|
|
3530
|
+
...assignedWorkerId ? { assignedWorkerId } : {},
|
|
3531
|
+
...config.token ? { token: config.token } : {}
|
|
3532
|
+
}),
|
|
2769
3533
|
transports: ["websocket"],
|
|
2770
3534
|
reconnection: true
|
|
2771
3535
|
});
|
|
2772
3536
|
let heartbeatTimer;
|
|
2773
3537
|
let reattachFallbackTimer;
|
|
3538
|
+
socket.on(WorkerEvent.ASSIGNED_ID, ({ workerId }) => {
|
|
3539
|
+
assignedWorkerId = workerId;
|
|
3540
|
+
src_default.info({ workerId: config.workerId, assignedWorkerId }, "worker runtime ID assigned");
|
|
3541
|
+
});
|
|
2774
3542
|
const sendHeartbeat = () => socket.emit(WorkerEvent.HEARTBEAT, buildHeartbeat(config, executor));
|
|
2775
3543
|
function startHeartbeat() {
|
|
2776
3544
|
if (heartbeatTimer) return;
|
|
@@ -2778,15 +3546,37 @@ function connectToSandbox(config, executor) {
|
|
|
2778
3546
|
heartbeatTimer = setInterval(sendHeartbeat, HEARTBEAT_INTERVAL_MS);
|
|
2779
3547
|
}
|
|
2780
3548
|
socket.on("connect", () => {
|
|
2781
|
-
src_default.info(
|
|
3549
|
+
src_default.info(
|
|
3550
|
+
{ workerId: config.workerId, assignedWorkerId, url: config.sandboxUrl },
|
|
3551
|
+
"connected to sandbox"
|
|
3552
|
+
);
|
|
2782
3553
|
socket.off(WorkerEvent.REATTACH);
|
|
2783
3554
|
reattachFallbackTimer = setTimeout(() => {
|
|
2784
|
-
src_default.warn(
|
|
3555
|
+
src_default.warn(
|
|
3556
|
+
{ workerId: config.workerId },
|
|
3557
|
+
"no reattach handshake from sandbox \u2014 starting heartbeat anyway"
|
|
3558
|
+
);
|
|
2785
3559
|
startHeartbeat();
|
|
2786
3560
|
}, REATTACH_FALLBACK_MS);
|
|
2787
3561
|
socket.once(WorkerEvent.REATTACH, (records) => {
|
|
2788
3562
|
clearTimeout(reattachFallbackTimer);
|
|
2789
|
-
executor.reattachAll(records).catch((error) => src_default.error({ err: error }, "reattach batch failed")).finally(
|
|
3563
|
+
executor.reattachAll(records).catch((error) => src_default.error({ err: error }, "reattach batch failed")).finally(() => {
|
|
3564
|
+
for (const record of records) {
|
|
3565
|
+
if (record.runMode !== "job" || !executor.get(record.vmId)) continue;
|
|
3566
|
+
try {
|
|
3567
|
+
executor.onExit(
|
|
3568
|
+
record.vmId,
|
|
3569
|
+
(result) => socket.emit(WorkerEvent.VM_RUN_EXITED, { vmId: record.vmId, ...result })
|
|
3570
|
+
);
|
|
3571
|
+
} catch (error) {
|
|
3572
|
+
src_default.warn(
|
|
3573
|
+
{ vmId: record.vmId, err: error },
|
|
3574
|
+
"reattached job vm has no onExit support"
|
|
3575
|
+
);
|
|
3576
|
+
}
|
|
3577
|
+
}
|
|
3578
|
+
startHeartbeat();
|
|
3579
|
+
});
|
|
2790
3580
|
});
|
|
2791
3581
|
});
|
|
2792
3582
|
socket.on("disconnect", (reason) => {
|
|
@@ -2797,7 +3587,10 @@ function connectToSandbox(config, executor) {
|
|
|
2797
3587
|
socket.off(WorkerEvent.REATTACH);
|
|
2798
3588
|
});
|
|
2799
3589
|
socket.on("connect_error", (error) => {
|
|
2800
|
-
src_default.error({
|
|
3590
|
+
src_default.error({
|
|
3591
|
+
err: error,
|
|
3592
|
+
sandboxUrl: config.sandboxUrl
|
|
3593
|
+
}, "sandbox connection error");
|
|
2801
3594
|
});
|
|
2802
3595
|
registerCommandHandlers(socket, executor);
|
|
2803
3596
|
return socket;
|
|
@@ -2831,7 +3624,11 @@ var VncRelay = class {
|
|
|
2831
3624
|
const rfbc = this.rfbc;
|
|
2832
3625
|
rfbc.on("connect", () => {
|
|
2833
3626
|
rfbc.autoUpdate = true;
|
|
2834
|
-
this.socket.emit(WorkerEvent.VNC_INIT, {
|
|
3627
|
+
this.socket.emit(WorkerEvent.VNC_INIT, {
|
|
3628
|
+
vmId: this.vmId,
|
|
3629
|
+
width: rfbc.width,
|
|
3630
|
+
height: rfbc.height
|
|
3631
|
+
});
|
|
2835
3632
|
this.interval = setInterval(() => {
|
|
2836
3633
|
rfbc.requestUpdate(false, 0, 0, rfbc.width, rfbc.height);
|
|
2837
3634
|
}, 300);
|
|
@@ -2866,8 +3663,15 @@ var VncRelay = class {
|
|
|
2866
3663
|
}
|
|
2867
3664
|
});
|
|
2868
3665
|
rfbc.on("resize", (rect) => {
|
|
2869
|
-
src_default.info(
|
|
2870
|
-
|
|
3666
|
+
src_default.info(
|
|
3667
|
+
{ vmId: this.vmId, width: rect.width, height: rect.height },
|
|
3668
|
+
"[VNC] desktop resized"
|
|
3669
|
+
);
|
|
3670
|
+
this.socket.emit(WorkerEvent.VNC_INIT, {
|
|
3671
|
+
vmId: this.vmId,
|
|
3672
|
+
width: rect.width,
|
|
3673
|
+
height: rect.height
|
|
3674
|
+
});
|
|
2871
3675
|
rfbc.requestUpdate(false, 0, 0, rect.width, rect.height);
|
|
2872
3676
|
});
|
|
2873
3677
|
rfbc.on("error", (err) => src_default.error({ vmId: this.vmId, err }, "[VNC] RFB error"));
|
|
@@ -2928,7 +3732,10 @@ var SerialRelay = class {
|
|
|
2928
3732
|
sock.once("connect", () => {
|
|
2929
3733
|
this.conn = sock;
|
|
2930
3734
|
src_default.info({ vmId: this.vmId, port }, "[Serial] TCP connected");
|
|
2931
|
-
sock.on(
|
|
3735
|
+
sock.on(
|
|
3736
|
+
"data",
|
|
3737
|
+
(d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
|
|
3738
|
+
);
|
|
2932
3739
|
sock.on("end", () => src_default.info({ vmId: this.vmId }, "[Serial] TCP ended"));
|
|
2933
3740
|
sock.on("error", (err) => src_default.error({ vmId: this.vmId, err }, "[Serial] TCP error"));
|
|
2934
3741
|
});
|
|
@@ -2938,8 +3745,14 @@ var SerialRelay = class {
|
|
|
2938
3745
|
}
|
|
2939
3746
|
connectPipe() {
|
|
2940
3747
|
const { process: proc } = this.source;
|
|
2941
|
-
proc.stdout?.on(
|
|
2942
|
-
|
|
3748
|
+
proc.stdout?.on(
|
|
3749
|
+
"data",
|
|
3750
|
+
(d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
|
|
3751
|
+
);
|
|
3752
|
+
proc.stderr?.on(
|
|
3753
|
+
"data",
|
|
3754
|
+
(d) => this.socket.emit(WorkerEvent.TERMINAL_DATA, { vmId: this.vmId, data: d })
|
|
3755
|
+
);
|
|
2943
3756
|
}
|
|
2944
3757
|
destroy() {
|
|
2945
3758
|
if (this.inputHandler) this.socket.off(WorkerEvent.TERMINAL_INPUT, this.inputHandler);
|
|
@@ -2982,10 +3795,37 @@ function attachConsoleRelays(executor, sandboxSocket) {
|
|
|
2982
3795
|
return originalDestroy(vmId);
|
|
2983
3796
|
};
|
|
2984
3797
|
}
|
|
3798
|
+
|
|
3799
|
+
// src/health-server.ts
|
|
3800
|
+
var import_http7 = __toESM(require("http"));
|
|
3801
|
+
async function startSandboxHealthServer(options) {
|
|
3802
|
+
const server = import_http7.default.createServer((req, res) => {
|
|
3803
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
3804
|
+
if (pathname === `${options.pathPrefix}/healthz`) {
|
|
3805
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
3806
|
+
res.end(JSON.stringify({ ok: true, status: "alive" }));
|
|
3807
|
+
return;
|
|
3808
|
+
}
|
|
3809
|
+
if (pathname === `${options.pathPrefix}/health`) {
|
|
3810
|
+
const ready = options.isReady();
|
|
3811
|
+
res.writeHead(ready ? 200 : 503, { "content-type": "application/json" });
|
|
3812
|
+
res.end(JSON.stringify({ ok: ready, status: ready ? "ready" : "not-ready" }));
|
|
3813
|
+
return;
|
|
3814
|
+
}
|
|
3815
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
3816
|
+
res.end(JSON.stringify({ ok: false, error: "not found" }));
|
|
3817
|
+
});
|
|
3818
|
+
await new Promise((resolve, reject) => {
|
|
3819
|
+
server.listen(options.port, options.host, resolve);
|
|
3820
|
+
server.once("error", reject);
|
|
3821
|
+
});
|
|
3822
|
+
return server;
|
|
3823
|
+
}
|
|
2985
3824
|
// Annotate the CommonJS export names for ESM import in node:
|
|
2986
3825
|
0 && (module.exports = {
|
|
2987
3826
|
IMAGE_FILES,
|
|
2988
3827
|
MultiEngineExecutor,
|
|
3828
|
+
PUBLISHED_IMAGE_STACKS,
|
|
2989
3829
|
SerialRelay,
|
|
2990
3830
|
VncRelay,
|
|
2991
3831
|
WORKER_VERSION,
|
|
@@ -3001,6 +3841,7 @@ function attachConsoleRelays(executor, sandboxSocket) {
|
|
|
3001
3841
|
probeEngines,
|
|
3002
3842
|
pullImages,
|
|
3003
3843
|
pushImages,
|
|
3004
|
-
resolveStackImages
|
|
3844
|
+
resolveStackImages,
|
|
3845
|
+
startSandboxHealthServer
|
|
3005
3846
|
});
|
|
3006
3847
|
//# sourceMappingURL=index.js.map
|